src/vnet/ip/ip4_reassembly.c

   1 /*
   2  * Copyright (c) 2017 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 /**
  17  * @file
  18  * @brief IPv4 Reassembly.
  19  *
  20  * This file contains the source code for IPv4 reassembly.
  21  */
  22
  23 #include <vppinfra/vec.h>
  24 #include <vnet/vnet.h>
  25 #include <vnet/ip/ip.h>
  26 #include <vppinfra/bihash_16_8.h>
  27 #include <vnet/ip/ip4_reassembly.h>
  28
  29 #define MSEC_PER_SEC 1000
  30 #define IP4_REASS_TIMEOUT_DEFAULT_MS 100
  31 #define IP4_REASS_EXPIRE_WALK_INTERVAL_DEFAULT_MS 10000 // 10 seconds default
  32 #define IP4_REASS_MAX_REASSEMBLIES_DEFAULT 1024
  33 #define IP4_REASS_HT_LOAD_FACTOR (0.75)
  34
  35 #define IP4_REASS_DEBUG_BUFFERS 0
  36 #if IP4_REASS_DEBUG_BUFFERS
  37 #define IP4_REASS_DEBUG_BUFFER(bi, what)             \
  38   do                                                 \
  39     {                                                \
  40       u32 _bi = bi;                                  \
  41       printf (#what "buffer %u", _bi);               \
  42       vlib_buffer_t *_b = vlib_get_buffer (vm, _bi); \
  43       while (_b->flags & VLIB_BUFFER_NEXT_PRESENT)   \
  44         {                                            \
  45           _bi = _b->next_buffer;                     \
  46           printf ("[%u]", _bi);                      \
  47           _b = vlib_get_buffer (vm, _bi);            \
  48         }                                            \
  49       printf ("\n");                                 \
  50       fflush (stdout);                               \
  51     }                                                \
  52   while (0)
  53 #else
  54 #define IP4_REASS_DEBUG_BUFFER(...)
  55 #endif
  56
  57 static vlib_node_registration_t ip4_reass_node;
  58
  59 typedef struct
  60 {
  61   union
  62   {
  63     struct
  64     {
  65       u32 xx_id;
  66       ip4_address_t src;
  67       ip4_address_t dst;
  68       u16 frag_id;
  69       u8 proto;
  70       u8 unused;
  71     };
  72     u64 as_u64[2];
  73   };
  74 } ip4_reass_key_t;
  75
  76 always_inline u32
  77 ip4_reass_buffer_get_data_offset_no_check (vlib_buffer_t * b)
  78 {
  79   vnet_buffer_opaque_t *vnb = vnet_buffer (b);
  80   return vnb->ip.reass.range_first - vnb->ip.reass.fragment_first;
  81 }
  82
  83 always_inline u32
  84 ip4_reass_buffer_get_data_offset (vlib_buffer_t * b)
  85 {
  86   vnet_buffer_opaque_t *vnb = vnet_buffer (b);
  87   ASSERT (vnb->ip.reass.range_first >= vnb->ip.reass.fragment_first);
  88   return ip4_reass_buffer_get_data_offset_no_check (b);
  89 }
  90
  91 always_inline u16
  92 ip4_reass_buffer_get_data_len_no_check (vlib_buffer_t * b)
  93 {
  94   vnet_buffer_opaque_t *vnb = vnet_buffer (b);
  95   return clib_min (vnb->ip.reass.range_last, vnb->ip.reass.fragment_last) -
  96     (vnb->ip.reass.fragment_first + ip4_reass_buffer_get_data_offset (b)) + 1;
  97 }
  98
  99 always_inline u16
 100 ip4_reass_buffer_get_data_len (vlib_buffer_t * b)
 101 {
 102   vnet_buffer_opaque_t *vnb = vnet_buffer (b);
 103   ASSERT (vnb->ip.reass.range_last > vnb->ip.reass.fragment_first);
 104   return ip4_reass_buffer_get_data_len_no_check (b);
 105 }
 106
 107 typedef struct
 108 {
 109   // hash table key
 110   ip4_reass_key_t key;
 111   // time when last packet was received
 112   f64 last_heard;
 113   // internal id of this reassembly
 114   u64 id;
 115   // buffer index of first buffer in this reassembly context
 116   u32 first_bi;
 117   // last octet of packet, ~0 until fragment without more_fragments arrives
 118   u32 last_packet_octet;
 119   // length of data collected so far
 120   u32 data_len;
 121   // trace operation counter
 122   u32 trace_op_counter;
 123   // next index - used by non-feature node
 124   u8 next_index;
 125   // minimum fragment length for this reassembly - used to estimate MTU
 126   u16 min_fragment_length;
 127 } ip4_reass_t;
 128
 129 typedef struct
 130 {
 131   ip4_reass_t *pool;
 132   u32 reass_n;
 133   u32 buffers_n;
 134   u32 id_counter;
 135   clib_spinlock_t lock;
 136 } ip4_reass_per_thread_t;
 137
 138 typedef struct
 139 {
 140   // IPv4 config
 141   u32 timeout_ms;
 142   f64 timeout;
 143   u32 expire_walk_interval_ms;
 144   u32 max_reass_n;
 145
 146   // IPv4 runtime
 147   clib_bihash_16_8_t hash;
 148   // per-thread data
 149   ip4_reass_per_thread_t *per_thread_data;
 150
 151   // convenience
 152   vlib_main_t *vlib_main;
 153   vnet_main_t *vnet_main;
 154
 155   // node index of ip4-drop node
 156   u32 ip4_drop_idx;
 157   u32 ip4_reass_expire_node_idx;
 158 } ip4_reass_main_t;
 159
 160 ip4_reass_main_t ip4_reass_main;
 161
 162 typedef enum
 163 {
 164   IP4_REASSEMBLY_NEXT_INPUT,
 165   IP4_REASSEMBLY_NEXT_DROP,
 166   IP4_REASSEMBLY_N_NEXT,
 167 } ip4_reass_next_t;
 168
 169 typedef enum
 170 {
 171   RANGE_NEW,
 172   RANGE_SHRINK,
 173   RANGE_DISCARD,
 174   RANGE_OVERLAP,
 175   FINALIZE,
 176 } ip4_reass_trace_operation_e;
 177
 178 typedef struct
 179 {
 180   u16 range_first;
 181   u16 range_last;
 182   u32 range_bi;
 183   i32 data_offset;
 184   u32 data_len;
 185   u32 first_bi;
 186 } ip4_reass_range_trace_t;
 187
 188 typedef struct
 189 {
 190   ip4_reass_trace_operation_e action;
 191   u32 reass_id;
 192   ip4_reass_range_trace_t trace_range;
 193   u32 size_diff;
 194   u32 op_id;
 195   u32 fragment_first;
 196   u32 fragment_last;
 197   u32 total_data_len;
 198 } ip4_reass_trace_t;
 199
 200 static void
 201 ip4_reass_trace_details (vlib_main_t * vm, u32 bi,
 202                          ip4_reass_range_trace_t * trace)
 203 {
 204   vlib_buffer_t *b = vlib_get_buffer (vm, bi);
 205   vnet_buffer_opaque_t *vnb = vnet_buffer (b);
 206   trace->range_first = vnb->ip.reass.range_first;
 207   trace->range_last = vnb->ip.reass.range_last;
 208   trace->data_offset = ip4_reass_buffer_get_data_offset_no_check (b);
 209   trace->data_len = ip4_reass_buffer_get_data_len_no_check (b);
 210   trace->range_bi = bi;
 211 }
 212
 213 static u8 *
 214 format_ip4_reass_range_trace (u8 * s, va_list * args)
 215 {
 216   ip4_reass_range_trace_t *trace = va_arg (*args, ip4_reass_range_trace_t *);
 217   s = format (s, "range: [%u, %u], off %d, len %u, bi %u", trace->range_first,
 218               trace->range_last, trace->data_offset, trace->data_len,
 219               trace->range_bi);
 220   return s;
 221 }
 222
 223 u8 *
 224 format_ip4_reass_trace (u8 * s, va_list * args)
 225 {
 226   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 227   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 228   ip4_reass_trace_t *t = va_arg (*args, ip4_reass_trace_t *);
 229   s = format (s, "reass id: %u, op id: %u ", t->reass_id, t->op_id);
 230   u32 indent = format_get_indent (s);
 231   s = format (s, "first bi: %u, data len: %u, ip/fragment[%u, %u]",
 232               t->trace_range.first_bi, t->total_data_len, t->fragment_first,
 233               t->fragment_last);
 234   switch (t->action)
 235     {
 236     case RANGE_SHRINK:
 237       s = format (s, "\n%Ushrink %U by %u", format_white_space, indent,
 238                   format_ip4_reass_range_trace, &t->trace_range,
 239                   t->size_diff);
 240       break;
 241     case RANGE_DISCARD:
 242       s = format (s, "\n%Udiscard %U", format_white_space, indent,
 243                   format_ip4_reass_range_trace, &t->trace_range);
 244       break;
 245     case RANGE_NEW:
 246       s = format (s, "\n%Unew %U", format_white_space, indent,
 247                   format_ip4_reass_range_trace, &t->trace_range);
 248       break;
 249     case RANGE_OVERLAP:
 250       s = format (s, "\n%Uoverlapping/ignored %U", format_white_space, indent,
 251                   format_ip4_reass_range_trace, &t->trace_range);
 252       break;
 253     case FINALIZE:
 254       s = format (s, "\n%Ufinalize reassembly", format_white_space, indent);
 255       break;
 256     }
 257   return s;
 258 }
 259
 260 static void
 261 ip4_reass_add_trace (vlib_main_t * vm, vlib_node_runtime_t * node,
 262                      ip4_reass_main_t * rm, ip4_reass_t * reass, u32 bi,
 263                      ip4_reass_trace_operation_e action, u32 size_diff)
 264 {
 265   vlib_buffer_t *b = vlib_get_buffer (vm, bi);
 266   vnet_buffer_opaque_t *vnb = vnet_buffer (b);
 267   if (pool_is_free_index (vm->trace_main.trace_buffer_pool, b->trace_index))
 268     {
 269       // this buffer's trace is gone
 270       b->flags &= ~VLIB_BUFFER_IS_TRACED;
 271       return;
 272     }
 273   ip4_reass_trace_t *t = vlib_add_trace (vm, node, b, sizeof (t[0]));
 274   t->reass_id = reass->id;
 275   t->action = action;
 276   ip4_reass_trace_details (vm, bi, &t->trace_range);
 277   t->size_diff = size_diff;
 278   t->op_id = reass->trace_op_counter;
 279   ++reass->trace_op_counter;
 280   t->fragment_first = vnb->ip.reass.fragment_first;
 281   t->fragment_last = vnb->ip.reass.fragment_last;
 282   t->trace_range.first_bi = reass->first_bi;
 283   t->total_data_len = reass->data_len;
 284 #if 0
 285   static u8 *s = NULL;
 286   s = format (s, "%U", format_ip4_reass_trace, NULL, NULL, t);
 287   printf ("%.*s\n", vec_len (s), s);
 288   fflush (stdout);
 289   vec_reset_length (s);
 290 #endif
 291 }
 292
 293 always_inline void
 294 ip4_reass_free (ip4_reass_main_t * rm, ip4_reass_per_thread_t * rt,
 295                 ip4_reass_t * reass)
 296 {
 297   clib_bihash_kv_16_8_t kv;
 298   kv.key[0] = reass->key.as_u64[0];
 299   kv.key[1] = reass->key.as_u64[1];
 300   clib_bihash_add_del_16_8 (&rm->hash, &kv, 0);
 301   pool_put (rt->pool, reass);
 302   --rt->reass_n;
 303 }
 304
 305 always_inline void
 306 ip4_reass_on_timeout (vlib_main_t * vm, ip4_reass_main_t * rm,
 307                       ip4_reass_t * reass, u32 ** vec_drop_timeout)
 308 {
 309   u32 range_bi = reass->first_bi;
 310   vlib_buffer_t *range_b;
 311   vnet_buffer_opaque_t *range_vnb;
 312   while (~0 != range_bi)
 313     {
 314       range_b = vlib_get_buffer (vm, range_bi);
 315       range_vnb = vnet_buffer (range_b);
 316       u32 bi = range_bi;
 317       while (~0 != bi)
 318         {
 319           vec_add1 (*vec_drop_timeout, bi);
 320           vlib_buffer_t *b = vlib_get_buffer (vm, bi);
 321           if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
 322             {
 323               bi = b->next_buffer;
 324               b->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
 325             }
 326           else
 327             {
 328               bi = ~0;
 329             }
 330         }
 331       range_bi = range_vnb->ip.reass.next_range_bi;
 332     }
 333 }
 334
 335 ip4_reass_t *
 336 ip4_reass_find_or_create (vlib_main_t * vm, ip4_reass_main_t * rm,
 337                           ip4_reass_per_thread_t * rt,
 338                           ip4_reass_key_t * k, u32 ** vec_drop_timeout)
 339 {
 340   ip4_reass_t *reass = NULL;
 341   f64 now = vlib_time_now (rm->vlib_main);
 342   clib_bihash_kv_16_8_t kv, value;
 343   kv.key[0] = k->as_u64[0];
 344   kv.key[1] = k->as_u64[1];
 345
 346   if (!clib_bihash_search_16_8 (&rm->hash, &kv, &value))
 347     {
 348       reass = pool_elt_at_index (rt->pool, value.value);
 349       if (now > reass->last_heard + rm->timeout)
 350         {
 351           ip4_reass_on_timeout (vm, rm, reass, vec_drop_timeout);
 352           ip4_reass_free (rm, rt, reass);
 353           reass = NULL;
 354         }
 355     }
 356
 357   if (reass)
 358     {
 359       reass->last_heard = now;
 360       return reass;
 361     }
 362
 363   if (rt->reass_n >= rm->max_reass_n)
 364     {
 365       reass = NULL;
 366       return reass;
 367     }
 368   else
 369     {
 370       pool_get (rt->pool, reass);
 371       memset (reass, 0, sizeof (*reass));
 372       reass->id =
 373         ((u64) os_get_thread_index () * 1000000000) + rt->id_counter;
 374       ++rt->id_counter;
 375       reass->first_bi = ~0;
 376       reass->last_packet_octet = ~0;
 377       reass->data_len = 0;
 378       ++rt->reass_n;
 379     }
 380
 381   reass->key.as_u64[0] = kv.key[0] = k->as_u64[0];
 382   reass->key.as_u64[1] = kv.key[1] = k->as_u64[1];
 383   kv.value = reass - rt->pool;
 384   reass->last_heard = now;
 385
 386   if (clib_bihash_add_del_16_8 (&rm->hash, &kv, 1))
 387     {
 388       ip4_reass_free (rm, rt, reass);
 389       reass = NULL;
 390     }
 391
 392   return reass;
 393 }
 394
 395 always_inline void
 396 ip4_reass_finalize (vlib_main_t * vm, vlib_node_runtime_t * node,
 397                     ip4_reass_main_t * rm, ip4_reass_per_thread_t * rt,
 398                     ip4_reass_t * reass, u32 * bi0, u32 * next0,
 399                     u32 * error0, u32 ** vec_drop_compress,
 400                     u32 ** vec_drop_overlap, bool is_feature)
 401 {
 402   ASSERT (~0 != reass->first_bi);
 403   vlib_buffer_t *first_b = vlib_get_buffer (vm, reass->first_bi);
 404   vlib_buffer_t *last_b = NULL;
 405   u32 sub_chain_bi = reass->first_bi;
 406   u32 total_length = 0;
 407   u32 buf_cnt = 0;
 408   u32 dropped_cnt = 0;
 409   do
 410     {
 411       u32 tmp_bi = sub_chain_bi;
 412       vlib_buffer_t *tmp = vlib_get_buffer (vm, tmp_bi);
 413       ip4_header_t *ip = vlib_buffer_get_current (tmp);
 414       u32 data_len = ip4_reass_buffer_get_data_len (tmp);
 415       u32 trim_front =
 416         ip4_header_bytes (ip) + ip4_reass_buffer_get_data_offset (tmp);
 417       u32 trim_end =
 418         vlib_buffer_length_in_chain (vm, tmp) - trim_front - data_len;
 419       if (tmp_bi == reass->first_bi)
 420         {
 421           /* first buffer - keep ip4 header */
 422           ASSERT (0 == ip4_reass_buffer_get_data_offset (tmp));
 423           trim_front = 0;
 424           trim_end = vlib_buffer_length_in_chain (vm, tmp) - data_len -
 425             ip4_header_bytes (ip);
 426           ASSERT (vlib_buffer_length_in_chain (vm, tmp) - trim_end > 0);
 427         }
 428       u32 keep_data =
 429         vlib_buffer_length_in_chain (vm, tmp) - trim_front - trim_end;
 430       while (1)
 431         {
 432           ++buf_cnt;
 433           if (trim_front)
 434             {
 435               if (trim_front > tmp->current_length)
 436                 {
 437                   /* drop whole buffer */
 438                   vec_add1 (*vec_drop_compress, tmp_bi);
 439                   ++dropped_cnt;
 440                   trim_front -= tmp->current_length;
 441                   ASSERT (tmp->flags & VLIB_BUFFER_NEXT_PRESENT);
 442                   tmp->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
 443                   tmp_bi = tmp->next_buffer;
 444                   tmp = vlib_get_buffer (vm, tmp_bi);
 445                   continue;
 446                 }
 447               else
 448                 {
 449                   vlib_buffer_advance (tmp, trim_front);
 450                   trim_front = 0;
 451                 }
 452             }
 453           if (keep_data)
 454             {
 455               if (last_b)
 456                 {
 457                   last_b->flags |= VLIB_BUFFER_NEXT_PRESENT;
 458                   last_b->next_buffer = tmp_bi;
 459                 }
 460               last_b = tmp;
 461               if (keep_data <= tmp->current_length)
 462                 {
 463                   tmp->current_length = keep_data;
 464                   keep_data = 0;
 465                 }
 466               else
 467                 {
 468                   keep_data -= tmp->current_length;
 469                   ASSERT (tmp->flags & VLIB_BUFFER_NEXT_PRESENT);
 470                 }
 471               total_length += tmp->current_length;
 472             }
 473           else
 474             {
 475               vec_add1 (*vec_drop_overlap, tmp_bi);
 476               ASSERT (reass->first_bi != tmp_bi);
 477               ++dropped_cnt;
 478             }
 479           if (tmp->flags & VLIB_BUFFER_NEXT_PRESENT)
 480             {
 481               tmp_bi = tmp->next_buffer;
 482               tmp = vlib_get_buffer (vm, tmp->next_buffer);
 483             }
 484           else
 485             {
 486               break;
 487             }
 488         }
 489       sub_chain_bi =
 490         vnet_buffer (vlib_get_buffer (vm, sub_chain_bi))->ip.
 491         reass.next_range_bi;
 492     }
 493   while (~0 != sub_chain_bi);
 494
 495   ASSERT (last_b != NULL);
 496   last_b->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
 497   ASSERT (rt->buffers_n >= (buf_cnt - dropped_cnt));
 498   rt->buffers_n -= buf_cnt - dropped_cnt;
 499   ASSERT (total_length >= first_b->current_length);
 500   total_length -= first_b->current_length;
 501   first_b->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
 502   first_b->total_length_not_including_first_buffer = total_length;
 503   ip4_header_t *ip = vlib_buffer_get_current (first_b);
 504   ip->flags_and_fragment_offset = 0;
 505   ip->length = clib_host_to_net_u16 (first_b->current_length + total_length);
 506   ip->checksum = ip4_header_checksum (ip);
 507   vlib_buffer_chain_compress (vm, first_b, vec_drop_compress);
 508   if (PREDICT_FALSE (first_b->flags & VLIB_BUFFER_IS_TRACED))
 509     {
 510       ip4_reass_add_trace (vm, node, rm, reass, reass->first_bi, FINALIZE, 0);
 511 #if 0
 512       // following code does a hexdump of packet fragments to stdout ...
 513       do
 514         {
 515           u32 bi = reass->first_bi;
 516           u8 *s = NULL;
 517           while (~0 != bi)
 518             {
 519               vlib_buffer_t *b = vlib_get_buffer (vm, bi);
 520               s = format (s, "%u: %U\n", bi, format_hexdump,
 521                           vlib_buffer_get_current (b), b->current_length);
 522               if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
 523                 {
 524                   bi = b->next_buffer;
 525                 }
 526               else
 527                 {
 528                   break;
 529                 }
 530             }
 531           printf ("%.*s\n", vec_len (s), s);
 532           fflush (stdout);
 533           vec_free (s);
 534         }
 535       while (0);
 536 #endif
 537     }
 538   *bi0 = reass->first_bi;
 539   if (is_feature)
 540     {
 541       *next0 = IP4_REASSEMBLY_NEXT_INPUT;
 542     }
 543   else
 544     {
 545       *next0 = reass->next_index;
 546     }
 547   vnet_buffer (first_b)->ip.reass.estimated_mtu = reass->min_fragment_length;
 548   *error0 = IP4_ERROR_NONE;
 549   ip4_reass_free (rm, rt, reass);
 550   reass = NULL;
 551 }
 552
 553 always_inline u32
 554 ip4_reass_get_buffer_chain_length (vlib_main_t * vm, vlib_buffer_t * b)
 555 {
 556   u32 len = 0;
 557   while (b)
 558     {
 559       ++len;
 560       if (PREDICT_FALSE (b->flags & VLIB_BUFFER_NEXT_PRESENT))
 561         {
 562           b = vlib_get_buffer (vm, b->next_buffer);
 563         }
 564       else
 565         {
 566           break;
 567         }
 568     }
 569   return len;
 570 }
 571
 572 always_inline void
 573 ip4_reass_insert_range_in_chain (vlib_main_t * vm,
 574                                  ip4_reass_main_t * rm,
 575                                  ip4_reass_per_thread_t * rt,
 576                                  ip4_reass_t * reass,
 577                                  u32 prev_range_bi, u32 new_next_bi)
 578 {
 579
 580   vlib_buffer_t *new_next_b = vlib_get_buffer (vm, new_next_bi);
 581   vnet_buffer_opaque_t *new_next_vnb = vnet_buffer (new_next_b);
 582   if (~0 != prev_range_bi)
 583     {
 584       vlib_buffer_t *prev_b = vlib_get_buffer (vm, prev_range_bi);
 585       vnet_buffer_opaque_t *prev_vnb = vnet_buffer (prev_b);
 586       new_next_vnb->ip.reass.next_range_bi = prev_vnb->ip.reass.next_range_bi;
 587       prev_vnb->ip.reass.next_range_bi = new_next_bi;
 588     }
 589   else
 590     {
 591       if (~0 != reass->first_bi)
 592         {
 593           new_next_vnb->ip.reass.next_range_bi = reass->first_bi;
 594         }
 595       reass->first_bi = new_next_bi;
 596     }
 597   reass->data_len += ip4_reass_buffer_get_data_len (new_next_b);
 598   rt->buffers_n += ip4_reass_get_buffer_chain_length (vm, new_next_b);
 599 }
 600
 601 always_inline void
 602 ip4_reass_remove_range_from_chain (vlib_main_t * vm,
 603                                    vlib_node_runtime_t * node,
 604                                    ip4_reass_main_t * rm,
 605                                    u32 ** vec_drop_overlap,
 606                                    ip4_reass_t * reass, u32 prev_range_bi,
 607                                    u32 discard_bi)
 608 {
 609   vlib_buffer_t *discard_b = vlib_get_buffer (vm, discard_bi);
 610   vnet_buffer_opaque_t *discard_vnb = vnet_buffer (discard_b);
 611   if (~0 != prev_range_bi)
 612     {
 613       vlib_buffer_t *prev_b = vlib_get_buffer (vm, prev_range_bi);
 614       vnet_buffer_opaque_t *prev_vnb = vnet_buffer (prev_b);
 615       ASSERT (prev_vnb->ip.reass.next_range_bi == discard_bi);
 616       prev_vnb->ip.reass.next_range_bi = discard_vnb->ip.reass.next_range_bi;
 617     }
 618   else
 619     {
 620       reass->first_bi = discard_vnb->ip.reass.next_range_bi;
 621     }
 622   reass->data_len -= ip4_reass_buffer_get_data_len (discard_b);
 623   while (1)
 624     {
 625       vec_add1 (*vec_drop_overlap, discard_bi);
 626       if (PREDICT_FALSE (discard_b->flags & VLIB_BUFFER_IS_TRACED))
 627         {
 628           ip4_reass_add_trace (vm, node, rm, reass, discard_bi, RANGE_DISCARD,
 629                                0);
 630         }
 631       if (discard_b->flags & VLIB_BUFFER_NEXT_PRESENT)
 632         {
 633           discard_b->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
 634           discard_bi = discard_b->next_buffer;
 635           discard_b = vlib_get_buffer (vm, discard_bi);
 636         }
 637       else
 638         {
 639           break;
 640         }
 641     }
 642 }
 643
 644 always_inline void
 645 ip4_reass_update (vlib_main_t * vm, vlib_node_runtime_t * node,
 646                   ip4_reass_main_t * rm, ip4_reass_per_thread_t * rt,
 647                   ip4_reass_t * reass, u32 * bi0, u32 * next0,
 648                   u32 * error0, u32 ** vec_drop_overlap,
 649                   u32 ** vec_drop_compress, bool is_feature)
 650 {
 651   int consumed = 0;
 652   vlib_buffer_t *fb = vlib_get_buffer (vm, *bi0);
 653   ip4_header_t *fip = vlib_buffer_get_current (fb);
 654   ASSERT (fb->current_length >= sizeof (*fip));
 655   vnet_buffer_opaque_t *fvnb = vnet_buffer (fb);
 656   reass->next_index = fvnb->ip.reass.next_index;        // store next_index before it's overwritten
 657   u32 fragment_first = fvnb->ip.reass.fragment_first =
 658     ip4_get_fragment_offset_bytes (fip);
 659   u32 fragment_length =
 660     clib_net_to_host_u16 (fip->length) - ip4_header_bytes (fip);
 661   u32 fragment_last = fvnb->ip.reass.fragment_last =
 662     fragment_first + fragment_length - 1;
 663   int more_fragments = ip4_get_fragment_more (fip);
 664   u32 candidate_range_bi = reass->first_bi;
 665   u32 prev_range_bi = ~0;
 666   fvnb->ip.reass.range_first = fragment_first;
 667   fvnb->ip.reass.range_last = fragment_last;
 668   fvnb->ip.reass.next_range_bi = ~0;
 669   if (!more_fragments)
 670     {
 671       reass->last_packet_octet = fragment_last;
 672     }
 673   if (~0 == reass->first_bi)
 674     {
 675       // starting a new reassembly
 676       ip4_reass_insert_range_in_chain (vm, rm, rt, reass, prev_range_bi,
 677                                        *bi0);
 678       if (PREDICT_FALSE (fb->flags & VLIB_BUFFER_IS_TRACED))
 679         {
 680           ip4_reass_add_trace (vm, node, rm, reass, *bi0, RANGE_NEW, 0);
 681         }
 682       *bi0 = ~0;
 683       reass->min_fragment_length = clib_net_to_host_u16 (fip->length);
 684       return;
 685     }
 686   reass->min_fragment_length = clib_min (clib_net_to_host_u16 (fip->length),
 687                                          fvnb->ip.reass.estimated_mtu);
 688   while (~0 != candidate_range_bi)
 689     {
 690       vlib_buffer_t *candidate_b = vlib_get_buffer (vm, candidate_range_bi);
 691       vnet_buffer_opaque_t *candidate_vnb = vnet_buffer (candidate_b);
 692       if (fragment_first > candidate_vnb->ip.reass.range_last)
 693         {
 694           // this fragments starts after candidate range
 695           prev_range_bi = candidate_range_bi;
 696           candidate_range_bi = candidate_vnb->ip.reass.next_range_bi;
 697           if (candidate_vnb->ip.reass.range_last < fragment_last &&
 698               ~0 == candidate_range_bi)
 699             {
 700               // special case - this fragment falls beyond all known ranges
 701               ip4_reass_insert_range_in_chain (vm, rm, rt, reass,
 702                                                prev_range_bi, *bi0);
 703               consumed = 1;
 704               break;
 705             }
 706           continue;
 707         }
 708       if (fragment_last < candidate_vnb->ip.reass.range_first)
 709         {
 710           // this fragment ends before candidate range without any overlap
 711           ip4_reass_insert_range_in_chain (vm, rm, rt, reass, prev_range_bi,
 712                                            *bi0);
 713           consumed = 1;
 714         }
 715       else
 716         {
 717           if (fragment_first >= candidate_vnb->ip.reass.range_first &&
 718               fragment_last <= candidate_vnb->ip.reass.range_last)
 719             {
 720               // this fragment is a (sub)part of existing range, ignore it
 721               if (PREDICT_FALSE (fb->flags & VLIB_BUFFER_IS_TRACED))
 722                 {
 723                   ip4_reass_add_trace (vm, node, rm, reass, *bi0,
 724                                        RANGE_OVERLAP, 0);
 725                 }
 726               break;
 727             }
 728           int discard_candidate = 0;
 729           if (fragment_first < candidate_vnb->ip.reass.range_first)
 730             {
 731               u32 overlap =
 732                 fragment_last - candidate_vnb->ip.reass.range_first + 1;
 733               if (overlap < ip4_reass_buffer_get_data_len (candidate_b))
 734                 {
 735                   candidate_vnb->ip.reass.range_first += overlap;
 736                   ASSERT (reass->data_len >= overlap);
 737                   reass->data_len -= overlap;
 738                   if (PREDICT_FALSE (fb->flags & VLIB_BUFFER_IS_TRACED))
 739                     {
 740                       ip4_reass_add_trace (vm, node, rm, reass,
 741                                            candidate_range_bi, RANGE_SHRINK,
 742                                            overlap);
 743                     }
 744                   ip4_reass_insert_range_in_chain (vm, rm, rt, reass,
 745                                                    prev_range_bi, *bi0);
 746                   consumed = 1;
 747                 }
 748               else
 749                 {
 750                   discard_candidate = 1;
 751                 }
 752             }
 753           else if (fragment_last > candidate_vnb->ip.reass.range_last)
 754             {
 755               u32 overlap =
 756                 candidate_vnb->ip.reass.range_last - fragment_first + 1;
 757               if (overlap < ip4_reass_buffer_get_data_len (candidate_b))
 758                 {
 759                   fvnb->ip.reass.range_first += overlap;
 760                   if (~0 != candidate_vnb->ip.reass.next_range_bi)
 761                     {
 762                       prev_range_bi = candidate_range_bi;
 763                       candidate_range_bi =
 764                         candidate_vnb->ip.reass.next_range_bi;
 765                       continue;
 766                     }
 767                   else
 768                     {
 769                       // special case - last range discarded
 770                       ip4_reass_insert_range_in_chain (vm, rm, rt, reass,
 771                                                        candidate_range_bi,
 772                                                        *bi0);
 773                       consumed = 1;
 774                     }
 775                 }
 776               else
 777                 {
 778                   discard_candidate = 1;
 779                 }
 780             }
 781           else
 782             {
 783               discard_candidate = 1;
 784             }
 785           if (discard_candidate)
 786             {
 787               u32 next_range_bi = candidate_vnb->ip.reass.next_range_bi;
 788               // discard candidate range, probe next range
 789               ip4_reass_remove_range_from_chain (vm, node, rm,
 790                                                  vec_drop_overlap, reass,
 791                                                  prev_range_bi,
 792                                                  candidate_range_bi);
 793               if (~0 != next_range_bi)
 794                 {
 795                   candidate_range_bi = next_range_bi;
 796                   continue;
 797                 }
 798               else
 799                 {
 800                   // special case - last range discarded
 801                   ip4_reass_insert_range_in_chain (vm, rm, rt, reass,
 802                                                    prev_range_bi, *bi0);
 803                   consumed = 1;
 804                 }
 805             }
 806         }
 807       break;
 808     }
 809   if (consumed)
 810     {
 811       if (PREDICT_FALSE (fb->flags & VLIB_BUFFER_IS_TRACED))
 812         {
 813           ip4_reass_add_trace (vm, node, rm, reass, *bi0, RANGE_NEW, 0);
 814         }
 815     }
 816   if (~0 != reass->last_packet_octet &&
 817       reass->data_len == reass->last_packet_octet + 1)
 818     {
 819       ip4_reass_finalize (vm, node, rm, rt, reass, bi0, next0, error0,
 820                           vec_drop_compress, vec_drop_overlap, is_feature);
 821     }
 822   else
 823     {
 824       if (consumed)
 825         {
 826           *bi0 = ~0;
 827         }
 828       else
 829         {
 830           *next0 = IP4_REASSEMBLY_NEXT_DROP;
 831           *error0 = IP4_ERROR_REASS_DUPLICATE_FRAGMENT;
 832         }
 833     }
 834 }
 835
 836 always_inline uword
 837 ip4_reassembly_inline (vlib_main_t * vm,
 838                        vlib_node_runtime_t * node,
 839                        vlib_frame_t * frame, bool is_feature)
 840 {
 841   u32 *from = vlib_frame_vector_args (frame);
 842   u32 n_left_from, n_left_to_next, *to_next, next_index;
 843   ip4_reass_main_t *rm = &ip4_reass_main;
 844   ip4_reass_per_thread_t *rt = &rm->per_thread_data[os_get_thread_index ()];
 845   clib_spinlock_lock (&rt->lock);
 846
 847   n_left_from = frame->n_vectors;
 848   next_index = node->cached_next_index;
 849   static u32 *vec_drop_timeout = NULL;  // indexes of buffers which timed out
 850   static u32 *vec_drop_overlap = NULL;  // indexes of buffers which were discarded due to overlap
 851   static u32 *vec_drop_compress = NULL; // indexes of buffers dicarded due to buffer compression
 852   while (n_left_from > 0 || vec_len (vec_drop_timeout) > 0 ||
 853          vec_len (vec_drop_overlap) > 0 || vec_len (vec_drop_compress) > 0)
 854     {
 855       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 856
 857       while (vec_len (vec_drop_timeout) > 0 && n_left_to_next > 0)
 858         {
 859           u32 bi = vec_pop (vec_drop_timeout);
 860           vlib_buffer_t *b = vlib_get_buffer (vm, bi);
 861           b->error = node->errors[IP4_ERROR_REASS_TIMEOUT];
 862           to_next[0] = bi;
 863           to_next += 1;
 864           n_left_to_next -= 1;
 865           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
 866                                            n_left_to_next, bi,
 867                                            IP4_REASSEMBLY_NEXT_DROP);
 868           IP4_REASS_DEBUG_BUFFER (bi, enqueue_drop_timeout);
 869           ASSERT (rt->buffers_n > 0);
 870           --rt->buffers_n;
 871         }
 872
 873       while (vec_len (vec_drop_overlap) > 0 && n_left_to_next > 0)
 874         {
 875           u32 bi = vec_pop (vec_drop_overlap);
 876           vlib_buffer_t *b = vlib_get_buffer (vm, bi);
 877           b->error = node->errors[IP4_ERROR_REASS_DUPLICATE_FRAGMENT];
 878           to_next[0] = bi;
 879           to_next += 1;
 880           n_left_to_next -= 1;
 881           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
 882                                            n_left_to_next, bi,
 883                                            IP4_REASSEMBLY_NEXT_DROP);
 884           IP4_REASS_DEBUG_BUFFER (bi, enqueue_drop_duplicate_fragment);
 885           ASSERT (rt->buffers_n > 0);
 886           --rt->buffers_n;
 887         }
 888
 889       while (vec_len (vec_drop_compress) > 0 && n_left_to_next > 0)
 890         {
 891           u32 bi = vec_pop (vec_drop_compress);
 892           vlib_buffer_t *b = vlib_get_buffer (vm, bi);
 893           b->error = node->errors[IP4_ERROR_NONE];
 894           to_next[0] = bi;
 895           to_next += 1;
 896           n_left_to_next -= 1;
 897           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
 898                                            n_left_to_next, bi,
 899                                            IP4_REASSEMBLY_NEXT_DROP);
 900           IP4_REASS_DEBUG_BUFFER (bi, enqueue_drop_compress);
 901           ASSERT (rt->buffers_n > 0);
 902           --rt->buffers_n;
 903         }
 904
 905       while (n_left_from > 0 && n_left_to_next > 0)
 906         {
 907           u32 bi0;
 908           vlib_buffer_t *b0;
 909           u32 next0;
 910           u32 error0 = IP4_ERROR_NONE;
 911
 912           bi0 = from[0];
 913           b0 = vlib_get_buffer (vm, bi0);
 914
 915           ip4_header_t *ip0 = vlib_buffer_get_current (b0);
 916           if (!ip4_get_fragment_more (ip0) && !ip4_get_fragment_offset (ip0))
 917             {
 918               // this is a whole packet - no fragmentation
 919               if (is_feature)
 920                 {
 921                   next0 = IP4_REASSEMBLY_NEXT_INPUT;
 922                 }
 923               else
 924                 {
 925                   next0 = vnet_buffer (b0)->ip.reass.next_index;
 926                 }
 927             }
 928           else
 929             {
 930               ip4_reass_key_t k;
 931               k.as_u64[0] =
 932                 (u64) vnet_buffer (b0)->sw_if_index[VLIB_RX] << 32 | (u64)
 933                 ip0->src_address.as_u32;
 934               k.as_u64[1] =
 935                 (u64) ip0->dst_address.
 936                 as_u32 << 32 | (u64) ip0->fragment_id << 16 | (u64) ip0->
 937                 protocol << 8;
 938
 939               ip4_reass_t *reass =
 940                 ip4_reass_find_or_create (vm, rm, rt, &k, &vec_drop_timeout);
 941
 942               if (reass)
 943                 {
 944                   ip4_reass_update (vm, node, rm, rt, reass, &bi0, &next0,
 945                                     &error0, &vec_drop_overlap,
 946                                     &vec_drop_compress, is_feature);
 947                 }
 948               else
 949                 {
 950                   next0 = IP4_REASSEMBLY_NEXT_DROP;
 951                   error0 = IP4_ERROR_REASS_LIMIT_REACHED;
 952                 }
 953
 954               b0->error = node->errors[error0];
 955             }
 956
 957           if (bi0 != ~0)
 958             {
 959               to_next[0] = bi0;
 960               to_next += 1;
 961               n_left_to_next -= 1;
 962               if (is_feature && IP4_ERROR_NONE == error0)
 963                 {
 964                   b0 = vlib_get_buffer (vm, bi0);
 965                   vnet_feature_next (&next0, b0);
 966                 }
 967               vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
 968                                                n_left_to_next, bi0, next0);
 969               IP4_REASS_DEBUG_BUFFER (bi0, enqueue_next);
 970             }
 971
 972           from += 1;
 973           n_left_from -= 1;
 974         }
 975
 976       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 977     }
 978
 979   clib_spinlock_unlock (&rt->lock);
 980   return frame->n_vectors;
 981 }
 982
 983 static char *ip4_reassembly_error_strings[] = {
 984 #define _(sym, string) string,
 985   foreach_ip4_error
 986 #undef _
 987 };
 988
 989 static uword
 990 ip4_reassembly (vlib_main_t * vm, vlib_node_runtime_t * node,
 991                 vlib_frame_t * frame)
 992 {
 993   return ip4_reassembly_inline (vm, node, frame, false /* is_feature */ );
 994 }
 995
 996 /* *INDENT-OFF* */
 997 VLIB_REGISTER_NODE (ip4_reass_node, static) = {
 998     .function = ip4_reassembly,
 999     .name = "ip4-reassembly",
1000     .vector_size = sizeof (u32),
1001     .format_trace = format_ip4_reass_trace,
1002     .n_errors = ARRAY_LEN (ip4_reassembly_error_strings),
1003     .error_strings = ip4_reassembly_error_strings,
1004     .n_next_nodes = IP4_REASSEMBLY_N_NEXT,
1005     .next_nodes =
1006         {
1007                 [IP4_REASSEMBLY_NEXT_INPUT] = "ip4-input",
1008                 [IP4_REASSEMBLY_NEXT_DROP] = "ip4-drop",
1009         },
1010 };
1011 /* *INDENT-ON* */
1012
1013 VLIB_NODE_FUNCTION_MULTIARCH (ip4_reass_node, ip4_reassembly);
1014
1015 static uword
1016 ip4_reassembly_feature (vlib_main_t * vm,
1017                         vlib_node_runtime_t * node, vlib_frame_t * frame)
1018 {
1019   return ip4_reassembly_inline (vm, node, frame, true /* is_feature */ );
1020 }
1021
1022 /* *INDENT-OFF* */
1023 VLIB_REGISTER_NODE (ip4_reass_node_feature, static) = {
1024     .function = ip4_reassembly_feature,
1025     .name = "ip4-reassembly-feature",
1026     .vector_size = sizeof (u32),
1027     .format_trace = format_ip4_reass_trace,
1028     .n_errors = ARRAY_LEN (ip4_reassembly_error_strings),
1029     .error_strings = ip4_reassembly_error_strings,
1030     .n_next_nodes = IP4_REASSEMBLY_N_NEXT,
1031     .next_nodes =
1032         {
1033                 [IP4_REASSEMBLY_NEXT_INPUT] = "ip4-input",
1034                 [IP4_REASSEMBLY_NEXT_DROP] = "ip4-drop",
1035         },
1036 };
1037 /* *INDENT-ON* */
1038
1039 VLIB_NODE_FUNCTION_MULTIARCH (ip4_reass_node_feature, ip4_reassembly_feature);
1040
1041 /* *INDENT-OFF* */
1042 VNET_FEATURE_INIT (ip4_reassembly_feature, static) = {
1043     .arc_name = "ip4-unicast",
1044     .node_name = "ip4-reassembly-feature",
1045     .runs_before = VNET_FEATURES ("ip4-lookup"),
1046     .runs_after = 0,
1047 };
1048 /* *INDENT-ON* */
1049
1050 always_inline u32
1051 ip4_reass_get_nbuckets ()
1052 {
1053   ip4_reass_main_t *rm = &ip4_reass_main;
1054   u32 nbuckets;
1055   u8 i;
1056
1057   nbuckets = (u32) (rm->max_reass_n / IP4_REASS_HT_LOAD_FACTOR);
1058
1059   for (i = 0; i < 31; i++)
1060     if ((1 << i) >= nbuckets)
1061       break;
1062   nbuckets = 1 << i;
1063
1064   return nbuckets;
1065 }
1066
1067 typedef enum
1068 {
1069   IP4_EVENT_CONFIG_CHANGED = 1,
1070 } ip4_reass_event_t;
1071
1072 typedef struct
1073 {
1074   int failure;
1075   clib_bihash_16_8_t *new_hash;
1076 } ip4_rehash_cb_ctx;
1077
1078 static void
1079 ip4_rehash_cb (clib_bihash_kv_16_8_t * kv, void *_ctx)
1080 {
1081   ip4_rehash_cb_ctx *ctx = _ctx;
1082   if (clib_bihash_add_del_16_8 (ctx->new_hash, kv, 1))
1083     {
1084       ctx->failure = 1;
1085     }
1086 }
1087
1088 static void
1089 ip4_reass_set_params (u32 timeout_ms, u32 max_reassemblies,
1090                       u32 expire_walk_interval_ms)
1091 {
1092   ip4_reass_main.timeout_ms = timeout_ms;
1093   ip4_reass_main.timeout = (f64) timeout_ms / (f64) MSEC_PER_SEC;
1094   ip4_reass_main.max_reass_n = max_reassemblies;
1095   ip4_reass_main.expire_walk_interval_ms = expire_walk_interval_ms;
1096 }
1097
1098 vnet_api_error_t
1099 ip4_reass_set (u32 timeout_ms, u32 max_reassemblies,
1100                u32 expire_walk_interval_ms)
1101 {
1102   u32 old_nbuckets = ip4_reass_get_nbuckets ();
1103   ip4_reass_set_params (timeout_ms, max_reassemblies,
1104                         expire_walk_interval_ms);
1105   vlib_process_signal_event (ip4_reass_main.vlib_main,
1106                              ip4_reass_main.ip4_reass_expire_node_idx,
1107                              IP4_EVENT_CONFIG_CHANGED, 0);
1108   u32 new_nbuckets = ip4_reass_get_nbuckets ();
1109   if (ip4_reass_main.max_reass_n > 0 && new_nbuckets > old_nbuckets)
1110     {
1111       clib_bihash_16_8_t new_hash;
1112       memset (&new_hash, 0, sizeof (new_hash));
1113       ip4_rehash_cb_ctx ctx;
1114       ctx.failure = 0;
1115       ctx.new_hash = &new_hash;
1116       clib_bihash_init_16_8 (&new_hash, "ip4-reass", new_nbuckets,
1117                              new_nbuckets * 1024);
1118       clib_bihash_foreach_key_value_pair_16_8 (&ip4_reass_main.hash,
1119                                                ip4_rehash_cb, &ctx);
1120       if (ctx.failure)
1121         {
1122           clib_bihash_free_16_8 (&new_hash);
1123           return -1;
1124         }
1125       else
1126         {
1127           clib_bihash_free_16_8 (&ip4_reass_main.hash);
1128           clib_memcpy (&ip4_reass_main.hash, &new_hash,
1129                        sizeof (ip4_reass_main.hash));
1130         }
1131     }
1132   return 0;
1133 }
1134
1135 vnet_api_error_t
1136 ip4_reass_get (u32 * timeout_ms, u32 * max_reassemblies,
1137                u32 * expire_walk_interval_ms)
1138 {
1139   *timeout_ms = ip4_reass_main.timeout_ms;
1140   *max_reassemblies = ip4_reass_main.max_reass_n;
1141   *expire_walk_interval_ms = ip4_reass_main.expire_walk_interval_ms;
1142   return 0;
1143 }
1144
1145 static clib_error_t *
1146 ip4_reass_init_function (vlib_main_t * vm)
1147 {
1148   ip4_reass_main_t *rm = &ip4_reass_main;
1149   clib_error_t *error = 0;
1150   u32 nbuckets;
1151   vlib_node_t *node;
1152
1153   rm->vlib_main = vm;
1154   rm->vnet_main = vnet_get_main ();
1155
1156   vec_validate (rm->per_thread_data, vlib_num_workers ());
1157   ip4_reass_per_thread_t *rt;
1158   vec_foreach (rt, rm->per_thread_data)
1159   {
1160     clib_spinlock_init (&rt->lock);
1161     pool_alloc (rt->pool, rm->max_reass_n);
1162   }
1163
1164   node = vlib_get_node_by_name (vm, (u8 *) "ip4-reassembly-expire-walk");
1165   ASSERT (node);
1166   rm->ip4_reass_expire_node_idx = node->index;
1167
1168   ip4_reass_set_params (IP4_REASS_TIMEOUT_DEFAULT_MS,
1169                         IP4_REASS_MAX_REASSEMBLIES_DEFAULT,
1170                         IP4_REASS_EXPIRE_WALK_INTERVAL_DEFAULT_MS);
1171
1172   nbuckets = ip4_reass_get_nbuckets ();
1173   clib_bihash_init_16_8 (&rm->hash, "ip4-reass", nbuckets, nbuckets * 1024);
1174
1175   node = vlib_get_node_by_name (vm, (u8 *) "ip4-drop");
1176   ASSERT (node);
1177   rm->ip4_drop_idx = node->index;
1178
1179   return error;
1180 }
1181
1182 VLIB_INIT_FUNCTION (ip4_reass_init_function);
1183
1184 static uword
1185 ip4_reass_walk_expired (vlib_main_t * vm,
1186                         vlib_node_runtime_t * node, vlib_frame_t * f)
1187 {
1188   ip4_reass_main_t *rm = &ip4_reass_main;
1189   uword event_type, *event_data = 0;
1190
1191   while (true)
1192     {
1193       vlib_process_wait_for_event_or_clock (vm,
1194                                             (f64) rm->expire_walk_interval_ms
1195                                             / (f64) MSEC_PER_SEC);
1196       event_type = vlib_process_get_events (vm, &event_data);
1197
1198       switch (event_type)
1199         {
1200         case ~0:                /* no events => timeout */
1201           /* nothing to do here */
1202           break;
1203         case IP4_EVENT_CONFIG_CHANGED:
1204           break;
1205         default:
1206           clib_warning ("BUG: event type 0x%wx", event_type);
1207           break;
1208         }
1209       f64 now = vlib_time_now (vm);
1210
1211       ip4_reass_t *reass;
1212       u32 *vec_drop_timeout = NULL;
1213       int *pool_indexes_to_free = NULL;
1214
1215       uword thread_index = 0;
1216       int index;
1217       const uword nthreads = vlib_num_workers () + 1;
1218       for (thread_index = 0; thread_index < nthreads; ++thread_index)
1219         {
1220           ip4_reass_per_thread_t *rt = &rm->per_thread_data[thread_index];
1221           clib_spinlock_lock (&rt->lock);
1222
1223           vec_reset_length (pool_indexes_to_free);
1224           /* *INDENT-OFF* */
1225           pool_foreach_index (index, rt->pool, ({
1226                                 reass = pool_elt_at_index (rt->pool, index);
1227                                 if (now > reass->last_heard + rm->timeout)
1228                                   {
1229                                     vec_add1 (pool_indexes_to_free, index);
1230                                   }
1231                               }));
1232           /* *INDENT-ON* */
1233           int *i;
1234           /* *INDENT-OFF* */
1235           vec_foreach (i, pool_indexes_to_free)
1236           {
1237             ip4_reass_t *reass = pool_elt_at_index (rt->pool, i[0]);
1238             u32 before = vec_len (vec_drop_timeout);
1239             vlib_buffer_t *b = vlib_get_buffer (vm, reass->first_bi);
1240             if (PREDICT_FALSE (b->flags & VLIB_BUFFER_IS_TRACED))
1241               {
1242                 if (pool_is_free_index (vm->trace_main.trace_buffer_pool,
1243                                         b->trace_index))
1244                   {
1245                     /* the trace is gone, don't trace this buffer anymore */
1246                     b->flags &= ~VLIB_BUFFER_IS_TRACED;
1247                   }
1248               }
1249             ip4_reass_on_timeout (vm, rm, reass, &vec_drop_timeout);
1250             u32 after = vec_len (vec_drop_timeout);
1251             ASSERT (rt->buffers_n >= (after - before));
1252             rt->buffers_n -= (after - before);
1253             ip4_reass_free (rm, rt, reass);
1254           }
1255           /* *INDENT-ON* */
1256
1257           clib_spinlock_unlock (&rt->lock);
1258         }
1259
1260       while (vec_len (vec_drop_timeout) > 0)
1261         {
1262           vlib_frame_t *f = vlib_get_frame_to_node (vm, rm->ip4_drop_idx);
1263           u32 *to_next = vlib_frame_vector_args (f);
1264           u32 n_left_to_next = VLIB_FRAME_SIZE - f->n_vectors;
1265           int trace_frame = 0;
1266           while (vec_len (vec_drop_timeout) > 0 && n_left_to_next > 0)
1267             {
1268               u32 bi = vec_pop (vec_drop_timeout);
1269               vlib_buffer_t *b = vlib_get_buffer (vm, bi);
1270               if (PREDICT_FALSE (b->flags & VLIB_BUFFER_IS_TRACED))
1271                 {
1272                   if (pool_is_free_index (vm->trace_main.trace_buffer_pool,
1273                                           b->trace_index))
1274                     {
1275                       /* the trace is gone, don't trace this buffer anymore */
1276                       b->flags &= ~VLIB_BUFFER_IS_TRACED;
1277                     }
1278                   else
1279                     {
1280                       trace_frame = 1;
1281                     }
1282                 }
1283               b->error = node->errors[IP4_ERROR_REASS_TIMEOUT];
1284               to_next[0] = bi;
1285               ++f->n_vectors;
1286               to_next += 1;
1287               n_left_to_next -= 1;
1288               IP4_REASS_DEBUG_BUFFER (bi, enqueue_drop_timeout_walk);
1289             }
1290           f->frame_flags |= (trace_frame * VLIB_FRAME_TRACE);
1291           vlib_put_frame_to_node (vm, rm->ip4_drop_idx, f);
1292         }
1293
1294       vec_free (pool_indexes_to_free);
1295       vec_free (vec_drop_timeout);
1296       if (event_data)
1297         {
1298           _vec_len (event_data) = 0;
1299         }
1300     }
1301
1302   return 0;
1303 }
1304
1305 static vlib_node_registration_t ip4_reass_expire_node;
1306
1307 /* *INDENT-OFF* */
1308 VLIB_REGISTER_NODE (ip4_reass_expire_node, static) = {
1309     .function = ip4_reass_walk_expired,
1310     .type = VLIB_NODE_TYPE_PROCESS,
1311     .name = "ip4-reassembly-expire-walk",
1312     .format_trace = format_ip4_reass_trace,
1313     .n_errors = ARRAY_LEN (ip4_reassembly_error_strings),
1314     .error_strings = ip4_reassembly_error_strings,
1315
1316 };
1317 /* *INDENT-ON* */
1318
1319 static u8 *
1320 format_ip4_reass_key (u8 * s, va_list * args)
1321 {
1322   ip4_reass_key_t *key = va_arg (*args, ip4_reass_key_t *);
1323   s = format (s, "xx_id: %u, src: %U, dst: %U, frag_id: %u, proto: %u",
1324               key->xx_id, format_ip4_address, &key->src, format_ip4_address,
1325               &key->dst, clib_net_to_host_u16 (key->frag_id), key->proto);
1326   return s;
1327 }
1328
1329 static u8 *
1330 format_ip4_reass (u8 * s, va_list * args)
1331 {
1332   vlib_main_t *vm = va_arg (*args, vlib_main_t *);
1333   ip4_reass_t *reass = va_arg (*args, ip4_reass_t *);
1334
1335   s = format (s, "ID: %lu, key: %U\n  first_bi: %u, data_len: %u, "
1336               "last_packet_octet: %u, trace_op_counter: %u\n",
1337               reass->id, format_ip4_reass_key, &reass->key, reass->first_bi,
1338               reass->data_len, reass->last_packet_octet,
1339               reass->trace_op_counter);
1340   u32 bi = reass->first_bi;
1341   u32 counter = 0;
1342   while (~0 != bi)
1343     {
1344       vlib_buffer_t *b = vlib_get_buffer (vm, bi);
1345       vnet_buffer_opaque_t *vnb = vnet_buffer (b);
1346       s = format (s, "  #%03u: range: [%u, %u], bi: %u, off: %d, len: %u, "
1347                   "fragment[%u, %u]\n",
1348                   counter, vnb->ip.reass.range_first,
1349                   vnb->ip.reass.range_last, bi,
1350                   ip4_reass_buffer_get_data_offset_no_check (b),
1351                   ip4_reass_buffer_get_data_len_no_check (b),
1352                   vnb->ip.reass.fragment_first, vnb->ip.reass.fragment_last);
1353       if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
1354         {
1355           bi = b->next_buffer;
1356         }
1357       else
1358         {
1359           bi = ~0;
1360         }
1361     }
1362   return s;
1363 }
1364
1365 static clib_error_t *
1366 show_ip4_reass (vlib_main_t * vm, unformat_input_t * input,
1367                 CLIB_UNUSED (vlib_cli_command_t * lmd))
1368 {
1369   ip4_reass_main_t *rm = &ip4_reass_main;
1370
1371   vlib_cli_output (vm, "---------------------");
1372   vlib_cli_output (vm, "IP4 reassembly status");
1373   vlib_cli_output (vm, "---------------------");
1374   bool details = false;
1375   if (unformat (input, "details"))
1376     {
1377       details = true;
1378     }
1379
1380   u32 sum_reass_n = 0;
1381   u64 sum_buffers_n = 0;
1382   ip4_reass_t *reass;
1383   uword thread_index;
1384   const uword nthreads = vlib_num_workers () + 1;
1385   for (thread_index = 0; thread_index < nthreads; ++thread_index)
1386     {
1387       ip4_reass_per_thread_t *rt = &rm->per_thread_data[thread_index];
1388       clib_spinlock_lock (&rt->lock);
1389       if (details)
1390         {
1391           /* *INDENT-OFF* */
1392           pool_foreach (reass, rt->pool, {
1393             vlib_cli_output (vm, "%U", format_ip4_reass, vm, reass);
1394           });
1395           /* *INDENT-ON* */
1396         }
1397       sum_reass_n += rt->reass_n;
1398       sum_buffers_n += rt->buffers_n;
1399       clib_spinlock_unlock (&rt->lock);
1400     }
1401   vlib_cli_output (vm, "---------------------");
1402   vlib_cli_output (vm, "Current IP4 reassemblies count: %lu\n",
1403                    (long unsigned) sum_reass_n);
1404   vlib_cli_output (vm,
1405                    "Maximum configured concurrent IP4 reassemblies per worker-thread: %lu\n",
1406                    (long unsigned) rm->max_reass_n);
1407   vlib_cli_output (vm, "Buffers in use: %lu\n",
1408                    (long unsigned) sum_buffers_n);
1409   return 0;
1410 }
1411
1412 /* *INDENT-OFF* */
1413 VLIB_CLI_COMMAND (show_ip4_reassembly_cmd, static) = {
1414     .path = "show ip4-reassembly",
1415     .short_help = "show ip4-reassembly [details]",
1416     .function = show_ip4_reass,
1417 };
1418 /* *INDENT-ON* */
1419
1420 vnet_api_error_t
1421 ip4_reass_enable_disable (u32 sw_if_index, u8 enable_disable)
1422 {
1423   return vnet_feature_enable_disable ("ip4-unicast", "ip4-reassembly-feature",
1424                                       sw_if_index, enable_disable, 0, 0);
1425 }
1426
1427 /*
1428  * fd.io coding-style-patch-verification: ON
1429  *
1430  * Local Variables:
1431  * eval: (c-set-style "gnu")
1432  * End:
1433  */