NAT: Refactoring / Housekeeping (VPP-1415)
[vpp.git] / src / plugins / nat / in2out_ed.c
1 /*
2  * Copyright (c) 2018 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /**
16  * @file
17  * @brief NAT44 endpoint-dependent inside to outside network translation
18  */
19
20 #include <vlib/vlib.h>
21 #include <vnet/vnet.h>
22 #include <vnet/pg/pg.h>
23 #include <vnet/ip/ip.h>
24 #include <vnet/ethernet/ethernet.h>
25 #include <vnet/fib/ip4_fib.h>
26 #include <vppinfra/error.h>
27 #include <nat/nat.h>
28 #include <nat/nat_ipfix_logging.h>
29 #include <nat/nat_reass.h>
30 #include <nat/nat_inlines.h>
31
32 #define foreach_nat_in2out_ed_error                       \
33 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
34 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
35 _(OUT_OF_PORTS, "Out of ports")                         \
36 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
37 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
38 _(DROP_FRAGMENT, "Drop fragment")                       \
39 _(MAX_REASS, "Maximum reassemblies exceeded")           \
40 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
41
42 typedef enum
43 {
44 #define _(sym,str) NAT_IN2OUT_ED_ERROR_##sym,
45   foreach_nat_in2out_ed_error
46 #undef _
47     NAT_IN2OUT_ED_N_ERROR,
48 } nat_in2out_ed_error_t;
49
50 static char *nat_in2out_ed_error_strings[] = {
51 #define _(sym,string) string,
52   foreach_nat_in2out_ed_error
53 #undef _
54 };
55
56 typedef enum
57 {
58   NAT_IN2OUT_ED_NEXT_LOOKUP,
59   NAT_IN2OUT_ED_NEXT_DROP,
60   NAT_IN2OUT_ED_NEXT_ICMP_ERROR,
61   NAT_IN2OUT_ED_NEXT_SLOW_PATH,
62   NAT_IN2OUT_ED_NEXT_REASS,
63   NAT_IN2OUT_ED_N_NEXT,
64 } nat_in2out_ed_next_t;
65
66 typedef struct
67 {
68   u32 sw_if_index;
69   u32 next_index;
70   u32 session_index;
71   u32 is_slow_path;
72 } nat_in2out_ed_trace_t;
73
74 vlib_node_registration_t nat44_ed_in2out_node;
75 vlib_node_registration_t nat44_ed_in2out_slowpath_node;
76 vlib_node_registration_t nat44_ed_in2out_output_node;
77 vlib_node_registration_t nat44_ed_in2out_output_slowpath_node;
78 vlib_node_registration_t nat44_ed_in2out_reass_node;
79
80 static u8 *
81 format_nat_in2out_ed_trace (u8 * s, va_list * args)
82 {
83   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
84   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
85   nat_in2out_ed_trace_t *t = va_arg (*args, nat_in2out_ed_trace_t *);
86   char *tag;
87
88   tag =
89     t->is_slow_path ? "NAT44_IN2OUT_ED_SLOW_PATH" :
90     "NAT44_IN2OUT_ED_FAST_PATH";
91
92   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
93               t->sw_if_index, t->next_index, t->session_index);
94
95   return s;
96 }
97
98 static_always_inline int
99 icmp_get_ed_key (ip4_header_t * ip0, nat_ed_ses_key_t * p_key0)
100 {
101   icmp46_header_t *icmp0;
102   nat_ed_ses_key_t key0;
103   icmp_echo_header_t *echo0, *inner_echo0 = 0;
104   ip4_header_t *inner_ip0 = 0;
105   void *l4_header = 0;
106   icmp46_header_t *inner_icmp0;
107
108   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
109   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
110
111   if (!icmp_is_error_message (icmp0))
112     {
113       key0.proto = IP_PROTOCOL_ICMP;
114       key0.l_addr = ip0->src_address;
115       key0.r_addr = ip0->dst_address;
116       key0.l_port = echo0->identifier;
117       key0.r_port = 0;
118     }
119   else
120     {
121       inner_ip0 = (ip4_header_t *) (echo0 + 1);
122       l4_header = ip4_next_header (inner_ip0);
123       key0.proto = inner_ip0->protocol;
124       key0.r_addr = inner_ip0->src_address;
125       key0.l_addr = inner_ip0->dst_address;
126       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
127         {
128         case SNAT_PROTOCOL_ICMP:
129           inner_icmp0 = (icmp46_header_t *) l4_header;
130           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
131           key0.r_port = 0;
132           key0.l_port = inner_echo0->identifier;
133           break;
134         case SNAT_PROTOCOL_UDP:
135         case SNAT_PROTOCOL_TCP:
136           key0.l_port = ((tcp_udp_header_t *) l4_header)->dst_port;
137           key0.r_port = ((tcp_udp_header_t *) l4_header)->src_port;
138           break;
139         default:
140           return NAT_IN2OUT_ED_ERROR_UNSUPPORTED_PROTOCOL;
141         }
142     }
143   *p_key0 = key0;
144   return 0;
145 }
146
147 int
148 nat44_i2o_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
149 {
150   snat_main_t *sm = &snat_main;
151   nat44_is_idle_session_ctx_t *ctx = arg;
152   snat_session_t *s;
153   u64 sess_timeout_time;
154   nat_ed_ses_key_t ed_key;
155   clib_bihash_kv_16_8_t ed_kv;
156   int i;
157   snat_address_t *a;
158   snat_session_key_t key;
159   snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
160                                                        ctx->thread_index);
161
162   s = pool_elt_at_index (tsm->sessions, kv->value);
163   sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
164   if (ctx->now >= sess_timeout_time)
165     {
166       if (is_fwd_bypass_session (s))
167         goto delete;
168
169       ed_key.l_addr = s->out2in.addr;
170       ed_key.r_addr = s->ext_host_addr;
171       ed_key.fib_index = s->out2in.fib_index;
172       if (snat_is_unk_proto_session (s))
173         {
174           ed_key.proto = s->in2out.port;
175           ed_key.r_port = 0;
176           ed_key.l_port = 0;
177         }
178       else
179         {
180           ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
181           ed_key.l_port = s->out2in.port;
182           ed_key.r_port = s->ext_host_port;
183         }
184       ed_kv.key[0] = ed_key.as_u64[0];
185       ed_kv.key[1] = ed_key.as_u64[1];
186       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
187         nat_log_warn ("out2in_ed key del failed");
188
189       if (snat_is_unk_proto_session (s))
190         goto delete;
191
192       snat_ipfix_logging_nat44_ses_delete (s->in2out.addr.as_u32,
193                                            s->out2in.addr.as_u32,
194                                            s->in2out.protocol,
195                                            s->in2out.port,
196                                            s->out2in.port,
197                                            s->in2out.fib_index);
198
199       if (is_twice_nat_session (s))
200         {
201           for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
202             {
203               key.protocol = s->in2out.protocol;
204               key.port = s->ext_host_nat_port;
205               a = sm->twice_nat_addresses + i;
206               if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
207                 {
208                   snat_free_outside_address_and_port (sm->twice_nat_addresses,
209                                                       ctx->thread_index,
210                                                       &key);
211                   break;
212                 }
213             }
214         }
215
216       if (snat_is_session_static (s))
217         goto delete;
218
219       snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
220                                           &s->out2in);
221     delete:
222       nat44_delete_session (sm, s, ctx->thread_index);
223       return 1;
224     }
225
226   return 0;
227 }
228
229 static inline u32
230 icmp_in2out_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
231                           ip4_header_t * ip0, icmp46_header_t * icmp0,
232                           u32 sw_if_index0, u32 rx_fib_index0,
233                           vlib_node_runtime_t * node, u32 next0, f64 now,
234                           u32 thread_index, snat_session_t ** p_s0)
235 {
236   next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
237                        next0, thread_index, p_s0, 0);
238   snat_session_t *s0 = *p_s0;
239   if (PREDICT_TRUE (next0 != NAT_IN2OUT_ED_NEXT_DROP && s0))
240     {
241       /* Hairpinning */
242       if (vnet_buffer (b0)->sw_if_index[VLIB_TX] == ~0)
243         snat_icmp_hairpinning (sm, b0, ip0, icmp0, sm->endpoint_dependent);
244       /* Accounting */
245       nat44_session_update_counters (s0, now,
246                                      vlib_buffer_length_in_chain
247                                      (sm->vlib_main, b0));
248     }
249   return next0;
250 }
251
252 static u32
253 slow_path_ed (snat_main_t * sm,
254               vlib_buffer_t * b,
255               u32 rx_fib_index,
256               clib_bihash_kv_16_8_t * kv,
257               snat_session_t ** sessionp,
258               vlib_node_runtime_t * node, u32 next, u32 thread_index, f64 now)
259 {
260   snat_session_t *s;
261   snat_user_t *u;
262   snat_session_key_t key0, key1;
263   lb_nat_type_t lb = 0, is_sm = 0;
264   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
265   nat_ed_ses_key_t *key = (nat_ed_ses_key_t *) kv->key;
266   u32 proto = ip_proto_to_snat_proto (key->proto);
267   nat_outside_fib_t *outside_fib;
268   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
269   fib_prefix_t pfx = {
270     .fp_proto = FIB_PROTOCOL_IP4,
271     .fp_len = 32,
272     .fp_addr = {
273                 .ip4.as_u32 = key->r_addr.as_u32,
274                 },
275   };
276   nat44_is_idle_session_ctx_t ctx;
277
278   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
279     {
280       b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
281       nat_ipfix_logging_max_sessions (sm->max_translations);
282       nat_log_notice ("maximum sessions exceeded");
283       return NAT_IN2OUT_ED_NEXT_DROP;
284     }
285
286   key0.addr = key->l_addr;
287   key0.port = key->l_port;
288   key1.protocol = key0.protocol = proto;
289   key0.fib_index = rx_fib_index;
290   key1.fib_index = sm->outside_fib_index;
291   /* First try to match static mapping by local address and port */
292   if (snat_static_mapping_match (sm, key0, &key1, 0, 0, 0, &lb, 0))
293     {
294       /* Try to create dynamic translation */
295       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index,
296                                                thread_index, &key1,
297                                                sm->port_per_thread,
298                                                tsm->snat_thread_index))
299         {
300           nat_log_notice ("addresses exhausted");
301           b->error = node->errors[NAT_IN2OUT_ED_ERROR_OUT_OF_PORTS];
302           return NAT_IN2OUT_ED_NEXT_DROP;
303         }
304     }
305   else
306     is_sm = 1;
307
308   u = nat_user_get_or_create (sm, &key->l_addr, rx_fib_index, thread_index);
309   if (!u)
310     {
311       nat_log_warn ("create NAT user failed");
312       if (!is_sm)
313         snat_free_outside_address_and_port (sm->addresses,
314                                             thread_index, &key1);
315       return NAT_IN2OUT_ED_NEXT_DROP;
316     }
317
318   s = nat_ed_session_alloc (sm, u, thread_index);
319   if (!s)
320     {
321       nat44_delete_user_with_no_session (sm, u, thread_index);
322       nat_log_warn ("create NAT session failed");
323       if (!is_sm)
324         snat_free_outside_address_and_port (sm->addresses,
325                                             thread_index, &key1);
326       return NAT_IN2OUT_ED_NEXT_DROP;
327     }
328
329   user_session_increment (sm, u, is_sm);
330   if (is_sm)
331     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
332   if (lb)
333     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
334   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
335   s->ext_host_addr = key->r_addr;
336   s->ext_host_port = key->r_port;
337   s->in2out = key0;
338   s->out2in = key1;
339   s->out2in.protocol = key0.protocol;
340
341   switch (vec_len (sm->outside_fibs))
342     {
343     case 0:
344       s->out2in.fib_index = sm->outside_fib_index;
345       break;
346     case 1:
347       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
348       break;
349     default:
350       /* *INDENT-OFF* */
351       vec_foreach (outside_fib, sm->outside_fibs)
352        {
353           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
354           if (FIB_NODE_INDEX_INVALID != fei)
355             {
356               if (fib_entry_get_resolving_interface (fei) != ~0)
357                 {
358                   s->out2in.fib_index = outside_fib->fib_index;
359                   break;
360                 }
361             }
362         }
363       /* *INDENT-ON* */
364       break;
365     }
366
367   /* Add to lookup tables */
368   kv->value = s - tsm->sessions;
369   ctx.now = now;
370   ctx.thread_index = thread_index;
371   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, kv,
372                                                nat44_i2o_ed_is_idle_session_cb,
373                                                &ctx))
374     nat_log_notice ("in2out-ed key add failed");
375
376   make_ed_kv (kv, &key1.addr, &key->r_addr, key->proto, s->out2in.fib_index,
377               key1.port, key->r_port);
378   kv->value = s - tsm->sessions;
379   if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->out2in_ed, kv,
380                                                nat44_o2i_ed_is_idle_session_cb,
381                                                &ctx))
382     nat_log_notice ("out2in-ed key add failed");
383
384   *sessionp = s;
385
386   /* log NAT event */
387   snat_ipfix_logging_nat44_ses_create (s->in2out.addr.as_u32,
388                                        s->out2in.addr.as_u32,
389                                        s->in2out.protocol,
390                                        s->in2out.port,
391                                        s->out2in.port, s->in2out.fib_index);
392   return next;
393 }
394
395 static_always_inline int
396 nat44_ed_not_translate (snat_main_t * sm, vlib_node_runtime_t * node,
397                         u32 sw_if_index, ip4_header_t * ip, u32 proto,
398                         u32 rx_fib_index, u32 thread_index)
399 {
400   udp_header_t *udp = ip4_next_header (ip);
401   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
402   clib_bihash_kv_16_8_t kv, value;
403   snat_session_key_t key0, key1;
404
405   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, ip->protocol,
406               sm->outside_fib_index, udp->dst_port, udp->src_port);
407
408   /* NAT packet aimed at external address if */
409   /* has active sessions */
410   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
411     {
412       key0.addr = ip->dst_address;
413       key0.port = udp->dst_port;
414       key0.protocol = proto;
415       key0.fib_index = sm->outside_fib_index;
416       /* or is static mappings */
417       if (!snat_static_mapping_match (sm, key0, &key1, 1, 0, 0, 0, 0))
418         return 0;
419     }
420   else
421     return 0;
422
423   if (sm->forwarding_enabled)
424     return 1;
425
426   return snat_not_translate_fast (sm, node, sw_if_index, ip, proto,
427                                   rx_fib_index);
428 }
429
430 static_always_inline int
431 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
432                                       u32 thread_index, f64 now,
433                                       vlib_main_t * vm, vlib_buffer_t * b)
434 {
435   nat_ed_ses_key_t key;
436   clib_bihash_kv_16_8_t kv, value;
437   udp_header_t *udp;
438   snat_session_t *s = 0;
439   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
440
441   if (!sm->forwarding_enabled)
442     return 0;
443
444   if (ip->protocol == IP_PROTOCOL_ICMP)
445     {
446       key.as_u64[0] = key.as_u64[1] = 0;
447       if (icmp_get_ed_key (ip, &key))
448         return 0;
449       key.fib_index = 0;
450       kv.key[0] = key.as_u64[0];
451       kv.key[1] = key.as_u64[1];
452     }
453   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
454     {
455       udp = ip4_next_header (ip);
456       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0,
457                   udp->src_port, udp->dst_port);
458     }
459   else
460     {
461       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0, 0,
462                   0);
463     }
464
465   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
466     {
467       s = pool_elt_at_index (tsm->sessions, value.value);
468       if (is_fwd_bypass_session (s))
469         {
470           if (ip->protocol == IP_PROTOCOL_TCP)
471             {
472               tcp_header_t *tcp = ip4_next_header (ip);
473               if (nat44_set_tcp_session_state_i2o (sm, s, tcp, thread_index))
474                 return 1;
475             }
476           /* Accounting */
477           nat44_session_update_counters (s, now,
478                                          vlib_buffer_length_in_chain (vm, b));
479           return 1;
480         }
481       else
482         return 0;
483     }
484
485   return 0;
486 }
487
488 static_always_inline int
489 nat44_ed_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip,
490                                        u8 proto, u16 src_port, u16 dst_port,
491                                        u32 thread_index, u32 rx_sw_if_index,
492                                        u32 tx_sw_if_index)
493 {
494   clib_bihash_kv_16_8_t kv, value;
495   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
496   snat_interface_t *i;
497   snat_session_t *s;
498   u32 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (rx_sw_if_index);
499   u32 tx_fib_index = ip4_fib_table_get_index_for_sw_if_index (tx_sw_if_index);
500
501   /* src NAT check */
502   make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto, tx_fib_index,
503               src_port, dst_port);
504   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
505     return 1;
506
507   /* dst NAT check */
508   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, proto, rx_fib_index,
509               dst_port, src_port);
510   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
511     {
512       s = pool_elt_at_index (tsm->sessions, value.value);
513       if (is_fwd_bypass_session (s))
514         return 0;
515
516       /* hairpinning */
517       /* *INDENT-OFF* */
518       pool_foreach (i, sm->output_feature_interfaces,
519       ({
520         if ((nat_interface_is_inside (i)) && (rx_sw_if_index == i->sw_if_index))
521            return 0;
522       }));
523       /* *INDENT-ON* */
524       return 1;
525     }
526
527   return 0;
528 }
529
530 u32
531 icmp_match_in2out_ed (snat_main_t * sm, vlib_node_runtime_t * node,
532                       u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
533                       u8 * p_proto, snat_session_key_t * p_value,
534                       u8 * p_dont_translate, void *d, void *e)
535 {
536   icmp46_header_t *icmp;
537   u32 sw_if_index;
538   u32 rx_fib_index;
539   nat_ed_ses_key_t key;
540   snat_session_t *s = 0;
541   u8 dont_translate = 0;
542   clib_bihash_kv_16_8_t kv, value;
543   u32 next = ~0;
544   int err;
545   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
546
547   icmp = (icmp46_header_t *) ip4_next_header (ip);
548   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
549   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
550
551   key.as_u64[0] = key.as_u64[1] = 0;
552   err = icmp_get_ed_key (ip, &key);
553   if (err != 0)
554     {
555       b->error = node->errors[err];
556       next = NAT_IN2OUT_ED_NEXT_DROP;
557       goto out;
558     }
559   key.fib_index = rx_fib_index;
560
561   kv.key[0] = key.as_u64[0];
562   kv.key[1] = key.as_u64[1];
563
564   if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
565     {
566       if (vnet_buffer (b)->sw_if_index[VLIB_TX] != ~0)
567         {
568           if (PREDICT_FALSE (nat44_ed_not_translate_output_feature (sm, ip,
569                                                                     key.proto,
570                                                                     key.
571                                                                     l_port,
572                                                                     key.
573                                                                     r_port,
574                                                                     thread_index,
575                                                                     sw_if_index,
576                                                                     vnet_buffer
577                                                                     (b)->
578                                                                     sw_if_index
579                                                                     [VLIB_TX])))
580             {
581               dont_translate = 1;
582               goto out;
583             }
584         }
585       else
586         {
587           if (PREDICT_FALSE (nat44_ed_not_translate (sm, node, sw_if_index,
588                                                      ip, SNAT_PROTOCOL_ICMP,
589                                                      rx_fib_index,
590                                                      thread_index)))
591             {
592               dont_translate = 1;
593               goto out;
594             }
595         }
596
597       if (PREDICT_FALSE (icmp_is_error_message (icmp)))
598         {
599           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
600           next = NAT_IN2OUT_ED_NEXT_DROP;
601           goto out;
602         }
603
604       next = slow_path_ed (sm, b, rx_fib_index, &kv, &s, node, next,
605                            thread_index, vlib_time_now (sm->vlib_main));
606
607       if (PREDICT_FALSE (next == NAT_IN2OUT_ED_NEXT_DROP))
608         goto out;
609     }
610   else
611     {
612       if (PREDICT_FALSE (icmp->type != ICMP4_echo_request &&
613                          icmp->type != ICMP4_echo_reply &&
614                          !icmp_is_error_message (icmp)))
615         {
616           b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
617           next = NAT_IN2OUT_ED_NEXT_DROP;
618           goto out;
619         }
620
621       s = pool_elt_at_index (tsm->sessions, value.value);
622     }
623
624   *p_proto = ip_proto_to_snat_proto (key.proto);
625 out:
626   if (s)
627     *p_value = s->out2in;
628   *p_dont_translate = dont_translate;
629   if (d)
630     *(snat_session_t **) d = s;
631   return next;
632 }
633
634 static snat_session_t *
635 nat44_ed_in2out_unknown_proto (snat_main_t * sm,
636                                vlib_buffer_t * b,
637                                ip4_header_t * ip,
638                                u32 rx_fib_index,
639                                u32 thread_index,
640                                f64 now,
641                                vlib_main_t * vm, vlib_node_runtime_t * node)
642 {
643   clib_bihash_kv_8_8_t kv, value;
644   clib_bihash_kv_16_8_t s_kv, s_value;
645   snat_static_mapping_t *m;
646   u32 old_addr, new_addr = 0;
647   ip_csum_t sum;
648   snat_user_t *u;
649   dlist_elt_t *head, *elt;
650   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
651   u32 elt_index, head_index, ses_index;
652   snat_session_t *s;
653   u32 outside_fib_index = sm->outside_fib_index;
654   int i;
655   u8 is_sm = 0;
656   nat_outside_fib_t *outside_fib;
657   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
658   fib_prefix_t pfx = {
659     .fp_proto = FIB_PROTOCOL_IP4,
660     .fp_len = 32,
661     .fp_addr = {
662                 .ip4.as_u32 = ip->dst_address.as_u32,
663                 },
664   };
665
666   switch (vec_len (sm->outside_fibs))
667     {
668     case 0:
669       outside_fib_index = sm->outside_fib_index;
670       break;
671     case 1:
672       outside_fib_index = sm->outside_fibs[0].fib_index;
673       break;
674     default:
675       /* *INDENT-OFF* */
676       vec_foreach (outside_fib, sm->outside_fibs)
677         {
678           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
679           if (FIB_NODE_INDEX_INVALID != fei)
680             {
681               if (fib_entry_get_resolving_interface (fei) != ~0)
682                 {
683                   outside_fib_index = outside_fib->fib_index;
684                   break;
685                 }
686             }
687         }
688       /* *INDENT-ON* */
689       break;
690     }
691   old_addr = ip->src_address.as_u32;
692
693   make_ed_kv (&s_kv, &ip->src_address, &ip->dst_address, ip->protocol,
694               rx_fib_index, 0, 0);
695
696   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &s_kv, &s_value))
697     {
698       s = pool_elt_at_index (tsm->sessions, s_value.value);
699       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
700     }
701   else
702     {
703       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
704         {
705           b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
706           nat_ipfix_logging_max_sessions (sm->max_translations);
707           nat_log_notice ("maximum sessions exceeded");
708           return 0;
709         }
710
711       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
712                                   thread_index);
713       if (!u)
714         {
715           nat_log_warn ("create NAT user failed");
716           return 0;
717         }
718
719       make_sm_kv (&kv, &ip->src_address, 0, rx_fib_index, 0);
720
721       /* Try to find static mapping first */
722       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
723         {
724           m = pool_elt_at_index (sm->static_mappings, value.value);
725           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
726           is_sm = 1;
727           goto create_ses;
728         }
729       /* Fallback to 3-tuple key */
730       else
731         {
732           /* Choose same out address as for TCP/UDP session to same destination */
733           head_index = u->sessions_per_user_list_head_index;
734           head = pool_elt_at_index (tsm->list_pool, head_index);
735           elt_index = head->next;
736           if (PREDICT_FALSE (elt_index == ~0))
737             ses_index = ~0;
738           else
739             {
740               elt = pool_elt_at_index (tsm->list_pool, elt_index);
741               ses_index = elt->value;
742             }
743
744           while (ses_index != ~0)
745             {
746               s = pool_elt_at_index (tsm->sessions, ses_index);
747               elt_index = elt->next;
748               elt = pool_elt_at_index (tsm->list_pool, elt_index);
749               ses_index = elt->value;
750
751               if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
752                 {
753                   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
754
755                   make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address,
756                               ip->protocol, outside_fib_index, 0, 0);
757                   if (clib_bihash_search_16_8
758                       (&tsm->out2in_ed, &s_kv, &s_value))
759                     goto create_ses;
760
761                   break;
762                 }
763             }
764
765           for (i = 0; i < vec_len (sm->addresses); i++)
766             {
767               make_ed_kv (&s_kv, &sm->addresses[i].addr, &ip->dst_address,
768                           ip->protocol, outside_fib_index, 0, 0);
769               if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
770                 {
771                   new_addr = ip->src_address.as_u32 =
772                     sm->addresses[i].addr.as_u32;
773                   goto create_ses;
774                 }
775             }
776           return 0;
777         }
778
779     create_ses:
780       s = nat_ed_session_alloc (sm, u, thread_index);
781       if (!s)
782         {
783           nat44_delete_user_with_no_session (sm, u, thread_index);
784           nat_log_warn ("create NAT session failed");
785           return 0;
786         }
787
788       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
789       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
790       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
791       s->out2in.addr.as_u32 = new_addr;
792       s->out2in.fib_index = outside_fib_index;
793       s->in2out.addr.as_u32 = old_addr;
794       s->in2out.fib_index = rx_fib_index;
795       s->in2out.port = s->out2in.port = ip->protocol;
796       if (is_sm)
797         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
798       user_session_increment (sm, u, is_sm);
799
800       /* Add to lookup tables */
801       make_ed_kv (&s_kv, &s->in2out.addr, &ip->dst_address, ip->protocol,
802                   rx_fib_index, 0, 0);
803       s_kv.value = s - tsm->sessions;
804       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
805         nat_log_notice ("in2out key add failed");
806
807       make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address, ip->protocol,
808                   outside_fib_index, 0, 0);
809       s_kv.value = s - tsm->sessions;
810       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
811         nat_log_notice ("out2in key add failed");
812     }
813
814   /* Update IP checksum */
815   sum = ip->checksum;
816   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
817   ip->checksum = ip_csum_fold (sum);
818
819   /* Accounting */
820   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b));
821
822   /* Hairpinning */
823   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
824     nat44_ed_hairpinning_unknown_proto (sm, b, ip);
825
826   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
827     vnet_buffer (b)->sw_if_index[VLIB_TX] = outside_fib_index;
828
829   return s;
830 }
831
832 static inline uword
833 nat44_ed_in2out_node_fn_inline (vlib_main_t * vm,
834                                 vlib_node_runtime_t * node,
835                                 vlib_frame_t * frame, int is_slow_path,
836                                 int is_output_feature)
837 {
838   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
839   nat_in2out_ed_next_t next_index;
840   snat_main_t *sm = &snat_main;
841   f64 now = vlib_time_now (vm);
842   u32 thread_index = vm->thread_index;
843   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
844
845   stats_node_index = is_slow_path ? nat44_ed_in2out_slowpath_node.index :
846     nat44_ed_in2out_node.index;
847
848   from = vlib_frame_vector_args (frame);
849   n_left_from = frame->n_vectors;
850   next_index = node->cached_next_index;
851
852   while (n_left_from > 0)
853     {
854       u32 n_left_to_next;
855
856       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
857
858       while (n_left_from >= 4 && n_left_to_next >= 2)
859         {
860           u32 bi0, bi1;
861           vlib_buffer_t *b0, *b1;
862           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
863             new_addr0, old_addr0;
864           u32 next1, sw_if_index1, rx_fib_index1, iph_offset1 = 0, proto1,
865             new_addr1, old_addr1;
866           u16 old_port0, new_port0, old_port1, new_port1;
867           ip4_header_t *ip0, *ip1;
868           udp_header_t *udp0, *udp1;
869           tcp_header_t *tcp0, *tcp1;
870           icmp46_header_t *icmp0, *icmp1;
871           snat_session_t *s0 = 0, *s1 = 0;
872           clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
873           ip_csum_t sum0, sum1;
874
875           /* Prefetch next iteration. */
876           {
877             vlib_buffer_t *p2, *p3;
878
879             p2 = vlib_get_buffer (vm, from[2]);
880             p3 = vlib_get_buffer (vm, from[3]);
881
882             vlib_prefetch_buffer_header (p2, LOAD);
883             vlib_prefetch_buffer_header (p3, LOAD);
884
885             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
886             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
887           }
888
889           /* speculatively enqueue b0 and b1 to the current next frame */
890           to_next[0] = bi0 = from[0];
891           to_next[1] = bi1 = from[1];
892           from += 2;
893           to_next += 2;
894           n_left_from -= 2;
895           n_left_to_next -= 2;
896
897           b0 = vlib_get_buffer (vm, bi0);
898           b1 = vlib_get_buffer (vm, bi1);
899
900           next0 = NAT_IN2OUT_ED_NEXT_LOOKUP;
901
902           if (is_output_feature)
903             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
904
905           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
906                                   iph_offset0);
907
908           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
909           rx_fib_index0 =
910             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
911                                                  sw_if_index0);
912
913           if (PREDICT_FALSE (ip0->ttl == 1))
914             {
915               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
916               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
917                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
918                                            0);
919               next0 = NAT_IN2OUT_ED_NEXT_ICMP_ERROR;
920               goto trace00;
921             }
922
923           udp0 = ip4_next_header (ip0);
924           tcp0 = (tcp_header_t *) udp0;
925           icmp0 = (icmp46_header_t *) udp0;
926           proto0 = ip_proto_to_snat_proto (ip0->protocol);
927
928           if (is_slow_path)
929             {
930               if (PREDICT_FALSE (proto0 == ~0))
931                 {
932                   s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
933                                                       rx_fib_index0,
934                                                       thread_index, now, vm,
935                                                       node);
936                   if (!s0)
937                     next0 = NAT_IN2OUT_ED_NEXT_DROP;
938                   goto trace00;
939                 }
940
941               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
942                 {
943                   next0 = icmp_in2out_ed_slow_path
944                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
945                      next0, now, thread_index, &s0);
946                   goto trace00;
947                 }
948             }
949           else
950             {
951               if (PREDICT_FALSE (proto0 == ~0))
952                 {
953                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
954                   goto trace00;
955                 }
956
957               if (ip4_is_fragment (ip0))
958                 {
959                   next0 = NAT_IN2OUT_ED_NEXT_REASS;
960                   goto trace00;
961                 }
962
963               if (is_output_feature)
964                 {
965                   if (PREDICT_FALSE
966                       (nat_not_translate_output_feature_fwd
967                        (sm, ip0, thread_index, now, vm, b0)))
968                     goto trace00;
969                 }
970
971               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
972                 {
973                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
974                   goto trace00;
975                 }
976             }
977
978           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
979                       ip0->protocol, rx_fib_index0, udp0->src_port,
980                       udp0->dst_port);
981
982           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
983             {
984               if (is_slow_path)
985                 {
986                   if (is_output_feature)
987                     {
988                       if (PREDICT_FALSE
989                           (nat44_ed_not_translate_output_feature
990                            (sm, ip0, ip0->protocol, udp0->src_port,
991                             udp0->dst_port, thread_index, sw_if_index0,
992                             vnet_buffer (b0)->sw_if_index[VLIB_TX])))
993                         goto trace00;
994                     }
995                   else
996                     {
997                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
998                                                                  sw_if_index0,
999                                                                  ip0, proto0,
1000                                                                  rx_fib_index0,
1001                                                                  thread_index)))
1002                         goto trace00;
1003                     }
1004
1005                   next0 =
1006                     slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
1007                                   next0, thread_index, now);
1008
1009                   if (PREDICT_FALSE (next0 == NAT_IN2OUT_ED_NEXT_DROP))
1010                     goto trace00;
1011                 }
1012               else
1013                 {
1014                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1015                   goto trace00;
1016                 }
1017             }
1018           else
1019             {
1020               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1021             }
1022
1023           b0->flags |= VNET_BUFFER_F_IS_NATED;
1024
1025           if (!is_output_feature)
1026             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1027
1028           old_addr0 = ip0->src_address.as_u32;
1029           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1030           sum0 = ip0->checksum;
1031           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1032                                  src_address);
1033           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1034             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1035                                    s0->ext_host_addr.as_u32, ip4_header_t,
1036                                    dst_address);
1037           ip0->checksum = ip_csum_fold (sum0);
1038
1039           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1040             {
1041               old_port0 = tcp0->src_port;
1042               new_port0 = tcp0->src_port = s0->out2in.port;
1043
1044               sum0 = tcp0->checksum;
1045               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1046                                      dst_address);
1047               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1048                                      length);
1049               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1050                 {
1051                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1052                                          s0->ext_host_addr.as_u32,
1053                                          ip4_header_t, dst_address);
1054                   sum0 = ip_csum_update (sum0, tcp0->dst_port,
1055                                          s0->ext_host_port, ip4_header_t,
1056                                          length);
1057                   tcp0->dst_port = s0->ext_host_port;
1058                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1059                 }
1060               mss_clamping (sm, tcp0, &sum0);
1061               tcp0->checksum = ip_csum_fold (sum0);
1062               if (nat44_set_tcp_session_state_i2o
1063                   (sm, s0, tcp0, thread_index))
1064                 goto trace00;
1065             }
1066           else
1067             {
1068               udp0->src_port = s0->out2in.port;
1069               udp0->checksum = 0;
1070               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1071                 {
1072                   udp0->dst_port = s0->ext_host_port;
1073                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1074                 }
1075             }
1076
1077           /* Accounting */
1078           nat44_session_update_counters (s0, now,
1079                                          vlib_buffer_length_in_chain (vm,
1080                                                                       b0));
1081
1082         trace00:
1083           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1084                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1085             {
1086               nat_in2out_ed_trace_t *t =
1087                 vlib_add_trace (vm, node, b0, sizeof (*t));
1088               t->is_slow_path = is_slow_path;
1089               t->sw_if_index = sw_if_index0;
1090               t->next_index = next0;
1091               t->session_index = ~0;
1092               if (s0)
1093                 t->session_index = s0 - tsm->sessions;
1094             }
1095
1096           pkts_processed += next0 != NAT_IN2OUT_ED_NEXT_DROP;
1097
1098
1099           next1 = NAT_IN2OUT_ED_NEXT_LOOKUP;
1100
1101           if (is_output_feature)
1102             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1103
1104           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1105                                   iph_offset1);
1106
1107           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1108           rx_fib_index1 =
1109             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1110                                                  sw_if_index1);
1111
1112           if (PREDICT_FALSE (ip1->ttl == 1))
1113             {
1114               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1115               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1116                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1117                                            0);
1118               next1 = NAT_IN2OUT_ED_NEXT_ICMP_ERROR;
1119               goto trace01;
1120             }
1121
1122           udp1 = ip4_next_header (ip1);
1123           tcp1 = (tcp_header_t *) udp1;
1124           icmp1 = (icmp46_header_t *) udp1;
1125           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1126
1127           if (is_slow_path)
1128             {
1129               if (PREDICT_FALSE (proto1 == ~0))
1130                 {
1131                   s1 = nat44_ed_in2out_unknown_proto (sm, b1, ip1,
1132                                                       rx_fib_index1,
1133                                                       thread_index, now, vm,
1134                                                       node);
1135                   if (!s1)
1136                     next1 = NAT_IN2OUT_ED_NEXT_DROP;
1137                   goto trace01;
1138                 }
1139
1140               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1141                 {
1142                   next1 = icmp_in2out_ed_slow_path
1143                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1144                      next1, now, thread_index, &s1);
1145                   goto trace01;
1146                 }
1147             }
1148           else
1149             {
1150               if (PREDICT_FALSE (proto1 == ~0))
1151                 {
1152                   next1 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1153                   goto trace01;
1154                 }
1155
1156               if (ip4_is_fragment (ip1))
1157                 {
1158                   next1 = NAT_IN2OUT_ED_NEXT_REASS;
1159                   goto trace01;
1160                 }
1161
1162               if (is_output_feature)
1163                 {
1164                   if (PREDICT_FALSE
1165                       (nat_not_translate_output_feature_fwd
1166                        (sm, ip1, thread_index, now, vm, b1)))
1167                     goto trace01;
1168                 }
1169
1170               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1171                 {
1172                   next1 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1173                   goto trace01;
1174                 }
1175             }
1176
1177           make_ed_kv (&kv1, &ip1->src_address, &ip1->dst_address,
1178                       ip1->protocol, rx_fib_index1, udp1->src_port,
1179                       udp1->dst_port);
1180
1181           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv1, &value1))
1182             {
1183               if (is_slow_path)
1184                 {
1185                   if (is_output_feature)
1186                     {
1187                       if (PREDICT_FALSE
1188                           (nat44_ed_not_translate_output_feature
1189                            (sm, ip1, ip1->protocol, udp1->src_port,
1190                             udp1->dst_port, thread_index, sw_if_index1,
1191                             vnet_buffer (b1)->sw_if_index[VLIB_TX])))
1192                         goto trace01;
1193                     }
1194                   else
1195                     {
1196                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1197                                                                  sw_if_index1,
1198                                                                  ip1, proto1,
1199                                                                  rx_fib_index1,
1200                                                                  thread_index)))
1201                         goto trace01;
1202                     }
1203
1204                   next1 =
1205                     slow_path_ed (sm, b1, rx_fib_index1, &kv1, &s1, node,
1206                                   next1, thread_index, now);
1207
1208                   if (PREDICT_FALSE (next1 == NAT_IN2OUT_ED_NEXT_DROP))
1209                     goto trace01;
1210                 }
1211               else
1212                 {
1213                   next1 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1214                   goto trace01;
1215                 }
1216             }
1217           else
1218             {
1219               s1 = pool_elt_at_index (tsm->sessions, value1.value);
1220             }
1221
1222           b1->flags |= VNET_BUFFER_F_IS_NATED;
1223
1224           if (!is_output_feature)
1225             vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1226
1227           old_addr1 = ip1->src_address.as_u32;
1228           new_addr1 = ip1->src_address.as_u32 = s1->out2in.addr.as_u32;
1229           sum1 = ip1->checksum;
1230           sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1231                                  src_address);
1232           if (PREDICT_FALSE (is_twice_nat_session (s1)))
1233             sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
1234                                    s1->ext_host_addr.as_u32, ip4_header_t,
1235                                    dst_address);
1236           ip1->checksum = ip_csum_fold (sum1);
1237
1238           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1239             {
1240               old_port1 = tcp1->src_port;
1241               new_port1 = tcp1->src_port = s1->out2in.port;
1242
1243               sum1 = tcp1->checksum;
1244               sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1245                                      dst_address);
1246               sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
1247                                      length);
1248               if (PREDICT_FALSE (is_twice_nat_session (s1)))
1249                 {
1250                   sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
1251                                          s1->ext_host_addr.as_u32,
1252                                          ip4_header_t, dst_address);
1253                   sum1 = ip_csum_update (sum1, tcp1->dst_port,
1254                                          s1->ext_host_port, ip4_header_t,
1255                                          length);
1256                   tcp1->dst_port = s1->ext_host_port;
1257                   ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
1258                 }
1259               tcp1->checksum = ip_csum_fold (sum1);
1260               mss_clamping (sm, tcp1, &sum1);
1261               if (nat44_set_tcp_session_state_i2o
1262                   (sm, s1, tcp1, thread_index))
1263                 goto trace01;
1264             }
1265           else
1266             {
1267               udp1->src_port = s1->out2in.port;
1268               udp1->checksum = 0;
1269               if (PREDICT_FALSE (is_twice_nat_session (s1)))
1270                 {
1271                   udp1->dst_port = s1->ext_host_port;
1272                   ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
1273                 }
1274             }
1275
1276           /* Accounting */
1277           nat44_session_update_counters (s1, now,
1278                                          vlib_buffer_length_in_chain (vm,
1279                                                                       b1));
1280
1281         trace01:
1282           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1283                              && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1284             {
1285               nat_in2out_ed_trace_t *t =
1286                 vlib_add_trace (vm, node, b1, sizeof (*t));
1287               t->is_slow_path = is_slow_path;
1288               t->sw_if_index = sw_if_index1;
1289               t->next_index = next1;
1290               t->session_index = ~0;
1291               if (s1)
1292                 t->session_index = s1 - tsm->sessions;
1293             }
1294
1295           pkts_processed += next1 != NAT_IN2OUT_ED_NEXT_DROP;
1296
1297           /* verify speculative enqueues, maybe switch current next frame */
1298           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1299                                            to_next, n_left_to_next,
1300                                            bi0, bi1, next0, next1);
1301         }
1302
1303       while (n_left_from > 0 && n_left_to_next > 0)
1304         {
1305           u32 bi0;
1306           vlib_buffer_t *b0;
1307           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
1308             new_addr0, old_addr0;
1309           u16 old_port0, new_port0;
1310           ip4_header_t *ip0;
1311           udp_header_t *udp0;
1312           tcp_header_t *tcp0;
1313           icmp46_header_t *icmp0;
1314           snat_session_t *s0 = 0;
1315           clib_bihash_kv_16_8_t kv0, value0;
1316           ip_csum_t sum0;
1317
1318           /* speculatively enqueue b0 to the current next frame */
1319           bi0 = from[0];
1320           to_next[0] = bi0;
1321           from += 1;
1322           to_next += 1;
1323           n_left_from -= 1;
1324           n_left_to_next -= 1;
1325
1326           b0 = vlib_get_buffer (vm, bi0);
1327           next0 = NAT_IN2OUT_ED_NEXT_LOOKUP;
1328
1329           if (is_output_feature)
1330             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1331
1332           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1333                                   iph_offset0);
1334
1335           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1336           rx_fib_index0 =
1337             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1338                                                  sw_if_index0);
1339
1340           if (PREDICT_FALSE (ip0->ttl == 1))
1341             {
1342               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1343               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1344                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1345                                            0);
1346               next0 = NAT_IN2OUT_ED_NEXT_ICMP_ERROR;
1347               goto trace0;
1348             }
1349
1350           udp0 = ip4_next_header (ip0);
1351           tcp0 = (tcp_header_t *) udp0;
1352           icmp0 = (icmp46_header_t *) udp0;
1353           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1354
1355           if (is_slow_path)
1356             {
1357               if (PREDICT_FALSE (proto0 == ~0))
1358                 {
1359                   s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
1360                                                       rx_fib_index0,
1361                                                       thread_index, now, vm,
1362                                                       node);
1363                   if (!s0)
1364                     next0 = NAT_IN2OUT_ED_NEXT_DROP;
1365                   goto trace0;
1366                 }
1367
1368               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1369                 {
1370                   next0 = icmp_in2out_ed_slow_path
1371                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1372                      next0, now, thread_index, &s0);
1373                   goto trace0;
1374                 }
1375             }
1376           else
1377             {
1378               if (PREDICT_FALSE (proto0 == ~0))
1379                 {
1380                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1381                   goto trace0;
1382                 }
1383
1384               if (ip4_is_fragment (ip0))
1385                 {
1386                   next0 = NAT_IN2OUT_ED_NEXT_REASS;
1387                   goto trace0;
1388                 }
1389
1390               if (is_output_feature)
1391                 {
1392                   if (PREDICT_FALSE
1393                       (nat_not_translate_output_feature_fwd
1394                        (sm, ip0, thread_index, now, vm, b0)))
1395                     goto trace0;
1396                 }
1397
1398               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1399                 {
1400                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1401                   goto trace0;
1402                 }
1403             }
1404
1405           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
1406                       ip0->protocol, rx_fib_index0, udp0->src_port,
1407                       udp0->dst_port);
1408
1409           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
1410             {
1411               if (is_slow_path)
1412                 {
1413                   if (is_output_feature)
1414                     {
1415                       if (PREDICT_FALSE
1416                           (nat44_ed_not_translate_output_feature
1417                            (sm, ip0, ip0->protocol, udp0->src_port,
1418                             udp0->dst_port, thread_index, sw_if_index0,
1419                             vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1420                         goto trace0;
1421                     }
1422                   else
1423                     {
1424                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1425                                                                  sw_if_index0,
1426                                                                  ip0, proto0,
1427                                                                  rx_fib_index0,
1428                                                                  thread_index)))
1429                         goto trace0;
1430                     }
1431
1432                   next0 =
1433                     slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
1434                                   next0, thread_index, now);
1435
1436                   if (PREDICT_FALSE (next0 == NAT_IN2OUT_ED_NEXT_DROP))
1437                     goto trace0;
1438                 }
1439               else
1440                 {
1441                   next0 = NAT_IN2OUT_ED_NEXT_SLOW_PATH;
1442                   goto trace0;
1443                 }
1444             }
1445           else
1446             {
1447               s0 = pool_elt_at_index (tsm->sessions, value0.value);
1448             }
1449
1450           b0->flags |= VNET_BUFFER_F_IS_NATED;
1451
1452           if (!is_output_feature)
1453             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1454
1455           old_addr0 = ip0->src_address.as_u32;
1456           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1457           sum0 = ip0->checksum;
1458           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1459                                  src_address);
1460           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1461             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1462                                    s0->ext_host_addr.as_u32, ip4_header_t,
1463                                    dst_address);
1464           ip0->checksum = ip_csum_fold (sum0);
1465
1466           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1467             {
1468               old_port0 = tcp0->src_port;
1469               new_port0 = tcp0->src_port = s0->out2in.port;
1470
1471               sum0 = tcp0->checksum;
1472               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1473                                      dst_address);
1474               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1475                                      length);
1476               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1477                 {
1478                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1479                                          s0->ext_host_addr.as_u32,
1480                                          ip4_header_t, dst_address);
1481                   sum0 = ip_csum_update (sum0, tcp0->dst_port,
1482                                          s0->ext_host_port, ip4_header_t,
1483                                          length);
1484                   tcp0->dst_port = s0->ext_host_port;
1485                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1486                 }
1487               mss_clamping (sm, tcp0, &sum0);
1488               tcp0->checksum = ip_csum_fold (sum0);
1489               if (nat44_set_tcp_session_state_i2o
1490                   (sm, s0, tcp0, thread_index))
1491                 goto trace0;
1492             }
1493           else
1494             {
1495               udp0->src_port = s0->out2in.port;
1496               udp0->checksum = 0;
1497               if (PREDICT_FALSE (is_twice_nat_session (s0)))
1498                 {
1499                   udp0->dst_port = s0->ext_host_port;
1500                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1501                 }
1502             }
1503
1504           /* Accounting */
1505           nat44_session_update_counters (s0, now,
1506                                          vlib_buffer_length_in_chain (vm,
1507                                                                       b0));
1508
1509         trace0:
1510           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1511                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1512             {
1513               nat_in2out_ed_trace_t *t =
1514                 vlib_add_trace (vm, node, b0, sizeof (*t));
1515               t->is_slow_path = is_slow_path;
1516               t->sw_if_index = sw_if_index0;
1517               t->next_index = next0;
1518               t->session_index = ~0;
1519               if (s0)
1520                 t->session_index = s0 - tsm->sessions;
1521             }
1522
1523           pkts_processed += next0 != NAT_IN2OUT_ED_NEXT_DROP;
1524
1525           /* verify speculative enqueue, maybe switch current next frame */
1526           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1527                                            to_next, n_left_to_next,
1528                                            bi0, next0);
1529         }
1530
1531       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1532     }
1533
1534   vlib_node_increment_counter (vm, stats_node_index,
1535                                NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
1536                                pkts_processed);
1537   return frame->n_vectors;
1538 }
1539
1540 static uword
1541 nat44_ed_in2out_fast_path_fn (vlib_main_t * vm,
1542                               vlib_node_runtime_t * node,
1543                               vlib_frame_t * frame)
1544 {
1545   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 0);
1546 }
1547
1548 /* *INDENT-OFF* */
1549 VLIB_REGISTER_NODE (nat44_ed_in2out_node) = {
1550   .function = nat44_ed_in2out_fast_path_fn,
1551   .name = "nat44-ed-in2out",
1552   .vector_size = sizeof (u32),
1553   .format_trace = format_nat_in2out_ed_trace,
1554   .type = VLIB_NODE_TYPE_INTERNAL,
1555   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1556   .error_strings = nat_in2out_ed_error_strings,
1557   .runtime_data_bytes = sizeof (snat_runtime_t),
1558   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1559   .next_nodes = {
1560     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1561     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "ip4-lookup",
1562     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
1563     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1564     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
1565   },
1566 };
1567 /* *INDENT-ON* */
1568
1569 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_node,
1570                               nat44_ed_in2out_fast_path_fn);
1571
1572 static uword
1573 nat44_ed_in2out_output_fast_path_fn (vlib_main_t * vm,
1574                                      vlib_node_runtime_t * node,
1575                                      vlib_frame_t * frame)
1576 {
1577   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 1);
1578 }
1579
1580 /* *INDENT-OFF* */
1581 VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = {
1582   .function = nat44_ed_in2out_output_fast_path_fn,
1583   .name = "nat44-ed-in2out-output",
1584   .vector_size = sizeof (u32),
1585   .format_trace = format_nat_in2out_ed_trace,
1586   .type = VLIB_NODE_TYPE_INTERNAL,
1587   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1588   .error_strings = nat_in2out_ed_error_strings,
1589   .runtime_data_bytes = sizeof (snat_runtime_t),
1590   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1591   .next_nodes = {
1592     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1593     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "interface-output",
1594     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
1595     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1596     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass-output",
1597   },
1598 };
1599 /* *INDENT-ON* */
1600
1601 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_node,
1602                               nat44_ed_in2out_output_fast_path_fn);
1603
1604 static uword
1605 nat44_ed_in2out_slow_path_fn (vlib_main_t * vm,
1606                               vlib_node_runtime_t * node,
1607                               vlib_frame_t * frame)
1608 {
1609   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 0);
1610 }
1611
1612 /* *INDENT-OFF* */
1613 VLIB_REGISTER_NODE (nat44_ed_in2out_slowpath_node) = {
1614   .function = nat44_ed_in2out_slow_path_fn,
1615   .name = "nat44-ed-in2out-slowpath",
1616   .vector_size = sizeof (u32),
1617   .format_trace = format_nat_in2out_ed_trace,
1618   .type = VLIB_NODE_TYPE_INTERNAL,
1619   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1620   .error_strings = nat_in2out_ed_error_strings,
1621   .runtime_data_bytes = sizeof (snat_runtime_t),
1622   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1623   .next_nodes = {
1624     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1625     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "ip4-lookup",
1626     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
1627     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1628     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
1629   },
1630 };
1631 /* *INDENT-ON* */
1632
1633 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_slowpath_node,
1634                               nat44_ed_in2out_slow_path_fn);
1635
1636 static uword
1637 nat44_ed_in2out_output_slow_path_fn (vlib_main_t * vm,
1638                                      vlib_node_runtime_t * node,
1639                                      vlib_frame_t * frame)
1640 {
1641   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 1);
1642 }
1643
1644 /* *INDENT-OFF* */
1645 VLIB_REGISTER_NODE (nat44_ed_in2out_output_slowpath_node) = {
1646   .function = nat44_ed_in2out_output_slow_path_fn,
1647   .name = "nat44-ed-in2out-output-slowpath",
1648   .vector_size = sizeof (u32),
1649   .format_trace = format_nat_in2out_ed_trace,
1650   .type = VLIB_NODE_TYPE_INTERNAL,
1651   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
1652   .error_strings = nat_in2out_ed_error_strings,
1653   .runtime_data_bytes = sizeof (snat_runtime_t),
1654   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
1655   .next_nodes = {
1656     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
1657     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "interface-output",
1658     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
1659     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1660     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
1661   },
1662 };
1663 /* *INDENT-ON* */
1664
1665 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_slowpath_node,
1666                               nat44_ed_in2out_output_slow_path_fn);
1667
1668 static inline uword
1669 nat44_ed_in2out_reass_node_fn_inline (vlib_main_t * vm,
1670                                       vlib_node_runtime_t * node,
1671                                       vlib_frame_t * frame,
1672                                       int is_output_feature)
1673 {
1674   u32 n_left_from, *from, *to_next;
1675   nat_in2out_ed_next_t next_index;
1676   u32 pkts_processed = 0;
1677   snat_main_t *sm = &snat_main;
1678   f64 now = vlib_time_now (vm);
1679   u32 thread_index = vm->thread_index;
1680   snat_main_per_thread_data_t *per_thread_data =
1681     &sm->per_thread_data[thread_index];
1682   u32 *fragments_to_drop = 0;
1683   u32 *fragments_to_loopback = 0;
1684
1685   from = vlib_frame_vector_args (frame);
1686   n_left_from = frame->n_vectors;
1687   next_index = node->cached_next_index;
1688
1689   while (n_left_from > 0)
1690     {
1691       u32 n_left_to_next;
1692
1693       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1694
1695       while (n_left_from > 0 && n_left_to_next > 0)
1696         {
1697           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1698           u32 iph_offset0 = 0;
1699           vlib_buffer_t *b0;
1700           u32 next0;
1701           u8 cached0 = 0;
1702           ip4_header_t *ip0 = 0;
1703           nat_reass_ip4_t *reass0;
1704           udp_header_t *udp0;
1705           tcp_header_t *tcp0;
1706           icmp46_header_t *icmp0;
1707           clib_bihash_kv_16_8_t kv0, value0;
1708           snat_session_t *s0 = 0;
1709           u16 old_port0, new_port0;
1710           ip_csum_t sum0;
1711
1712           /* speculatively enqueue b0 to the current next frame */
1713           bi0 = from[0];
1714           to_next[0] = bi0;
1715           from += 1;
1716           to_next += 1;
1717           n_left_from -= 1;
1718           n_left_to_next -= 1;
1719
1720           b0 = vlib_get_buffer (vm, bi0);
1721
1722           next0 = NAT_IN2OUT_ED_NEXT_LOOKUP;
1723
1724           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1725           rx_fib_index0 =
1726             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1727                                                  sw_if_index0);
1728
1729           if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
1730             {
1731               next0 = NAT_IN2OUT_ED_NEXT_DROP;
1732               b0->error = node->errors[NAT_IN2OUT_ED_ERROR_DROP_FRAGMENT];
1733               goto trace0;
1734             }
1735
1736           if (is_output_feature)
1737             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1738
1739           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1740                                   iph_offset0);
1741
1742           udp0 = ip4_next_header (ip0);
1743           tcp0 = (tcp_header_t *) udp0;
1744           icmp0 = (icmp46_header_t *) udp0;
1745           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1746
1747           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1748                                                  ip0->dst_address,
1749                                                  ip0->fragment_id,
1750                                                  ip0->protocol,
1751                                                  1, &fragments_to_drop);
1752
1753           if (PREDICT_FALSE (!reass0))
1754             {
1755               next0 = NAT_IN2OUT_ED_NEXT_DROP;
1756               b0->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_REASS];
1757               nat_log_notice ("maximum reassemblies exceeded");
1758               goto trace0;
1759             }
1760
1761           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1762             {
1763               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1764                 {
1765                   if (is_output_feature)
1766                     {
1767                       if (PREDICT_FALSE
1768                           (nat_not_translate_output_feature_fwd
1769                            (sm, ip0, thread_index, now, vm, b0)))
1770                         reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1771                       goto trace0;
1772                     }
1773
1774                   next0 = icmp_in2out_ed_slow_path
1775                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1776                      next0, now, thread_index, &s0);
1777
1778                   if (PREDICT_TRUE (next0 != NAT_IN2OUT_ED_NEXT_DROP))
1779                     {
1780                       if (s0)
1781                         reass0->sess_index = s0 - per_thread_data->sessions;
1782                       else
1783                         reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1784                       nat_ip4_reass_get_frags (reass0,
1785                                                &fragments_to_loopback);
1786                     }
1787
1788                   goto trace0;
1789                 }
1790
1791               make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
1792                           ip0->protocol, rx_fib_index0, udp0->src_port,
1793                           udp0->dst_port);
1794
1795               if (clib_bihash_search_16_8
1796                   (&per_thread_data->in2out_ed, &kv0, &value0))
1797                 {
1798                   if (is_output_feature)
1799                     {
1800                       if (PREDICT_FALSE
1801                           (nat44_ed_not_translate_output_feature
1802                            (sm, ip0, ip0->protocol, udp0->src_port,
1803                             udp0->dst_port, thread_index, sw_if_index0,
1804                             vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1805                         {
1806                           reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1807                           nat_ip4_reass_get_frags (reass0,
1808                                                    &fragments_to_loopback);
1809                           goto trace0;
1810                         }
1811                     }
1812                   else
1813                     {
1814                       if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1815                                                                  sw_if_index0,
1816                                                                  ip0, proto0,
1817                                                                  rx_fib_index0,
1818                                                                  thread_index)))
1819                         {
1820                           reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1821                           nat_ip4_reass_get_frags (reass0,
1822                                                    &fragments_to_loopback);
1823                           goto trace0;
1824                         }
1825                     }
1826
1827                   next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0,
1828                                         &s0, node, next0, thread_index, now);
1829
1830                   if (PREDICT_FALSE (next0 == NAT_IN2OUT_ED_NEXT_DROP))
1831                     goto trace0;
1832
1833                   reass0->sess_index = s0 - per_thread_data->sessions;
1834                 }
1835               else
1836                 {
1837                   s0 = pool_elt_at_index (per_thread_data->sessions,
1838                                           value0.value);
1839                   reass0->sess_index = value0.value;
1840                 }
1841               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1842             }
1843           else
1844             {
1845               if (reass0->flags & NAT_REASS_FLAG_ED_DONT_TRANSLATE)
1846                 goto trace0;
1847               if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1848                 {
1849                   if (nat_ip4_reass_add_fragment
1850                       (reass0, bi0, &fragments_to_drop))
1851                     {
1852                       b0->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_FRAG];
1853                       nat_log_notice
1854                         ("maximum fragments per reassembly exceeded");
1855                       next0 = NAT_IN2OUT_ED_NEXT_DROP;
1856                       goto trace0;
1857                     }
1858                   cached0 = 1;
1859                   goto trace0;
1860                 }
1861               s0 = pool_elt_at_index (per_thread_data->sessions,
1862                                       reass0->sess_index);
1863             }
1864
1865           old_addr0 = ip0->src_address.as_u32;
1866           ip0->src_address = s0->out2in.addr;
1867           new_addr0 = ip0->src_address.as_u32;
1868           if (!is_output_feature)
1869             vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1870
1871           sum0 = ip0->checksum;
1872           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1873                                  ip4_header_t,
1874                                  src_address /* changed member */ );
1875           if (PREDICT_FALSE (is_twice_nat_session (s0)))
1876             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1877                                    s0->ext_host_addr.as_u32, ip4_header_t,
1878                                    dst_address);
1879           ip0->checksum = ip_csum_fold (sum0);
1880
1881           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1882             {
1883               if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1884                 {
1885                   old_port0 = tcp0->src_port;
1886                   tcp0->src_port = s0->out2in.port;
1887                   new_port0 = tcp0->src_port;
1888
1889                   sum0 = tcp0->checksum;
1890                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1891                                          ip4_header_t,
1892                                          dst_address /* changed member */ );
1893                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1894                                          ip4_header_t /* cheat */ ,
1895                                          length /* changed member */ );
1896                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1897                     {
1898                       sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1899                                              s0->ext_host_addr.as_u32,
1900                                              ip4_header_t, dst_address);
1901                       sum0 = ip_csum_update (sum0, tcp0->dst_port,
1902                                              s0->ext_host_port, ip4_header_t,
1903                                              length);
1904                       tcp0->dst_port = s0->ext_host_port;
1905                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1906                     }
1907                   tcp0->checksum = ip_csum_fold (sum0);
1908                 }
1909               else
1910                 {
1911                   old_port0 = udp0->src_port;
1912                   udp0->src_port = s0->out2in.port;
1913                   udp0->checksum = 0;
1914                   if (PREDICT_FALSE (is_twice_nat_session (s0)))
1915                     {
1916                       udp0->dst_port = s0->ext_host_port;
1917                       ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1918                     }
1919                 }
1920             }
1921
1922           /* Hairpinning */
1923           if (PREDICT_TRUE (proto0 != SNAT_PROTOCOL_ICMP))
1924             nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
1925                                      s0->ext_host_port, proto0, 1);
1926           else
1927             snat_icmp_hairpinning (sm, b0, ip0, icmp0, 1);
1928
1929           /* Accounting */
1930           nat44_session_update_counters (s0, now,
1931                                          vlib_buffer_length_in_chain (vm,
1932                                                                       b0));
1933
1934         trace0:
1935           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1936                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1937             {
1938               nat44_reass_trace_t *t =
1939                 vlib_add_trace (vm, node, b0, sizeof (*t));
1940               t->cached = cached0;
1941               t->sw_if_index = sw_if_index0;
1942               t->next_index = next0;
1943             }
1944
1945           if (cached0)
1946             {
1947               n_left_to_next++;
1948               to_next--;
1949             }
1950           else
1951             {
1952               pkts_processed += next0 != NAT_IN2OUT_ED_NEXT_DROP;
1953
1954               /* verify speculative enqueue, maybe switch current next frame */
1955               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1956                                                to_next, n_left_to_next,
1957                                                bi0, next0);
1958             }
1959
1960           if (n_left_from == 0 && vec_len (fragments_to_loopback))
1961             {
1962               from = vlib_frame_vector_args (frame);
1963               u32 len = vec_len (fragments_to_loopback);
1964               if (len <= VLIB_FRAME_SIZE)
1965                 {
1966                   clib_memcpy (from, fragments_to_loopback,
1967                                sizeof (u32) * len);
1968                   n_left_from = len;
1969                   vec_reset_length (fragments_to_loopback);
1970                 }
1971               else
1972                 {
1973                   clib_memcpy (from,
1974                                fragments_to_loopback + (len -
1975                                                         VLIB_FRAME_SIZE),
1976                                sizeof (u32) * VLIB_FRAME_SIZE);
1977                   n_left_from = VLIB_FRAME_SIZE;
1978                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1979                 }
1980             }
1981         }
1982
1983       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1984     }
1985
1986   vlib_node_increment_counter (vm, nat44_ed_in2out_reass_node.index,
1987                                NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
1988                                pkts_processed);
1989
1990   nat_send_all_to_node (vm, fragments_to_drop, node,
1991                         &node->errors[NAT_IN2OUT_ED_ERROR_DROP_FRAGMENT],
1992                         NAT_IN2OUT_ED_NEXT_DROP);
1993
1994   vec_free (fragments_to_drop);
1995   vec_free (fragments_to_loopback);
1996   return frame->n_vectors;
1997 }
1998
1999 static uword
2000 nat44_ed_in2out_reass_node_fn (vlib_main_t * vm,
2001                                vlib_node_runtime_t * node,
2002                                vlib_frame_t * frame)
2003 {
2004   return nat44_ed_in2out_reass_node_fn_inline (vm, node, frame, 0);
2005 }
2006
2007 /* *INDENT-OFF* */
2008 VLIB_REGISTER_NODE (nat44_ed_in2out_reass_node) = {
2009   .function = nat44_ed_in2out_reass_node_fn,
2010   .name = "nat44-ed-in2out-reass",
2011   .vector_size = sizeof (u32),
2012   .format_trace = format_nat44_reass_trace,
2013   .type = VLIB_NODE_TYPE_INTERNAL,
2014   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
2015   .error_strings = nat_in2out_ed_error_strings,
2016   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
2017   .next_nodes = {
2018     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
2019     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "ip4-lookup",
2020     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2021     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2022     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
2023   },
2024 };
2025 /* *INDENT-ON* */
2026
2027 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_reass_node,
2028                               nat44_ed_in2out_reass_node_fn);
2029
2030 static uword
2031 nat44_ed_in2out_reass_output_node_fn (vlib_main_t * vm,
2032                                       vlib_node_runtime_t * node,
2033                                       vlib_frame_t * frame)
2034 {
2035   return nat44_ed_in2out_reass_node_fn_inline (vm, node, frame, 1);
2036 }
2037
2038 /* *INDENT-OFF* */
2039 VLIB_REGISTER_NODE (nat44_ed_in2out_reass_output_node) = {
2040   .function = nat44_ed_in2out_reass_output_node_fn,
2041   .name = "nat44-ed-in2out-reass-output",
2042   .vector_size = sizeof (u32),
2043   .format_trace = format_nat44_reass_trace,
2044   .type = VLIB_NODE_TYPE_INTERNAL,
2045   .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
2046   .error_strings = nat_in2out_ed_error_strings,
2047   .n_next_nodes = NAT_IN2OUT_ED_N_NEXT,
2048   .next_nodes = {
2049     [NAT_IN2OUT_ED_NEXT_DROP] = "error-drop",
2050     [NAT_IN2OUT_ED_NEXT_LOOKUP] = "interface-output",
2051     [NAT_IN2OUT_ED_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2052     [NAT_IN2OUT_ED_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2053     [NAT_IN2OUT_ED_NEXT_REASS] = "nat44-ed-in2out-reass",
2054   },
2055 };
2056 /* *INDENT-ON* */
2057
2058 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_reass_output_node,
2059                               nat44_ed_in2out_reass_output_node_fn);
2060
2061 /*
2062  * fd.io coding-style-patch-verification: ON
2063  *
2064  * Local Variables:
2065  * eval: (c-set-style "gnu")
2066  * End:
2067  */