Remove c-11 memcpy checks from perf-critical code
[vpp.git] / src / plugins / nat / nat44_classify.c
1 /*
2  * Copyright (c) 2018 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /**
16  * @file
17  * @brief Classify for one armed NAT44 (in+out interface)
18  */
19
20 #include <vlib/vlib.h>
21 #include <vnet/vnet.h>
22 #include <vnet/fib/ip4_fib.h>
23 #include <nat/nat.h>
24 #include <nat/nat_reass.h>
25 #include <nat/nat_inlines.h>
26
27 vlib_node_registration_t nat44_classify_node;
28 vlib_node_registration_t nat44_ed_classify_node;
29 vlib_node_registration_t nat44_det_classify_node;
30 vlib_node_registration_t nat44_handoff_classify_node;
31
32 #define foreach_nat44_classify_error                      \
33 _(MAX_REASS, "Maximum reassemblies exceeded")             \
34 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
35
36 typedef enum
37 {
38 #define _(sym,str) NAT44_CLASSIFY_ERROR_##sym,
39   foreach_nat44_classify_error
40 #undef _
41     NAT44_CLASSIFY_N_ERROR,
42 } nat44_classify_error_t;
43
44 static char *nat44_classify_error_strings[] = {
45 #define _(sym,string) string,
46   foreach_nat44_classify_error
47 #undef _
48 };
49
50 typedef enum
51 {
52   NAT44_CLASSIFY_NEXT_IN2OUT,
53   NAT44_CLASSIFY_NEXT_OUT2IN,
54   NAT44_CLASSIFY_NEXT_DROP,
55   NAT44_CLASSIFY_N_NEXT,
56 } nat44_classify_next_t;
57
58 typedef struct
59 {
60   u8 next_in2out;
61   u8 cached;
62 } nat44_classify_trace_t;
63
64 static u8 *
65 format_nat44_classify_trace (u8 * s, va_list * args)
66 {
67   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
68   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
69   nat44_classify_trace_t *t = va_arg (*args, nat44_classify_trace_t *);
70   char *next;
71
72   if (t->cached)
73     s = format (s, "nat44-classify: fragment cached");
74   else
75     {
76       next = t->next_in2out ? "nat44-in2out" : "nat44-out2in";
77       s = format (s, "nat44-classify: next %s", next);
78     }
79
80   return s;
81 }
82
83 static inline uword
84 nat44_classify_node_fn_inline (vlib_main_t * vm,
85                                vlib_node_runtime_t * node,
86                                vlib_frame_t * frame, int is_ed)
87 {
88   u32 n_left_from, *from, *to_next;
89   nat44_classify_next_t next_index;
90   snat_main_t *sm = &snat_main;
91   snat_static_mapping_t *m;
92   u32 thread_index = vm->thread_index;
93   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
94   u32 *fragments_to_drop = 0;
95   u32 *fragments_to_loopback = 0;
96
97   from = vlib_frame_vector_args (frame);
98   n_left_from = frame->n_vectors;
99   next_index = node->cached_next_index;
100
101   while (n_left_from > 0)
102     {
103       u32 n_left_to_next;
104
105       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
106
107       while (n_left_from > 0 && n_left_to_next > 0)
108         {
109           u32 bi0;
110           vlib_buffer_t *b0;
111           u32 next0 = NAT44_CLASSIFY_NEXT_IN2OUT, sw_if_index0, rx_fib_index0;
112           ip4_header_t *ip0;
113           snat_address_t *ap;
114           snat_session_key_t m_key0;
115           clib_bihash_kv_8_8_t kv0, value0;
116           clib_bihash_kv_16_8_t ed_kv0, ed_value0;
117           udp_header_t *udp0;
118           nat_reass_ip4_t *reass0;
119           u8 cached0 = 0;
120
121           /* speculatively enqueue b0 to the current next frame */
122           bi0 = from[0];
123           to_next[0] = bi0;
124           from += 1;
125           to_next += 1;
126           n_left_from -= 1;
127           n_left_to_next -= 1;
128
129           b0 = vlib_get_buffer (vm, bi0);
130           ip0 = vlib_buffer_get_current (b0);
131           udp0 = ip4_next_header (ip0);
132
133           if (is_ed && ip0->protocol != IP_PROTOCOL_ICMP)
134             {
135               if (!ip4_is_fragment (ip0) || ip4_is_first_fragment (ip0))
136                 {
137                   /* process leading fragment/whole packet (with L4 header) */
138                   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
139                   rx_fib_index0 =
140                     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
141                                                          sw_if_index0);
142                   make_ed_kv (&ed_kv0, &ip0->src_address, &ip0->dst_address,
143                               ip0->protocol, rx_fib_index0, udp0->src_port,
144                               udp0->dst_port);
145                   if (ip4_is_fragment (ip0))
146                     {
147                       reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
148                                                              ip0->dst_address,
149                                                              ip0->fragment_id,
150                                                              ip0->protocol,
151                                                              1,
152                                                              &fragments_to_drop);
153                       if (PREDICT_FALSE (!reass0))
154                         {
155                           next0 = NAT44_CLASSIFY_NEXT_DROP;
156                           b0->error =
157                             node->errors[NAT44_CLASSIFY_ERROR_MAX_REASS];
158                           nat_log_notice ("maximum reassemblies exceeded");
159                           goto enqueue0;
160                         }
161                       if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &ed_kv0,
162                                                     &ed_value0))
163                         {
164                           /* session exists so classify as IN2OUT,
165                            * save this information for future fragments and set
166                            * past fragments to be looped over and reprocessed */
167                           reass0->sess_index = ed_value0.value;
168                           reass0->classify_next =
169                             NAT_REASS_IP4_CLASSIFY_NEXT_IN2OUT;
170                           nat_ip4_reass_get_frags (reass0,
171                                                    &fragments_to_loopback);
172                           goto enqueue0;
173                         }
174                       else
175                         {
176                           /* session doesn't exist so continue in the code,
177                            * save this information for future fragments and set
178                            * past fragments to be looped over and reprocessed */
179                           reass0->flags |=
180                             NAT_REASS_FLAG_CLASSIFY_ED_CONTINUE;
181                           nat_ip4_reass_get_frags (reass0,
182                                                    &fragments_to_loopback);
183                         }
184                     }
185                   else
186                     {
187                       /* process whole packet */
188                       if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &ed_kv0,
189                                                     &ed_value0))
190                         goto enqueue0;
191                       /* session doesn't exist so continue in code */
192                     }
193                 }
194               else
195                 {
196                   /* process non-first fragment */
197                   reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
198                                                          ip0->dst_address,
199                                                          ip0->fragment_id,
200                                                          ip0->protocol,
201                                                          1,
202                                                          &fragments_to_drop);
203                   if (PREDICT_FALSE (!reass0))
204                     {
205                       next0 = NAT44_CLASSIFY_NEXT_DROP;
206                       b0->error =
207                         node->errors[NAT44_CLASSIFY_ERROR_MAX_REASS];
208                       nat_log_notice ("maximum reassemblies exceeded");
209                       goto enqueue0;
210                     }
211                   /* check if first fragment has arrived */
212                   if (reass0->classify_next == NAT_REASS_IP4_CLASSIFY_NONE &&
213                       !(reass0->flags & NAT_REASS_FLAG_CLASSIFY_ED_CONTINUE))
214                     {
215                       /* first fragment still hasn't arrived, cache this fragment */
216                       if (nat_ip4_reass_add_fragment (reass0, bi0,
217                                                       &fragments_to_drop))
218                         {
219                           b0->error =
220                             node->errors[NAT44_CLASSIFY_ERROR_MAX_FRAG];
221                           nat_log_notice
222                             ("maximum fragments per reassembly exceeded");
223                           next0 = NAT44_CLASSIFY_NEXT_DROP;
224                           goto enqueue0;
225                         }
226                       cached0 = 1;
227                       goto enqueue0;
228                     }
229                   if (reass0->classify_next ==
230                       NAT_REASS_IP4_CLASSIFY_NEXT_IN2OUT)
231                     goto enqueue0;
232                   /* flag NAT_REASS_FLAG_CLASSIFY_ED_CONTINUE is set
233                    * so keep the default next0 and continue in code to
234                    * potentially find other classification for this packet */
235                 }
236             }
237
238           /* *INDENT-OFF* */
239           vec_foreach (ap, sm->addresses)
240             {
241               if (ip0->dst_address.as_u32 == ap->addr.as_u32)
242                 {
243                   next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
244                   goto enqueue0;
245                 }
246             }
247           /* *INDENT-ON* */
248
249           if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
250             {
251               m_key0.addr = ip0->dst_address;
252               m_key0.port = 0;
253               m_key0.protocol = 0;
254               m_key0.fib_index = 0;
255               kv0.key = m_key0.as_u64;
256               /* try to classify the fragment based on IP header alone */
257               if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external,
258                                            &kv0, &value0))
259                 {
260                   m = pool_elt_at_index (sm->static_mappings, value0.value);
261                   if (m->local_addr.as_u32 != m->external_addr.as_u32)
262                     next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
263                   goto enqueue0;
264                 }
265               if (!ip4_is_fragment (ip0) || ip4_is_first_fragment (ip0))
266                 {
267                   /* process leading fragment/whole packet (with L4 header) */
268                   m_key0.port = clib_net_to_host_u16 (udp0->dst_port);
269                   m_key0.protocol = ip_proto_to_snat_proto (ip0->protocol);
270                   kv0.key = m_key0.as_u64;
271                   if (!clib_bihash_search_8_8
272                       (&sm->static_mapping_by_external, &kv0, &value0))
273                     {
274                       m =
275                         pool_elt_at_index (sm->static_mappings, value0.value);
276                       if (m->local_addr.as_u32 != m->external_addr.as_u32)
277                         next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
278                     }
279                   if (ip4_is_fragment (ip0))
280                     {
281                       reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
282                                                              ip0->dst_address,
283                                                              ip0->fragment_id,
284                                                              ip0->protocol,
285                                                              1,
286                                                              &fragments_to_drop);
287                       if (PREDICT_FALSE (!reass0))
288                         {
289                           next0 = NAT44_CLASSIFY_NEXT_DROP;
290                           b0->error =
291                             node->errors[NAT44_CLASSIFY_ERROR_MAX_REASS];
292                           nat_log_notice ("maximum reassemblies exceeded");
293                           goto enqueue0;
294                         }
295                       /* save classification for future fragments and set past
296                        * fragments to be looped over and reprocessed */
297                       if (next0 == NAT44_CLASSIFY_NEXT_OUT2IN)
298                         reass0->classify_next =
299                           NAT_REASS_IP4_CLASSIFY_NEXT_OUT2IN;
300                       else
301                         reass0->classify_next =
302                           NAT_REASS_IP4_CLASSIFY_NEXT_IN2OUT;
303                       nat_ip4_reass_get_frags (reass0,
304                                                &fragments_to_loopback);
305                     }
306                 }
307               else
308                 {
309                   /* process non-first fragment */
310                   reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
311                                                          ip0->dst_address,
312                                                          ip0->fragment_id,
313                                                          ip0->protocol,
314                                                          1,
315                                                          &fragments_to_drop);
316                   if (PREDICT_FALSE (!reass0))
317                     {
318                       next0 = NAT44_CLASSIFY_NEXT_DROP;
319                       b0->error =
320                         node->errors[NAT44_CLASSIFY_ERROR_MAX_REASS];
321                       nat_log_notice ("maximum reassemblies exceeded");
322                       goto enqueue0;
323                     }
324                   if (reass0->classify_next == NAT_REASS_IP4_CLASSIFY_NONE)
325                     /* first fragment still hasn't arrived */
326                     {
327                       if (nat_ip4_reass_add_fragment (reass0, bi0,
328                                                       &fragments_to_drop))
329                         {
330                           b0->error =
331                             node->errors[NAT44_CLASSIFY_ERROR_MAX_FRAG];
332                           nat_log_notice
333                             ("maximum fragments per reassembly exceeded");
334                           next0 = NAT44_CLASSIFY_NEXT_DROP;
335                           goto enqueue0;
336                         }
337                       cached0 = 1;
338                       goto enqueue0;
339                     }
340                   else if (reass0->classify_next ==
341                            NAT_REASS_IP4_CLASSIFY_NEXT_OUT2IN)
342                     next0 = NAT44_CLASSIFY_NEXT_OUT2IN;
343                   else if (reass0->classify_next ==
344                            NAT_REASS_IP4_CLASSIFY_NEXT_IN2OUT)
345                     next0 = NAT44_CLASSIFY_NEXT_IN2OUT;
346                 }
347             }
348
349         enqueue0:
350           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
351                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
352             {
353               nat44_classify_trace_t *t =
354                 vlib_add_trace (vm, node, b0, sizeof (*t));
355               t->cached = cached0;
356               if (!cached0)
357                 t->next_in2out = next0 == NAT44_CLASSIFY_NEXT_IN2OUT ? 1 : 0;
358             }
359
360           if (cached0)
361             {
362               n_left_to_next++;
363               to_next--;
364             }
365           else
366             /* verify speculative enqueue, maybe switch current next frame */
367             vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
368                                              to_next, n_left_to_next,
369                                              bi0, next0);
370
371           if (n_left_from == 0 && vec_len (fragments_to_loopback))
372             {
373               from = vlib_frame_vector_args (frame);
374               u32 len = vec_len (fragments_to_loopback);
375               if (len <= VLIB_FRAME_SIZE)
376                 {
377                   clib_memcpy_fast (from, fragments_to_loopback,
378                                     sizeof (u32) * len);
379                   n_left_from = len;
380                   vec_reset_length (fragments_to_loopback);
381                 }
382               else
383                 {
384                   clib_memcpy_fast (from, fragments_to_loopback +
385                                     (len - VLIB_FRAME_SIZE),
386                                     sizeof (u32) * VLIB_FRAME_SIZE);
387                   n_left_from = VLIB_FRAME_SIZE;
388                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
389                 }
390             }
391         }
392
393       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
394     }
395
396   nat_send_all_to_node (vm, fragments_to_drop, node, 0,
397                         NAT44_CLASSIFY_NEXT_DROP);
398
399   vec_free (fragments_to_drop);
400
401   return frame->n_vectors;
402 }
403
404 static uword
405 nat44_classify_node_fn (vlib_main_t * vm,
406                         vlib_node_runtime_t * node, vlib_frame_t * frame)
407 {
408   return nat44_classify_node_fn_inline (vm, node, frame, 0);
409 }
410
411 /* *INDENT-OFF* */
412 VLIB_REGISTER_NODE (nat44_classify_node) = {
413   .function = nat44_classify_node_fn,
414   .name = "nat44-classify",
415   .vector_size = sizeof (u32),
416   .format_trace = format_nat44_classify_trace,
417   .type = VLIB_NODE_TYPE_INTERNAL,
418   .n_errors = ARRAY_LEN(nat44_classify_error_strings),
419   .error_strings = nat44_classify_error_strings,
420   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
421   .next_nodes = {
422     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-in2out",
423     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-out2in",
424     [NAT44_CLASSIFY_NEXT_DROP] = "error-drop",
425   },
426 };
427 /* *INDENT-ON* */
428
429 VLIB_NODE_FUNCTION_MULTIARCH (nat44_classify_node, nat44_classify_node_fn);
430 static uword
431 nat44_ed_classify_node_fn (vlib_main_t * vm,
432                            vlib_node_runtime_t * node, vlib_frame_t * frame)
433 {
434   return nat44_classify_node_fn_inline (vm, node, frame, 1);
435 }
436
437 /* *INDENT-OFF* */
438 VLIB_REGISTER_NODE (nat44_ed_classify_node) = {
439   .function = nat44_ed_classify_node_fn,
440   .name = "nat44-ed-classify",
441   .vector_size = sizeof (u32),
442   .format_trace = format_nat44_classify_trace,
443   .type = VLIB_NODE_TYPE_INTERNAL,
444   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
445   .next_nodes = {
446     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-ed-in2out",
447     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-ed-out2in",
448     [NAT44_CLASSIFY_NEXT_DROP] = "error-drop",
449   },
450 };
451 /* *INDENT-ON* */
452
453 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_classify_node,
454                               nat44_ed_classify_node_fn);
455
456 static uword
457 nat44_det_classify_node_fn (vlib_main_t * vm,
458                             vlib_node_runtime_t * node, vlib_frame_t * frame)
459 {
460   return nat44_classify_node_fn_inline (vm, node, frame, 0);
461 }
462
463 /* *INDENT-OFF* */
464 VLIB_REGISTER_NODE (nat44_det_classify_node) = {
465   .function = nat44_det_classify_node_fn,
466   .name = "nat44-det-classify",
467   .vector_size = sizeof (u32),
468   .format_trace = format_nat44_classify_trace,
469   .type = VLIB_NODE_TYPE_INTERNAL,
470   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
471   .next_nodes = {
472     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-det-in2out",
473     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-det-out2in",
474     [NAT44_CLASSIFY_NEXT_DROP] = "error-drop",
475   },
476 };
477 /* *INDENT-ON* */
478
479 VLIB_NODE_FUNCTION_MULTIARCH (nat44_det_classify_node,
480                               nat44_det_classify_node_fn);
481
482 static uword
483 nat44_handoff_classify_node_fn (vlib_main_t * vm,
484                                 vlib_node_runtime_t * node,
485                                 vlib_frame_t * frame)
486 {
487   return nat44_classify_node_fn_inline (vm, node, frame, 0);
488 }
489
490 /* *INDENT-OFF* */
491 VLIB_REGISTER_NODE (nat44_handoff_classify_node) = {
492   .function = nat44_handoff_classify_node_fn,
493   .name = "nat44-handoff-classify",
494   .vector_size = sizeof (u32),
495   .format_trace = format_nat44_classify_trace,
496   .type = VLIB_NODE_TYPE_INTERNAL,
497   .n_next_nodes = NAT44_CLASSIFY_N_NEXT,
498   .next_nodes = {
499     [NAT44_CLASSIFY_NEXT_IN2OUT] = "nat44-in2out-worker-handoff",
500     [NAT44_CLASSIFY_NEXT_OUT2IN] = "nat44-out2in-worker-handoff",
501     [NAT44_CLASSIFY_NEXT_DROP] = "error-drop",
502   },
503 };
504
505 VLIB_NODE_FUNCTION_MULTIARCH (nat44_handoff_classify_node,
506                               nat44_handoff_classify_node_fn);
507 /* *INDENT-ON* */
508
509 /*
510  * fd.io coding-style-patch-verification: ON
511  *
512  * Local Variables:
513  * eval: (c-set-style "gnu")
514  * End:
515  */