2 * Copyright (c) 2020 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #ifndef __CNAT_NODE_H__
17 #define __CNAT_NODE_H__
19 #include <vlibmemory/api.h>
20 #include <vnet/dpo/load_balance.h>
21 #include <vnet/dpo/load_balance_map.h>
23 #include <cnat/cnat_session.h>
24 #include <cnat/cnat_client.h>
25 #include <cnat/cnat_inline.h>
26 #include <cnat/cnat_translation.h>
28 #include <vnet/ip/ip4_inlines.h>
29 #include <vnet/ip/ip6_inlines.h>
31 typedef uword (*cnat_node_sub_t) (vlib_main_t * vm,
32 vlib_node_runtime_t * node,
34 cnat_node_ctx_t * ctx, int rv,
35 cnat_session_t * session);
37 typedef struct cnat_trace_element_t_
39 cnat_session_t session;
40 cnat_translation_t tr;
41 u32 sw_if_index[VLIB_N_RX_TX];
42 u32 snat_policy_result;
44 } cnat_trace_element_t;
46 typedef enum cnat_trace_element_flag_t_
48 CNAT_TRACE_SESSION_FOUND = (1 << 0),
49 CNAT_TRACE_SESSION_CREATED = (1 << 1),
50 CNAT_TRACE_TRANSLATION_FOUND = (1 << 2),
51 CNAT_TRACE_NO_NAT = (1 << 3),
52 } cnat_trace_element_flag_t;
54 static_always_inline void
55 cnat_add_trace (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_buffer_t *b,
56 cnat_session_t *session, const cnat_translation_t *ct,
59 cnat_trace_element_t *t;
61 flags |= CNAT_TRACE_TRANSLATION_FOUND;
63 t = vlib_add_trace (vm, node, b, sizeof (*t));
64 t->sw_if_index[VLIB_RX] = vnet_buffer (b)->sw_if_index[VLIB_RX];
65 t->sw_if_index[VLIB_TX] = vnet_buffer (b)->sw_if_index[VLIB_TX];
67 if (flags & (CNAT_TRACE_SESSION_FOUND | CNAT_TRACE_SESSION_CREATED))
68 clib_memcpy (&t->session, session, sizeof (t->session));
69 if (flags & CNAT_TRACE_TRANSLATION_FOUND)
70 clib_memcpy (&t->tr, ct, sizeof (cnat_translation_t));
75 format_cnat_trace (u8 *s, va_list *args)
77 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
78 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
79 cnat_trace_element_t *t = va_arg (*args, cnat_trace_element_t *);
80 u32 indent = format_get_indent (s);
81 vnet_main_t *vnm = vnet_get_main ();
83 if (t->flags & CNAT_TRACE_SESSION_CREATED)
84 s = format (s, "created session");
85 else if (t->flags & CNAT_TRACE_SESSION_FOUND)
86 s = format (s, "found session");
88 s = format (s, "session not found");
90 if (t->flags & (CNAT_TRACE_NO_NAT))
91 s = format (s, " [policy:skip]");
93 s = format (s, "\n%Uin:%U out:%U ", format_white_space, indent,
94 format_vnet_sw_if_index_name, vnm, t->sw_if_index[VLIB_RX],
95 format_vnet_sw_if_index_name, vnm, t->sw_if_index[VLIB_TX]);
97 if (t->flags & (CNAT_TRACE_SESSION_CREATED | CNAT_TRACE_SESSION_FOUND))
98 s = format (s, "\n%U%U", format_white_space, indent, format_cnat_session,
101 if (t->flags & CNAT_TRACE_TRANSLATION_FOUND)
102 s = format (s, "\n%Utranslation: %U", format_white_space, indent,
103 format_cnat_translation, &t->tr, 0);
108 static_always_inline u8
109 icmp_type_is_error_message (u8 icmp_type)
113 case ICMP4_destination_unreachable:
114 case ICMP4_time_exceeded:
115 case ICMP4_parameter_problem:
116 case ICMP4_source_quench:
118 case ICMP4_alternate_host_address:
124 static_always_inline u8
125 icmp_type_is_echo (u8 icmp_type)
129 case ICMP4_echo_request:
130 case ICMP4_echo_reply:
136 static_always_inline u8
137 icmp6_type_is_echo (u8 icmp_type)
141 case ICMP6_echo_request:
142 case ICMP6_echo_reply:
148 static_always_inline u8
149 icmp6_type_is_error_message (u8 icmp_type)
153 case ICMP6_destination_unreachable:
154 case ICMP6_time_exceeded:
155 case ICMP6_parameter_problem:
161 static_always_inline u8
162 cmp_ip6_address (const ip6_address_t * a1, const ip6_address_t * a2)
164 return ((a1->as_u64[0] == a2->as_u64[0])
165 && (a1->as_u64[1] == a2->as_u64[1]));
169 * Inline translation functions
172 static_always_inline u8
173 has_ip6_address (ip6_address_t * a)
175 return ((0 != a->as_u64[0]) || (0 != a->as_u64[1]));
178 static_always_inline void
179 cnat_ip4_translate_l4 (ip4_header_t * ip4, udp_header_t * udp,
181 ip4_address_t new_addr[VLIB_N_DIR],
182 u16 new_port[VLIB_N_DIR])
184 u16 old_port[VLIB_N_DIR];
185 ip4_address_t old_addr[VLIB_N_DIR];
187 /* Fastpath no checksum */
188 if (PREDICT_TRUE (0 == *sum))
190 udp->dst_port = new_port[VLIB_TX];
191 udp->src_port = new_port[VLIB_RX];
195 old_port[VLIB_TX] = udp->dst_port;
196 old_port[VLIB_RX] = udp->src_port;
197 old_addr[VLIB_TX] = ip4->dst_address;
198 old_addr[VLIB_RX] = ip4->src_address;
200 if (new_addr[VLIB_TX].as_u32)
203 ip_csum_update (*sum, old_addr[VLIB_TX].as_u32,
204 new_addr[VLIB_TX].as_u32, ip4_header_t, dst_address);
206 if (new_port[VLIB_TX])
208 udp->dst_port = new_port[VLIB_TX];
209 *sum = ip_csum_update (*sum, old_port[VLIB_TX], new_port[VLIB_TX],
210 ip4_header_t /* cheat */ ,
211 length /* changed member */ );
213 if (new_addr[VLIB_RX].as_u32)
216 ip_csum_update (*sum, old_addr[VLIB_RX].as_u32,
217 new_addr[VLIB_RX].as_u32, ip4_header_t, src_address);
219 if (new_port[VLIB_RX])
221 udp->src_port = new_port[VLIB_RX];
222 *sum = ip_csum_update (*sum, old_port[VLIB_RX], new_port[VLIB_RX],
223 ip4_header_t /* cheat */ ,
224 length /* changed member */ );
228 static_always_inline void
229 cnat_ip4_translate_l3 (ip4_header_t * ip4, ip4_address_t new_addr[VLIB_N_DIR])
231 ip4_address_t old_addr[VLIB_N_DIR];
234 old_addr[VLIB_TX] = ip4->dst_address;
235 old_addr[VLIB_RX] = ip4->src_address;
238 if (new_addr[VLIB_TX].as_u32)
240 ip4->dst_address = new_addr[VLIB_TX];
242 ip_csum_update (sum, old_addr[VLIB_TX].as_u32,
243 new_addr[VLIB_TX].as_u32, ip4_header_t, dst_address);
245 if (new_addr[VLIB_RX].as_u32)
247 ip4->src_address = new_addr[VLIB_RX];
249 ip_csum_update (sum, old_addr[VLIB_RX].as_u32,
250 new_addr[VLIB_RX].as_u32, ip4_header_t, src_address);
252 ip4->checksum = ip_csum_fold (sum);
255 static_always_inline void
256 cnat_tcp_update_session_lifetime (tcp_header_t * tcp, u32 index)
258 cnat_main_t *cm = &cnat_main;
259 if (PREDICT_FALSE (tcp_fin (tcp)))
261 cnat_timestamp_set_lifetime (index, CNAT_DEFAULT_TCP_RST_TIMEOUT);
264 if (PREDICT_FALSE (tcp_rst (tcp)))
266 cnat_timestamp_set_lifetime (index, CNAT_DEFAULT_TCP_RST_TIMEOUT);
269 if (PREDICT_FALSE (tcp_syn (tcp) && tcp_ack (tcp)))
271 cnat_timestamp_set_lifetime (index, cm->tcp_max_age);
275 static_always_inline void
276 cnat_translation_icmp4_echo (ip4_header_t * ip4, icmp46_header_t * icmp,
277 ip4_address_t new_addr[VLIB_N_DIR],
278 u16 new_port[VLIB_N_DIR])
282 cnat_echo_header_t *echo = (cnat_echo_header_t *) (icmp + 1);
284 cnat_ip4_translate_l3 (ip4, new_addr);
285 old_port = echo->identifier;
286 echo->identifier = new_port[VLIB_RX];
288 sum = icmp->checksum;
289 sum = ip_csum_update (sum, old_port, new_port[VLIB_RX],
290 ip4_header_t /* cheat */ ,
291 length /* changed member */ );
293 icmp->checksum = ip_csum_fold (sum);
296 static_always_inline void
297 cnat_translation_icmp4_error (ip4_header_t * outer_ip4,
298 icmp46_header_t * icmp,
299 ip4_address_t outer_new_addr[VLIB_N_DIR],
300 u16 outer_new_port[VLIB_N_DIR],
303 ip4_address_t new_addr[VLIB_N_DIR];
304 ip4_address_t old_addr[VLIB_N_DIR];
305 u16 new_port[VLIB_N_DIR];
306 u16 old_port[VLIB_N_DIR];
307 ip_csum_t sum, old_ip_sum, inner_l4_sum, inner_l4_old_sum;
309 ip4_header_t *ip4 = (ip4_header_t *) (icmp + 2);
310 udp_header_t *udp = (udp_header_t *) (ip4 + 1);
311 tcp_header_t *tcp = (tcp_header_t *) udp;
313 /* Swap inner ports */
314 new_addr[VLIB_TX] = outer_new_addr[VLIB_RX];
315 new_addr[VLIB_RX] = outer_new_addr[VLIB_TX];
316 new_port[VLIB_TX] = outer_new_port[VLIB_RX];
317 new_port[VLIB_RX] = outer_new_port[VLIB_TX];
319 old_addr[VLIB_TX] = ip4->dst_address;
320 old_addr[VLIB_RX] = ip4->src_address;
321 old_port[VLIB_RX] = udp->src_port;
322 old_port[VLIB_TX] = udp->dst_port;
324 sum = icmp->checksum;
325 old_ip_sum = ip4->checksum;
327 /* translate outer ip. */
329 outer_new_addr[VLIB_RX] = outer_ip4->src_address;
330 cnat_ip4_translate_l3 (outer_ip4, outer_new_addr);
332 if (ip4->protocol == IP_PROTOCOL_TCP)
334 inner_l4_old_sum = inner_l4_sum = tcp->checksum;
335 cnat_ip4_translate_l4 (ip4, udp, &inner_l4_sum, new_addr, new_port);
336 tcp->checksum = ip_csum_fold (inner_l4_sum);
338 else if (ip4->protocol == IP_PROTOCOL_UDP)
340 inner_l4_old_sum = inner_l4_sum = udp->checksum;
341 cnat_ip4_translate_l4 (ip4, udp, &inner_l4_sum, new_addr, new_port);
342 udp->checksum = ip_csum_fold (inner_l4_sum);
347 /* UDP/TCP checksum changed */
348 sum = ip_csum_update (sum, inner_l4_old_sum, inner_l4_sum,
349 ip4_header_t, checksum);
351 /* UDP/TCP Ports changed */
352 if (old_port[VLIB_TX] && new_port[VLIB_TX])
353 sum = ip_csum_update (sum, old_port[VLIB_TX], new_port[VLIB_TX],
354 ip4_header_t /* cheat */ ,
355 length /* changed member */ );
357 if (old_port[VLIB_RX] && new_port[VLIB_RX])
358 sum = ip_csum_update (sum, old_port[VLIB_RX], new_port[VLIB_RX],
359 ip4_header_t /* cheat */ ,
360 length /* changed member */ );
363 cnat_ip4_translate_l3 (ip4, new_addr);
364 ip_csum_t new_ip_sum = ip4->checksum;
365 /* IP checksum changed */
366 sum = ip_csum_update (sum, old_ip_sum, new_ip_sum, ip4_header_t, checksum);
368 /* IP src/dst addr changed */
369 if (new_addr[VLIB_TX].as_u32)
371 ip_csum_update (sum, old_addr[VLIB_TX].as_u32, new_addr[VLIB_TX].as_u32,
372 ip4_header_t, dst_address);
374 if (new_addr[VLIB_RX].as_u32)
376 ip_csum_update (sum, old_addr[VLIB_RX].as_u32, new_addr[VLIB_RX].as_u32,
377 ip4_header_t, src_address);
379 icmp->checksum = ip_csum_fold (sum);
382 static_always_inline void
383 cnat_translation_ip4 (const cnat_session_t * session,
384 ip4_header_t * ip4, udp_header_t * udp)
386 tcp_header_t *tcp = (tcp_header_t *) udp;
387 ip4_address_t new_addr[VLIB_N_DIR];
388 u16 new_port[VLIB_N_DIR];
390 new_addr[VLIB_TX] = session->value.cs_ip[VLIB_TX].ip4;
391 new_addr[VLIB_RX] = session->value.cs_ip[VLIB_RX].ip4;
392 new_port[VLIB_TX] = session->value.cs_port[VLIB_TX];
393 new_port[VLIB_RX] = session->value.cs_port[VLIB_RX];
395 if (ip4->protocol == IP_PROTOCOL_TCP)
397 ip_csum_t sum = tcp->checksum;
398 cnat_ip4_translate_l4 (ip4, udp, &sum, new_addr, new_port);
399 tcp->checksum = ip_csum_fold (sum);
400 cnat_ip4_translate_l3 (ip4, new_addr);
401 cnat_tcp_update_session_lifetime (tcp, session->value.cs_ts_index);
403 else if (ip4->protocol == IP_PROTOCOL_UDP)
405 ip_csum_t sum = udp->checksum;
406 cnat_ip4_translate_l4 (ip4, udp, &sum, new_addr, new_port);
407 udp->checksum = ip_csum_fold (sum);
408 cnat_ip4_translate_l3 (ip4, new_addr);
410 else if (ip4->protocol == IP_PROTOCOL_ICMP)
412 icmp46_header_t *icmp = (icmp46_header_t *) udp;
413 if (icmp_type_is_error_message (icmp->type))
415 /* SNAT only if src_addr was translated */
417 (ip4->src_address.as_u32 ==
418 session->key.cs_ip[VLIB_RX].ip4.as_u32);
419 cnat_translation_icmp4_error (ip4, icmp, new_addr, new_port,
422 else if (icmp_type_is_echo (icmp->type))
423 cnat_translation_icmp4_echo (ip4, icmp, new_addr, new_port);
427 static_always_inline void
428 cnat_ip6_translate_l3 (ip6_header_t * ip6, ip6_address_t new_addr[VLIB_N_DIR])
430 if (has_ip6_address (&new_addr[VLIB_TX]))
431 ip6_address_copy (&ip6->dst_address, &new_addr[VLIB_TX]);
432 if (has_ip6_address (&new_addr[VLIB_RX]))
433 ip6_address_copy (&ip6->src_address, &new_addr[VLIB_RX]);
436 static_always_inline void
437 cnat_ip6_translate_l4 (ip6_header_t * ip6, udp_header_t * udp,
439 ip6_address_t new_addr[VLIB_N_DIR],
440 u16 new_port[VLIB_N_DIR])
442 u16 old_port[VLIB_N_DIR];
443 ip6_address_t old_addr[VLIB_N_DIR];
445 /* Fastpath no checksum */
446 if (PREDICT_TRUE (0 == *sum))
448 udp->dst_port = new_port[VLIB_TX];
449 udp->src_port = new_port[VLIB_RX];
453 old_port[VLIB_TX] = udp->dst_port;
454 old_port[VLIB_RX] = udp->src_port;
455 ip6_address_copy (&old_addr[VLIB_TX], &ip6->dst_address);
456 ip6_address_copy (&old_addr[VLIB_RX], &ip6->src_address);
458 if (has_ip6_address (&new_addr[VLIB_TX]))
460 *sum = ip_csum_add_even (*sum, new_addr[VLIB_TX].as_u64[0]);
461 *sum = ip_csum_add_even (*sum, new_addr[VLIB_TX].as_u64[1]);
462 *sum = ip_csum_sub_even (*sum, old_addr[VLIB_TX].as_u64[0]);
463 *sum = ip_csum_sub_even (*sum, old_addr[VLIB_TX].as_u64[1]);
466 if (new_port[VLIB_TX])
468 udp->dst_port = new_port[VLIB_TX];
469 *sum = ip_csum_update (*sum, old_port[VLIB_TX], new_port[VLIB_TX],
470 ip4_header_t /* cheat */ ,
471 length /* changed member */ );
473 if (has_ip6_address (&new_addr[VLIB_RX]))
475 *sum = ip_csum_add_even (*sum, new_addr[VLIB_RX].as_u64[0]);
476 *sum = ip_csum_add_even (*sum, new_addr[VLIB_RX].as_u64[1]);
477 *sum = ip_csum_sub_even (*sum, old_addr[VLIB_RX].as_u64[0]);
478 *sum = ip_csum_sub_even (*sum, old_addr[VLIB_RX].as_u64[1]);
481 if (new_port[VLIB_RX])
483 udp->src_port = new_port[VLIB_RX];
484 *sum = ip_csum_update (*sum, old_port[VLIB_RX], new_port[VLIB_RX],
485 ip4_header_t /* cheat */ ,
486 length /* changed member */ );
490 static_always_inline void
491 cnat_translation_icmp6_echo (ip6_header_t * ip6, icmp46_header_t * icmp,
492 ip6_address_t new_addr[VLIB_N_DIR],
493 u16 new_port[VLIB_N_DIR])
495 cnat_echo_header_t *echo = (cnat_echo_header_t *) (icmp + 1);
496 ip6_address_t old_addr[VLIB_N_DIR];
499 old_port = echo->identifier;
500 ip6_address_copy (&old_addr[VLIB_TX], &ip6->dst_address);
501 ip6_address_copy (&old_addr[VLIB_RX], &ip6->src_address);
503 sum = icmp->checksum;
505 cnat_ip6_translate_l3 (ip6, new_addr);
506 if (has_ip6_address (&new_addr[VLIB_TX]))
508 sum = ip_csum_add_even (sum, new_addr[VLIB_TX].as_u64[0]);
509 sum = ip_csum_add_even (sum, new_addr[VLIB_TX].as_u64[1]);
510 sum = ip_csum_sub_even (sum, old_addr[VLIB_TX].as_u64[0]);
511 sum = ip_csum_sub_even (sum, old_addr[VLIB_TX].as_u64[1]);
514 if (has_ip6_address (&new_addr[VLIB_RX]))
516 sum = ip_csum_add_even (sum, new_addr[VLIB_RX].as_u64[0]);
517 sum = ip_csum_add_even (sum, new_addr[VLIB_RX].as_u64[1]);
518 sum = ip_csum_sub_even (sum, old_addr[VLIB_RX].as_u64[0]);
519 sum = ip_csum_sub_even (sum, old_addr[VLIB_RX].as_u64[1]);
522 echo->identifier = new_port[VLIB_RX];
523 sum = ip_csum_update (sum, old_port, new_port[VLIB_RX],
524 ip4_header_t /* cheat */ ,
525 length /* changed member */ );
527 icmp->checksum = ip_csum_fold (sum);
530 static_always_inline void
531 cnat_translation_icmp6_error (ip6_header_t * outer_ip6,
532 icmp46_header_t * icmp,
533 ip6_address_t outer_new_addr[VLIB_N_DIR],
534 u16 outer_new_port[VLIB_N_DIR],
537 ip6_address_t new_addr[VLIB_N_DIR];
538 ip6_address_t old_addr[VLIB_N_DIR];
539 ip6_address_t outer_old_addr[VLIB_N_DIR];
540 u16 new_port[VLIB_N_DIR];
541 u16 old_port[VLIB_N_DIR];
542 ip_csum_t sum, inner_l4_sum, inner_l4_old_sum;
544 if (!icmp6_type_is_error_message (icmp->type))
547 ip6_header_t *ip6 = (ip6_header_t *) (icmp + 2);
548 udp_header_t *udp = (udp_header_t *) (ip6 + 1);
549 tcp_header_t *tcp = (tcp_header_t *) udp;
551 /* Swap inner ports */
552 ip6_address_copy (&new_addr[VLIB_RX], &outer_new_addr[VLIB_TX]);
553 ip6_address_copy (&new_addr[VLIB_TX], &outer_new_addr[VLIB_RX]);
554 new_port[VLIB_TX] = outer_new_port[VLIB_RX];
555 new_port[VLIB_RX] = outer_new_port[VLIB_TX];
557 ip6_address_copy (&old_addr[VLIB_TX], &ip6->dst_address);
558 ip6_address_copy (&old_addr[VLIB_RX], &ip6->src_address);
559 old_port[VLIB_RX] = udp->src_port;
560 old_port[VLIB_TX] = udp->dst_port;
562 sum = icmp->checksum;
563 /* Translate outer ip */
564 ip6_address_copy (&outer_old_addr[VLIB_TX], &outer_ip6->dst_address);
565 ip6_address_copy (&outer_old_addr[VLIB_RX], &outer_ip6->src_address);
567 ip6_address_copy (&outer_new_addr[VLIB_RX], &outer_ip6->src_address);
568 cnat_ip6_translate_l3 (outer_ip6, outer_new_addr);
569 if (has_ip6_address (&outer_new_addr[VLIB_TX]))
571 sum = ip_csum_add_even (sum, outer_new_addr[VLIB_TX].as_u64[0]);
572 sum = ip_csum_add_even (sum, outer_new_addr[VLIB_TX].as_u64[1]);
573 sum = ip_csum_sub_even (sum, outer_old_addr[VLIB_TX].as_u64[0]);
574 sum = ip_csum_sub_even (sum, outer_old_addr[VLIB_TX].as_u64[1]);
577 if (has_ip6_address (&outer_new_addr[VLIB_RX]))
579 sum = ip_csum_add_even (sum, outer_new_addr[VLIB_RX].as_u64[0]);
580 sum = ip_csum_add_even (sum, outer_new_addr[VLIB_RX].as_u64[1]);
581 sum = ip_csum_sub_even (sum, outer_old_addr[VLIB_RX].as_u64[0]);
582 sum = ip_csum_sub_even (sum, outer_old_addr[VLIB_RX].as_u64[1]);
585 /* Translate inner TCP / UDP */
586 if (ip6->protocol == IP_PROTOCOL_TCP)
588 inner_l4_old_sum = inner_l4_sum = tcp->checksum;
589 cnat_ip6_translate_l4 (ip6, udp, &inner_l4_sum, new_addr, new_port);
590 tcp->checksum = ip_csum_fold (inner_l4_sum);
592 else if (ip6->protocol == IP_PROTOCOL_UDP)
594 inner_l4_old_sum = inner_l4_sum = udp->checksum;
595 cnat_ip6_translate_l4 (ip6, udp, &inner_l4_sum, new_addr, new_port);
596 udp->checksum = ip_csum_fold (inner_l4_sum);
601 /* UDP/TCP checksum changed */
602 sum = ip_csum_update (sum, inner_l4_old_sum, inner_l4_sum,
603 ip4_header_t /* cheat */ ,
606 /* UDP/TCP Ports changed */
607 if (old_port[VLIB_TX] && new_port[VLIB_TX])
608 sum = ip_csum_update (sum, old_port[VLIB_TX], new_port[VLIB_TX],
609 ip4_header_t /* cheat */ ,
610 length /* changed member */ );
612 if (old_port[VLIB_RX] && new_port[VLIB_RX])
613 sum = ip_csum_update (sum, old_port[VLIB_RX], new_port[VLIB_RX],
614 ip4_header_t /* cheat */ ,
615 length /* changed member */ );
618 cnat_ip6_translate_l3 (ip6, new_addr);
619 /* IP src/dst addr changed */
620 if (has_ip6_address (&new_addr[VLIB_TX]))
622 sum = ip_csum_add_even (sum, new_addr[VLIB_TX].as_u64[0]);
623 sum = ip_csum_add_even (sum, new_addr[VLIB_TX].as_u64[1]);
624 sum = ip_csum_sub_even (sum, old_addr[VLIB_TX].as_u64[0]);
625 sum = ip_csum_sub_even (sum, old_addr[VLIB_TX].as_u64[1]);
628 if (has_ip6_address (&new_addr[VLIB_RX]))
630 sum = ip_csum_add_even (sum, new_addr[VLIB_RX].as_u64[0]);
631 sum = ip_csum_add_even (sum, new_addr[VLIB_RX].as_u64[1]);
632 sum = ip_csum_sub_even (sum, old_addr[VLIB_RX].as_u64[0]);
633 sum = ip_csum_sub_even (sum, old_addr[VLIB_RX].as_u64[1]);
636 icmp->checksum = ip_csum_fold (sum);
639 static_always_inline void
640 cnat_translation_ip6 (const cnat_session_t * session,
641 ip6_header_t * ip6, udp_header_t * udp)
643 tcp_header_t *tcp = (tcp_header_t *) udp;
644 ip6_address_t new_addr[VLIB_N_DIR];
645 u16 new_port[VLIB_N_DIR];
647 ip6_address_copy (&new_addr[VLIB_TX], &session->value.cs_ip[VLIB_TX].ip6);
648 ip6_address_copy (&new_addr[VLIB_RX], &session->value.cs_ip[VLIB_RX].ip6);
649 new_port[VLIB_TX] = session->value.cs_port[VLIB_TX];
650 new_port[VLIB_RX] = session->value.cs_port[VLIB_RX];
652 if (ip6->protocol == IP_PROTOCOL_TCP)
654 ip_csum_t sum = tcp->checksum;
655 cnat_ip6_translate_l4 (ip6, udp, &sum, new_addr, new_port);
656 tcp->checksum = ip_csum_fold (sum);
657 cnat_ip6_translate_l3 (ip6, new_addr);
658 cnat_tcp_update_session_lifetime (tcp, session->value.cs_ts_index);
660 else if (ip6->protocol == IP_PROTOCOL_UDP)
662 ip_csum_t sum = udp->checksum;
663 cnat_ip6_translate_l4 (ip6, udp, &sum, new_addr, new_port);
664 udp->checksum = ip_csum_fold (sum);
665 cnat_ip6_translate_l3 (ip6, new_addr);
667 else if (ip6->protocol == IP_PROTOCOL_ICMP6)
669 icmp46_header_t *icmp = (icmp46_header_t *) udp;
670 if (icmp6_type_is_error_message (icmp->type))
672 /* SNAT only if src_addr was translated */
673 u8 snat_outer_ip = cmp_ip6_address (&ip6->src_address,
676 cnat_translation_icmp6_error (ip6, icmp, new_addr, new_port,
679 else if (icmp6_type_is_echo (icmp->type))
680 cnat_translation_icmp6_echo (ip6, icmp, new_addr, new_port);
684 static_always_inline void
685 cnat_session_make_key (vlib_buffer_t *b, ip_address_family_t af,
686 cnat_session_location_t cs_loc, cnat_bihash_kv_t *bkey)
689 cnat_session_t *session = (cnat_session_t *) bkey;
691 session->key.cs_af = af;
693 session->key.cs_loc = cs_loc;
694 session->key.__cs_pad = 0;
695 if (cs_loc == CNAT_LOCATION_OUTPUT)
697 iph_offset = vnet_buffer (b)->ip.save_rewrite_length;
702 ip4 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b) + iph_offset);
704 if (PREDICT_FALSE (ip4->protocol == IP_PROTOCOL_ICMP))
706 icmp46_header_t *icmp = (icmp46_header_t *) (ip4 + 1);
707 if (icmp_type_is_error_message (icmp->type))
709 ip4 = (ip4_header_t *) (icmp + 2); /* Use inner packet */
710 udp = (udp_header_t *) (ip4 + 1);
711 /* Swap dst & src for search as ICMP payload is reversed */
712 ip46_address_set_ip4 (&session->key.cs_ip[VLIB_RX],
714 ip46_address_set_ip4 (&session->key.cs_ip[VLIB_TX],
716 session->key.cs_proto = ip4->protocol;
717 session->key.cs_port[VLIB_TX] = udp->src_port;
718 session->key.cs_port[VLIB_RX] = udp->dst_port;
720 else if (icmp_type_is_echo (icmp->type))
722 cnat_echo_header_t *echo = (cnat_echo_header_t *) (icmp + 1);
723 ip46_address_set_ip4 (&session->key.cs_ip[VLIB_TX],
725 ip46_address_set_ip4 (&session->key.cs_ip[VLIB_RX],
727 session->key.cs_proto = ip4->protocol;
728 session->key.cs_port[VLIB_TX] = echo->identifier;
729 session->key.cs_port[VLIB_RX] = echo->identifier;
734 else if (ip4->protocol == IP_PROTOCOL_UDP ||
735 ip4->protocol == IP_PROTOCOL_TCP)
737 udp = (udp_header_t *) (ip4 + 1);
738 ip46_address_set_ip4 (&session->key.cs_ip[VLIB_TX],
740 ip46_address_set_ip4 (&session->key.cs_ip[VLIB_RX],
742 session->key.cs_proto = ip4->protocol;
743 session->key.cs_port[VLIB_RX] = udp->src_port;
744 session->key.cs_port[VLIB_TX] = udp->dst_port;
752 ip6 = (ip6_header_t *) ((u8 *) vlib_buffer_get_current (b) + iph_offset);
753 if (PREDICT_FALSE (ip6->protocol == IP_PROTOCOL_ICMP6))
755 icmp46_header_t *icmp = (icmp46_header_t *) (ip6 + 1);
756 if (icmp6_type_is_error_message (icmp->type))
758 ip6 = (ip6_header_t *) (icmp + 2); /* Use inner packet */
759 udp = (udp_header_t *) (ip6 + 1);
760 /* Swap dst & src for search as ICMP payload is reversed */
761 ip46_address_set_ip6 (&session->key.cs_ip[VLIB_RX],
763 ip46_address_set_ip6 (&session->key.cs_ip[VLIB_TX],
765 session->key.cs_proto = ip6->protocol;
766 session->key.cs_port[VLIB_TX] = udp->src_port;
767 session->key.cs_port[VLIB_RX] = udp->dst_port;
769 else if (icmp6_type_is_echo (icmp->type))
771 cnat_echo_header_t *echo = (cnat_echo_header_t *) (icmp + 1);
772 ip46_address_set_ip6 (&session->key.cs_ip[VLIB_TX],
774 ip46_address_set_ip6 (&session->key.cs_ip[VLIB_RX],
776 session->key.cs_proto = ip6->protocol;
777 session->key.cs_port[VLIB_TX] = echo->identifier;
778 session->key.cs_port[VLIB_RX] = echo->identifier;
783 else if (ip6->protocol == IP_PROTOCOL_UDP ||
784 ip6->protocol == IP_PROTOCOL_TCP)
786 udp = (udp_header_t *) (ip6 + 1);
787 ip46_address_set_ip6 (&session->key.cs_ip[VLIB_TX],
789 ip46_address_set_ip6 (&session->key.cs_ip[VLIB_RX],
791 session->key.cs_port[VLIB_RX] = udp->src_port;
792 session->key.cs_port[VLIB_TX] = udp->dst_port;
793 session->key.cs_proto = ip6->protocol;
801 /* Ensure we dont find anything */
802 session->key.cs_proto = 0;
806 static_always_inline cnat_ep_trk_t *
807 cnat_load_balance (const cnat_translation_t *ct, ip_address_family_t af,
808 ip4_header_t *ip4, ip6_header_t *ip6, u32 *dpoi_index)
810 cnat_main_t *cm = &cnat_main;
811 const load_balance_t *lb0;
812 const dpo_id_t *dpo0;
813 u32 hash_c0, bucket0;
815 lb0 = load_balance_get (ct->ct_lb.dpoi_index);
816 if (PREDICT_FALSE (!lb0->lb_n_buckets))
819 /* session table miss */
820 hash_c0 = (AF_IP4 == af ? ip4_compute_flow_hash (ip4, lb0->lb_hash_config) :
821 ip6_compute_flow_hash (ip6, lb0->lb_hash_config));
823 if (PREDICT_FALSE (ct->lb_type == CNAT_LB_MAGLEV))
824 bucket0 = ct->lb_maglev[hash_c0 % cm->maglev_len];
826 bucket0 = hash_c0 % lb0->lb_n_buckets;
828 dpo0 = load_balance_get_fwd_bucket (lb0, bucket0);
830 *dpoi_index = dpo0->dpoi_index;
832 return &ct->ct_active_paths[bucket0];
836 * Create NAT sessions
837 * rsession_location is the location the (return) session will be
840 static_always_inline void
841 cnat_session_create (cnat_session_t *session, cnat_node_ctx_t *ctx,
842 cnat_session_location_t rsession_location,
846 cnat_bihash_kv_t rkey;
847 cnat_session_t *rsession = (cnat_session_t *) & rkey;
848 cnat_bihash_kv_t *bkey = (cnat_bihash_kv_t *) session;
849 cnat_bihash_kv_t rvalue;
852 session->value.cs_ts_index = cnat_timestamp_new (ctx->now);
853 cnat_bihash_add_del (&cnat_session_db, bkey, 1);
855 if (!(rsession_flags & CNAT_SESSION_FLAG_NO_CLIENT))
857 /* is this the first time we've seen this source address */
858 cc = (AF_IP4 == ctx->af ?
859 cnat_client_ip4_find (&session->value.cs_ip[VLIB_RX].ip4) :
860 cnat_client_ip6_find (&session->value.cs_ip[VLIB_RX].ip6));
868 addr.version = ctx->af;
869 ip46_address_copy (&addr.ip, &session->value.cs_ip[VLIB_RX]);
872 clib_spinlock_lock (&cnat_client_db.throttle_lock);
874 p = hash_get_mem (cnat_client_db.throttle_mem, &addr);
878 hash_set_mem (cnat_client_db.throttle_mem, &addr, refcnt);
881 hash_set_mem_alloc (&cnat_client_db.throttle_mem, &addr, 0);
883 clib_spinlock_unlock (&cnat_client_db.throttle_lock);
885 /* fire client create to the main thread */
887 vl_api_rpc_call_main_thread (cnat_client_learn, (u8 *) &addr,
892 /* Refcount reverse session */
893 cnat_client_cnt_session (cc);
897 /* create the reverse flow key */
898 ip46_address_copy (&rsession->key.cs_ip[VLIB_RX],
899 &session->value.cs_ip[VLIB_TX]);
900 ip46_address_copy (&rsession->key.cs_ip[VLIB_TX],
901 &session->value.cs_ip[VLIB_RX]);
902 rsession->key.cs_proto = session->key.cs_proto;
903 rsession->key.cs_loc = rsession_location;
904 rsession->key.__cs_pad = 0;
905 rsession->key.cs_af = ctx->af;
906 rsession->key.cs_port[VLIB_RX] = session->value.cs_port[VLIB_TX];
907 rsession->key.cs_port[VLIB_TX] = session->value.cs_port[VLIB_RX];
909 /* First search for existing reverse session */
910 rv = cnat_bihash_search_i2 (&cnat_session_db, &rkey, &rvalue);
913 /* Reverse session already exists
914 cleanup before creating for refcnts */
915 cnat_session_t *found_rsession = (cnat_session_t *) & rvalue;
916 cnat_session_free (found_rsession);
918 /* add the reverse flow */
919 ip46_address_copy (&rsession->value.cs_ip[VLIB_RX],
920 &session->key.cs_ip[VLIB_TX]);
921 ip46_address_copy (&rsession->value.cs_ip[VLIB_TX],
922 &session->key.cs_ip[VLIB_RX]);
923 rsession->value.cs_ts_index = session->value.cs_ts_index;
924 rsession->value.cs_lbi = INDEX_INVALID;
925 rsession->value.flags = rsession_flags;
926 rsession->value.cs_port[VLIB_TX] = session->key.cs_port[VLIB_RX];
927 rsession->value.cs_port[VLIB_RX] = session->key.cs_port[VLIB_TX];
929 cnat_bihash_add_del (&cnat_session_db, &rkey, 1);
933 cnat_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
934 vlib_frame_t *frame, cnat_node_sub_t cnat_sub,
935 ip_address_family_t af, cnat_session_location_t cs_loc,
938 u32 n_left, *from, thread_index;
939 vlib_buffer_t *bufs[VLIB_FRAME_SIZE];
940 vlib_buffer_t **b = bufs;
941 u16 nexts[VLIB_FRAME_SIZE], *next;
944 thread_index = vm->thread_index;
945 from = vlib_frame_vector_args (frame);
946 n_left = frame->n_vectors;
948 vlib_get_buffers (vm, from, bufs, n_left);
949 now = vlib_time_now (vm);
950 cnat_session_t *session[4];
951 cnat_bihash_kv_t bkey[4], bvalue[4];
955 cnat_node_ctx_t ctx = { now, thread_index, af, do_trace };
959 /* Kickstart our state */
960 cnat_session_make_key (b[3], af, cs_loc, &bkey[3]);
961 cnat_session_make_key (b[2], af, cs_loc, &bkey[2]);
962 cnat_session_make_key (b[1], af, cs_loc, &bkey[1]);
963 cnat_session_make_key (b[0], af, cs_loc, &bkey[0]);
965 hash[3] = cnat_bihash_hash (&bkey[3]);
966 hash[2] = cnat_bihash_hash (&bkey[2]);
967 hash[1] = cnat_bihash_hash (&bkey[1]);
968 hash[0] = cnat_bihash_hash (&bkey[0]);
975 vlib_prefetch_buffer_header (b[11], LOAD);
976 vlib_prefetch_buffer_header (b[10], LOAD);
977 vlib_prefetch_buffer_header (b[9], LOAD);
978 vlib_prefetch_buffer_header (b[8], LOAD);
981 rv[3] = cnat_bihash_search_i2_hash (&cnat_session_db, hash[3], &bkey[3],
983 session[3] = (cnat_session_t *) (rv[3] ? &bkey[3] : &bvalue[3]);
984 next[3] = cnat_sub (vm, node, b[3], &ctx, rv[3], session[3]);
986 rv[2] = cnat_bihash_search_i2_hash (&cnat_session_db, hash[2], &bkey[2],
988 session[2] = (cnat_session_t *) (rv[2] ? &bkey[2] : &bvalue[2]);
989 next[2] = cnat_sub (vm, node, b[2], &ctx, rv[2], session[2]);
991 rv[1] = cnat_bihash_search_i2_hash (&cnat_session_db, hash[1], &bkey[1],
993 session[1] = (cnat_session_t *) (rv[1] ? &bkey[1] : &bvalue[1]);
994 next[1] = cnat_sub (vm, node, b[1], &ctx, rv[1], session[1]);
996 rv[0] = cnat_bihash_search_i2_hash (&cnat_session_db, hash[0], &bkey[0],
998 session[0] = (cnat_session_t *) (rv[0] ? &bkey[0] : &bvalue[0]);
999 next[0] = cnat_sub (vm, node, b[0], &ctx, rv[0], session[0]);
1001 cnat_session_make_key (b[7], af, cs_loc, &bkey[3]);
1002 cnat_session_make_key (b[6], af, cs_loc, &bkey[2]);
1003 cnat_session_make_key (b[5], af, cs_loc, &bkey[1]);
1004 cnat_session_make_key (b[4], af, cs_loc, &bkey[0]);
1006 hash[3] = cnat_bihash_hash (&bkey[3]);
1007 hash[2] = cnat_bihash_hash (&bkey[2]);
1008 hash[1] = cnat_bihash_hash (&bkey[1]);
1009 hash[0] = cnat_bihash_hash (&bkey[0]);
1011 cnat_bihash_prefetch_bucket (&cnat_session_db, hash[3]);
1012 cnat_bihash_prefetch_bucket (&cnat_session_db, hash[2]);
1013 cnat_bihash_prefetch_bucket (&cnat_session_db, hash[1]);
1014 cnat_bihash_prefetch_bucket (&cnat_session_db, hash[0]);
1016 cnat_bihash_prefetch_data (&cnat_session_db, hash[3]);
1017 cnat_bihash_prefetch_data (&cnat_session_db, hash[2]);
1018 cnat_bihash_prefetch_data (&cnat_session_db, hash[1]);
1019 cnat_bihash_prefetch_data (&cnat_session_db, hash[0]);
1028 cnat_session_make_key (b[0], af, cs_loc, &bkey[0]);
1029 rv[0] = cnat_bihash_search_i2 (&cnat_session_db, &bkey[0], &bvalue[0]);
1031 session[0] = (cnat_session_t *) (rv[0] ? &bkey[0] : &bvalue[0]);
1032 next[0] = cnat_sub (vm, node, b[0], &ctx, rv[0], session[0]);
1039 vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
1041 return frame->n_vectors;
1045 * fd.io coding-style-patch-verification: ON
1048 * eval: (c-set-style "gnu")