2 * Copyright (c) 2017 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
17 * @brief NAT64 IPv6 to IPv4 translation (inside to outside network)
20 #include <nat/nat64.h>
21 #include <nat/nat_reass.h>
22 #include <nat/nat_inlines.h>
23 #include <vnet/ip/ip6_to_ip4.h>
24 #include <vnet/fib/fib_table.h>
31 } nat64_in2out_trace_t;
34 format_nat64_in2out_trace (u8 * s, va_list * args)
36 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
37 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
38 nat64_in2out_trace_t *t = va_arg (*args, nat64_in2out_trace_t *);
41 tag = t->is_slow_path ? "NAT64-in2out-slowpath" : "NAT64-in2out";
44 format (s, "%s: sw_if_index %d, next index %d", tag, t->sw_if_index,
55 } nat64_in2out_reass_trace_t;
58 format_nat64_in2out_reass_trace (u8 * s, va_list * args)
60 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
61 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
62 nat64_in2out_reass_trace_t *t =
63 va_arg (*args, nat64_in2out_reass_trace_t *);
66 format (s, "NAT64-in2out-reass: sw_if_index %d, next index %d, status %s",
67 t->sw_if_index, t->next_index,
68 t->cached ? "cached" : "translated");
73 vlib_node_registration_t nat64_in2out_node;
74 vlib_node_registration_t nat64_in2out_slowpath_node;
75 vlib_node_registration_t nat64_in2out_reass_node;
76 vlib_node_registration_t nat64_in2out_handoff_node;
78 #define foreach_nat64_in2out_error \
79 _(UNSUPPORTED_PROTOCOL, "unsupported protocol") \
80 _(IN2OUT_PACKETS, "good in2out packets processed") \
81 _(NO_TRANSLATION, "no translation") \
82 _(UNKNOWN, "unknown") \
83 _(DROP_FRAGMENT, "Drop fragment") \
84 _(MAX_REASS, "Maximum reassemblies exceeded") \
85 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
90 #define _(sym,str) NAT64_IN2OUT_ERROR_##sym,
91 foreach_nat64_in2out_error
94 } nat64_in2out_error_t;
96 static char *nat64_in2out_error_strings[] = {
97 #define _(sym,string) string,
98 foreach_nat64_in2out_error
104 NAT64_IN2OUT_NEXT_IP4_LOOKUP,
105 NAT64_IN2OUT_NEXT_IP6_LOOKUP,
106 NAT64_IN2OUT_NEXT_DROP,
107 NAT64_IN2OUT_NEXT_SLOWPATH,
108 NAT64_IN2OUT_NEXT_REASS,
110 } nat64_in2out_next_t;
112 typedef struct nat64_in2out_set_ctx_t_
117 } nat64_in2out_set_ctx_t;
120 nat64_not_translate (u32 sw_if_index, ip6_address_t ip6_addr)
123 ip6_main_t *im6 = &ip6_main;
124 ip_lookup_main_t *lm6 = &im6->lookup_main;
125 ip_interface_address_t *ia = 0;
128 foreach_ip_interface_address (lm6, ia, sw_if_index, 0,
130 addr = ip_interface_address_get_address (lm6, ia);
131 if (0 == ip6_address_compare (addr, &ip6_addr))
140 * @brief Check whether is a hairpinning.
142 * If the destination IP address of the packet is an IPv4 address assigned to
143 * the NAT64 itself, then the packet is a hairpin packet.
145 * param dst_addr Destination address of the packet.
147 * @returns 1 if hairpinning, otherwise 0.
149 static_always_inline int
150 is_hairpinning (ip6_address_t * dst_addr)
152 nat64_main_t *nm = &nat64_main;
155 for (i = 0; i < vec_len (nm->addr_pool); i++)
157 if (nm->addr_pool[i].addr.as_u32 == dst_addr->as_u32[3])
165 nat64_in2out_tcp_udp_set_cb (ip6_header_t * ip6, ip4_header_t * ip4,
168 nat64_main_t *nm = &nat64_main;
169 nat64_in2out_set_ctx_t *ctx = arg;
170 nat64_db_bib_entry_t *bibe;
171 nat64_db_st_entry_t *ste;
172 ip46_address_t saddr, daddr;
173 u32 sw_if_index, fib_index;
174 udp_header_t *udp = ip6_next_header (ip6);
175 u8 proto = ip6->protocol;
176 u16 sport = udp->src_port;
177 u16 dport = udp->dst_port;
178 nat64_db_t *db = &nm->db[ctx->thread_index];
180 sw_if_index = vnet_buffer (ctx->b)->sw_if_index[VLIB_RX];
182 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
184 saddr.as_u64[0] = ip6->src_address.as_u64[0];
185 saddr.as_u64[1] = ip6->src_address.as_u64[1];
186 daddr.as_u64[0] = ip6->dst_address.as_u64[0];
187 daddr.as_u64[1] = ip6->dst_address.as_u64[1];
190 nat64_db_st_entry_find (db, &saddr, &daddr, sport, dport, proto,
195 bibe = nat64_db_bib_entry_by_index (db, proto, ste->bibe_index);
201 bibe = nat64_db_bib_entry_find (db, &saddr, sport, proto, fib_index, 1);
206 ip4_address_t out_addr;
207 if (nat64_alloc_out_addr_and_port
208 (fib_index, ip_proto_to_snat_proto (proto), &out_addr,
209 &out_port, ctx->thread_index))
213 nat64_db_bib_entry_create (db, &ip6->src_address, &out_addr,
214 sport, out_port, fib_index, proto, 0);
219 nat64_extract_ip4 (&ip6->dst_address, &daddr.ip4, fib_index);
221 nat64_db_st_entry_create (db, bibe, &ip6->dst_address,
227 ip4->src_address.as_u32 = bibe->out_addr.as_u32;
228 udp->src_port = bibe->out_port;
230 ip4->dst_address.as_u32 = ste->out_r_addr.as_u32;
232 if (proto == IP_PROTOCOL_TCP)
236 tcp_header_t *tcp = ip6_next_header (ip6);
238 nat64_tcp_session_set_state (ste, tcp, 1);
239 checksum = &tcp->checksum;
240 csum = ip_csum_sub_even (*checksum, sport);
241 csum = ip_csum_add_even (csum, udp->src_port);
242 *checksum = ip_csum_fold (csum);
245 nat64_session_reset_timeout (ste, ctx->vm);
251 nat64_in2out_icmp_set_cb (ip6_header_t * ip6, ip4_header_t * ip4, void *arg)
253 nat64_main_t *nm = &nat64_main;
254 nat64_in2out_set_ctx_t *ctx = arg;
255 nat64_db_bib_entry_t *bibe;
256 nat64_db_st_entry_t *ste;
257 ip46_address_t saddr, daddr;
258 u32 sw_if_index, fib_index;
259 icmp46_header_t *icmp = ip6_next_header (ip6);
260 nat64_db_t *db = &nm->db[ctx->thread_index];
262 sw_if_index = vnet_buffer (ctx->b)->sw_if_index[VLIB_RX];
264 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
266 saddr.as_u64[0] = ip6->src_address.as_u64[0];
267 saddr.as_u64[1] = ip6->src_address.as_u64[1];
268 daddr.as_u64[0] = ip6->dst_address.as_u64[0];
269 daddr.as_u64[1] = ip6->dst_address.as_u64[1];
271 if (icmp->type == ICMP4_echo_request || icmp->type == ICMP4_echo_reply)
273 u16 in_id = ((u16 *) (icmp))[2];
275 nat64_db_st_entry_find (db, &saddr, &daddr, in_id, 0,
276 IP_PROTOCOL_ICMP, fib_index, 1);
281 nat64_db_bib_entry_by_index (db, IP_PROTOCOL_ICMP,
289 nat64_db_bib_entry_find (db, &saddr, in_id,
290 IP_PROTOCOL_ICMP, fib_index, 1);
295 ip4_address_t out_addr;
296 if (nat64_alloc_out_addr_and_port
297 (fib_index, SNAT_PROTOCOL_ICMP, &out_addr, &out_id,
302 nat64_db_bib_entry_create (db, &ip6->src_address,
303 &out_addr, in_id, out_id,
304 fib_index, IP_PROTOCOL_ICMP, 0);
309 nat64_extract_ip4 (&ip6->dst_address, &daddr.ip4, fib_index);
311 nat64_db_st_entry_create (db, bibe, &ip6->dst_address,
317 nat64_session_reset_timeout (ste, ctx->vm);
319 ip4->src_address.as_u32 = bibe->out_addr.as_u32;
320 ((u16 *) (icmp))[2] = bibe->out_port;
322 ip4->dst_address.as_u32 = ste->out_r_addr.as_u32;
326 if (!vec_len (nm->addr_pool))
329 ip4->src_address.as_u32 = nm->addr_pool[0].addr.as_u32;
330 nat64_extract_ip4 (&ip6->dst_address, &ip4->dst_address, fib_index);
337 nat64_in2out_inner_icmp_set_cb (ip6_header_t * ip6, ip4_header_t * ip4,
340 nat64_main_t *nm = &nat64_main;
341 nat64_in2out_set_ctx_t *ctx = arg;
342 nat64_db_st_entry_t *ste;
343 nat64_db_bib_entry_t *bibe;
344 ip46_address_t saddr, daddr;
345 u32 sw_if_index, fib_index;
346 u8 proto = ip6->protocol;
347 nat64_db_t *db = &nm->db[ctx->thread_index];
349 sw_if_index = vnet_buffer (ctx->b)->sw_if_index[VLIB_RX];
351 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
353 saddr.as_u64[0] = ip6->src_address.as_u64[0];
354 saddr.as_u64[1] = ip6->src_address.as_u64[1];
355 daddr.as_u64[0] = ip6->dst_address.as_u64[0];
356 daddr.as_u64[1] = ip6->dst_address.as_u64[1];
358 if (proto == IP_PROTOCOL_ICMP6)
360 icmp46_header_t *icmp = ip6_next_header (ip6);
361 u16 in_id = ((u16 *) (icmp))[2];
362 proto = IP_PROTOCOL_ICMP;
365 (icmp->type == ICMP4_echo_request
366 || icmp->type == ICMP4_echo_reply))
370 nat64_db_st_entry_find (db, &daddr, &saddr, in_id, 0, proto,
375 bibe = nat64_db_bib_entry_by_index (db, proto, ste->bibe_index);
379 ip4->dst_address.as_u32 = bibe->out_addr.as_u32;
380 ((u16 *) (icmp))[2] = bibe->out_port;
381 ip4->src_address.as_u32 = ste->out_r_addr.as_u32;
385 udp_header_t *udp = ip6_next_header (ip6);
386 tcp_header_t *tcp = ip6_next_header (ip6);
390 u16 sport = udp->src_port;
391 u16 dport = udp->dst_port;
394 nat64_db_st_entry_find (db, &daddr, &saddr, dport, sport, proto,
399 bibe = nat64_db_bib_entry_by_index (db, proto, ste->bibe_index);
403 ip4->dst_address.as_u32 = bibe->out_addr.as_u32;
404 udp->dst_port = bibe->out_port;
405 ip4->src_address.as_u32 = ste->out_r_addr.as_u32;
407 if (proto == IP_PROTOCOL_TCP)
408 checksum = &tcp->checksum;
410 checksum = &udp->checksum;
411 csum = ip_csum_sub_even (*checksum, dport);
412 csum = ip_csum_add_even (csum, udp->dst_port);
413 *checksum = ip_csum_fold (csum);
419 typedef struct unk_proto_st_walk_ctx_t_
421 ip6_address_t src_addr;
422 ip6_address_t dst_addr;
423 ip4_address_t out_addr;
427 } unk_proto_st_walk_ctx_t;
430 unk_proto_st_walk (nat64_db_st_entry_t * ste, void *arg)
432 nat64_main_t *nm = &nat64_main;
433 unk_proto_st_walk_ctx_t *ctx = arg;
434 nat64_db_bib_entry_t *bibe;
435 ip46_address_t saddr, daddr;
436 nat64_db_t *db = &nm->db[ctx->thread_index];
438 if (ip46_address_is_equal (&ste->in_r_addr, &ctx->dst_addr))
440 bibe = nat64_db_bib_entry_by_index (db, ste->proto, ste->bibe_index);
444 if (ip46_address_is_equal (&bibe->in_addr, &ctx->src_addr)
445 && bibe->fib_index == ctx->fib_index)
447 memset (&saddr, 0, sizeof (saddr));
448 saddr.ip4.as_u32 = bibe->out_addr.as_u32;
449 memset (&daddr, 0, sizeof (daddr));
450 nat64_extract_ip4 (&ctx->dst_addr, &daddr.ip4, ctx->fib_index);
452 if (nat64_db_st_entry_find
453 (db, &daddr, &saddr, 0, 0, ctx->proto, ctx->fib_index, 0))
456 ctx->out_addr.as_u32 = bibe->out_addr.as_u32;
465 nat64_in2out_unk_proto_set_cb (ip6_header_t * ip6, ip4_header_t * ip4,
468 nat64_main_t *nm = &nat64_main;
469 nat64_in2out_set_ctx_t *s_ctx = arg;
470 nat64_db_bib_entry_t *bibe;
471 nat64_db_st_entry_t *ste;
472 ip46_address_t saddr, daddr, addr;
473 u32 sw_if_index, fib_index;
474 u8 proto = ip6->protocol;
476 nat64_db_t *db = &nm->db[s_ctx->thread_index];
478 sw_if_index = vnet_buffer (s_ctx->b)->sw_if_index[VLIB_RX];
480 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
482 saddr.as_u64[0] = ip6->src_address.as_u64[0];
483 saddr.as_u64[1] = ip6->src_address.as_u64[1];
484 daddr.as_u64[0] = ip6->dst_address.as_u64[0];
485 daddr.as_u64[1] = ip6->dst_address.as_u64[1];
488 nat64_db_st_entry_find (db, &saddr, &daddr, 0, 0, proto, fib_index, 1);
492 bibe = nat64_db_bib_entry_by_index (db, proto, ste->bibe_index);
498 bibe = nat64_db_bib_entry_find (db, &saddr, 0, proto, fib_index, 1);
502 /* Choose same out address as for TCP/UDP session to same dst */
503 unk_proto_st_walk_ctx_t ctx = {
504 .src_addr.as_u64[0] = ip6->src_address.as_u64[0],
505 .src_addr.as_u64[1] = ip6->src_address.as_u64[1],
506 .dst_addr.as_u64[0] = ip6->dst_address.as_u64[0],
507 .dst_addr.as_u64[1] = ip6->dst_address.as_u64[1],
508 .out_addr.as_u32 = 0,
509 .fib_index = fib_index,
511 .thread_index = s_ctx->thread_index,
514 nat64_db_st_walk (db, IP_PROTOCOL_TCP, unk_proto_st_walk, &ctx);
516 if (!ctx.out_addr.as_u32)
517 nat64_db_st_walk (db, IP_PROTOCOL_UDP, unk_proto_st_walk, &ctx);
519 /* Verify if out address is not already in use for protocol */
520 memset (&addr, 0, sizeof (addr));
521 addr.ip4.as_u32 = ctx.out_addr.as_u32;
522 if (nat64_db_bib_entry_find (db, &addr, 0, proto, 0, 0))
523 ctx.out_addr.as_u32 = 0;
525 if (!ctx.out_addr.as_u32)
527 for (i = 0; i < vec_len (nm->addr_pool); i++)
529 addr.ip4.as_u32 = nm->addr_pool[i].addr.as_u32;
530 if (!nat64_db_bib_entry_find (db, &addr, 0, proto, 0, 0))
535 if (!ctx.out_addr.as_u32)
539 nat64_db_bib_entry_create (db, &ip6->src_address,
540 &ctx.out_addr, 0, 0, fib_index, proto,
546 nat64_extract_ip4 (&ip6->dst_address, &daddr.ip4, fib_index);
548 nat64_db_st_entry_create (db, bibe, &ip6->dst_address, &daddr.ip4, 0);
553 nat64_session_reset_timeout (ste, s_ctx->vm);
555 ip4->src_address.as_u32 = bibe->out_addr.as_u32;
556 ip4->dst_address.as_u32 = ste->out_r_addr.as_u32;
564 nat64_in2out_tcp_udp_hairpinning (vlib_main_t * vm, vlib_buffer_t * b,
565 ip6_header_t * ip6, u32 thread_index)
567 nat64_main_t *nm = &nat64_main;
568 nat64_db_bib_entry_t *bibe;
569 nat64_db_st_entry_t *ste;
570 ip46_address_t saddr, daddr;
571 u32 sw_if_index, fib_index;
572 udp_header_t *udp = ip6_next_header (ip6);
573 tcp_header_t *tcp = ip6_next_header (ip6);
574 u8 proto = ip6->protocol;
575 u16 sport = udp->src_port;
576 u16 dport = udp->dst_port;
579 nat64_db_t *db = &nm->db[thread_index];
581 sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
583 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
585 saddr.as_u64[0] = ip6->src_address.as_u64[0];
586 saddr.as_u64[1] = ip6->src_address.as_u64[1];
587 daddr.as_u64[0] = ip6->dst_address.as_u64[0];
588 daddr.as_u64[1] = ip6->dst_address.as_u64[1];
590 if (proto == IP_PROTOCOL_UDP)
591 checksum = &udp->checksum;
593 checksum = &tcp->checksum;
595 csum = ip_csum_sub_even (*checksum, ip6->src_address.as_u64[0]);
596 csum = ip_csum_sub_even (csum, ip6->src_address.as_u64[1]);
597 csum = ip_csum_sub_even (csum, ip6->dst_address.as_u64[0]);
598 csum = ip_csum_sub_even (csum, ip6->dst_address.as_u64[1]);
599 csum = ip_csum_sub_even (csum, sport);
600 csum = ip_csum_sub_even (csum, dport);
603 nat64_db_st_entry_find (db, &saddr, &daddr, sport, dport, proto,
608 bibe = nat64_db_bib_entry_by_index (db, proto, ste->bibe_index);
614 bibe = nat64_db_bib_entry_find (db, &saddr, sport, proto, fib_index, 1);
619 ip4_address_t out_addr;
620 if (nat64_alloc_out_addr_and_port
621 (fib_index, ip_proto_to_snat_proto (proto), &out_addr,
622 &out_port, thread_index))
626 nat64_db_bib_entry_create (db, &ip6->src_address, &out_addr,
627 sport, out_port, fib_index, proto, 0);
632 nat64_extract_ip4 (&ip6->dst_address, &daddr.ip4, fib_index);
634 nat64_db_st_entry_create (db, bibe, &ip6->dst_address,
640 if (proto == IP_PROTOCOL_TCP)
641 nat64_tcp_session_set_state (ste, tcp, 1);
643 nat64_session_reset_timeout (ste, vm);
645 sport = udp->src_port = bibe->out_port;
646 nat64_compose_ip6 (&ip6->src_address, &bibe->out_addr, fib_index);
648 memset (&daddr, 0, sizeof (daddr));
649 daddr.ip4.as_u32 = ste->out_r_addr.as_u32;
653 vec_foreach (db, nm->db)
655 bibe = nat64_db_bib_entry_find (db, &daddr, dport, proto, 0, 0);
665 ip6->dst_address.as_u64[0] = bibe->in_addr.as_u64[0];
666 ip6->dst_address.as_u64[1] = bibe->in_addr.as_u64[1];
667 udp->dst_port = bibe->in_port;
669 csum = ip_csum_add_even (csum, ip6->src_address.as_u64[0]);
670 csum = ip_csum_add_even (csum, ip6->src_address.as_u64[1]);
671 csum = ip_csum_add_even (csum, ip6->dst_address.as_u64[0]);
672 csum = ip_csum_add_even (csum, ip6->dst_address.as_u64[1]);
673 csum = ip_csum_add_even (csum, udp->src_port);
674 csum = ip_csum_add_even (csum, udp->dst_port);
675 *checksum = ip_csum_fold (csum);
681 nat64_in2out_icmp_hairpinning (vlib_main_t * vm, vlib_buffer_t * b,
682 ip6_header_t * ip6, u32 thread_index)
684 nat64_main_t *nm = &nat64_main;
685 nat64_db_bib_entry_t *bibe;
686 nat64_db_st_entry_t *ste;
687 icmp46_header_t *icmp = ip6_next_header (ip6);
688 ip6_header_t *inner_ip6;
689 ip46_address_t saddr, daddr;
690 u32 sw_if_index, fib_index;
694 u16 *checksum, sport, dport;
696 nat64_db_t *db = &nm->db[thread_index];
698 if (icmp->type == ICMP6_echo_request || icmp->type == ICMP6_echo_reply)
701 inner_ip6 = (ip6_header_t *) u8_ptr_add (icmp, 8);
703 proto = inner_ip6->protocol;
705 if (proto == IP_PROTOCOL_ICMP6)
708 sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
710 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
712 saddr.as_u64[0] = inner_ip6->src_address.as_u64[0];
713 saddr.as_u64[1] = inner_ip6->src_address.as_u64[1];
714 daddr.as_u64[0] = inner_ip6->dst_address.as_u64[0];
715 daddr.as_u64[1] = inner_ip6->dst_address.as_u64[1];
717 udp = ip6_next_header (inner_ip6);
718 tcp = ip6_next_header (inner_ip6);
720 sport = udp->src_port;
721 dport = udp->dst_port;
723 if (proto == IP_PROTOCOL_UDP)
724 checksum = &udp->checksum;
726 checksum = &tcp->checksum;
728 csum = ip_csum_sub_even (*checksum, inner_ip6->src_address.as_u64[0]);
729 csum = ip_csum_sub_even (csum, inner_ip6->src_address.as_u64[1]);
730 csum = ip_csum_sub_even (csum, inner_ip6->dst_address.as_u64[0]);
731 csum = ip_csum_sub_even (csum, inner_ip6->dst_address.as_u64[1]);
732 csum = ip_csum_sub_even (csum, sport);
733 csum = ip_csum_sub_even (csum, dport);
736 nat64_db_st_entry_find (db, &daddr, &saddr, dport, sport, proto,
741 bibe = nat64_db_bib_entry_by_index (db, proto, ste->bibe_index);
745 dport = udp->dst_port = bibe->out_port;
746 nat64_compose_ip6 (&inner_ip6->dst_address, &bibe->out_addr, fib_index);
748 memset (&saddr, 0, sizeof (saddr));
749 memset (&daddr, 0, sizeof (daddr));
750 saddr.ip4.as_u32 = ste->out_r_addr.as_u32;
751 daddr.ip4.as_u32 = bibe->out_addr.as_u32;
755 vec_foreach (db, nm->db)
757 ste = nat64_db_st_entry_find (db, &saddr, &daddr, sport, dport, proto,
768 bibe = nat64_db_bib_entry_by_index (db, proto, ste->bibe_index);
772 inner_ip6->src_address.as_u64[0] = bibe->in_addr.as_u64[0];
773 inner_ip6->src_address.as_u64[1] = bibe->in_addr.as_u64[1];
774 udp->src_port = bibe->in_port;
776 csum = ip_csum_add_even (csum, inner_ip6->src_address.as_u64[0]);
777 csum = ip_csum_add_even (csum, inner_ip6->src_address.as_u64[1]);
778 csum = ip_csum_add_even (csum, inner_ip6->dst_address.as_u64[0]);
779 csum = ip_csum_add_even (csum, inner_ip6->dst_address.as_u64[1]);
780 csum = ip_csum_add_even (csum, udp->src_port);
781 csum = ip_csum_add_even (csum, udp->dst_port);
782 *checksum = ip_csum_fold (csum);
784 if (!vec_len (nm->addr_pool))
787 nat64_compose_ip6 (&ip6->src_address, &nm->addr_pool[0].addr, fib_index);
788 ip6->dst_address.as_u64[0] = inner_ip6->src_address.as_u64[0];
789 ip6->dst_address.as_u64[1] = inner_ip6->src_address.as_u64[1];
792 csum = ip_csum_with_carry (0, ip6->payload_length);
793 csum = ip_csum_with_carry (csum, clib_host_to_net_u16 (ip6->protocol));
794 csum = ip_csum_with_carry (csum, ip6->src_address.as_u64[0]);
795 csum = ip_csum_with_carry (csum, ip6->src_address.as_u64[1]);
796 csum = ip_csum_with_carry (csum, ip6->dst_address.as_u64[0]);
797 csum = ip_csum_with_carry (csum, ip6->dst_address.as_u64[1]);
799 ip_incremental_checksum (csum, icmp,
800 clib_net_to_host_u16 (ip6->payload_length));
801 icmp->checksum = ~ip_csum_fold (csum);
807 nat64_in2out_unk_proto_hairpinning (vlib_main_t * vm, vlib_buffer_t * b,
808 ip6_header_t * ip6, u32 thread_index)
810 nat64_main_t *nm = &nat64_main;
811 nat64_db_bib_entry_t *bibe;
812 nat64_db_st_entry_t *ste;
813 ip46_address_t saddr, daddr, addr;
814 u32 sw_if_index, fib_index;
815 u8 proto = ip6->protocol;
817 nat64_db_t *db = &nm->db[thread_index];
819 sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
821 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
823 saddr.as_u64[0] = ip6->src_address.as_u64[0];
824 saddr.as_u64[1] = ip6->src_address.as_u64[1];
825 daddr.as_u64[0] = ip6->dst_address.as_u64[0];
826 daddr.as_u64[1] = ip6->dst_address.as_u64[1];
829 nat64_db_st_entry_find (db, &saddr, &daddr, 0, 0, proto, fib_index, 1);
833 bibe = nat64_db_bib_entry_by_index (db, proto, ste->bibe_index);
839 bibe = nat64_db_bib_entry_find (db, &saddr, 0, proto, fib_index, 1);
843 /* Choose same out address as for TCP/UDP session to same dst */
844 unk_proto_st_walk_ctx_t ctx = {
845 .src_addr.as_u64[0] = ip6->src_address.as_u64[0],
846 .src_addr.as_u64[1] = ip6->src_address.as_u64[1],
847 .dst_addr.as_u64[0] = ip6->dst_address.as_u64[0],
848 .dst_addr.as_u64[1] = ip6->dst_address.as_u64[1],
849 .out_addr.as_u32 = 0,
850 .fib_index = fib_index,
852 .thread_index = thread_index,
855 nat64_db_st_walk (db, IP_PROTOCOL_TCP, unk_proto_st_walk, &ctx);
857 if (!ctx.out_addr.as_u32)
858 nat64_db_st_walk (db, IP_PROTOCOL_UDP, unk_proto_st_walk, &ctx);
860 /* Verify if out address is not already in use for protocol */
861 memset (&addr, 0, sizeof (addr));
862 addr.ip4.as_u32 = ctx.out_addr.as_u32;
863 if (nat64_db_bib_entry_find (db, &addr, 0, proto, 0, 0))
864 ctx.out_addr.as_u32 = 0;
866 if (!ctx.out_addr.as_u32)
868 for (i = 0; i < vec_len (nm->addr_pool); i++)
870 addr.ip4.as_u32 = nm->addr_pool[i].addr.as_u32;
871 if (!nat64_db_bib_entry_find (db, &addr, 0, proto, 0, 0))
876 if (!ctx.out_addr.as_u32)
880 nat64_db_bib_entry_create (db, &ip6->src_address,
881 &ctx.out_addr, 0, 0, fib_index, proto,
887 nat64_extract_ip4 (&ip6->dst_address, &daddr.ip4, fib_index);
889 nat64_db_st_entry_create (db, bibe, &ip6->dst_address, &daddr.ip4, 0);
894 nat64_session_reset_timeout (ste, vm);
896 nat64_compose_ip6 (&ip6->src_address, &bibe->out_addr, fib_index);
898 memset (&daddr, 0, sizeof (daddr));
899 daddr.ip4.as_u32 = ste->out_r_addr.as_u32;
903 vec_foreach (db, nm->db)
905 bibe = nat64_db_bib_entry_find (db, &daddr, 0, proto, 0, 0);
915 ip6->dst_address.as_u64[0] = bibe->in_addr.as_u64[0];
916 ip6->dst_address.as_u64[1] = bibe->in_addr.as_u64[1];
922 nat64_in2out_node_fn_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
923 vlib_frame_t * frame, u8 is_slow_path)
925 u32 n_left_from, *from, *to_next;
926 nat64_in2out_next_t next_index;
927 u32 pkts_processed = 0;
928 u32 stats_node_index;
929 u32 thread_index = vm->thread_index;
932 is_slow_path ? nat64_in2out_slowpath_node.index : nat64_in2out_node.index;
934 from = vlib_frame_vector_args (frame);
935 n_left_from = frame->n_vectors;
936 next_index = node->cached_next_index;
938 while (n_left_from > 0)
942 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
944 while (n_left_from > 0 && n_left_to_next > 0)
950 u16 l4_offset0, frag_offset0;
953 nat64_in2out_set_ctx_t ctx0;
956 /* speculatively enqueue b0 to the current next frame */
964 b0 = vlib_get_buffer (vm, bi0);
965 ip60 = vlib_buffer_get_current (b0);
969 ctx0.thread_index = thread_index;
971 next0 = NAT64_IN2OUT_NEXT_IP4_LOOKUP;
975 (ip60, b0->current_length, &l4_protocol0, &l4_offset0,
978 next0 = NAT64_IN2OUT_NEXT_DROP;
979 b0->error = node->errors[NAT64_IN2OUT_ERROR_UNKNOWN];
983 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
985 if (nat64_not_translate (sw_if_index0, ip60->dst_address))
987 next0 = NAT64_IN2OUT_NEXT_IP6_LOOKUP;
991 proto0 = ip_proto_to_snat_proto (l4_protocol0);
995 if (PREDICT_TRUE (proto0 == ~0))
997 if (is_hairpinning (&ip60->dst_address))
999 next0 = NAT64_IN2OUT_NEXT_IP6_LOOKUP;
1000 if (nat64_in2out_unk_proto_hairpinning
1001 (vm, b0, ip60, thread_index))
1003 next0 = NAT64_IN2OUT_NEXT_DROP;
1005 node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
1010 if (ip6_to_ip4 (b0, nat64_in2out_unk_proto_set_cb, &ctx0))
1012 next0 = NAT64_IN2OUT_NEXT_DROP;
1014 node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
1022 if (PREDICT_FALSE (proto0 == ~0))
1024 next0 = NAT64_IN2OUT_NEXT_SLOWPATH;
1030 (ip60->protocol == IP_PROTOCOL_IPV6_FRAGMENTATION))
1032 next0 = NAT64_IN2OUT_NEXT_REASS;
1036 if (proto0 == SNAT_PROTOCOL_ICMP)
1038 if (is_hairpinning (&ip60->dst_address))
1040 next0 = NAT64_IN2OUT_NEXT_IP6_LOOKUP;
1041 if (nat64_in2out_icmp_hairpinning
1042 (vm, b0, ip60, thread_index))
1044 next0 = NAT64_IN2OUT_NEXT_DROP;
1046 node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
1052 (b0, nat64_in2out_icmp_set_cb, &ctx0,
1053 nat64_in2out_inner_icmp_set_cb, &ctx0))
1055 next0 = NAT64_IN2OUT_NEXT_DROP;
1056 b0->error = node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
1060 else if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
1062 if (is_hairpinning (&ip60->dst_address))
1064 next0 = NAT64_IN2OUT_NEXT_IP6_LOOKUP;
1065 if (nat64_in2out_tcp_udp_hairpinning
1066 (vm, b0, ip60, thread_index))
1068 next0 = NAT64_IN2OUT_NEXT_DROP;
1070 node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
1075 if (ip6_to_ip4_tcp_udp
1076 (b0, nat64_in2out_tcp_udp_set_cb, &ctx0, 0))
1078 next0 = NAT64_IN2OUT_NEXT_DROP;
1079 b0->error = node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
1085 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1086 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1088 nat64_in2out_trace_t *t =
1089 vlib_add_trace (vm, node, b0, sizeof (*t));
1090 t->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1091 t->next_index = next0;
1092 t->is_slow_path = is_slow_path;
1095 pkts_processed += next0 != NAT64_IN2OUT_NEXT_DROP;
1097 /* verify speculative enqueue, maybe switch current next frame */
1098 vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
1099 n_left_to_next, bi0, next0);
1101 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1103 vlib_node_increment_counter (vm, stats_node_index,
1104 NAT64_IN2OUT_ERROR_IN2OUT_PACKETS,
1106 return frame->n_vectors;
1110 nat64_in2out_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
1111 vlib_frame_t * frame)
1113 return nat64_in2out_node_fn_inline (vm, node, frame, 0);
1117 VLIB_REGISTER_NODE (nat64_in2out_node) = {
1118 .function = nat64_in2out_node_fn,
1119 .name = "nat64-in2out",
1120 .vector_size = sizeof (u32),
1121 .format_trace = format_nat64_in2out_trace,
1122 .type = VLIB_NODE_TYPE_INTERNAL,
1123 .n_errors = ARRAY_LEN (nat64_in2out_error_strings),
1124 .error_strings = nat64_in2out_error_strings,
1125 .n_next_nodes = NAT64_IN2OUT_N_NEXT,
1126 /* edit / add dispositions here */
1128 [NAT64_IN2OUT_NEXT_DROP] = "error-drop",
1129 [NAT64_IN2OUT_NEXT_IP4_LOOKUP] = "ip4-lookup",
1130 [NAT64_IN2OUT_NEXT_IP6_LOOKUP] = "ip6-lookup",
1131 [NAT64_IN2OUT_NEXT_SLOWPATH] = "nat64-in2out-slowpath",
1132 [NAT64_IN2OUT_NEXT_REASS] = "nat64-in2out-reass",
1137 VLIB_NODE_FUNCTION_MULTIARCH (nat64_in2out_node, nat64_in2out_node_fn);
1140 nat64_in2out_slowpath_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
1141 vlib_frame_t * frame)
1143 return nat64_in2out_node_fn_inline (vm, node, frame, 1);
1147 VLIB_REGISTER_NODE (nat64_in2out_slowpath_node) = {
1148 .function = nat64_in2out_slowpath_node_fn,
1149 .name = "nat64-in2out-slowpath",
1150 .vector_size = sizeof (u32),
1151 .format_trace = format_nat64_in2out_trace,
1152 .type = VLIB_NODE_TYPE_INTERNAL,
1153 .n_errors = ARRAY_LEN (nat64_in2out_error_strings),
1154 .error_strings = nat64_in2out_error_strings,
1155 .n_next_nodes = NAT64_IN2OUT_N_NEXT,
1156 /* edit / add dispositions here */
1158 [NAT64_IN2OUT_NEXT_DROP] = "error-drop",
1159 [NAT64_IN2OUT_NEXT_IP4_LOOKUP] = "ip4-lookup",
1160 [NAT64_IN2OUT_NEXT_IP6_LOOKUP] = "ip6-lookup",
1161 [NAT64_IN2OUT_NEXT_SLOWPATH] = "nat64-in2out-slowpath",
1162 [NAT64_IN2OUT_NEXT_REASS] = "nat64-in2out-reass",
1167 VLIB_NODE_FUNCTION_MULTIARCH (nat64_in2out_slowpath_node,
1168 nat64_in2out_slowpath_node_fn);
1170 typedef struct nat64_in2out_frag_set_ctx_t_
1178 } nat64_in2out_frag_set_ctx_t;
1181 nat64_in2out_frag_set_cb (ip6_header_t * ip6, ip4_header_t * ip4, void *arg)
1183 nat64_main_t *nm = &nat64_main;
1184 nat64_in2out_frag_set_ctx_t *ctx = arg;
1185 nat64_db_st_entry_t *ste;
1186 nat64_db_bib_entry_t *bibe;
1188 nat64_db_t *db = &nm->db[ctx->thread_index];
1190 ste = nat64_db_st_entry_by_index (db, ctx->proto, ctx->sess_index);
1194 bibe = nat64_db_bib_entry_by_index (db, ctx->proto, ste->bibe_index);
1198 nat64_session_reset_timeout (ste, ctx->vm);
1200 if (ctx->first_frag)
1202 udp = (udp_header_t *) u8_ptr_add (ip6, ctx->l4_offset);
1204 if (ctx->proto == IP_PROTOCOL_TCP)
1208 tcp_header_t *tcp = (tcp_header_t *) udp;
1210 nat64_tcp_session_set_state (ste, tcp, 1);
1211 checksum = &tcp->checksum;
1212 csum = ip_csum_sub_even (*checksum, tcp->src_port);
1213 csum = ip_csum_sub_even (csum, ip6->src_address.as_u64[0]);
1214 csum = ip_csum_sub_even (csum, ip6->src_address.as_u64[1]);
1215 csum = ip_csum_sub_even (csum, ip6->dst_address.as_u64[0]);
1216 csum = ip_csum_sub_even (csum, ip6->dst_address.as_u64[1]);
1217 csum = ip_csum_add_even (csum, bibe->out_port);
1218 csum = ip_csum_add_even (csum, bibe->out_addr.as_u32);
1219 csum = ip_csum_add_even (csum, ste->out_r_addr.as_u32);
1220 *checksum = ip_csum_fold (csum);
1223 udp->src_port = bibe->out_port;
1226 ip4->src_address.as_u32 = bibe->out_addr.as_u32;
1227 ip4->dst_address.as_u32 = ste->out_r_addr.as_u32;
1233 nat64_in2out_frag_hairpinning (vlib_buffer_t * b, ip6_header_t * ip6,
1234 nat64_in2out_frag_set_ctx_t * ctx)
1236 nat64_main_t *nm = &nat64_main;
1237 nat64_db_st_entry_t *ste;
1238 nat64_db_bib_entry_t *bibe;
1239 udp_header_t *udp = (udp_header_t *) u8_ptr_add (ip6, ctx->l4_offset);
1240 tcp_header_t *tcp = (tcp_header_t *) udp;
1241 u16 sport = udp->src_port;
1242 u16 dport = udp->dst_port;
1245 ip46_address_t daddr;
1246 nat64_db_t *db = &nm->db[ctx->thread_index];
1248 if (ctx->first_frag)
1250 if (ctx->proto == IP_PROTOCOL_UDP)
1251 checksum = &udp->checksum;
1253 checksum = &tcp->checksum;
1255 csum = ip_csum_sub_even (*checksum, ip6->src_address.as_u64[0]);
1256 csum = ip_csum_sub_even (csum, ip6->src_address.as_u64[1]);
1257 csum = ip_csum_sub_even (csum, ip6->dst_address.as_u64[0]);
1258 csum = ip_csum_sub_even (csum, ip6->dst_address.as_u64[1]);
1259 csum = ip_csum_sub_even (csum, sport);
1260 csum = ip_csum_sub_even (csum, dport);
1263 ste = nat64_db_st_entry_by_index (db, ctx->proto, ctx->sess_index);
1267 bibe = nat64_db_bib_entry_by_index (db, ctx->proto, ste->bibe_index);
1271 if (ctx->proto == IP_PROTOCOL_TCP)
1272 nat64_tcp_session_set_state (ste, tcp, 1);
1274 nat64_session_reset_timeout (ste, ctx->vm);
1276 sport = bibe->out_port;
1277 dport = ste->r_port;
1279 nat64_compose_ip6 (&ip6->src_address, &bibe->out_addr, bibe->fib_index);
1281 memset (&daddr, 0, sizeof (daddr));
1282 daddr.ip4.as_u32 = ste->out_r_addr.as_u32;
1286 vec_foreach (db, nm->db)
1288 bibe = nat64_db_bib_entry_find (db, &daddr, dport, ctx->proto, 0, 0);
1298 ip6->dst_address.as_u64[0] = bibe->in_addr.as_u64[0];
1299 ip6->dst_address.as_u64[1] = bibe->in_addr.as_u64[1];
1301 if (ctx->first_frag)
1303 udp->dst_port = bibe->in_port;
1304 udp->src_port = sport;
1305 csum = ip_csum_add_even (csum, ip6->src_address.as_u64[0]);
1306 csum = ip_csum_add_even (csum, ip6->src_address.as_u64[1]);
1307 csum = ip_csum_add_even (csum, ip6->dst_address.as_u64[0]);
1308 csum = ip_csum_add_even (csum, ip6->dst_address.as_u64[1]);
1309 csum = ip_csum_add_even (csum, udp->src_port);
1310 csum = ip_csum_add_even (csum, udp->dst_port);
1311 *checksum = ip_csum_fold (csum);
1318 nat64_in2out_reass_node_fn (vlib_main_t * vm,
1319 vlib_node_runtime_t * node, vlib_frame_t * frame)
1321 u32 n_left_from, *from, *to_next;
1322 nat64_in2out_next_t next_index;
1323 u32 pkts_processed = 0;
1324 u32 *fragments_to_drop = 0;
1325 u32 *fragments_to_loopback = 0;
1326 nat64_main_t *nm = &nat64_main;
1327 u32 thread_index = vm->thread_index;
1329 from = vlib_frame_vector_args (frame);
1330 n_left_from = frame->n_vectors;
1331 next_index = node->cached_next_index;
1333 while (n_left_from > 0)
1337 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1339 while (n_left_from > 0 && n_left_to_next > 0)
1346 u16 l4_offset0, frag_offset0;
1348 nat_reass_ip6_t *reass0;
1349 ip6_frag_hdr_t *frag0;
1350 nat64_db_bib_entry_t *bibe0;
1351 nat64_db_st_entry_t *ste0;
1353 snat_protocol_t proto0;
1354 u32 sw_if_index0, fib_index0;
1355 ip46_address_t saddr0, daddr0;
1356 nat64_in2out_frag_set_ctx_t ctx0;
1357 nat64_db_t *db = &nm->db[thread_index];
1359 /* speculatively enqueue b0 to the current next frame */
1365 n_left_to_next -= 1;
1367 b0 = vlib_get_buffer (vm, bi0);
1368 next0 = NAT64_IN2OUT_NEXT_IP4_LOOKUP;
1370 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1372 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6,
1375 ctx0.thread_index = thread_index;
1377 if (PREDICT_FALSE (nat_reass_is_drop_frag (1)))
1379 next0 = NAT64_IN2OUT_NEXT_DROP;
1380 b0->error = node->errors[NAT64_IN2OUT_ERROR_DROP_FRAGMENT];
1384 ip60 = (ip6_header_t *) vlib_buffer_get_current (b0);
1388 (ip60, b0->current_length, &l4_protocol0, &l4_offset0,
1391 next0 = NAT64_IN2OUT_NEXT_DROP;
1392 b0->error = node->errors[NAT64_IN2OUT_ERROR_UNKNOWN];
1397 (!(l4_protocol0 == IP_PROTOCOL_TCP
1398 || l4_protocol0 == IP_PROTOCOL_UDP)))
1400 next0 = NAT64_IN2OUT_NEXT_DROP;
1401 b0->error = node->errors[NAT64_IN2OUT_ERROR_DROP_FRAGMENT];
1405 udp0 = (udp_header_t *) u8_ptr_add (ip60, l4_offset0);
1406 frag0 = (ip6_frag_hdr_t *) u8_ptr_add (ip60, frag_offset0);
1407 proto0 = ip_proto_to_snat_proto (l4_protocol0);
1409 reass0 = nat_ip6_reass_find_or_create (ip60->src_address,
1411 frag0->identification,
1413 1, &fragments_to_drop);
1415 if (PREDICT_FALSE (!reass0))
1417 next0 = NAT64_IN2OUT_NEXT_DROP;
1418 b0->error = node->errors[NAT64_IN2OUT_ERROR_MAX_REASS];
1422 if (PREDICT_TRUE (ip6_frag_hdr_offset (frag0)))
1424 ctx0.first_frag = 0;
1425 if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1427 if (nat_ip6_reass_add_fragment (reass0, bi0))
1429 b0->error = node->errors[NAT64_IN2OUT_ERROR_MAX_FRAG];
1430 next0 = NAT64_IN2OUT_NEXT_DROP;
1439 ctx0.first_frag = 1;
1441 saddr0.as_u64[0] = ip60->src_address.as_u64[0];
1442 saddr0.as_u64[1] = ip60->src_address.as_u64[1];
1443 daddr0.as_u64[0] = ip60->dst_address.as_u64[0];
1444 daddr0.as_u64[1] = ip60->dst_address.as_u64[1];
1447 nat64_db_st_entry_find (db, &saddr0, &daddr0,
1448 udp0->src_port, udp0->dst_port,
1449 l4_protocol0, fib_index0, 1);
1453 nat64_db_bib_entry_find (db, &saddr0, udp0->src_port,
1454 l4_protocol0, fib_index0, 1);
1458 ip4_address_t out_addr0;
1459 if (nat64_alloc_out_addr_and_port
1460 (fib_index0, proto0, &out_addr0, &out_port0,
1463 next0 = NAT64_IN2OUT_NEXT_DROP;
1465 node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
1470 nat64_db_bib_entry_create (db,
1472 &out_addr0, udp0->src_port,
1473 out_port0, fib_index0,
1477 next0 = NAT64_IN2OUT_NEXT_DROP;
1479 node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
1483 nat64_extract_ip4 (&ip60->dst_address, &daddr0.ip4,
1486 nat64_db_st_entry_create (db, bibe0,
1487 &ip60->dst_address, &daddr0.ip4,
1491 next0 = NAT64_IN2OUT_NEXT_DROP;
1493 node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
1497 reass0->sess_index = nat64_db_st_entry_get_index (db, ste0);
1499 nat_ip6_reass_get_frags (reass0, &fragments_to_loopback);
1502 ctx0.sess_index = reass0->sess_index;
1503 ctx0.proto = l4_protocol0;
1505 ctx0.l4_offset = l4_offset0;
1507 if (PREDICT_FALSE (is_hairpinning (&ip60->dst_address)))
1509 next0 = NAT64_IN2OUT_NEXT_IP6_LOOKUP;
1510 if (nat64_in2out_frag_hairpinning (b0, ip60, &ctx0))
1512 next0 = NAT64_IN2OUT_NEXT_DROP;
1513 b0->error = node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
1519 if (ip6_to_ip4_fragmented (b0, nat64_in2out_frag_set_cb, &ctx0))
1521 next0 = NAT64_IN2OUT_NEXT_DROP;
1522 b0->error = node->errors[NAT64_IN2OUT_ERROR_UNKNOWN];
1529 ((node->flags & VLIB_NODE_FLAG_TRACE)
1530 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1532 nat64_in2out_reass_trace_t *t =
1533 vlib_add_trace (vm, node, b0, sizeof (*t));
1534 t->cached = cached0;
1535 t->sw_if_index = sw_if_index0;
1536 t->next_index = next0;
1546 pkts_processed += next0 != NAT64_IN2OUT_NEXT_DROP;
1548 /* verify speculative enqueue, maybe switch current next frame */
1549 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1550 to_next, n_left_to_next,
1554 if (n_left_from == 0 && vec_len (fragments_to_loopback))
1556 from = vlib_frame_vector_args (frame);
1557 u32 len = vec_len (fragments_to_loopback);
1558 if (len <= VLIB_FRAME_SIZE)
1560 clib_memcpy (from, fragments_to_loopback,
1561 sizeof (u32) * len);
1563 vec_reset_length (fragments_to_loopback);
1568 fragments_to_loopback + (len -
1570 sizeof (u32) * VLIB_FRAME_SIZE);
1571 n_left_from = VLIB_FRAME_SIZE;
1572 _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1577 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1580 vlib_node_increment_counter (vm, nat64_in2out_reass_node.index,
1581 NAT64_IN2OUT_ERROR_IN2OUT_PACKETS,
1584 nat_send_all_to_node (vm, fragments_to_drop, node,
1585 &node->errors[NAT64_IN2OUT_ERROR_DROP_FRAGMENT],
1586 NAT64_IN2OUT_NEXT_DROP);
1588 vec_free (fragments_to_drop);
1589 vec_free (fragments_to_loopback);
1590 return frame->n_vectors;
1594 VLIB_REGISTER_NODE (nat64_in2out_reass_node) = {
1595 .function = nat64_in2out_reass_node_fn,
1596 .name = "nat64-in2out-reass",
1597 .vector_size = sizeof (u32),
1598 .format_trace = format_nat64_in2out_reass_trace,
1599 .type = VLIB_NODE_TYPE_INTERNAL,
1600 .n_errors = ARRAY_LEN (nat64_in2out_error_strings),
1601 .error_strings = nat64_in2out_error_strings,
1602 .n_next_nodes = NAT64_IN2OUT_N_NEXT,
1603 /* edit / add dispositions here */
1605 [NAT64_IN2OUT_NEXT_DROP] = "error-drop",
1606 [NAT64_IN2OUT_NEXT_IP4_LOOKUP] = "ip4-lookup",
1607 [NAT64_IN2OUT_NEXT_IP6_LOOKUP] = "ip6-lookup",
1608 [NAT64_IN2OUT_NEXT_SLOWPATH] = "nat64-in2out-slowpath",
1609 [NAT64_IN2OUT_NEXT_REASS] = "nat64-in2out-reass",
1614 VLIB_NODE_FUNCTION_MULTIARCH (nat64_in2out_reass_node,
1615 nat64_in2out_reass_node_fn);
1619 u32 next_worker_index;
1621 } nat64_in2out_handoff_trace_t;
1624 format_nat64_in2out_handoff_trace (u8 * s, va_list * args)
1626 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1627 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1628 nat64_in2out_handoff_trace_t *t =
1629 va_arg (*args, nat64_in2out_handoff_trace_t *);
1632 m = t->do_handoff ? "next worker" : "same worker";
1633 s = format (s, "NAT64-IN2OUT-HANDOFF: %s %d", m, t->next_worker_index);
1639 nat64_in2out_handoff_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
1640 vlib_frame_t * frame)
1642 nat64_main_t *nm = &nat64_main;
1643 vlib_thread_main_t *tm = vlib_get_thread_main ();
1644 u32 n_left_from, *from, *to_next = 0, *to_next_drop = 0;
1645 static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
1646 static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
1648 vlib_frame_queue_elt_t *hf = 0;
1649 vlib_frame_queue_t *fq;
1650 vlib_frame_t *f = 0, *d = 0;
1652 u32 n_left_to_next_worker = 0, *to_next_worker = 0;
1653 u32 next_worker_index = 0;
1654 u32 current_worker_index = ~0;
1655 u32 thread_index = vm->thread_index;
1659 fq_index = nm->fq_in2out_index;
1660 to_node_index = nat64_in2out_node.index;
1662 if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
1664 vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
1666 vec_validate_init_empty (congested_handoff_queue_by_worker_index,
1667 tm->n_vlib_mains - 1,
1668 (vlib_frame_queue_t *) (~0));
1671 from = vlib_frame_vector_args (frame);
1672 n_left_from = frame->n_vectors;
1674 while (n_left_from > 0)
1685 b0 = vlib_get_buffer (vm, bi0);
1687 ip0 = vlib_buffer_get_current (b0);
1689 next_worker_index = nat64_get_worker_in2out (&ip0->src_address);
1691 if (PREDICT_FALSE (next_worker_index != thread_index))
1695 if (next_worker_index != current_worker_index)
1698 is_vlib_frame_queue_congested (fq_index, next_worker_index,
1700 congested_handoff_queue_by_worker_index);
1704 /* if this is 1st frame */
1707 d = vlib_get_frame_to_node (vm, nm->error_node_index);
1708 to_next_drop = vlib_frame_vector_args (d);
1711 to_next_drop[0] = bi0;
1718 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1721 vlib_get_worker_handoff_queue_elt (fq_index,
1723 handoff_queue_elt_by_worker_index);
1724 n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
1725 to_next_worker = &hf->buffer_index[hf->n_vectors];
1726 current_worker_index = next_worker_index;
1729 ASSERT (to_next_worker != 0);
1731 /* enqueue to correct worker thread */
1732 to_next_worker[0] = bi0;
1734 n_left_to_next_worker--;
1736 if (n_left_to_next_worker == 0)
1738 hf->n_vectors = VLIB_FRAME_SIZE;
1739 vlib_put_frame_queue_elt (hf);
1740 current_worker_index = ~0;
1741 handoff_queue_elt_by_worker_index[next_worker_index] = 0;
1748 /* if this is 1st frame */
1751 f = vlib_get_frame_to_node (vm, to_node_index);
1752 to_next = vlib_frame_vector_args (f);
1762 ((node->flags & VLIB_NODE_FLAG_TRACE)
1763 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1765 nat64_in2out_handoff_trace_t *t =
1766 vlib_add_trace (vm, node, b0, sizeof (*t));
1767 t->next_worker_index = next_worker_index;
1768 t->do_handoff = do_handoff;
1773 vlib_put_frame_to_node (vm, to_node_index, f);
1776 vlib_put_frame_to_node (vm, nm->error_node_index, d);
1779 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1781 /* Ship frames to the worker nodes */
1782 for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
1784 if (handoff_queue_elt_by_worker_index[i])
1786 hf = handoff_queue_elt_by_worker_index[i];
1788 * It works better to let the handoff node
1789 * rate-adapt, always ship the handoff queue element.
1791 if (1 || hf->n_vectors == hf->last_n_vectors)
1793 vlib_put_frame_queue_elt (hf);
1794 handoff_queue_elt_by_worker_index[i] = 0;
1797 hf->last_n_vectors = hf->n_vectors;
1799 congested_handoff_queue_by_worker_index[i] =
1800 (vlib_frame_queue_t *) (~0);
1803 current_worker_index = ~0;
1804 return frame->n_vectors;
1808 VLIB_REGISTER_NODE (nat64_in2out_handoff_node) = {
1809 .function = nat64_in2out_handoff_node_fn,
1810 .name = "nat64-in2out-handoff",
1811 .vector_size = sizeof (u32),
1812 .format_trace = format_nat64_in2out_handoff_trace,
1813 .type = VLIB_NODE_TYPE_INTERNAL,
1823 VLIB_NODE_FUNCTION_MULTIARCH (nat64_in2out_handoff_node,
1824 nat64_in2out_handoff_node_fn);
1827 * fd.io coding-style-patch-verification: ON
1830 * eval: (c-set-style "gnu")