src/plugins/nat/nat64_in2out.c

   1 /*
   2  * Copyright (c) 2017 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT64 IPv6 to IPv4 translation (inside to outside network)
  18  */
  19
  20 #include <nat/nat64.h>
  21 #include <nat/nat_reass.h>
  22 #include <nat/nat_inlines.h>
  23 #include <vnet/ip/ip6_to_ip4.h>
  24 #include <vnet/fib/fib_table.h>
  25
  26 typedef struct
  27 {
  28   u32 sw_if_index;
  29   u32 next_index;
  30   u8 is_slow_path;
  31 } nat64_in2out_trace_t;
  32
  33 static u8 *
  34 format_nat64_in2out_trace (u8 * s, va_list * args)
  35 {
  36   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  37   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  38   nat64_in2out_trace_t *t = va_arg (*args, nat64_in2out_trace_t *);
  39   char *tag;
  40
  41   tag = t->is_slow_path ? "NAT64-in2out-slowpath" : "NAT64-in2out";
  42
  43   s =
  44     format (s, "%s: sw_if_index %d, next index %d", tag, t->sw_if_index,
  45             t->next_index);
  46
  47   return s;
  48 }
  49
  50 typedef struct
  51 {
  52   u32 sw_if_index;
  53   u32 next_index;
  54   u8 cached;
  55 } nat64_in2out_reass_trace_t;
  56
  57 static u8 *
  58 format_nat64_in2out_reass_trace (u8 * s, va_list * args)
  59 {
  60   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  61   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  62   nat64_in2out_reass_trace_t *t =
  63     va_arg (*args, nat64_in2out_reass_trace_t *);
  64
  65   s =
  66     format (s, "NAT64-in2out-reass: sw_if_index %d, next index %d, status %s",
  67             t->sw_if_index, t->next_index,
  68             t->cached ? "cached" : "translated");
  69
  70   return s;
  71 }
  72
  73 vlib_node_registration_t nat64_in2out_node;
  74 vlib_node_registration_t nat64_in2out_slowpath_node;
  75 vlib_node_registration_t nat64_in2out_reass_node;
  76 vlib_node_registration_t nat64_in2out_handoff_node;
  77
  78 #define foreach_nat64_in2out_error                       \
  79 _(UNSUPPORTED_PROTOCOL, "unsupported protocol")          \
  80 _(IN2OUT_PACKETS, "good in2out packets processed")       \
  81 _(NO_TRANSLATION, "no translation")                      \
  82 _(UNKNOWN, "unknown")                                    \
  83 _(DROP_FRAGMENT, "drop fragment")                        \
  84 _(MAX_REASS, "maximum reassemblies exceeded")            \
  85 _(MAX_FRAG, "maximum fragments per reassembly exceeded") \
  86 _(TCP_PACKETS, "TCP packets")                            \
  87 _(UDP_PACKETS, "UDP packets")                            \
  88 _(ICMP_PACKETS, "ICMP packets")                          \
  89 _(OTHER_PACKETS, "other protocol packets")               \
  90 _(FRAGMENTS, "fragments")                                \
  91 _(CACHED_FRAGMENTS, "cached fragments")                  \
  92 _(PROCESSED_FRAGMENTS, "processed fragments")
  93
  94
  95 typedef enum
  96 {
  97 #define _(sym,str) NAT64_IN2OUT_ERROR_##sym,
  98   foreach_nat64_in2out_error
  99 #undef _
 100     NAT64_IN2OUT_N_ERROR,
 101 } nat64_in2out_error_t;
 102
 103 static char *nat64_in2out_error_strings[] = {
 104 #define _(sym,string) string,
 105   foreach_nat64_in2out_error
 106 #undef _
 107 };
 108
 109 typedef enum
 110 {
 111   NAT64_IN2OUT_NEXT_IP4_LOOKUP,
 112   NAT64_IN2OUT_NEXT_IP6_LOOKUP,
 113   NAT64_IN2OUT_NEXT_DROP,
 114   NAT64_IN2OUT_NEXT_SLOWPATH,
 115   NAT64_IN2OUT_NEXT_REASS,
 116   NAT64_IN2OUT_N_NEXT,
 117 } nat64_in2out_next_t;
 118
 119 typedef struct nat64_in2out_set_ctx_t_
 120 {
 121   vlib_buffer_t *b;
 122   vlib_main_t *vm;
 123   u32 thread_index;
 124 } nat64_in2out_set_ctx_t;
 125
 126 static inline u8
 127 nat64_not_translate (u32 sw_if_index, ip6_address_t ip6_addr)
 128 {
 129   ip6_address_t *addr;
 130   ip6_main_t *im6 = &ip6_main;
 131   ip_lookup_main_t *lm6 = &im6->lookup_main;
 132   ip_interface_address_t *ia = 0;
 133
 134   /* *INDENT-OFF* */
 135   foreach_ip_interface_address (lm6, ia, sw_if_index, 0,
 136   ({
 137         addr = ip_interface_address_get_address (lm6, ia);
 138         if (0 == ip6_address_compare (addr, &ip6_addr))
 139                 return 1;
 140   }));
 141   /* *INDENT-ON* */
 142
 143   return 0;
 144 }
 145
 146 /**
 147  * @brief Check whether is a hairpinning.
 148  *
 149  * If the destination IP address of the packet is an IPv4 address assigned to
 150  * the NAT64 itself, then the packet is a hairpin packet.
 151  *
 152  * param dst_addr Destination address of the packet.
 153  *
 154  * @returns 1 if hairpinning, otherwise 0.
 155  */
 156 static_always_inline int
 157 is_hairpinning (ip6_address_t * dst_addr)
 158 {
 159   nat64_main_t *nm = &nat64_main;
 160   int i;
 161
 162   for (i = 0; i < vec_len (nm->addr_pool); i++)
 163     {
 164       if (nm->addr_pool[i].addr.as_u32 == dst_addr->as_u32[3])
 165         return 1;
 166     }
 167
 168   return 0;
 169 }
 170
 171 static int
 172 nat64_in2out_tcp_udp_set_cb (ip6_header_t * ip6, ip4_header_t * ip4,
 173                              void *arg)
 174 {
 175   nat64_main_t *nm = &nat64_main;
 176   nat64_in2out_set_ctx_t *ctx = arg;
 177   nat64_db_bib_entry_t *bibe;
 178   nat64_db_st_entry_t *ste;
 179   ip46_address_t saddr, daddr;
 180   u32 sw_if_index, fib_index;
 181   udp_header_t *udp = ip6_next_header (ip6);
 182   u8 proto = ip6->protocol;
 183   u16 sport = udp->src_port;
 184   u16 dport = udp->dst_port;
 185   nat64_db_t *db = &nm->db[ctx->thread_index];
 186
 187   sw_if_index = vnet_buffer (ctx->b)->sw_if_index[VLIB_RX];
 188   fib_index =
 189     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
 190
 191   saddr.as_u64[0] = ip6->src_address.as_u64[0];
 192   saddr.as_u64[1] = ip6->src_address.as_u64[1];
 193   daddr.as_u64[0] = ip6->dst_address.as_u64[0];
 194   daddr.as_u64[1] = ip6->dst_address.as_u64[1];
 195
 196   ste =
 197     nat64_db_st_entry_find (db, &saddr, &daddr, sport, dport, proto,
 198                             fib_index, 1);
 199
 200   if (ste)
 201     {
 202       bibe = nat64_db_bib_entry_by_index (db, proto, ste->bibe_index);
 203       if (!bibe)
 204         return -1;
 205     }
 206   else
 207     {
 208       bibe = nat64_db_bib_entry_find (db, &saddr, sport, proto, fib_index, 1);
 209
 210       if (!bibe)
 211         {
 212           u16 out_port;
 213           ip4_address_t out_addr;
 214           if (nat64_alloc_out_addr_and_port
 215               (fib_index, ip_proto_to_snat_proto (proto), &out_addr,
 216                &out_port, ctx->thread_index))
 217             return -1;
 218
 219           bibe =
 220             nat64_db_bib_entry_create (ctx->thread_index, db,
 221                                        &ip6->src_address, &out_addr, sport,
 222                                        out_port, fib_index, proto, 0);
 223           if (!bibe)
 224             return -1;
 225
 226           vlib_set_simple_counter (&nm->total_bibs, ctx->thread_index, 0,
 227                                    db->bib.bib_entries_num);
 228         }
 229
 230       nat64_extract_ip4 (&ip6->dst_address, &daddr.ip4, fib_index);
 231       ste =
 232         nat64_db_st_entry_create (ctx->thread_index, db, bibe,
 233                                   &ip6->dst_address, &daddr.ip4, dport);
 234       if (!ste)
 235         return -1;
 236
 237       vlib_set_simple_counter (&nm->total_sessions, ctx->thread_index, 0,
 238                                db->st.st_entries_num);
 239     }
 240
 241   ip4->src_address.as_u32 = bibe->out_addr.as_u32;
 242   udp->src_port = bibe->out_port;
 243
 244   ip4->dst_address.as_u32 = ste->out_r_addr.as_u32;
 245
 246   if (proto == IP_PROTOCOL_TCP)
 247     {
 248       u16 *checksum;
 249       ip_csum_t csum;
 250       tcp_header_t *tcp = ip6_next_header (ip6);
 251
 252       nat64_tcp_session_set_state (ste, tcp, 1);
 253       checksum = &tcp->checksum;
 254       csum = ip_csum_sub_even (*checksum, sport);
 255       csum = ip_csum_add_even (csum, udp->src_port);
 256       mss_clamping (nm->sm, tcp, &csum);
 257       *checksum = ip_csum_fold (csum);
 258     }
 259
 260   nat64_session_reset_timeout (ste, ctx->vm);
 261
 262   return 0;
 263 }
 264
 265 static int
 266 nat64_in2out_icmp_set_cb (ip6_header_t * ip6, ip4_header_t * ip4, void *arg)
 267 {
 268   nat64_main_t *nm = &nat64_main;
 269   nat64_in2out_set_ctx_t *ctx = arg;
 270   nat64_db_bib_entry_t *bibe;
 271   nat64_db_st_entry_t *ste;
 272   ip46_address_t saddr, daddr;
 273   u32 sw_if_index, fib_index;
 274   icmp46_header_t *icmp = ip6_next_header (ip6);
 275   nat64_db_t *db = &nm->db[ctx->thread_index];
 276
 277   sw_if_index = vnet_buffer (ctx->b)->sw_if_index[VLIB_RX];
 278   fib_index =
 279     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
 280
 281   saddr.as_u64[0] = ip6->src_address.as_u64[0];
 282   saddr.as_u64[1] = ip6->src_address.as_u64[1];
 283   daddr.as_u64[0] = ip6->dst_address.as_u64[0];
 284   daddr.as_u64[1] = ip6->dst_address.as_u64[1];
 285
 286   if (icmp->type == ICMP4_echo_request || icmp->type == ICMP4_echo_reply)
 287     {
 288       u16 in_id = ((u16 *) (icmp))[2];
 289       ste =
 290         nat64_db_st_entry_find (db, &saddr, &daddr, in_id, 0,
 291                                 IP_PROTOCOL_ICMP, fib_index, 1);
 292
 293       if (ste)
 294         {
 295           bibe =
 296             nat64_db_bib_entry_by_index (db, IP_PROTOCOL_ICMP,
 297                                          ste->bibe_index);
 298           if (!bibe)
 299             return -1;
 300         }
 301       else
 302         {
 303           bibe =
 304             nat64_db_bib_entry_find (db, &saddr, in_id,
 305                                      IP_PROTOCOL_ICMP, fib_index, 1);
 306
 307           if (!bibe)
 308             {
 309               u16 out_id;
 310               ip4_address_t out_addr;
 311               if (nat64_alloc_out_addr_and_port
 312                   (fib_index, SNAT_PROTOCOL_ICMP, &out_addr, &out_id,
 313                    ctx->thread_index))
 314                 return -1;
 315
 316               bibe =
 317                 nat64_db_bib_entry_create (ctx->thread_index, db,
 318                                            &ip6->src_address, &out_addr,
 319                                            in_id, out_id, fib_index,
 320                                            IP_PROTOCOL_ICMP, 0);
 321               if (!bibe)
 322                 return -1;
 323
 324               vlib_set_simple_counter (&nm->total_bibs, ctx->thread_index, 0,
 325                                        db->bib.bib_entries_num);
 326             }
 327
 328           nat64_extract_ip4 (&ip6->dst_address, &daddr.ip4, fib_index);
 329           ste =
 330             nat64_db_st_entry_create (ctx->thread_index, db, bibe,
 331                                       &ip6->dst_address, &daddr.ip4, 0);
 332           if (!ste)
 333             return -1;
 334
 335           vlib_set_simple_counter (&nm->total_sessions, ctx->thread_index, 0,
 336                                    db->st.st_entries_num);
 337         }
 338
 339       nat64_session_reset_timeout (ste, ctx->vm);
 340
 341       ip4->src_address.as_u32 = bibe->out_addr.as_u32;
 342       ((u16 *) (icmp))[2] = bibe->out_port;
 343
 344       ip4->dst_address.as_u32 = ste->out_r_addr.as_u32;
 345     }
 346   else
 347     {
 348       if (!vec_len (nm->addr_pool))
 349         return -1;
 350
 351       ip4->src_address.as_u32 = nm->addr_pool[0].addr.as_u32;
 352       nat64_extract_ip4 (&ip6->dst_address, &ip4->dst_address, fib_index);
 353     }
 354
 355   return 0;
 356 }
 357
 358 static int
 359 nat64_in2out_inner_icmp_set_cb (ip6_header_t * ip6, ip4_header_t * ip4,
 360                                 void *arg)
 361 {
 362   nat64_main_t *nm = &nat64_main;
 363   nat64_in2out_set_ctx_t *ctx = arg;
 364   nat64_db_st_entry_t *ste;
 365   nat64_db_bib_entry_t *bibe;
 366   ip46_address_t saddr, daddr;
 367   u32 sw_if_index, fib_index;
 368   u8 proto = ip6->protocol;
 369   nat64_db_t *db = &nm->db[ctx->thread_index];
 370
 371   sw_if_index = vnet_buffer (ctx->b)->sw_if_index[VLIB_RX];
 372   fib_index =
 373     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
 374
 375   saddr.as_u64[0] = ip6->src_address.as_u64[0];
 376   saddr.as_u64[1] = ip6->src_address.as_u64[1];
 377   daddr.as_u64[0] = ip6->dst_address.as_u64[0];
 378   daddr.as_u64[1] = ip6->dst_address.as_u64[1];
 379
 380   if (proto == IP_PROTOCOL_ICMP6)
 381     {
 382       icmp46_header_t *icmp = ip6_next_header (ip6);
 383       u16 in_id = ((u16 *) (icmp))[2];
 384       proto = IP_PROTOCOL_ICMP;
 385
 386       if (!
 387           (icmp->type == ICMP4_echo_request
 388            || icmp->type == ICMP4_echo_reply))
 389         return -1;
 390
 391       ste =
 392         nat64_db_st_entry_find (db, &daddr, &saddr, in_id, 0, proto,
 393                                 fib_index, 1);
 394       if (!ste)
 395         return -1;
 396
 397       bibe = nat64_db_bib_entry_by_index (db, proto, ste->bibe_index);
 398       if (!bibe)
 399         return -1;
 400
 401       ip4->dst_address.as_u32 = bibe->out_addr.as_u32;
 402       ((u16 *) (icmp))[2] = bibe->out_port;
 403       ip4->src_address.as_u32 = ste->out_r_addr.as_u32;
 404     }
 405   else
 406     {
 407       udp_header_t *udp = ip6_next_header (ip6);
 408       tcp_header_t *tcp = ip6_next_header (ip6);
 409       u16 *checksum;
 410       ip_csum_t csum;
 411
 412       u16 sport = udp->src_port;
 413       u16 dport = udp->dst_port;
 414
 415       ste =
 416         nat64_db_st_entry_find (db, &daddr, &saddr, dport, sport, proto,
 417                                 fib_index, 1);
 418       if (!ste)
 419         return -1;
 420
 421       bibe = nat64_db_bib_entry_by_index (db, proto, ste->bibe_index);
 422       if (!bibe)
 423         return -1;
 424
 425       ip4->dst_address.as_u32 = bibe->out_addr.as_u32;
 426       udp->dst_port = bibe->out_port;
 427       ip4->src_address.as_u32 = ste->out_r_addr.as_u32;
 428
 429       if (proto == IP_PROTOCOL_TCP)
 430         checksum = &tcp->checksum;
 431       else
 432         checksum = &udp->checksum;
 433       csum = ip_csum_sub_even (*checksum, dport);
 434       csum = ip_csum_add_even (csum, udp->dst_port);
 435       *checksum = ip_csum_fold (csum);
 436     }
 437
 438   return 0;
 439 }
 440
 441 typedef struct unk_proto_st_walk_ctx_t_
 442 {
 443   ip6_address_t src_addr;
 444   ip6_address_t dst_addr;
 445   ip4_address_t out_addr;
 446   u32 fib_index;
 447   u32 thread_index;
 448   u8 proto;
 449 } unk_proto_st_walk_ctx_t;
 450
 451 static int
 452 unk_proto_st_walk (nat64_db_st_entry_t * ste, void *arg)
 453 {
 454   nat64_main_t *nm = &nat64_main;
 455   unk_proto_st_walk_ctx_t *ctx = arg;
 456   nat64_db_bib_entry_t *bibe;
 457   ip46_address_t saddr, daddr;
 458   nat64_db_t *db = &nm->db[ctx->thread_index];
 459
 460   if (ip46_address_is_equal (&ste->in_r_addr, &ctx->dst_addr))
 461     {
 462       bibe = nat64_db_bib_entry_by_index (db, ste->proto, ste->bibe_index);
 463       if (!bibe)
 464         return -1;
 465
 466       if (ip46_address_is_equal (&bibe->in_addr, &ctx->src_addr)
 467           && bibe->fib_index == ctx->fib_index)
 468         {
 469           clib_memset (&saddr, 0, sizeof (saddr));
 470           saddr.ip4.as_u32 = bibe->out_addr.as_u32;
 471           clib_memset (&daddr, 0, sizeof (daddr));
 472           nat64_extract_ip4 (&ctx->dst_addr, &daddr.ip4, ctx->fib_index);
 473
 474           if (nat64_db_st_entry_find
 475               (db, &daddr, &saddr, 0, 0, ctx->proto, ctx->fib_index, 0))
 476             return -1;
 477
 478           ctx->out_addr.as_u32 = bibe->out_addr.as_u32;
 479           return 1;
 480         }
 481     }
 482
 483   return 0;
 484 }
 485
 486 static int
 487 nat64_in2out_unk_proto_set_cb (ip6_header_t * ip6, ip4_header_t * ip4,
 488                                void *arg)
 489 {
 490   nat64_main_t *nm = &nat64_main;
 491   nat64_in2out_set_ctx_t *s_ctx = arg;
 492   nat64_db_bib_entry_t *bibe;
 493   nat64_db_st_entry_t *ste;
 494   ip46_address_t saddr, daddr, addr;
 495   u32 sw_if_index, fib_index;
 496   u8 proto = ip6->protocol;
 497   int i;
 498   nat64_db_t *db = &nm->db[s_ctx->thread_index];
 499
 500   sw_if_index = vnet_buffer (s_ctx->b)->sw_if_index[VLIB_RX];
 501   fib_index =
 502     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
 503
 504   saddr.as_u64[0] = ip6->src_address.as_u64[0];
 505   saddr.as_u64[1] = ip6->src_address.as_u64[1];
 506   daddr.as_u64[0] = ip6->dst_address.as_u64[0];
 507   daddr.as_u64[1] = ip6->dst_address.as_u64[1];
 508
 509   ste =
 510     nat64_db_st_entry_find (db, &saddr, &daddr, 0, 0, proto, fib_index, 1);
 511
 512   if (ste)
 513     {
 514       bibe = nat64_db_bib_entry_by_index (db, proto, ste->bibe_index);
 515       if (!bibe)
 516         return -1;
 517     }
 518   else
 519     {
 520       bibe = nat64_db_bib_entry_find (db, &saddr, 0, proto, fib_index, 1);
 521
 522       if (!bibe)
 523         {
 524           /* Choose same out address as for TCP/UDP session to same dst */
 525           unk_proto_st_walk_ctx_t ctx = {
 526             .src_addr.as_u64[0] = ip6->src_address.as_u64[0],
 527             .src_addr.as_u64[1] = ip6->src_address.as_u64[1],
 528             .dst_addr.as_u64[0] = ip6->dst_address.as_u64[0],
 529             .dst_addr.as_u64[1] = ip6->dst_address.as_u64[1],
 530             .out_addr.as_u32 = 0,
 531             .fib_index = fib_index,
 532             .proto = proto,
 533             .thread_index = s_ctx->thread_index,
 534           };
 535
 536           nat64_db_st_walk (db, IP_PROTOCOL_TCP, unk_proto_st_walk, &ctx);
 537
 538           if (!ctx.out_addr.as_u32)
 539             nat64_db_st_walk (db, IP_PROTOCOL_UDP, unk_proto_st_walk, &ctx);
 540
 541           /* Verify if out address is not already in use for protocol */
 542           clib_memset (&addr, 0, sizeof (addr));
 543           addr.ip4.as_u32 = ctx.out_addr.as_u32;
 544           if (nat64_db_bib_entry_find (db, &addr, 0, proto, 0, 0))
 545             ctx.out_addr.as_u32 = 0;
 546
 547           if (!ctx.out_addr.as_u32)
 548             {
 549               for (i = 0; i < vec_len (nm->addr_pool); i++)
 550                 {
 551                   addr.ip4.as_u32 = nm->addr_pool[i].addr.as_u32;
 552                   if (!nat64_db_bib_entry_find (db, &addr, 0, proto, 0, 0))
 553                     break;
 554                 }
 555             }
 556
 557           if (!ctx.out_addr.as_u32)
 558             return -1;
 559
 560           bibe =
 561             nat64_db_bib_entry_create (s_ctx->thread_index, db,
 562                                        &ip6->src_address, &ctx.out_addr,
 563                                        0, 0, fib_index, proto, 0);
 564           if (!bibe)
 565             return -1;
 566
 567           vlib_set_simple_counter (&nm->total_bibs, s_ctx->thread_index, 0,
 568                                    db->bib.bib_entries_num);
 569         }
 570
 571       nat64_extract_ip4 (&ip6->dst_address, &daddr.ip4, fib_index);
 572       ste =
 573         nat64_db_st_entry_create (s_ctx->thread_index, db, bibe,
 574                                   &ip6->dst_address, &daddr.ip4, 0);
 575       if (!ste)
 576         return -1;
 577
 578       vlib_set_simple_counter (&nm->total_sessions, s_ctx->thread_index, 0,
 579                                db->st.st_entries_num);
 580     }
 581
 582   nat64_session_reset_timeout (ste, s_ctx->vm);
 583
 584   ip4->src_address.as_u32 = bibe->out_addr.as_u32;
 585   ip4->dst_address.as_u32 = ste->out_r_addr.as_u32;
 586
 587   return 0;
 588 }
 589
 590
 591
 592 static int
 593 nat64_in2out_tcp_udp_hairpinning (vlib_main_t * vm, vlib_buffer_t * b,
 594                                   ip6_header_t * ip6, u32 thread_index)
 595 {
 596   nat64_main_t *nm = &nat64_main;
 597   nat64_db_bib_entry_t *bibe;
 598   nat64_db_st_entry_t *ste;
 599   ip46_address_t saddr, daddr;
 600   u32 sw_if_index, fib_index;
 601   udp_header_t *udp = ip6_next_header (ip6);
 602   tcp_header_t *tcp = ip6_next_header (ip6);
 603   u8 proto = ip6->protocol;
 604   u16 sport = udp->src_port;
 605   u16 dport = udp->dst_port;
 606   u16 *checksum;
 607   ip_csum_t csum;
 608   nat64_db_t *db = &nm->db[thread_index];
 609
 610   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 611   fib_index =
 612     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
 613
 614   saddr.as_u64[0] = ip6->src_address.as_u64[0];
 615   saddr.as_u64[1] = ip6->src_address.as_u64[1];
 616   daddr.as_u64[0] = ip6->dst_address.as_u64[0];
 617   daddr.as_u64[1] = ip6->dst_address.as_u64[1];
 618
 619   if (proto == IP_PROTOCOL_UDP)
 620     checksum = &udp->checksum;
 621   else
 622     checksum = &tcp->checksum;
 623
 624   csum = ip_csum_sub_even (*checksum, ip6->src_address.as_u64[0]);
 625   csum = ip_csum_sub_even (csum, ip6->src_address.as_u64[1]);
 626   csum = ip_csum_sub_even (csum, ip6->dst_address.as_u64[0]);
 627   csum = ip_csum_sub_even (csum, ip6->dst_address.as_u64[1]);
 628   csum = ip_csum_sub_even (csum, sport);
 629   csum = ip_csum_sub_even (csum, dport);
 630
 631   ste =
 632     nat64_db_st_entry_find (db, &saddr, &daddr, sport, dport, proto,
 633                             fib_index, 1);
 634
 635   if (ste)
 636     {
 637       bibe = nat64_db_bib_entry_by_index (db, proto, ste->bibe_index);
 638       if (!bibe)
 639         return -1;
 640     }
 641   else
 642     {
 643       bibe = nat64_db_bib_entry_find (db, &saddr, sport, proto, fib_index, 1);
 644
 645       if (!bibe)
 646         {
 647           u16 out_port;
 648           ip4_address_t out_addr;
 649           if (nat64_alloc_out_addr_and_port
 650               (fib_index, ip_proto_to_snat_proto (proto), &out_addr,
 651                &out_port, thread_index))
 652             return -1;
 653
 654           bibe =
 655             nat64_db_bib_entry_create (thread_index, db, &ip6->src_address,
 656                                        &out_addr, sport, out_port, fib_index,
 657                                        proto, 0);
 658           if (!bibe)
 659             return -1;
 660
 661           vlib_set_simple_counter (&nm->total_bibs, thread_index, 0,
 662                                    db->bib.bib_entries_num);
 663         }
 664
 665       nat64_extract_ip4 (&ip6->dst_address, &daddr.ip4, fib_index);
 666       ste =
 667         nat64_db_st_entry_create (thread_index, db, bibe, &ip6->dst_address,
 668                                   &daddr.ip4, dport);
 669       if (!ste)
 670         return -1;
 671
 672       vlib_set_simple_counter (&nm->total_sessions, thread_index, 0,
 673                                db->st.st_entries_num);
 674     }
 675
 676   if (proto == IP_PROTOCOL_TCP)
 677     nat64_tcp_session_set_state (ste, tcp, 1);
 678
 679   nat64_session_reset_timeout (ste, vm);
 680
 681   sport = udp->src_port = bibe->out_port;
 682   nat64_compose_ip6 (&ip6->src_address, &bibe->out_addr, fib_index);
 683
 684   clib_memset (&daddr, 0, sizeof (daddr));
 685   daddr.ip4.as_u32 = ste->out_r_addr.as_u32;
 686
 687   bibe = 0;
 688   /* *INDENT-OFF* */
 689   vec_foreach (db, nm->db)
 690     {
 691       bibe = nat64_db_bib_entry_find (db, &daddr, dport, proto, 0, 0);
 692
 693       if (bibe)
 694         break;
 695     }
 696   /* *INDENT-ON* */
 697
 698   if (!bibe)
 699     return -1;
 700
 701   ip6->dst_address.as_u64[0] = bibe->in_addr.as_u64[0];
 702   ip6->dst_address.as_u64[1] = bibe->in_addr.as_u64[1];
 703   udp->dst_port = bibe->in_port;
 704
 705   csum = ip_csum_add_even (csum, ip6->src_address.as_u64[0]);
 706   csum = ip_csum_add_even (csum, ip6->src_address.as_u64[1]);
 707   csum = ip_csum_add_even (csum, ip6->dst_address.as_u64[0]);
 708   csum = ip_csum_add_even (csum, ip6->dst_address.as_u64[1]);
 709   csum = ip_csum_add_even (csum, udp->src_port);
 710   csum = ip_csum_add_even (csum, udp->dst_port);
 711   *checksum = ip_csum_fold (csum);
 712
 713   return 0;
 714 }
 715
 716 static int
 717 nat64_in2out_icmp_hairpinning (vlib_main_t * vm, vlib_buffer_t * b,
 718                                ip6_header_t * ip6, u32 thread_index)
 719 {
 720   nat64_main_t *nm = &nat64_main;
 721   nat64_db_bib_entry_t *bibe;
 722   nat64_db_st_entry_t *ste;
 723   icmp46_header_t *icmp = ip6_next_header (ip6);
 724   ip6_header_t *inner_ip6;
 725   ip46_address_t saddr, daddr;
 726   u32 sw_if_index, fib_index;
 727   u8 proto;
 728   udp_header_t *udp;
 729   tcp_header_t *tcp;
 730   u16 *checksum, sport, dport;
 731   ip_csum_t csum;
 732   nat64_db_t *db = &nm->db[thread_index];
 733
 734   if (icmp->type == ICMP6_echo_request || icmp->type == ICMP6_echo_reply)
 735     return -1;
 736
 737   inner_ip6 = (ip6_header_t *) u8_ptr_add (icmp, 8);
 738
 739   proto = inner_ip6->protocol;
 740
 741   if (proto == IP_PROTOCOL_ICMP6)
 742     return -1;
 743
 744   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 745   fib_index =
 746     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
 747
 748   saddr.as_u64[0] = inner_ip6->src_address.as_u64[0];
 749   saddr.as_u64[1] = inner_ip6->src_address.as_u64[1];
 750   daddr.as_u64[0] = inner_ip6->dst_address.as_u64[0];
 751   daddr.as_u64[1] = inner_ip6->dst_address.as_u64[1];
 752
 753   udp = ip6_next_header (inner_ip6);
 754   tcp = ip6_next_header (inner_ip6);
 755
 756   sport = udp->src_port;
 757   dport = udp->dst_port;
 758
 759   if (proto == IP_PROTOCOL_UDP)
 760     checksum = &udp->checksum;
 761   else
 762     checksum = &tcp->checksum;
 763
 764   csum = ip_csum_sub_even (*checksum, inner_ip6->src_address.as_u64[0]);
 765   csum = ip_csum_sub_even (csum, inner_ip6->src_address.as_u64[1]);
 766   csum = ip_csum_sub_even (csum, inner_ip6->dst_address.as_u64[0]);
 767   csum = ip_csum_sub_even (csum, inner_ip6->dst_address.as_u64[1]);
 768   csum = ip_csum_sub_even (csum, sport);
 769   csum = ip_csum_sub_even (csum, dport);
 770
 771   ste =
 772     nat64_db_st_entry_find (db, &daddr, &saddr, dport, sport, proto,
 773                             fib_index, 1);
 774   if (!ste)
 775     return -1;
 776
 777   bibe = nat64_db_bib_entry_by_index (db, proto, ste->bibe_index);
 778   if (!bibe)
 779     return -1;
 780
 781   dport = udp->dst_port = bibe->out_port;
 782   nat64_compose_ip6 (&inner_ip6->dst_address, &bibe->out_addr, fib_index);
 783
 784   clib_memset (&saddr, 0, sizeof (saddr));
 785   clib_memset (&daddr, 0, sizeof (daddr));
 786   saddr.ip4.as_u32 = ste->out_r_addr.as_u32;
 787   daddr.ip4.as_u32 = bibe->out_addr.as_u32;
 788
 789   ste = 0;
 790   /* *INDENT-OFF* */
 791   vec_foreach (db, nm->db)
 792     {
 793       ste = nat64_db_st_entry_find (db, &saddr, &daddr, sport, dport, proto,
 794                                     0, 0);
 795
 796       if (ste)
 797         break;
 798     }
 799   /* *INDENT-ON* */
 800
 801   if (!ste)
 802     return -1;
 803
 804   bibe = nat64_db_bib_entry_by_index (db, proto, ste->bibe_index);
 805   if (!bibe)
 806     return -1;
 807
 808   inner_ip6->src_address.as_u64[0] = bibe->in_addr.as_u64[0];
 809   inner_ip6->src_address.as_u64[1] = bibe->in_addr.as_u64[1];
 810   udp->src_port = bibe->in_port;
 811
 812   csum = ip_csum_add_even (csum, inner_ip6->src_address.as_u64[0]);
 813   csum = ip_csum_add_even (csum, inner_ip6->src_address.as_u64[1]);
 814   csum = ip_csum_add_even (csum, inner_ip6->dst_address.as_u64[0]);
 815   csum = ip_csum_add_even (csum, inner_ip6->dst_address.as_u64[1]);
 816   csum = ip_csum_add_even (csum, udp->src_port);
 817   csum = ip_csum_add_even (csum, udp->dst_port);
 818   *checksum = ip_csum_fold (csum);
 819
 820   if (!vec_len (nm->addr_pool))
 821     return -1;
 822
 823   nat64_compose_ip6 (&ip6->src_address, &nm->addr_pool[0].addr, fib_index);
 824   ip6->dst_address.as_u64[0] = inner_ip6->src_address.as_u64[0];
 825   ip6->dst_address.as_u64[1] = inner_ip6->src_address.as_u64[1];
 826
 827   icmp->checksum = 0;
 828   csum = ip_csum_with_carry (0, ip6->payload_length);
 829   csum = ip_csum_with_carry (csum, clib_host_to_net_u16 (ip6->protocol));
 830   csum = ip_csum_with_carry (csum, ip6->src_address.as_u64[0]);
 831   csum = ip_csum_with_carry (csum, ip6->src_address.as_u64[1]);
 832   csum = ip_csum_with_carry (csum, ip6->dst_address.as_u64[0]);
 833   csum = ip_csum_with_carry (csum, ip6->dst_address.as_u64[1]);
 834   csum =
 835     ip_incremental_checksum (csum, icmp,
 836                              clib_net_to_host_u16 (ip6->payload_length));
 837   icmp->checksum = ~ip_csum_fold (csum);
 838
 839   return 0;
 840 }
 841
 842 static int
 843 nat64_in2out_unk_proto_hairpinning (vlib_main_t * vm, vlib_buffer_t * b,
 844                                     ip6_header_t * ip6, u32 thread_index)
 845 {
 846   nat64_main_t *nm = &nat64_main;
 847   nat64_db_bib_entry_t *bibe;
 848   nat64_db_st_entry_t *ste;
 849   ip46_address_t saddr, daddr, addr;
 850   u32 sw_if_index, fib_index;
 851   u8 proto = ip6->protocol;
 852   int i;
 853   nat64_db_t *db = &nm->db[thread_index];
 854
 855   sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
 856   fib_index =
 857     fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6, sw_if_index);
 858
 859   saddr.as_u64[0] = ip6->src_address.as_u64[0];
 860   saddr.as_u64[1] = ip6->src_address.as_u64[1];
 861   daddr.as_u64[0] = ip6->dst_address.as_u64[0];
 862   daddr.as_u64[1] = ip6->dst_address.as_u64[1];
 863
 864   ste =
 865     nat64_db_st_entry_find (db, &saddr, &daddr, 0, 0, proto, fib_index, 1);
 866
 867   if (ste)
 868     {
 869       bibe = nat64_db_bib_entry_by_index (db, proto, ste->bibe_index);
 870       if (!bibe)
 871         return -1;
 872     }
 873   else
 874     {
 875       bibe = nat64_db_bib_entry_find (db, &saddr, 0, proto, fib_index, 1);
 876
 877       if (!bibe)
 878         {
 879           /* Choose same out address as for TCP/UDP session to same dst */
 880           unk_proto_st_walk_ctx_t ctx = {
 881             .src_addr.as_u64[0] = ip6->src_address.as_u64[0],
 882             .src_addr.as_u64[1] = ip6->src_address.as_u64[1],
 883             .dst_addr.as_u64[0] = ip6->dst_address.as_u64[0],
 884             .dst_addr.as_u64[1] = ip6->dst_address.as_u64[1],
 885             .out_addr.as_u32 = 0,
 886             .fib_index = fib_index,
 887             .proto = proto,
 888             .thread_index = thread_index,
 889           };
 890
 891           nat64_db_st_walk (db, IP_PROTOCOL_TCP, unk_proto_st_walk, &ctx);
 892
 893           if (!ctx.out_addr.as_u32)
 894             nat64_db_st_walk (db, IP_PROTOCOL_UDP, unk_proto_st_walk, &ctx);
 895
 896           /* Verify if out address is not already in use for protocol */
 897           clib_memset (&addr, 0, sizeof (addr));
 898           addr.ip4.as_u32 = ctx.out_addr.as_u32;
 899           if (nat64_db_bib_entry_find (db, &addr, 0, proto, 0, 0))
 900             ctx.out_addr.as_u32 = 0;
 901
 902           if (!ctx.out_addr.as_u32)
 903             {
 904               for (i = 0; i < vec_len (nm->addr_pool); i++)
 905                 {
 906                   addr.ip4.as_u32 = nm->addr_pool[i].addr.as_u32;
 907                   if (!nat64_db_bib_entry_find (db, &addr, 0, proto, 0, 0))
 908                     break;
 909                 }
 910             }
 911
 912           if (!ctx.out_addr.as_u32)
 913             return -1;
 914
 915           bibe =
 916             nat64_db_bib_entry_create (thread_index, db, &ip6->src_address,
 917                                        &ctx.out_addr, 0, 0, fib_index, proto,
 918                                        0);
 919           if (!bibe)
 920             return -1;
 921
 922           vlib_set_simple_counter (&nm->total_bibs, thread_index, 0,
 923                                    db->bib.bib_entries_num);
 924         }
 925
 926       nat64_extract_ip4 (&ip6->dst_address, &daddr.ip4, fib_index);
 927       ste =
 928         nat64_db_st_entry_create (thread_index, db, bibe, &ip6->dst_address,
 929                                   &daddr.ip4, 0);
 930       if (!ste)
 931         return -1;
 932
 933       vlib_set_simple_counter (&nm->total_sessions, thread_index, 0,
 934                                db->st.st_entries_num);
 935     }
 936
 937   nat64_session_reset_timeout (ste, vm);
 938
 939   nat64_compose_ip6 (&ip6->src_address, &bibe->out_addr, fib_index);
 940
 941   clib_memset (&daddr, 0, sizeof (daddr));
 942   daddr.ip4.as_u32 = ste->out_r_addr.as_u32;
 943
 944   bibe = 0;
 945   /* *INDENT-OFF* */
 946   vec_foreach (db, nm->db)
 947     {
 948       bibe = nat64_db_bib_entry_find (db, &daddr, 0, proto, 0, 0);
 949
 950       if (bibe)
 951         break;
 952     }
 953   /* *INDENT-ON* */
 954
 955   if (!bibe)
 956     return -1;
 957
 958   ip6->dst_address.as_u64[0] = bibe->in_addr.as_u64[0];
 959   ip6->dst_address.as_u64[1] = bibe->in_addr.as_u64[1];
 960
 961   return 0;
 962 }
 963
 964 static inline uword
 965 nat64_in2out_node_fn_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 966                              vlib_frame_t * frame, u8 is_slow_path)
 967 {
 968   u32 n_left_from, *from, *to_next;
 969   nat64_in2out_next_t next_index;
 970   u32 pkts_processed = 0;
 971   u32 stats_node_index;
 972   u32 thread_index = vm->thread_index;
 973   u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
 974     0, fragments = 0;
 975
 976   stats_node_index =
 977     is_slow_path ? nat64_in2out_slowpath_node.index : nat64_in2out_node.index;
 978
 979   from = vlib_frame_vector_args (frame);
 980   n_left_from = frame->n_vectors;
 981   next_index = node->cached_next_index;
 982
 983   while (n_left_from > 0)
 984     {
 985       u32 n_left_to_next;
 986
 987       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 988
 989       while (n_left_from > 0 && n_left_to_next > 0)
 990         {
 991           u32 bi0;
 992           vlib_buffer_t *b0;
 993           u32 next0;
 994           ip6_header_t *ip60;
 995           u16 l4_offset0, frag_offset0;
 996           u8 l4_protocol0;
 997           u32 proto0;
 998           nat64_in2out_set_ctx_t ctx0;
 999           u32 sw_if_index0;
1000
1001           /* speculatively enqueue b0 to the current next frame */
1002           bi0 = from[0];
1003           to_next[0] = bi0;
1004           from += 1;
1005           to_next += 1;
1006           n_left_from -= 1;
1007           n_left_to_next -= 1;
1008
1009           b0 = vlib_get_buffer (vm, bi0);
1010           ip60 = vlib_buffer_get_current (b0);
1011
1012           ctx0.b = b0;
1013           ctx0.vm = vm;
1014           ctx0.thread_index = thread_index;
1015
1016           next0 = NAT64_IN2OUT_NEXT_IP4_LOOKUP;
1017
1018           if (PREDICT_FALSE
1019               (ip6_parse
1020                (ip60, b0->current_length, &l4_protocol0, &l4_offset0,
1021                 &frag_offset0)))
1022             {
1023               next0 = NAT64_IN2OUT_NEXT_DROP;
1024               b0->error = node->errors[NAT64_IN2OUT_ERROR_UNKNOWN];
1025               goto trace0;
1026             }
1027
1028           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1029
1030           if (nat64_not_translate (sw_if_index0, ip60->dst_address))
1031             {
1032               next0 = NAT64_IN2OUT_NEXT_IP6_LOOKUP;
1033               goto trace0;
1034             }
1035
1036           proto0 = ip_proto_to_snat_proto (l4_protocol0);
1037
1038           if (is_slow_path)
1039             {
1040               if (PREDICT_TRUE (proto0 == ~0))
1041                 {
1042                   other_packets++;
1043                   if (is_hairpinning (&ip60->dst_address))
1044                     {
1045                       next0 = NAT64_IN2OUT_NEXT_IP6_LOOKUP;
1046                       if (nat64_in2out_unk_proto_hairpinning
1047                           (vm, b0, ip60, thread_index))
1048                         {
1049                           next0 = NAT64_IN2OUT_NEXT_DROP;
1050                           b0->error =
1051                             node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
1052                         }
1053                       goto trace0;
1054                     }
1055
1056                   if (ip6_to_ip4 (b0, nat64_in2out_unk_proto_set_cb, &ctx0))
1057                     {
1058                       next0 = NAT64_IN2OUT_NEXT_DROP;
1059                       b0->error =
1060                         node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
1061                       goto trace0;
1062                     }
1063                 }
1064               goto trace0;
1065             }
1066           else
1067             {
1068               if (PREDICT_FALSE (proto0 == ~0))
1069                 {
1070                   next0 = NAT64_IN2OUT_NEXT_SLOWPATH;
1071                   goto trace0;
1072                 }
1073             }
1074
1075           if (PREDICT_FALSE
1076               (ip60->protocol == IP_PROTOCOL_IPV6_FRAGMENTATION))
1077             {
1078               next0 = NAT64_IN2OUT_NEXT_REASS;
1079               fragments++;
1080               goto trace0;
1081             }
1082
1083           if (proto0 == SNAT_PROTOCOL_ICMP)
1084             {
1085               icmp_packets++;
1086               if (is_hairpinning (&ip60->dst_address))
1087                 {
1088                   next0 = NAT64_IN2OUT_NEXT_IP6_LOOKUP;
1089                   if (nat64_in2out_icmp_hairpinning
1090                       (vm, b0, ip60, thread_index))
1091                     {
1092                       next0 = NAT64_IN2OUT_NEXT_DROP;
1093                       b0->error =
1094                         node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
1095                     }
1096                   goto trace0;
1097                 }
1098
1099               if (icmp6_to_icmp
1100                   (b0, nat64_in2out_icmp_set_cb, &ctx0,
1101                    nat64_in2out_inner_icmp_set_cb, &ctx0))
1102                 {
1103                   next0 = NAT64_IN2OUT_NEXT_DROP;
1104                   b0->error = node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
1105                   goto trace0;
1106                 }
1107             }
1108           else if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
1109             {
1110               if (proto0 == SNAT_PROTOCOL_TCP)
1111                 tcp_packets++;
1112               else
1113                 udp_packets++;
1114
1115               if (is_hairpinning (&ip60->dst_address))
1116                 {
1117                   next0 = NAT64_IN2OUT_NEXT_IP6_LOOKUP;
1118                   if (nat64_in2out_tcp_udp_hairpinning
1119                       (vm, b0, ip60, thread_index))
1120                     {
1121                       next0 = NAT64_IN2OUT_NEXT_DROP;
1122                       b0->error =
1123                         node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
1124                     }
1125                   goto trace0;
1126                 }
1127
1128               if (ip6_to_ip4_tcp_udp
1129                   (b0, nat64_in2out_tcp_udp_set_cb, &ctx0, 0))
1130                 {
1131                   next0 = NAT64_IN2OUT_NEXT_DROP;
1132                   b0->error = node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
1133                   goto trace0;
1134                 }
1135             }
1136
1137         trace0:
1138           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1139                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1140             {
1141               nat64_in2out_trace_t *t =
1142                 vlib_add_trace (vm, node, b0, sizeof (*t));
1143               t->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1144               t->next_index = next0;
1145               t->is_slow_path = is_slow_path;
1146             }
1147
1148           pkts_processed += next0 == NAT64_IN2OUT_NEXT_IP4_LOOKUP;
1149
1150           /* verify speculative enqueue, maybe switch current next frame */
1151           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
1152                                            n_left_to_next, bi0, next0);
1153         }
1154       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1155     }
1156   vlib_node_increment_counter (vm, stats_node_index,
1157                                NAT64_IN2OUT_ERROR_IN2OUT_PACKETS,
1158                                pkts_processed);
1159   vlib_node_increment_counter (vm, stats_node_index,
1160                                NAT64_IN2OUT_ERROR_TCP_PACKETS, tcp_packets);
1161   vlib_node_increment_counter (vm, stats_node_index,
1162                                NAT64_IN2OUT_ERROR_UDP_PACKETS, tcp_packets);
1163   vlib_node_increment_counter (vm, stats_node_index,
1164                                NAT64_IN2OUT_ERROR_ICMP_PACKETS, icmp_packets);
1165   vlib_node_increment_counter (vm, stats_node_index,
1166                                NAT64_IN2OUT_ERROR_OTHER_PACKETS,
1167                                other_packets);
1168   vlib_node_increment_counter (vm, stats_node_index,
1169                                NAT64_IN2OUT_ERROR_FRAGMENTS, fragments);
1170
1171   return frame->n_vectors;
1172 }
1173
1174 static uword
1175 nat64_in2out_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
1176                       vlib_frame_t * frame)
1177 {
1178   return nat64_in2out_node_fn_inline (vm, node, frame, 0);
1179 }
1180
1181 /* *INDENT-OFF* */
1182 VLIB_REGISTER_NODE (nat64_in2out_node) = {
1183   .function = nat64_in2out_node_fn,
1184   .name = "nat64-in2out",
1185   .vector_size = sizeof (u32),
1186   .format_trace = format_nat64_in2out_trace,
1187   .type = VLIB_NODE_TYPE_INTERNAL,
1188   .n_errors = ARRAY_LEN (nat64_in2out_error_strings),
1189   .error_strings = nat64_in2out_error_strings,
1190   .n_next_nodes = NAT64_IN2OUT_N_NEXT,
1191   /* edit / add dispositions here */
1192   .next_nodes = {
1193     [NAT64_IN2OUT_NEXT_DROP] = "error-drop",
1194     [NAT64_IN2OUT_NEXT_IP4_LOOKUP] = "ip4-lookup",
1195     [NAT64_IN2OUT_NEXT_IP6_LOOKUP] = "ip6-lookup",
1196     [NAT64_IN2OUT_NEXT_SLOWPATH] = "nat64-in2out-slowpath",
1197     [NAT64_IN2OUT_NEXT_REASS] = "nat64-in2out-reass",
1198   },
1199 };
1200 /* *INDENT-ON* */
1201
1202 VLIB_NODE_FUNCTION_MULTIARCH (nat64_in2out_node, nat64_in2out_node_fn);
1203
1204 static uword
1205 nat64_in2out_slowpath_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
1206                                vlib_frame_t * frame)
1207 {
1208   return nat64_in2out_node_fn_inline (vm, node, frame, 1);
1209 }
1210
1211 /* *INDENT-OFF* */
1212 VLIB_REGISTER_NODE (nat64_in2out_slowpath_node) = {
1213   .function = nat64_in2out_slowpath_node_fn,
1214   .name = "nat64-in2out-slowpath",
1215   .vector_size = sizeof (u32),
1216   .format_trace = format_nat64_in2out_trace,
1217   .type = VLIB_NODE_TYPE_INTERNAL,
1218   .n_errors = ARRAY_LEN (nat64_in2out_error_strings),
1219   .error_strings = nat64_in2out_error_strings,
1220   .n_next_nodes = NAT64_IN2OUT_N_NEXT,
1221   /* edit / add dispositions here */
1222   .next_nodes = {
1223     [NAT64_IN2OUT_NEXT_DROP] = "error-drop",
1224     [NAT64_IN2OUT_NEXT_IP4_LOOKUP] = "ip4-lookup",
1225     [NAT64_IN2OUT_NEXT_IP6_LOOKUP] = "ip6-lookup",
1226     [NAT64_IN2OUT_NEXT_SLOWPATH] = "nat64-in2out-slowpath",
1227     [NAT64_IN2OUT_NEXT_REASS] = "nat64-in2out-reass",
1228   },
1229 };
1230 /* *INDENT-ON* */
1231
1232 VLIB_NODE_FUNCTION_MULTIARCH (nat64_in2out_slowpath_node,
1233                               nat64_in2out_slowpath_node_fn);
1234
1235 typedef struct nat64_in2out_frag_set_ctx_t_
1236 {
1237   vlib_main_t *vm;
1238   u32 sess_index;
1239   u32 thread_index;
1240   u16 l4_offset;
1241   u8 proto;
1242   u8 first_frag;
1243 } nat64_in2out_frag_set_ctx_t;
1244
1245 static int
1246 nat64_in2out_frag_set_cb (ip6_header_t * ip6, ip4_header_t * ip4, void *arg)
1247 {
1248   nat64_main_t *nm = &nat64_main;
1249   nat64_in2out_frag_set_ctx_t *ctx = arg;
1250   nat64_db_st_entry_t *ste;
1251   nat64_db_bib_entry_t *bibe;
1252   udp_header_t *udp;
1253   nat64_db_t *db = &nm->db[ctx->thread_index];
1254
1255   ste = nat64_db_st_entry_by_index (db, ctx->proto, ctx->sess_index);
1256   if (!ste)
1257     return -1;
1258
1259   bibe = nat64_db_bib_entry_by_index (db, ctx->proto, ste->bibe_index);
1260   if (!bibe)
1261     return -1;
1262
1263   nat64_session_reset_timeout (ste, ctx->vm);
1264
1265   if (ctx->first_frag)
1266     {
1267       udp = (udp_header_t *) u8_ptr_add (ip6, ctx->l4_offset);
1268
1269       if (ctx->proto == IP_PROTOCOL_TCP)
1270         {
1271           u16 *checksum;
1272           ip_csum_t csum;
1273           tcp_header_t *tcp = (tcp_header_t *) udp;
1274
1275           nat64_tcp_session_set_state (ste, tcp, 1);
1276           checksum = &tcp->checksum;
1277           csum = ip_csum_sub_even (*checksum, tcp->src_port);
1278           csum = ip_csum_sub_even (csum, ip6->src_address.as_u64[0]);
1279           csum = ip_csum_sub_even (csum, ip6->src_address.as_u64[1]);
1280           csum = ip_csum_sub_even (csum, ip6->dst_address.as_u64[0]);
1281           csum = ip_csum_sub_even (csum, ip6->dst_address.as_u64[1]);
1282           csum = ip_csum_add_even (csum, bibe->out_port);
1283           csum = ip_csum_add_even (csum, bibe->out_addr.as_u32);
1284           csum = ip_csum_add_even (csum, ste->out_r_addr.as_u32);
1285           *checksum = ip_csum_fold (csum);
1286         }
1287
1288       udp->src_port = bibe->out_port;
1289     }
1290
1291   ip4->src_address.as_u32 = bibe->out_addr.as_u32;
1292   ip4->dst_address.as_u32 = ste->out_r_addr.as_u32;
1293
1294   return 0;
1295 }
1296
1297 static int
1298 nat64_in2out_frag_hairpinning (vlib_buffer_t * b, ip6_header_t * ip6,
1299                                nat64_in2out_frag_set_ctx_t * ctx)
1300 {
1301   nat64_main_t *nm = &nat64_main;
1302   nat64_db_st_entry_t *ste;
1303   nat64_db_bib_entry_t *bibe;
1304   udp_header_t *udp = (udp_header_t *) u8_ptr_add (ip6, ctx->l4_offset);
1305   tcp_header_t *tcp = (tcp_header_t *) udp;
1306   u16 sport = udp->src_port;
1307   u16 dport = udp->dst_port;
1308   u16 *checksum;
1309   ip_csum_t csum;
1310   ip46_address_t daddr;
1311   nat64_db_t *db = &nm->db[ctx->thread_index];
1312
1313   if (ctx->first_frag)
1314     {
1315       if (ctx->proto == IP_PROTOCOL_UDP)
1316         checksum = &udp->checksum;
1317       else
1318         checksum = &tcp->checksum;
1319
1320       csum = ip_csum_sub_even (*checksum, ip6->src_address.as_u64[0]);
1321       csum = ip_csum_sub_even (csum, ip6->src_address.as_u64[1]);
1322       csum = ip_csum_sub_even (csum, ip6->dst_address.as_u64[0]);
1323       csum = ip_csum_sub_even (csum, ip6->dst_address.as_u64[1]);
1324       csum = ip_csum_sub_even (csum, sport);
1325       csum = ip_csum_sub_even (csum, dport);
1326     }
1327
1328   ste = nat64_db_st_entry_by_index (db, ctx->proto, ctx->sess_index);
1329   if (!ste)
1330     return -1;
1331
1332   bibe = nat64_db_bib_entry_by_index (db, ctx->proto, ste->bibe_index);
1333   if (!bibe)
1334     return -1;
1335
1336   if (ctx->proto == IP_PROTOCOL_TCP)
1337     nat64_tcp_session_set_state (ste, tcp, 1);
1338
1339   nat64_session_reset_timeout (ste, ctx->vm);
1340
1341   sport = bibe->out_port;
1342   dport = ste->r_port;
1343
1344   nat64_compose_ip6 (&ip6->src_address, &bibe->out_addr, bibe->fib_index);
1345
1346   clib_memset (&daddr, 0, sizeof (daddr));
1347   daddr.ip4.as_u32 = ste->out_r_addr.as_u32;
1348
1349   bibe = 0;
1350   /* *INDENT-OFF* */
1351   vec_foreach (db, nm->db)
1352     {
1353       bibe = nat64_db_bib_entry_find (db, &daddr, dport, ctx->proto, 0, 0);
1354
1355       if (bibe)
1356         break;
1357     }
1358   /* *INDENT-ON* */
1359
1360   if (!bibe)
1361     return -1;
1362
1363   ip6->dst_address.as_u64[0] = bibe->in_addr.as_u64[0];
1364   ip6->dst_address.as_u64[1] = bibe->in_addr.as_u64[1];
1365
1366   if (ctx->first_frag)
1367     {
1368       udp->dst_port = bibe->in_port;
1369       udp->src_port = sport;
1370       csum = ip_csum_add_even (csum, ip6->src_address.as_u64[0]);
1371       csum = ip_csum_add_even (csum, ip6->src_address.as_u64[1]);
1372       csum = ip_csum_add_even (csum, ip6->dst_address.as_u64[0]);
1373       csum = ip_csum_add_even (csum, ip6->dst_address.as_u64[1]);
1374       csum = ip_csum_add_even (csum, udp->src_port);
1375       csum = ip_csum_add_even (csum, udp->dst_port);
1376       *checksum = ip_csum_fold (csum);
1377     }
1378
1379   return 0;
1380 }
1381
1382 static uword
1383 nat64_in2out_reass_node_fn (vlib_main_t * vm,
1384                             vlib_node_runtime_t * node, vlib_frame_t * frame)
1385 {
1386   u32 n_left_from, *from, *to_next;
1387   nat64_in2out_next_t next_index;
1388   u32 pkts_processed = 0, cached_fragments = 0;
1389   u32 *fragments_to_drop = 0;
1390   u32 *fragments_to_loopback = 0;
1391   nat64_main_t *nm = &nat64_main;
1392   u32 thread_index = vm->thread_index;
1393
1394   from = vlib_frame_vector_args (frame);
1395   n_left_from = frame->n_vectors;
1396   next_index = node->cached_next_index;
1397
1398   while (n_left_from > 0)
1399     {
1400       u32 n_left_to_next;
1401
1402       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1403
1404       while (n_left_from > 0 && n_left_to_next > 0)
1405         {
1406           u32 bi0;
1407           vlib_buffer_t *b0;
1408           u32 next0;
1409           u8 cached0 = 0;
1410           ip6_header_t *ip60;
1411           u16 l4_offset0, frag_offset0;
1412           u8 l4_protocol0;
1413           nat_reass_ip6_t *reass0;
1414           ip6_frag_hdr_t *frag0;
1415           nat64_db_bib_entry_t *bibe0;
1416           nat64_db_st_entry_t *ste0;
1417           udp_header_t *udp0;
1418           snat_protocol_t proto0;
1419           u32 sw_if_index0, fib_index0;
1420           ip46_address_t saddr0, daddr0;
1421           nat64_in2out_frag_set_ctx_t ctx0;
1422           nat64_db_t *db = &nm->db[thread_index];
1423
1424           /* speculatively enqueue b0 to the current next frame */
1425           bi0 = from[0];
1426           to_next[0] = bi0;
1427           from += 1;
1428           to_next += 1;
1429           n_left_from -= 1;
1430           n_left_to_next -= 1;
1431
1432           b0 = vlib_get_buffer (vm, bi0);
1433           next0 = NAT64_IN2OUT_NEXT_IP4_LOOKUP;
1434
1435           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1436           fib_index0 =
1437             fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP6,
1438                                                  sw_if_index0);
1439
1440           ctx0.thread_index = thread_index;
1441
1442           if (PREDICT_FALSE (nat_reass_is_drop_frag (1)))
1443             {
1444               next0 = NAT64_IN2OUT_NEXT_DROP;
1445               b0->error = node->errors[NAT64_IN2OUT_ERROR_DROP_FRAGMENT];
1446               goto trace0;
1447             }
1448
1449           ip60 = (ip6_header_t *) vlib_buffer_get_current (b0);
1450
1451           if (PREDICT_FALSE
1452               (ip6_parse
1453                (ip60, b0->current_length, &l4_protocol0, &l4_offset0,
1454                 &frag_offset0)))
1455             {
1456               next0 = NAT64_IN2OUT_NEXT_DROP;
1457               b0->error = node->errors[NAT64_IN2OUT_ERROR_UNKNOWN];
1458               goto trace0;
1459             }
1460
1461           if (PREDICT_FALSE
1462               (!(l4_protocol0 == IP_PROTOCOL_TCP
1463                  || l4_protocol0 == IP_PROTOCOL_UDP)))
1464             {
1465               next0 = NAT64_IN2OUT_NEXT_DROP;
1466               b0->error = node->errors[NAT64_IN2OUT_ERROR_DROP_FRAGMENT];
1467               goto trace0;
1468             }
1469
1470           udp0 = (udp_header_t *) u8_ptr_add (ip60, l4_offset0);
1471           frag0 = (ip6_frag_hdr_t *) u8_ptr_add (ip60, frag_offset0);
1472           proto0 = ip_proto_to_snat_proto (l4_protocol0);
1473
1474           reass0 = nat_ip6_reass_find_or_create (ip60->src_address,
1475                                                  ip60->dst_address,
1476                                                  frag0->identification,
1477                                                  l4_protocol0,
1478                                                  1, &fragments_to_drop);
1479
1480           if (PREDICT_FALSE (!reass0))
1481             {
1482               next0 = NAT64_IN2OUT_NEXT_DROP;
1483               b0->error = node->errors[NAT64_IN2OUT_ERROR_MAX_REASS];
1484               goto trace0;
1485             }
1486
1487           if (PREDICT_TRUE (ip6_frag_hdr_offset (frag0)))
1488             {
1489               ctx0.first_frag = 0;
1490               if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1491                 {
1492                   if (nat_ip6_reass_add_fragment
1493                       (thread_index, reass0, bi0, &fragments_to_drop))
1494                     {
1495                       b0->error = node->errors[NAT64_IN2OUT_ERROR_MAX_FRAG];
1496                       next0 = NAT64_IN2OUT_NEXT_DROP;
1497                       goto trace0;
1498                     }
1499                   cached0 = 1;
1500                   goto trace0;
1501                 }
1502             }
1503           else
1504             {
1505               ctx0.first_frag = 1;
1506
1507               saddr0.as_u64[0] = ip60->src_address.as_u64[0];
1508               saddr0.as_u64[1] = ip60->src_address.as_u64[1];
1509               daddr0.as_u64[0] = ip60->dst_address.as_u64[0];
1510               daddr0.as_u64[1] = ip60->dst_address.as_u64[1];
1511
1512               ste0 =
1513                 nat64_db_st_entry_find (db, &saddr0, &daddr0,
1514                                         udp0->src_port, udp0->dst_port,
1515                                         l4_protocol0, fib_index0, 1);
1516               if (!ste0)
1517                 {
1518                   bibe0 =
1519                     nat64_db_bib_entry_find (db, &saddr0, udp0->src_port,
1520                                              l4_protocol0, fib_index0, 1);
1521                   if (!bibe0)
1522                     {
1523                       u16 out_port0;
1524                       ip4_address_t out_addr0;
1525                       if (nat64_alloc_out_addr_and_port
1526                           (fib_index0, proto0, &out_addr0, &out_port0,
1527                            thread_index))
1528                         {
1529                           next0 = NAT64_IN2OUT_NEXT_DROP;
1530                           b0->error =
1531                             node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
1532                           goto trace0;
1533                         }
1534
1535                       bibe0 =
1536                         nat64_db_bib_entry_create (thread_index, db,
1537                                                    &ip60->src_address,
1538                                                    &out_addr0, udp0->src_port,
1539                                                    out_port0, fib_index0,
1540                                                    l4_protocol0, 0);
1541                       if (!bibe0)
1542                         {
1543                           next0 = NAT64_IN2OUT_NEXT_DROP;
1544                           b0->error =
1545                             node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
1546                           goto trace0;
1547                         }
1548                       vlib_set_simple_counter (&nm->total_bibs, thread_index,
1549                                                0, db->bib.bib_entries_num);
1550                     }
1551                   nat64_extract_ip4 (&ip60->dst_address, &daddr0.ip4,
1552                                      fib_index0);
1553                   ste0 =
1554                     nat64_db_st_entry_create (thread_index, db, bibe0,
1555                                               &ip60->dst_address, &daddr0.ip4,
1556                                               udp0->dst_port);
1557                   if (!ste0)
1558                     {
1559                       next0 = NAT64_IN2OUT_NEXT_DROP;
1560                       b0->error =
1561                         node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
1562                       goto trace0;
1563                     }
1564
1565                   vlib_set_simple_counter (&nm->total_sessions, thread_index,
1566                                            0, db->st.st_entries_num);
1567                 }
1568               reass0->sess_index = nat64_db_st_entry_get_index (db, ste0);
1569
1570               nat_ip6_reass_get_frags (reass0, &fragments_to_loopback);
1571             }
1572
1573           ctx0.sess_index = reass0->sess_index;
1574           ctx0.proto = l4_protocol0;
1575           ctx0.vm = vm;
1576           ctx0.l4_offset = l4_offset0;
1577
1578           if (PREDICT_FALSE (is_hairpinning (&ip60->dst_address)))
1579             {
1580               next0 = NAT64_IN2OUT_NEXT_IP6_LOOKUP;
1581               if (nat64_in2out_frag_hairpinning (b0, ip60, &ctx0))
1582                 {
1583                   next0 = NAT64_IN2OUT_NEXT_DROP;
1584                   b0->error = node->errors[NAT64_IN2OUT_ERROR_NO_TRANSLATION];
1585                 }
1586               goto trace0;
1587             }
1588           else
1589             {
1590               if (ip6_to_ip4_fragmented (b0, nat64_in2out_frag_set_cb, &ctx0))
1591                 {
1592                   next0 = NAT64_IN2OUT_NEXT_DROP;
1593                   b0->error = node->errors[NAT64_IN2OUT_ERROR_UNKNOWN];
1594                   goto trace0;
1595                 }
1596             }
1597
1598         trace0:
1599           if (PREDICT_FALSE
1600               ((node->flags & VLIB_NODE_FLAG_TRACE)
1601                && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1602             {
1603               nat64_in2out_reass_trace_t *t =
1604                 vlib_add_trace (vm, node, b0, sizeof (*t));
1605               t->cached = cached0;
1606               t->sw_if_index = sw_if_index0;
1607               t->next_index = next0;
1608             }
1609
1610           if (cached0)
1611             {
1612               n_left_to_next++;
1613               to_next--;
1614               cached_fragments++;
1615             }
1616           else
1617             {
1618               pkts_processed += next0 != NAT64_IN2OUT_NEXT_DROP;
1619
1620               /* verify speculative enqueue, maybe switch current next frame */
1621               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1622                                                to_next, n_left_to_next,
1623                                                bi0, next0);
1624             }
1625
1626           if (n_left_from == 0 && vec_len (fragments_to_loopback))
1627             {
1628               from = vlib_frame_vector_args (frame);
1629               u32 len = vec_len (fragments_to_loopback);
1630               if (len <= VLIB_FRAME_SIZE)
1631                 {
1632                   clib_memcpy_fast (from, fragments_to_loopback,
1633                                     sizeof (u32) * len);
1634                   n_left_from = len;
1635                   vec_reset_length (fragments_to_loopback);
1636                 }
1637               else
1638                 {
1639                   clib_memcpy_fast (from, fragments_to_loopback +
1640                                     (len - VLIB_FRAME_SIZE),
1641                                     sizeof (u32) * VLIB_FRAME_SIZE);
1642                   n_left_from = VLIB_FRAME_SIZE;
1643                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1644                 }
1645             }
1646         }
1647
1648       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1649     }
1650
1651   vlib_node_increment_counter (vm, nat64_in2out_reass_node.index,
1652                                NAT64_IN2OUT_ERROR_PROCESSED_FRAGMENTS,
1653                                pkts_processed);
1654   vlib_node_increment_counter (vm, nat64_in2out_reass_node.index,
1655                                NAT64_IN2OUT_ERROR_CACHED_FRAGMENTS,
1656                                cached_fragments);
1657
1658   nat_send_all_to_node (vm, fragments_to_drop, node,
1659                         &node->errors[NAT64_IN2OUT_ERROR_DROP_FRAGMENT],
1660                         NAT64_IN2OUT_NEXT_DROP);
1661
1662   vec_free (fragments_to_drop);
1663   vec_free (fragments_to_loopback);
1664   return frame->n_vectors;
1665 }
1666
1667 /* *INDENT-OFF* */
1668 VLIB_REGISTER_NODE (nat64_in2out_reass_node) = {
1669   .function = nat64_in2out_reass_node_fn,
1670   .name = "nat64-in2out-reass",
1671   .vector_size = sizeof (u32),
1672   .format_trace = format_nat64_in2out_reass_trace,
1673   .type = VLIB_NODE_TYPE_INTERNAL,
1674   .n_errors = ARRAY_LEN (nat64_in2out_error_strings),
1675   .error_strings = nat64_in2out_error_strings,
1676   .n_next_nodes = NAT64_IN2OUT_N_NEXT,
1677   /* edit / add dispositions here */
1678   .next_nodes = {
1679     [NAT64_IN2OUT_NEXT_DROP] = "error-drop",
1680     [NAT64_IN2OUT_NEXT_IP4_LOOKUP] = "ip4-lookup",
1681     [NAT64_IN2OUT_NEXT_IP6_LOOKUP] = "ip6-lookup",
1682     [NAT64_IN2OUT_NEXT_SLOWPATH] = "nat64-in2out-slowpath",
1683     [NAT64_IN2OUT_NEXT_REASS] = "nat64-in2out-reass",
1684   },
1685 };
1686 /* *INDENT-ON* */
1687
1688 VLIB_NODE_FUNCTION_MULTIARCH (nat64_in2out_reass_node,
1689                               nat64_in2out_reass_node_fn);
1690
1691 #define foreach_nat64_in2out_handoff_error                       \
1692 _(CONGESTION_DROP, "congestion drop")                            \
1693 _(SAME_WORKER, "same worker")                                    \
1694 _(DO_HANDOFF, "do handoff")
1695
1696 typedef enum
1697 {
1698 #define _(sym,str) NAT64_IN2OUT_HANDOFF_ERROR_##sym,
1699   foreach_nat64_in2out_handoff_error
1700 #undef _
1701     NAT64_IN2OUT_HANDOFF_N_ERROR,
1702 } nat64_in2out_handoff_error_t;
1703
1704 static char *nat64_in2out_handoff_error_strings[] = {
1705 #define _(sym,string) string,
1706   foreach_nat64_in2out_handoff_error
1707 #undef _
1708 };
1709
1710 typedef struct
1711 {
1712   u32 next_worker_index;
1713 } nat64_in2out_handoff_trace_t;
1714
1715 static u8 *
1716 format_nat64_in2out_handoff_trace (u8 * s, va_list * args)
1717 {
1718   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1719   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1720   nat64_in2out_handoff_trace_t *t =
1721     va_arg (*args, nat64_in2out_handoff_trace_t *);
1722
1723   s =
1724     format (s, "NAT64-IN2OUT-HANDOFF: next-worker %d", t->next_worker_index);
1725
1726   return s;
1727 }
1728
1729 static inline uword
1730 nat64_in2out_handoff_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
1731                               vlib_frame_t * frame)
1732 {
1733   nat64_main_t *nm = &nat64_main;
1734   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
1735   u32 n_enq, n_left_from, *from;
1736   u16 thread_indices[VLIB_FRAME_SIZE], *ti;
1737   u32 fq_index;
1738   u32 thread_index = vm->thread_index;
1739   u32 do_handoff = 0, same_worker = 0;
1740
1741   from = vlib_frame_vector_args (frame);
1742   n_left_from = frame->n_vectors;
1743   vlib_get_buffers (vm, from, bufs, n_left_from);
1744
1745   b = bufs;
1746   ti = thread_indices;
1747
1748   fq_index = nm->fq_in2out_index;
1749
1750   while (n_left_from > 0)
1751     {
1752       ip6_header_t *ip0;
1753
1754       ip0 = vlib_buffer_get_current (b[0]);
1755       ti[0] = nat64_get_worker_in2out (&ip0->src_address);
1756
1757       if (ti[0] != thread_index)
1758         do_handoff++;
1759       else
1760         same_worker++;
1761
1762       if (PREDICT_FALSE
1763           ((node->flags & VLIB_NODE_FLAG_TRACE)
1764            && (b[0]->flags & VLIB_BUFFER_IS_TRACED)))
1765         {
1766           nat64_in2out_handoff_trace_t *t =
1767             vlib_add_trace (vm, node, b[0], sizeof (*t));
1768           t->next_worker_index = ti[0];
1769         }
1770
1771       n_left_from -= 1;
1772       ti += 1;
1773       b += 1;
1774     }
1775
1776   n_enq =
1777     vlib_buffer_enqueue_to_thread (vm, fq_index, from, thread_indices,
1778                                    frame->n_vectors, 1);
1779
1780   if (n_enq < frame->n_vectors)
1781     vlib_node_increment_counter (vm, node->node_index,
1782                                  NAT64_IN2OUT_HANDOFF_ERROR_CONGESTION_DROP,
1783                                  frame->n_vectors - n_enq);
1784   vlib_node_increment_counter (vm, node->node_index,
1785                                NAT64_IN2OUT_HANDOFF_ERROR_SAME_WORKER,
1786                                same_worker);
1787   vlib_node_increment_counter (vm, node->node_index,
1788                                NAT64_IN2OUT_HANDOFF_ERROR_DO_HANDOFF,
1789                                do_handoff);
1790
1791   return frame->n_vectors;
1792 }
1793
1794 /* *INDENT-OFF* */
1795 VLIB_REGISTER_NODE (nat64_in2out_handoff_node) = {
1796   .function = nat64_in2out_handoff_node_fn,
1797   .name = "nat64-in2out-handoff",
1798   .vector_size = sizeof (u32),
1799   .format_trace = format_nat64_in2out_handoff_trace,
1800   .type = VLIB_NODE_TYPE_INTERNAL,
1801   .n_errors = ARRAY_LEN(nat64_in2out_handoff_error_strings),
1802   .error_strings = nat64_in2out_handoff_error_strings,
1803
1804   .n_next_nodes = 1,
1805
1806   .next_nodes = {
1807     [0] = "error-drop",
1808   },
1809 };
1810 /* *INDENT-ON* */
1811
1812 VLIB_NODE_FUNCTION_MULTIARCH (nat64_in2out_handoff_node,
1813                               nat64_in2out_handoff_node_fn);
1814
1815 /*
1816  * fd.io coding-style-patch-verification: ON
1817  *
1818  * Local Variables:
1819  * eval: (c-set-style "gnu")
1820  * End:
1821  */