+ lb_main_t *lbm = &lb_main;
+ u32 n_left_from, *from, next_index, *to_next, n_left_to_next;
+
+ from = vlib_frame_vector_args (frame);
+ n_left_from = frame->n_vectors;
+ next_index = node->cached_next_index;
+
+ while (n_left_from > 0)
+ {
+ vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
+
+ while (n_left_from > 0 && n_left_to_next > 0)
+ {
+ u32 pi0;
+ vlib_buffer_t *p0;
+ udp_header_t * udp_0;
+ uword * entry0;
+
+ if (PREDICT_TRUE(n_left_from > 1))
+ {
+ vlib_buffer_t *p1 = vlib_get_buffer (vm, from[1]);
+ //Prefetch for encap, next
+ CLIB_PREFETCH(vlib_buffer_get_current (p1) - 64, 64, STORE);
+ }
+
+ if (PREDICT_TRUE(n_left_from > 2))
+ {
+ vlib_buffer_t *p2;
+ p2 = vlib_get_buffer (vm, from[2]);
+ /* prefetch packet header and data */
+ vlib_prefetch_buffer_header(p2, STORE);
+ CLIB_PREFETCH(vlib_buffer_get_current (p2), 64, STORE);
+ }
+
+ pi0 = to_next[0] = from[0];
+ from += 1;
+ n_left_from -= 1;
+ to_next += 1;
+ n_left_to_next -= 1;
+
+ p0 = vlib_get_buffer (vm, pi0);
+
+ if (is_input_v4)
+ {
+ ip4_header_t *ip40;
+ vlib_buffer_advance (
+ p0, -(word) (sizeof(udp_header_t) + sizeof(ip4_header_t)));
+ ip40 = vlib_buffer_get_current (p0);
+ udp_0 = (udp_header_t *) (ip40 + 1);
+ }
+ else
+ {
+ ip6_header_t *ip60;
+ vlib_buffer_advance (
+ p0, -(word) (sizeof(udp_header_t) + sizeof(ip6_header_t)));
+ ip60 = vlib_buffer_get_current (p0);
+ udp_0 = (udp_header_t *) (ip60 + 1);
+ }
+
+ entry0 = hash_get_mem(lbm->vip_index_by_nodeport, &(udp_0->dst_port));
+
+ //Enqueue to next
+ vnet_buffer(p0)->ip.adj_index[VLIB_TX] = entry0[0];
+
+ if (PREDICT_FALSE(p0->flags & VLIB_BUFFER_IS_TRACED))
+ {
+ lb_nodeport_trace_t *tr = vlib_add_trace (vm, node, p0,
+ sizeof(*tr));
+ tr->vip_index = entry0[0];
+ tr->node_port = (u32) clib_net_to_host_u16 (udp_0->dst_port);
+ }
+
+ vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
+ n_left_to_next, pi0,
+ is_input_v4 ?
+ LB4_NODEPORT_NEXT_IP4_NAT4 : LB6_NODEPORT_NEXT_IP6_NAT6);
+ }
+ vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+ }
+
+ return frame->n_vectors;
+
+}
+
+/**
+ * @brief Match NAT44 static mapping.
+ *
+ * @param sm NAT main.
+ * @param match Address and port to match.
+ * @param index index to the pool.
+ *
+ * @returns 0 if match found, otherwise -1.
+ */
+int
+lb_nat44_mapping_match (lb_main_t *lbm, lb_snat4_key_t * match, u32 *index)
+{
+ clib_bihash_kv_8_8_t kv4, value;
+ clib_bihash_8_8_t *mapping_hash = &lbm->mapping_by_as4;
+
+ kv4.key = match->as_u64;
+ kv4.value = 0;
+ if (clib_bihash_search_8_8 (mapping_hash, &kv4, &value))
+ {
+ return 1;
+ }
+
+ *index = value.value;
+ return 0;
+}
+
+/**
+ * @brief Match NAT66 static mapping.
+ *
+ * @param sm NAT main.
+ * @param match Address and port to match.
+ * @param mapping External or local address and port of the matched mapping.
+ *
+ * @returns 0 if match found otherwise 1.
+ */
+int
+lb_nat66_mapping_match (lb_main_t *lbm, lb_snat6_key_t * match, u32 *index)
+{
+ clib_bihash_kv_24_8_t kv6, value;
+ lb_snat6_key_t m_key6;
+ clib_bihash_24_8_t *mapping_hash = &lbm->mapping_by_as6;
+
+ m_key6.addr.as_u64[0] = match->addr.as_u64[0];
+ m_key6.addr.as_u64[1] = match->addr.as_u64[1];
+ m_key6.port = match->port;
+ m_key6.protocol = 0;
+ m_key6.fib_index = 0;
+
+ kv6.key[0] = m_key6.as_u64[0];
+ kv6.key[1] = m_key6.as_u64[1];
+ kv6.key[2] = m_key6.as_u64[2];
+ kv6.value = 0;
+ if (clib_bihash_search_24_8 (mapping_hash, &kv6, &value))
+ {
+ return 1;
+ }
+
+ *index = value.value;
+ return 0;
+}
+
+static uword
+lb_nat_in2out_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
+ vlib_frame_t * frame, u32 is_nat4)
+{
+ u32 n_left_from, *from, *to_next;
+ u32 next_index;
+ u32 pkts_processed = 0;
+ lb_main_t *lbm = &lb_main;
+ u32 stats_node_index;
+
+ stats_node_index =
+ is_nat4 ? lb_nat4_in2out_node.index : lb_nat6_in2out_node.index;
+
+ from = vlib_frame_vector_args (frame);
+ n_left_from = frame->n_vectors;
+ next_index = node->cached_next_index;
+
+ while (n_left_from > 0)
+ {
+ u32 n_left_to_next;
+
+ vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
+
+ while (n_left_from > 0 && n_left_to_next > 0)
+ {
+ u32 bi0;
+ vlib_buffer_t * b0;
+ u32 next0;
+ u32 sw_if_index0;
+ ip_csum_t csum;
+ u16 old_port0, new_port0;
+ udp_header_t * udp0;
+ tcp_header_t * tcp0;
+
+ u32 proto0;
+ u32 rx_fib_index0;
+
+ /* speculatively enqueue b0 to the current next frame */
+ bi0 = from[0];
+ to_next[0] = bi0;
+ from += 1;
+ to_next += 1;
+ n_left_from -= 1;
+ n_left_to_next -= 1;
+
+ b0 = vlib_get_buffer (vm, bi0);
+ next0 = LB_NAT4_IN2OUT_NEXT_LOOKUP;
+ sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
+ rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (
+ sw_if_index0);
+
+ if (is_nat4)
+ {
+ ip4_header_t * ip40;
+ u32 old_addr0, new_addr0;
+ lb_snat4_key_t key40;
+ lb_snat_mapping_t *sm40;
+ u32 index40;
+
+ ip40 = vlib_buffer_get_current (b0);
+ udp0 = ip4_next_header (ip40);
+ tcp0 = (tcp_header_t *) udp0;
+ proto0 = lb_ip_proto_to_nat_proto (ip40->protocol);
+
+ key40.addr = ip40->src_address;
+ key40.protocol = proto0;
+ key40.port = udp0->src_port;
+ key40.fib_index = rx_fib_index0;
+
+ if (lb_nat44_mapping_match (lbm, &key40, &index40))
+ {
+ next0 = LB_NAT4_IN2OUT_NEXT_DROP;
+ goto trace0;
+ }
+
+ sm40 = pool_elt_at_index(lbm->snat_mappings, index40);
+ new_addr0 = sm40->src_ip.ip4.as_u32;
+ new_port0 = sm40->src_port;
+ vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm40->fib_index;
+ old_addr0 = ip40->src_address.as_u32;
+ ip40->src_address.as_u32 = new_addr0;
+
+ csum = ip40->checksum;
+ csum = ip_csum_sub_even (csum, old_addr0);
+ csum = ip_csum_add_even (csum, new_addr0);
+ ip40->checksum = ip_csum_fold (csum);
+
+ if (PREDICT_TRUE(proto0 == LB_NAT_PROTOCOL_TCP))
+ {
+ old_port0 = tcp0->src_port;
+ tcp0->src_port = new_port0;
+
+ csum = tcp0->checksum;
+ csum = ip_csum_sub_even (csum, old_addr0);
+ csum = ip_csum_sub_even (csum, old_port0);
+ csum = ip_csum_add_even (csum, new_addr0);
+ csum = ip_csum_add_even (csum, new_port0);
+ tcp0->checksum = ip_csum_fold (csum);
+ }
+ else if (PREDICT_TRUE(proto0 == LB_NAT_PROTOCOL_UDP))
+ {
+ old_port0 = udp0->src_port;
+ udp0->src_port = new_port0;
+
+ csum = udp0->checksum;
+ csum = ip_csum_sub_even (csum, old_addr0);
+ csum = ip_csum_sub_even (csum, old_port0);
+ csum = ip_csum_add_even (csum, new_addr0);
+ csum = ip_csum_add_even (csum, new_port0);
+ udp0->checksum = ip_csum_fold (csum);
+ }
+
+ pkts_processed += next0 != LB_NAT4_IN2OUT_NEXT_DROP;
+ }
+ else
+ {
+ ip6_header_t * ip60;
+ ip6_address_t old_addr0, new_addr0;
+ lb_snat6_key_t key60;
+ lb_snat_mapping_t *sm60;
+ u32 index60;
+
+ ip60 = vlib_buffer_get_current (b0);
+ udp0 = ip6_next_header (ip60);
+ tcp0 = (tcp_header_t *) udp0;
+ proto0 = lb_ip_proto_to_nat_proto (ip60->protocol);
+
+ key60.addr.as_u64[0] = ip60->src_address.as_u64[0];
+ key60.addr.as_u64[1] = ip60->src_address.as_u64[1];
+ key60.protocol = proto0;
+ key60.port = udp0->src_port;
+ key60.fib_index = rx_fib_index0;
+
+ if (lb_nat66_mapping_match (lbm, &key60, &index60))
+ {
+ next0 = LB_NAT6_IN2OUT_NEXT_DROP;
+ goto trace0;
+ }
+
+ sm60 = pool_elt_at_index(lbm->snat_mappings, index60);
+ new_addr0.as_u64[0] = sm60->src_ip.as_u64[0];
+ new_addr0.as_u64[1] = sm60->src_ip.as_u64[1];
+ new_port0 = sm60->src_port;
+ vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm60->fib_index;
+ old_addr0.as_u64[0] = ip60->src_address.as_u64[0];
+ old_addr0.as_u64[1] = ip60->src_address.as_u64[1];
+ ip60->src_address.as_u64[0] = new_addr0.as_u64[0];
+ ip60->src_address.as_u64[1] = new_addr0.as_u64[1];
+
+ if (PREDICT_TRUE(proto0 == LB_NAT_PROTOCOL_TCP))
+ {
+ old_port0 = tcp0->src_port;
+ tcp0->src_port = new_port0;
+
+ csum = tcp0->checksum;
+ csum = ip_csum_sub_even (csum, old_addr0.as_u64[0]);
+ csum = ip_csum_sub_even (csum, old_addr0.as_u64[1]);
+ csum = ip_csum_add_even (csum, new_addr0.as_u64[0]);
+ csum = ip_csum_add_even (csum, new_addr0.as_u64[1]);
+ csum = ip_csum_sub_even (csum, old_port0);
+ csum = ip_csum_add_even (csum, new_port0);
+ tcp0->checksum = ip_csum_fold (csum);
+ }
+ else if (PREDICT_TRUE(proto0 == LB_NAT_PROTOCOL_UDP))
+ {
+ old_port0 = udp0->src_port;
+ udp0->src_port = new_port0;
+
+ csum = udp0->checksum;
+ csum = ip_csum_sub_even (csum, old_addr0.as_u64[0]);
+ csum = ip_csum_sub_even (csum, old_addr0.as_u64[1]);
+ csum = ip_csum_add_even (csum, new_addr0.as_u64[0]);
+ csum = ip_csum_add_even (csum, new_addr0.as_u64[1]);
+ csum = ip_csum_sub_even (csum, old_port0);
+ csum = ip_csum_add_even (csum, new_port0);
+ udp0->checksum = ip_csum_fold (csum);
+ }
+
+ pkts_processed += next0 != LB_NAT4_IN2OUT_NEXT_DROP;
+ }
+
+ trace0: if (PREDICT_FALSE(
+ (node->flags & VLIB_NODE_FLAG_TRACE) && (b0->flags & VLIB_BUFFER_IS_TRACED)))
+ {
+ lb_nat_trace_t *t = vlib_add_trace (vm, node, b0, sizeof(*t));
+ t->rx_sw_if_index = sw_if_index0;
+ t->next_index = next0;
+ }
+
+ /* verify speculative enqueue, maybe switch current next frame */
+ vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
+ n_left_to_next, bi0, next0);
+ }
+
+ vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+ }
+
+ vlib_node_increment_counter (vm, stats_node_index,
+ LB_NAT_IN2OUT_ERROR_IN2OUT_PACKETS,
+ pkts_processed);
+ return frame->n_vectors;