- snat_session_key_t key0, sm0;
- clib_bihash_kv_8_8_t kv0, value0;
- snat_worker_key_t k0;
- u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
- ip_csum_t sum0;
- snat_session_t *s0;
-
- if (!icmp_is_error_message (icmp0))
- {
- icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
- u16 icmp_id0 = echo0->identifier;
- key0.addr = ip0->dst_address;
- key0.port = icmp_id0;
- key0.protocol = SNAT_PROTOCOL_ICMP;
- key0.fib_index = sm->outside_fib_index;
- kv0.key = key0.as_u64;
-
- /* Check if destination is in active sessions */
- if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
- {
- /* or static mappings */
- if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
- {
- new_dst_addr0 = sm0.addr.as_u32;
- vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
- }
- }
- else
- {
- si = value0.value;
- if (sm->num_workers > 1)
- {
- k0.addr = ip0->dst_address;
- k0.port = icmp_id0;
- k0.fib_index = sm->outside_fib_index;
- kv0.key = k0.as_u64;
- if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
- ASSERT(0);
- else
- ti = value0.value;
- }
- else
- ti = sm->num_workers;
-
- s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
- new_dst_addr0 = s0->in2out.addr.as_u32;
- vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
- echo0->identifier = s0->in2out.port;
- sum0 = icmp0->checksum;
- sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
- icmp_echo_header_t, identifier);
- icmp0->checksum = ip_csum_fold (sum0);
- }
-
- /* Destination is behind the same NAT, use internal address and port */
- if (new_dst_addr0)
- {
- old_dst_addr0 = ip0->dst_address.as_u32;
- ip0->dst_address.as_u32 = new_dst_addr0;
- sum0 = ip0->checksum;
- sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
- ip4_header_t, dst_address);
- ip0->checksum = ip_csum_fold (sum0);
- }
- }
-
-}
-
-static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
- vlib_buffer_t * b0,
- ip4_header_t * ip0,
- icmp46_header_t * icmp0,
- u32 sw_if_index0,
- u32 rx_fib_index0,
- vlib_node_runtime_t * node,
- u32 next0,
- f64 now,
- u32 thread_index,
- snat_session_t ** p_s0)
-{
- next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
- next0, thread_index, p_s0, 0);
- snat_session_t * s0 = *p_s0;
- if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
- {
- /* Hairpinning */
- if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
- snat_icmp_hairpinning(sm, b0, ip0, icmp0);
- /* Accounting */
- s0->last_heard = now;
- s0->total_pkts++;
- s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
- /* Per-user LRU list maintenance for dynamic translations */
- if (!snat_is_session_static (s0))
- {
- clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
- s0->per_user_index);
- clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
- s0->per_user_list_head_index,
- s0->per_user_index);
- }
- }
- return next0;
-}
-static inline void
-snat_hairpinning_unknown_proto (snat_main_t *sm,
- vlib_buffer_t * b,
- ip4_header_t * ip)
-{
- u32 old_addr, new_addr = 0, ti = 0;
- clib_bihash_kv_8_8_t kv, value;
- clib_bihash_kv_16_8_t s_kv, s_value;
- nat_ed_ses_key_t key;
- snat_session_key_t m_key;
- snat_worker_key_t w_key;
- snat_static_mapping_t *m;
- ip_csum_t sum;
- snat_session_t *s;
-
- old_addr = ip->dst_address.as_u32;
- key.l_addr.as_u32 = ip->dst_address.as_u32;
- key.r_addr.as_u32 = ip->src_address.as_u32;
- key.fib_index = sm->outside_fib_index;
- key.proto = ip->protocol;
- key.rsvd = 0;
- key.l_port = 0;
- s_kv.key[0] = key.as_u64[0];
- s_kv.key[1] = key.as_u64[1];
- if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
- {
- m_key.addr = ip->dst_address;
- m_key.fib_index = sm->outside_fib_index;
- m_key.port = 0;
- m_key.protocol = 0;
- kv.key = m_key.as_u64;
- if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
- return;
-
- m = pool_elt_at_index (sm->static_mappings, value.value);
- if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
- vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
- new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
- }
- else
- {
- if (sm->num_workers > 1)
- {
- w_key.addr = ip->dst_address;
- w_key.port = 0;
- w_key.fib_index = sm->outside_fib_index;
- kv.key = w_key.as_u64;
- if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv, &value))
- return;
- else
- ti = value.value;
- }
- else
- ti = sm->num_workers;
-
- s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
- if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
- vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
- new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
- }
- sum = ip->checksum;
- sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
- ip->checksum = ip_csum_fold (sum);
-}
-
-static void
-snat_in2out_unknown_proto (snat_main_t *sm,
- vlib_buffer_t * b,
- ip4_header_t * ip,
- u32 rx_fib_index,
- u32 thread_index,
- f64 now,
- vlib_main_t * vm)
-{
- clib_bihash_kv_8_8_t kv, value;
- clib_bihash_kv_16_8_t s_kv, s_value;
- snat_static_mapping_t *m;
- snat_session_key_t m_key;
- u32 old_addr, new_addr = 0;
- ip_csum_t sum;
- snat_user_key_t u_key;
- snat_user_t *u;
- dlist_elt_t *head, *elt, *oldest;
- snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
- u32 elt_index, head_index, ses_index, oldest_index;
- snat_session_t * s;
- nat_ed_ses_key_t key;
- u32 address_index = ~0;
- int i;
- u8 is_sm = 0;
-
- old_addr = ip->src_address.as_u32;
-
- key.l_addr = ip->src_address;
- key.r_addr = ip->dst_address;
- key.fib_index = rx_fib_index;
- key.proto = ip->protocol;
- key.rsvd = 0;
- key.l_port = 0;
- s_kv.key[0] = key.as_u64[0];
- s_kv.key[1] = key.as_u64[1];
-
- if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
- {
- s = pool_elt_at_index (tsm->sessions, s_value.value);
- new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
- }
- else
- {
- u_key.addr = ip->src_address;
- u_key.fib_index = rx_fib_index;
- kv.key = u_key.as_u64;
-
- /* Ever heard of the "user" = src ip4 address before? */
- if (clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
- {
- /* no, make a new one */
- pool_get (tsm->users, u);
- memset (u, 0, sizeof (*u));
- u->addr = ip->src_address;
- u->fib_index = rx_fib_index;
-
- pool_get (tsm->list_pool, head);
- u->sessions_per_user_list_head_index = head - tsm->list_pool;
-
- clib_dlist_init (tsm->list_pool,
- u->sessions_per_user_list_head_index);
-
- kv.value = u - tsm->users;
-
- /* add user */
- clib_bihash_add_del_8_8 (&sm->user_hash, &kv, 1);
- }
- else
- {
- u = pool_elt_at_index (tsm->users, value.value);
- }
-
- m_key.addr = ip->src_address;
- m_key.port = 0;
- m_key.protocol = 0;
- m_key.fib_index = rx_fib_index;
- kv.key = m_key.as_u64;
-
- /* Try to find static mapping first */
- if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
- {
- m = pool_elt_at_index (sm->static_mappings, value.value);
- new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
- is_sm = 1;
- goto create_ses;
- }
- /* Fallback to 3-tuple key */
- else
- {
- /* Choose same out address as for TCP/UDP session to same destination */
- if (!clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
- {
- head_index = u->sessions_per_user_list_head_index;
- head = pool_elt_at_index (tsm->list_pool, head_index);
- elt_index = head->next;
- elt = pool_elt_at_index (tsm->list_pool, elt_index);
- ses_index = elt->value;
- while (ses_index != ~0)
- {
- s = pool_elt_at_index (tsm->sessions, ses_index);
- elt_index = elt->next;
- elt = pool_elt_at_index (tsm->list_pool, elt_index);
- ses_index = elt->value;
-
- if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
- {
- new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
- address_index = s->outside_address_index;
-
- key.fib_index = sm->outside_fib_index;
- key.l_addr.as_u32 = new_addr;
- s_kv.key[0] = key.as_u64[0];
- s_kv.key[1] = key.as_u64[1];
- if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
- break;
-
- goto create_ses;
- }
- }
- }
- key.fib_index = sm->outside_fib_index;
- for (i = 0; i < vec_len (sm->addresses); i++)
- {
- key.l_addr.as_u32 = sm->addresses[i].addr.as_u32;
- s_kv.key[0] = key.as_u64[0];
- s_kv.key[1] = key.as_u64[1];
- if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
- {
- new_addr = ip->src_address.as_u32 = key.l_addr.as_u32;
- address_index = i;
- goto create_ses;
- }
- }
- return;
- }
-
-create_ses:
- /* Over quota? Recycle the least recently used dynamic translation */
- if (u->nsessions >= sm->max_translations_per_user && !is_sm)
- {
- /* Remove the oldest dynamic translation */
- do {
- oldest_index = clib_dlist_remove_head (
- tsm->list_pool, u->sessions_per_user_list_head_index);
-
- ASSERT (oldest_index != ~0);
-
- /* add it back to the end of the LRU list */
- clib_dlist_addtail (tsm->list_pool,
- u->sessions_per_user_list_head_index,
- oldest_index);
- /* Get the list element */
- oldest = pool_elt_at_index (tsm->list_pool, oldest_index);
-
- /* Get the session index from the list element */
- ses_index = oldest->value;
-
- /* Get the session */
- s = pool_elt_at_index (tsm->sessions, ses_index);
- } while (snat_is_session_static (s));
-
- if (snat_is_unk_proto_session (s))
- {
- /* Remove from lookup tables */
- key.l_addr = s->in2out.addr;
- key.r_addr = s->ext_host_addr;
- key.fib_index = s->in2out.fib_index;
- key.proto = s->in2out.port;
- s_kv.key[0] = key.as_u64[0];
- s_kv.key[1] = key.as_u64[1];
- if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 0))
- clib_warning ("in2out key del failed");
-
- key.l_addr = s->out2in.addr;
- key.fib_index = s->out2in.fib_index;
- s_kv.key[0] = key.as_u64[0];
- s_kv.key[1] = key.as_u64[1];
- if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 0))
- clib_warning ("out2in key del failed");
- }
- else
- {
- /* log NAT event */
- snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
- s->out2in.addr.as_u32,
- s->in2out.protocol,
- s->in2out.port,
- s->out2in.port,
- s->in2out.fib_index);
-
- snat_free_outside_address_and_port (sm, thread_index, &s->out2in,
- s->outside_address_index);
-
- /* Remove in2out, out2in keys */
- kv.key = s->in2out.as_u64;
- if (clib_bihash_add_del_8_8 (&sm->in2out, &kv, 0))
- clib_warning ("in2out key del failed");
- kv.key = s->out2in.as_u64;
- if (clib_bihash_add_del_8_8 (&sm->out2in, &kv, 0))
- clib_warning ("out2in key del failed");
- }
- }
- else
- {
- /* Create a new session */
- pool_get (tsm->sessions, s);
- memset (s, 0, sizeof (*s));
-
- /* Create list elts */
- pool_get (tsm->list_pool, elt);
- clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
- elt->value = s - tsm->sessions;
- s->per_user_index = elt - tsm->list_pool;
- s->per_user_list_head_index = u->sessions_per_user_list_head_index;
- clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
- s->per_user_index);
- }
-
- s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
- s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
- s->outside_address_index = address_index;
- s->out2in.addr.as_u32 = new_addr;
- s->out2in.fib_index = sm->outside_fib_index;
- s->in2out.addr.as_u32 = old_addr;
- s->in2out.fib_index = rx_fib_index;
- s->in2out.port = s->out2in.port = ip->protocol;
- if (is_sm)
- {
- u->nstaticsessions++;
- s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
- }
- else
- {
- u->nsessions++;
- }
-
- /* Add to lookup tables */
- key.l_addr.as_u32 = old_addr;
- key.r_addr = ip->dst_address;
- key.proto = ip->protocol;
- key.fib_index = rx_fib_index;
- s_kv.key[0] = key.as_u64[0];
- s_kv.key[1] = key.as_u64[1];
- s_kv.value = s - tsm->sessions;
- if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
- clib_warning ("in2out key add failed");
-
- key.l_addr.as_u32 = new_addr;
- key.fib_index = sm->outside_fib_index;
- s_kv.key[0] = key.as_u64[0];
- s_kv.key[1] = key.as_u64[1];
- if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
- clib_warning ("out2in key add failed");
- }
-
- /* Update IP checksum */
- sum = ip->checksum;
- sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
- ip->checksum = ip_csum_fold (sum);
-
- /* Accounting */
- s->last_heard = now;
- s->total_pkts++;
- s->total_bytes += vlib_buffer_length_in_chain (vm, b);
- /* Per-user LRU list maintenance */
- clib_dlist_remove (tsm->list_pool, s->per_user_index);
- clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
- s->per_user_index);
-
- /* Hairpinning */
- if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
- snat_hairpinning_unknown_proto(sm, b, ip);
-
- if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
- vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
-}
-
-static snat_session_t *
-snat_in2out_lb (snat_main_t *sm,
- vlib_buffer_t * b,
- ip4_header_t * ip,
- u32 rx_fib_index,
- u32 thread_index,
- f64 now,
- vlib_main_t * vm)
-{
- nat_ed_ses_key_t key;
- clib_bihash_kv_16_8_t s_kv, s_value;
- udp_header_t *udp = ip4_next_header (ip);
- tcp_header_t *tcp = (tcp_header_t *) udp;
- snat_session_t *s = 0;
- snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
- u32 old_addr, new_addr;
- u16 new_port, old_port;
- ip_csum_t sum;
- u32 proto = ip_proto_to_snat_proto (ip->protocol);
- snat_session_key_t e_key, l_key;
- clib_bihash_kv_8_8_t kv, value;
- snat_user_key_t u_key;
- snat_user_t *u;
- dlist_elt_t *head, *elt;
-
- old_addr = ip->src_address.as_u32;
-
- key.l_addr = ip->src_address;
- key.r_addr = ip->dst_address;
- key.fib_index = rx_fib_index;
- key.proto = ip->protocol;
- key.rsvd = 0;
- key.l_port = udp->src_port;
- s_kv.key[0] = key.as_u64[0];
- s_kv.key[1] = key.as_u64[1];
-
- if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
- {
- s = pool_elt_at_index (tsm->sessions, s_value.value);
- }
- else
- {
- l_key.addr = ip->src_address;
- l_key.port = udp->src_port;
- l_key.protocol = proto;
- l_key.fib_index = rx_fib_index;
- if (snat_static_mapping_match(sm, l_key, &e_key, 0, 0))
- return 0;
-
- u_key.addr = ip->src_address;
- u_key.fib_index = rx_fib_index;
- kv.key = u_key.as_u64;
-
- /* Ever heard of the "user" = src ip4 address before? */
- if (clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
- {
- /* no, make a new one */
- pool_get (tsm->users, u);
- memset (u, 0, sizeof (*u));
- u->addr = ip->src_address;
- u->fib_index = rx_fib_index;
-
- pool_get (tsm->list_pool, head);
- u->sessions_per_user_list_head_index = head - tsm->list_pool;
-
- clib_dlist_init (tsm->list_pool,
- u->sessions_per_user_list_head_index);
-
- kv.value = u - tsm->users;
-
- /* add user */
- if (clib_bihash_add_del_8_8 (&sm->user_hash, &kv, 1))
- clib_warning ("user key add failed");
- }
- else
- {
- u = pool_elt_at_index (tsm->users, value.value);
- }
-
- /* Create a new session */
- pool_get (tsm->sessions, s);
- memset (s, 0, sizeof (*s));
-
- s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
- s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
- s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
- s->outside_address_index = ~0;
- s->in2out = l_key;
- s->out2in = e_key;
- u->nstaticsessions++;
-
- /* Create list elts */
- pool_get (tsm->list_pool, elt);
- clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
- elt->value = s - tsm->sessions;
- s->per_user_index = elt - tsm->list_pool;
- s->per_user_list_head_index = u->sessions_per_user_list_head_index;
- clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
- s->per_user_index);
-
- /* Add to lookup tables */
- s_kv.value = s - tsm->sessions;
- if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
- clib_warning ("in2out-ed key add failed");
-
- key.l_addr = e_key.addr;
- key.fib_index = e_key.fib_index;
- key.l_port = e_key.port;
- s_kv.key[0] = key.as_u64[0];
- s_kv.key[1] = key.as_u64[1];
- if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
- clib_warning ("out2in-ed key add failed");
- }
-
- new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
-
- /* Update IP checksum */
- sum = ip->checksum;
- sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
- ip->checksum = ip_csum_fold (sum);
-
- if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
- {
- old_port = tcp->src_port;
- tcp->src_port = s->out2in.port;
- new_port = tcp->src_port;
-
- sum = tcp->checksum;
- sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
- sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
- tcp->checksum = ip_csum_fold(sum);
- }
- else
- {
- udp->src_port = s->out2in.port;
- udp->checksum = 0;
- }
-
- if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
- vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
-
- /* Accounting */
- s->last_heard = now;
- s->total_pkts++;
- s->total_bytes += vlib_buffer_length_in_chain (vm, b);
- return s;
-}
-
-static inline uword
-snat_in2out_node_fn_inline (vlib_main_t * vm,
- vlib_node_runtime_t * node,
- vlib_frame_t * frame, int is_slow_path,
- int is_output_feature)
-{
- u32 n_left_from, * from, * to_next;