2 * Copyright (c) 2017 Intel and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "POD IS" BPODIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
17 * kp-plugin implements a MagLev-like load balancer.
18 * http://research.google.com/pubs/pub44824.html
20 * It hasn't been tested for interoperability with the original MagLev
21 * but intends to provide similar functionality.
22 * The kube-proxy receives traffic destined to VIP (Virtual IP)
23 * addresses from one or multiple(ECMP) routers.
24 * The kube-proxy tunnels the traffic toward many application servers
25 * ensuring session stickyness (i.e. that a single sessions is tunneled
26 * towards a single application server).
30 #ifndef KP_PLUGIN_KP_KP_H_
31 #define KP_PLUGIN_KP_KP_H_
33 #include <vnet/util/refcount.h>
34 #include <vnet/vnet.h>
35 #include <vnet/ip/ip.h>
36 #include <vnet/dpo/dpo.h>
37 #include <vnet/fib/fib_table.h>
38 #include <vppinfra/bihash_8_8.h>
40 #include <kubeproxy/kphash.h>
42 #define KP_DEFAULT_PER_CPU_STICKY_BUCKETS 1 << 10
43 #define KP_DEFAULT_FLOW_TIMEOUT 40
44 #define KP_MAPPING_BUCKETS 1024
45 #define KP_MAPPING_MEMORY_SIZE 64<<20
53 KP_NAT4_IN2OUT_NEXT_DROP,
54 KP_NAT4_IN2OUT_NEXT_LOOKUP,
55 KP_NAT4_IN2OUT_N_NEXT,
56 } kp_nat4_in2out_next_t;
58 #define foreach_kp_nat_in2out_error \
59 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \
60 _(IN2OUT_PACKETS, "Good in2out packets processed") \
61 _(NO_TRANSLATION, "No translation")
64 #define _(sym,str) KP_NAT_IN2OUT_ERROR_##sym,
65 foreach_kp_nat_in2out_error
67 KP_NAT_IN2OUT_N_ERROR,
68 } kp_nat_in2out_error_t;
71 * kube-proxy supports three types of service
75 KP_SVR_TYPE_NODEIP_PORT,
81 KP_NODEPORT_NEXT_IP4_NAT4,
82 KP_NODEPORT_NEXT_IP4_NAT6,
83 KP_NODEPORT_NEXT_IP6_NAT4,
84 KP_NODEPORT_NEXT_IP6_NAT6,
85 KP_NODEPORT_NEXT_DROP,
90 * Each VIP is configured with a set of PODs
94 * Registration to FIB event.
99 * Destination address used to transfer traffic towards to that POD.
100 * The address is also used pod ID and pseudo-random
101 * seed for the load-balancing process.
103 ip46_address_t address;
106 * PODs are indexed by address and VIP Index.
107 * Which means there will be duplicated if the same server
108 * address is used for multiple VIPs.
113 * Some per-POD flags.
114 * For now only KP_POD_FLAGS_USED is defined.
118 #define KP_POD_FLAGS_USED 0x1
121 * Rotating timestamp of when KP_POD_FLAGS_USED flag was last set.
123 * POD removal is based on garbage collection and reference counting.
124 * When an POD is removed, there is a race between configuration core
125 * and worker cores which may still add a reference while it should not
126 * be used. This timestamp is used to not remove the POD while a race condition
132 * The FIB entry index for the next-hop
134 fib_node_index_t next_hop_fib_entry_index;
137 * The child index on the FIB entry
139 u32 next_hop_child_index;
142 * The next DPO in the graph to follow.
148 format_function_t format_kp_pod;
152 } kp_new_flow_entry_t;
154 #define kp_foreach_vip_counter \
155 _(NEXT_PACKET, "packet from existing sessions", 0) \
156 _(FIRST_PACKET, "first session packet", 1) \
157 _(UNTRACKED_PACKET, "untracked packet", 2) \
158 _(NO_SERVER, "no server configured", 3)
161 #define _(a,b,c) KP_VIP_COUNTER_##a = c,
162 kp_foreach_vip_counter
168 * kube-proxy supports IPv4 and IPv6 traffic
172 KP_VIP_TYPE_IP4_NAT44,
173 KP_VIP_TYPE_IP4_NAT46,
174 KP_VIP_TYPE_IP6_NAT64,
175 KP_VIP_TYPE_IP6_NAT66,
179 format_function_t format_kp_vip_type;
180 unformat_function_t unformat_kp_vip_type;
183 * Load balancing service is provided per VIP.
184 * In this data model, a VIP can be a whole prefix.
185 * But load balancing only
186 * occurs on a per-source-address/port basis. Meaning that if a given source
187 * reuses the same port for multiple destinations within the same VIP,
188 * they will be considered as a single flow.
195 * Vector mapping (flow-hash & new_connect_table_mask) to POD index.
196 * This is used for new flows.
198 kp_new_flow_entry_t *new_flow_table;
201 * New flows table length - 1
202 * (length MUST be a power of 2)
204 u32 new_flow_table_mask;
207 * last time garbage collection was run to free the PODs.
209 u32 last_garbage_collection;
214 * A Virtual IP represents a given service delivered
215 * by a set of PODs. It can be a single
216 * address or a prefix.
217 * IPv4 prefixes are encoded using IPv4-in-IPv6 embedded address
218 * (i.e. ::/96 prefix).
220 ip46_address_t prefix;
223 * The VIP prefix length.
224 * In case of IPv4, plen = 96 + ip4_plen.
229 * Service port. network byte order
234 * Pod's port corresponding to specific service. network byte order
239 * Node's port, can access service via NodeIP:node_port. network byte order
245 * The type of traffic for this.
246 * KP_TYPE_UNDEFINED if unknown.
251 * Flags related to this VIP.
252 * KP_VIP_FLAGS_USED means the VIP is active.
253 * When it is not set, the VIP in the process of being removed.
254 * We cannot immediately remove a VIP because the VIP index still may be stored
255 * in the adjacency index.
258 #define KP_VIP_FLAGS_USED 0x1
261 * Pool of POD indexes used for this VIP.
262 * This also includes PODs that have been removed (but are still referenced).
269 * mapping from nodeport to vip_index
277 #define kp_vip_is_ip4(vip) ((vip)->type == KP_VIP_TYPE_IP4_NAT44 \
278 || (vip)->type == KP_VIP_TYPE_IP4_NAT46)
279 #define kp_vip_is_nat4(vip) ((vip)->type == KP_VIP_TYPE_IP6_NAT64 \
280 || (vip)->type == KP_VIP_TYPE_IP4_NAT44)
281 format_function_t format_kp_vip;
282 format_function_t format_kp_vip_detailed;
284 #define foreach_kp_nat_protocol \
285 _(UDP, 0, udp, "udp") \
286 _(TCP, 1, tcp, "tcp")
289 #define _(N, i, n, s) KP_NAT_PROTOCOL_##N = i,
290 foreach_kp_nat_protocol
295 kp_ip_proto_to_nat_proto (u8 ip_proto)
299 nat_proto = (ip_proto == IP_PROTOCOL_UDP) ? KP_NAT_PROTOCOL_UDP : nat_proto;
300 nat_proto = (ip_proto == IP_PROTOCOL_TCP) ? KP_NAT_PROTOCOL_TCP : nat_proto;
305 /* Key for Pod's egress SNAT */
322 ip6_address_t prefix;
329 kp_svr_type_t svr_type;
331 ip46_address_t node_ip;
332 ip46_address_t pod_ip;
336 u16 port; /* Network byte order */
337 u16 node_port; /* Network byte order */
338 u16 target_port; /* Network byte order */
345 * Each CPU has its own sticky flow hash table.
346 * One single table is used for all VIPs.
348 kp_hash_t *sticky_ht;
354 * Pool of all Virtual IPs
360 * PODs are referenced by address and vip index.
361 * The first element (index 0) is special and used only to fill
362 * new_flow_tables when no POD has been configured.
367 * Each POD has an associated reference counter.
368 * As pods[0] has a special meaning, its associated counter
369 * starts at 0 and is decremented instead. i.e. do not use it.
371 vlib_refcount_t pod_refcount;
373 /* hash lookup vip_index by key: {u16: nodeport} */
374 uword * nodeport_by_key;
378 * Some global data is per-cpu
380 kp_per_cpu_t *per_cpu;
383 * Node next index for IP adjacencies, for each of the traffic types.
385 u32 ip_lookup_next_index[KP_VIP_N_TYPES];
388 * Number of buckets in the per-cpu sticky hash table.
390 u32 per_cpu_sticky_buckets;
393 * Flow timeout in seconds.
400 vlib_simple_counter_main_t vip_counters[KP_N_VIP_COUNTERS];
403 * DPO used to send packet from IP4/6 lookup to KP node.
405 dpo_type_t dpo_nat4_type;
406 dpo_type_t dpo_nat6_type;
409 * Node type for registering to fib changes.
411 fib_node_type_t fib_node_type;
413 /* Find a static mapping by pod IP : target_port */
414 clib_bihash_8_8_t mapping_by_pod;
416 /* Static mapping pool */
417 kp_snat_mapping_t * snat_mappings;
420 * API dynamically registered base ID.
424 volatile u32 *writer_lock;
427 vlib_main_t *vlib_main;
428 vnet_main_t *vnet_main;
431 #define ip46_address_type(ip46) (ip46_address_is_ip4(ip46)?IP46_TYPE_IP4:IP46_TYPE_IP6)
432 #define ip46_prefix_is_ip4(ip46, len) ((len) >= 96 && ip46_address_is_ip4(ip46))
433 #define ip46_prefix_type(ip46, len) (ip46_prefix_is_ip4(ip46, len)?IP46_TYPE_IP4:IP46_TYPE_IP6)
435 void ip46_prefix_normalize(ip46_address_t *prefix, u8 plen);
436 uword unformat_ip46_prefix (unformat_input_t * input, va_list * args);
437 u8 *format_ip46_prefix (u8 * s, va_list * args);
440 extern kp_main_t kp_main;
441 extern vlib_node_registration_t kp4_node;
442 extern vlib_node_registration_t kp6_node;
443 extern vlib_node_registration_t kp4_nodeport_node;
444 extern vlib_node_registration_t kp6_nodeport_node;
445 extern vlib_node_registration_t kp_nat4_in2out_node;
448 * Fix global kube-proxy parameters.
449 * @return 0 on success. VNET_KP_ERR_XXX on error
451 int kp_conf(u32 sticky_buckets, u32 flow_timeout);
453 int kp_vip_add(ip46_address_t *prefix, u8 plen, kp_vip_type_t type,
454 u32 new_length, u32 *vip_index,
455 u16 port, u16 target_port, u16 node_port);
456 int kp_vip_del(u32 vip_index);
458 int kp_vip_find_index(ip46_address_t *prefix, u8 plen, u32 *vip_index);
460 #define kp_vip_get_by_index(index) (pool_is_free_index(kp_main.vips, index)?NULL:pool_elt_at_index(kp_main.vips, index))
462 int kp_vip_add_pods(u32 vip_index, ip46_address_t *addresses, u32 n);
463 int kp_vip_del_pods(u32 vip_index, ip46_address_t *addresses, u32 n);
465 u32 kp_hash_time_now(vlib_main_t * vm);
467 void kp_garbage_collection();
469 int kp_nat4_interface_add_del (u32 sw_if_index, int is_del);
471 format_function_t format_kp_main;
473 #endif /* KP_PLUGIN_KP_KP_H_ */