2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
18 #include <vnet/gre/packet.h>
19 #include <lb/lbhash.h>
21 #define foreach_lb_error \
23 _(PROTO_NOT_SUPPORTED, "protocol not supported") \
24 _(NO_SERVER, "no configured application server")
27 #define _(sym,str) LB_ERROR_##sym,
33 static char *lb_error_strings[] = {
34 #define _(sym,string) string,
51 /* u8 *lb_format_adjacency(u8 * s, va_list * va) */
53 /* lb_main_t *lbm = &lb_main; */
54 /* __attribute((unused)) ip_lookup_main_t *lm = va_arg (*va, ip_lookup_main_t *); */
55 /* ip_adjacency_t *adj = va_arg (*va, ip_adjacency_t *); */
56 /* lb_adj_data_t *ad = (lb_adj_data_t *) &adj->opaque; */
57 /* __attribute__((unused)) lb_vip_t *vip = pool_elt_at_index (lbm->vips, ad->vip_index); */
58 /* return format(s, "vip idx:%d", ad->vip_index); */
62 format_lb_trace (u8 * s, va_list * args)
64 lb_main_t *lbm = &lb_main;
65 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
66 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
67 lb_trace_t *t = va_arg (*args, lb_trace_t *);
68 s = format(s, "lb vip[%d]: %U\n", t->vip_index, format_lb_vip, &lbm->vips[t->vip_index]);
69 s = format(s, "lb as[%d]: %U\n", t->as_index, format_lb_as, &lbm->ass[t->as_index]);
73 lb_hash_t *lb_get_sticky_table(u32 cpu_index)
75 lb_main_t *lbm = &lb_main;
76 lb_hash_t *sticky_ht = lbm->per_cpu[cpu_index].sticky_ht;
77 //Check if size changed
78 if (PREDICT_FALSE(sticky_ht && (lbm->per_cpu_sticky_buckets != lb_hash_nbuckets(sticky_ht)))) {
80 //Dereference everything in there
82 lb_hash_foreach_entry(sticky_ht, e) {
83 vlib_refcount_add(&lbm->as_refcount, cpu_index, e->value, -1);
84 vlib_refcount_add(&lbm->as_refcount, cpu_index, 0, -1);
87 lb_hash_free(sticky_ht);
92 if (PREDICT_FALSE(sticky_ht == NULL)) {
93 lbm->per_cpu[cpu_index].sticky_ht = lb_hash_alloc(lbm->per_cpu_sticky_buckets, lbm->flow_timeout);
94 sticky_ht = lbm->per_cpu[cpu_index].sticky_ht;
95 clib_warning("Regenerated sticky table %p", sticky_ht);
101 sticky_ht->timeout = lbm->flow_timeout;
105 static_always_inline uword
106 lb_node_fn (vlib_main_t * vm,
107 vlib_node_runtime_t * node, vlib_frame_t * frame,
108 u8 is_input_v4, //Compile-time parameter stating that is input is v4 (or v6)
109 u8 is_encap_v4) //Compile-time parameter stating that is GRE encap is v4 (or v6)
111 ip_lookup_main_t *lm = (is_input_v4)?&ip4_main.lookup_main:&ip6_main.lookup_main;
112 lb_main_t *lbm = &lb_main;
113 vlib_node_runtime_t *error_node = node;
114 u32 n_left_from, *from, next_index, *to_next, n_left_to_next;
115 u32 cpu_index = os_get_cpu_number();
116 u32 lb_time = lb_hash_time_now(vm);
118 lb_hash_t *sticky_ht = lb_get_sticky_table(cpu_index);
119 from = vlib_frame_vector_args (frame);
120 n_left_from = frame->n_vectors;
121 next_index = node->cached_next_index;
123 while (n_left_from > 0)
125 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
126 while (n_left_from > 0 && n_left_to_next > 0)
130 ip_adjacency_t *adj0;
136 u32 value0, available_index0, hash0;
138 lb_error_t error0 = LB_ERROR_NONE;
139 lb_next_t next0 = LB_NEXT_LOOKUP;
141 if (PREDICT_TRUE(n_left_from > 1))
144 p2 = vlib_get_buffer(vm, from[1]);
145 vlib_prefetch_buffer_header(p2, STORE);
146 /* IPv4 + 8 = 28. possibly plus -40 */
147 CLIB_PREFETCH (vlib_buffer_get_current(p2) - 40, 128, STORE);
150 pi0 = to_next[0] = from[0];
156 p0 = vlib_get_buffer (vm, pi0);
157 adj0 = ip_get_adjacency (lm, vnet_buffer (p0)->ip.adj_index[VLIB_TX]);
158 ad0 = (lb_adj_data_t *) &adj0->opaque;
159 vip0 = pool_elt_at_index (lbm->vips, ad0->vip_index);
163 ip40 = vlib_buffer_get_current (p0);
164 len0 = clib_net_to_host_u16(ip40->length);
165 key0[0] = (u64) ip40->src_address.as_u32;
166 key0[1] = (u64) ip40->dst_address.as_u32;
169 key0[4] = ((u64)((udp_header_t *)(ip40 + 1))->src_port << 32) |
170 ((u64)((udp_header_t *)(ip40 + 1))->dst_port << 16);
172 hash0 = lb_hash_hash(key0);
175 ip60 = vlib_buffer_get_current (p0);
176 len0 = clib_net_to_host_u16(ip60->payload_length) + sizeof(ip6_header_t);
177 key0[0] = ip60->src_address.as_u64[0];
178 key0[1] = ip60->src_address.as_u64[1];
179 key0[2] = ip60->dst_address.as_u64[0];
180 key0[3] = ip60->dst_address.as_u64[1];
181 key0[4] = ((u64)((udp_header_t *)(ip60 + 1))->src_port << 32) |
182 ((u64)((udp_header_t *)(ip60 + 1))->dst_port << 16);
184 hash0 = lb_hash_hash(key0);
187 //NOTE: This is an ugly trick to not include the VIP index in the hash calculation
188 //but actually use it in the key determination.
189 key0[4] |= ((vip0 - lbm->vips));
191 lb_hash_get(sticky_ht, key0, hash0, lb_time, &available_index0, &value0);
192 if (PREDICT_TRUE(value0 != ~0)) {
193 //Found an existing entry
194 as0 = &lbm->ass[value0];
195 } else if (PREDICT_TRUE(available_index0 != ~0)) {
196 //There is an available slot for a new flow
197 as0 = &lbm->ass[vip0->new_flow_table[hash0 & vip0->new_flow_table_mask].as_index];
198 if (PREDICT_FALSE(as0 == lbm->ass)) { //Special first element
199 error0 = LB_ERROR_NO_SERVER;
200 next0 = LB_NEXT_DROP;
202 vlib_increment_simple_counter(&lbm->vip_counters[LB_VIP_COUNTER_TRACKED_SESSION],
203 cpu_index, vip0 - lbm->vips, 1);
206 //TODO: There are race conditions with as0 and vip0 manipulation.
207 //Configuration may be changed, vectors resized, etc...
209 //Dereference previously used
210 vlib_refcount_add(&lbm->as_refcount, cpu_index, lb_hash_available_value(sticky_ht, available_index0), -1);
211 vlib_refcount_add(&lbm->as_refcount, cpu_index, as0 - lbm->ass, 1);
214 //Note that when there is no AS configured, an entry is configured anyway.
215 //But no configured AS is not something that should happen
216 lb_hash_put(sticky_ht, key0, as0 - lbm->ass, available_index0, lb_time);
218 //Could not store new entry in the table
219 as0 = &lbm->ass[vip0->new_flow_table[hash0 & vip0->new_flow_table_mask].as_index];
220 vlib_increment_simple_counter(&lbm->vip_counters[LB_VIP_COUNTER_UNTRACKED_PACKET],
221 cpu_index, vip0 - lbm->vips, 1);
227 vlib_buffer_advance(p0, - sizeof(ip4_header_t) - sizeof(gre_header_t));
228 ip40 = vlib_buffer_get_current(p0);
229 gre0 = (gre_header_t *)(ip40 + 1);
230 ip40->src_address = lbm->ip4_src_address;
231 ip40->dst_address = as0->address.ip4;
232 ip40->ip_version_and_header_length = 0x45;
234 ip40->length = clib_host_to_net_u16(len0 + sizeof(gre_header_t) + sizeof(ip4_header_t));
235 ip40->protocol = IP_PROTOCOL_GRE;
236 ip40->checksum = ip4_header_checksum (ip40);
239 vlib_buffer_advance(p0, - sizeof(ip6_header_t) - sizeof(gre_header_t));
240 ip60 = vlib_buffer_get_current(p0);
241 gre0 = (gre_header_t *)(ip60 + 1);
242 ip60->dst_address = as0->address.ip6;
243 ip60->src_address = lbm->ip6_src_address;
244 ip60->hop_limit = 128;
245 ip60->ip_version_traffic_class_and_flow_label = clib_host_to_net_u32 (0x6<<28);
246 ip60->payload_length = clib_host_to_net_u16(len0 + sizeof(gre_header_t));
247 ip60->protocol = IP_PROTOCOL_GRE;
250 gre0->flags_and_version = 0;
251 gre0->protocol = (is_input_v4)?
252 clib_host_to_net_u16(0x0800):
253 clib_host_to_net_u16(0x86DD);
255 vnet_buffer(p0)->ip.adj_index[VLIB_TX] = as0->adj_index;
256 next0 = (as0->adj_index != ~0)?LB_NEXT_REWRITE:next0;
258 if (PREDICT_FALSE (p0->flags & VLIB_BUFFER_IS_TRACED))
260 lb_trace_t *tr = vlib_add_trace (vm, node, p0, sizeof (*tr));
261 tr->as_index = as0 - lbm->ass;
262 tr->vip_index = ad0->vip_index;
265 p0->error = error_node->errors[error0];
266 vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
267 n_left_to_next, pi0, next0);
269 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
272 return frame->n_vectors;
276 lb6_gre6_node_fn (vlib_main_t * vm,
277 vlib_node_runtime_t * node, vlib_frame_t * frame)
279 return lb_node_fn(vm, node, frame, 0, 0);
283 lb6_gre4_node_fn (vlib_main_t * vm,
284 vlib_node_runtime_t * node, vlib_frame_t * frame)
286 return lb_node_fn(vm, node, frame, 0, 1);
290 lb4_gre6_node_fn (vlib_main_t * vm,
291 vlib_node_runtime_t * node, vlib_frame_t * frame)
293 return lb_node_fn(vm, node, frame, 1, 0);
297 lb4_gre4_node_fn (vlib_main_t * vm,
298 vlib_node_runtime_t * node, vlib_frame_t * frame)
300 return lb_node_fn(vm, node, frame, 1, 1);
303 VLIB_REGISTER_NODE (lb6_gre6_node) =
305 .function = lb6_gre6_node_fn,
307 .vector_size = sizeof (u32),
308 .format_trace = format_lb_trace,
310 .n_errors = LB_N_ERROR,
311 .error_strings = lb_error_strings,
313 .n_next_nodes = LB_N_NEXT,
316 [LB_NEXT_LOOKUP] = "ip6-lookup",
317 [LB_NEXT_REWRITE] = "ip6-rewrite",
318 [LB_NEXT_DROP] = "error-drop"
322 /* VNET_IP6_REGISTER_ADJACENCY(lb6_gre6) = { */
323 /* .node_name = "lb6-gre6", */
324 /* .fn = lb_format_adjacency, */
325 /* .next_index = &lb_main.ip_lookup_next_index[LB_VIP_TYPE_IP6_GRE6] */
328 VLIB_REGISTER_NODE (lb6_gre4_node) =
330 .function = lb6_gre4_node_fn,
332 .vector_size = sizeof (u32),
333 .format_trace = format_lb_trace,
335 .n_errors = LB_N_ERROR,
336 .error_strings = lb_error_strings,
338 .n_next_nodes = LB_N_NEXT,
341 [LB_NEXT_LOOKUP] = "ip4-lookup",
342 [LB_NEXT_REWRITE]= "ip4-rewrite-transit",
343 [LB_NEXT_DROP] = "error-drop"
347 /* VNET_IP6_REGISTER_ADJACENCY(lb6_gre4) = { */
348 /* .node_name = "lb6-gre4", */
349 /* .fn = lb_format_adjacency, */
350 /* .next_index = &lb_main.ip_lookup_next_index[LB_VIP_TYPE_IP6_GRE4] */
353 VLIB_REGISTER_NODE (lb4_gre6_node) =
355 .function = lb4_gre6_node_fn,
357 .vector_size = sizeof (u32),
358 .format_trace = format_lb_trace,
360 .n_errors = LB_N_ERROR,
361 .error_strings = lb_error_strings,
363 .n_next_nodes = LB_N_NEXT,
366 [LB_NEXT_LOOKUP] = "ip6-lookup",
367 [LB_NEXT_REWRITE] = "ip6-rewrite",
368 [LB_NEXT_DROP] = "error-drop"
372 /* VNET_IP4_REGISTER_ADJACENCY(lb4_gre6) = { */
373 /* .node_name = "lb4-gre6", */
374 /* .fn = lb_format_adjacency, */
375 /* .next_index = &lb_main.ip_lookup_next_index[LB_VIP_TYPE_IP4_GRE6] */
378 VLIB_REGISTER_NODE (lb4_gre4_node) =
380 .function = lb4_gre4_node_fn,
382 .vector_size = sizeof (u32),
383 .format_trace = format_lb_trace,
385 .n_errors = LB_N_ERROR,
386 .error_strings = lb_error_strings,
388 .n_next_nodes = LB_N_NEXT,
391 [LB_NEXT_LOOKUP] = "ip4-lookup",
392 [LB_NEXT_REWRITE]= "ip4-rewrite-transit",
393 [LB_NEXT_DROP] = "error-drop"
397 /* VNET_IP4_REGISTER_ADJACENCY(lb4_gre4) = { */
398 /* .node_name = "lb4-gre4", */
399 /* .fn = lb_format_adjacency, */
400 /* .next_index = &lb_main.ip_lookup_next_index[LB_VIP_TYPE_IP4_GRE4] */