2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vnet/ip/lookup.h>
17 #include <vnet/dpo/load_balance.h>
18 #include <vnet/dpo/load_balance_map.h>
19 #include <vnet/dpo/drop_dpo.h>
20 #include <vppinfra/math.h> /* for fabs */
21 #include <vnet/adj/adj.h>
22 #include <vnet/adj/adj_internal.h>
23 #include <vnet/fib/fib_urpf_list.h>
24 #include <vnet/bier/bier_hdr_inlines.h>
27 * distribution error tolerance for load-balancing
29 const f64 multipath_next_hop_error_tolerance = 0.1;
34 #define LB_DBG(_lb, _fmt, _args...) \
37 clib_warning("lb:[%s]:" _fmt, \
38 load_balance_format(load_balance_get_index((_lb)), \
44 #define LB_DBG(_p, _fmt, _args...)
49 * Pool of all DPOs. It's not static so the DP can have fast access
51 load_balance_t *load_balance_pool;
54 * The one instance of load-balance main
56 load_balance_main_t load_balance_main;
59 load_balance_get_multipath_tolerance (void)
61 return (multipath_next_hop_error_tolerance);
65 load_balance_get_index (const load_balance_t *lb)
67 return (lb - load_balance_pool);
70 static inline dpo_id_t*
71 load_balance_get_buckets (load_balance_t *lb)
73 if (LB_HAS_INLINE_BUCKETS(lb))
75 return (lb->lb_buckets_inline);
79 return (lb->lb_buckets);
83 static load_balance_t *
84 load_balance_alloc_i (void)
88 pool_get_aligned(load_balance_pool, lb, CLIB_CACHE_LINE_BYTES);
89 memset(lb, 0, sizeof(*lb));
91 lb->lb_map = INDEX_INVALID;
92 lb->lb_urpf = INDEX_INVALID;
93 vlib_validate_combined_counter(&(load_balance_main.lbm_to_counters),
94 load_balance_get_index(lb));
95 vlib_validate_combined_counter(&(load_balance_main.lbm_via_counters),
96 load_balance_get_index(lb));
97 vlib_zero_combined_counter(&(load_balance_main.lbm_to_counters),
98 load_balance_get_index(lb));
99 vlib_zero_combined_counter(&(load_balance_main.lbm_via_counters),
100 load_balance_get_index(lb));
106 load_balance_format (index_t lbi,
107 load_balance_format_flags_t flags,
111 vlib_counter_t to, via;
116 lb = load_balance_get(lbi);
117 vlib_get_combined_counter(&(load_balance_main.lbm_to_counters), lbi, &to);
118 vlib_get_combined_counter(&(load_balance_main.lbm_via_counters), lbi, &via);
119 buckets = load_balance_get_buckets(lb);
121 s = format(s, "%U: ", format_dpo_type, DPO_LOAD_BALANCE);
122 s = format(s, "[proto:%U ", format_dpo_proto, lb->lb_proto);
123 s = format(s, "index:%d buckets:%d ", lbi, lb->lb_n_buckets);
124 s = format(s, "uRPF:%d ", lb->lb_urpf);
125 s = format(s, "to:[%Ld:%Ld]", to.packets, to.bytes);
126 if (0 != via.packets)
128 s = format(s, " via:[%Ld:%Ld]",
129 via.packets, via.bytes);
133 if (INDEX_INVALID != lb->lb_map)
135 s = format(s, "\n%U%U",
136 format_white_space, indent+4,
137 format_load_balance_map, lb->lb_map, indent+4);
139 for (i = 0; i < lb->lb_n_buckets; i++)
141 s = format(s, "\n%U[%d] %U",
142 format_white_space, indent+2,
145 &buckets[i], indent+6);
151 format_load_balance (u8 * s, va_list * args)
153 index_t lbi = va_arg(*args, index_t);
154 load_balance_format_flags_t flags = va_arg(*args, load_balance_format_flags_t);
156 return (load_balance_format(lbi, flags, 0, s));
159 format_load_balance_dpo (u8 * s, va_list * args)
161 index_t lbi = va_arg(*args, index_t);
162 u32 indent = va_arg(*args, u32);
164 return (load_balance_format(lbi, LOAD_BALANCE_FORMAT_DETAIL, indent, s));
168 static load_balance_t *
169 load_balance_create_i (u32 num_buckets,
170 dpo_proto_t lb_proto,
171 flow_hash_config_t fhc)
175 lb = load_balance_alloc_i();
176 lb->lb_hash_config = fhc;
177 lb->lb_n_buckets = num_buckets;
178 lb->lb_n_buckets_minus_1 = num_buckets-1;
179 lb->lb_proto = lb_proto;
181 if (!LB_HAS_INLINE_BUCKETS(lb))
183 vec_validate_aligned(lb->lb_buckets,
184 lb->lb_n_buckets - 1,
185 CLIB_CACHE_LINE_BYTES);
188 LB_DBG(lb, "create");
194 load_balance_create (u32 n_buckets,
195 dpo_proto_t lb_proto,
196 flow_hash_config_t fhc)
198 return (load_balance_get_index(load_balance_create_i(n_buckets, lb_proto, fhc)));
202 load_balance_set_bucket_i (load_balance_t *lb,
205 const dpo_id_t *next)
207 dpo_stack(DPO_LOAD_BALANCE, lb->lb_proto, &buckets[bucket], next);
211 load_balance_set_bucket (index_t lbi,
213 const dpo_id_t *next)
218 lb = load_balance_get(lbi);
219 buckets = load_balance_get_buckets(lb);
221 ASSERT(bucket < lb->lb_n_buckets);
223 load_balance_set_bucket_i(lb, bucket, buckets, next);
227 load_balance_is_drop (const dpo_id_t *dpo)
231 if (DPO_LOAD_BALANCE != dpo->dpoi_type)
234 lb = load_balance_get(dpo->dpoi_index);
236 if (1 == lb->lb_n_buckets)
238 return (dpo_is_drop(load_balance_get_bucket_i(lb, 0)));
244 load_balance_n_buckets (index_t lbi)
248 lb = load_balance_get(lbi);
250 return (lb->lb_n_buckets);
254 load_balance_set_fib_entry_flags (index_t lbi,
255 fib_entry_flag_t flags)
259 lb = load_balance_get(lbi);
260 lb->lb_fib_entry_flags = flags;
265 load_balance_set_urpf (index_t lbi,
271 lb = load_balance_get(lbi);
274 * packets in flight we see this change. but it's atomic, so :P
279 fib_urpf_list_unlock(old);
280 fib_urpf_list_lock(urpf);
284 load_balance_get_urpf (index_t lbi)
288 lb = load_balance_get(lbi);
290 return (lb->lb_urpf);
294 load_balance_get_bucket (index_t lbi,
299 lb = load_balance_get(lbi);
301 return (load_balance_get_bucket_i(lb, bucket));
305 next_hop_sort_by_weight (const load_balance_path_t * n1,
306 const load_balance_path_t * n2)
308 return ((int) n1->path_weight - (int) n2->path_weight);
311 /* Given next hop vector is over-written with normalized one with sorted weights and
312 with weights corresponding to the number of adjacencies for each next hop.
313 Returns number of adjacencies in block. */
315 ip_multipath_normalize_next_hops (const load_balance_path_t * raw_next_hops,
316 load_balance_path_t ** normalized_next_hops,
318 f64 multipath_next_hop_error_tolerance)
320 load_balance_path_t * nhs;
321 uword n_nhs, n_adj, n_adj_left, i, sum_weight;
324 n_nhs = vec_len (raw_next_hops);
329 /* Allocate enough space for 2 copies; we'll use second copy to save original weights. */
330 nhs = *normalized_next_hops;
331 vec_validate (nhs, 2*n_nhs - 1);
333 /* Fast path: 1 next hop in block. */
337 nhs[0] = raw_next_hops[0];
338 nhs[0].path_weight = 1;
346 int cmp = next_hop_sort_by_weight (&raw_next_hops[0], &raw_next_hops[1]) < 0;
349 nhs[0] = raw_next_hops[cmp];
350 nhs[1] = raw_next_hops[cmp ^ 1];
352 /* Fast path: equal cost multipath with 2 next hops. */
353 if (nhs[0].path_weight == nhs[1].path_weight)
355 nhs[0].path_weight = nhs[1].path_weight = 1;
363 clib_memcpy (nhs, raw_next_hops, n_nhs * sizeof (raw_next_hops[0]));
364 qsort (nhs, n_nhs, sizeof (nhs[0]), (void *) next_hop_sort_by_weight);
367 /* Find total weight to normalize weights. */
369 for (i = 0; i < n_nhs; i++)
370 sum_weight += nhs[i].path_weight;
372 /* In the unlikely case that all weights are given as 0, set them all to 1. */
375 for (i = 0; i < n_nhs; i++)
376 nhs[i].path_weight = 1;
380 /* Save copies of all next hop weights to avoid being overwritten in loop below. */
381 for (i = 0; i < n_nhs; i++)
382 nhs[n_nhs + i].path_weight = nhs[i].path_weight;
384 /* Try larger and larger power of 2 sized adjacency blocks until we
385 find one where traffic flows to within 1% of specified weights. */
386 for (n_adj = max_pow2 (n_nhs); ; n_adj *= 2)
390 norm = n_adj / ((f64) sum_weight);
392 for (i = 0; i < n_nhs; i++)
394 f64 nf = nhs[n_nhs + i].path_weight * norm; /* use saved weights */
395 word n = flt_round_nearest (nf);
397 n = n > n_adj_left ? n_adj_left : n;
399 error += fabs (nf - n);
400 nhs[i].path_weight = n;
402 if (0 == nhs[i].path_weight)
405 * when the weight skew is high (norm is small) and n == nf.
406 * without this correction the path with a low weight would have
407 * no represenation in the load-balanace - don't want that.
408 * If the weight skew is high so the load-balance has many buckets
409 * to allow it. pays ya money takes ya choice.
416 nhs[0].path_weight += n_adj_left;
418 /* Less than 5% average error per adjacency with this size adjacency block? */
419 if (error <= multipath_next_hop_error_tolerance*n_adj)
421 /* Truncate any next hops with zero weight. */
428 /* Save vector for next call. */
429 *normalized_next_hops = nhs;
430 *sum_weight_in = sum_weight;
434 static load_balance_path_t *
435 load_balance_multipath_next_hop_fixup (const load_balance_path_t *nhs,
436 dpo_proto_t drop_proto)
438 if (0 == vec_len(nhs))
440 load_balance_path_t *new_nhs = NULL, *nh;
443 * we need something for the load-balance. so use the drop
445 vec_add2(new_nhs, nh, 1);
448 dpo_copy(&nh->path_dpo, drop_dpo_get(drop_proto));
457 * Fill in adjacencies in block based on corresponding
458 * next hop adjacencies.
461 load_balance_fill_buckets (load_balance_t *lb,
462 load_balance_path_t *nhs,
466 load_balance_path_t * nh;
472 * the next-hops have normalised weights. that means their sum is the number
473 * of buckets we need to fill.
475 vec_foreach (nh, nhs)
477 for (ii = 0; ii < nh->path_weight; ii++)
479 ASSERT(bucket < n_buckets);
480 load_balance_set_bucket_i(lb, bucket++, buckets, &nh->path_dpo);
486 load_balance_set_n_buckets (load_balance_t *lb,
489 lb->lb_n_buckets = n_buckets;
490 lb->lb_n_buckets_minus_1 = n_buckets-1;
494 load_balance_multipath_update (const dpo_id_t *dpo,
495 const load_balance_path_t * raw_nhs,
496 load_balance_flags_t flags)
498 load_balance_path_t *nh, *nhs, *fixed_nhs;
499 u32 sum_of_weights, n_buckets, ii;
500 index_t lbmi, old_lbmi;
506 ASSERT(DPO_LOAD_BALANCE == dpo->dpoi_type);
507 lb = load_balance_get(dpo->dpoi_index);
508 fixed_nhs = load_balance_multipath_next_hop_fixup(raw_nhs, lb->lb_proto);
510 ip_multipath_normalize_next_hops((NULL == fixed_nhs ?
515 multipath_next_hop_error_tolerance);
517 ASSERT (n_buckets >= vec_len (raw_nhs));
520 * Save the old load-balance map used, and get a new one if required.
522 old_lbmi = lb->lb_map;
523 if (flags & LOAD_BALANCE_FLAG_USES_MAP)
525 lbmi = load_balance_map_add_or_lock(n_buckets, sum_of_weights, nhs);
529 lbmi = INDEX_INVALID;
532 if (0 == lb->lb_n_buckets)
535 * first time initialisation. no packets inflight, so we can write
538 load_balance_set_n_buckets(lb, n_buckets);
540 if (!LB_HAS_INLINE_BUCKETS(lb))
541 vec_validate_aligned(lb->lb_buckets,
542 lb->lb_n_buckets - 1,
543 CLIB_CACHE_LINE_BYTES);
545 load_balance_fill_buckets(lb, nhs,
546 load_balance_get_buckets(lb),
553 * This is a modification of an existing load-balance.
554 * We need to ensure that packets inflight see a consistent state, that
555 * is the number of reported buckets the LB has (read from
556 * lb_n_buckets_minus_1) is not more than it actually has. So if the
557 * number of buckets is increasing, we must update the bucket array first,
558 * then the reported number. vice-versa if the number of buckets goes down.
560 if (n_buckets == lb->lb_n_buckets)
563 * no change in the number of buckets. we can simply fill what
564 * is new over what is old.
566 load_balance_fill_buckets(lb, nhs,
567 load_balance_get_buckets(lb),
571 else if (n_buckets > lb->lb_n_buckets)
574 * we have more buckets. the old load-balance map (if there is one)
575 * will remain valid, i.e. mapping to indices within range, so we
578 if (n_buckets > LB_NUM_INLINE_BUCKETS &&
579 lb->lb_n_buckets <= LB_NUM_INLINE_BUCKETS)
582 * the new increased number of buckets is crossing the threshold
583 * from the inline storage to out-line. Alloc the outline buckets
584 * first, then fixup the number. then reset the inlines.
586 ASSERT(NULL == lb->lb_buckets);
587 vec_validate_aligned(lb->lb_buckets,
589 CLIB_CACHE_LINE_BYTES);
591 load_balance_fill_buckets(lb, nhs,
594 CLIB_MEMORY_BARRIER();
595 load_balance_set_n_buckets(lb, n_buckets);
597 CLIB_MEMORY_BARRIER();
599 for (ii = 0; ii < LB_NUM_INLINE_BUCKETS; ii++)
601 dpo_reset(&lb->lb_buckets_inline[ii]);
606 if (n_buckets <= LB_NUM_INLINE_BUCKETS)
609 * we are not crossing the threshold and it's still inline buckets.
610 * we can write the new on the old..
612 load_balance_fill_buckets(lb, nhs,
613 load_balance_get_buckets(lb),
615 CLIB_MEMORY_BARRIER();
616 load_balance_set_n_buckets(lb, n_buckets);
621 * we are not crossing the threshold. We need a new bucket array to
622 * hold the increased number of choices.
624 dpo_id_t *new_buckets, *old_buckets, *tmp_dpo;
627 old_buckets = load_balance_get_buckets(lb);
629 vec_validate_aligned(new_buckets,
631 CLIB_CACHE_LINE_BYTES);
633 load_balance_fill_buckets(lb, nhs, new_buckets, n_buckets);
634 CLIB_MEMORY_BARRIER();
635 lb->lb_buckets = new_buckets;
636 CLIB_MEMORY_BARRIER();
637 load_balance_set_n_buckets(lb, n_buckets);
639 vec_foreach(tmp_dpo, old_buckets)
643 vec_free(old_buckets);
648 * buckets fixed. ready for the MAP update.
655 * bucket size shrinkage.
656 * Any map we have will be based on the old
657 * larger number of buckets, so will be translating to indices
658 * out of range. So the new MAP must be installed first.
661 CLIB_MEMORY_BARRIER();
664 if (n_buckets <= LB_NUM_INLINE_BUCKETS &&
665 lb->lb_n_buckets > LB_NUM_INLINE_BUCKETS)
668 * the new decreased number of buckets is crossing the threshold
669 * from out-line storage to inline:
670 * 1 - Fill the inline buckets,
671 * 2 - fixup the number (and this point the inline buckets are
673 * 3 - free the outline buckets
675 load_balance_fill_buckets(lb, nhs,
676 lb->lb_buckets_inline,
678 CLIB_MEMORY_BARRIER();
679 load_balance_set_n_buckets(lb, n_buckets);
680 CLIB_MEMORY_BARRIER();
682 vec_foreach(tmp_dpo, lb->lb_buckets)
686 vec_free(lb->lb_buckets);
691 * not crossing the threshold.
692 * 1 - update the number to the smaller size
693 * 2 - write the new buckets
694 * 3 - reset those no longer used.
699 old_n_buckets = lb->lb_n_buckets;
700 buckets = load_balance_get_buckets(lb);
702 load_balance_set_n_buckets(lb, n_buckets);
703 CLIB_MEMORY_BARRIER();
705 load_balance_fill_buckets(lb, nhs,
709 for (ii = n_buckets; ii < old_n_buckets; ii++)
711 dpo_reset(&buckets[ii]);
717 vec_foreach (nh, nhs)
719 dpo_reset(&nh->path_dpo);
724 load_balance_map_unlock(old_lbmi);
728 load_balance_lock (dpo_id_t *dpo)
732 lb = load_balance_get(dpo->dpoi_index);
738 load_balance_destroy (load_balance_t *lb)
743 buckets = load_balance_get_buckets(lb);
745 for (i = 0; i < lb->lb_n_buckets; i++)
747 dpo_reset(&buckets[i]);
750 LB_DBG(lb, "destroy");
751 if (!LB_HAS_INLINE_BUCKETS(lb))
753 vec_free(lb->lb_buckets);
756 fib_urpf_list_unlock(lb->lb_urpf);
757 load_balance_map_unlock(lb->lb_map);
759 pool_put(load_balance_pool, lb);
763 load_balance_unlock (dpo_id_t *dpo)
767 lb = load_balance_get(dpo->dpoi_index);
771 if (0 == lb->lb_locks)
773 load_balance_destroy(lb);
778 load_balance_mem_show (void)
780 fib_show_memory_usage("load-balance",
781 pool_elts(load_balance_pool),
782 pool_len(load_balance_pool),
783 sizeof(load_balance_t));
784 load_balance_map_show_mem();
787 const static dpo_vft_t lb_vft = {
788 .dv_lock = load_balance_lock,
789 .dv_unlock = load_balance_unlock,
790 .dv_format = format_load_balance_dpo,
791 .dv_mem_show = load_balance_mem_show,
795 * @brief The per-protocol VLIB graph nodes that are assigned to a load-balance
798 * this means that these graph nodes are ones from which a load-balance is the
799 * parent object in the DPO-graph.
801 * We do not list all the load-balance nodes, such as the *-lookup. instead
802 * we are relying on the correct use of the .sibling_of field when setting
803 * up these sibling nodes.
805 const static char* const load_balance_ip4_nodes[] =
810 const static char* const load_balance_ip6_nodes[] =
815 const static char* const load_balance_mpls_nodes[] =
820 const static char* const load_balance_l2_nodes[] =
825 const static char* const load_balance_nsh_nodes[] =
830 const static char* const load_balance_bier_nodes[] =
835 const static char* const * const load_balance_nodes[DPO_PROTO_NUM] =
837 [DPO_PROTO_IP4] = load_balance_ip4_nodes,
838 [DPO_PROTO_IP6] = load_balance_ip6_nodes,
839 [DPO_PROTO_MPLS] = load_balance_mpls_nodes,
840 [DPO_PROTO_ETHERNET] = load_balance_l2_nodes,
841 [DPO_PROTO_NSH] = load_balance_nsh_nodes,
842 [DPO_PROTO_BIER] = load_balance_bier_nodes,
846 load_balance_module_init (void)
850 dpo_register(DPO_LOAD_BALANCE, &lb_vft, load_balance_nodes);
853 * Special LB with index zero. we need to define this since the v4 mtrie
854 * assumes an index of 0 implies the ply is empty. therefore all 'real'
855 * adjs need a non-zero index.
856 * This should never be used, but just in case, stack it on a drop.
858 lbi = load_balance_create(1, DPO_PROTO_IP4, 0);
859 load_balance_set_bucket(lbi, 0, drop_dpo_get(DPO_PROTO_IP4));
861 load_balance_map_module_init();
864 static clib_error_t *
865 load_balance_show (vlib_main_t * vm,
866 unformat_input_t * input,
867 vlib_cli_command_t * cmd)
869 index_t lbi = INDEX_INVALID;
871 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
873 if (unformat (input, "%d", &lbi))
879 if (INDEX_INVALID != lbi)
881 vlib_cli_output (vm, "%U", format_load_balance, lbi,
882 LOAD_BALANCE_FORMAT_DETAIL);
888 pool_foreach(lb, load_balance_pool,
890 vlib_cli_output (vm, "%U", format_load_balance,
891 load_balance_get_index(lb),
892 LOAD_BALANCE_FORMAT_NONE);
899 VLIB_CLI_COMMAND (load_balance_show_command, static) = {
900 .path = "show load-balance",
901 .short_help = "show load-balance [<index>]",
902 .function = load_balance_show,
907 ip_flow_hash (void *data)
909 ip4_header_t *iph = (ip4_header_t *) data;
911 if ((iph->ip_version_and_header_length & 0xF0) == 0x40)
912 return ip4_compute_flow_hash (iph, IP_FLOW_HASH_DEFAULT);
914 return ip6_compute_flow_hash ((ip6_header_t *) iph, IP_FLOW_HASH_DEFAULT);
920 return (*((u64 *) m) & 0xffffffffffff);
924 l2_flow_hash (vlib_buffer_t * b0)
926 ethernet_header_t *eh;
928 uword is_ip, eh_size;
931 eh = vlib_buffer_get_current (b0);
932 eh_type = clib_net_to_host_u16 (eh->type);
933 eh_size = ethernet_buffer_header_size (b0);
935 is_ip = (eh_type == ETHERNET_TYPE_IP4 || eh_type == ETHERNET_TYPE_IP6);
937 /* since we have 2 cache lines, use them */
939 a = ip_flow_hash ((u8 *) vlib_buffer_get_current (b0) + eh_size);
943 b = mac_to_u64 ((u8 *) eh->dst_address);
944 c = mac_to_u64 ((u8 *) eh->src_address);
945 hash_mix64 (a, b, c);
950 typedef struct load_balance_trace_t_
953 } load_balance_trace_t;
956 load_balance_inline (vlib_main_t * vm,
957 vlib_node_runtime_t * node,
958 vlib_frame_t * frame,
961 u32 n_left_from, next_index, *from, *to_next;
963 from = vlib_frame_vector_args (frame);
964 n_left_from = frame->n_vectors;
966 next_index = node->cached_next_index;
968 while (n_left_from > 0)
972 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
974 while (n_left_from > 0 && n_left_to_next > 0)
977 u32 bi0, lbi0, next0;
978 const dpo_id_t *dpo0;
979 const load_balance_t *lb0;
988 b0 = vlib_get_buffer (vm, bi0);
990 /* lookup dst + src mac */
991 lbi0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
992 lb0 = load_balance_get(lbi0);
996 vnet_buffer(b0)->ip.flow_hash = l2_flow_hash(b0);
1001 const bier_hdr_t *bh0 = vlib_buffer_get_current(b0);
1002 vnet_buffer(b0)->ip.flow_hash = bier_hdr_get_entropy(bh0);
1005 dpo0 = load_balance_get_bucket_i(lb0,
1006 vnet_buffer(b0)->ip.flow_hash &
1007 (lb0->lb_n_buckets_minus_1));
1009 next0 = dpo0->dpoi_next_node;
1010 vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
1012 if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
1014 load_balance_trace_t *tr = vlib_add_trace (vm, node, b0,
1016 tr->lb_index = lbi0;
1018 vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
1019 n_left_to_next, bi0, next0);
1022 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1025 return frame->n_vectors;
1029 l2_load_balance (vlib_main_t * vm,
1030 vlib_node_runtime_t * node,
1031 vlib_frame_t * frame)
1033 return (load_balance_inline(vm, node, frame, 1));
1037 format_l2_load_balance_trace (u8 * s, va_list * args)
1039 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1040 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1041 load_balance_trace_t *t = va_arg (*args, load_balance_trace_t *);
1043 s = format (s, "L2-load-balance: index %d", t->lb_index);
1050 VLIB_REGISTER_NODE (l2_load_balance_node) = {
1051 .function = l2_load_balance,
1052 .name = "l2-load-balance",
1053 .vector_size = sizeof (u32),
1055 .format_trace = format_l2_load_balance_trace,
1063 nsh_load_balance (vlib_main_t * vm,
1064 vlib_node_runtime_t * node,
1065 vlib_frame_t * frame)
1067 u32 n_left_from, next_index, *from, *to_next;
1069 from = vlib_frame_vector_args (frame);
1070 n_left_from = frame->n_vectors;
1072 next_index = node->cached_next_index;
1074 while (n_left_from > 0)
1078 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1080 while (n_left_from > 0 && n_left_to_next > 0)
1083 u32 bi0, lbi0, next0, *nsh0;
1084 const dpo_id_t *dpo0;
1085 const load_balance_t *lb0;
1092 n_left_to_next -= 1;
1094 b0 = vlib_get_buffer (vm, bi0);
1096 lbi0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
1097 lb0 = load_balance_get(lbi0);
1099 /* SPI + SI are the second word of the NSH header */
1100 nsh0 = vlib_buffer_get_current (b0);
1101 vnet_buffer(b0)->ip.flow_hash = nsh0[1] % lb0->lb_n_buckets;
1103 dpo0 = load_balance_get_bucket_i(lb0,
1104 vnet_buffer(b0)->ip.flow_hash &
1105 (lb0->lb_n_buckets_minus_1));
1107 next0 = dpo0->dpoi_next_node;
1108 vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
1110 if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
1112 load_balance_trace_t *tr = vlib_add_trace (vm, node, b0,
1114 tr->lb_index = lbi0;
1116 vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
1117 n_left_to_next, bi0, next0);
1120 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1123 return frame->n_vectors;
1127 format_nsh_load_balance_trace (u8 * s, va_list * args)
1129 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1130 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1131 load_balance_trace_t *t = va_arg (*args, load_balance_trace_t *);
1133 s = format (s, "NSH-load-balance: index %d", t->lb_index);
1140 VLIB_REGISTER_NODE (nsh_load_balance_node) = {
1141 .function = nsh_load_balance,
1142 .name = "nsh-load-balance",
1143 .vector_size = sizeof (u32),
1145 .format_trace = format_nsh_load_balance_trace,
1153 format_bier_load_balance_trace (u8 * s, va_list * args)
1155 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1156 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1157 load_balance_trace_t *t = va_arg (*args, load_balance_trace_t *);
1159 s = format (s, "BIER-load-balance: index %d", t->lb_index);
1164 bier_load_balance (vlib_main_t * vm,
1165 vlib_node_runtime_t * node,
1166 vlib_frame_t * frame)
1168 return (load_balance_inline(vm, node, frame, 0));
1174 VLIB_REGISTER_NODE (bier_load_balance_node) = {
1175 .function = bier_load_balance,
1176 .name = "bier-load-balance",
1177 .vector_size = sizeof (u32),
1179 .format_trace = format_bier_load_balance_trace,
1180 .sibling_of = "mpls-load-balance",