2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vnet/ip/lookup.h>
17 #include <vnet/dpo/load_balance.h>
18 #include <vnet/dpo/load_balance_map.h>
19 #include <vnet/dpo/drop_dpo.h>
20 #include <vppinfra/math.h> /* for fabs */
21 #include <vnet/adj/adj.h>
22 #include <vnet/adj/adj_internal.h>
23 #include <vnet/fib/fib_urpf_list.h>
24 #include <vnet/bier/bier_fwd.h>
25 #include <vnet/fib/mpls_fib.h>
28 * distribution error tolerance for load-balancing
30 const f64 multipath_next_hop_error_tolerance = 0.1;
32 static const char *load_balance_attr_names[] = LOAD_BALANCE_ATTR_NAMES;
37 vlib_log_class_t load_balance_logger;
39 #define LB_DBG(_lb, _fmt, _args...) \
41 vlib_log_debug(load_balance_logger, \
43 format_load_balance, load_balance_get_index(_lb), \
44 LOAD_BALANCE_FORMAT_NONE, \
49 * Pool of all DPOs. It's not static so the DP can have fast access
51 load_balance_t *load_balance_pool;
54 * The one instance of load-balance main
56 load_balance_main_t load_balance_main = {
59 .stat_segment_name = "/net/route/to",
63 .stat_segment_name = "/net/route/via",
68 load_balance_get_multipath_tolerance (void)
70 return (multipath_next_hop_error_tolerance);
74 load_balance_get_index (const load_balance_t *lb)
76 return (lb - load_balance_pool);
79 static inline dpo_id_t*
80 load_balance_get_buckets (load_balance_t *lb)
82 if (LB_HAS_INLINE_BUCKETS(lb))
84 return (lb->lb_buckets_inline);
88 return (lb->lb_buckets);
92 static load_balance_t *
93 load_balance_alloc_i (void)
96 u8 need_barrier_sync = 0;
97 vlib_main_t *vm = vlib_get_main();
98 ASSERT (vm->thread_index == 0);
100 pool_get_aligned_will_expand (load_balance_pool, need_barrier_sync,
101 CLIB_CACHE_LINE_BYTES);
102 if (need_barrier_sync)
103 vlib_worker_thread_barrier_sync (vm);
105 pool_get_aligned(load_balance_pool, lb, CLIB_CACHE_LINE_BYTES);
106 clib_memset(lb, 0, sizeof(*lb));
108 lb->lb_map = INDEX_INVALID;
109 lb->lb_urpf = INDEX_INVALID;
111 if (need_barrier_sync == 0)
113 need_barrier_sync += vlib_validate_combined_counter_will_expand
114 (&(load_balance_main.lbm_to_counters),
115 load_balance_get_index(lb));
116 need_barrier_sync += vlib_validate_combined_counter_will_expand
117 (&(load_balance_main.lbm_via_counters),
118 load_balance_get_index(lb));
119 if (need_barrier_sync)
120 vlib_worker_thread_barrier_sync (vm);
123 vlib_validate_combined_counter(&(load_balance_main.lbm_to_counters),
124 load_balance_get_index(lb));
125 vlib_validate_combined_counter(&(load_balance_main.lbm_via_counters),
126 load_balance_get_index(lb));
127 vlib_zero_combined_counter(&(load_balance_main.lbm_to_counters),
128 load_balance_get_index(lb));
129 vlib_zero_combined_counter(&(load_balance_main.lbm_via_counters),
130 load_balance_get_index(lb));
132 if (need_barrier_sync)
133 vlib_worker_thread_barrier_release (vm);
139 load_balance_format (index_t lbi,
140 load_balance_format_flags_t flags,
144 vlib_counter_t to, via;
149 lb = load_balance_get(lbi);
150 vlib_get_combined_counter(&(load_balance_main.lbm_to_counters), lbi, &to);
151 vlib_get_combined_counter(&(load_balance_main.lbm_via_counters), lbi, &via);
152 buckets = load_balance_get_buckets(lb);
154 s = format(s, "%U: ", format_dpo_type, DPO_LOAD_BALANCE);
155 s = format(s, "[proto:%U ", format_dpo_proto, lb->lb_proto);
156 s = format(s, "index:%d buckets:%d ", lbi, lb->lb_n_buckets);
157 s = format(s, "uRPF:%d ", lb->lb_urpf);
160 load_balance_attr_t attr;
162 s = format(s, "flags:[");
164 FOR_EACH_LOAD_BALANCE_ATTR(attr)
166 if (lb->lb_flags & (1 << attr))
168 s = format (s, "%s", load_balance_attr_names[attr]);
173 s = format(s, "to:[%Ld:%Ld]", to.packets, to.bytes);
174 if (0 != via.packets)
176 s = format(s, " via:[%Ld:%Ld]",
177 via.packets, via.bytes);
181 if (INDEX_INVALID != lb->lb_map)
183 s = format(s, "\n%U%U",
184 format_white_space, indent+4,
185 format_load_balance_map, lb->lb_map, indent+4);
187 for (i = 0; i < lb->lb_n_buckets; i++)
189 s = format(s, "\n%U[%d] %U",
190 format_white_space, indent+2,
193 &buckets[i], indent+6);
199 format_load_balance (u8 * s, va_list * args)
201 index_t lbi = va_arg(*args, index_t);
202 load_balance_format_flags_t flags = va_arg(*args, load_balance_format_flags_t);
204 return (load_balance_format(lbi, flags, 0, s));
208 format_load_balance_dpo (u8 * s, va_list * args)
210 index_t lbi = va_arg(*args, index_t);
211 u32 indent = va_arg(*args, u32);
213 return (load_balance_format(lbi, LOAD_BALANCE_FORMAT_DETAIL, indent, s));
217 load_balance_get_default_flow_hash (dpo_proto_t lb_proto)
223 return (IP_FLOW_HASH_DEFAULT);
226 return (MPLS_FLOW_HASH_DEFAULT);
228 case DPO_PROTO_ETHERNET:
237 static load_balance_t *
238 load_balance_create_i (u32 num_buckets,
239 dpo_proto_t lb_proto,
240 flow_hash_config_t fhc)
244 lb = load_balance_alloc_i();
245 lb->lb_hash_config = fhc;
246 lb->lb_n_buckets = num_buckets;
247 lb->lb_n_buckets_minus_1 = num_buckets-1;
248 lb->lb_proto = lb_proto;
250 if (!LB_HAS_INLINE_BUCKETS(lb))
252 vec_validate_aligned(lb->lb_buckets,
253 lb->lb_n_buckets - 1,
254 CLIB_CACHE_LINE_BYTES);
257 LB_DBG(lb, "create");
263 load_balance_create (u32 n_buckets,
264 dpo_proto_t lb_proto,
265 flow_hash_config_t fhc)
267 return (load_balance_get_index(load_balance_create_i(n_buckets, lb_proto, fhc)));
271 load_balance_set_bucket_i (load_balance_t *lb,
274 const dpo_id_t *next)
276 dpo_stack(DPO_LOAD_BALANCE, lb->lb_proto, &buckets[bucket], next);
280 load_balance_set_bucket (index_t lbi,
282 const dpo_id_t *next)
287 lb = load_balance_get(lbi);
288 buckets = load_balance_get_buckets(lb);
290 ASSERT(bucket < lb->lb_n_buckets);
292 load_balance_set_bucket_i(lb, bucket, buckets, next);
296 load_balance_is_drop (const dpo_id_t *dpo)
300 if (DPO_LOAD_BALANCE != dpo->dpoi_type)
303 lb = load_balance_get(dpo->dpoi_index);
305 if (1 == lb->lb_n_buckets)
307 return (dpo_is_drop(load_balance_get_bucket_i(lb, 0)));
313 load_balance_n_buckets (index_t lbi)
317 lb = load_balance_get(lbi);
319 return (lb->lb_n_buckets);
323 load_balance_set_fib_entry_flags (index_t lbi,
324 fib_entry_flag_t flags)
328 lb = load_balance_get(lbi);
329 lb->lb_fib_entry_flags = flags;
334 load_balance_set_urpf (index_t lbi,
340 lb = load_balance_get(lbi);
343 * packets in flight we see this change. but it's atomic, so :P
348 fib_urpf_list_unlock(old);
349 fib_urpf_list_lock(urpf);
353 load_balance_get_urpf (index_t lbi)
357 lb = load_balance_get(lbi);
359 return (lb->lb_urpf);
363 load_balance_get_bucket (index_t lbi,
368 lb = load_balance_get(lbi);
370 return (load_balance_get_bucket_i(lb, bucket));
374 next_hop_sort_by_weight (const load_balance_path_t * n1,
375 const load_balance_path_t * n2)
377 return ((int) n1->path_weight - (int) n2->path_weight);
380 /* Given next hop vector is over-written with normalized one with sorted weights and
381 with weights corresponding to the number of adjacencies for each next hop.
382 Returns number of adjacencies in block. */
384 ip_multipath_normalize_next_hops (const load_balance_path_t * raw_next_hops,
385 load_balance_path_t ** normalized_next_hops,
387 f64 multipath_next_hop_error_tolerance)
389 load_balance_path_t * nhs;
390 uword n_nhs, n_adj, n_adj_left, i, sum_weight;
393 n_nhs = vec_len (raw_next_hops);
398 /* Allocate enough space for 2 copies; we'll use second copy to save original weights. */
399 nhs = *normalized_next_hops;
400 vec_validate (nhs, 2*n_nhs - 1);
402 /* Fast path: 1 next hop in block. */
406 nhs[0] = raw_next_hops[0];
407 nhs[0].path_weight = 1;
415 int cmp = next_hop_sort_by_weight (&raw_next_hops[0], &raw_next_hops[1]) < 0;
418 nhs[0] = raw_next_hops[cmp];
419 nhs[1] = raw_next_hops[cmp ^ 1];
421 /* Fast path: equal cost multipath with 2 next hops. */
422 if (nhs[0].path_weight == nhs[1].path_weight)
424 nhs[0].path_weight = nhs[1].path_weight = 1;
432 clib_memcpy_fast (nhs, raw_next_hops, n_nhs * sizeof (raw_next_hops[0]));
433 qsort (nhs, n_nhs, sizeof (nhs[0]), (void *) next_hop_sort_by_weight);
436 /* Find total weight to normalize weights. */
438 for (i = 0; i < n_nhs; i++)
439 sum_weight += nhs[i].path_weight;
441 /* In the unlikely case that all weights are given as 0, set them all to 1. */
444 for (i = 0; i < n_nhs; i++)
445 nhs[i].path_weight = 1;
449 /* Save copies of all next hop weights to avoid being overwritten in loop below. */
450 for (i = 0; i < n_nhs; i++)
451 nhs[n_nhs + i].path_weight = nhs[i].path_weight;
453 /* Try larger and larger power of 2 sized adjacency blocks until we
454 find one where traffic flows to within 1% of specified weights. */
455 for (n_adj = max_pow2 (n_nhs); ; n_adj *= 2)
459 norm = n_adj / ((f64) sum_weight);
461 for (i = 0; i < n_nhs; i++)
463 f64 nf = nhs[n_nhs + i].path_weight * norm; /* use saved weights */
464 word n = flt_round_nearest (nf);
466 n = n > n_adj_left ? n_adj_left : n;
468 error += fabs (nf - n);
469 nhs[i].path_weight = n;
471 if (0 == nhs[i].path_weight)
474 * when the weight skew is high (norm is small) and n == nf.
475 * without this correction the path with a low weight would have
476 * no representation in the load-balanace - don't want that.
477 * If the weight skew is high so the load-balance has many buckets
478 * to allow it. pays ya money takes ya choice.
485 nhs[0].path_weight += n_adj_left;
487 /* Less than 5% average error per adjacency with this size adjacency block? */
488 if (error <= multipath_next_hop_error_tolerance*n_adj)
490 /* Truncate any next hops with zero weight. */
497 /* Save vector for next call. */
498 *normalized_next_hops = nhs;
499 *sum_weight_in = sum_weight;
503 static load_balance_path_t *
504 load_balance_multipath_next_hop_fixup (const load_balance_path_t *nhs,
505 dpo_proto_t drop_proto)
507 if (0 == vec_len(nhs))
509 load_balance_path_t *new_nhs = NULL, *nh;
512 * we need something for the load-balance. so use the drop
514 vec_add2(new_nhs, nh, 1);
517 dpo_copy(&nh->path_dpo, drop_dpo_get(drop_proto));
526 * Fill in adjacencies in block based on corresponding
527 * next hop adjacencies.
530 load_balance_fill_buckets_norm (load_balance_t *lb,
531 load_balance_path_t *nhs,
535 load_balance_path_t *nh;
541 * the next-hops have normalised weights. that means their sum is the number
542 * of buckets we need to fill.
544 vec_foreach (nh, nhs)
546 for (ii = 0; ii < nh->path_weight; ii++)
548 ASSERT(bucket < n_buckets);
549 load_balance_set_bucket_i(lb, bucket++, buckets, &nh->path_dpo);
554 load_balance_fill_buckets_sticky (load_balance_t *lb,
555 load_balance_path_t *nhs,
559 load_balance_path_t *nh, *fwding_paths;
560 u16 ii, bucket, fpath;
565 vec_foreach (nh, nhs)
567 if (!dpo_is_drop(&nh->path_dpo))
569 vec_add1(fwding_paths, *nh);
572 if (vec_len(fwding_paths) == 0)
573 fwding_paths = vec_dup(nhs);
576 * the next-hops have normalised weights. that means their sum is the number
577 * of buckets we need to fill.
579 vec_foreach (nh, nhs)
581 for (ii = 0; ii < nh->path_weight; ii++)
583 ASSERT(bucket < n_buckets);
584 if (!dpo_is_drop(&nh->path_dpo))
586 load_balance_set_bucket_i(lb, bucket++, buckets, &nh->path_dpo);
590 /* fill the bucks from the next up path */
591 load_balance_set_bucket_i(lb, bucket++, buckets, &fwding_paths[fpath].path_dpo);
592 fpath = (fpath + 1) % vec_len(fwding_paths);
597 vec_free(fwding_paths);
601 load_balance_fill_buckets (load_balance_t *lb,
602 load_balance_path_t *nhs,
605 load_balance_flags_t flags)
607 if (flags & LOAD_BALANCE_FLAG_STICKY)
609 load_balance_fill_buckets_sticky(lb, nhs, buckets, n_buckets);
613 load_balance_fill_buckets_norm(lb, nhs, buckets, n_buckets);
618 load_balance_set_n_buckets (load_balance_t *lb,
621 lb->lb_n_buckets = n_buckets;
622 lb->lb_n_buckets_minus_1 = n_buckets-1;
626 load_balance_multipath_update (const dpo_id_t *dpo,
627 const load_balance_path_t * raw_nhs,
628 load_balance_flags_t flags)
630 load_balance_path_t *nh, *nhs, *fixed_nhs;
631 u32 sum_of_weights, n_buckets, ii;
632 index_t lbmi, old_lbmi;
638 ASSERT(DPO_LOAD_BALANCE == dpo->dpoi_type);
639 lb = load_balance_get(dpo->dpoi_index);
640 lb->lb_flags = flags;
641 fixed_nhs = load_balance_multipath_next_hop_fixup(raw_nhs, lb->lb_proto);
643 ip_multipath_normalize_next_hops((NULL == fixed_nhs ?
648 multipath_next_hop_error_tolerance);
650 ASSERT (n_buckets >= vec_len (raw_nhs));
653 * Save the old load-balance map used, and get a new one if required.
655 old_lbmi = lb->lb_map;
656 if (flags & LOAD_BALANCE_FLAG_USES_MAP)
658 lbmi = load_balance_map_add_or_lock(n_buckets, sum_of_weights, nhs);
662 lbmi = INDEX_INVALID;
665 if (0 == lb->lb_n_buckets)
668 * first time initialisation. no packets inflight, so we can write
671 load_balance_set_n_buckets(lb, n_buckets);
673 if (!LB_HAS_INLINE_BUCKETS(lb))
674 vec_validate_aligned(lb->lb_buckets,
675 lb->lb_n_buckets - 1,
676 CLIB_CACHE_LINE_BYTES);
678 load_balance_fill_buckets(lb, nhs,
679 load_balance_get_buckets(lb),
686 * This is a modification of an existing load-balance.
687 * We need to ensure that packets inflight see a consistent state, that
688 * is the number of reported buckets the LB has (read from
689 * lb_n_buckets_minus_1) is not more than it actually has. So if the
690 * number of buckets is increasing, we must update the bucket array first,
691 * then the reported number. vice-versa if the number of buckets goes down.
693 if (n_buckets == lb->lb_n_buckets)
696 * no change in the number of buckets. we can simply fill what
697 * is new over what is old.
699 load_balance_fill_buckets(lb, nhs,
700 load_balance_get_buckets(lb),
704 else if (n_buckets > lb->lb_n_buckets)
707 * we have more buckets. the old load-balance map (if there is one)
708 * will remain valid, i.e. mapping to indices within range, so we
711 if (n_buckets > LB_NUM_INLINE_BUCKETS &&
712 lb->lb_n_buckets <= LB_NUM_INLINE_BUCKETS)
715 * the new increased number of buckets is crossing the threshold
716 * from the inline storage to out-line. Alloc the outline buckets
717 * first, then fixup the number. then reset the inlines.
719 ASSERT(NULL == lb->lb_buckets);
720 vec_validate_aligned(lb->lb_buckets,
722 CLIB_CACHE_LINE_BYTES);
724 load_balance_fill_buckets(lb, nhs,
727 CLIB_MEMORY_BARRIER();
728 load_balance_set_n_buckets(lb, n_buckets);
730 CLIB_MEMORY_BARRIER();
732 for (ii = 0; ii < LB_NUM_INLINE_BUCKETS; ii++)
734 dpo_reset(&lb->lb_buckets_inline[ii]);
739 if (n_buckets <= LB_NUM_INLINE_BUCKETS)
742 * we are not crossing the threshold and it's still inline buckets.
743 * we can write the new on the old..
745 load_balance_fill_buckets(lb, nhs,
746 load_balance_get_buckets(lb),
748 CLIB_MEMORY_BARRIER();
749 load_balance_set_n_buckets(lb, n_buckets);
754 * we are not crossing the threshold. We need a new bucket array to
755 * hold the increased number of choices.
757 dpo_id_t *new_buckets, *old_buckets, *tmp_dpo;
760 old_buckets = load_balance_get_buckets(lb);
762 vec_validate_aligned(new_buckets,
764 CLIB_CACHE_LINE_BYTES);
766 load_balance_fill_buckets(lb, nhs, new_buckets,
768 CLIB_MEMORY_BARRIER();
769 lb->lb_buckets = new_buckets;
770 CLIB_MEMORY_BARRIER();
771 load_balance_set_n_buckets(lb, n_buckets);
773 vec_foreach(tmp_dpo, old_buckets)
777 vec_free(old_buckets);
782 * buckets fixed. ready for the MAP update.
789 * bucket size shrinkage.
790 * Any map we have will be based on the old
791 * larger number of buckets, so will be translating to indices
792 * out of range. So the new MAP must be installed first.
795 CLIB_MEMORY_BARRIER();
798 if (n_buckets <= LB_NUM_INLINE_BUCKETS &&
799 lb->lb_n_buckets > LB_NUM_INLINE_BUCKETS)
802 * the new decreased number of buckets is crossing the threshold
803 * from out-line storage to inline:
804 * 1 - Fill the inline buckets,
805 * 2 - fixup the number (and this point the inline buckets are
807 * 3 - free the outline buckets
809 load_balance_fill_buckets(lb, nhs,
810 lb->lb_buckets_inline,
812 CLIB_MEMORY_BARRIER();
813 load_balance_set_n_buckets(lb, n_buckets);
814 CLIB_MEMORY_BARRIER();
816 vec_foreach(tmp_dpo, lb->lb_buckets)
820 vec_free(lb->lb_buckets);
825 * not crossing the threshold.
826 * 1 - update the number to the smaller size
827 * 2 - write the new buckets
828 * 3 - reset those no longer used.
833 old_n_buckets = lb->lb_n_buckets;
834 buckets = load_balance_get_buckets(lb);
836 load_balance_set_n_buckets(lb, n_buckets);
837 CLIB_MEMORY_BARRIER();
839 load_balance_fill_buckets(lb, nhs, buckets,
842 for (ii = n_buckets; ii < old_n_buckets; ii++)
844 dpo_reset(&buckets[ii]);
850 vec_foreach (nh, nhs)
852 dpo_reset(&nh->path_dpo);
857 load_balance_map_unlock(old_lbmi);
861 load_balance_lock (dpo_id_t *dpo)
865 lb = load_balance_get(dpo->dpoi_index);
871 load_balance_destroy (load_balance_t *lb)
876 buckets = load_balance_get_buckets(lb);
878 for (i = 0; i < lb->lb_n_buckets; i++)
880 dpo_reset(&buckets[i]);
883 LB_DBG(lb, "destroy");
884 if (!LB_HAS_INLINE_BUCKETS(lb))
886 vec_free(lb->lb_buckets);
889 fib_urpf_list_unlock(lb->lb_urpf);
890 load_balance_map_unlock(lb->lb_map);
892 pool_put(load_balance_pool, lb);
896 load_balance_unlock (dpo_id_t *dpo)
900 lb = load_balance_get(dpo->dpoi_index);
904 if (0 == lb->lb_locks)
906 load_balance_destroy(lb);
911 load_balance_mem_show (void)
913 fib_show_memory_usage("load-balance",
914 pool_elts(load_balance_pool),
915 pool_len(load_balance_pool),
916 sizeof(load_balance_t));
917 load_balance_map_show_mem();
920 const static dpo_vft_t lb_vft = {
921 .dv_lock = load_balance_lock,
922 .dv_unlock = load_balance_unlock,
923 .dv_format = format_load_balance_dpo,
924 .dv_mem_show = load_balance_mem_show,
928 * @brief The per-protocol VLIB graph nodes that are assigned to a load-balance
931 * this means that these graph nodes are ones from which a load-balance is the
932 * parent object in the DPO-graph.
934 * We do not list all the load-balance nodes, such as the *-lookup. instead
935 * we are relying on the correct use of the .sibling_of field when setting
936 * up these sibling nodes.
938 const static char* const load_balance_ip4_nodes[] =
943 const static char* const load_balance_ip6_nodes[] =
948 const static char* const load_balance_mpls_nodes[] =
953 const static char* const load_balance_l2_nodes[] =
958 const static char* const load_balance_nsh_nodes[] =
963 const static char* const load_balance_bier_nodes[] =
968 const static char* const * const load_balance_nodes[DPO_PROTO_NUM] =
970 [DPO_PROTO_IP4] = load_balance_ip4_nodes,
971 [DPO_PROTO_IP6] = load_balance_ip6_nodes,
972 [DPO_PROTO_MPLS] = load_balance_mpls_nodes,
973 [DPO_PROTO_ETHERNET] = load_balance_l2_nodes,
974 [DPO_PROTO_NSH] = load_balance_nsh_nodes,
975 [DPO_PROTO_BIER] = load_balance_bier_nodes,
979 load_balance_module_init (void)
983 dpo_register(DPO_LOAD_BALANCE, &lb_vft, load_balance_nodes);
986 * Special LB with index zero. we need to define this since the v4 mtrie
987 * assumes an index of 0 implies the ply is empty. therefore all 'real'
988 * adjs need a non-zero index.
989 * This should never be used, but just in case, stack it on a drop.
991 lbi = load_balance_create(1, DPO_PROTO_IP4, 0);
992 load_balance_set_bucket(lbi, 0, drop_dpo_get(DPO_PROTO_IP4));
994 load_balance_logger =
995 vlib_log_register_class("dpo", "load-balance");
997 load_balance_map_module_init();
1000 static clib_error_t *
1001 load_balance_show (vlib_main_t * vm,
1002 unformat_input_t * input,
1003 vlib_cli_command_t * cmd)
1005 index_t lbi = INDEX_INVALID;
1007 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1009 if (unformat (input, "%d", &lbi))
1015 if (INDEX_INVALID != lbi)
1017 if (pool_is_free_index(load_balance_pool, lbi))
1019 vlib_cli_output (vm, "no such load-balance:%d", lbi);
1023 vlib_cli_output (vm, "%U", format_load_balance, lbi,
1024 LOAD_BALANCE_FORMAT_DETAIL);
1031 pool_foreach(lb, load_balance_pool,
1033 vlib_cli_output (vm, "%U", format_load_balance,
1034 load_balance_get_index(lb),
1035 LOAD_BALANCE_FORMAT_NONE);
1042 VLIB_CLI_COMMAND (load_balance_show_command, static) = {
1043 .path = "show load-balance",
1044 .short_help = "show load-balance [<index>]",
1045 .function = load_balance_show,
1050 ip_flow_hash (void *data)
1052 ip4_header_t *iph = (ip4_header_t *) data;
1054 if ((iph->ip_version_and_header_length & 0xF0) == 0x40)
1055 return ip4_compute_flow_hash (iph, IP_FLOW_HASH_DEFAULT);
1057 return ip6_compute_flow_hash ((ip6_header_t *) iph, IP_FLOW_HASH_DEFAULT);
1063 return (*((u64 *) m) & 0xffffffffffff);
1067 l2_flow_hash (vlib_buffer_t * b0)
1069 ethernet_header_t *eh;
1071 uword is_ip, eh_size;
1074 eh = vlib_buffer_get_current (b0);
1075 eh_type = clib_net_to_host_u16 (eh->type);
1076 eh_size = ethernet_buffer_header_size (b0);
1078 is_ip = (eh_type == ETHERNET_TYPE_IP4 || eh_type == ETHERNET_TYPE_IP6);
1080 /* since we have 2 cache lines, use them */
1082 a = ip_flow_hash ((u8 *) vlib_buffer_get_current (b0) + eh_size);
1086 b = mac_to_u64 ((u8 *) eh->dst_address);
1087 c = mac_to_u64 ((u8 *) eh->src_address);
1088 hash_mix64 (a, b, c);
1093 typedef struct load_balance_trace_t_
1096 } load_balance_trace_t;
1099 load_balance_inline (vlib_main_t * vm,
1100 vlib_node_runtime_t * node,
1101 vlib_frame_t * frame,
1104 u32 n_left_from, next_index, *from, *to_next;
1106 from = vlib_frame_vector_args (frame);
1107 n_left_from = frame->n_vectors;
1109 next_index = node->cached_next_index;
1111 while (n_left_from > 0)
1115 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1117 while (n_left_from > 0 && n_left_to_next > 0)
1120 u32 bi0, lbi0, next0;
1121 const dpo_id_t *dpo0;
1122 const load_balance_t *lb0;
1129 n_left_to_next -= 1;
1131 b0 = vlib_get_buffer (vm, bi0);
1133 /* lookup dst + src mac */
1134 lbi0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
1135 lb0 = load_balance_get(lbi0);
1139 vnet_buffer(b0)->ip.flow_hash = l2_flow_hash(b0);
1144 const bier_hdr_t *bh0 = vlib_buffer_get_current(b0);
1145 vnet_buffer(b0)->ip.flow_hash = bier_compute_flow_hash(bh0);
1148 dpo0 = load_balance_get_bucket_i(lb0,
1149 vnet_buffer(b0)->ip.flow_hash &
1150 (lb0->lb_n_buckets_minus_1));
1152 next0 = dpo0->dpoi_next_node;
1153 vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
1155 if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
1157 load_balance_trace_t *tr = vlib_add_trace (vm, node, b0,
1159 tr->lb_index = lbi0;
1161 vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
1162 n_left_to_next, bi0, next0);
1165 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1168 return frame->n_vectors;
1172 l2_load_balance (vlib_main_t * vm,
1173 vlib_node_runtime_t * node,
1174 vlib_frame_t * frame)
1176 return (load_balance_inline(vm, node, frame, 1));
1180 format_l2_load_balance_trace (u8 * s, va_list * args)
1182 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1183 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1184 load_balance_trace_t *t = va_arg (*args, load_balance_trace_t *);
1186 s = format (s, "L2-load-balance: index %d", t->lb_index);
1193 VLIB_REGISTER_NODE (l2_load_balance_node) = {
1194 .function = l2_load_balance,
1195 .name = "l2-load-balance",
1196 .vector_size = sizeof (u32),
1198 .format_trace = format_l2_load_balance_trace,
1206 nsh_load_balance (vlib_main_t * vm,
1207 vlib_node_runtime_t * node,
1208 vlib_frame_t * frame)
1210 u32 n_left_from, next_index, *from, *to_next;
1212 from = vlib_frame_vector_args (frame);
1213 n_left_from = frame->n_vectors;
1215 next_index = node->cached_next_index;
1217 while (n_left_from > 0)
1221 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1223 while (n_left_from > 0 && n_left_to_next > 0)
1226 u32 bi0, lbi0, next0, *nsh0;
1227 const dpo_id_t *dpo0;
1228 const load_balance_t *lb0;
1235 n_left_to_next -= 1;
1237 b0 = vlib_get_buffer (vm, bi0);
1239 lbi0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
1240 lb0 = load_balance_get(lbi0);
1242 /* SPI + SI are the second word of the NSH header */
1243 nsh0 = vlib_buffer_get_current (b0);
1244 vnet_buffer(b0)->ip.flow_hash = nsh0[1] % lb0->lb_n_buckets;
1246 dpo0 = load_balance_get_bucket_i(lb0,
1247 vnet_buffer(b0)->ip.flow_hash &
1248 (lb0->lb_n_buckets_minus_1));
1250 next0 = dpo0->dpoi_next_node;
1251 vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
1253 if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
1255 load_balance_trace_t *tr = vlib_add_trace (vm, node, b0,
1257 tr->lb_index = lbi0;
1259 vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
1260 n_left_to_next, bi0, next0);
1263 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1266 return frame->n_vectors;
1270 format_nsh_load_balance_trace (u8 * s, va_list * args)
1272 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1273 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1274 load_balance_trace_t *t = va_arg (*args, load_balance_trace_t *);
1276 s = format (s, "NSH-load-balance: index %d", t->lb_index);
1283 VLIB_REGISTER_NODE (nsh_load_balance_node) = {
1284 .function = nsh_load_balance,
1285 .name = "nsh-load-balance",
1286 .vector_size = sizeof (u32),
1288 .format_trace = format_nsh_load_balance_trace,
1296 format_bier_load_balance_trace (u8 * s, va_list * args)
1298 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1299 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1300 load_balance_trace_t *t = va_arg (*args, load_balance_trace_t *);
1302 s = format (s, "BIER-load-balance: index %d", t->lb_index);
1307 bier_load_balance (vlib_main_t * vm,
1308 vlib_node_runtime_t * node,
1309 vlib_frame_t * frame)
1311 return (load_balance_inline(vm, node, frame, 0));
1317 VLIB_REGISTER_NODE (bier_load_balance_node) = {
1318 .function = bier_load_balance,
1319 .name = "bier-load-balance",
1320 .vector_size = sizeof (u32),
1322 .format_trace = format_bier_load_balance_trace,
1323 .sibling_of = "mpls-load-balance",