2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vnet/dpo/load_balance.h>
17 #include <vnet/dpo/load_balance_map.h>
18 #include <vnet/dpo/drop_dpo.h>
19 #include <vppinfra/math.h> /* for fabs */
20 #include <vnet/adj/adj.h>
21 #include <vnet/adj/adj_internal.h>
22 #include <vnet/fib/fib_urpf_list.h>
23 #include <vnet/bier/bier_fwd.h>
24 #include <vnet/fib/mpls_fib.h>
25 #include <vnet/ip/ip4_inlines.h>
26 #include <vnet/ip/ip6_inlines.h>
31 * distribution error tolerance for load-balancing
33 const f64 multipath_next_hop_error_tolerance = 0.1;
35 static const char *load_balance_attr_names[] = LOAD_BALANCE_ATTR_NAMES;
40 vlib_log_class_t load_balance_logger;
42 #define LB_DBG(_lb, _fmt, _args...) \
44 vlib_log_debug(load_balance_logger, \
46 format_load_balance, load_balance_get_index(_lb), \
47 LOAD_BALANCE_FORMAT_NONE, \
52 * Pool of all DPOs. It's not static so the DP can have fast access
54 load_balance_t *load_balance_pool;
57 * The one instance of load-balance main
59 load_balance_main_t load_balance_main = {
62 .stat_segment_name = "/net/route/to",
66 .stat_segment_name = "/net/route/via",
71 load_balance_get_multipath_tolerance (void)
73 return (multipath_next_hop_error_tolerance);
77 load_balance_get_index (const load_balance_t *lb)
79 return (lb - load_balance_pool);
82 static inline dpo_id_t*
83 load_balance_get_buckets (load_balance_t *lb)
85 if (LB_HAS_INLINE_BUCKETS(lb))
87 return (lb->lb_buckets_inline);
91 return (lb->lb_buckets);
95 static load_balance_t *
96 load_balance_alloc_i (void)
99 u8 need_barrier_sync = 0;
100 vlib_main_t *vm = vlib_get_main();
101 ASSERT (vm->thread_index == 0);
103 need_barrier_sync = pool_get_will_expand (load_balance_pool);
105 if (need_barrier_sync)
106 vlib_worker_thread_barrier_sync (vm);
108 pool_get_aligned(load_balance_pool, lb, CLIB_CACHE_LINE_BYTES);
109 clib_memset(lb, 0, sizeof(*lb));
111 lb->lb_map = INDEX_INVALID;
112 lb->lb_urpf = INDEX_INVALID;
114 if (need_barrier_sync == 0)
116 need_barrier_sync += vlib_validate_combined_counter_will_expand
117 (&(load_balance_main.lbm_to_counters),
118 load_balance_get_index(lb));
119 need_barrier_sync += vlib_validate_combined_counter_will_expand
120 (&(load_balance_main.lbm_via_counters),
121 load_balance_get_index(lb));
122 if (need_barrier_sync)
123 vlib_worker_thread_barrier_sync (vm);
126 vlib_validate_combined_counter(&(load_balance_main.lbm_to_counters),
127 load_balance_get_index(lb));
128 vlib_validate_combined_counter(&(load_balance_main.lbm_via_counters),
129 load_balance_get_index(lb));
130 vlib_zero_combined_counter(&(load_balance_main.lbm_to_counters),
131 load_balance_get_index(lb));
132 vlib_zero_combined_counter(&(load_balance_main.lbm_via_counters),
133 load_balance_get_index(lb));
135 if (need_barrier_sync)
136 vlib_worker_thread_barrier_release (vm);
142 load_balance_format (index_t lbi,
143 load_balance_format_flags_t flags,
147 vlib_counter_t to, via;
152 lb = load_balance_get_or_null(lbi);
155 s = format(s, "DELETED lb:%u", lbi);
159 vlib_get_combined_counter(&(load_balance_main.lbm_to_counters), lbi, &to);
160 vlib_get_combined_counter(&(load_balance_main.lbm_via_counters), lbi, &via);
161 buckets = load_balance_get_buckets(lb);
163 s = format(s, "%U: ", format_dpo_type, DPO_LOAD_BALANCE);
164 s = format(s, "[proto:%U ", format_dpo_proto, lb->lb_proto);
165 s = format(s, "index:%d buckets:%d ", lbi, lb->lb_n_buckets);
166 s = format(s, "uRPF:%d ", lb->lb_urpf);
169 load_balance_attr_t attr;
171 s = format(s, "flags:[");
173 FOR_EACH_LOAD_BALANCE_ATTR(attr)
175 if (lb->lb_flags & (1 << attr))
177 s = format (s, "%s", load_balance_attr_names[attr]);
182 s = format(s, "to:[%Ld:%Ld]", to.packets, to.bytes);
183 if (0 != via.packets)
185 s = format(s, " via:[%Ld:%Ld]",
186 via.packets, via.bytes);
190 if (INDEX_INVALID != lb->lb_map)
192 s = format(s, "\n%U%U",
193 format_white_space, indent+4,
194 format_load_balance_map, lb->lb_map, indent+4);
196 for (i = 0; i < lb->lb_n_buckets; i++)
198 s = format(s, "\n%U[%d] %U",
199 format_white_space, indent+2,
202 &buckets[i], indent+6);
208 format_load_balance (u8 * s, va_list * args)
210 index_t lbi = va_arg(*args, index_t);
211 load_balance_format_flags_t flags = va_arg(*args, load_balance_format_flags_t);
213 return (load_balance_format(lbi, flags, 0, s));
217 format_load_balance_dpo (u8 * s, va_list * args)
219 index_t lbi = va_arg(*args, index_t);
220 u32 indent = va_arg(*args, u32);
222 return (load_balance_format(lbi, LOAD_BALANCE_FORMAT_DETAIL, indent, s));
226 load_balance_get_default_flow_hash (dpo_proto_t lb_proto)
232 return (IP_FLOW_HASH_DEFAULT);
235 return (MPLS_FLOW_HASH_DEFAULT);
237 case DPO_PROTO_ETHERNET:
246 static load_balance_t *
247 load_balance_create_i (u32 num_buckets,
248 dpo_proto_t lb_proto,
249 flow_hash_config_t fhc)
253 ASSERT (num_buckets <= LB_MAX_BUCKETS);
255 lb = load_balance_alloc_i();
256 lb->lb_hash_config = fhc;
257 lb->lb_n_buckets = num_buckets;
258 lb->lb_n_buckets_minus_1 = num_buckets-1;
259 lb->lb_proto = lb_proto;
261 if (!LB_HAS_INLINE_BUCKETS(lb))
263 vec_validate_aligned(lb->lb_buckets,
264 lb->lb_n_buckets - 1,
265 CLIB_CACHE_LINE_BYTES);
268 LB_DBG(lb, "create");
274 load_balance_create (u32 n_buckets,
275 dpo_proto_t lb_proto,
276 flow_hash_config_t fhc)
278 return (load_balance_get_index(load_balance_create_i(n_buckets, lb_proto, fhc)));
282 load_balance_set_bucket_i (load_balance_t *lb,
285 const dpo_id_t *next)
287 dpo_stack(DPO_LOAD_BALANCE, lb->lb_proto, &buckets[bucket], next);
291 load_balance_set_bucket (index_t lbi,
293 const dpo_id_t *next)
298 lb = load_balance_get(lbi);
299 buckets = load_balance_get_buckets(lb);
301 ASSERT(bucket < lb->lb_n_buckets);
303 load_balance_set_bucket_i(lb, bucket, buckets, next);
307 load_balance_is_drop (const dpo_id_t *dpo)
311 if (DPO_LOAD_BALANCE != dpo->dpoi_type)
314 lb = load_balance_get(dpo->dpoi_index);
316 if (1 == lb->lb_n_buckets)
318 return (dpo_is_drop(load_balance_get_bucket_i(lb, 0)));
324 load_balance_n_buckets (index_t lbi)
328 lb = load_balance_get(lbi);
330 return (lb->lb_n_buckets);
334 load_balance_set_fib_entry_flags (index_t lbi,
335 fib_entry_flag_t flags)
339 lb = load_balance_get(lbi);
340 lb->lb_fib_entry_flags = flags;
345 load_balance_set_urpf (index_t lbi,
351 lb = load_balance_get(lbi);
354 * packets in flight we see this change. but it's atomic, so :P
359 fib_urpf_list_unlock(old);
360 fib_urpf_list_lock(urpf);
364 load_balance_get_urpf (index_t lbi)
368 lb = load_balance_get(lbi);
370 return (lb->lb_urpf);
374 load_balance_get_bucket (index_t lbi,
379 lb = load_balance_get(lbi);
381 return (load_balance_get_bucket_i(lb, bucket));
385 next_hop_sort_by_weight (const load_balance_path_t * n1,
386 const load_balance_path_t * n2)
388 return ((int) n1->path_weight - (int) n2->path_weight);
391 /* Given next hop vector is over-written with normalized one with sorted weights and
392 with weights corresponding to the number of adjacencies for each next hop.
393 Returns number of adjacencies in block. */
395 ip_multipath_normalize_next_hops (const load_balance_path_t * raw_next_hops,
396 load_balance_path_t ** normalized_next_hops,
398 f64 multipath_next_hop_error_tolerance)
400 load_balance_path_t * nhs;
401 uword n_nhs, n_adj, n_adj_left, i, sum_weight;
404 n_nhs = vec_len (raw_next_hops);
409 /* Allocate enough space for 2 copies; we'll use second copy to save original weights. */
410 nhs = *normalized_next_hops;
411 vec_validate (nhs, 2*n_nhs - 1);
413 /* Fast path: 1 next hop in block. */
417 nhs[0] = raw_next_hops[0];
418 nhs[0].path_weight = 1;
419 vec_set_len (nhs, 1);
426 int cmp = next_hop_sort_by_weight (&raw_next_hops[0], &raw_next_hops[1]) < 0;
429 nhs[0] = raw_next_hops[cmp];
430 nhs[1] = raw_next_hops[cmp ^ 1];
432 /* Fast path: equal cost multipath with 2 next hops. */
433 if (nhs[0].path_weight == nhs[1].path_weight)
435 nhs[0].path_weight = nhs[1].path_weight = 1;
436 vec_set_len (nhs, 2);
443 clib_memcpy_fast (nhs, raw_next_hops, n_nhs * sizeof (raw_next_hops[0]));
444 qsort (nhs, n_nhs, sizeof (nhs[0]), (void *) next_hop_sort_by_weight);
447 /* Find total weight to normalize weights. */
449 for (i = 0; i < n_nhs; i++)
450 sum_weight += nhs[i].path_weight;
452 /* In the unlikely case that all weights are given as 0, set them all to 1. */
455 for (i = 0; i < n_nhs; i++)
456 nhs[i].path_weight = 1;
460 /* Save copies of all next hop weights to avoid being overwritten in loop below. */
461 for (i = 0; i < n_nhs; i++)
462 nhs[n_nhs + i].path_weight = nhs[i].path_weight;
464 /* Try larger and larger power of 2 sized adjacency blocks until we
465 find one where traffic flows to within 1% of specified weights. */
466 for (n_adj = clib_min(max_pow2 (n_nhs), LB_MAX_BUCKETS); ; n_adj *= 2)
468 ASSERT (n_adj <= LB_MAX_BUCKETS);
471 norm = n_adj / ((f64) sum_weight);
473 for (i = 0; i < n_nhs; i++)
475 f64 nf = nhs[n_nhs + i].path_weight * norm; /* use saved weights */
476 word n = flt_round_nearest (nf);
478 n = n > n_adj_left ? n_adj_left : n;
480 error += fabs (nf - n);
481 nhs[i].path_weight = n;
483 if (0 == nhs[i].path_weight)
486 * when the weight skew is high (norm is small) and n == nf.
487 * without this correction the path with a low weight would have
488 * no representation in the load-balanace - don't want that.
489 * If the weight skew is high so the load-balance has many buckets
490 * to allow it. pays ya money takes ya choice.
497 nhs[0].path_weight += n_adj_left;
499 /* Less than 1% average error per adjacency with this size adjacency block,
500 * or did we reached the maximum number of buckets we support? */
501 if (error <= multipath_next_hop_error_tolerance*n_adj ||
502 n_adj >= LB_MAX_BUCKETS)
506 /* Truncate any next hops in excess */
507 vlib_log_err(load_balance_logger,
508 "Too many paths for load-balance, truncating %d -> %d",
510 for (int j = i; j < n_nhs; j++)
511 dpo_reset (&vec_elt(nhs, j).path_dpo);
513 vec_set_len (nhs, i);
519 /* Save vector for next call. */
520 *normalized_next_hops = nhs;
521 *sum_weight_in = sum_weight;
525 static load_balance_path_t *
526 load_balance_multipath_next_hop_fixup (const load_balance_path_t *nhs,
527 dpo_proto_t drop_proto)
529 if (0 == vec_len(nhs))
531 load_balance_path_t *new_nhs = NULL, *nh;
534 * we need something for the load-balance. so use the drop
536 vec_add2(new_nhs, nh, 1);
539 dpo_copy(&nh->path_dpo, drop_dpo_get(drop_proto));
548 * Fill in adjacencies in block based on corresponding
549 * next hop adjacencies.
552 load_balance_fill_buckets_norm (load_balance_t *lb,
553 load_balance_path_t *nhs,
557 load_balance_path_t *nh;
563 * the next-hops have normalised weights. that means their sum is the number
564 * of buckets we need to fill.
566 vec_foreach (nh, nhs)
568 for (ii = 0; ii < nh->path_weight; ii++)
570 ASSERT(bucket < n_buckets);
571 load_balance_set_bucket_i(lb, bucket++, buckets, &nh->path_dpo);
576 load_balance_fill_buckets_sticky (load_balance_t *lb,
577 load_balance_path_t *nhs,
581 load_balance_path_t *nh, *fwding_paths;
582 u16 ii, bucket, fpath;
587 vec_foreach (nh, nhs)
589 if (!dpo_is_drop(&nh->path_dpo))
591 vec_add1(fwding_paths, *nh);
594 if (vec_len(fwding_paths) == 0)
595 fwding_paths = vec_dup(nhs);
598 * the next-hops have normalised weights. that means their sum is the number
599 * of buckets we need to fill.
601 vec_foreach (nh, nhs)
603 for (ii = 0; ii < nh->path_weight; ii++)
605 ASSERT(bucket < n_buckets);
606 if (!dpo_is_drop(&nh->path_dpo))
608 load_balance_set_bucket_i(lb, bucket++, buckets, &nh->path_dpo);
612 /* fill the bucks from the next up path */
613 load_balance_set_bucket_i(lb, bucket++, buckets, &fwding_paths[fpath].path_dpo);
614 ASSERT(vec_len(fwding_paths) > 0);
615 fpath = (fpath + 1) % vec_len(fwding_paths);
620 vec_free(fwding_paths);
624 load_balance_fill_buckets (load_balance_t *lb,
625 load_balance_path_t *nhs,
628 load_balance_flags_t flags)
630 if (flags & LOAD_BALANCE_FLAG_STICKY)
632 load_balance_fill_buckets_sticky(lb, nhs, buckets, n_buckets);
636 load_balance_fill_buckets_norm(lb, nhs, buckets, n_buckets);
641 load_balance_set_n_buckets (load_balance_t *lb,
644 ASSERT (n_buckets <= LB_MAX_BUCKETS);
645 lb->lb_n_buckets = n_buckets;
646 lb->lb_n_buckets_minus_1 = n_buckets-1;
650 load_balance_multipath_update (const dpo_id_t *dpo,
651 const load_balance_path_t * raw_nhs,
652 load_balance_flags_t flags)
654 load_balance_path_t *nh, *nhs, *fixed_nhs;
655 u32 sum_of_weights, n_buckets, ii;
656 index_t lbmi, old_lbmi;
662 ASSERT(DPO_LOAD_BALANCE == dpo->dpoi_type);
663 lb = load_balance_get(dpo->dpoi_index);
664 lb->lb_flags = flags;
665 fixed_nhs = load_balance_multipath_next_hop_fixup(raw_nhs, lb->lb_proto);
667 ip_multipath_normalize_next_hops((NULL == fixed_nhs ?
672 multipath_next_hop_error_tolerance);
675 * Save the old load-balance map used, and get a new one if required.
677 old_lbmi = lb->lb_map;
678 if (flags & LOAD_BALANCE_FLAG_USES_MAP)
680 lbmi = load_balance_map_add_or_lock(n_buckets, sum_of_weights, nhs);
684 lbmi = INDEX_INVALID;
687 if (0 == lb->lb_n_buckets)
690 * first time initialisation. no packets inflight, so we can write
693 load_balance_set_n_buckets(lb, n_buckets);
695 if (!LB_HAS_INLINE_BUCKETS(lb))
696 vec_validate_aligned(lb->lb_buckets,
697 lb->lb_n_buckets - 1,
698 CLIB_CACHE_LINE_BYTES);
700 load_balance_fill_buckets(lb, nhs,
701 load_balance_get_buckets(lb),
708 * This is a modification of an existing load-balance.
709 * We need to ensure that packets inflight see a consistent state, that
710 * is the number of reported buckets the LB has (read from
711 * lb_n_buckets_minus_1) is not more than it actually has. So if the
712 * number of buckets is increasing, we must update the bucket array first,
713 * then the reported number. vice-versa if the number of buckets goes down.
715 if (n_buckets == lb->lb_n_buckets)
718 * no change in the number of buckets. we can simply fill what
719 * is new over what is old.
721 load_balance_fill_buckets(lb, nhs,
722 load_balance_get_buckets(lb),
726 else if (n_buckets > lb->lb_n_buckets)
729 * we have more buckets. the old load-balance map (if there is one)
730 * will remain valid, i.e. mapping to indices within range, so we
733 if (n_buckets > LB_NUM_INLINE_BUCKETS &&
734 lb->lb_n_buckets <= LB_NUM_INLINE_BUCKETS)
737 * the new increased number of buckets is crossing the threshold
738 * from the inline storage to out-line. Alloc the outline buckets
739 * first, then fixup the number. then reset the inlines.
741 ASSERT(NULL == lb->lb_buckets);
742 vec_validate_aligned(lb->lb_buckets,
744 CLIB_CACHE_LINE_BYTES);
746 load_balance_fill_buckets(lb, nhs,
749 CLIB_MEMORY_BARRIER();
750 load_balance_set_n_buckets(lb, n_buckets);
752 CLIB_MEMORY_BARRIER();
754 for (ii = 0; ii < LB_NUM_INLINE_BUCKETS; ii++)
756 dpo_reset(&lb->lb_buckets_inline[ii]);
761 if (n_buckets <= LB_NUM_INLINE_BUCKETS)
764 * we are not crossing the threshold and it's still inline buckets.
765 * we can write the new on the old..
767 load_balance_fill_buckets(lb, nhs,
768 load_balance_get_buckets(lb),
770 CLIB_MEMORY_BARRIER();
771 load_balance_set_n_buckets(lb, n_buckets);
776 * we are not crossing the threshold. We need a new bucket array to
777 * hold the increased number of choices.
779 dpo_id_t *new_buckets, *old_buckets, *tmp_dpo;
782 old_buckets = load_balance_get_buckets(lb);
784 vec_validate_aligned(new_buckets,
786 CLIB_CACHE_LINE_BYTES);
788 load_balance_fill_buckets(lb, nhs, new_buckets,
790 CLIB_MEMORY_BARRIER();
791 lb->lb_buckets = new_buckets;
792 CLIB_MEMORY_BARRIER();
793 load_balance_set_n_buckets(lb, n_buckets);
795 vec_foreach(tmp_dpo, old_buckets)
799 vec_free(old_buckets);
804 * buckets fixed. ready for the MAP update.
811 * bucket size shrinkage.
812 * Any map we have will be based on the old
813 * larger number of buckets, so will be translating to indices
814 * out of range. So the new MAP must be installed first.
817 CLIB_MEMORY_BARRIER();
820 if (n_buckets <= LB_NUM_INLINE_BUCKETS &&
821 lb->lb_n_buckets > LB_NUM_INLINE_BUCKETS)
824 * the new decreased number of buckets is crossing the threshold
825 * from out-line storage to inline:
826 * 1 - Fill the inline buckets,
827 * 2 - fixup the number (and this point the inline buckets are
829 * 3 - free the outline buckets
831 load_balance_fill_buckets(lb, nhs,
832 lb->lb_buckets_inline,
834 CLIB_MEMORY_BARRIER();
835 load_balance_set_n_buckets(lb, n_buckets);
836 CLIB_MEMORY_BARRIER();
838 vec_foreach(tmp_dpo, lb->lb_buckets)
842 vec_free(lb->lb_buckets);
847 * not crossing the threshold.
848 * 1 - update the number to the smaller size
849 * 2 - write the new buckets
850 * 3 - reset those no longer used.
855 old_n_buckets = lb->lb_n_buckets;
856 buckets = load_balance_get_buckets(lb);
858 load_balance_set_n_buckets(lb, n_buckets);
859 CLIB_MEMORY_BARRIER();
861 load_balance_fill_buckets(lb, nhs, buckets,
864 for (ii = n_buckets; ii < old_n_buckets; ii++)
866 dpo_reset(&buckets[ii]);
872 vec_foreach (nh, nhs)
874 dpo_reset(&nh->path_dpo);
879 load_balance_map_unlock(old_lbmi);
883 load_balance_lock (dpo_id_t *dpo)
887 lb = load_balance_get(dpo->dpoi_index);
893 load_balance_destroy (load_balance_t *lb)
898 buckets = load_balance_get_buckets(lb);
900 for (i = 0; i < lb->lb_n_buckets; i++)
902 dpo_reset(&buckets[i]);
905 LB_DBG(lb, "destroy");
906 if (!LB_HAS_INLINE_BUCKETS(lb))
908 vec_free(lb->lb_buckets);
911 fib_urpf_list_unlock(lb->lb_urpf);
912 load_balance_map_unlock(lb->lb_map);
914 pool_put(load_balance_pool, lb);
918 load_balance_unlock (dpo_id_t *dpo)
922 lb = load_balance_get(dpo->dpoi_index);
926 if (0 == lb->lb_locks)
928 load_balance_destroy(lb);
933 load_balance_mem_show (void)
935 fib_show_memory_usage("load-balance",
936 pool_elts(load_balance_pool),
937 pool_len(load_balance_pool),
938 sizeof(load_balance_t));
939 load_balance_map_show_mem();
943 load_balance_dpo_get_mtu (const dpo_id_t *dpo)
945 const dpo_id_t *buckets;
949 lb = load_balance_get(dpo->dpoi_index);
950 buckets = load_balance_get_buckets(lb);
952 for (i = 0; i < lb->lb_n_buckets; i++)
954 mtu = clib_min (mtu, dpo_get_mtu (&buckets[i]));
960 const static dpo_vft_t lb_vft = {
961 .dv_lock = load_balance_lock,
962 .dv_unlock = load_balance_unlock,
963 .dv_format = format_load_balance_dpo,
964 .dv_mem_show = load_balance_mem_show,
965 .dv_get_mtu = load_balance_dpo_get_mtu,
969 * @brief The per-protocol VLIB graph nodes that are assigned to a load-balance
972 * this means that these graph nodes are ones from which a load-balance is the
973 * parent object in the DPO-graph.
975 * We do not list all the load-balance nodes, such as the *-lookup. instead
976 * we are relying on the correct use of the .sibling_of field when setting
977 * up these sibling nodes.
979 const static char* const load_balance_ip4_nodes[] =
984 const static char* const load_balance_ip6_nodes[] =
989 const static char* const load_balance_mpls_nodes[] =
994 const static char* const load_balance_l2_nodes[] =
999 const static char* const load_balance_nsh_nodes[] =
1004 const static char* const load_balance_bier_nodes[] =
1006 "bier-load-balance",
1009 const static char* const * const load_balance_nodes[DPO_PROTO_NUM] =
1011 [DPO_PROTO_IP4] = load_balance_ip4_nodes,
1012 [DPO_PROTO_IP6] = load_balance_ip6_nodes,
1013 [DPO_PROTO_MPLS] = load_balance_mpls_nodes,
1014 [DPO_PROTO_ETHERNET] = load_balance_l2_nodes,
1015 [DPO_PROTO_NSH] = load_balance_nsh_nodes,
1016 [DPO_PROTO_BIER] = load_balance_bier_nodes,
1020 load_balance_module_init (void)
1024 dpo_register(DPO_LOAD_BALANCE, &lb_vft, load_balance_nodes);
1027 * Special LB with index zero. we need to define this since the v4 mtrie
1028 * assumes an index of 0 implies the ply is empty. therefore all 'real'
1029 * adjs need a non-zero index.
1030 * This should never be used, but just in case, stack it on a drop.
1032 lbi = load_balance_create(1, DPO_PROTO_IP4, 0);
1033 load_balance_set_bucket(lbi, 0, drop_dpo_get(DPO_PROTO_IP4));
1035 load_balance_logger =
1036 vlib_log_register_class("dpo", "load-balance");
1038 load_balance_map_module_init();
1041 static clib_error_t *
1042 load_balance_show (vlib_main_t * vm,
1043 unformat_input_t * input,
1044 vlib_cli_command_t * cmd)
1046 index_t lbi = INDEX_INVALID;
1048 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1050 if (unformat (input, "%d", &lbi))
1056 if (INDEX_INVALID != lbi)
1058 if (pool_is_free_index(load_balance_pool, lbi))
1060 vlib_cli_output (vm, "no such load-balance:%d", lbi);
1064 vlib_cli_output (vm, "%U", format_load_balance, lbi,
1065 LOAD_BALANCE_FORMAT_DETAIL);
1072 pool_foreach (lb, load_balance_pool)
1074 vlib_cli_output (vm, "%U", format_load_balance,
1075 load_balance_get_index(lb),
1076 LOAD_BALANCE_FORMAT_NONE);
1083 VLIB_CLI_COMMAND (load_balance_show_command, static) = {
1084 .path = "show load-balance",
1085 .short_help = "show load-balance [<index>]",
1086 .function = load_balance_show,
1091 ip_flow_hash (void *data)
1093 ip4_header_t *iph = (ip4_header_t *) data;
1095 if ((iph->ip_version_and_header_length & 0xF0) == 0x40)
1096 return ip4_compute_flow_hash (iph, IP_FLOW_HASH_DEFAULT);
1098 return ip6_compute_flow_hash ((ip6_header_t *) iph, IP_FLOW_HASH_DEFAULT);
1104 return (*((u64 *) m) & 0xffffffffffff);
1108 l2_flow_hash (vlib_buffer_t * b0)
1110 ethernet_header_t *eh;
1112 uword is_ip, eh_size;
1115 eh = vlib_buffer_get_current (b0);
1116 eh_type = clib_net_to_host_u16 (eh->type);
1117 eh_size = ethernet_buffer_header_size (b0);
1119 is_ip = (eh_type == ETHERNET_TYPE_IP4 || eh_type == ETHERNET_TYPE_IP6);
1121 /* since we have 2 cache lines, use them */
1123 a = ip_flow_hash ((u8 *) vlib_buffer_get_current (b0) + eh_size);
1127 b = mac_to_u64 ((u8 *) eh->dst_address);
1128 c = mac_to_u64 ((u8 *) eh->src_address);
1129 hash_mix64 (a, b, c);
1134 typedef struct load_balance_trace_t_
1137 } load_balance_trace_t;
1140 load_balance_inline (vlib_main_t * vm,
1141 vlib_node_runtime_t * node,
1142 vlib_frame_t * frame,
1145 u32 n_left_from, next_index, *from, *to_next;
1147 from = vlib_frame_vector_args (frame);
1148 n_left_from = frame->n_vectors;
1150 next_index = node->cached_next_index;
1152 while (n_left_from > 0)
1156 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1158 while (n_left_from > 0 && n_left_to_next > 0)
1161 u32 bi0, lbi0, next0;
1162 const dpo_id_t *dpo0;
1163 const load_balance_t *lb0;
1170 n_left_to_next -= 1;
1172 b0 = vlib_get_buffer (vm, bi0);
1174 /* lookup dst + src mac */
1175 lbi0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
1176 lb0 = load_balance_get(lbi0);
1180 vnet_buffer(b0)->ip.flow_hash = l2_flow_hash(b0);
1185 const bier_hdr_t *bh0 = vlib_buffer_get_current(b0);
1186 vnet_buffer(b0)->ip.flow_hash = bier_compute_flow_hash(bh0);
1189 dpo0 = load_balance_get_bucket_i(lb0,
1190 vnet_buffer(b0)->ip.flow_hash &
1191 (lb0->lb_n_buckets_minus_1));
1193 next0 = dpo0->dpoi_next_node;
1194 vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
1196 if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
1198 load_balance_trace_t *tr = vlib_add_trace (vm, node, b0,
1200 tr->lb_index = lbi0;
1202 vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
1203 n_left_to_next, bi0, next0);
1206 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1209 return frame->n_vectors;
1213 l2_load_balance (vlib_main_t * vm,
1214 vlib_node_runtime_t * node,
1215 vlib_frame_t * frame)
1217 return (load_balance_inline(vm, node, frame, 1));
1221 format_l2_load_balance_trace (u8 * s, va_list * args)
1223 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1224 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1225 load_balance_trace_t *t = va_arg (*args, load_balance_trace_t *);
1227 s = format (s, "L2-load-balance: index %d", t->lb_index);
1234 VLIB_REGISTER_NODE (l2_load_balance_node) = {
1235 .function = l2_load_balance,
1236 .name = "l2-load-balance",
1237 .vector_size = sizeof (u32),
1239 .format_trace = format_l2_load_balance_trace,
1247 nsh_load_balance (vlib_main_t * vm,
1248 vlib_node_runtime_t * node,
1249 vlib_frame_t * frame)
1251 u32 n_left_from, next_index, *from, *to_next;
1253 from = vlib_frame_vector_args (frame);
1254 n_left_from = frame->n_vectors;
1256 next_index = node->cached_next_index;
1258 while (n_left_from > 0)
1262 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1264 while (n_left_from > 0 && n_left_to_next > 0)
1267 u32 bi0, lbi0, next0, *nsh0;
1268 const dpo_id_t *dpo0;
1269 const load_balance_t *lb0;
1276 n_left_to_next -= 1;
1278 b0 = vlib_get_buffer (vm, bi0);
1280 lbi0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
1281 lb0 = load_balance_get(lbi0);
1283 /* SPI + SI are the second word of the NSH header */
1284 nsh0 = vlib_buffer_get_current (b0);
1285 vnet_buffer(b0)->ip.flow_hash = nsh0[1] % lb0->lb_n_buckets;
1287 dpo0 = load_balance_get_bucket_i(lb0,
1288 vnet_buffer(b0)->ip.flow_hash &
1289 (lb0->lb_n_buckets_minus_1));
1291 next0 = dpo0->dpoi_next_node;
1292 vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
1294 if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
1296 load_balance_trace_t *tr = vlib_add_trace (vm, node, b0,
1298 tr->lb_index = lbi0;
1300 vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
1301 n_left_to_next, bi0, next0);
1304 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1307 return frame->n_vectors;
1311 format_nsh_load_balance_trace (u8 * s, va_list * args)
1313 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1314 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1315 load_balance_trace_t *t = va_arg (*args, load_balance_trace_t *);
1317 s = format (s, "NSH-load-balance: index %d", t->lb_index);
1324 VLIB_REGISTER_NODE (nsh_load_balance_node) = {
1325 .function = nsh_load_balance,
1326 .name = "nsh-load-balance",
1327 .vector_size = sizeof (u32),
1329 .format_trace = format_nsh_load_balance_trace,
1337 format_bier_load_balance_trace (u8 * s, va_list * args)
1339 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1340 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1341 load_balance_trace_t *t = va_arg (*args, load_balance_trace_t *);
1343 s = format (s, "BIER-load-balance: index %d", t->lb_index);
1348 bier_load_balance (vlib_main_t * vm,
1349 vlib_node_runtime_t * node,
1350 vlib_frame_t * frame)
1352 return (load_balance_inline(vm, node, frame, 0));
1358 VLIB_REGISTER_NODE (bier_load_balance_node) = {
1359 .function = bier_load_balance,
1360 .name = "bier-load-balance",
1361 .vector_size = sizeof (u32),
1363 .format_trace = format_bier_load_balance_trace,
1364 .sibling_of = "mpls-load-balance",