2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
15 #include <vnet/vxlan/vxlan.h>
16 #include <vnet/ip/format.h>
17 #include <vnet/fib/fib_entry.h>
18 #include <vnet/fib/fib_table.h>
24 * VXLAN provides the features needed to allow L2 bridge domains (BDs)
25 * to span multiple servers. This is done by building an L2 overlay on
26 * top of an L3 network underlay using VXLAN tunnels.
28 * This makes it possible for servers to be co-located in the same data
29 * center or be separated geographically as long as they are reachable
30 * through the underlay L3 network.
32 * You can refer to this kind of L2 overlay bridge domain as a VXLAN
33 * (Virtual eXtensible VLAN) segment.
37 vxlan_main_t vxlan_main;
39 u8 * format_vxlan_tunnel (u8 * s, va_list * args)
41 vxlan_tunnel_t * t = va_arg (*args, vxlan_tunnel_t *);
42 vxlan_main_t * ngm = &vxlan_main;
45 "[%d] src %U dst %U vni %d encap_fib_index %d sw_if_index %d "
46 "fib_entry_index %d\n",
48 format_ip46_address, &t->src, IP46_TYPE_ANY,
49 format_ip46_address, &t->dst, IP46_TYPE_ANY,
50 t->vni, t->encap_fib_index, t->sw_if_index, t->fib_entry_index);
54 static u8 * format_vxlan_name (u8 * s, va_list * args)
56 u32 dev_instance = va_arg (*args, u32);
57 return format (s, "vxlan_tunnel%d", dev_instance);
60 static uword dummy_interface_tx (vlib_main_t * vm,
61 vlib_node_runtime_t * node,
64 clib_warning ("you shouldn't be here, leaking buffers...");
65 return frame->n_vectors;
69 vxlan_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
71 if (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP)
72 vnet_hw_interface_set_flags (vnm, hw_if_index, VNET_HW_INTERFACE_FLAG_LINK_UP);
74 vnet_hw_interface_set_flags (vnm, hw_if_index, 0);
76 return /* no error */ 0;
79 VNET_DEVICE_CLASS (vxlan_device_class,static) = {
81 .format_device_name = format_vxlan_name,
82 .format_tx_trace = format_vxlan_encap_trace,
83 .tx_function = dummy_interface_tx,
84 .admin_up_down_function = vxlan_interface_admin_up_down,
87 static u8 * format_vxlan_header_with_length (u8 * s, va_list * args)
89 u32 dev_instance = va_arg (*args, u32);
90 s = format (s, "unimplemented dev %u", dev_instance);
94 VNET_HW_INTERFACE_CLASS (vxlan_hw_class) = {
96 .format_header = format_vxlan_header_with_length,
97 .build_rewrite = default_build_rewrite,
101 static vxlan_tunnel_t *
102 vxlan_tunnel_from_fib_node (fib_node_t *node)
105 ASSERT(FIB_NODE_TYPE_VXLAN_TUNNEL == node->fn_type);
107 return ((vxlan_tunnel_t*) (((char*)node) -
108 STRUCT_OFFSET_OF(vxlan_tunnel_t, node)));
112 * Function definition to backwalk a FIB node -
113 * Here we will restack the new dpo of VXLAN DIP to encap node.
115 static fib_node_back_walk_rc_t
116 vxlan_tunnel_back_walk (fib_node_t *node,
117 fib_node_back_walk_ctx_t *ctx)
119 vxlan_tunnel_t *t = vxlan_tunnel_from_fib_node(node);
120 dpo_id_t dpo = DPO_INVALID;
122 if (ip46_address_is_ip4(&t->dst)) {
123 fib_entry_contribute_forwarding
124 (t->fib_entry_index, FIB_FORW_CHAIN_TYPE_UNICAST_IP4, &dpo);
126 (vxlan4_encap_node.index, &t->next_dpo, &dpo);
128 fib_entry_contribute_forwarding
129 (t->fib_entry_index, FIB_FORW_CHAIN_TYPE_UNICAST_IP6, &dpo);
131 (vxlan6_encap_node.index, &t->next_dpo, &dpo);
135 return (FIB_NODE_BACK_WALK_CONTINUE);
139 * Function definition to get a FIB node from its index
142 vxlan_tunnel_fib_node_get (fib_node_index_t index)
145 vxlan_main_t * vxm = &vxlan_main;
147 t = pool_elt_at_index(vxm->tunnels, index);
153 * Function definition to inform the FIB node that its last lock has gone.
156 vxlan_tunnel_last_lock_gone (fib_node_t *node)
159 * The VXLAN tunnel is a root of the graph. As such
160 * it never has children and thus is never locked.
166 * Virtual function table registered by VXLAN tunnels
167 * for participation in the FIB object graph.
169 const static fib_node_vft_t vxlan_vft = {
170 .fnv_get = vxlan_tunnel_fib_node_get,
171 .fnv_last_lock = vxlan_tunnel_last_lock_gone,
172 .fnv_back_walk = vxlan_tunnel_back_walk,
176 #define foreach_copy_field \
180 #define foreach_copy_ipv4 { \
185 #define foreach_copy_ipv6 { \
186 _(src.ip6.as_u64[0]) \
187 _(src.ip6.as_u64[1]) \
188 _(dst.ip6.as_u64[0]) \
189 _(dst.ip6.as_u64[1]) \
192 static int vxlan4_rewrite (vxlan_tunnel_t * t)
196 ip4_vxlan_header_t * h0;
197 int len = sizeof (*h0);
199 vec_validate_aligned (rw, len-1, CLIB_CACHE_LINE_BYTES);
201 h0 = (ip4_vxlan_header_t *) rw;
203 /* Fixed portion of the (outer) ip4 header */
205 ip0->ip_version_and_header_length = 0x45;
207 ip0->protocol = IP_PROTOCOL_UDP;
209 /* we fix up the ip4 header length and checksum after-the-fact */
210 ip0->src_address.as_u32 = t->src.ip4.as_u32;
211 ip0->dst_address.as_u32 = t->dst.ip4.as_u32;
212 ip0->checksum = ip4_header_checksum (ip0);
214 /* UDP header, randomize src port on something, maybe? */
215 h0->udp.src_port = clib_host_to_net_u16 (4789);
216 h0->udp.dst_port = clib_host_to_net_u16 (UDP_DST_PORT_vxlan);
219 vnet_set_vni_and_flags(&h0->vxlan, t->vni);
225 static int vxlan6_rewrite (vxlan_tunnel_t * t)
229 ip6_vxlan_header_t * h0;
230 int len = sizeof (*h0);
232 vec_validate_aligned (rw, len-1, CLIB_CACHE_LINE_BYTES);
234 h0 = (ip6_vxlan_header_t *) rw;
236 /* Fixed portion of the (outer) ip6 header */
238 ip0->ip_version_traffic_class_and_flow_label = clib_host_to_net_u32(6 << 28);
239 ip0->hop_limit = 255;
240 ip0->protocol = IP_PROTOCOL_UDP;
242 ip0->src_address.as_u64[0] = t->src.ip6.as_u64[0];
243 ip0->src_address.as_u64[1] = t->src.ip6.as_u64[1];
244 ip0->dst_address.as_u64[0] = t->dst.ip6.as_u64[0];
245 ip0->dst_address.as_u64[1] = t->dst.ip6.as_u64[1];
247 /* UDP header, randomize src port on something, maybe? */
248 h0->udp.src_port = clib_host_to_net_u16 (4789);
249 h0->udp.dst_port = clib_host_to_net_u16 (UDP_DST_PORT_vxlan);
252 vnet_set_vni_and_flags(&h0->vxlan, t->vni);
258 int vnet_vxlan_add_del_tunnel
259 (vnet_vxlan_add_del_tunnel_args_t *a, u32 * sw_if_indexp)
261 vxlan_main_t * vxm = &vxlan_main;
262 vxlan_tunnel_t *t = 0;
263 vnet_main_t * vnm = vxm->vnet_main;
264 vnet_hw_interface_t * hi;
266 u32 hw_if_index = ~0;
267 u32 sw_if_index = ~0;
269 vxlan4_tunnel_key_t key4;
270 vxlan6_tunnel_key_t key6;
271 u32 is_ip6 = a->is_ip6;
274 key4.src = a->dst.ip4.as_u32; /* decap src in key is encap dst in config */
275 key4.vni = clib_host_to_net_u32 (a->vni << 8);
277 p = hash_get (vxm->vxlan4_tunnel_by_key, key4.as_u64);
279 key6.src.as_u64[0] = a->dst.ip6.as_u64[0];
280 key6.src.as_u64[1] = a->dst.ip6.as_u64[1];
281 key6.vni = clib_host_to_net_u32 (a->vni << 8);
283 p = hash_get_mem (vxm->vxlan6_tunnel_by_key, &key6);
288 l2input_main_t * l2im = &l2input_main;
290 /* adding a tunnel: tunnel must not already exist */
292 return VNET_API_ERROR_TUNNEL_EXIST;
294 pool_get_aligned (vxm->tunnels, t, CLIB_CACHE_LINE_BYTES);
295 memset (t, 0, sizeof (*t));
297 /* copy from arg structure */
298 #define _(x) t->x = a->x;
300 if (!is_ip6) foreach_copy_ipv4
301 else foreach_copy_ipv6
307 t->key6 = clib_mem_alloc (sizeof(vxlan6_tunnel_key_t));
308 clib_memcpy (t->key6, &key6, sizeof(key6));
312 t->key4 = 0; /* not yet used */
316 rv = vxlan4_rewrite (t);
318 rv = vxlan6_rewrite (t);
323 pool_put (vxm->tunnels, t);
328 hash_set (vxm->vxlan4_tunnel_by_key, key4.as_u64, t - vxm->tunnels);
330 hash_set_mem (vxm->vxlan6_tunnel_by_key, t->key6, t - vxm->tunnels);
332 if (vec_len (vxm->free_vxlan_tunnel_hw_if_indices) > 0)
334 vnet_interface_main_t * im = &vnm->interface_main;
335 hw_if_index = vxm->free_vxlan_tunnel_hw_if_indices
336 [vec_len (vxm->free_vxlan_tunnel_hw_if_indices)-1];
337 _vec_len (vxm->free_vxlan_tunnel_hw_if_indices) -= 1;
339 hi = vnet_get_hw_interface (vnm, hw_if_index);
340 hi->dev_instance = t - vxm->tunnels;
341 hi->hw_instance = hi->dev_instance;
343 /* clear old stats of freed tunnel before reuse */
344 sw_if_index = hi->sw_if_index;
345 vnet_interface_counter_lock(im);
346 vlib_zero_combined_counter
347 (&im->combined_sw_if_counters[VNET_INTERFACE_COUNTER_TX], sw_if_index);
348 vlib_zero_combined_counter
349 (&im->combined_sw_if_counters[VNET_INTERFACE_COUNTER_RX], sw_if_index);
350 vlib_zero_simple_counter
351 (&im->sw_if_counters[VNET_INTERFACE_COUNTER_DROP], sw_if_index);
352 vnet_interface_counter_unlock(im);
356 hw_if_index = vnet_register_interface
357 (vnm, vxlan_device_class.index, t - vxm->tunnels,
358 vxlan_hw_class.index, t - vxm->tunnels);
359 hi = vnet_get_hw_interface (vnm, hw_if_index);
362 t->hw_if_index = hw_if_index;
363 t->sw_if_index = sw_if_index = hi->sw_if_index;
365 vec_validate_init_empty (vxm->tunnel_index_by_sw_if_index, sw_if_index, ~0);
366 vxm->tunnel_index_by_sw_if_index[sw_if_index] = t - vxm->tunnels;
368 /* setup l2 input config with l2 feature and bd 0 to drop packet */
369 vec_validate (l2im->configs, sw_if_index);
370 l2im->configs[sw_if_index].feature_bitmap = L2INPUT_FEAT_DROP;
371 l2im->configs[sw_if_index].bd_index = 0;
373 vnet_sw_interface_set_flags (vnm, sw_if_index,
374 VNET_SW_INTERFACE_FLAG_ADMIN_UP);
376 * source the FIB entry for the tunnel's destination
377 * and become a child thereof. The tunnel will then get poked
378 * when the forwarding for the entry updates, and the tunnel can
379 * re-stack accordingly
381 fib_node_init(&t->node, FIB_NODE_TYPE_VXLAN_TUNNEL);
384 dpo_id_t dpo = DPO_INVALID;
385 const fib_prefix_t tun_dst_pfx =
388 .fp_proto = FIB_PROTOCOL_IP4,
395 t->fib_entry_index = fib_table_entry_special_add
396 (t->encap_fib_index, &tun_dst_pfx,
397 FIB_SOURCE_RR, FIB_ENTRY_FLAG_NONE, ADJ_INDEX_INVALID);
398 t->sibling_index = fib_entry_child_add
399 (t->fib_entry_index, FIB_NODE_TYPE_VXLAN_TUNNEL, t - vxm->tunnels);
400 fib_entry_contribute_forwarding
401 (t->fib_entry_index, FIB_FORW_CHAIN_TYPE_UNICAST_IP4, &dpo);
402 dpo_stack_from_node (vxlan4_encap_node.index, &t->next_dpo, &dpo);
405 /* Set vxlan tunnel output node to ip4 version */
406 hi->output_node_index = vxlan4_encap_node.index;
410 dpo_id_t dpo = DPO_INVALID;
411 const fib_prefix_t tun_dst_pfx =
414 .fp_proto = FIB_PROTOCOL_IP6,
421 t->fib_entry_index = fib_table_entry_special_add
422 (t->encap_fib_index, &tun_dst_pfx,
423 FIB_SOURCE_RR, FIB_ENTRY_FLAG_NONE, ADJ_INDEX_INVALID);
424 t->sibling_index = fib_entry_child_add
425 (t->fib_entry_index, FIB_NODE_TYPE_VXLAN_TUNNEL, t - vxm->tunnels);
426 fib_entry_contribute_forwarding
427 (t->fib_entry_index, FIB_FORW_CHAIN_TYPE_UNICAST_IP6, &dpo);
429 (vxlan6_encap_node.index, &t->next_dpo, &dpo);
432 /* Set vxlan tunnel output node to ip6 version */
433 hi->output_node_index = vxlan6_encap_node.index;
438 /* deleting a tunnel: tunnel must exist */
440 return VNET_API_ERROR_NO_SUCH_ENTRY;
442 t = pool_elt_at_index (vxm->tunnels, p[0]);
444 vnet_sw_interface_set_flags (vnm, t->sw_if_index, 0 /* down */);
445 /* make sure tunnel is removed from l2 bd or xconnect */
446 set_int_l2_mode(vxm->vlib_main, vnm, MODE_L3, t->sw_if_index, 0, 0, 0, 0);
447 vec_add1 (vxm->free_vxlan_tunnel_hw_if_indices, t->hw_if_index);
449 vxm->tunnel_index_by_sw_if_index[t->sw_if_index] = ~0;
451 fib_entry_child_remove(t->fib_entry_index, t->sibling_index);
452 fib_table_entry_delete_index(t->fib_entry_index, FIB_SOURCE_RR);
453 fib_node_deinit(&t->node);
457 hash_unset (vxm->vxlan4_tunnel_by_key, key4.as_u64);
461 hash_unset_mem (vxm->vxlan6_tunnel_by_key, t->key6);
462 clib_mem_free (t->key6);
464 vec_free (t->rewrite);
465 pool_put (vxm->tunnels, t);
469 *sw_if_indexp = sw_if_index;
474 static u32 fib4_index_from_fib_id (u32 fib_id)
476 ip4_main_t * im = &ip4_main;
479 p = hash_get (im->fib_index_by_table_id, fib_id);
486 static u32 fib6_index_from_fib_id (u32 fib_id)
488 ip6_main_t * im = &ip6_main;
491 p = hash_get (im->fib_index_by_table_id, fib_id);
498 static uword unformat_decap_next (unformat_input_t * input, va_list * args)
500 u32 * result = va_arg (*args, u32 *);
503 if (unformat (input, "l2"))
504 *result = VXLAN_INPUT_NEXT_L2_INPUT;
505 else if (unformat (input, "%d", &tmp))
512 static clib_error_t *
513 vxlan_add_del_tunnel_command_fn (vlib_main_t * vm,
514 unformat_input_t * input,
515 vlib_cli_command_t * cmd)
517 unformat_input_t _line_input, * line_input = &_line_input;
518 ip46_address_t src, dst;
524 u32 encap_fib_index = 0;
525 u32 decap_next_index = ~0;
529 vnet_vxlan_add_del_tunnel_args_t _a, * a = &_a;
532 /* Get a line of input. */
533 if (! unformat_user (input, unformat_line_input, line_input))
536 while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) {
537 if (unformat (line_input, "del"))
541 else if (unformat (line_input, "src %U",
542 unformat_ip4_address, &src.ip4))
547 else if (unformat (line_input, "dst %U",
548 unformat_ip4_address, &dst.ip4))
553 else if (unformat (line_input, "src %U",
554 unformat_ip6_address, &src.ip6))
559 else if (unformat (line_input, "dst %U",
560 unformat_ip6_address, &dst.ip6))
565 else if (unformat (line_input, "encap-vrf-id %d", &tmp))
568 encap_fib_index = fib6_index_from_fib_id (tmp);
570 encap_fib_index = fib4_index_from_fib_id (tmp);
571 if (encap_fib_index == ~0)
572 return clib_error_return (0, "nonexistent encap-vrf-id %d", tmp);
574 else if (unformat (line_input, "decap-next %U", unformat_decap_next,
577 else if (unformat (line_input, "vni %d", &vni))
580 return clib_error_return (0, "vni %d out of range", vni);
583 return clib_error_return (0, "parse error: '%U'",
584 format_unformat_error, line_input);
587 unformat_free (line_input);
590 return clib_error_return (0, "tunnel src address not specified");
593 return clib_error_return (0, "tunnel dst address not specified");
595 if (ipv4_set && ipv6_set)
596 return clib_error_return (0, "both IPv4 and IPv6 addresses specified");
598 if ((ipv4_set && memcmp(&src.ip4, &dst.ip4, sizeof(src.ip4)) == 0) ||
599 (ipv6_set && memcmp(&src.ip6, &dst.ip6, sizeof(src.ip6)) == 0))
600 return clib_error_return (0, "src and dst addresses are identical");
603 return clib_error_return (0, "vni not specified");
605 memset (a, 0, sizeof (*a));
608 a->is_ip6 = ipv6_set;
610 #define _(x) a->x = x;
612 if (ipv4_set) foreach_copy_ipv4
613 else foreach_copy_ipv6
616 rv = vnet_vxlan_add_del_tunnel (a, &sw_if_index);
622 vlib_cli_output(vm, "%U\n", format_vnet_sw_if_index_name,
623 vnet_get_main(), sw_if_index);
626 case VNET_API_ERROR_TUNNEL_EXIST:
627 return clib_error_return (0, "tunnel already exists...");
629 case VNET_API_ERROR_NO_SUCH_ENTRY:
630 return clib_error_return (0, "tunnel does not exist...");
633 return clib_error_return
634 (0, "vnet_vxlan_add_del_tunnel returned %d", rv);
641 * Add or delete a VXLAN Tunnel.
643 * VXLAN provides the features needed to allow L2 bridge domains (BDs)
644 * to span multiple servers. This is done by building an L2 overlay on
645 * top of an L3 network underlay using VXLAN tunnels.
647 * This makes it possible for servers to be co-located in the same data
648 * center or be separated geographically as long as they are reachable
649 * through the underlay L3 network.
651 * You can refer to this kind of L2 overlay bridge domain as a VXLAN
652 * (Virtual eXtensible VLAN) segment.
655 * Example of how to create a VXLAN Tunnel:
656 * @cliexcmd{create vxlan tunnel src 10.0.3.1 dst 10.0.3.3 vni 13 encap-vrf-id 7}
657 * Example of how to delete a VXLAN Tunnel:
658 * @cliexcmd{create vxlan tunnel src 10.0.3.1 dst 10.0.3.3 vni 13 del}
661 VLIB_CLI_COMMAND (create_vxlan_tunnel_command, static) = {
662 .path = "create vxlan tunnel",
664 "create vxlan tunnel src <local-vtep-addr> dst <remote-vtep-addr> vni <nn>"
665 " [encap-vrf-id <nn>]",
666 .function = vxlan_add_del_tunnel_command_fn,
670 static clib_error_t *
671 show_vxlan_tunnel_command_fn (vlib_main_t * vm,
672 unformat_input_t * input,
673 vlib_cli_command_t * cmd)
675 vxlan_main_t * vxm = &vxlan_main;
678 if (pool_elts (vxm->tunnels) == 0)
679 vlib_cli_output (vm, "No vxlan tunnels configured...");
681 pool_foreach (t, vxm->tunnels,
683 vlib_cli_output (vm, "%U", format_vxlan_tunnel, t);
690 * Display all the VXLAN Tunnel entries.
693 * Example of how to display the VXLAN Tunnel entries:
694 * @cliexstart{show vxlan tunnel}
695 * [0] src 10.0.3.1 dst 10.0.3.3 vni 13 encap_fib_index 0 sw_if_index 5
699 VLIB_CLI_COMMAND (show_vxlan_tunnel_command, static) = {
700 .path = "show vxlan tunnel",
701 .short_help = "show vxlan tunnel",
702 .function = show_vxlan_tunnel_command_fn,
707 clib_error_t *vxlan_init (vlib_main_t *vm)
709 vxlan_main_t * vxm = &vxlan_main;
711 vxm->vnet_main = vnet_get_main();
714 /* initialize the ip6 hash */
715 vxm->vxlan6_tunnel_by_key = hash_create_mem(0,
716 sizeof(vxlan6_tunnel_key_t),
719 udp_register_dst_port (vm, UDP_DST_PORT_vxlan,
720 vxlan4_input_node.index, /* is_ip4 */ 1);
721 udp_register_dst_port (vm, UDP_DST_PORT_vxlan6,
722 vxlan6_input_node.index, /* is_ip4 */ 0);
724 fib_node_register_type(FIB_NODE_TYPE_VXLAN_TUNNEL, &vxlan_vft);
729 VLIB_INIT_FUNCTION(vxlan_init);