2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
15 #include <vnet/vxlan/vxlan.h>
16 #include <vnet/ip/format.h>
17 #include <vnet/fib/fib_entry.h>
18 #include <vnet/fib/fib_table.h>
19 #include <vnet/dpo/receive_dpo.h>
25 * VXLAN provides the features needed to allow L2 bridge domains (BDs)
26 * to span multiple servers. This is done by building an L2 overlay on
27 * top of an L3 network underlay using VXLAN tunnels.
29 * This makes it possible for servers to be co-located in the same data
30 * center or be separated geographically as long as they are reachable
31 * through the underlay L3 network.
33 * You can refer to this kind of L2 overlay bridge domain as a VXLAN
34 * (Virtual eXtensible VLAN) segment.
38 vxlan_main_t vxlan_main;
40 u8 * format_vxlan_tunnel (u8 * s, va_list * args)
42 vxlan_tunnel_t * t = va_arg (*args, vxlan_tunnel_t *);
43 vxlan_main_t * ngm = &vxlan_main;
46 "[%d] src %U dst %U vni %d encap_fib_index %d sw_if_index %d "
47 "fib_entry_index %d\n",
49 format_ip46_address, &t->src, IP46_TYPE_ANY,
50 format_ip46_address, &t->dst, IP46_TYPE_ANY,
51 t->vni, t->encap_fib_index, t->sw_if_index, t->fib_entry_index);
55 static u8 * format_vxlan_name (u8 * s, va_list * args)
57 u32 dev_instance = va_arg (*args, u32);
58 return format (s, "vxlan_tunnel%d", dev_instance);
61 static uword dummy_interface_tx (vlib_main_t * vm,
62 vlib_node_runtime_t * node,
65 clib_warning ("you shouldn't be here, leaking buffers...");
66 return frame->n_vectors;
70 vxlan_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
72 if (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP)
73 vnet_hw_interface_set_flags (vnm, hw_if_index, VNET_HW_INTERFACE_FLAG_LINK_UP);
75 vnet_hw_interface_set_flags (vnm, hw_if_index, 0);
77 return /* no error */ 0;
80 VNET_DEVICE_CLASS (vxlan_device_class,static) = {
82 .format_device_name = format_vxlan_name,
83 .format_tx_trace = format_vxlan_encap_trace,
84 .tx_function = dummy_interface_tx,
85 .admin_up_down_function = vxlan_interface_admin_up_down,
88 static u8 * format_vxlan_header_with_length (u8 * s, va_list * args)
90 u32 dev_instance = va_arg (*args, u32);
91 s = format (s, "unimplemented dev %u", dev_instance);
95 VNET_HW_INTERFACE_CLASS (vxlan_hw_class) = {
97 .format_header = format_vxlan_header_with_length,
98 .build_rewrite = default_build_rewrite,
102 vxlan_tunnel_restack_dpo(vxlan_tunnel_t * t)
104 dpo_id_t dpo = DPO_INVALID;
105 u32 encap_index = ip46_address_is_ip4(&t->dst) ?
106 vxlan4_encap_node.index : vxlan6_encap_node.index;
107 fib_forward_chain_type_t forw_type = ip46_address_is_ip4(&t->dst) ?
108 FIB_FORW_CHAIN_TYPE_UNICAST_IP4 : FIB_FORW_CHAIN_TYPE_UNICAST_IP6;
110 fib_entry_contribute_forwarding (t->fib_entry_index, forw_type, &dpo);
111 dpo_stack_from_node (encap_index, &t->next_dpo, &dpo);
115 static vxlan_tunnel_t *
116 vxlan_tunnel_from_fib_node (fib_node_t *node)
119 ASSERT(FIB_NODE_TYPE_VXLAN_TUNNEL == node->fn_type);
121 return ((vxlan_tunnel_t*) (((char*)node) -
122 STRUCT_OFFSET_OF(vxlan_tunnel_t, node)));
126 * Function definition to backwalk a FIB node -
127 * Here we will restack the new dpo of VXLAN DIP to encap node.
129 static fib_node_back_walk_rc_t
130 vxlan_tunnel_back_walk (fib_node_t *node,
131 fib_node_back_walk_ctx_t *ctx)
133 vxlan_tunnel_restack_dpo(vxlan_tunnel_from_fib_node(node));
134 return (FIB_NODE_BACK_WALK_CONTINUE);
138 * Function definition to get a FIB node from its index
141 vxlan_tunnel_fib_node_get (fib_node_index_t index)
144 vxlan_main_t * vxm = &vxlan_main;
146 t = pool_elt_at_index(vxm->tunnels, index);
152 * Function definition to inform the FIB node that its last lock has gone.
155 vxlan_tunnel_last_lock_gone (fib_node_t *node)
158 * The VXLAN tunnel is a root of the graph. As such
159 * it never has children and thus is never locked.
165 * Virtual function table registered by VXLAN tunnels
166 * for participation in the FIB object graph.
168 const static fib_node_vft_t vxlan_vft = {
169 .fnv_get = vxlan_tunnel_fib_node_get,
170 .fnv_last_lock = vxlan_tunnel_last_lock_gone,
171 .fnv_back_walk = vxlan_tunnel_back_walk,
175 #define foreach_copy_field \
177 _(mcast_sw_if_index) \
182 static int vxlan4_rewrite (vxlan_tunnel_t * t)
186 ip4_vxlan_header_t * h0;
187 int len = sizeof (*h0);
189 vec_validate_aligned (rw, len-1, CLIB_CACHE_LINE_BYTES);
191 h0 = (ip4_vxlan_header_t *) rw;
193 /* Fixed portion of the (outer) ip4 header */
195 ip0->ip_version_and_header_length = 0x45;
197 ip0->protocol = IP_PROTOCOL_UDP;
199 /* we fix up the ip4 header length and checksum after-the-fact */
200 ip0->src_address.as_u32 = t->src.ip4.as_u32;
201 ip0->dst_address.as_u32 = t->dst.ip4.as_u32;
202 ip0->checksum = ip4_header_checksum (ip0);
204 /* UDP header, randomize src port on something, maybe? */
205 h0->udp.src_port = clib_host_to_net_u16 (4789);
206 h0->udp.dst_port = clib_host_to_net_u16 (UDP_DST_PORT_vxlan);
209 vnet_set_vni_and_flags(&h0->vxlan, t->vni);
215 static int vxlan6_rewrite (vxlan_tunnel_t * t)
219 ip6_vxlan_header_t * h0;
220 int len = sizeof (*h0);
222 vec_validate_aligned (rw, len-1, CLIB_CACHE_LINE_BYTES);
224 h0 = (ip6_vxlan_header_t *) rw;
226 /* Fixed portion of the (outer) ip6 header */
228 ip0->ip_version_traffic_class_and_flow_label = clib_host_to_net_u32(6 << 28);
229 ip0->hop_limit = 255;
230 ip0->protocol = IP_PROTOCOL_UDP;
232 ip0->src_address.as_u64[0] = t->src.ip6.as_u64[0];
233 ip0->src_address.as_u64[1] = t->src.ip6.as_u64[1];
234 ip0->dst_address.as_u64[0] = t->dst.ip6.as_u64[0];
235 ip0->dst_address.as_u64[1] = t->dst.ip6.as_u64[1];
237 /* UDP header, randomize src port on something, maybe? */
238 h0->udp.src_port = clib_host_to_net_u16 (4789);
239 h0->udp.dst_port = clib_host_to_net_u16 (UDP_DST_PORT_vxlan);
242 vnet_set_vni_and_flags(&h0->vxlan, t->vni);
248 int vnet_vxlan_add_del_tunnel
249 (vnet_vxlan_add_del_tunnel_args_t *a, u32 * sw_if_indexp)
251 vxlan_main_t * vxm = &vxlan_main;
252 vxlan_tunnel_t *t = 0;
253 vnet_main_t * vnm = vxm->vnet_main;
254 vnet_hw_interface_t * hi;
256 u32 hw_if_index = ~0;
257 u32 sw_if_index = ~0;
259 vxlan4_tunnel_key_t key4;
260 vxlan6_tunnel_key_t key6;
261 u32 is_ip6 = a->is_ip6;
264 key4.src = a->dst.ip4.as_u32; /* decap src in key is encap dst in config */
265 key4.vni = clib_host_to_net_u32 (a->vni << 8);
267 p = hash_get (vxm->vxlan4_tunnel_by_key, key4.as_u64);
268 pvtep = hash_get (vxm->vtep4, a->src.ip4.as_u32);
270 key6.src.as_u64[0] = a->dst.ip6.as_u64[0];
271 key6.src.as_u64[1] = a->dst.ip6.as_u64[1];
272 key6.vni = clib_host_to_net_u32 (a->vni << 8);
274 p = hash_get_mem (vxm->vxlan6_tunnel_by_key, &key6);
275 pvtep = NULL; /* ip6 vxlan-bypass not yet implemented */
280 l2input_main_t * l2im = &l2input_main;
282 /* adding a tunnel: tunnel must not already exist */
284 return VNET_API_ERROR_TUNNEL_EXIST;
286 pool_get_aligned (vxm->tunnels, t, CLIB_CACHE_LINE_BYTES);
287 memset (t, 0, sizeof (*t));
289 /* copy from arg structure */
290 #define _(x) t->x = a->x;
297 t->key6 = clib_mem_alloc (sizeof(vxlan6_tunnel_key_t));
298 clib_memcpy (t->key6, &key6, sizeof(key6));
302 t->key4 = 0; /* not yet used */
306 rv = vxlan4_rewrite (t);
308 rv = vxlan6_rewrite (t);
313 pool_put (vxm->tunnels, t);
319 hash_set (vxm->vxlan4_tunnel_by_key, key4.as_u64, t - vxm->tunnels);
320 if (pvtep) pvtep[0]++;
321 else hash_set (vxm->vtep4, a->src.ip4.as_u32, 1);
324 hash_set_mem (vxm->vxlan6_tunnel_by_key, t->key6, t - vxm->tunnels);
326 if (vec_len (vxm->free_vxlan_tunnel_hw_if_indices) > 0)
328 vnet_interface_main_t * im = &vnm->interface_main;
329 hw_if_index = vxm->free_vxlan_tunnel_hw_if_indices
330 [vec_len (vxm->free_vxlan_tunnel_hw_if_indices)-1];
331 _vec_len (vxm->free_vxlan_tunnel_hw_if_indices) -= 1;
333 hi = vnet_get_hw_interface (vnm, hw_if_index);
334 hi->dev_instance = t - vxm->tunnels;
335 hi->hw_instance = hi->dev_instance;
337 /* clear old stats of freed tunnel before reuse */
338 sw_if_index = hi->sw_if_index;
339 vnet_interface_counter_lock(im);
340 vlib_zero_combined_counter
341 (&im->combined_sw_if_counters[VNET_INTERFACE_COUNTER_TX], sw_if_index);
342 vlib_zero_combined_counter
343 (&im->combined_sw_if_counters[VNET_INTERFACE_COUNTER_RX], sw_if_index);
344 vlib_zero_simple_counter
345 (&im->sw_if_counters[VNET_INTERFACE_COUNTER_DROP], sw_if_index);
346 vnet_interface_counter_unlock(im);
350 hw_if_index = vnet_register_interface
351 (vnm, vxlan_device_class.index, t - vxm->tunnels,
352 vxlan_hw_class.index, t - vxm->tunnels);
353 hi = vnet_get_hw_interface (vnm, hw_if_index);
356 t->hw_if_index = hw_if_index;
357 t->sw_if_index = sw_if_index = hi->sw_if_index;
359 vec_validate_init_empty (vxm->tunnel_index_by_sw_if_index, sw_if_index, ~0);
360 vxm->tunnel_index_by_sw_if_index[sw_if_index] = t - vxm->tunnels;
362 /* setup l2 input config with l2 feature and bd 0 to drop packet */
363 vec_validate (l2im->configs, sw_if_index);
364 l2im->configs[sw_if_index].feature_bitmap = L2INPUT_FEAT_DROP;
365 l2im->configs[sw_if_index].bd_index = 0;
367 vnet_sw_interface_set_flags (vnm, sw_if_index,
368 VNET_SW_INTERFACE_FLAG_ADMIN_UP);
369 fib_node_init(&t->node, FIB_NODE_TYPE_VXLAN_TUNNEL);
370 fib_prefix_t tun_dst_pfx;
371 u32 encap_index = !is_ip6 ?
372 vxlan4_encap_node.index : vxlan6_encap_node.index;
373 vnet_flood_class_t flood_class = VNET_FLOOD_CLASS_TUNNEL_NORMAL;
375 fib_prefix_from_ip46_addr(&t->dst, &tun_dst_pfx);
376 if (ip46_address_is_multicast(&t->dst))
381 ip4_multicast_ethernet_address(mcast_mac, &t->dst.ip4);
382 fp = FIB_PROTOCOL_IP4;
384 ip6_multicast_ethernet_address(mcast_mac, t->dst.ip6.as_u32[0]);
385 fp = FIB_PROTOCOL_IP6;
387 t->mcast_adj_index = adj_rewrite_add_and_lock
388 (fp, fib_proto_to_link(fp), t->mcast_sw_if_index, mcast_mac);
390 flood_class = VNET_FLOOD_CLASS_TUNNEL_MASTER;
392 /* Stack mcast dst mac addr rewrite on encap */
393 dpo_proto_t dproto = fib_proto_to_dpo(fp);
394 dpo_id_t dpo = DPO_INVALID;
396 dpo_set (&dpo, DPO_ADJACENCY, dproto, t->mcast_adj_index);
397 dpo_stack_from_node (encap_index, &t->next_dpo, &dpo);
400 /* Add local mcast adj. */
401 receive_dpo_add_or_lock(dproto, ~0, NULL, &dpo);
402 t->fib_entry_index = fib_table_entry_special_dpo_add
403 (t->encap_fib_index, &tun_dst_pfx, FIB_SOURCE_SPECIAL, FIB_ENTRY_FLAG_NONE, &dpo);
407 * source the FIB entry for the tunnel's destination
408 * and become a child thereof. The tunnel will then get poked
409 * when the forwarding for the entry updates, and the tunnel can
410 * re-stack accordingly
412 t->fib_entry_index = fib_table_entry_special_add
413 (t->encap_fib_index, &tun_dst_pfx, FIB_SOURCE_RR, FIB_ENTRY_FLAG_NONE, ADJ_INDEX_INVALID);
414 t->sibling_index = fib_entry_child_add
415 (t->fib_entry_index, FIB_NODE_TYPE_VXLAN_TUNNEL, t - vxm->tunnels);
416 vxlan_tunnel_restack_dpo(t);
418 /* Set vxlan tunnel output node */
419 hi->output_node_index = encap_index;
421 vnet_get_sw_interface (vnet_get_main(), sw_if_index)->flood_class = flood_class;
425 /* deleting a tunnel: tunnel must exist */
427 return VNET_API_ERROR_NO_SUCH_ENTRY;
429 t = pool_elt_at_index (vxm->tunnels, p[0]);
431 vnet_sw_interface_set_flags (vnm, t->sw_if_index, 0 /* down */);
432 /* make sure tunnel is removed from l2 bd or xconnect */
433 set_int_l2_mode(vxm->vlib_main, vnm, MODE_L3, t->sw_if_index, 0, 0, 0, 0);
434 vec_add1 (vxm->free_vxlan_tunnel_hw_if_indices, t->hw_if_index);
436 vxm->tunnel_index_by_sw_if_index[t->sw_if_index] = ~0;
438 if (ip46_address_is_multicast(&t->dst))
440 adj_unlock(t->mcast_adj_index);
441 fib_table_entry_delete_index(t->fib_entry_index, FIB_SOURCE_SPECIAL);
445 fib_entry_child_remove(t->fib_entry_index, t->sibling_index);
446 fib_table_entry_delete_index(t->fib_entry_index, FIB_SOURCE_RR);
448 fib_node_deinit(&t->node);
452 hash_unset (vxm->vxlan4_tunnel_by_key, key4.as_u64);
457 hash_unset (vxm->vtep4, a->src.ip4.as_u32);
462 hash_unset_mem (vxm->vxlan6_tunnel_by_key, t->key6);
463 clib_mem_free (t->key6);
465 vec_free (t->rewrite);
466 pool_put (vxm->tunnels, t);
470 *sw_if_indexp = sw_if_index;
475 static u32 fib4_index_from_fib_id (u32 fib_id)
477 ip4_main_t * im = &ip4_main;
480 p = hash_get (im->fib_index_by_table_id, fib_id);
487 static u32 fib6_index_from_fib_id (u32 fib_id)
489 ip6_main_t * im = &ip6_main;
492 p = hash_get (im->fib_index_by_table_id, fib_id);
499 static uword unformat_decap_next (unformat_input_t * input, va_list * args)
501 u32 * result = va_arg (*args, u32 *);
504 if (unformat (input, "l2"))
505 *result = VXLAN_INPUT_NEXT_L2_INPUT;
506 else if (unformat (input, "%d", &tmp))
513 static clib_error_t *
514 vxlan_add_del_tunnel_command_fn (vlib_main_t * vm,
515 unformat_input_t * input,
516 vlib_cli_command_t * cmd)
518 unformat_input_t _line_input, * line_input = &_line_input;
519 ip46_address_t src , dst;
526 u32 encap_fib_index = 0;
527 u32 mcast_sw_if_index = ~0;
528 u32 decap_next_index = ~0;
532 vnet_vxlan_add_del_tunnel_args_t _a, * a = &_a;
533 u32 tunnel_sw_if_index;
535 /* Cant "universally zero init" (={0}) due to GCC bug 53119 */
536 memset(&src, 0, sizeof src);
537 memset(&dst, 0, sizeof dst);
539 /* Get a line of input. */
540 if (! unformat_user (input, unformat_line_input, line_input))
543 while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) {
544 if (unformat (line_input, "del"))
548 else if (unformat (line_input, "src %U",
549 unformat_ip4_address, &src.ip4))
554 else if (unformat (line_input, "dst %U",
555 unformat_ip4_address, &dst.ip4))
560 else if (unformat (line_input, "src %U",
561 unformat_ip6_address, &src.ip6))
566 else if (unformat (line_input, "dst %U",
567 unformat_ip6_address, &dst.ip6))
572 else if (unformat (line_input, "group %U %U",
573 unformat_ip4_address, &dst.ip4,
574 unformat_vnet_sw_interface,
575 vnet_get_main(), &mcast_sw_if_index))
577 grp_set = dst_set = 1;
580 else if (unformat (line_input, "group %U %U",
581 unformat_ip6_address, &dst.ip6,
582 unformat_vnet_sw_interface,
583 vnet_get_main(), &mcast_sw_if_index))
585 grp_set = dst_set = 1;
588 else if (unformat (line_input, "encap-vrf-id %d", &tmp))
591 encap_fib_index = fib6_index_from_fib_id (tmp);
593 encap_fib_index = fib4_index_from_fib_id (tmp);
594 if (encap_fib_index == ~0)
595 return clib_error_return (0, "nonexistent encap-vrf-id %d", tmp);
597 else if (unformat (line_input, "decap-next %U", unformat_decap_next,
600 else if (unformat (line_input, "vni %d", &vni))
603 return clib_error_return (0, "vni %d out of range", vni);
606 return clib_error_return (0, "parse error: '%U'",
607 format_unformat_error, line_input);
610 unformat_free (line_input);
613 return clib_error_return (0, "tunnel src address not specified");
616 return clib_error_return (0, "tunnel dst address not specified");
618 if (grp_set && !ip46_address_is_multicast(&dst))
619 return clib_error_return (0, "tunnel group address not multicast");
621 if (grp_set && mcast_sw_if_index == ~0)
622 return clib_error_return (0, "tunnel nonexistent multicast device");
624 if (ipv4_set && ipv6_set)
625 return clib_error_return (0, "both IPv4 and IPv6 addresses specified");
627 if (ip46_address_cmp(&src, &dst) == 0)
628 return clib_error_return (0, "src and dst addresses are identical");
631 return clib_error_return (0, "vni not specified");
633 memset (a, 0, sizeof (*a));
636 a->is_ip6 = ipv6_set;
638 #define _(x) a->x = x;
642 rv = vnet_vxlan_add_del_tunnel (a, &tunnel_sw_if_index);
648 vlib_cli_output(vm, "%U\n", format_vnet_sw_if_index_name,
649 vnet_get_main(), tunnel_sw_if_index);
652 case VNET_API_ERROR_TUNNEL_EXIST:
653 return clib_error_return (0, "tunnel already exists...");
655 case VNET_API_ERROR_NO_SUCH_ENTRY:
656 return clib_error_return (0, "tunnel does not exist...");
659 return clib_error_return
660 (0, "vnet_vxlan_add_del_tunnel returned %d", rv);
667 * Add or delete a VXLAN Tunnel.
669 * VXLAN provides the features needed to allow L2 bridge domains (BDs)
670 * to span multiple servers. This is done by building an L2 overlay on
671 * top of an L3 network underlay using VXLAN tunnels.
673 * This makes it possible for servers to be co-located in the same data
674 * center or be separated geographically as long as they are reachable
675 * through the underlay L3 network.
677 * You can refer to this kind of L2 overlay bridge domain as a VXLAN
678 * (Virtual eXtensible VLAN) segment.
681 * Example of how to create a VXLAN Tunnel:
682 * @cliexcmd{create vxlan tunnel src 10.0.3.1 dst 10.0.3.3 vni 13 encap-vrf-id 7}
683 * Example of how to delete a VXLAN Tunnel:
684 * @cliexcmd{create vxlan tunnel src 10.0.3.1 dst 10.0.3.3 vni 13 del}
687 VLIB_CLI_COMMAND (create_vxlan_tunnel_command, static) = {
688 .path = "create vxlan tunnel",
690 "create vxlan tunnel src <local-vtep-addr>"
691 " {dst <remote-vtep-addr>|group <mcast-vtep-addr> <intf-name>} vni <nn>"
692 " [encap-vrf-id <nn>]",
693 .function = vxlan_add_del_tunnel_command_fn,
697 static clib_error_t *
698 show_vxlan_tunnel_command_fn (vlib_main_t * vm,
699 unformat_input_t * input,
700 vlib_cli_command_t * cmd)
702 vxlan_main_t * vxm = &vxlan_main;
705 if (pool_elts (vxm->tunnels) == 0)
706 vlib_cli_output (vm, "No vxlan tunnels configured...");
708 pool_foreach (t, vxm->tunnels,
710 vlib_cli_output (vm, "%U", format_vxlan_tunnel, t);
717 * Display all the VXLAN Tunnel entries.
720 * Example of how to display the VXLAN Tunnel entries:
721 * @cliexstart{show vxlan tunnel}
722 * [0] src 10.0.3.1 dst 10.0.3.3 vni 13 encap_fib_index 0 sw_if_index 5
726 VLIB_CLI_COMMAND (show_vxlan_tunnel_command, static) = {
727 .path = "show vxlan tunnel",
728 .short_help = "show vxlan tunnel",
729 .function = show_vxlan_tunnel_command_fn,
734 clib_error_t *vxlan_init (vlib_main_t *vm)
736 vxlan_main_t * vxm = &vxlan_main;
738 vxm->vnet_main = vnet_get_main();
741 /* initialize the ip6 hash */
742 vxm->vxlan6_tunnel_by_key = hash_create_mem(0,
743 sizeof(vxlan6_tunnel_key_t),
746 udp_register_dst_port (vm, UDP_DST_PORT_vxlan,
747 vxlan4_input_node.index, /* is_ip4 */ 1);
748 udp_register_dst_port (vm, UDP_DST_PORT_vxlan6,
749 vxlan6_input_node.index, /* is_ip4 */ 0);
751 fib_node_register_type(FIB_NODE_TYPE_VXLAN_TUNNEL, &vxlan_vft);
756 VLIB_INIT_FUNCTION(vxlan_init);