DPDK_MARCH="native"
endif
+DPDK_TUNE = $(strip $($(PLATFORM)_mtune))
+ifeq ($(DPDK_TUNE),)
+ DPDK_MARCH="generic"
+endif
+
ifneq (,$(findstring debug,$(TAG)))
DPDK_DEBUG=y
else
DPDK_BUILD_DIR=$(PACKAGE_BUILD_DIR) \
DPDK_INSTALL_DIR=$(PACKAGE_INSTALL_DIR) \
DPDK_MARCH=$(DPDK_MARCH) \
+ DPDK_TUNE=$(DPDK_TUNE) \
DPDK_DEBUG=$(DPDK_DEBUG)
# vector packet processor
vpp_arch = native
+vpp_march = corei7 # Nehalem Instruction set
+vpp_mtune = corei7-avx # Optimize for Sandy Bridge
+vpp_dpdk_arch = corei7
vpp_native_tools = vppapigen
vpp_uses_dpdk = yes
vpp_debug_TAG_LDFLAGS = -g -O0 -DCLIB_DEBUG -DFORTIFY_SOURCE=2 -march=$(MARCH) \
-fstack-protector-all -fPIC -Werror
-vpp_TAG_CFLAGS = -g -O2 -DFORTIFY_SOURCE=2 -march=$(MARCH) \
+vpp_TAG_CFLAGS = -g -O2 -DFORTIFY_SOURCE=2 -march=$(MARCH) -mtune=$(MTUNE) \
-fstack-protector -fPIC -Werror
-vpp_TAG_LDFLAGS = -g -O2 -DFORTIFY_SOURCE=2 -march=$(MARCH) \
+vpp_TAG_LDFLAGS = -g -O2 -DFORTIFY_SOURCE=2 -march=$(MARCH) -mtune=$(MTUNE) \
-fstack-protector -fPIC -Werror
vpp_gcov_TAG_CFLAGS = -g -O0 -DCLIB_DEBUG -march=$(MARCH) \
# vector packet processor
vpp_lite_arch = native
+vpp_lite_march = corei7 # Nehalem Instruction set
+vpp_lite_mtune = corei7-avx # Optimize for Sandy Bridge
vpp_lite_native_tools = vppapigen
vpp_lite_uses_dpdk = no
vpp_lite_debug_TAG_LDFLAGS = -g -O0 -DCLIB_DEBUG -DFORTIFY_SOURCE=2 -march=$(MARCH) \
-fstack-protector-all -fPIC -Werror
-vpp_lite_TAG_CFLAGS = -g -O2 -DFORTIFY_SOURCE=2 -march=$(MARCH) \
+vpp_lite_TAG_CFLAGS = -g -O2 -DFORTIFY_SOURCE=2 -march=$(MARCH) -mtune=$(MTUNE) \
-fstack-protector -fPIC -Werror
-vpp_lite_TAG_LDFLAGS = -g -O2 -DFORTIFY_SOURCE=2 -march=$(MARCH) \
+vpp_lite_TAG_LDFLAGS = -g -O2 -DFORTIFY_SOURCE=2 -march=$(MARCH) -mtune=$(MTUNE) \
-fstack-protector -fPIC -Werror
endif
export MARCH
+MTUNE = $(strip $($(PLATFORM)_mtune))
+ifeq ($(MTUNE),)
+ MTUNE = generic
+endif
+
######################################################################
# Generic build stuff
######################################################################
DPDK_PKTMBUF_HEADROOM ?= 128
DPDK_DOWNLOAD_DIR ?= $(HOME)/Downloads
DPDK_MARCH ?= native
+DPDK_TUNE ?= generic
DPDK_DEBUG ?= n
B := $(DPDK_BUILD_DIR)
DPDK_EXTRA_LDFLAGS := -g
ifeq ($(DPDK_DEBUG),n)
-DPDK_EXTRA_CFLAGS := -g
+DPDK_EXTRA_CFLAGS := -g -mtune=$(DPDK_TUNE)
else
DPDK_EXTRA_CFLAGS := -g -O0
endif
#ifndef included_vlib_node_h
#define included_vlib_node_h
+#include <vppinfra/cpu.h>
#include <vppinfra/longjmp.h>
#include <vppinfra/timing_wheel.h>
#include <vlib/trace.h> /* for vlib_trace_filter_t */
} \
__VA_ARGS__ vlib_node_registration_t x
+#if CLIB_DEBUG > 0
+#define VLIB_NODE_FUNCTION_CLONE_TEMPLATE(arch, fn)
+#define VLIB_NODE_FUNCTION_MULTIARCH_CLONE(fn)
+#define VLIB_NODE_FUNCTION_MULTIARCH(node, fn)
+#else
+#define VLIB_NODE_FUNCTION_CLONE_TEMPLATE(arch, fn, tgt) \
+ uword \
+ __attribute__ ((flatten)) \
+ __attribute__ ((target (tgt))) \
+ CLIB_CPU_OPTIMIZED \
+ fn ## _ ## arch ( struct vlib_main_t * vm, \
+ struct vlib_node_runtime_t * node, \
+ struct vlib_frame_t * frame) \
+ { return fn (vm, node, frame); }
+
+#define VLIB_NODE_FUNCTION_MULTIARCH_CLONE(fn) \
+ foreach_march_variant(VLIB_NODE_FUNCTION_CLONE_TEMPLATE, fn)
+
+#define VLIB_NODE_FUNCTION_MULTIARCH(node, fn) \
+ VLIB_NODE_FUNCTION_MULTIARCH_CLONE(fn) \
+ CLIB_MULTIARCH_SELECT_FN(fn, static inline) \
+ static void __attribute__((__constructor__)) \
+ __vlib_node_function_multiarch_select_##node (void) \
+ { node.function = fn ## _multiarch_select(); }
+#endif
+
always_inline vlib_node_registration_t *
vlib_node_next_registered (vlib_node_registration_t * c)
{
.next_nodes = IP4_LOOKUP_NEXT_NODES,
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip4_classify_node, ip4_classify)
+
static uword
ip6_classify (vlib_main_t * vm,
vlib_node_runtime_t * node,
.next_nodes = IP6_LOOKUP_NEXT_NODES,
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip6_classify_node, ip6_classify)
+
static clib_error_t *
ip_classify_init (vlib_main_t * vm)
{
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip4_cop_whitelist_node, ip4_cop_whitelist_node_fn)
+
static clib_error_t *
ip4_whitelist_init (vlib_main_t * vm)
{
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip6_cop_whitelist_node, ip6_cop_whitelist_node_fn)
+
static clib_error_t *
ip6_whitelist_init (vlib_main_t * vm)
{
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (cop_input_node, cop_input_node_fn)
+
#define foreach_cop_stub \
_(default-cop-whitelist, default_cop_whitelist)
.subif_add_del_function = af_packet_subif_add_del_function,
.no_flatten_output_chains = 1,
};
+
+VLIB_DEVICE_TX_FUNCTION_MULTIARCH (af_packet_device_class,
+ af_packet_interface_tx)
return n_rx_packets;
}
-
VLIB_REGISTER_NODE (af_packet_input_node) = {
.function = af_packet_input_fn,
.name = "af-packet-input",
[AF_PACKET_INPUT_NEXT_ETHERNET_INPUT] = "ethernet-input",
},
};
+
+VLIB_NODE_FUNCTION_MULTIARCH (af_packet_input_node, af_packet_input_fn)
+
dpdk_main_t * dm = &dpdk_main;
vlib_thread_main_t * tm = vlib_get_thread_main();
clib_error_t * error = NULL;
+ vlib_node_runtime_t * rt = vlib_node_get_runtime (vm, dpdk_input_node.index);
if (unformat(input, "enable")) {
if (unformat(input, "dpdk")) {
format_unformat_error, input);
}
+ if (dm->efd.enabled)
+ rt->function = dpdk_input_efd_multiarch_select();
+ else if (dm->use_rss)
+ rt->function = dpdk_input_rss_multiarch_select();
+ else
+ rt->function = dpdk_input_multiarch_select();
+
return error;
}
.name_renumber = dpdk_device_renumber,
};
+VLIB_DEVICE_TX_FUNCTION_MULTIARCH (dpdk_device_class,
+ dpdk_interface_tx)
+
void dpdk_set_flowcontrol_callback (vlib_main_t *vm,
dpdk_flowcontrol_callback_t callback)
{
i8 dpdk_get_cpu_socket (vnet_hw_interface_t *hi);
-uword
-dpdk_input_rss (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * f);
+void * dpdk_input_multiarch_select();
+void * dpdk_input_rss_multiarch_select();
+void * dpdk_input_efd_multiarch_select();
clib_error_t*
dpdk_get_hw_interface_stats (u32 hw_if_index, struct rte_eth_stats* dest);
clib_error_t * error = 0;
dpdk_main_t * dm = &dpdk_main;
vlib_thread_main_t * tm = vlib_get_thread_main();
+ vlib_node_runtime_t * rt = vlib_node_get_runtime (vm, dpdk_input_node.index);
u8 * s, * tmp = 0;
u8 * pci_dev_id = 0;
u8 * rte_cmd = 0, * ethname = 0;
}
if (dm->use_rss)
- {
- vlib_node_runtime_t * rt = vlib_node_get_runtime (vm, dpdk_input_node.index);
- rt->function = dpdk_input_rss;
- }
+ rt->function = dpdk_input_rss_multiarch_select();
+ else
+ rt->function = dpdk_input_multiarch_select();
done:
return error;
}
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (handoff_dispatch_node, handoff_dispatch_node_fn)
+
clib_error_t *handoff_dispatch_init (vlib_main_t *vm)
{
handoff_dispatch_main_t * mp = &handoff_dispatch_main;
dpdk_device_t * xd,
vlib_node_runtime_t * node,
u32 cpu_index,
- u16 queue_id)
+ u16 queue_id,
+ int use_efd)
{
u32 n_buffers;
u32 next_index = DPDK_RX_NEXT_ETHERNET_INPUT;
if (n_buffers == 0)
{
/* check if EFD (dpdk) is enabled */
- if (PREDICT_FALSE(dm->efd.enabled))
+ if (PREDICT_FALSE(use_efd && dm->efd.enabled))
{
/* reset a few stats */
xd->efd_agent.last_poll_time = 0;
/* Check for congestion if EFD (Early-Fast-Discard) is enabled
* in any mode (e.g. dpdk, monitor, or drop_all)
*/
- if (PREDICT_FALSE(dm->efd.enabled))
+ if (PREDICT_FALSE(use_efd && dm->efd.enabled))
{
/* update EFD counters */
dpdk_efd_update_counters(xd, n_buffers, dm->efd.enabled);
{
xd = vec_elt_at_index(dm->devices, dq->device);
ASSERT(dq->queue_id == 0);
- n_rx_packets += dpdk_device_input (dm, xd, node, cpu_index, 0);
+ n_rx_packets += dpdk_device_input (dm, xd, node, cpu_index, 0, 0);
}
VIRL_SPEED_LIMIT()
vec_foreach (dq, dm->devices_by_cpu[cpu_index])
{
xd = vec_elt_at_index(dm->devices, dq->device);
- n_rx_packets += dpdk_device_input (dm, xd, node, cpu_index, dq->queue_id);
+ n_rx_packets += dpdk_device_input (dm, xd, node, cpu_index, dq->queue_id, 0);
}
VIRL_SPEED_LIMIT()
return n_rx_packets;
}
+uword
+dpdk_input_efd (vlib_main_t * vm,
+ vlib_node_runtime_t * node,
+ vlib_frame_t * f)
+{
+ dpdk_main_t * dm = &dpdk_main;
+ dpdk_device_t * xd;
+ uword n_rx_packets = 0;
+ dpdk_device_and_queue_t * dq;
+ u32 cpu_index = os_get_cpu_number();
+
+ /*
+ * Poll all devices on this cpu for input/interrupts.
+ */
+ vec_foreach (dq, dm->devices_by_cpu[cpu_index])
+ {
+ xd = vec_elt_at_index(dm->devices, dq->device);
+ n_rx_packets += dpdk_device_input (dm, xd, node, cpu_index, dq->queue_id, 1);
+ }
+
+ VIRL_SPEED_LIMIT()
+
+ return n_rx_packets;
+}
+
+
VLIB_REGISTER_NODE (dpdk_input_node) = {
.function = dpdk_input,
.type = VLIB_NODE_TYPE_INPUT,
},
};
+
+/* handle dpdk_input_rss alternative function */
+VLIB_NODE_FUNCTION_MULTIARCH_CLONE(dpdk_input)
+VLIB_NODE_FUNCTION_MULTIARCH_CLONE(dpdk_input_rss)
+VLIB_NODE_FUNCTION_MULTIARCH_CLONE(dpdk_input_efd)
+
+/* this macro defines dpdk_input_rss_multiarch_select() */
+CLIB_MULTIARCH_SELECT_FN(dpdk_input);
+CLIB_MULTIARCH_SELECT_FN(dpdk_input_rss);
+CLIB_MULTIARCH_SELECT_FN(dpdk_input_efd);
+
/*
* Override the next nodes for the dpdk input nodes.
* Must be invoked prior to VLIB_INIT_FUNCTION calls.
.subif_add_del_function = netmap_subif_add_del_function,
.no_flatten_output_chains = 1,
};
+
+VLIB_DEVICE_TX_FUNCTION_MULTIARCH(netmap_device_class,
+ netmap_interface_tx)
return n_rx_packets;
}
-
VLIB_REGISTER_NODE (netmap_input_node) = {
.function = netmap_input_fn,
.name = "netmap-input",
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (netmap_input_node, netmap_input_fn)
+
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (ssvm_eth_input_node, ssvm_eth_input_node_fn)
+
.rx_redirect_to_node = ssvm_eth_set_interface_next_node,
.no_flatten_output_chains = 1,
};
+
+VLIB_DEVICE_TX_FUNCTION_MULTIARCH (ssvm_eth_device_class,
+ ssvm_eth_interface_tx)
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (vhost_user_input_node, vhost_user_input)
+
static uword
vhost_user_intfc_tx (vlib_main_t * vm,
vlib_node_runtime_t * node,
.no_flatten_output_chains = 1,
};
+VLIB_DEVICE_TX_FUNCTION_MULTIARCH (vhost_user_dev_class,
+ vhost_user_intfc_tx)
+
static uword
vhost_user_process (vlib_main_t * vm,
vlib_node_runtime_t * rt,
.unformat_buffer = unformat_ethernet_header,
};
+VLIB_NODE_FUNCTION_MULTIARCH (ethernet_input_node, ethernet_input)
+
VLIB_REGISTER_NODE (ethernet_input_type_node,static) = {
.function = ethernet_input_type,
.name = "ethernet-input-type",
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (ethernet_input_type_node, ethernet_input_type)
+
VLIB_REGISTER_NODE (ethernet_input_not_l2_node,static) = {
.function = ethernet_input_not_l2,
.name = "ethernet-input-not-l2",
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (ethernet_input_not_l2_node, ethernet_input_not_l2)
+
void ethernet_set_rx_redirect (vnet_main_t * vnm,
vnet_hw_interface_t * hi,
u32 enable)
#endif
};
+VLIB_DEVICE_TX_FUNCTION_MULTIARCH (gre_device_class,
+ gre_interface_tx)
+
VNET_HW_INTERFACE_CLASS (gre_hw_interface_class) = {
.name = "GRE",
.unformat_buffer = unformat_gre_header,
};
+VLIB_NODE_FUNCTION_MULTIARCH (gre_input_node, gre_input)
+
void
gre_register_input_protocol (vlib_main_t * vm,
gre_protocol_t protocol,
} \
__VA_ARGS__ vnet_device_class_t x
+#define VLIB_DEVICE_TX_FUNCTION_CLONE_TEMPLATE(arch, fn, tgt) \
+ uword \
+ __attribute__ ((flatten)) \
+ __attribute__ ((target (tgt))) \
+ CLIB_CPU_OPTIMIZED \
+ fn ## _ ## arch ( vlib_main_t * vm, \
+ vlib_node_runtime_t * node, \
+ vlib_frame_t * frame) \
+ { return fn (vm, node, frame); }
+
+#define VLIB_DEVICE_TX_FUNCTION_MULTIARCH_CLONE(fn) \
+ foreach_march_variant(VLIB_DEVICE_TX_FUNCTION_CLONE_TEMPLATE, fn)
+
+#if CLIB_DEBUG > 0
+#define VLIB_MULTIARCH_CLONE_AND_SELECT_FN(fn,...)
+#define VLIB_DEVICE_TX_FUNCTION_MULTIARCH(dev, fn)
+#else
+#define VLIB_DEVICE_TX_FUNCTION_MULTIARCH(dev, fn) \
+ VLIB_DEVICE_TX_FUNCTION_MULTIARCH_CLONE(fn) \
+ CLIB_MULTIARCH_SELECT_FN(fn, static inline) \
+ static void __attribute__((__constructor__)) \
+ __vlib_device_tx_function_multiarch_select_##dev (void) \
+ { dev.tx_function = fn ## _multiarch_select(); }
+#endif
+
+
/* Layer-2 (e.g. Ethernet) interface class. */
typedef struct _vnet_hw_interface_class {
/* Index into main vector. */
.validate_frame = validate_error_frame,
};
+VLIB_NODE_FUNCTION_MULTIARCH (drop_buffers, process_drop)
+
VLIB_REGISTER_NODE (punt_buffers,static) = {
.function = process_punt,
.flags = (VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH
.validate_frame = validate_error_frame,
};
+VLIB_NODE_FUNCTION_MULTIARCH (punt_buffers, process_punt)
+
VLIB_REGISTER_NODE (vnet_per_buffer_interface_output_node,static) = {
.function = vnet_per_buffer_interface_output,
.name = "interface-output",
.vector_size = sizeof (u32),
};
+VLIB_NODE_FUNCTION_MULTIARCH (vnet_per_buffer_interface_output_node, vnet_per_buffer_interface_output)
+
clib_error_t *
vnet_per_buffer_interface_output_hw_interface_add_del (vnet_main_t * vnm,
u32 hw_if_index,
.next_nodes = IP4_LOOKUP_NEXT_NODES,
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip4_lookup_node, ip4_lookup)
+
static uword
ip4_indirect (vlib_main_t * vm,
vlib_node_runtime_t * node,
.next_nodes = IP4_LOOKUP_NEXT_NODES,
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip4_indirect_node, ip4_indirect)
+
/* Global IP4 main. */
ip4_main_t ip4_main;
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip4_drop_node, ip4_drop)
+
VLIB_REGISTER_NODE (ip4_punt_node,static) = {
.function = ip4_punt,
.name = "ip4-punt",
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip4_punt_node, ip4_punt)
+
VLIB_REGISTER_NODE (ip4_miss_node,static) = {
.function = ip4_miss,
.name = "ip4-miss",
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip4_miss_node, ip4_miss)
+
/* Compute TCP/UDP/ICMP4 checksum in software. */
u16
ip4_tcp_udp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0,
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip4_local_node, ip4_local)
+
void ip4_register_protocol (u32 protocol, u32 node_index)
{
vlib_main_t * vm = vlib_get_main();
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_node, ip4_rewrite_transit)
+
VLIB_REGISTER_NODE (ip4_rewrite_local_node,static) = {
.function = ip4_rewrite_local,
.name = "ip4-rewrite-local",
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_local_node, ip4_rewrite_local)
+
static clib_error_t *
add_del_interface_table (vlib_main_t * vm,
unformat_input_t * input,
.next_nodes = IP4_LOOKUP_NEXT_NODES,
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip4_lookup_multicast_node, ip4_lookup_multicast)
+
VLIB_REGISTER_NODE (ip4_multicast_node,static) = {
.function = ip4_drop,
.name = "ip4-multicast",
.next_nodes = IP4_LOOKUP_NEXT_NODES,
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip4_hop_by_hop_node, ip4_hop_by_hop_node_fn)
+
VLIB_REGISTER_NODE (ip4_add_hop_by_hop_node) = {
.function = ip4_hop_by_hop_node_fn,
.name = "ip4-add-hop-by-hop",
.format_trace = format_ip4_input_trace,
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip4_input_node, ip4_input)
+
VLIB_REGISTER_NODE (ip4_input_no_checksum_node,static) = {
.function = ip4_input_no_checksum,
.name = "ip4-input-no-checksum",
.format_trace = format_ip4_input_trace,
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip4_input_no_checksum_node, ip4_input_no_checksum)
+
static clib_error_t * ip4_init (vlib_main_t * vm)
{
clib_error_t * error;
.format_trace = format_ip4_source_check_trace,
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip4_check_source_reachable_via_any,
+ ip4_source_check_reachable_via_any)
+
VLIB_REGISTER_NODE (ip4_check_source_reachable_via_rx) = {
.function = ip4_source_check_reachable_via_rx,
.name = "ip4-source-check-via-rx",
.format_trace = format_ip4_source_check_trace,
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip4_check_source_reachable_via_rx,
+ ip4_source_check_reachable_via_rx)
+
static clib_error_t *
set_ip_source_check (vlib_main_t * vm,
unformat_input_t * input,
.next_nodes = IP6_LOOKUP_NEXT_NODES,
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip6_lookup_node, ip6_lookup)
+
static uword
ip6_indirect (vlib_main_t * vm,
vlib_node_runtime_t * node,
.next_nodes = IP6_LOOKUP_NEXT_NODES,
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip6_indirect_node, ip6_indirect)
+
typedef struct {
/* Adjacency taken. */
u32 adj_index;
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip6_drop_node, ip6_drop)
+
VLIB_REGISTER_NODE (ip6_punt_node,static) = {
.function = ip6_punt,
.name = "ip6-punt",
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip6_punt_node, ip6_punt)
+
VLIB_REGISTER_NODE (ip6_miss_node,static) = {
.function = ip6_miss,
.name = "ip6-miss",
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip6_miss_node, ip6_miss)
+
VLIB_REGISTER_NODE (ip6_multicast_node,static) = {
.function = ip6_drop,
.name = "ip6-multicast",
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip6_local_node, ip6_local)
+
void ip6_register_protocol (u32 protocol, u32 node_index)
{
vlib_main_t * vm = vlib_get_main();
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_node, ip6_rewrite_transit)
+
VLIB_REGISTER_NODE (ip6_rewrite_local_node,static) = {
.function = ip6_rewrite_local,
.name = "ip6-rewrite-local",
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_local_node, ip6_rewrite_local)
+
/* Global IP6 main. */
ip6_main_t ip6_main;
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip6_hop_by_hop_node, ip6_hop_by_hop_node_fn)
+
/* The main h-b-h tracer will be invoked, no need to do much here */
typedef struct {
u32 next_index;
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip6_add_hop_by_hop_node, ip6_add_hop_by_hop_node_fn)
/* The main h-b-h tracer was already invoked, no need to do much here */
typedef struct {
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip6_pop_hop_by_hop_node,
+ ip6_pop_hop_by_hop_node_fn)
static clib_error_t *
ip6_hop_by_hop_init (vlib_main_t * vm)
.format_trace = format_ip6_input_trace,
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip6_input_node, ip6_input)
+
static clib_error_t * ip6_init (vlib_main_t * vm)
{
ethernet_register_input_type (vm, ETHERNET_TYPE_IP6,
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip4_inacl_node, ip4_inacl)
+
static uword
ip6_inacl (vlib_main_t * vm,
vlib_node_runtime_t * node,
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (ip6_inacl_node, ip6_inacl)
+
static clib_error_t *
ip_inacl_init (vlib_main_t * vm)
{
.unformat_buffer = unformat_udp_header,
};
+VLIB_NODE_FUNCTION_MULTIARCH (udp4_input_node, udp4_input)
+
VLIB_REGISTER_NODE (udp6_input_node) = {
.function = udp6_input,
.name = "ip6-udp-lookup",
.unformat_buffer = unformat_udp_header,
};
+VLIB_NODE_FUNCTION_MULTIARCH (udp6_input_node, udp6_input)
+
static void add_dst_port (udp_main_t * um,
udp_dst_port_t dst_port,
char * dst_port_name, u8 is_ip4)
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (esp_decrypt_node, esp_decrypt_node_fn)
+
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (esp_encrypt_node, esp_encrypt_node_fn)
+
[IPSEC_IF_INPUT_NEXT_ESP_DECRYPT] = "esp-decrypt",
[IPSEC_IF_INPUT_NEXT_DROP] = "error-drop",
},
-};
\ No newline at end of file
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (ipsec_if_input_node, ipsec_if_input_node_fn)
+
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (ipsec_if_output_node, ipsec_if_output_node_fn)
+
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (ipsec_input_ip4_node,
+ ipsec_input_ip4_node_fn)
+
static vlib_node_registration_t ipsec_input_ip6_node;
#undef _
},
};
+
+VLIB_NODE_FUNCTION_MULTIARCH (ipsec_input_ip6_node,
+ ipsec_input_ip6_node_fn)
+
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (ipsec_output_node, ipsec_output_node_fn)
+
#else /* IPSEC > 1 */
/* Dummy ipsec output node, in case when IPSec is disabled */
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (l2_classify_node, l2_classify_node_fn)
+
clib_error_t *l2_classify_init (vlib_main_t *vm)
{
l2_classify_main_t * cm = &l2_classify_main;
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (l2_efp_filter_node, l2_efp_filter_node_fn)
+
clib_error_t *l2_efp_filter_init (vlib_main_t *vm)
{
l2_efp_filter_main_t * mp = &l2_efp_filter_main;
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (l2flood_node, l2flood_node_fn)
+
clib_error_t *l2flood_init (vlib_main_t *vm)
{
l2flood_main_t * mp = &l2flood_main;
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (l2fwd_node, l2fwd_node_fn)
+
clib_error_t *l2fwd_init (vlib_main_t *vm)
{
l2fwd_main_t * mp = &l2fwd_main;
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (l2input_node, l2input_node_fn)
+
clib_error_t *l2input_init (vlib_main_t *vm)
{
l2input_main_t * mp = &l2input_main;
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (l2_inacl_node, l2_inacl_node_fn)
+
clib_error_t *l2_inacl_init (vlib_main_t *vm)
{
l2_inacl_main_t * mp = &l2_inacl_main;
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (l2_invtr_node, l2_invtr_node_fn)
+
clib_error_t *l2_invtr_init (vlib_main_t *vm)
{
l2_invtr_main_t * mp = &l2_invtr_main;
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (l2learn_node, l2learn_node_fn)
clib_error_t *l2learn_init (vlib_main_t *vm)
{
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (l2output_node, l2output_node_fn)
+
clib_error_t *l2output_init (vlib_main_t *vm)
{
l2output_main_t * mp = &l2output_main;
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (l2_outacl_node, l2_outacl_node_fn)
+
clib_error_t *l2_outacl_init (vlib_main_t *vm)
{
l2_outacl_main_t * mp = &l2_outacl_main;
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (l2_patch_node, l2_patch_node_fn)
+
int vnet_l2_patch_add_del (u32 rx_sw_if_index, u32 tx_sw_if_index, int is_add)
{
l2_patch_main_t * l2pm = &l2_patch_main;
.next_nodes = { [L2_RW_NEXT_DROP] = "error-drop"},
};
+VLIB_NODE_FUNCTION_MULTIARCH (l2_rw_node, l2_rw_node_fn)
+
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (l2_xcrw_node, l2_xcrw_node_fn)
+
clib_error_t *l2_xcrw_init (vlib_main_t *vm)
{
l2_xcrw_main_t * mp = &l2_xcrw_main;
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (l2t_decap_node, l2t_decap_node_fn)
+
void l2tp_decap_init (void)
{
ip6_register_protocol (IP_PROTOCOL_L2TP, l2t_decap_node.index);
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (l2t_encap_node, l2t_encap_node_fn)
+
void l2tp_encap_init (vlib_main_t * vm)
{
l2tp_encap_runtime_t * rt;
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (li_hit_node, li_hit_node_fn)
+
#else
#include <vlib/vlib.h>
#endif
};
+VLIB_DEVICE_TX_FUNCTION_MULTIARCH (mpls_gre_device_class,
+ mpls_gre_interface_tx)
+
VNET_HW_INTERFACE_CLASS (mpls_gre_hw_interface_class) = {
.name = "MPLS-GRE",
.format_header = format_mpls_gre_header_with_length,
#endif
};
+VLIB_DEVICE_TX_FUNCTION_MULTIARCH (mpls_eth_device_class,
+ mpls_eth_interface_tx)
VNET_HW_INTERFACE_CLASS (mpls_eth_hw_interface_class) = {
.name = "MPLS-ETH",
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (mpls_post_rewrite_node, mpls_post_rewrite)
+
static u8 * mpls_gre_rewrite (mpls_main_t *mm, mpls_gre_tunnel_t * t)
{
ip4_header_t * ip0;
.unformat_buffer = unformat_mpls_gre_header,
};
+VLIB_NODE_FUNCTION_MULTIARCH (mpls_input_node, mpls_input)
+
static uword
mpls_ethernet_input (vlib_main_t * vm,
vlib_node_runtime_t * node,
.unformat_buffer = unformat_mpls_gre_header,
};
+VLIB_NODE_FUNCTION_MULTIARCH (mpls_ethernet_input_node, mpls_ethernet_input)
+
static void
mpls_setup_nodes (vlib_main_t * vm)
{
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (mpls_policy_encap_node, mpls_policy_encap)
+
static clib_error_t *
mpls_policy_encap_init (vlib_main_t * vm)
{
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (policer_by_sw_if_index_node,
+ vnet_policer_by_sw_if_index);
+
int test_policer_add_del (u32 rx_sw_if_index, u8 *config_name,
int is_add)
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (sr_rewrite_node, sr_rewrite)
+
static int ip6_delete_route_no_next_hop (ip6_address_t *dst_address_arg,
u32 dst_address_length,
u32 rx_table_id)
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (sr_fix_dst_addr_node, sr_fix_dst_addr)
+
static clib_error_t * sr_init (vlib_main_t * vm)
{
ip6_sr_main_t * sm = &sr_main;
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (sr_local_node, sr_local)
+
ip6_sr_main_t * sr_get_main (vlib_main_t * vm)
{
vlib_call_init_function (vm, sr_init);
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (sr_replicate_node, sr_replicate_node_fn)
+
clib_error_t *sr_replicate_init (vlib_main_t *vm)
{
sr_replicate_main_t *msm = &sr_replicate_main;
// $$$$ .unformat_buffer = unformat_vxlan_header,
};
+VLIB_NODE_FUNCTION_MULTIARCH (vxlan4_input_node, vxlan4_input)
+
VLIB_REGISTER_NODE (vxlan6_input_node) = {
.function = vxlan6_input,
.name = "vxlan6-input",
.format_trace = format_vxlan_rx_trace,
// $$$$ .unformat_buffer = unformat_vxlan_header,
};
+
+VLIB_NODE_FUNCTION_MULTIARCH (vxlan6_input_node, vxlan6_input)
+
[VXLAN_ENCAP_NEXT_DROP] = "error-drop",
},
};
+
+VLIB_NODE_FUNCTION_MULTIARCH (vxlan_encap_node, vxlan_encap)
+
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (sw6_ip6_node, ip6_l2t_node_fn)
+
static clib_error_t *
l2tp_config (vlib_main_t * vm, unformat_input_t * input)
{
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (l2t_l2_node, l2t_l2_node_fn)
+
_("Compiler", "%s", vpe_compiler);
_("CPU model name", "%U", format_cpu_model_name);
_("CPU microarchitecture", "%U", format_cpu_uarch);
+ _("CPU flags", "%U", format_cpu_flags);
_("Current PID", "%d", getpid());
#if DPDK > 0
_("DPDK Version", "%s", rte_version());
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
+#include <vppinfra/cpu.h>
#include <vlib/vlib.h>
#include <vlib/unix/unix.h>
#include <vnet/plugin/plugin.h>
u32 size;
void vlib_set_get_handoff_structure_cb (void *cb);
-#if __x86_64__ && !defined(__clang__)
- __builtin_cpu_init ();
+#if __x86_64__
const char * msg = "ERROR: This binary requires CPU with %s extensions.\n";
#define _(a,b) \
- if (!__builtin_cpu_supports(a)) \
+ if (!clib_cpu_supports_ ## a ()) \
{ \
fprintf(stderr, msg, b); \
exit(1); \
}
#if __AVX2__
- _("avx2", "AVX2")
+ _(avx2, "AVX2")
#endif
#if __AVX__
- _("avx", "AVX")
+ _(avx, "AVX")
#endif
#if __SSE4_2__
- _("sse4.2", "SSE4.2")
+ _(sse42, "SSE4.2")
#endif
#if __SSE4_1__
- _("sse4.1", "SSE4.1")
+ _(sse41, "SSE4.1")
#endif
#if __SSSE3__
- _("ssse3", "SSSE3")
+ _(ssse3, "SSSE3")
#endif
#if __SSE3__
- _("sse3", "SSE3")
+ _(sse3, "SSE3")
#endif
#undef _
#endif
#include <vppinfra/format.h>
#include <vppinfra/cpu.h>
-#if __x86_64__
-#include <cpuid.h>
-#endif
-
#define foreach_x86_cpu_uarch \
_(0x06, 0x4f, "Broadwell", "Broadwell-EP/EX") \
_(0x06, 0x3d, "Broadwell", "Broadwell") \
#else /* ! __x86_64__ */
return format (s, "unknown");
#endif
-}
\ No newline at end of file
+}
+
+u8 *
+format_cpu_flags (u8 * s, va_list * args)
+{
+#if __x86_64__
+#define _(flag, func, reg, bit) \
+ if (clib_cpu_supports_ ## flag()) \
+ s = format (s, #flag " ");
+ foreach_x86_64_flags
+ return s;
+#undef _
+#else /* ! __x86_64__ */
+ return format (s, "unknown");
+#endif
+}
+
+
#ifndef included_clib_cpu_h
#define included_clib_cpu_h
+#include <vppinfra/format.h>
+
+/*
+ * multiarchitecture support. Adding new entry will produce
+ * new graph node function variant optimized for specific cpu
+ * microarchitecture.
+ * Order is important for runtime selection, as 1st match wins...
+ */
+
+#if __x86_64__ && CLIB_DEBUG == 0
+#define foreach_march_variant(macro, x) \
+ macro(avx2, x, "arch=core-avx2")
+#else
+#define foreach_march_variant(macro, x)
+#endif
+
+
+#if __GNUC__ > 4 && !__clang__
+#define CLIB_CPU_OPTIMIZED __attribute__ ((optimize ("tree-vectorize")))
+#else
+#define CLIB_CPU_OPTIMIZED
+#endif
+
+
+#define CLIB_MULTIARCH_ARCH_CHECK(arch, fn, tgt) \
+ if (clib_cpu_supports_ ## arch()) \
+ return & fn ## _ ##arch;
+
+#define CLIB_MULTIARCH_SELECT_FN(fn,...) \
+ __VA_ARGS__ void * fn ## _multiarch_select(void) \
+{ \
+ foreach_march_variant(CLIB_MULTIARCH_ARCH_CHECK, fn) \
+ return & fn; \
+}
+
+#if __x86_64__
+#include "cpuid.h"
+
+#define foreach_x86_64_flags \
+_ (sse3, 1, ecx, 0) \
+_ (ssse3, 1, ecx, 9) \
+_ (sse41, 1, ecx, 19) \
+_ (sse42, 1, ecx, 20) \
+_ (avx, 1, ecx, 28) \
+_ (avx2, 7, ebx, 5) \
+_ (avx512f, 7, ebx, 16) \
+_ (aes, 1, ecx, 25) \
+_ (sha, 7, ebx, 29)
+
+static inline int
+clib_get_cpuid(const u32 lev, u32 * eax, u32 *ebx, u32 * ecx, u32 * edx)
+{
+ if ((u32) __get_cpuid_max (0x80000000 & lev, 0) < lev)
+ return 0;
+ if (lev == 7)
+ __cpuid_count(lev, 0, *eax, *ebx, *ecx, *edx);
+ else
+ __cpuid(lev, *eax, *ebx, *ecx, *edx);
+ return 1;
+}
+
+
+#define _(flag, func, reg, bit) \
+static inline int \
+clib_cpu_supports_ ## flag() \
+{ \
+ u32 __attribute__((unused)) eax, ebx = 0, ecx = 0, edx = 0; \
+ clib_get_cpuid (func, &eax, &ebx, &ecx, &edx); \
+ \
+ return ((reg & (1 << bit)) != 0); \
+}
+ foreach_x86_64_flags
+#undef _
+#endif
+
format_function_t format_cpu_uarch;
format_function_t format_cpu_model_name;
+format_function_t format_cpu_flags;
-#endif
\ No newline at end of file
+#endif
Code Review / vpp.git / commitdiff