X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Fdpdk%2Fdevice%2Finit.c;h=2d21bfd91caca5948a1eb149d5c8589242e0a164;hb=4403690cda44134af3b9ea78d33a5cbf78a5acc9;hp=39d919e21747ef6cb1be78f949182b79dcd7cc53;hpb=250b95b71babdfb558554c788a82cf45ccc34ab8;p=vpp.git diff --git a/src/plugins/dpdk/device/init.c b/src/plugins/dpdk/device/init.c index 39d919e2174..2d21bfd91ca 100755 --- a/src/plugins/dpdk/device/init.c +++ b/src/plugins/dpdk/device/init.c @@ -35,269 +35,6 @@ dpdk_main_t dpdk_main; -#include -#include - -/* define message IDs */ -#include - -#define vl_typedefs /* define message structures */ -#include -#undef vl_typedefs - -#define vl_endianfun /* define message structures */ -#include -#undef vl_endianfun - -#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) - -/* Get the API version number. */ -#define vl_api_version(n,v) static u32 api_version=(v); -#include -#undef vl_api_version - -/* Macro to finish up custom dump fns */ -#define FINISH \ - vec_add1 (s, 0); \ - vl_print (handle, (char *)s); \ - vec_free (s); \ - return handle; - -#include - -static void - vl_api_sw_interface_set_dpdk_hqos_pipe_t_handler - (vl_api_sw_interface_set_dpdk_hqos_pipe_t * mp) -{ - vl_api_sw_interface_set_dpdk_hqos_pipe_reply_t *rmp; - int rv = 0; - - dpdk_main_t *dm = &dpdk_main; - dpdk_device_t *xd; - - u32 sw_if_index = ntohl (mp->sw_if_index); - u32 subport = ntohl (mp->subport); - u32 pipe = ntohl (mp->pipe); - u32 profile = ntohl (mp->profile); - vnet_hw_interface_t *hw; - - VALIDATE_SW_IF_INDEX (mp); - - /* hw_if & dpdk device */ - hw = vnet_get_sup_hw_interface (dm->vnet_main, sw_if_index); - - xd = vec_elt_at_index (dm->devices, hw->dev_instance); - - rv = rte_sched_pipe_config (xd->hqos_ht->hqos, subport, pipe, profile); - - BAD_SW_IF_INDEX_LABEL; - - REPLY_MACRO (VL_API_SW_INTERFACE_SET_DPDK_HQOS_PIPE_REPLY); -} - -static void *vl_api_sw_interface_set_dpdk_hqos_pipe_t_print - (vl_api_sw_interface_set_dpdk_hqos_pipe_t * mp, void *handle) -{ - u8 *s; - - s = format (0, "SCRIPT: sw_interface_set_dpdk_hqos_pipe "); - - s = format (s, "sw_if_index %u ", ntohl (mp->sw_if_index)); - - s = format (s, "subport %u pipe %u profile %u ", - ntohl (mp->subport), ntohl (mp->pipe), ntohl (mp->profile)); - - FINISH; -} - -static void - vl_api_sw_interface_set_dpdk_hqos_subport_t_handler - (vl_api_sw_interface_set_dpdk_hqos_subport_t * mp) -{ - vl_api_sw_interface_set_dpdk_hqos_subport_reply_t *rmp; - int rv = 0; - - dpdk_main_t *dm = &dpdk_main; - dpdk_device_t *xd; - struct rte_sched_subport_params p; - - u32 sw_if_index = ntohl (mp->sw_if_index); - u32 subport = ntohl (mp->subport); - p.tb_rate = ntohl (mp->tb_rate); - p.tb_size = ntohl (mp->tb_size); - p.tc_rate[0] = ntohl (mp->tc_rate[0]); - p.tc_rate[1] = ntohl (mp->tc_rate[1]); - p.tc_rate[2] = ntohl (mp->tc_rate[2]); - p.tc_rate[3] = ntohl (mp->tc_rate[3]); - p.tc_period = ntohl (mp->tc_period); - - vnet_hw_interface_t *hw; - - VALIDATE_SW_IF_INDEX (mp); - - /* hw_if & dpdk device */ - hw = vnet_get_sup_hw_interface (dm->vnet_main, sw_if_index); - - xd = vec_elt_at_index (dm->devices, hw->dev_instance); - - rv = rte_sched_subport_config (xd->hqos_ht->hqos, subport, &p); - - BAD_SW_IF_INDEX_LABEL; - - REPLY_MACRO (VL_API_SW_INTERFACE_SET_DPDK_HQOS_SUBPORT_REPLY); -} - -static void *vl_api_sw_interface_set_dpdk_hqos_subport_t_print - (vl_api_sw_interface_set_dpdk_hqos_subport_t * mp, void *handle) -{ - u8 *s; - - s = format (0, "SCRIPT: sw_interface_set_dpdk_hqos_subport "); - - s = format (s, "sw_if_index %u ", ntohl (mp->sw_if_index)); - - s = - format (s, - "subport %u rate %u bkt_size %u tc0 %u tc1 %u tc2 %u tc3 %u period %u", - ntohl (mp->subport), ntohl (mp->tb_rate), ntohl (mp->tb_size), - ntohl (mp->tc_rate[0]), ntohl (mp->tc_rate[1]), - ntohl (mp->tc_rate[2]), ntohl (mp->tc_rate[3]), - ntohl (mp->tc_period)); - - FINISH; -} - -static void - vl_api_sw_interface_set_dpdk_hqos_tctbl_t_handler - (vl_api_sw_interface_set_dpdk_hqos_tctbl_t * mp) -{ - vl_api_sw_interface_set_dpdk_hqos_tctbl_reply_t *rmp; - int rv = 0; - - dpdk_main_t *dm = &dpdk_main; - vlib_thread_main_t *tm = vlib_get_thread_main (); - dpdk_device_t *xd; - - u32 sw_if_index = ntohl (mp->sw_if_index); - u32 entry = ntohl (mp->entry); - u32 tc = ntohl (mp->tc); - u32 queue = ntohl (mp->queue); - u32 val, i; - - vnet_hw_interface_t *hw; - - VALIDATE_SW_IF_INDEX (mp); - - /* hw_if & dpdk device */ - hw = vnet_get_sup_hw_interface (dm->vnet_main, sw_if_index); - - xd = vec_elt_at_index (dm->devices, hw->dev_instance); - - if (tc >= RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE) - { - clib_warning ("invalid traffic class !!"); - rv = VNET_API_ERROR_INVALID_VALUE; - goto done; - } - if (queue >= RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS) - { - clib_warning ("invalid queue !!"); - rv = VNET_API_ERROR_INVALID_VALUE; - goto done; - } - - /* Detect the set of worker threads */ - uword *p = hash_get_mem (tm->thread_registrations_by_name, "workers"); - - if (p == 0) - { - clib_warning ("worker thread registration AWOL !!"); - rv = VNET_API_ERROR_INVALID_VALUE_2; - goto done; - } - - vlib_thread_registration_t *tr = (vlib_thread_registration_t *) p[0]; - int worker_thread_first = tr->first_index; - int worker_thread_count = tr->count; - - val = tc * RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS + queue; - for (i = 0; i < worker_thread_count; i++) - xd->hqos_wt[worker_thread_first + i].hqos_tc_table[entry] = val; - - BAD_SW_IF_INDEX_LABEL; -done: - - REPLY_MACRO (VL_API_SW_INTERFACE_SET_DPDK_HQOS_TCTBL_REPLY); -} - -static void *vl_api_sw_interface_set_dpdk_hqos_tctbl_t_print - (vl_api_sw_interface_set_dpdk_hqos_tctbl_t * mp, void *handle) -{ - u8 *s; - - s = format (0, "SCRIPT: sw_interface_set_dpdk_hqos_tctbl "); - - s = format (s, "sw_if_index %u ", ntohl (mp->sw_if_index)); - - s = format (s, "entry %u tc %u queue %u", - ntohl (mp->entry), ntohl (mp->tc), ntohl (mp->queue)); - - FINISH; -} - -#define foreach_dpdk_plugin_api_msg \ -_(SW_INTERFACE_SET_DPDK_HQOS_PIPE, sw_interface_set_dpdk_hqos_pipe) \ -_(SW_INTERFACE_SET_DPDK_HQOS_SUBPORT, sw_interface_set_dpdk_hqos_subport) \ -_(SW_INTERFACE_SET_DPDK_HQOS_TCTBL, sw_interface_set_dpdk_hqos_tctbl) - -/* Set up the API message handling tables */ -static clib_error_t * -dpdk_plugin_api_hookup (vlib_main_t * vm) -{ - dpdk_main_t *dm __attribute__ ((unused)) = &dpdk_main; -#define _(N,n) \ - vl_msg_api_set_handlers((VL_API_##N + dm->msg_id_base), \ - #n, \ - vl_api_##n##_t_handler, \ - vl_noop_handler, \ - vl_api_##n##_t_endian, \ - vl_api_##n##_t_print, \ - sizeof(vl_api_##n##_t), 1); - foreach_dpdk_plugin_api_msg; -#undef _ - return 0; -} - -#define vl_msg_name_crc_list -#include -#undef vl_msg_name_crc_list - -static void -setup_message_id_table (dpdk_main_t * dm, api_main_t * am) -{ -#define _(id,n,crc) \ - vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id + dm->msg_id_base); - foreach_vl_msg_name_crc_dpdk; -#undef _ -} - -// TODO -/* -static void plugin_custom_dump_configure (dpdk_main_t * dm) -{ -#define _(n,f) dm->api_main->msg_print_handlers \ - [VL_API_##n + dm->msg_id_base] \ - = (void *) vl_api_##f##_t_print; - foreach_dpdk_plugin_api_msg; -#undef _ -} -*/ -/* force linker to link functions used by vlib and declared weak */ -void *vlib_weakly_linked_functions[] = { - &rte_pktmbuf_init, - &rte_pktmbuf_pool_init, -}; - #define LINK_STATE_ELOGS 0 #define DEFAULT_HUGE_DIR "/run/vpp/hugepages" @@ -328,7 +65,7 @@ dpdk_port_setup (dpdk_main_t * dm, dpdk_device_t * xd) if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) { - vnet_hw_interface_set_flags (dm->vnet_main, xd->vlib_hw_if_index, 0); + vnet_hw_interface_set_flags (dm->vnet_main, xd->hw_if_index, 0); rte_eth_dev_stop (xd->device_index); } @@ -359,20 +96,20 @@ dpdk_port_setup (dpdk_main_t * dm, dpdk_device_t * xd) for (j = 0; j < xd->rx_q_used; j++) { + uword tidx = vnet_get_device_input_thread_index (dm->vnet_main, + xd->hw_if_index, j); + unsigned lcore = vlib_worker_threads[tidx].lcore_id; + u16 socket_id = rte_lcore_to_socket_id (lcore); rv = rte_eth_rx_queue_setup (xd->device_index, j, xd->nb_rx_desc, xd->cpu_socket, 0, - dm-> - pktmbuf_pools[xd->cpu_socket_id_by_queue - [j]]); + dm->pktmbuf_pools[socket_id]); /* retry with any other CPU socket */ if (rv < 0) rv = rte_eth_rx_queue_setup (xd->device_index, j, xd->nb_rx_desc, SOCKET_ID_ANY, 0, - dm-> - pktmbuf_pools[xd->cpu_socket_id_by_queue - [j]]); + dm->pktmbuf_pools[socket_id]); if (rv < 0) return clib_error_return (0, "rte_eth_rx_queue_setup[%d]: err %d", xd->device_index, rv); @@ -452,7 +189,7 @@ dpdk_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hi, u32 flags) return old; } -void +static void dpdk_device_lock_init (dpdk_device_t * xd) { int q; @@ -465,17 +202,6 @@ dpdk_device_lock_init (dpdk_device_t * xd) } } -void -dpdk_device_lock_free (dpdk_device_t * xd) -{ - int q; - - for (q = 0; q < vec_len (xd->lockp); q++) - clib_mem_free ((void *) xd->lockp[q]); - vec_free (xd->lockp); - xd->lockp = 0; -} - static clib_error_t * dpdk_lib_init (dpdk_main_t * dm) { @@ -485,35 +211,20 @@ dpdk_lib_init (dpdk_main_t * dm) clib_error_t *error; vlib_main_t *vm = vlib_get_main (); vlib_thread_main_t *tm = vlib_get_thread_main (); + vnet_device_main_t *vdm = &vnet_device_main; vnet_sw_interface_t *sw; vnet_hw_interface_t *hi; dpdk_device_t *xd; vlib_pci_addr_t last_pci_addr; u32 last_pci_addr_port = 0; - vlib_thread_registration_t *tr, *tr_hqos; - uword *p, *p_hqos; + vlib_thread_registration_t *tr_hqos; + uword *p_hqos; - u32 next_cpu = 0, next_hqos_cpu = 0; + u32 next_hqos_cpu = 0; u8 af_packet_port_id = 0; u8 bond_ether_port_id = 0; last_pci_addr.as_u32 = ~0; - dm->input_cpu_first_index = 0; - dm->input_cpu_count = 1; - - /* find out which cpus will be used for input */ - p = hash_get_mem (tm->thread_registrations_by_name, "workers"); - tr = p ? (vlib_thread_registration_t *) p[0] : 0; - - if (tr && tr->count > 0) - { - dm->input_cpu_first_index = tr->first_index; - dm->input_cpu_count = tr->count; - } - - vec_validate_aligned (dm->devices_by_cpu, tm->n_vlib_mains - 1, - CLIB_CACHE_LINE_BYTES); - dm->hqos_cpu_first_index = 0; dm->hqos_cpu_count = 0; @@ -924,48 +635,6 @@ dpdk_lib_init (dpdk_main_t * dm) dpdk_device_and_queue_t *dq; int q; - if (devconf->workers) - { - int i; - q = 0; - /* *INDENT-OFF* */ - clib_bitmap_foreach (i, devconf->workers, ({ - int cpu = dm->input_cpu_first_index + i; - unsigned lcore = vlib_worker_threads[cpu].lcore_id; - vec_validate(xd->cpu_socket_id_by_queue, q); - xd->cpu_socket_id_by_queue[q] = rte_lcore_to_socket_id(lcore); - vec_add2(dm->devices_by_cpu[cpu], dq, 1); - dq->device = xd->device_index; - dq->queue_id = q++; - })); - /* *INDENT-ON* */ - } - else - for (q = 0; q < xd->rx_q_used; q++) - { - int cpu = dm->input_cpu_first_index + next_cpu; - unsigned lcore = vlib_worker_threads[cpu].lcore_id; - - /* - * numa node for worker thread handling this queue - * needed for taking buffers from the right mempool - */ - vec_validate (xd->cpu_socket_id_by_queue, q); - xd->cpu_socket_id_by_queue[q] = rte_lcore_to_socket_id (lcore); - - /* - * construct vector of (device,queue) pairs for each worker thread - */ - vec_add2 (dm->devices_by_cpu[cpu], dq, 1); - dq->device = xd->device_index; - dq->queue_id = q; - - next_cpu++; - if (next_cpu == dm->input_cpu_count) - next_cpu = 0; - } - - if (devconf->hqos_enabled) { xd->flags |= DPDK_DEVICE_FLAG_HQOS; @@ -1022,17 +691,6 @@ dpdk_lib_init (dpdk_main_t * dm) vec_validate_aligned (xd->d_trace_buffers, tm->n_vlib_mains, CLIB_CACHE_LINE_BYTES); - rv = dpdk_port_setup (dm, xd); - - if (rv) - return rv; - - if (devconf->hqos_enabled) - { - rv = dpdk_port_setup_hqos (xd, &devconf->hqos); - if (rv) - return rv; - } /* count the number of descriptors used for this device */ nb_desc += xd->nb_rx_desc + xd->nb_tx_desc * xd->tx_q_used; @@ -1040,13 +698,46 @@ dpdk_lib_init (dpdk_main_t * dm) error = ethernet_register_interface (dm->vnet_main, dpdk_device_class.index, xd->device_index, /* ethernet address */ addr, - &xd->vlib_hw_if_index, dpdk_flag_change); + &xd->hw_if_index, dpdk_flag_change); if (error) return error; - sw = vnet_get_hw_sw_interface (dm->vnet_main, xd->vlib_hw_if_index); + sw = vnet_get_hw_sw_interface (dm->vnet_main, xd->hw_if_index); xd->vlib_sw_if_index = sw->sw_if_index; - hi = vnet_get_hw_interface (dm->vnet_main, xd->vlib_hw_if_index); + vnet_hw_interface_set_input_node (dm->vnet_main, xd->hw_if_index, + dpdk_input_node.index); + + if (devconf->workers) + { + int i; + q = 0; + /* *INDENT-OFF* */ + clib_bitmap_foreach (i, devconf->workers, ({ + vnet_hw_interface_assign_rx_thread (dm->vnet_main, xd->hw_if_index, q++, + vdm->first_worker_thread_index + i); + })); + /* *INDENT-ON* */ + } + else + for (q = 0; q < xd->rx_q_used; q++) + { + vnet_hw_interface_assign_rx_thread (dm->vnet_main, xd->hw_if_index, q, /* any */ + ~1); + } + + hi = vnet_get_hw_interface (dm->vnet_main, xd->hw_if_index); + + rv = dpdk_port_setup (dm, xd); + + if (rv) + return rv; + + if (devconf->hqos_enabled) + { + rv = dpdk_port_setup_hqos (xd, &devconf->hqos); + if (rv) + return rv; + } /* * For cisco VIC vNIC, set default to VLAN strip enabled, unless @@ -1304,9 +995,6 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) else if (unformat (input, "no-multi-seg")) conf->no_multi_seg = 1; - else if (unformat (input, "enable-cryptodev")) - conf->cryptodev = 1; - else if (unformat (input, "dev default %U", unformat_vlib_cli_sub_input, &sub_input)) { @@ -1340,8 +1028,6 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) } else if (unformat (input, "num-mbufs %d", &conf->num_mbufs)) ; - else if (unformat (input, "kni %d", &conf->num_kni)) - ; else if (unformat (input, "uio-driver %s", &conf->uio_driver_name)) ; else if (unformat (input, "socket-mem %s", &socket_mem)) @@ -1723,13 +1409,13 @@ dpdk_update_link_state (dpdk_device_t * xd, f64 now) ed->sw_if_index = xd->vlib_sw_if_index; ed->admin_up = (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) != 0; ed->old_link_state = (u8) - vnet_hw_interface_is_link_up (vnm, xd->vlib_hw_if_index); + vnet_hw_interface_is_link_up (vnm, xd->hw_if_index); ed->new_link_state = (u8) xd->link.link_status; } if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) && ((xd->link.link_status != 0) ^ - vnet_hw_interface_is_link_up (vnm, xd->vlib_hw_if_index))) + vnet_hw_interface_is_link_up (vnm, xd->hw_if_index))) { hw_flags_chg = 1; hw_flags |= (xd->link.link_status ? VNET_HW_INTERFACE_FLAG_LINK_UP : 0); @@ -1798,7 +1484,7 @@ dpdk_update_link_state (dpdk_device_t * xd, f64 now) ed->sw_if_index = xd->vlib_sw_if_index; ed->flags = hw_flags; } - vnet_hw_interface_set_flags (vnm, xd->vlib_hw_if_index, hw_flags); + vnet_hw_interface_set_flags (vnm, xd->hw_if_index, hw_flags); } } @@ -1815,23 +1501,6 @@ dpdk_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f) error = dpdk_lib_init (dm); - /* - * Turn on the input node if we found some devices to drive - * and we're not running worker threads or i/o threads - */ - - if (error == 0 && vec_len (dm->devices) > 0) - { - if (tm->n_vlib_mains == 1) - vlib_node_set_state (vm, dpdk_input_node.index, - VLIB_NODE_STATE_POLLING); - else - for (i = 0; i < tm->n_vlib_mains; i++) - if (vec_len (dm->devices_by_cpu[i]) > 0) - vlib_node_set_state (vlib_mains[i], dpdk_input_node.index, - VLIB_NODE_STATE_POLLING); - } - if (error) clib_error_report (error); @@ -1881,7 +1550,7 @@ dpdk_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f) /* Populate MAC of bonded interface in VPP hw tables */ bhi = vnet_get_hw_interface - (vnm, dm->devices[i].vlib_hw_if_index); + (vnm, dm->devices[i].hw_if_index); bei = pool_elt_at_index (em->interfaces, bhi->hw_instance); clib_memcpy (bhi->hw_address, addr, 6); @@ -1910,10 +1579,9 @@ dpdk_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f) } /* Set slaves bitmap for bonded interface */ bhi->bond_info = clib_bitmap_set - (bhi->bond_info, sdev->vlib_hw_if_index, 1); + (bhi->bond_info, sdev->hw_if_index, 1); /* Set slave link flags on slave interface */ - shi = vnet_get_hw_interface - (vnm, sdev->vlib_hw_if_index); + shi = vnet_get_hw_interface (vnm, sdev->hw_if_index); ssi = vnet_get_sw_interface (vnm, sdev->vlib_sw_if_index); sei = pool_elt_at_index @@ -1978,29 +1646,7 @@ VLIB_REGISTER_NODE (dpdk_process_node,static) = { }; /* *INDENT-ON* */ -int -dpdk_set_stat_poll_interval (f64 interval) -{ - if (interval < DPDK_MIN_STATS_POLL_INTERVAL) - return (VNET_API_ERROR_INVALID_VALUE); - - dpdk_main.stat_poll_interval = interval; - - return 0; -} - -int -dpdk_set_link_state_poll_interval (f64 interval) -{ - if (interval < DPDK_MIN_LINK_POLL_INTERVAL) - return (VNET_API_ERROR_INVALID_VALUE); - - dpdk_main.link_state_poll_interval = interval; - - return 0; -} - -clib_error_t * +static clib_error_t * dpdk_init (vlib_main_t * vm) { dpdk_main_t *dm = &dpdk_main; @@ -2017,26 +1663,10 @@ dpdk_init (vlib_main_t * vm) STATIC_ASSERT (offsetof (frame_queue_trace_t, cacheline0) == 0, "Cache line marker must be 1st element in frame_queue_trace_t"); - u8 *name; - name = format (0, "dpdk_%08x%c", api_version, 0); - - /* Ask for a correctly-sized block of API message decode slots */ - dm->msg_id_base = vl_msg_api_get_msg_ids - ((char *) name, VL_MSG_FIRST_AVAILABLE); - vec_free (name); - dm->vlib_main = vm; dm->vnet_main = vnet_get_main (); dm->conf = &dpdk_config_main; - error = dpdk_plugin_api_hookup (vm); - - /* Add our API messages to the global name_crc hash table */ - setup_message_id_table (dm, &api_main); - -// TODO -// plugin_custom_dump_configure (dm); - ei = vlib_get_node_by_name (vm, (u8 *) "ethernet-input"); if (ei == 0) return clib_error_return (0, "ethernet-input node AWOL"); @@ -2047,11 +1677,6 @@ dpdk_init (vlib_main_t * vm) dm->conf->num_mbufs = dm->conf->num_mbufs ? dm->conf->num_mbufs : NB_MBUF; vec_add1 (dm->conf->eal_init_args, (u8 *) "vnet"); - dm->dpdk_device_by_kni_port_id = hash_create (0, sizeof (uword)); - dm->vu_sw_if_index_by_listener_fd = hash_create (0, sizeof (uword)); - dm->vu_sw_if_index_by_sock_fd = hash_create (0, sizeof (uword)); - - /* $$$ use n_thread_stacks since it's known-good at this point */ vec_validate (dm->recycle, tm->n_thread_stacks - 1); /* Default vlib_buffer_t flags, DISABLES tcp/udp checksumming... */