if (!strcasecmp(single_name, name)) {
found = 1;
break;
- } else if (!strcasecmp(single_name, "UNKNOWN"))
- break;
- else if (single_name == NULL)
- break;
+ }
single_offload <<= 1;
}
.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
.level = 0,
.types = rss_hf,
- .key_len = 0,
+ .key_len = sizeof(action_rss_data->key),
.queue_num = RTE_MIN(nb_rxq, ACTION_RSS_QUEUE_NUM),
- .key = NULL,
+ .key = action_rss_data->key,
.queue = action_rss_data->queue,
},
+ .key = "testpmd's default RSS hash key, "
+ "override it for better balancing",
.queue = { 0 },
};
for (i = 0; i < action_rss_data->conf.queue_num; ++i)
action_rss_data->queue[i] = i;
+ if (!port_id_is_invalid(ctx->port, DISABLED_WARN) &&
+ ctx->port != (portid_t)RTE_PORT_ALL) {
+ struct rte_eth_dev_info info;
+
+ rte_eth_dev_info_get(ctx->port, &info);
+ action_rss_data->conf.key_len =
+ RTE_MIN(sizeof(action_rss_data->key),
+ info.hash_key_size);
+ }
action->conf = &action_rss_data->conf;
return ret;
}
}
static int
-parse_dscp_table_entries(char *str, enum rte_mtr_color *dscp_table)
+parse_dscp_table_entries(char *str, enum rte_mtr_color **dscp_table)
{
char *token;
int i = 0;
return 0;
/* Allocate memory for dscp table */
- dscp_table = (enum rte_mtr_color *)malloc(MAX_DSCP_TABLE_ENTRIES *
+ *dscp_table = (enum rte_mtr_color *)malloc(MAX_DSCP_TABLE_ENTRIES *
sizeof(enum rte_mtr_color));
- if (dscp_table == NULL)
+ if (*dscp_table == NULL)
return -1;
while (1) {
if (strcmp(token, "G") == 0 ||
strcmp(token, "g") == 0)
- dscp_table[i++] = RTE_MTR_GREEN;
+ *dscp_table[i++] = RTE_MTR_GREEN;
else if (strcmp(token, "Y") == 0 ||
strcmp(token, "y") == 0)
- dscp_table[i++] = RTE_MTR_YELLOW;
+ *dscp_table[i++] = RTE_MTR_YELLOW;
else if (strcmp(token, "R") == 0 ||
strcmp(token, "r") == 0)
- dscp_table[i++] = RTE_MTR_RED;
+ *dscp_table[i++] = RTE_MTR_RED;
else {
- free(dscp_table);
+ free(*dscp_table);
return -1;
}
if (i == MAX_DSCP_TABLE_ENTRIES)
token = strtok_r(str, PARSE_DELIMITER, &str);
if (token == NULL) {
- free(dscp_table);
+ free(*dscp_table);
return -1;
}
}
static int
parse_meter_color_str(char *c_str, uint32_t *use_prev_meter_color,
- enum rte_mtr_color *dscp_table)
+ enum rte_mtr_color **dscp_table)
{
char *token;
uint64_t previous_mtr_color = 0;
static int
parse_multi_token_string(char *t_str, uint16_t *port_id,
- uint32_t *mtr_id, enum rte_mtr_color *dscp_table)
+ uint32_t *mtr_id, enum rte_mtr_color **dscp_table)
{
char *token;
uint64_t val;
params.meter_profile_id = res->profile_id;
/* Parse meter input color string params */
- ret = parse_meter_color_str(c_str, &use_prev_meter_color, dscp_table);
+ ret = parse_meter_color_str(c_str, &use_prev_meter_color, &dscp_table);
if (ret) {
printf(" Meter input color params string parse error\n");
return;
int ret;
/* Parse string */
- ret = parse_multi_token_string(t_str, &port_id, &mtr_id, dscp_table);
+ ret = parse_multi_token_string(t_str, &port_id, &mtr_id, &dscp_table);
if (ret) {
printf(" Multi token string parse error\n");
return;
if (info->l4_proto == IPPROTO_TCP) {
tcp_hdr = (struct tcp_hdr *)((char *)ipv4_hdr + info->l3_len);
info->l4_len = (tcp_hdr->data_off & 0xf0) >> 2;
- } else
+ } else if (info->l4_proto == IPPROTO_UDP)
+ info->l4_len = sizeof(struct udp_hdr);
+ else
info->l4_len = 0;
}
if (info->l4_proto == IPPROTO_TCP) {
tcp_hdr = (struct tcp_hdr *)((char *)ipv6_hdr + info->l3_len);
info->l4_len = (tcp_hdr->data_off & 0xf0) >> 2;
- } else
+ } else if (info->l4_proto == IPPROTO_UDP)
+ info->l4_len = sizeof(struct udp_hdr);
+ else
info->l4_len = 0;
}
error->message,
shaper_profile_id);
+ free(tnp.shared_shaper_id);
return -1;
}
tnp.shaper_profile_id = shaper_profile_id;
error->message,
h->tc_node_id[pos][k]);
+ free(tnp.shared_shaper_id);
return -1;
}
shaper_profile_id++;
MAPFILE=$1
OBJFILE=$2
+# added check for "make -C test/" usage
+if [ ! -e $MAPFILE ] || [ ! -f $OBJFILE ]
+then
+ exit 0
+fi
+
if [ -d $MAPFILE ]
then
exit 0
# Auto-generate a Makefile in build directory
# Args:
# $1: path of project src root
-# $2: path of build dir (can be relative to $1)
echo "# Automatically generated by gen-build-mk.sh"
echo
echo "MAKEFLAGS += --no-print-directory"
echo
echo "all:"
-echo " @\$(MAKE) -C $1 O=$2"
+echo " @\$(MAKE) -C $1 O=\$(CURDIR)"
echo
echo "%::"
-echo " @\$(MAKE) -C $1 O=$2 \$@"
+echo " @\$(MAKE) -C $1 O=\$(CURDIR) \$@"
enabled (``rxq_cqe_comp_en``) at the same time, RSS hash result is not fully
supported. Some Rx packets may not have PKT_RX_RSS_HASH.
+- IPv6 Multicast messages are not supported on VM, while promiscuous mode
+ and allmulticast mode are both set to off.
+ To receive IPv6 Multicast messages on VM, explicitly set the relevant
+ MAC address using rte_eth_dev_mac_addr_add() API.
+
Statistics
----------
A nonzero value enables the DV flow steering assuming it is supported
by the driver.
+ The DV flow steering is not supported on switchdev mode.
Disabled by default.
}
dev->device.bus = &rte_vdev_bus;
- dev->device.devargs = devargs;
dev->device.numa_node = SOCKET_ID_ANY;
dev->device.name = devargs->name;
goto fail;
}
- TAILQ_INSERT_TAIL(&vdev_device_list, dev, next);
if (init)
- rte_devargs_insert(devargs);
+ rte_devargs_insert(&devargs);
+ dev->device.devargs = devargs;
+ TAILQ_INSERT_TAIL(&vdev_device_list, dev, next);
if (p_dev)
*p_dev = dev;
char chan_path[PATH_MAX], subchan_path[PATH_MAX];
struct dirent *ent;
DIR *chan_dir;
+ int err;
snprintf(chan_path, sizeof(chan_path),
"%s/%s/channels",
while ((ent = readdir(chan_dir))) {
unsigned long relid, subid, monid;
char *endp;
- int err;
if (ent->d_name[0] == '.')
continue;
if (err) {
VMBUS_LOG(NOTICE, "invalid subchannel id %lu",
subid);
- closedir(chan_dir);
- return err;
+ goto fail;
}
if (subid == 0)
if (err) {
VMBUS_LOG(NOTICE, "invalid monitor id %lu",
monid);
- return err;
+ goto fail;
}
err = vmbus_chan_create(dev, relid, subid, monid, subchan);
if (err) {
VMBUS_LOG(NOTICE, "subchannel setup failed");
- return err;
+ goto fail;
}
break;
}
closedir(chan_dir);
return (ent == NULL) ? -ENOENT : 0;
+fail:
+ closedir(chan_dir);
+ return err;
}
/** Create a ring to place processed operations on */
static struct rte_ring *
aesni_mb_pmd_qp_create_processed_ops_ring(struct aesni_mb_qp *qp,
- const char *str, unsigned int ring_size, int socket_id)
+ unsigned int ring_size, int socket_id)
{
struct rte_ring *r;
char ring_name[RTE_CRYPTODEV_NAME_MAX_LEN];
- unsigned int n = snprintf(ring_name, sizeof(ring_name),
- "%s_%s",
- qp->name, str);
+ unsigned int n = snprintf(ring_name, sizeof(ring_name), "%s", qp->name);
if (n >= sizeof(ring_name))
return NULL;
qp->op_fns = &job_ops[internals->vector_mode];
qp->ingress_queue = aesni_mb_pmd_qp_create_processed_ops_ring(qp,
- "ingress", qp_conf->nb_descriptors, socket_id);
+ qp_conf->nb_descriptors, socket_id);
if (qp->ingress_queue == NULL) {
ret = -1;
goto qp_setup_cleanup;
while (uio_module_names[i] != NULL) {
while (fgets(buf, sizeof(buf), fp) != NULL) {
if (!strncmp(buf, uio_module_names[i],
- strlen(uio_module_names[i])))
+ strlen(uio_module_names[i]))) {
+ fclose(fp);
return i;
+ }
}
i++;
rewind(fp);
}
+ fclose(fp);
printf("Insert igb_uio or uio_pci_generic kernel module(s)");
return -1;/* uio not inserted */
}
if (rte_eal_process_type() == RTE_PROC_PRIMARY)
rte_free(cryptodev->data->dev_private);
- cryptodev->device = NULL;
cryptodev->device->driver = NULL;
+ cryptodev->device = NULL;
cryptodev->data = NULL;
/* free metapool memory */
#include <rte_common.h>
#include <rte_errno.h>
#include <rte_memzone.h>
+#include <rte_string_fns.h>
#include "otx_cryptodev_hw_access.h"
#include "otx_cryptodev_mbox.h"
/* Bar0 base address */
cptvf->reg_base = reg_base;
- strncpy(cptvf->dev_name, name, 32);
+
+ /* Save device name */
+ strlcpy(cptvf->dev_name, name, (sizeof(cptvf->dev_name)));
cptvf->pdev = pdev;
}
ret = otx_cpt_get_resource(cptvf, 0, &instance);
- if (ret != 0) {
+ if (ret != 0 || instance == NULL) {
CPT_LOG_ERR("Error getting instance handle from device %s : "
"ret = %d", dev->data->name, ret);
return ret;
uint32_t hw_lock_control_reg;
int cnt;
+#ifndef RTE_LIBRTE_BNX2X_DEBUG_PERIODIC
if (resource)
PMD_INIT_FUNC_TRACE(sc);
+#else
+ PMD_INIT_FUNC_TRACE(sc);
+#endif
/* validate the resource is within range */
if (resource > HW_LOCK_MAX_RESOURCE_VALUE) {
int func = SC_FUNC(sc);
uint32_t hw_lock_control_reg;
+#ifndef RTE_LIBRTE_BNX2X_DEBUG_PERIODIC
if (resource)
PMD_INIT_FUNC_TRACE(sc);
+#else
+ PMD_INIT_FUNC_TRACE(sc);
+#endif
/* validate the resource is within range */
if (resource > HW_LOCK_MAX_RESOURCE_VALUE) {
}
bnx2x_link_report(sc);
} else {
- bnx2x_link_report(sc);
+ bnx2x_link_report_locked(sc);
bnx2x_stats_handle(sc, STATS_EVENT_LINK_UP);
}
}
uint32_t fw, hw_lock_reg, hw_lock_val;
uint32_t rc = 0;
+ PMD_INIT_FUNC_TRACE(sc);
+
/*
* Clear HW from errors which may have resulted from an interrupted
* DMAE transaction.
bnx2x_prev_interrupted_dmae(sc);
/* Release previously held locks */
- if (SC_FUNC(sc) <= 5)
- hw_lock_reg = (MISC_REG_DRIVER_CONTROL_1 + SC_FUNC(sc) * 8);
- else
- hw_lock_reg =
- (MISC_REG_DRIVER_CONTROL_7 + (SC_FUNC(sc) - 6) * 8);
+ hw_lock_reg = (SC_FUNC(sc) <= 5) ?
+ (MISC_REG_DRIVER_CONTROL_1 + SC_FUNC(sc) * 8) :
+ (MISC_REG_DRIVER_CONTROL_7 + (SC_FUNC(sc) - 6) * 8);
hw_lock_val = (REG_RD(sc, hw_lock_reg));
if (hw_lock_val) {
if (hw_lock_val & HW_LOCK_RESOURCE_NVRAM) {
+ PMD_DRV_LOG(DEBUG, sc, "Releasing previously held NVRAM lock\n");
REG_WR(sc, MCP_REG_MCPR_NVM_SW_ARB,
(MCPR_NVM_SW_ARB_ARB_REQ_CLR1 << SC_PORT(sc)));
}
+ PMD_DRV_LOG(DEBUG, sc, "Releasing previously held HW lock\n");
REG_WR(sc, hw_lock_reg, 0xffffffff);
}
if (MCPR_ACCESS_LOCK_LOCK & REG_RD(sc, MCP_REG_MCPR_ACCESS_LOCK)) {
+ PMD_DRV_LOG(DEBUG, sc, "Releasing previously held ALR\n");
REG_WR(sc, MCP_REG_MCPR_ACCESS_LOCK, 0);
}
sc->fw_seq =
(SHMEM_RD(sc, func_mb[SC_FW_MB_IDX(sc)].drv_mb_header) &
DRV_MSG_SEQ_NUMBER_MASK);
+ PMD_DRV_LOG(DEBUG, sc, "prev unload fw_seq 0x%04x",
+ sc->fw_seq);
bnx2x_prev_unload(sc);
}
extern int bnx2x_logtype_init;
#define PMD_INIT_LOG(level, sc, fmt, args...) \
- RTE_LOG(level, PMD, \
+ rte_log(RTE_LOG_ ## level, bnx2x_logtype_init, \
"[bnx2x_pmd: %s] %s() " fmt "\n", (sc)->devinfo.name, __func__, ##args)
#define PMD_INIT_FUNC_TRACE(sc) PMD_INIT_LOG(DEBUG, sc, " >>")
+extern int bnx2x_logtype_driver;
+#define PMD_DRV_LOG_RAW(level, sc, fmt, args...) \
+ rte_log(RTE_LOG_ ## level, bnx2x_logtype_driver, \
+ "[%s:%d(%s)] " fmt, __func__, __LINE__, \
+ (sc)->devinfo.name ? (sc)->devinfo.name : "", ## args)
+
+#define PMD_DRV_LOG(level, sc, fmt, args...) \
+ PMD_DRV_LOG_RAW(level, sc, fmt "\n", ## args)
+
#ifdef RTE_LIBRTE_BNX2X_DEBUG_RX
#define PMD_RX_LOG(level, fmt, args...) \
- RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args)
+ rte_log(RTE_LOG_ ## level, bnx2x_logtype_driver, \
+ "%s(): " fmt "\n", __func__, ## args)
#else
#define PMD_RX_LOG(level, fmt, args...) do { } while(0)
#endif
#ifdef RTE_LIBRTE_BNX2X_DEBUG_TX
#define PMD_TX_LOG(level, fmt, args...) \
- RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args)
+ rte_log(RTE_LOG_ ## level, bnx2x_logtype_driver, \
+ "%s(): " fmt "\n", __func__, ## args)
#else
#define PMD_TX_LOG(level, fmt, args...) do { } while(0)
#endif
-#ifdef RTE_LIBRTE_BNX2X_DEBUG_TX_FREE
-#define PMD_TX_FREE_LOG(level, fmt, args...) \
- RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args)
-#else
-#define PMD_TX_FREE_LOG(level, fmt, args...) do { } while(0)
-#endif
-
-extern int bnx2x_logtype_driver;
-#define PMD_DRV_LOG_RAW(level, sc, fmt, args...) \
- RTE_LOG(level, PMD, "[%s:%d(%s)] " fmt, __func__, __LINE__, \
- (sc)->devinfo.name ? (sc)->devinfo.name : "", ## args)
-
-#define PMD_DRV_LOG(level, sc, fmt, args...) \
- PMD_DRV_LOG_RAW(level, sc, fmt "\n", ## args)
-
#ifdef RTE_LIBRTE_BNX2X_DEBUG_PERIODIC
#define PMD_DEBUG_PERIODIC_LOG(level, sc, fmt, args...) \
- RTE_LOG(level, PMD, "%s(%s): " fmt "\n", __func__, \
+ rte_log(RTE_LOG_ ## level, bnx2x_logtype_driver, \
+ "%s(%s): " fmt "\n", __func__, \
(sc)->devinfo.name ? (sc)->devinfo.name : "", ## args)
#else
#define PMD_DEBUG_PERIODIC_LOG(level, sc, fmt, args...) do { } while (0)
internals->link_status_polling_enabled = 0;
for (i = 0; i < internals->slave_count; i++) {
- internals->slaves[i].last_link_status = 0;
- rte_eth_dev_stop(internals->slaves[i].port_id);
- deactivate_slave(eth_dev, internals->slaves[i].port_id);
+ uint16_t slave_id = internals->slaves[i].port_id;
+ if (find_slave_by_id(internals->active_slaves,
+ internals->active_slave_count, slave_id) !=
+ internals->active_slave_count) {
+ internals->slaves[i].last_link_status = 0;
+ rte_eth_dev_stop(slave_id);
+ deactivate_slave(eth_dev, slave_id);
+ }
}
}
u64 ntuple = 0;
u16 tcp_proto = IPPROTO_TCP; /* TCP Protocol Number */
- if (tp->port_shift >= 0)
- ntuple |= (u64)f->fs.mask.iport << tp->port_shift;
+ if (tp->port_shift >= 0 && f->fs.mask.iport)
+ ntuple |= (u64)f->fs.val.iport << tp->port_shift;
if (tp->protocol_shift >= 0) {
if (!f->fs.val.proto)
if (tp->macmatch_shift >= 0 && f->fs.mask.macidx)
ntuple |= (u64)(f->fs.val.macidx) << tp->macmatch_shift;
- if (ntuple != tp->hash_filter_mask)
- return 0;
-
return ntuple;
}
#define __CXGBE_FILL_FS(__v, __m, fs, elem, e) \
do { \
- if (!((fs)->val.elem || (fs)->mask.elem)) { \
- (fs)->val.elem = (__v); \
- (fs)->mask.elem = (__m); \
- } else { \
+ if ((fs)->mask.elem && ((fs)->val.elem != (__v))) \
return rte_flow_error_set(e, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, \
- NULL, "a filter can be specified" \
- " only once"); \
- } \
+ NULL, "Redefined match item with" \
+ " different values found"); \
+ (fs)->val.elem = (__v); \
+ (fs)->mask.elem = (__m); \
} while (0)
#define __CXGBE_FILL_FS_MEMCPY(__v, __m, fs, elem) \
/* Poll the FW for reply */
err = cxgbe_poll_for_completion(&adap->sge.fw_evtq,
- CXGBE_FLOW_POLL_US,
+ CXGBE_FLOW_POLL_MS,
CXGBE_FLOW_POLL_CNT,
&ctx.completion);
if (err) {
/* Poll the FW for reply */
err = cxgbe_poll_for_completion(&adap->sge.fw_evtq,
- CXGBE_FLOW_POLL_US,
+ CXGBE_FLOW_POLL_MS,
CXGBE_FLOW_POLL_CNT,
&ctx.completion);
if (err) {
#include "mps_tcam.h"
#include "cxgbe.h"
-#define CXGBE_FLOW_POLL_US 10
-#define CXGBE_FLOW_POLL_CNT 10
+/* Max poll time is 100 * 100msec = 10 sec */
+#define CXGBE_FLOW_POLL_MS 100 /* 100 milliseconds */
+#define CXGBE_FLOW_POLL_CNT 100 /* Max number of times to poll */
struct chrte_fparse {
int (*fptr)(const void *mask, /* currently supported mask */
/**
* cxgbe_poll_for_completion: Poll rxq for completion
* @q: rxq to poll
- * @us: microseconds to delay
+ * @ms: milliseconds to delay
* @cnt: number of times to poll
* @c: completion to check for 'done' status
*
* Polls the rxq for reples until completion is done or the count
* expires.
*/
-int cxgbe_poll_for_completion(struct sge_rspq *q, unsigned int us,
+int cxgbe_poll_for_completion(struct sge_rspq *q, unsigned int ms,
unsigned int cnt, struct t4_completion *c)
{
unsigned int i;
- unsigned int work_done, budget = 4;
+ unsigned int work_done, budget = 32;
if (!c)
return -EINVAL;
return 0;
}
t4_os_unlock(&c->lock);
- udelay(us);
+ rte_delay_ms(ms);
}
return -ETIMEDOUT;
}
int link_start(struct port_info *pi)
{
struct adapter *adapter = pi->adapter;
- int ret;
+ u64 conf_offloads;
unsigned int mtu;
+ int ret;
mtu = pi->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len -
(ETHER_HDR_LEN + ETHER_CRC_LEN);
+ conf_offloads = pi->eth_dev->data->dev_conf.rxmode.offloads;
+
/*
* We do not set address filters and promiscuity here, the stack does
* that step explicitly.
*/
- ret = t4_set_rxmode(adapter, adapter->mbox, pi->viid, mtu, -1, -1,
- -1, 1, true);
+ ret = t4_set_rxmode(adapter, adapter->mbox, pi->viid, mtu, -1, -1, -1,
+ !!(conf_offloads & DEV_RX_OFFLOAD_VLAN_STRIP),
+ true);
if (ret == 0) {
ret = cxgbe_mpstcam_modify(pi, (int)pi->xact_addr_filt,
(u8 *)&pi->eth_dev->data->mac_addrs[0]);
return err;
}
+static int eth_cxgbevf_dev_uninit(struct rte_eth_dev *eth_dev)
+{
+ struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+ struct adapter *adap = pi->adapter;
+
+ /* Free up other ports and all resources */
+ cxgbe_close(adap);
+ return 0;
+}
+
static int eth_cxgbevf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
struct rte_pci_device *pci_dev)
{
static int eth_cxgbevf_pci_remove(struct rte_pci_device *pci_dev)
{
- return rte_eth_dev_pci_generic_remove(pci_dev, NULL);
+ return rte_eth_dev_pci_generic_remove(pci_dev, eth_cxgbevf_dev_uninit);
}
static struct rte_pci_driver rte_cxgbevf_pmd = {
#include "t4_regs.h"
#include "t4_msg.h"
#include "cxgbe.h"
+#include "mps_tcam.h"
/*
* Figure out how many Ports and Queue Sets we can support. This depends on
print_adapter_info(adapter);
print_port_info(adapter);
+ adapter->mpstcam = t4_init_mpstcam(adapter);
+ if (!adapter->mpstcam)
+ dev_warn(adapter,
+ "VF could not allocate mps tcam table. Continuing\n");
+
err = init_rss(adapter);
if (err)
goto out_free;
void t4_cleanup_mpstcam(struct adapter *adap)
{
- if (adap->mpstcam) {
- t4_os_free(adap->mpstcam->entry);
+ if (adap->mpstcam)
t4_os_free(adap->mpstcam);
- }
}
if (ret_val != E1000_SUCCESS)
nvm_word = E1000_INVM_DEFAULT_AL;
tmp_nvm = nvm_word | E1000_INVM_PLL_WO_VAL;
+ phy_word = E1000_PHY_PLL_UNCONF;
for (i = 0; i < E1000_MAX_PLL_TRIES; i++) {
/* check current state directly from internal PHY */
e1000_read_phy_reg_gs40g(hw, (E1000_PHY_PLL_FREQ_PAGE |
ena_interrupt_handler_rte,
adapter);
- /*
- * Pass the information to the rte_eth_dev_close() that it should also
- * release the private port resources.
- */
- dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
/*
* MAC is not allocated dynamically. Setting NULL should prevent from
* release of the resource in the rte_eth_dev_release_port().
ether_addr_copy((struct ether_addr *)get_feat_ctx.dev_attr.mac_addr,
(struct ether_addr *)adapter->mac_addr);
+ /*
+ * Pass the information to the rte_eth_dev_close() that it should also
+ * release the private port resources.
+ */
+ eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
+
adapter->drv_stats = rte_zmalloc("adapter stats",
sizeof(*adapter->drv_stats),
RTE_CACHE_LINE_SIZE);
for (i = 0; i != nb_pkts; i++) {
m = tx_pkts[i];
- if (unlikely(m->pkt_len > ENIC_TX_MAX_PKT_SIZE)) {
- rte_errno = EINVAL;
- return i;
- }
ol_flags = m->ol_flags;
+ if (!(ol_flags & PKT_TX_TCP_SEG)) {
+ if (unlikely(m->pkt_len > ENIC_TX_MAX_PKT_SIZE)) {
+ rte_errno = EINVAL;
+ return i;
+ }
+ } else {
+ uint16_t header_len;
+
+ header_len = m->l2_len + m->l3_len + m->l4_len;
+ if (m->tso_segsz + header_len > ENIC_TX_MAX_PKT_SIZE) {
+ rte_errno = EINVAL;
+ return i;
+ }
+ }
+
if (ol_flags & wq->tx_offload_notsup_mask) {
rte_errno = ENOTSUP;
return i;
return 0;
}
-/* fls = find last set bit = 32 minus the number of leading zeros */
-#ifndef fls
-#define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
-#endif
-
static void
fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
{
macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
nb_queue_pools = macvlan->nb_queue_pools;
- pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
- rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
+ pool_len = nb_queue_pools ? rte_fls_u32(nb_queue_pools - 1) : 0;
+ rss_len = rte_fls_u32(dev->data->nb_rx_queues - 1) - pool_len;
/* GLORT 0x0-0x3F are used by PF and VMDQ, 0x40-0x7F used by FD */
dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
/* Flow Director configurations, only queue number is valid. */
- dglortdec = fls(dev->data->nb_rx_queues - 1);
+ dglortdec = rte_fls_u32(dev->data->nb_rx_queues - 1);
dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
(hw->mac.dglort_map + GLORT_FD_Q_BASE);
FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
if (in->key_len > RTE_DIM(out->key) ||
in->queue_num > RTE_DIM(out->queue))
return -EINVAL;
+ if (!in->key && in->key_len)
+ return -EINVAL;
+ if (in->key)
+ out->conf.key = memcpy(out->key, in->key, in->key_len);
out->conf = (struct rte_flow_action_rss){
.func = in->func,
.level = in->level,
.types = in->types,
.key_len = in->key_len,
.queue_num = in->queue_num,
- .key = memcpy(out->key, in->key, in->key_len),
.queue = memcpy(out->queue, in->queue,
sizeof(*in->queue) * in->queue_num),
};
return -EINVAL;
}
+ /* Stop the link setup handler before resetting the HW. */
+ rte_eal_alarm_cancel(ixgbe_dev_setup_link_alarm_handler, dev);
+
/* disable uio/vfio intr/eventfd mapping */
rte_intr_disable(intr_handle);
if (err)
goto error;
- ixgbe_dev_link_update(dev, 0);
-
skip_link_setup:
if (rte_intr_allow_others(intr_handle)) {
"please call hierarchy_commit() "
"before starting the port");
+ /*
+ * Update link status right before return, because it may
+ * start link configuration process in a separate thread.
+ */
+ ixgbe_dev_link_update(dev, 0);
+
return 0;
error:
ixgbevf_check_link(struct ixgbe_hw *hw, ixgbe_link_speed *speed,
int *link_up, int wait_to_complete)
{
- /**
- * for a quick link status checking, wait_to_compelet == 0,
- * skip PF link status checking
- */
- bool no_pflink_check = wait_to_complete == 0;
struct ixgbe_mbx_info *mbx = &hw->mbx;
struct ixgbe_mac_info *mac = &hw->mac;
uint32_t links_reg, in_msg;
*speed = IXGBE_LINK_SPEED_UNKNOWN;
}
- if (no_pflink_check) {
- if (*speed == IXGBE_LINK_SPEED_UNKNOWN)
- mac->get_link_status = true;
- else
- mac->get_link_status = false;
-
- goto out;
- }
/* if the read failed it could just be a mailbox collision, best wait
* until we are called again and don't report an error
*/
if (!(in_msg & IXGBE_VT_MSGTYPE_CTS)) {
/* msg is not CTS and is NACK we must have lost CTS status */
if (in_msg & IXGBE_VT_MSGTYPE_NACK)
- ret_val = -1;
+ mac->get_link_status = false;
goto out;
}
PMD_INIT_FUNC_TRACE();
+ /* Stop the link setup handler before resetting the HW. */
+ rte_eal_alarm_cancel(ixgbe_dev_setup_link_alarm_handler, dev);
+
err = hw->mac.ops.reset_hw(hw);
if (err) {
PMD_INIT_LOG(ERR, "Unable to reset vf hardware (%d)", err);
ixgbevf_dev_rxtx_start(dev);
- ixgbevf_dev_link_update(dev, 0);
-
/* check and configure queue intr-vector mapping */
if (rte_intr_cap_multiple(intr_handle) &&
dev->data->dev_conf.intr_conf.rxq) {
/* Re-enable interrupt for VF */
ixgbevf_intr_enable(dev);
+ /*
+ * Update link status right before return, because it may
+ * start link configuration process in a separate thread.
+ */
+ ixgbevf_dev_link_update(dev, 0);
+
return 0;
}
CFLAGS += -D_XOPEN_SOURCE=600
CFLAGS += $(WERROR_FLAGS)
CFLAGS += -Wno-strict-prototypes
-CFLAGS += $(shell pkg-config --cflags libmnl)
+CFLAGS += $(shell command -v pkg-config > /dev/null 2>&1 && pkg-config --cflags libmnl)
ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLOPEN_DEPS),y)
CFLAGS += -DMLX5_GLUE='"$(LIB_GLUE)"'
CFLAGS += -DMLX5_GLUE_VERSION='"$(LIB_GLUE_VERSION)"'
else
LDLIBS += -libverbs -lmlx5
endif
-LDLIBS += $(shell pkg-config --libs libmnl)
+LDLIBS += $(shell command -v pkg-config > /dev/null 2>&1 && pkg-config --libs libmnl || echo "-lmnl")
LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
LDLIBS += -lrte_bus_pci
}
memset(priv, 0, sizeof(*priv));
priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
- /*
- * flag to rte_eth_dev_close() that it should release the port resources
- * (calling rte_eth_dev_release_port()) in addition to closing it.
- */
- dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
/*
* Reset mac_addrs to NULL such that it is not freed as part of
* rte_eth_dev_release_port(). mac_addrs is part of dev_private so
err = ENOMEM;
goto error;
}
+ /* Flag to call rte_eth_dev_release_port() in rte_eth_dev_close(). */
+ eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
if (priv->representor) {
eth_dev->data->dev_flags |= RTE_ETH_DEV_REPRESENTOR;
eth_dev->data->representor_id = priv->representor_id;
"L3 cannot follow an L4 layer.");
if (!mask)
mask = &rte_flow_item_ipv4_mask;
+ else if (mask->hdr.next_proto_id != 0 &&
+ mask->hdr.next_proto_id != 0xff)
+ return rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
+ "partial mask is not supported"
+ " for protocol");
ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
(const uint8_t *)&nic_mask,
sizeof(struct rte_flow_item_ipv4),
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ITEM, item,
"L3 cannot follow an L4 layer.");
- /*
- * IPv6 is not recognised by the NIC inside a GRE tunnel.
- * Such support has to be disabled as the rule will be
- * accepted. Issue reproduced with Mellanox OFED 4.3-3.0.2.1 and
- * Mellanox OFED 4.4-1.0.0.0.
- */
- if (tunnel && item_flags & MLX5_FLOW_LAYER_GRE)
- return rte_flow_error_set(error, ENOTSUP,
- RTE_FLOW_ERROR_TYPE_ITEM, item,
- "IPv6 inside a GRE tunnel is"
- " not recognised.");
if (!mask)
mask = &rte_flow_item_ipv6_mask;
ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
FLOW_FDIR_CMP(f1, f2, l3_mask) ||
FLOW_FDIR_CMP(f1, f2, l4) ||
FLOW_FDIR_CMP(f1, f2, l4_mask) ||
- FLOW_FDIR_CMP(f1, f2, actions[0]))
+ FLOW_FDIR_CMP(f1, f2, actions[0].type))
return 1;
if (f1->actions[0].type == RTE_FLOW_ACTION_TYPE_QUEUE &&
FLOW_FDIR_CMP(f1, f2, queue))
MLX5_FLOW_LAYER_OUTER_L3_IPV4;
if (items->mask != NULL &&
((const struct rte_flow_item_ipv4 *)
- items->mask)->hdr.next_proto_id)
+ items->mask)->hdr.next_proto_id) {
next_protocol =
((const struct rte_flow_item_ipv4 *)
(items->spec))->hdr.next_proto_id;
+ next_protocol &=
+ ((const struct rte_flow_item_ipv4 *)
+ (items->mask))->hdr.next_proto_id;
+ } else {
+ /* Reset for inner layer. */
+ next_protocol = 0xff;
+ }
break;
case RTE_FLOW_ITEM_TYPE_IPV6:
ret = mlx5_flow_validate_item_ipv6(items, item_flags,
MLX5_FLOW_LAYER_OUTER_L3_IPV6;
if (items->mask != NULL &&
((const struct rte_flow_item_ipv6 *)
- items->mask)->hdr.proto)
+ items->mask)->hdr.proto) {
next_protocol =
((const struct rte_flow_item_ipv6 *)
items->spec)->hdr.proto;
+ next_protocol &=
+ ((const struct rte_flow_item_ipv6 *)
+ items->mask)->hdr.proto;
+ } else {
+ /* Reset for inner layer. */
+ next_protocol = 0xff;
+ }
break;
case RTE_FLOW_ITEM_TYPE_TCP:
ret = mlx5_flow_validate_item_tcp
return flow;
}
+#ifndef NDEBUG
+/**
+ * Sanity check for match mask and value. Similar to check_valid_spec() in
+ * kernel driver. If unmasked bit is present in value, it returns failure.
+ *
+ * @param match_mask
+ * pointer to match mask buffer.
+ * @param match_value
+ * pointer to match value buffer.
+ *
+ * @return
+ * 0 if valid, -EINVAL otherwise.
+ */
+static int
+flow_dv_check_valid_spec(void *match_mask, void *match_value)
+{
+ uint8_t *m = match_mask;
+ uint8_t *v = match_value;
+ unsigned int i;
+
+ for (i = 0; i < MLX5_ST_SZ_DB(fte_match_param); ++i) {
+ if (v[i] & ~m[i]) {
+ DRV_LOG(ERR,
+ "match_value differs from match_criteria"
+ " %p[%u] != %p[%u]",
+ match_value, i, match_mask, i);
+ return -EINVAL;
+ }
+ }
+ return 0;
+}
+#endif
+
/**
* Add Ethernet item to matcher and to the value.
*
if (priority == MLX5_FLOW_PRIO_RSVD)
priority = priv->config.flow_prio - 1;
- for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
- int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
- void *match_mask = matcher.mask.buf;
- void *match_value = dev_flow->dv.value.buf;
-
- switch (items->type) {
- case RTE_FLOW_ITEM_TYPE_ETH:
- flow_dv_translate_item_eth(match_mask, match_value,
- items, tunnel);
- matcher.priority = MLX5_PRIORITY_MAP_L2;
- item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
- MLX5_FLOW_LAYER_OUTER_L2;
- break;
- case RTE_FLOW_ITEM_TYPE_VLAN:
- flow_dv_translate_item_vlan(match_mask, match_value,
- items, tunnel);
- matcher.priority = MLX5_PRIORITY_MAP_L2;
- item_flags |= tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
- MLX5_FLOW_LAYER_INNER_VLAN) :
- (MLX5_FLOW_LAYER_OUTER_L2 |
- MLX5_FLOW_LAYER_OUTER_VLAN);
- break;
- case RTE_FLOW_ITEM_TYPE_IPV4:
- flow_dv_translate_item_ipv4(match_mask, match_value,
- items, tunnel);
- matcher.priority = MLX5_PRIORITY_MAP_L3;
- dev_flow->dv.hash_fields |=
- mlx5_flow_hashfields_adjust
- (dev_flow, tunnel,
- MLX5_IPV4_LAYER_TYPES,
- MLX5_IPV4_IBV_RX_HASH);
- item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
- MLX5_FLOW_LAYER_OUTER_L3_IPV4;
- break;
- case RTE_FLOW_ITEM_TYPE_IPV6:
- flow_dv_translate_item_ipv6(match_mask, match_value,
- items, tunnel);
- matcher.priority = MLX5_PRIORITY_MAP_L3;
- dev_flow->dv.hash_fields |=
- mlx5_flow_hashfields_adjust
- (dev_flow, tunnel,
- MLX5_IPV6_LAYER_TYPES,
- MLX5_IPV6_IBV_RX_HASH);
- item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
- MLX5_FLOW_LAYER_OUTER_L3_IPV6;
- break;
- case RTE_FLOW_ITEM_TYPE_TCP:
- flow_dv_translate_item_tcp(match_mask, match_value,
- items, tunnel);
- matcher.priority = MLX5_PRIORITY_MAP_L4;
- dev_flow->dv.hash_fields |=
- mlx5_flow_hashfields_adjust
- (dev_flow, tunnel, ETH_RSS_TCP,
- IBV_RX_HASH_SRC_PORT_TCP |
- IBV_RX_HASH_DST_PORT_TCP);
- item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
- MLX5_FLOW_LAYER_OUTER_L4_TCP;
- break;
- case RTE_FLOW_ITEM_TYPE_UDP:
- flow_dv_translate_item_udp(match_mask, match_value,
- items, tunnel);
- matcher.priority = MLX5_PRIORITY_MAP_L4;
- dev_flow->verbs.hash_fields |=
- mlx5_flow_hashfields_adjust
- (dev_flow, tunnel, ETH_RSS_UDP,
- IBV_RX_HASH_SRC_PORT_UDP |
- IBV_RX_HASH_DST_PORT_UDP);
- item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
- MLX5_FLOW_LAYER_OUTER_L4_UDP;
- break;
- case RTE_FLOW_ITEM_TYPE_GRE:
- flow_dv_translate_item_gre(match_mask, match_value,
- items, tunnel);
- item_flags |= MLX5_FLOW_LAYER_GRE;
- break;
- case RTE_FLOW_ITEM_TYPE_NVGRE:
- flow_dv_translate_item_nvgre(match_mask, match_value,
- items, tunnel);
- item_flags |= MLX5_FLOW_LAYER_GRE;
- break;
- case RTE_FLOW_ITEM_TYPE_VXLAN:
- flow_dv_translate_item_vxlan(match_mask, match_value,
- items, tunnel);
- item_flags |= MLX5_FLOW_LAYER_VXLAN;
- break;
- case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
- flow_dv_translate_item_vxlan(match_mask, match_value,
- items, tunnel);
- item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
- break;
- case RTE_FLOW_ITEM_TYPE_META:
- flow_dv_translate_item_meta(match_mask, match_value,
- items);
- item_flags |= MLX5_FLOW_ITEM_METADATA;
- break;
- default:
- break;
- }
- }
- dev_flow->layers = item_flags;
- /* Register matcher. */
- matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
- matcher.mask.size);
- matcher.priority = mlx5_flow_adjust_priority(dev, priority,
- matcher.priority);
- matcher.egress = attr->egress;
- if (flow_dv_matcher_register(dev, &matcher, dev_flow, error))
- return -rte_errno;
for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
const struct rte_flow_action_queue *queue;
const struct rte_flow_action_rss *rss;
}
dev_flow->dv.actions_n = actions_n;
flow->actions = action_flags;
+ for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
+ int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
+ void *match_mask = matcher.mask.buf;
+ void *match_value = dev_flow->dv.value.buf;
+
+ switch (items->type) {
+ case RTE_FLOW_ITEM_TYPE_ETH:
+ flow_dv_translate_item_eth(match_mask, match_value,
+ items, tunnel);
+ matcher.priority = MLX5_PRIORITY_MAP_L2;
+ item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
+ MLX5_FLOW_LAYER_OUTER_L2;
+ break;
+ case RTE_FLOW_ITEM_TYPE_VLAN:
+ flow_dv_translate_item_vlan(match_mask, match_value,
+ items, tunnel);
+ matcher.priority = MLX5_PRIORITY_MAP_L2;
+ item_flags |= tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
+ MLX5_FLOW_LAYER_INNER_VLAN) :
+ (MLX5_FLOW_LAYER_OUTER_L2 |
+ MLX5_FLOW_LAYER_OUTER_VLAN);
+ break;
+ case RTE_FLOW_ITEM_TYPE_IPV4:
+ flow_dv_translate_item_ipv4(match_mask, match_value,
+ items, tunnel);
+ matcher.priority = MLX5_PRIORITY_MAP_L3;
+ dev_flow->dv.hash_fields |=
+ mlx5_flow_hashfields_adjust
+ (dev_flow, tunnel,
+ MLX5_IPV4_LAYER_TYPES,
+ MLX5_IPV4_IBV_RX_HASH);
+ item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
+ MLX5_FLOW_LAYER_OUTER_L3_IPV4;
+ break;
+ case RTE_FLOW_ITEM_TYPE_IPV6:
+ flow_dv_translate_item_ipv6(match_mask, match_value,
+ items, tunnel);
+ matcher.priority = MLX5_PRIORITY_MAP_L3;
+ dev_flow->dv.hash_fields |=
+ mlx5_flow_hashfields_adjust
+ (dev_flow, tunnel,
+ MLX5_IPV6_LAYER_TYPES,
+ MLX5_IPV6_IBV_RX_HASH);
+ item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
+ MLX5_FLOW_LAYER_OUTER_L3_IPV6;
+ break;
+ case RTE_FLOW_ITEM_TYPE_TCP:
+ flow_dv_translate_item_tcp(match_mask, match_value,
+ items, tunnel);
+ matcher.priority = MLX5_PRIORITY_MAP_L4;
+ dev_flow->dv.hash_fields |=
+ mlx5_flow_hashfields_adjust
+ (dev_flow, tunnel, ETH_RSS_TCP,
+ IBV_RX_HASH_SRC_PORT_TCP |
+ IBV_RX_HASH_DST_PORT_TCP);
+ item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
+ MLX5_FLOW_LAYER_OUTER_L4_TCP;
+ break;
+ case RTE_FLOW_ITEM_TYPE_UDP:
+ flow_dv_translate_item_udp(match_mask, match_value,
+ items, tunnel);
+ matcher.priority = MLX5_PRIORITY_MAP_L4;
+ dev_flow->dv.hash_fields |=
+ mlx5_flow_hashfields_adjust
+ (dev_flow, tunnel, ETH_RSS_UDP,
+ IBV_RX_HASH_SRC_PORT_UDP |
+ IBV_RX_HASH_DST_PORT_UDP);
+ item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
+ MLX5_FLOW_LAYER_OUTER_L4_UDP;
+ break;
+ case RTE_FLOW_ITEM_TYPE_GRE:
+ flow_dv_translate_item_gre(match_mask, match_value,
+ items, tunnel);
+ item_flags |= MLX5_FLOW_LAYER_GRE;
+ break;
+ case RTE_FLOW_ITEM_TYPE_NVGRE:
+ flow_dv_translate_item_nvgre(match_mask, match_value,
+ items, tunnel);
+ item_flags |= MLX5_FLOW_LAYER_GRE;
+ break;
+ case RTE_FLOW_ITEM_TYPE_VXLAN:
+ flow_dv_translate_item_vxlan(match_mask, match_value,
+ items, tunnel);
+ item_flags |= MLX5_FLOW_LAYER_VXLAN;
+ break;
+ case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
+ flow_dv_translate_item_vxlan(match_mask, match_value,
+ items, tunnel);
+ item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
+ break;
+ case RTE_FLOW_ITEM_TYPE_META:
+ flow_dv_translate_item_meta(match_mask, match_value,
+ items);
+ item_flags |= MLX5_FLOW_ITEM_METADATA;
+ break;
+ default:
+ break;
+ }
+ }
+ assert(!flow_dv_check_valid_spec(matcher.mask.buf,
+ dev_flow->dv.value.buf));
+ dev_flow->layers = item_flags;
+ /* Register matcher. */
+ matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
+ matcher.mask.size);
+ matcher.priority = mlx5_flow_adjust_priority(dev, priority,
+ matcher.priority);
+ matcher.egress = attr->egress;
+ if (flow_dv_matcher_register(dev, &matcher, dev_flow, error))
+ return -rte_errno;
return 0;
}
} else if (flow->actions &
(MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) {
struct mlx5_hrxq *hrxq;
+
hrxq = mlx5_hrxq_get(dev, flow->key,
MLX5_RSS_HASH_KEY_LEN,
dv->hash_fields,
return nlh;
}
-/**
- * Set NLM_F_ACK flags in the last netlink command in buffer.
- * Only last command in the buffer will be acked by system.
- *
- * @param[in, out] buf
- * Pointer to buffer with netlink commands.
- */
-static void
-flow_tcf_setack_nlcmd(struct tcf_nlcb_buf *buf)
-{
- struct nlmsghdr *nlh;
- uint32_t size = 0;
-
- assert(buf->size);
- do {
- nlh = (struct nlmsghdr *)&buf->msg[size];
- size += NLMSG_ALIGN(nlh->nlmsg_len);
- if (size >= buf->size) {
- nlh->nlmsg_flags |= NLM_F_ACK;
- break;
- }
- } while (true);
-}
-
/**
* Send the buffers with prepared netlink commands. Scans the list and
* sends all found buffers. Buffers are sent and freed anyway in order
flow_tcf_send_nlcmd(struct mlx5_flow_tcf_context *tcf,
struct tcf_nlcb_context *ctx)
{
- struct tcf_nlcb_buf *bc, *bn;
- struct nlmsghdr *nlh;
+ struct tcf_nlcb_buf *bc = LIST_FIRST(&ctx->nlbuf);
int ret = 0;
- bc = LIST_FIRST(&ctx->nlbuf);
while (bc) {
+ struct tcf_nlcb_buf *bn = LIST_NEXT(bc, next);
+ struct nlmsghdr *nlh;
+ uint32_t msg = 0;
int rc;
- bn = LIST_NEXT(bc, next);
- if (bc->size) {
- flow_tcf_setack_nlcmd(bc);
- nlh = (struct nlmsghdr *)&bc->msg;
- rc = flow_tcf_nl_ack(tcf, nlh, bc->size, NULL, NULL);
- if (rc && !ret)
- ret = rc;
+ while (msg < bc->size) {
+ /*
+ * Send Netlink commands from buffer in one by one
+ * fashion. If we send multiple rule deletion commands
+ * in one Netlink message and some error occurs it may
+ * cause multiple ACK error messages and break sequence
+ * numbers of Netlink communication, because we expect
+ * the only one ACK reply.
+ */
+ assert((bc->size - msg) >= sizeof(struct nlmsghdr));
+ nlh = (struct nlmsghdr *)&bc->msg[msg];
+ assert((bc->size - msg) >= nlh->nlmsg_len);
+ msg += nlh->nlmsg_len;
+ rc = flow_tcf_nl_ack(tcf, nlh, 0, NULL, NULL);
+ if (rc) {
+ DRV_LOG(WARNING,
+ "netlink: cleanup error %d", rc);
+ if (!ret)
+ ret = rc;
+ }
}
rte_free(bc);
bc = bn;
struct nlattr *na_local = NULL;
struct nlattr *na_peer = NULL;
unsigned char family;
+ uint32_t size;
if (nlh->nlmsg_type != RTM_NEWADDR) {
rte_errno = EINVAL;
if (!na_local || !na_peer)
return 1;
/* Local rule found with scope link, permanent and assigned peer. */
- cmd = flow_tcf_alloc_nlcmd(ctx, MNL_ALIGN(sizeof(struct nlmsghdr)) +
- MNL_ALIGN(sizeof(struct ifaddrmsg)) +
- (family == AF_INET6
- ? 2 * SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN)
- : 2 * SZ_NLATTR_TYPE_OF(uint32_t)));
+ size = MNL_ALIGN(sizeof(struct nlmsghdr)) +
+ MNL_ALIGN(sizeof(struct ifaddrmsg)) +
+ (family == AF_INET6 ? 2 * SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN)
+ : 2 * SZ_NLATTR_TYPE_OF(uint32_t));
+ cmd = flow_tcf_alloc_nlcmd(ctx, size);
if (!cmd) {
rte_errno = ENOMEM;
return -rte_errno;
mnl_attr_put(cmd, IFA_ADDRESS, IPV6_ADDR_LEN,
mnl_attr_get_payload(na_peer));
}
+ assert(size == cmd->nlmsg_len);
return 1;
}
struct nlattr *na_ip = NULL;
struct nlattr *na_mac = NULL;
unsigned char family;
+ uint32_t size;
if (nlh->nlmsg_type != RTM_NEWNEIGH) {
rte_errno = EINVAL;
if (!na_mac || !na_ip)
return 1;
/* Neigh rule with permenent attribute found. */
- cmd = flow_tcf_alloc_nlcmd(ctx, MNL_ALIGN(sizeof(struct nlmsghdr)) +
- MNL_ALIGN(sizeof(struct ndmsg)) +
- SZ_NLATTR_DATA_OF(ETHER_ADDR_LEN) +
- (family == AF_INET6
- ? SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN)
- : SZ_NLATTR_TYPE_OF(uint32_t)));
+ size = MNL_ALIGN(sizeof(struct nlmsghdr)) +
+ MNL_ALIGN(sizeof(struct ndmsg)) +
+ SZ_NLATTR_DATA_OF(ETHER_ADDR_LEN) +
+ (family == AF_INET6 ? SZ_NLATTR_DATA_OF(IPV6_ADDR_LEN)
+ : SZ_NLATTR_TYPE_OF(uint32_t));
+ cmd = flow_tcf_alloc_nlcmd(ctx, size);
if (!cmd) {
rte_errno = ENOMEM;
return -rte_errno;
}
mnl_attr_put(cmd, NDA_LLADDR, ETHER_ADDR_LEN,
mnl_attr_get_payload(na_mac));
+ assert(size == cmd->nlmsg_len);
return 1;
}
struct nlattr *na_vxlan = NULL;
bool found = false;
unsigned int vxindex;
+ uint32_t size;
if (nlh->nlmsg_type != RTM_NEWLINK) {
rte_errno = EINVAL;
return 1;
/* Attached VXLAN device found, store the command to delete. */
vxindex = ifm->ifi_index;
- cmd = flow_tcf_alloc_nlcmd(ctx, MNL_ALIGN(sizeof(struct nlmsghdr)) +
- MNL_ALIGN(sizeof(struct ifinfomsg)));
- if (!nlh) {
+ size = MNL_ALIGN(sizeof(struct nlmsghdr)) +
+ MNL_ALIGN(sizeof(struct ifinfomsg));
+ cmd = flow_tcf_alloc_nlcmd(ctx, size);
+ if (!cmd) {
rte_errno = ENOMEM;
return -rte_errno;
}
ifm = mnl_nlmsg_put_extra_header(cmd, sizeof(*ifm));
ifm->ifi_family = AF_UNSPEC;
ifm->ifi_index = vxindex;
+ assert(size == cmd->nlmsg_len);
return 1;
}
dev_flow->tcf.applied = 1;
return 0;
}
+ if (dev_flow->tcf.tunnel) {
+ /* Rollback the VTEP configuration if rule apply failed. */
+ assert(dev_flow->tcf.tunnel->vtep);
+ flow_tcf_vtep_release(ctx, dev_flow->tcf.tunnel->vtep,
+ dev_flow);
+ dev_flow->tcf.tunnel->vtep = NULL;
+ }
return rte_flow_error_set(error, rte_errno,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
"netlink: failed to create TC flow rule");
MLX5_FLOW_LAYER_OUTER_L3_IPV4;
if (items->mask != NULL &&
((const struct rte_flow_item_ipv4 *)
- items->mask)->hdr.next_proto_id)
+ items->mask)->hdr.next_proto_id) {
next_protocol =
((const struct rte_flow_item_ipv4 *)
(items->spec))->hdr.next_proto_id;
+ next_protocol &=
+ ((const struct rte_flow_item_ipv4 *)
+ (items->mask))->hdr.next_proto_id;
+ } else {
+ /* Reset for inner layer. */
+ next_protocol = 0xff;
+ }
break;
case RTE_FLOW_ITEM_TYPE_IPV6:
ret = mlx5_flow_validate_item_ipv6(items, item_flags,
MLX5_FLOW_LAYER_OUTER_L3_IPV6;
if (items->mask != NULL &&
((const struct rte_flow_item_ipv6 *)
- items->mask)->hdr.proto)
+ items->mask)->hdr.proto) {
next_protocol =
((const struct rte_flow_item_ipv6 *)
items->spec)->hdr.proto;
+ next_protocol &=
+ ((const struct rte_flow_item_ipv6 *)
+ items->mask)->hdr.proto;
+ } else {
+ /* Reset for inner layer. */
+ next_protocol = 0xff;
+ }
break;
case RTE_FLOW_ITEM_TYPE_UDP:
ret = mlx5_flow_validate_item_udp(items, item_flags,
error);
if (ret < 0)
return ret;
- if (next_protocol != 0xff &&
- next_protocol != IPPROTO_MPLS)
- return rte_flow_error_set
- (error, EINVAL,
- RTE_FLOW_ERROR_TYPE_ITEM, items,
- "protocol filtering not compatible"
- " with MPLS layer");
item_flags |= MLX5_FLOW_LAYER_MPLS;
break;
default:
#include "mlx5_defs.h"
+/*
+ * Compilation workaround for PPC64 when AltiVec is fully enabled, e.g. std=c11.
+ * Otherwise there would be a type conflict between stdbool and altivec.
+ */
+#if defined(__PPC64__) && !defined(__APPLE_ALTIVEC__)
+#undef bool
+/* redefine as in stdbool.h */
+#define bool _Bool
+#endif
+
/* Bit-field manipulation. */
#define BITFIELD_DECLARE(bf, type, size) \
type bf[(((size_t)(size) / (sizeof(type) * CHAR_BIT)) + \
#include <rte_byteorder.h>
-#define OCTTX_PACKET_WQE_SKIP 128
-#define OCTTX_PACKET_FIRST_SKIP 240
+#define OCTTX_PACKET_WQE_SKIP 128
+#define OCTTX_PACKET_FIRST_SKIP_MAXREGVAL 496
+#define OCTTX_PACKET_FIRST_SKIP_MAXLEN 512
+#define OCTTX_PACKET_FIRST_SKIP_ADJUST(x) \
+ (RTE_MIN(x, OCTTX_PACKET_FIRST_SKIP_MAXREGVAL))
+#define OCTTX_PACKET_FIRST_SKIP_SUM(p) \
+ (OCTTX_PACKET_WQE_SKIP \
+ + rte_pktmbuf_priv_size(p) \
+ + RTE_PKTMBUF_HEADROOM)
+#define OCTTX_PACKET_FIRST_SKIP(p) \
+ OCTTX_PACKET_FIRST_SKIP_ADJUST(OCTTX_PACKET_FIRST_SKIP_SUM(p))
#define OCTTX_PACKET_LATER_SKIP 128
/* WQE descriptor */
pktbuf_conf.mmask.f_cache_mode = 1;
pktbuf_conf.wqe_skip = OCTTX_PACKET_WQE_SKIP;
- pktbuf_conf.first_skip = OCTTX_PACKET_FIRST_SKIP;
+ pktbuf_conf.first_skip = OCTTX_PACKET_FIRST_SKIP(mb_pool);
pktbuf_conf.later_skip = OCTTX_PACKET_LATER_SKIP;
pktbuf_conf.mbuff_size = (mb_pool->elt_size -
RTE_PKTMBUF_HEADROOM -
+ rte_pktmbuf_priv_size(mb_pool) -
sizeof(struct rte_mbuf));
pktbuf_conf.cache_mode = PKI_OPC_MODE_STF2_STT;
void qede_rx_queue_release(void *rx_queue)
{
struct qede_rx_queue *rxq = rx_queue;
- struct qede_dev *qdev = rxq->qdev;
- struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
-
- PMD_INIT_FUNC_TRACE(edev);
+ struct qede_dev *qdev;
+ struct ecore_dev *edev;
if (rxq) {
+ qdev = rxq->qdev;
+ edev = QEDE_INIT_EDEV(qdev);
+ PMD_INIT_FUNC_TRACE(edev);
qede_rx_queue_release_mbufs(rxq);
qdev->ops->common->chain_free(edev, &rxq->rx_bd_ring);
qdev->ops->common->chain_free(edev, &rxq->rx_comp_ring);
void qede_tx_queue_release(void *tx_queue)
{
struct qede_tx_queue *txq = tx_queue;
- struct qede_dev *qdev = txq->qdev;
- struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
-
- PMD_INIT_FUNC_TRACE(edev);
+ struct qede_dev *qdev;
+ struct ecore_dev *edev;
if (txq) {
+ qdev = txq->qdev;
+ edev = QEDE_INIT_EDEV(qdev);
+ PMD_INIT_FUNC_TRACE(edev);
qede_tx_queue_release_mbufs(txq);
qdev->ops->common->chain_free(edev, &txq->tx_pbl);
rte_free(txq->sw_tx_ring);
}
}
if (ol_flags & QEDE_TX_OFFLOAD_NOTSUP_MASK) {
+ /* We support only limited tunnel protocols */
+ if (ol_flags & PKT_TX_TUNNEL_MASK) {
+ uint64_t temp;
+
+ temp = ol_flags & PKT_TX_TUNNEL_MASK;
+ if (temp == PKT_TX_TUNNEL_VXLAN ||
+ temp == PKT_TX_TUNNEL_GENEVE ||
+ temp == PKT_TX_TUNNEL_MPLSINUDP ||
+ temp == PKT_TX_TUNNEL_GRE)
+ break;
+ }
+
rte_errno = -ENOTSUP;
break;
}
#define QEDE_TX_OFFLOAD_MASK (QEDE_TX_CSUM_OFFLOAD_MASK | \
PKT_TX_VLAN_PKT | \
- PKT_TX_TUNNEL_VXLAN | \
- PKT_TX_TUNNEL_GENEVE | \
- PKT_TX_TUNNEL_MPLSINUDP | \
- PKT_TX_TUNNEL_GRE)
+ PKT_TX_TUNNEL_MASK)
#define QEDE_TX_OFFLOAD_NOTSUP_MASK \
(PKT_TX_OFFLOAD_MASK ^ QEDE_TX_OFFLOAD_MASK)
efx_filter_flags_t efs_flags;
uint16_t efs_dmaq_id;
uint32_t efs_rss_context;
+ uint32_t efs_mark;
+ /* Fields below here are hashed for software filter lookup */
uint16_t efs_outer_vid;
uint16_t efs_inner_vid;
uint8_t efs_loc_mac[EFX_MAC_ADDR_LEN];
efx_oword_t efs_loc_host;
uint8_t efs_vni_or_vsid[EFX_VNI_OR_VSID_LEN];
uint8_t efs_ifrm_loc_mac[EFX_MAC_ADDR_LEN];
- uint32_t efs_mark;
} efx_filter_spec_t;
return fd;
error:
- if (fd > 0)
+ if (fd >= 0)
close(fd);
return -1;
}
TAP_LOG(ERR, "Remote feature requires flow support.");
goto error_exit;
}
+ rte_eth_dev_probing_finish(dev);
return 0;
error_remote:
}
if (setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &sndbuf_size, sizeof(int))) {
TAP_LOG(ERR, "Unable to set socket buffer send size");
+ close(fd);
return -1;
}
if (setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &rcvbuf_size, sizeof(int))) {
TAP_LOG(ERR, "Unable to set socket buffer receive size");
+ close(fd);
return -1;
}
if (bind(fd, (struct sockaddr *)&local, sizeof(local)) < 0) {
TAP_LOG(ERR, "Unable to bind to the netlink socket");
+ close(fd);
return -1;
}
return fd;
entry->buff[0] = sqe.buff[0];
}
+static inline void __hot
+fill_sq_desc_header_zero_w1(union sq_entry_t *entry,
+ struct rte_mbuf *pkt)
+{
+ fill_sq_desc_header(entry, pkt);
+ entry->buff[1] = 0ULL;
+}
+
void __hot
nicvf_single_pool_free_xmited_buffers(struct nicvf_txq *sq)
{
used_bufs += nb_segs;
txbuffs[tail] = NULL;
- fill_sq_desc_header(desc_ptr + tail, pkt);
+ fill_sq_desc_header_zero_w1(desc_ptr + tail, pkt);
tail = (tail + 1) & qlen_mask;
txbuffs[tail] = pkt;
hw->common_cfg = get_cfg_addr(dev, &cap);
break;
case VIRTIO_PCI_CAP_NOTIFY_CFG:
- rte_pci_read_config(dev, &hw->notify_off_multiplier,
+ ret = rte_pci_read_config(dev,
+ &hw->notify_off_multiplier,
4, pos + sizeof(cap));
- hw->notify_base = get_cfg_addr(dev, &cap);
+ if (ret != 4)
+ PMD_INIT_LOG(DEBUG,
+ "failed to read notify_off_multiplier, ret %d",
+ ret);
+ else
+ hw->notify_base = get_cfg_addr(dev, &cap);
break;
case VIRTIO_PCI_CAP_DEVICE_CFG:
hw->dev_cfg = get_cfg_addr(dev, &cap);
}
} else {
PMD_INIT_LOG(ERR, "arg %s is mandatory for virtio_user",
- VIRTIO_USER_ARG_QUEUE_SIZE);
+ VIRTIO_USER_ARG_PATH);
goto end;
}
hw->perm_addr[0], hw->perm_addr[1], hw->perm_addr[2],
hw->perm_addr[3], hw->perm_addr[4], hw->perm_addr[5]);
+ /* Flag to call rte_eth_dev_release_port() in rte_eth_dev_close(). */
+ eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
+
/* Put device in Quiesce Mode */
VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV);
vmxnet3_dev_stop(dev);
vmxnet3_free_queues(dev);
-
- /*
- * flag to rte_eth_dev_close() that it should release the port resources
- * (calling rte_eth_dev_release_port()) in addition to closing it.
- */
- dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
}
static void
static int
writeback_tdes_hex_str(const char *key, char *dst, struct fips_val *val)
{
- struct fips_val tmp_val;
+ struct fips_val tmp_val = {0};
tmp_val.len = 8;
#define TDES_EXTERN_ITER 400
#define TDES_INTERN_ITER 10000
struct fips_val val, val_key;
- uint8_t prev_out[TDES_BLOCK_SIZE];
- uint8_t prev_prev_out[TDES_BLOCK_SIZE];
- uint8_t prev_in[TDES_BLOCK_SIZE];
+ uint8_t prev_out[TDES_BLOCK_SIZE] = {0};
+ uint8_t prev_prev_out[TDES_BLOCK_SIZE] = {0};
+ uint8_t prev_in[TDES_BLOCK_SIZE] = {0};
uint32_t i, j, k;
int ret;
struct rte_eth_dev_info dev_info;
rte_eth_dev_info_get(port_id, &dev_info);
- port_conf.txmode.offloads &= dev_info.rx_offload_capa;
+ port_conf.txmode.offloads &= dev_info.tx_offload_capa;
printf(":: initializing port: %d\n", port_id);
ret = rte_eth_dev_configure(port_id,
nr_queues, nr_queues, &port_conf);
return 0;
cli_rule_file_process_free:
- *rule_list = NULL;
- *n_rules = rule_id;
- *line_number = line_id;
+ if (rule_list != NULL)
+ *rule_list = NULL;
- for ( ; ; ) {
- struct table_rule *rule;
+ if (n_rules != NULL)
+ *n_rules = rule_id;
- rule = TAILQ_FIRST(list);
- if (rule == NULL)
- break;
+ if (line_number != NULL)
+ *line_number = line_id;
- TAILQ_REMOVE(list, rule, node);
- free(rule);
- }
+ if (list != NULL)
+ for ( ; ; ) {
+ struct table_rule *rule;
+
+ rule = TAILQ_FIRST(list);
+ if (rule == NULL)
+ break;
+
+ TAILQ_REMOVE(list, rule, node);
+ free(rule);
+ }
if (f)
fclose(f);
rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n");
if (init_power_library())
- rte_exit(EXIT_FAILURE, "init_power_library failed\n");
+ RTE_LOG(ERR, L3FWD_POWER, "init_power_library failed\n");
if (update_lcore_params() < 0)
rte_exit(EXIT_FAILURE, "update_lcore_params failed\n");
emit_bytes(st, &v, sizeof(v));
}
+/*
+ * emit OPCODE+REGIDX byte
+ */
+static void
+emit_opcode(struct bpf_jit_state *st, uint8_t ops, uint32_t reg)
+{
+ uint8_t v;
+
+ v = ops | (reg & 7);
+ emit_bytes(st, &v, sizeof(v));
+}
+
+
/*
* emit xchg %<sreg>, %<dreg>
*/
emit_ld_imm64(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm0,
uint32_t imm1)
{
+ uint32_t op;
+
const uint8_t ops = 0xB8;
- if (imm1 == 0) {
- emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, dreg, imm0);
- return;
- }
+ op = (imm1 == 0) ? BPF_ALU : EBPF_ALU64;
- emit_rex(st, EBPF_ALU64, 0, dreg);
- emit_bytes(st, &ops, sizeof(ops));
- emit_modregrm(st, MOD_DIRECT, 0, dreg);
+ emit_rex(st, op, 0, dreg);
+ emit_opcode(st, ops, dreg);
emit_imm(st, imm0, sizeof(imm0));
- emit_imm(st, imm1, sizeof(imm1));
+ if (imm1 != 0)
+ emit_imm(st, imm1, sizeof(imm1));
}
/*
+++ /dev/null
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2010-2017 Intel Corporation
- */
-
-#include <rte_memcpy.h>
-#include <rte_cpuflags.h>
-#include <rte_log.h>
-
-void *(*rte_memcpy_ptr)(void *dst, const void *src, size_t n) = NULL;
-
-RTE_INIT(rte_memcpy_init)
-{
-#ifdef CC_SUPPORT_AVX512F
- if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F)) {
- rte_memcpy_ptr = rte_memcpy_avx512f;
- RTE_LOG(DEBUG, EAL, "AVX512 memcpy is using!\n");
- return;
- }
-#endif
-#ifdef CC_SUPPORT_AVX2
- if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) {
- rte_memcpy_ptr = rte_memcpy_avx2;
- RTE_LOG(DEBUG, EAL, "AVX2 memcpy is using!\n");
- return;
- }
-#endif
- rte_memcpy_ptr = rte_memcpy_sse;
- RTE_LOG(DEBUG, EAL, "Default SSE/AVX memcpy is using!\n");
-}
goto err_devarg;
}
- ret = rte_devargs_insert(da);
+ ret = rte_devargs_insert(&da);
if (ret)
goto err_devarg;
+ /* the rte_devargs will be referenced in the matching rte_device */
ret = da->bus->scan();
if (ret)
goto err_devarg;
}
int __rte_experimental
-rte_devargs_insert(struct rte_devargs *da)
+rte_devargs_insert(struct rte_devargs **da)
{
- int ret;
+ struct rte_devargs *listed_da;
+ void *tmp;
+
+ if (*da == NULL || (*da)->bus == NULL)
+ return -1;
- ret = rte_devargs_remove(da);
- if (ret < 0)
- return ret;
- TAILQ_INSERT_TAIL(&devargs_list, da, next);
+ TAILQ_FOREACH_SAFE(listed_da, &devargs_list, next, tmp) {
+ if (listed_da == *da)
+ /* devargs already in the list */
+ return 0;
+ if (strcmp(listed_da->bus->name, (*da)->bus->name) == 0 &&
+ strcmp(listed_da->name, (*da)->name) == 0) {
+ /* device already in devargs list, must be updated */
+ listed_da->type = (*da)->type;
+ listed_da->policy = (*da)->policy;
+ free(listed_da->args);
+ listed_da->args = (*da)->args;
+ listed_da->bus = (*da)->bus;
+ listed_da->cls = (*da)->cls;
+ listed_da->bus_str = (*da)->bus_str;
+ listed_da->cls_str = (*da)->cls_str;
+ listed_da->data = (*da)->data;
+ /* replace provided devargs with found one */
+ free(*da);
+ *da = listed_da;
+ return 0;
+ }
+ }
+ /* new device in the list */
+ TAILQ_INSERT_TAIL(&devargs_list, *da, next);
return 0;
}
return 1;
}
-#if defined(RTE_ARCH_64)
#define MAX_DMA_MASK_BITS 63
-#else
-#define MAX_DMA_MASK_BITS 31
-#endif
/* check memseg iovas are within the required range based on dma mask */
static int __rte_experimental
uint64_t mask;
int ret;
- /* sanity check */
+ /* Sanity check. We only check width can be managed with 64 bits
+ * variables. Indeed any higher value is likely wrong. */
if (maskbits > MAX_DMA_MASK_BITS) {
RTE_LOG(ERR, EAL, "wrong dma mask size %u (Max: %u)\n",
maskbits, MAX_DMA_MASK_BITS);
{
struct rte_mp_msg *reply_msg;
struct pending_request *pending_req, *exist;
- int ret;
+ int ret = -1;
pending_req = calloc(1, sizeof(*pending_req));
reply_msg = calloc(1, sizeof(*reply_msg));
goto fail;
}
+ /*
+ * set the alarm before sending message. there are two possible error
+ * scenarios to consider here:
+ *
+ * - if the alarm set fails, we free the memory right there
+ * - if the alarm set succeeds but sending message fails, then the alarm
+ * will trigger and clean up the memory
+ *
+ * Even if the alarm triggers too early (i.e. immediately), we're still
+ * holding the lock to pending requests queue, so the interrupt thread
+ * will just spin until we release the lock, and either release the
+ * memory, or doesn't find any pending requests in the queue because we
+ * never added any due to send message failure.
+ */
+ if (rte_eal_alarm_set(ts->tv_sec * 1000000 + ts->tv_nsec / 1000,
+ async_reply_handle, pending_req) < 0) {
+ RTE_LOG(ERR, EAL, "Fail to set alarm for request %s:%s\n",
+ dst, req->name);
+ ret = -1;
+ goto fail;
+ }
+
ret = send_msg(dst, req, MP_REQ);
if (ret < 0) {
RTE_LOG(ERR, EAL, "Fail to send request %s:%s\n",
param->user_reply.nb_sent++;
- if (rte_eal_alarm_set(ts->tv_sec * 1000000 + ts->tv_nsec / 1000,
- async_reply_handle, pending_req) < 0) {
- RTE_LOG(ERR, EAL, "Fail to set alarm for request %s:%s\n",
- dst, req->name);
- rte_panic("Fix the above shit to properly free all memory\n");
- }
-
return 0;
fail:
free(pending_req);
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2012,2013 Intel Corporation
+ */
+
#ifndef _RTE_RTM_H_
#define _RTE_RTM_H_ 1
-/*
- * Copyright (c) 2012,2013 Intel Corporation
- * Author: Andi Kleen
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that: (1) source code distributions
- * retain the above copyright notice and this paragraph in its entirety, (2)
- * distributions including binary code include the above copyright notice and
- * this paragraph in its entirety in the documentation or other materials
- * provided with the distribution
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
- */
/* Official RTM intrinsics interface matching gcc/icc, but works
on older gcc compatible compilers and binutils. */
#include "rte_branch_prediction.h"
#include "rte_common.h"
#include "rte_pause.h"
+#include "rte_cycles.h"
-#define RTE_RTM_MAX_RETRIES (10)
+#define RTE_RTM_MAX_RETRIES (20)
#define RTE_XABORT_LOCK_BUSY (0xff)
#ifndef RTE_FORCE_INTRINSICS
static inline int
rte_try_tm(volatile int *lock)
{
- int retries;
+ int i, retries;
if (!rte_rtm_supported)
return 0;
while (*lock)
rte_pause();
- if ((status & RTE_XABORT_EXPLICIT) &&
- (RTE_XABORT_CODE(status) == RTE_XABORT_LOCK_BUSY))
+ if ((status & RTE_XABORT_CONFLICT) ||
+ ((status & RTE_XABORT_EXPLICIT) &&
+ (RTE_XABORT_CODE(status) == RTE_XABORT_LOCK_BUSY))) {
+ /* add a small delay before retrying, basing the
+ * delay on the number of times we've already tried,
+ * to give a back-off type of behaviour. We
+ * randomize trycount by taking bits from the tsc count
+ */
+ int try_count = RTE_RTM_MAX_RETRIES - retries;
+ int pause_count = (rte_rdtsc() & 0x7) | 1;
+ pause_count <<= try_count;
+ for (i = 0; i < pause_count; i++)
+ rte_pause();
continue;
+ }
if ((status & RTE_XABORT_RETRY) == 0) /* do not retry */
break;
return rte_bsf32(v);
}
+
+/**
+ * Return the last (most-significant) bit set.
+ *
+ * @note The last (most significant) bit is at position 32.
+ * @note rte_fls_u32(0) = 0, rte_fls_u32(1) = 1, rte_fls_u32(0x80000000) = 32
+ *
+ * @param x
+ * The input parameter.
+ * @return
+ * The last (most-significant) bit set, or 0 if the input is 0.
+ */
+static inline int
+rte_fls_u32(uint32_t x)
+{
+ return (x == 0) ? 0 : 32 - __builtin_clz(x);
+}
+
+
#ifndef offsetof
/** Return the offset of a field in a structure. */
#define offsetof(TYPE, MEMBER) __builtin_offsetof (TYPE, MEMBER)
*
* @param da
* The devargs structure to insert.
+ * If a devargs for the same device is already inserted,
+ * it will be updated and returned. It means *da pointer can change.
*
* @return
* - 0 on success
*/
__rte_experimental
int
-rte_devargs_insert(struct rte_devargs *da);
+rte_devargs_insert(struct rte_devargs **da);
/**
* Add a device to the user device list
* 0-15 = release candidates
* 16 = release
*/
-#define RTE_VER_RELEASE 2
+#define RTE_VER_RELEASE 3
/**
* Macro to compute a version number usable for comparisons
#include "rte_reciprocal.h"
-/* find largest set bit.
- * portable and slow but does not matter for this usage.
- */
-static inline int fls(uint32_t x)
-{
- int b;
-
- for (b = 31; b >= 0; --b) {
- if (x & (1u << b))
- return b + 1;
- }
-
- return 0;
-}
-
struct rte_reciprocal rte_reciprocal_value(uint32_t d)
{
struct rte_reciprocal R;
uint64_t m;
int l;
- l = fls(d - 1);
+ l = rte_fls_u32(d - 1);
m = ((1ULL << 32) * ((1ULL << l) - d));
m /= d;
#define NS_PER_US 1000
#define US_PER_MS 1000
#define MS_PER_S 1000
+#ifndef US_PER_S
#define US_PER_S (US_PER_MS * MS_PER_S)
+#endif
#ifdef CLOCK_MONOTONIC_RAW /* Defined in glibc bits/time.h */
#define CLOCK_TYPE_ID CLOCK_MONOTONIC_RAW
tmp_hp = NULL;
munmap(hugepage, nr_hugefiles * sizeof(struct hugepage_file));
+ hugepage = NULL;
/* we're not going to allocate more pages, so release VA space for
* unused memseg lists
{
struct rte_eth_dev *dev;
struct rte_eth_dev_info dev_info;
- struct rte_eth_conf local_conf = *dev_conf;
+ struct rte_eth_conf orig_conf;
int diag;
+ int ret;
RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
+ if (dev->data->dev_started) {
+ RTE_ETHDEV_LOG(ERR,
+ "Port %u must be stopped to allow configuration\n",
+ port_id);
+ return -EBUSY;
+ }
+
+ /* Store original config, as rollback required on failure */
+ memcpy(&orig_conf, &dev->data->dev_conf, sizeof(dev->data->dev_conf));
+
+ /*
+ * Copy the dev_conf parameter into the dev structure.
+ * rte_eth_dev_info_get() requires dev_conf, copy it before dev_info get
+ */
+ memcpy(&dev->data->dev_conf, dev_conf, sizeof(dev->data->dev_conf));
+
rte_eth_dev_info_get(port_id, &dev_info);
/* If number of queues specified by application for both Rx and Tx is
RTE_ETHDEV_LOG(ERR,
"Number of RX queues requested (%u) is greater than max supported(%d)\n",
nb_rx_q, RTE_MAX_QUEUES_PER_PORT);
- return -EINVAL;
+ ret = -EINVAL;
+ goto rollback;
}
if (nb_tx_q > RTE_MAX_QUEUES_PER_PORT) {
RTE_ETHDEV_LOG(ERR,
"Number of TX queues requested (%u) is greater than max supported(%d)\n",
nb_tx_q, RTE_MAX_QUEUES_PER_PORT);
- return -EINVAL;
- }
-
- if (dev->data->dev_started) {
- RTE_ETHDEV_LOG(ERR,
- "Port %u must be stopped to allow configuration\n",
- port_id);
- return -EBUSY;
+ ret = -EINVAL;
+ goto rollback;
}
- /* Copy the dev_conf parameter into the dev structure */
- memcpy(&dev->data->dev_conf, &local_conf, sizeof(dev->data->dev_conf));
-
/*
* Check that the numbers of RX and TX queues are not greater
* than the maximum number of RX and TX queues supported by the
if (nb_rx_q > dev_info.max_rx_queues) {
RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%u nb_rx_queues=%u > %u\n",
port_id, nb_rx_q, dev_info.max_rx_queues);
- return -EINVAL;
+ ret = -EINVAL;
+ goto rollback;
}
if (nb_tx_q > dev_info.max_tx_queues) {
RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%u nb_tx_queues=%u > %u\n",
port_id, nb_tx_q, dev_info.max_tx_queues);
- return -EINVAL;
+ ret = -EINVAL;
+ goto rollback;
}
/* Check that the device supports requested interrupts */
(!(dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC))) {
RTE_ETHDEV_LOG(ERR, "Driver %s does not support lsc\n",
dev->device->driver->name);
- return -EINVAL;
+ ret = -EINVAL;
+ goto rollback;
}
if ((dev_conf->intr_conf.rmv == 1) &&
(!(dev->data->dev_flags & RTE_ETH_DEV_INTR_RMV))) {
RTE_ETHDEV_LOG(ERR, "Driver %s does not support rmv\n",
dev->device->driver->name);
- return -EINVAL;
+ ret = -EINVAL;
+ goto rollback;
}
/*
* If jumbo frames are enabled, check that the maximum RX packet
* length is supported by the configured device.
*/
- if (local_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
+ if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
if (dev_conf->rxmode.max_rx_pkt_len > dev_info.max_rx_pktlen) {
RTE_ETHDEV_LOG(ERR,
"Ethdev port_id=%u max_rx_pkt_len %u > max valid value %u\n",
port_id, dev_conf->rxmode.max_rx_pkt_len,
dev_info.max_rx_pktlen);
- return -EINVAL;
+ ret = -EINVAL;
+ goto rollback;
} else if (dev_conf->rxmode.max_rx_pkt_len < ETHER_MIN_LEN) {
RTE_ETHDEV_LOG(ERR,
"Ethdev port_id=%u max_rx_pkt_len %u < min valid value %u\n",
port_id, dev_conf->rxmode.max_rx_pkt_len,
(unsigned)ETHER_MIN_LEN);
- return -EINVAL;
+ ret = -EINVAL;
+ goto rollback;
}
} else {
if (dev_conf->rxmode.max_rx_pkt_len < ETHER_MIN_LEN ||
}
/* Any requested offloading must be within its device capabilities */
- if ((local_conf.rxmode.offloads & dev_info.rx_offload_capa) !=
- local_conf.rxmode.offloads) {
+ if ((dev_conf->rxmode.offloads & dev_info.rx_offload_capa) !=
+ dev_conf->rxmode.offloads) {
RTE_ETHDEV_LOG(ERR,
"Ethdev port_id=%u requested Rx offloads 0x%"PRIx64" doesn't match Rx offloads "
"capabilities 0x%"PRIx64" in %s()\n",
- port_id, local_conf.rxmode.offloads,
+ port_id, dev_conf->rxmode.offloads,
dev_info.rx_offload_capa,
__func__);
- return -EINVAL;
+ ret = -EINVAL;
+ goto rollback;
}
- if ((local_conf.txmode.offloads & dev_info.tx_offload_capa) !=
- local_conf.txmode.offloads) {
+ if ((dev_conf->txmode.offloads & dev_info.tx_offload_capa) !=
+ dev_conf->txmode.offloads) {
RTE_ETHDEV_LOG(ERR,
"Ethdev port_id=%u requested Tx offloads 0x%"PRIx64" doesn't match Tx offloads "
"capabilities 0x%"PRIx64" in %s()\n",
- port_id, local_conf.txmode.offloads,
+ port_id, dev_conf->txmode.offloads,
dev_info.tx_offload_capa,
__func__);
- return -EINVAL;
+ ret = -EINVAL;
+ goto rollback;
}
/* Check that device supports requested rss hash functions. */
"Ethdev port_id=%u invalid rss_hf: 0x%"PRIx64", valid value: 0x%"PRIx64"\n",
port_id, dev_conf->rx_adv_conf.rss_conf.rss_hf,
dev_info.flow_type_rss_offloads);
- return -EINVAL;
+ ret = -EINVAL;
+ goto rollback;
}
/*
RTE_ETHDEV_LOG(ERR,
"Port%u rte_eth_dev_rx_queue_config = %d\n",
port_id, diag);
- return diag;
+ ret = diag;
+ goto rollback;
}
diag = rte_eth_dev_tx_queue_config(dev, nb_tx_q);
"Port%u rte_eth_dev_tx_queue_config = %d\n",
port_id, diag);
rte_eth_dev_rx_queue_config(dev, 0);
- return diag;
+ ret = diag;
+ goto rollback;
}
diag = (*dev->dev_ops->dev_configure)(dev);
port_id, diag);
rte_eth_dev_rx_queue_config(dev, 0);
rte_eth_dev_tx_queue_config(dev, 0);
- return eth_err(port_id, diag);
+ ret = eth_err(port_id, diag);
+ goto rollback;
}
/* Initialize Rx profiling if enabled at compilation time. */
port_id, diag);
rte_eth_dev_rx_queue_config(dev, 0);
rte_eth_dev_tx_queue_config(dev, 0);
- return eth_err(port_id, diag);
+ ret = eth_err(port_id, diag);
+ goto rollback;
}
return 0;
+
+rollback:
+ memcpy(&dev->data->dev_conf, &orig_conf, sizeof(dev->data->dev_conf));
+
+ return ret;
}
void
* Copyright(c) 2015 Intel Corporation
*/
+#include <rte_vect.h>
+
/* Functions to compare multiple of 16 byte keys (up to 128 bytes) */
static int
rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unused)
#include <rte_common.h>
#include <rte_memory.h> /* for definition of RTE_CACHE_LINE_SIZE */
#include <rte_log.h>
-#include <rte_memcpy.h>
#include <rte_prefetch.h>
#include <rte_branch_prediction.h>
#include <rte_malloc.h>
new_k = RTE_PTR_ADD(keys, (uintptr_t)slot_id * h->key_entry_size);
new_idx = (uint32_t)((uintptr_t) slot_id);
/* Copy key */
- rte_memcpy(new_k->key, key, h->key_len);
+ memcpy(new_k->key, key, h->key_len);
/* Key can be of arbitrary length, so it is not possible to store
* it atomically. Hence the new key element's memory stores
* (key as well as data) should be complete before it is referenced.
return ret;
}
+/* Search one bucket to find the match key - uses rw lock */
+static inline int32_t
+search_one_bucket_l(const struct rte_hash *h, const void *key,
+ uint16_t sig, void **data,
+ const struct rte_hash_bucket *bkt)
+{
+ int i;
+ struct rte_hash_key *k, *keys = h->key_store;
+
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (bkt->sig_current[i] == sig &&
+ bkt->key_idx[i] != EMPTY_SLOT) {
+ k = (struct rte_hash_key *) ((char *)keys +
+ bkt->key_idx[i] * h->key_entry_size);
+
+ if (rte_hash_cmp_eq(key, k->key, h) == 0) {
+ if (data != NULL)
+ *data = k->pdata;
+ /*
+ * Return index where key is stored,
+ * subtracting the first dummy index
+ */
+ return bkt->key_idx[i] - 1;
+ }
+ }
+ }
+ return -1;
+}
+
/* Search one bucket to find the match key */
static inline int32_t
-search_one_bucket(const struct rte_hash *h, const void *key, uint16_t sig,
+search_one_bucket_lf(const struct rte_hash *h, const void *key, uint16_t sig,
void **data, const struct rte_hash_bucket *bkt)
{
int i;
}
static inline int32_t
-__rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
- hash_sig_t sig, void **data)
+__rte_hash_lookup_with_hash_l(const struct rte_hash *h, const void *key,
+ hash_sig_t sig, void **data)
{
uint32_t prim_bucket_idx, sec_bucket_idx;
struct rte_hash_bucket *bkt, *cur_bkt;
- uint32_t cnt_b, cnt_a;
int ret;
uint16_t short_sig;
prim_bucket_idx = get_prim_bucket_index(h, sig);
sec_bucket_idx = get_alt_bucket_index(h, prim_bucket_idx, short_sig);
+ bkt = &h->buckets[prim_bucket_idx];
+
__hash_rw_reader_lock(h);
+ /* Check if key is in primary location */
+ ret = search_one_bucket_l(h, key, short_sig, data, bkt);
+ if (ret != -1) {
+ __hash_rw_reader_unlock(h);
+ return ret;
+ }
+ /* Calculate secondary hash */
+ bkt = &h->buckets[sec_bucket_idx];
+
+ /* Check if key is in secondary location */
+ FOR_EACH_BUCKET(cur_bkt, bkt) {
+ ret = search_one_bucket_l(h, key, short_sig,
+ data, cur_bkt);
+ if (ret != -1) {
+ __hash_rw_reader_unlock(h);
+ return ret;
+ }
+ }
+
+ __hash_rw_reader_unlock(h);
+
+ return -ENOENT;
+}
+
+static inline int32_t
+__rte_hash_lookup_with_hash_lf(const struct rte_hash *h, const void *key,
+ hash_sig_t sig, void **data)
+{
+ uint32_t prim_bucket_idx, sec_bucket_idx;
+ struct rte_hash_bucket *bkt, *cur_bkt;
+ uint32_t cnt_b, cnt_a;
+ int ret;
+ uint16_t short_sig;
+
+ short_sig = get_short_sig(sig);
+ prim_bucket_idx = get_prim_bucket_index(h, sig);
+ sec_bucket_idx = get_alt_bucket_index(h, prim_bucket_idx, short_sig);
+
do {
/* Load the table change counter before the lookup
* starts. Acquire semantics will make sure that
/* Check if key is in primary location */
bkt = &h->buckets[prim_bucket_idx];
- ret = search_one_bucket(h, key, short_sig, data, bkt);
+ ret = search_one_bucket_lf(h, key, short_sig, data, bkt);
if (ret != -1) {
__hash_rw_reader_unlock(h);
return ret;
/* Check if key is in secondary location */
FOR_EACH_BUCKET(cur_bkt, bkt) {
- ret = search_one_bucket(h, key, short_sig,
+ ret = search_one_bucket_lf(h, key, short_sig,
data, cur_bkt);
if (ret != -1) {
__hash_rw_reader_unlock(h);
__ATOMIC_ACQUIRE);
} while (cnt_b != cnt_a);
- __hash_rw_reader_unlock(h);
-
return -ENOENT;
}
+static inline int32_t
+__rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
+ hash_sig_t sig, void **data)
+{
+ if (h->readwrite_concur_lf_support)
+ return __rte_hash_lookup_with_hash_lf(h, key, sig, data);
+ else
+ return __rte_hash_lookup_with_hash_l(h, key, sig, data);
+}
+
int32_t
rte_hash_lookup_with_hash(const struct rte_hash *h,
const void *key, hash_sig_t sig)
#define PREFETCH_OFFSET 4
static inline void
-__rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
+__rte_hash_lookup_bulk_l(const struct rte_hash *h, const void **keys,
+ int32_t num_keys, int32_t *positions,
+ uint64_t *hit_mask, void *data[])
+{
+ uint64_t hits = 0;
+ int32_t i;
+ int32_t ret;
+ uint32_t prim_hash[RTE_HASH_LOOKUP_BULK_MAX];
+ uint32_t prim_index[RTE_HASH_LOOKUP_BULK_MAX];
+ uint32_t sec_index[RTE_HASH_LOOKUP_BULK_MAX];
+ uint16_t sig[RTE_HASH_LOOKUP_BULK_MAX];
+ const struct rte_hash_bucket *primary_bkt[RTE_HASH_LOOKUP_BULK_MAX];
+ const struct rte_hash_bucket *secondary_bkt[RTE_HASH_LOOKUP_BULK_MAX];
+ uint32_t prim_hitmask[RTE_HASH_LOOKUP_BULK_MAX] = {0};
+ uint32_t sec_hitmask[RTE_HASH_LOOKUP_BULK_MAX] = {0};
+ struct rte_hash_bucket *cur_bkt, *next_bkt;
+
+ /* Prefetch first keys */
+ for (i = 0; i < PREFETCH_OFFSET && i < num_keys; i++)
+ rte_prefetch0(keys[i]);
+
+ /*
+ * Prefetch rest of the keys, calculate primary and
+ * secondary bucket and prefetch them
+ */
+ for (i = 0; i < (num_keys - PREFETCH_OFFSET); i++) {
+ rte_prefetch0(keys[i + PREFETCH_OFFSET]);
+
+ prim_hash[i] = rte_hash_hash(h, keys[i]);
+
+ sig[i] = get_short_sig(prim_hash[i]);
+ prim_index[i] = get_prim_bucket_index(h, prim_hash[i]);
+ sec_index[i] = get_alt_bucket_index(h, prim_index[i], sig[i]);
+
+ primary_bkt[i] = &h->buckets[prim_index[i]];
+ secondary_bkt[i] = &h->buckets[sec_index[i]];
+
+ rte_prefetch0(primary_bkt[i]);
+ rte_prefetch0(secondary_bkt[i]);
+ }
+
+ /* Calculate and prefetch rest of the buckets */
+ for (; i < num_keys; i++) {
+ prim_hash[i] = rte_hash_hash(h, keys[i]);
+
+ sig[i] = get_short_sig(prim_hash[i]);
+ prim_index[i] = get_prim_bucket_index(h, prim_hash[i]);
+ sec_index[i] = get_alt_bucket_index(h, prim_index[i], sig[i]);
+
+ primary_bkt[i] = &h->buckets[prim_index[i]];
+ secondary_bkt[i] = &h->buckets[sec_index[i]];
+
+ rte_prefetch0(primary_bkt[i]);
+ rte_prefetch0(secondary_bkt[i]);
+ }
+
+ __hash_rw_reader_lock(h);
+
+ /* Compare signatures and prefetch key slot of first hit */
+ for (i = 0; i < num_keys; i++) {
+ compare_signatures(&prim_hitmask[i], &sec_hitmask[i],
+ primary_bkt[i], secondary_bkt[i],
+ sig[i], h->sig_cmp_fn);
+
+ if (prim_hitmask[i]) {
+ uint32_t first_hit =
+ __builtin_ctzl(prim_hitmask[i])
+ >> 1;
+ uint32_t key_idx =
+ primary_bkt[i]->key_idx[first_hit];
+ const struct rte_hash_key *key_slot =
+ (const struct rte_hash_key *)(
+ (const char *)h->key_store +
+ key_idx * h->key_entry_size);
+ rte_prefetch0(key_slot);
+ continue;
+ }
+
+ if (sec_hitmask[i]) {
+ uint32_t first_hit =
+ __builtin_ctzl(sec_hitmask[i])
+ >> 1;
+ uint32_t key_idx =
+ secondary_bkt[i]->key_idx[first_hit];
+ const struct rte_hash_key *key_slot =
+ (const struct rte_hash_key *)(
+ (const char *)h->key_store +
+ key_idx * h->key_entry_size);
+ rte_prefetch0(key_slot);
+ }
+ }
+
+ /* Compare keys, first hits in primary first */
+ for (i = 0; i < num_keys; i++) {
+ positions[i] = -ENOENT;
+ while (prim_hitmask[i]) {
+ uint32_t hit_index =
+ __builtin_ctzl(prim_hitmask[i])
+ >> 1;
+ uint32_t key_idx =
+ primary_bkt[i]->key_idx[hit_index];
+ const struct rte_hash_key *key_slot =
+ (const struct rte_hash_key *)(
+ (const char *)h->key_store +
+ key_idx * h->key_entry_size);
+
+ /*
+ * If key index is 0, do not compare key,
+ * as it is checking the dummy slot
+ */
+ if (!!key_idx &
+ !rte_hash_cmp_eq(
+ key_slot->key, keys[i], h)) {
+ if (data != NULL)
+ data[i] = key_slot->pdata;
+
+ hits |= 1ULL << i;
+ positions[i] = key_idx - 1;
+ goto next_key;
+ }
+ prim_hitmask[i] &= ~(3ULL << (hit_index << 1));
+ }
+
+ while (sec_hitmask[i]) {
+ uint32_t hit_index =
+ __builtin_ctzl(sec_hitmask[i])
+ >> 1;
+ uint32_t key_idx =
+ secondary_bkt[i]->key_idx[hit_index];
+ const struct rte_hash_key *key_slot =
+ (const struct rte_hash_key *)(
+ (const char *)h->key_store +
+ key_idx * h->key_entry_size);
+
+ /*
+ * If key index is 0, do not compare key,
+ * as it is checking the dummy slot
+ */
+
+ if (!!key_idx &
+ !rte_hash_cmp_eq(
+ key_slot->key, keys[i], h)) {
+ if (data != NULL)
+ data[i] = key_slot->pdata;
+
+ hits |= 1ULL << i;
+ positions[i] = key_idx - 1;
+ goto next_key;
+ }
+ sec_hitmask[i] &= ~(3ULL << (hit_index << 1));
+ }
+next_key:
+ continue;
+ }
+
+ /* all found, do not need to go through ext bkt */
+ if ((hits == ((1ULL << num_keys) - 1)) || !h->ext_table_support) {
+ if (hit_mask != NULL)
+ *hit_mask = hits;
+ __hash_rw_reader_unlock(h);
+ return;
+ }
+
+ /* need to check ext buckets for match */
+ for (i = 0; i < num_keys; i++) {
+ if ((hits & (1ULL << i)) != 0)
+ continue;
+ next_bkt = secondary_bkt[i]->next;
+ FOR_EACH_BUCKET(cur_bkt, next_bkt) {
+ if (data != NULL)
+ ret = search_one_bucket_l(h, keys[i],
+ sig[i], &data[i], cur_bkt);
+ else
+ ret = search_one_bucket_l(h, keys[i],
+ sig[i], NULL, cur_bkt);
+ if (ret != -1) {
+ positions[i] = ret;
+ hits |= 1ULL << i;
+ break;
+ }
+ }
+ }
+
+ __hash_rw_reader_unlock(h);
+
+ if (hit_mask != NULL)
+ *hit_mask = hits;
+}
+
+static inline void
+__rte_hash_lookup_bulk_lf(const struct rte_hash *h, const void **keys,
int32_t num_keys, int32_t *positions,
uint64_t *hit_mask, void *data[])
{
rte_prefetch0(secondary_bkt[i]);
}
- __hash_rw_reader_lock(h);
do {
/* Load the table change counter before the lookup
* starts. Acquire semantics will make sure that
next_bkt = secondary_bkt[i]->next;
FOR_EACH_BUCKET(cur_bkt, next_bkt) {
if (data != NULL)
- ret = search_one_bucket(h, keys[i],
+ ret = search_one_bucket_lf(h, keys[i],
sig[i], &data[i], cur_bkt);
else
- ret = search_one_bucket(h, keys[i],
+ ret = search_one_bucket_lf(h, keys[i],
sig[i], NULL, cur_bkt);
if (ret != -1) {
positions[i] = ret;
}
}
- __hash_rw_reader_unlock(h);
-
if (hit_mask != NULL)
*hit_mask = hits;
}
+static inline void
+__rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
+ int32_t num_keys, int32_t *positions,
+ uint64_t *hit_mask, void *data[])
+{
+ if (h->readwrite_concur_lf_support)
+ return __rte_hash_lookup_bulk_lf(h, keys, num_keys,
+ positions, hit_mask, data);
+ else
+ return __rte_hash_lookup_bulk_l(h, keys, num_keys,
+ positions, hit_mask, data);
+}
+
int
rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
uint32_t num_keys, int32_t *positions)
uint8_t *u8 = _u8;
char *end;
+ /* empty string is an error though strtoul() returns 0 */
+ if (*in == '\0')
+ return NULL;
+
errno = 0;
val = strtoul(in, &end, 16);
if (errno != 0 || end[0] != dlm || val > UINT8_MAX) {
rte_cryptodev_info_get(cdev_id, &dev_info);
- for (i = 0;; i++) {
+ for (i = 0; dev_info.capabilities[i].op != RTE_CRYPTO_OP_TYPE_UNDEFINED;
+ i++) {
cap = &dev_info.capabilities[i];
- if (!cap)
- break;
if (cap->sym.xform_type != xform->type)
continue;
unsigned int max = n;
int success;
- *old_head = __atomic_load_n(&r->prod.head, __ATOMIC_ACQUIRE);
+ *old_head = __atomic_load_n(&r->prod.head, __ATOMIC_RELAXED);
do {
/* Reset n to the initial burst count */
n = max;
+ /* Ensure the head is read before tail */
+ __atomic_thread_fence(__ATOMIC_ACQUIRE);
+
/* load-acquire synchronize with store-release of ht->tail
* in update_tail.
*/
/* on failure, *old_head is updated */
success = __atomic_compare_exchange_n(&r->prod.head,
old_head, *new_head,
- 0, __ATOMIC_ACQUIRE,
+ 0, __ATOMIC_RELAXED,
__ATOMIC_RELAXED);
} while (unlikely(success == 0));
return n;
int success;
/* move cons.head atomically */
- *old_head = __atomic_load_n(&r->cons.head, __ATOMIC_ACQUIRE);
+ *old_head = __atomic_load_n(&r->cons.head, __ATOMIC_RELAXED);
do {
/* Restore n as it may change every loop */
n = max;
+ /* Ensure the head is read before tail */
+ __atomic_thread_fence(__ATOMIC_ACQUIRE);
+
/* this load-acquire synchronize with store-release of ht->tail
* in update_tail.
*/
/* on failure, *old_head will be updated */
success = __atomic_compare_exchange_n(&r->cons.head,
old_head, *new_head,
- 0, __ATOMIC_ACQUIRE,
+ 0, __ATOMIC_RELAXED,
__ATOMIC_RELAXED);
} while (unlikely(success == 0));
return n;
struct rte_hash *session_map;
struct rte_mempool *mbuf_pool;
struct rte_mempool *sess_pool;
+ struct rte_mempool *wb_pool;
/** DPDK cryptodev ID */
uint8_t cid;
uint8_t option;
} __rte_cache_aligned;
+struct vhost_crypto_writeback_data {
+ uint8_t *src;
+ uint8_t *dst;
+ uint64_t len;
+ struct vhost_crypto_writeback_data *next;
+};
+
struct vhost_crypto_data_req {
struct vring_desc *head;
struct virtio_net *dev;
struct virtio_crypto_inhdr *inhdr;
struct vhost_virtqueue *vq;
- struct vring_desc *wb_desc;
- uint16_t wb_len;
+ struct vhost_crypto_writeback_data *wb;
+ struct rte_mempool *wb_pool;
uint16_t desc_idx;
uint16_t len;
uint16_t zero_copy;
left -= desc->len;
}
- if (unlikely(left > 0)) {
- VC_LOG_ERR("Incorrect virtio descriptor");
+ if (unlikely(left > 0))
return -1;
- }
*cur_desc = &head[desc->next];
return 0;
}
+static __rte_always_inline void *
+get_data_ptr(struct vhost_crypto_data_req *vc_req, struct vring_desc *cur_desc,
+ uint8_t perm)
+{
+ void *data;
+ uint64_t dlen = cur_desc->len;
+
+ data = IOVA_TO_VVA(void *, vc_req, cur_desc->addr, &dlen, perm);
+ if (unlikely(!data || dlen != cur_desc->len)) {
+ VC_LOG_ERR("Failed to map object");
+ return NULL;
+ }
+
+ return data;
+}
+
static int
copy_data(void *dst_data, struct vhost_crypto_data_req *vc_req,
struct vring_desc **cur_desc, uint32_t size)
dlen = to_copy;
src = IOVA_TO_VVA(uint8_t *, vc_req, desc->addr, &dlen,
VHOST_ACCESS_RO);
- if (unlikely(!src || !dlen)) {
- VC_LOG_ERR("Failed to map descriptor");
+ if (unlikely(!src || !dlen))
return -1;
- }
rte_memcpy((uint8_t *)data, src, dlen);
data += dlen;
return 0;
}
-static __rte_always_inline void *
-get_data_ptr(struct vhost_crypto_data_req *vc_req, struct vring_desc **cur_desc,
- uint32_t size, uint8_t perm)
+static void
+write_back_data(struct vhost_crypto_data_req *vc_req)
{
- void *data;
- uint64_t dlen = (*cur_desc)->len;
-
- data = IOVA_TO_VVA(void *, vc_req, (*cur_desc)->addr, &dlen, perm);
- if (unlikely(!data || dlen != (*cur_desc)->len)) {
- VC_LOG_ERR("Failed to map object");
- return NULL;
+ struct vhost_crypto_writeback_data *wb_data = vc_req->wb, *wb_last;
+
+ while (wb_data) {
+ rte_prefetch0(wb_data->next);
+ rte_memcpy(wb_data->dst, wb_data->src, wb_data->len);
+ wb_last = wb_data;
+ wb_data = wb_data->next;
+ rte_mempool_put(vc_req->wb_pool, wb_last);
}
+}
- if (unlikely(move_desc(vc_req->head, cur_desc, size) < 0))
- return NULL;
+static void
+free_wb_data(struct vhost_crypto_writeback_data *wb_data,
+ struct rte_mempool *mp)
+{
+ while (wb_data->next != NULL)
+ free_wb_data(wb_data->next, mp);
- return data;
+ rte_mempool_put(mp, wb_data);
}
-static int
-write_back_data(struct rte_crypto_op *op, struct vhost_crypto_data_req *vc_req)
+/**
+ * The function will allocate a vhost_crypto_writeback_data linked list
+ * containing the source and destination data pointers for the write back
+ * operation after dequeued from Cryptodev PMD queues.
+ *
+ * @param vc_req
+ * The vhost crypto data request pointer
+ * @param cur_desc
+ * The pointer of the current in use descriptor pointer. The content of
+ * cur_desc is expected to be updated after the function execution.
+ * @param end_wb_data
+ * The last write back data element to be returned. It is used only in cipher
+ * and hash chain operations.
+ * @param src
+ * The source data pointer
+ * @param offset
+ * The offset to both source and destination data. For source data the offset
+ * is the number of bytes between src and start point of cipher operation. For
+ * destination data the offset is the number of bytes from *cur_desc->addr
+ * to the point where the src will be written to.
+ * @param write_back_len
+ * The size of the write back length.
+ * @return
+ * The pointer to the start of the write back data linked list.
+ */
+static struct vhost_crypto_writeback_data *
+prepare_write_back_data(struct vhost_crypto_data_req *vc_req,
+ struct vring_desc **cur_desc,
+ struct vhost_crypto_writeback_data **end_wb_data,
+ uint8_t *src,
+ uint32_t offset,
+ uint64_t write_back_len)
{
- struct rte_mbuf *mbuf = op->sym->m_dst;
- struct vring_desc *head = vc_req->head;
- struct vring_desc *desc = vc_req->wb_desc;
- int left = vc_req->wb_len;
- uint32_t to_write;
- uint8_t *src_data = mbuf->buf_addr, *dst;
+ struct vhost_crypto_writeback_data *wb_data, *head;
+ struct vring_desc *desc = *cur_desc;
uint64_t dlen;
+ uint8_t *dst;
+ int ret;
- rte_prefetch0(&head[desc->next]);
- to_write = RTE_MIN(desc->len, (uint32_t)left);
- dlen = desc->len;
- dst = IOVA_TO_VVA(uint8_t *, vc_req, desc->addr, &dlen,
- VHOST_ACCESS_RW);
- if (unlikely(!dst || dlen != desc->len)) {
- VC_LOG_ERR("Failed to map descriptor");
- return -1;
+ ret = rte_mempool_get(vc_req->wb_pool, (void **)&head);
+ if (unlikely(ret < 0)) {
+ VC_LOG_ERR("no memory");
+ goto error_exit;
}
- rte_memcpy(dst, src_data, to_write);
- left -= to_write;
- src_data += to_write;
+ wb_data = head;
- while ((desc->flags & VRING_DESC_F_NEXT) && left > 0) {
- desc = &head[desc->next];
- rte_prefetch0(&head[desc->next]);
- to_write = RTE_MIN(desc->len, (uint32_t)left);
+ if (likely(desc->len > offset)) {
+ wb_data->src = src + offset;
dlen = desc->len;
- dst = IOVA_TO_VVA(uint8_t *, vc_req, desc->addr, &dlen,
- VHOST_ACCESS_RW);
+ dst = IOVA_TO_VVA(uint8_t *, vc_req, desc->addr,
+ &dlen, VHOST_ACCESS_RW) + offset;
if (unlikely(!dst || dlen != desc->len)) {
VC_LOG_ERR("Failed to map descriptor");
- return -1;
+ goto error_exit;
}
- rte_memcpy(dst, src_data, to_write);
- left -= to_write;
- src_data += to_write;
- }
+ wb_data->dst = dst;
+ wb_data->len = desc->len - offset;
+ write_back_len -= wb_data->len;
+ src += offset + wb_data->len;
+ offset = 0;
+
+ if (unlikely(write_back_len)) {
+ ret = rte_mempool_get(vc_req->wb_pool,
+ (void **)&(wb_data->next));
+ if (unlikely(ret < 0)) {
+ VC_LOG_ERR("no memory");
+ goto error_exit;
+ }
- if (unlikely(left < 0)) {
- VC_LOG_ERR("Incorrect virtio descriptor");
- return -1;
+ wb_data = wb_data->next;
+ } else
+ wb_data->next = NULL;
+ } else
+ offset -= desc->len;
+
+ while (write_back_len) {
+ desc = &vc_req->head[desc->next];
+ if (unlikely(!(desc->flags & VRING_DESC_F_WRITE))) {
+ VC_LOG_ERR("incorrect descriptor");
+ goto error_exit;
+ }
+
+ if (desc->len <= offset) {
+ offset -= desc->len;
+ continue;
+ }
+
+ dlen = desc->len;
+ dst = IOVA_TO_VVA(uint8_t *, vc_req, desc->addr, &dlen,
+ VHOST_ACCESS_RW) + offset;
+ if (unlikely(dst == NULL || dlen != desc->len)) {
+ VC_LOG_ERR("Failed to map descriptor");
+ goto error_exit;
+ }
+
+ wb_data->src = src;
+ wb_data->dst = dst;
+ wb_data->len = RTE_MIN(desc->len - offset, write_back_len);
+ write_back_len -= wb_data->len;
+ src += wb_data->len;
+ offset = 0;
+
+ if (write_back_len) {
+ ret = rte_mempool_get(vc_req->wb_pool,
+ (void **)&(wb_data->next));
+ if (unlikely(ret < 0)) {
+ VC_LOG_ERR("no memory");
+ goto error_exit;
+ }
+
+ wb_data = wb_data->next;
+ } else
+ wb_data->next = NULL;
}
- return 0;
+ *cur_desc = &vc_req->head[desc->next];
+
+ *end_wb_data = wb_data;
+
+ return head;
+
+error_exit:
+ if (head)
+ free_wb_data(head, vc_req->wb_pool);
+
+ return NULL;
}
static uint8_t
struct vring_desc *cur_desc)
{
struct vring_desc *desc = cur_desc;
+ struct vhost_crypto_writeback_data *ewb = NULL;
struct rte_mbuf *m_src = op->sym->m_src, *m_dst = op->sym->m_dst;
uint8_t *iv_data = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET);
uint8_t ret = 0;
case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE:
m_src->buf_iova = gpa_to_hpa(vcrypto->dev, desc->addr,
cipher->para.src_data_len);
- m_src->buf_addr = get_data_ptr(vc_req, &desc,
- cipher->para.src_data_len, VHOST_ACCESS_RO);
+ m_src->buf_addr = get_data_ptr(vc_req, desc, VHOST_ACCESS_RO);
if (unlikely(m_src->buf_iova == 0 ||
m_src->buf_addr == NULL)) {
VC_LOG_ERR("zero_copy may fail due to cross page data");
ret = VIRTIO_CRYPTO_ERR;
goto error_exit;
}
+
+ if (unlikely(move_desc(vc_req->head, &desc,
+ cipher->para.src_data_len) < 0)) {
+ VC_LOG_ERR("Incorrect descriptor");
+ ret = VIRTIO_CRYPTO_ERR;
+ goto error_exit;
+ }
+
break;
case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE:
+ vc_req->wb_pool = vcrypto->wb_pool;
+
if (unlikely(cipher->para.src_data_len >
RTE_MBUF_DEFAULT_BUF_SIZE)) {
VC_LOG_ERR("Not enough space to do data copy");
case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE:
m_dst->buf_iova = gpa_to_hpa(vcrypto->dev,
desc->addr, cipher->para.dst_data_len);
- m_dst->buf_addr = get_data_ptr(vc_req, &desc,
- cipher->para.dst_data_len, VHOST_ACCESS_RW);
+ m_dst->buf_addr = get_data_ptr(vc_req, desc, VHOST_ACCESS_RW);
if (unlikely(m_dst->buf_iova == 0 || m_dst->buf_addr == NULL)) {
VC_LOG_ERR("zero_copy may fail due to cross page data");
ret = VIRTIO_CRYPTO_ERR;
goto error_exit;
}
+ if (unlikely(move_desc(vc_req->head, &desc,
+ cipher->para.dst_data_len) < 0)) {
+ VC_LOG_ERR("Incorrect descriptor");
+ ret = VIRTIO_CRYPTO_ERR;
+ goto error_exit;
+ }
+
m_dst->data_len = cipher->para.dst_data_len;
break;
case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE:
- vc_req->wb_desc = desc;
- vc_req->wb_len = cipher->para.dst_data_len;
- if (unlikely(move_desc(vc_req->head, &desc,
- vc_req->wb_len) < 0)) {
+ vc_req->wb = prepare_write_back_data(vc_req, &desc, &ewb,
+ rte_pktmbuf_mtod(m_src, uint8_t *), 0,
+ cipher->para.dst_data_len);
+ if (unlikely(vc_req->wb == NULL)) {
ret = VIRTIO_CRYPTO_ERR;
goto error_exit;
}
+
break;
default:
ret = VIRTIO_CRYPTO_BADMSG;
op->sym->cipher.data.offset = 0;
op->sym->cipher.data.length = cipher->para.src_data_len;
- vc_req->inhdr = get_data_ptr(vc_req, &desc, INHDR_LEN, VHOST_ACCESS_WO);
+ vc_req->inhdr = get_data_ptr(vc_req, desc, VHOST_ACCESS_WO);
if (unlikely(vc_req->inhdr == NULL)) {
ret = VIRTIO_CRYPTO_BADMSG;
goto error_exit;
return 0;
error_exit:
+ if (vc_req->wb)
+ free_wb_data(vc_req->wb, vc_req->wb_pool);
+
vc_req->len = INHDR_LEN;
return ret;
}
struct virtio_crypto_alg_chain_data_req *chain,
struct vring_desc *cur_desc)
{
- struct vring_desc *desc = cur_desc;
+ struct vring_desc *desc = cur_desc, *digest_desc;
+ struct vhost_crypto_writeback_data *ewb = NULL, *ewb2 = NULL;
struct rte_mbuf *m_src = op->sym->m_src, *m_dst = op->sym->m_dst;
uint8_t *iv_data = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET);
uint32_t digest_offset;
}
m_src->data_len = chain->para.src_data_len;
- m_dst->data_len = chain->para.dst_data_len;
switch (vcrypto->option) {
case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE:
+ m_dst->data_len = chain->para.dst_data_len;
+
m_src->buf_iova = gpa_to_hpa(vcrypto->dev, desc->addr,
chain->para.src_data_len);
- m_src->buf_addr = get_data_ptr(vc_req, &desc,
- chain->para.src_data_len, VHOST_ACCESS_RO);
+ m_src->buf_addr = get_data_ptr(vc_req, desc, VHOST_ACCESS_RO);
if (unlikely(m_src->buf_iova == 0 || m_src->buf_addr == NULL)) {
VC_LOG_ERR("zero_copy may fail due to cross page data");
ret = VIRTIO_CRYPTO_ERR;
goto error_exit;
}
+
+ if (unlikely(move_desc(vc_req->head, &desc,
+ chain->para.src_data_len) < 0)) {
+ VC_LOG_ERR("Incorrect descriptor");
+ ret = VIRTIO_CRYPTO_ERR;
+ goto error_exit;
+ }
break;
case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE:
+ vc_req->wb_pool = vcrypto->wb_pool;
+
if (unlikely(chain->para.src_data_len >
RTE_MBUF_DEFAULT_BUF_SIZE)) {
VC_LOG_ERR("Not enough space to do data copy");
ret = VIRTIO_CRYPTO_BADMSG;
goto error_exit;
}
+
break;
default:
ret = VIRTIO_CRYPTO_BADMSG;
case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE:
m_dst->buf_iova = gpa_to_hpa(vcrypto->dev,
desc->addr, chain->para.dst_data_len);
- m_dst->buf_addr = get_data_ptr(vc_req, &desc,
- chain->para.dst_data_len, VHOST_ACCESS_RW);
+ m_dst->buf_addr = get_data_ptr(vc_req, desc, VHOST_ACCESS_RW);
if (unlikely(m_dst->buf_iova == 0 || m_dst->buf_addr == NULL)) {
VC_LOG_ERR("zero_copy may fail due to cross page data");
ret = VIRTIO_CRYPTO_ERR;
goto error_exit;
}
+ if (unlikely(move_desc(vc_req->head, &desc,
+ chain->para.dst_data_len) < 0)) {
+ VC_LOG_ERR("Incorrect descriptor");
+ ret = VIRTIO_CRYPTO_ERR;
+ goto error_exit;
+ }
+
op->sym->auth.digest.phys_addr = gpa_to_hpa(vcrypto->dev,
desc->addr, chain->para.hash_result_len);
- op->sym->auth.digest.data = get_data_ptr(vc_req, &desc,
- chain->para.hash_result_len, VHOST_ACCESS_RW);
+ op->sym->auth.digest.data = get_data_ptr(vc_req, desc,
+ VHOST_ACCESS_RW);
if (unlikely(op->sym->auth.digest.phys_addr == 0)) {
VC_LOG_ERR("zero_copy may fail due to cross page data");
ret = VIRTIO_CRYPTO_ERR;
goto error_exit;
}
+
+ if (unlikely(move_desc(vc_req->head, &desc,
+ chain->para.hash_result_len) < 0)) {
+ VC_LOG_ERR("Incorrect descriptor");
+ ret = VIRTIO_CRYPTO_ERR;
+ goto error_exit;
+ }
+
break;
case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE:
- digest_offset = m_dst->data_len;
- digest_addr = rte_pktmbuf_mtod_offset(m_dst, void *,
- digest_offset);
+ vc_req->wb = prepare_write_back_data(vc_req, &desc, &ewb,
+ rte_pktmbuf_mtod(m_src, uint8_t *),
+ chain->para.cipher_start_src_offset,
+ chain->para.dst_data_len -
+ chain->para.cipher_start_src_offset);
+ if (unlikely(vc_req->wb == NULL)) {
+ ret = VIRTIO_CRYPTO_ERR;
+ goto error_exit;
+ }
- vc_req->wb_desc = desc;
- vc_req->wb_len = m_dst->data_len + chain->para.hash_result_len;
+ digest_offset = m_src->data_len;
+ digest_addr = rte_pktmbuf_mtod_offset(m_src, void *,
+ digest_offset);
+ digest_desc = desc;
- if (unlikely(move_desc(vc_req->head, &desc,
- chain->para.dst_data_len) < 0)) {
- ret = VIRTIO_CRYPTO_BADMSG;
+ /** create a wb_data for digest */
+ ewb->next = prepare_write_back_data(vc_req, &desc, &ewb2,
+ digest_addr, 0, chain->para.hash_result_len);
+ if (unlikely(ewb->next == NULL)) {
+ ret = VIRTIO_CRYPTO_ERR;
goto error_exit;
}
- if (unlikely(copy_data(digest_addr, vc_req, &desc,
+ if (unlikely(copy_data(digest_addr, vc_req, &digest_desc,
chain->para.hash_result_len)) < 0) {
ret = VIRTIO_CRYPTO_BADMSG;
goto error_exit;
}
op->sym->auth.digest.data = digest_addr;
- op->sym->auth.digest.phys_addr = rte_pktmbuf_iova_offset(m_dst,
+ op->sym->auth.digest.phys_addr = rte_pktmbuf_iova_offset(m_src,
digest_offset);
break;
default:
}
/* record inhdr */
- vc_req->inhdr = get_data_ptr(vc_req, &desc, INHDR_LEN, VHOST_ACCESS_WO);
+ vc_req->inhdr = get_data_ptr(vc_req, desc, VHOST_ACCESS_WO);
if (unlikely(vc_req->inhdr == NULL)) {
ret = VIRTIO_CRYPTO_BADMSG;
goto error_exit;
return 0;
error_exit:
+ if (vc_req->wb)
+ free_wb_data(vc_req->wb, vc_req->wb_pool);
vc_req->len = INHDR_LEN;
return ret;
}
vc_req->head = head;
vc_req->zero_copy = vcrypto->option;
- req = get_data_ptr(vc_req, &desc, sizeof(*req), VHOST_ACCESS_RO);
+ req = get_data_ptr(vc_req, desc, VHOST_ACCESS_RO);
if (unlikely(req == NULL)) {
switch (vcrypto->option) {
case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE:
VC_LOG_ERR("Invalid option");
goto error_exit;
}
+ } else {
+ if (unlikely(move_desc(vc_req->head, &desc,
+ sizeof(*req)) < 0)) {
+ VC_LOG_ERR("Incorrect descriptor");
+ goto error_exit;
+ }
}
switch (req->header.opcode) {
struct rte_mbuf *m_dst = op->sym->m_dst;
struct vhost_crypto_data_req *vc_req = rte_mbuf_to_priv(m_src);
uint16_t desc_idx;
- int ret = 0;
if (unlikely(!vc_req)) {
VC_LOG_ERR("Failed to retrieve vc_req");
if (unlikely(op->status != RTE_CRYPTO_OP_STATUS_SUCCESS))
vc_req->inhdr->status = VIRTIO_CRYPTO_ERR;
else {
- if (vc_req->zero_copy == 0) {
- ret = write_back_data(op, vc_req);
- if (unlikely(ret != 0))
- vc_req->inhdr->status = VIRTIO_CRYPTO_ERR;
- }
+ if (vc_req->zero_copy == 0)
+ write_back_data(vc_req);
}
vc_req->vq->used->ring[desc_idx].id = desc_idx;
vc_req->vq->used->ring[desc_idx].len = vc_req->len;
- rte_mempool_put(m_dst->pool, (void *)m_dst);
rte_mempool_put(m_src->pool, (void *)m_src);
+ if (m_dst)
+ rte_mempool_put(m_dst->pool, (void *)m_dst);
+
return vc_req->vq;
}
goto error_exit;
}
+ snprintf(name, 127, "WB_POOL_VM_%u", (uint32_t)vid);
+ vcrypto->wb_pool = rte_mempool_create(name,
+ VHOST_CRYPTO_MBUF_POOL_SIZE,
+ sizeof(struct vhost_crypto_writeback_data),
+ 128, 0, NULL, NULL, NULL, NULL,
+ rte_socket_id(), 0);
+ if (!vcrypto->wb_pool) {
+ VC_LOG_ERR("Failed to creath mempool");
+ ret = -ENOMEM;
+ goto error_exit;
+ }
+
dev->extern_data = vcrypto;
dev->extern_ops.pre_msg_handle = NULL;
dev->extern_ops.post_msg_handle = vhost_crypto_msg_post_handler;
rte_hash_free(vcrypto->session_map);
rte_mempool_free(vcrypto->mbuf_pool);
+ rte_mempool_free(vcrypto->wb_pool);
rte_free(vcrypto);
dev->extern_data = NULL;
if (vcrypto->option == (uint8_t)option)
return 0;
- if (!(rte_mempool_full(vcrypto->mbuf_pool))) {
+ if (!(rte_mempool_full(vcrypto->mbuf_pool)) ||
+ !(rte_mempool_full(vcrypto->wb_pool))) {
VC_LOG_ERR("Cannot update zero copy as mempool is not full");
return -EINVAL;
}
+ if (option == RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE) {
+ char name[128];
+
+ snprintf(name, 127, "WB_POOL_VM_%u", (uint32_t)vid);
+ vcrypto->wb_pool = rte_mempool_create(name,
+ VHOST_CRYPTO_MBUF_POOL_SIZE,
+ sizeof(struct vhost_crypto_writeback_data),
+ 128, 0, NULL, NULL, NULL, NULL,
+ rte_socket_id(), 0);
+ if (!vcrypto->wb_pool) {
+ VC_LOG_ERR("Failed to creath mbuf pool");
+ return -ENOMEM;
+ }
+ } else {
+ rte_mempool_free(vcrypto->wb_pool);
+ vcrypto->wb_pool = NULL;
+ }
+
vcrypto->option = (uint8_t)option;
return 0;
struct vhost_virtqueue *vq;
uint16_t avail_idx;
uint16_t start_idx;
- uint16_t required;
uint16_t count;
- uint16_t i;
+ uint16_t i = 0;
if (unlikely(dev == NULL)) {
VC_LOG_ERR("Invalid vid %i", vid);
/* for zero copy, we need 2 empty mbufs for src and dst, otherwise
* we need only 1 mbuf as src and dst
*/
- required = count * 2;
- if (unlikely(rte_mempool_get_bulk(vcrypto->mbuf_pool, (void **)mbufs,
- required) < 0)) {
- VC_LOG_ERR("Insufficient memory");
- return -ENOMEM;
- }
+ switch (vcrypto->option) {
+ case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE:
+ if (unlikely(rte_mempool_get_bulk(vcrypto->mbuf_pool,
+ (void **)mbufs, count * 2) < 0)) {
+ VC_LOG_ERR("Insufficient memory");
+ return -ENOMEM;
+ }
- for (i = 0; i < count; i++) {
- uint16_t used_idx = (start_idx + i) & (vq->size - 1);
- uint16_t desc_idx = vq->avail->ring[used_idx];
- struct vring_desc *head = &vq->desc[desc_idx];
- struct rte_crypto_op *op = ops[i];
+ for (i = 0; i < count; i++) {
+ uint16_t used_idx = (start_idx + i) & (vq->size - 1);
+ uint16_t desc_idx = vq->avail->ring[used_idx];
+ struct vring_desc *head = &vq->desc[desc_idx];
+ struct rte_crypto_op *op = ops[i];
- op->sym->m_src = mbufs[i * 2];
- op->sym->m_dst = mbufs[i * 2 + 1];
- op->sym->m_src->data_off = 0;
- op->sym->m_dst->data_off = 0;
+ op->sym->m_src = mbufs[i * 2];
+ op->sym->m_dst = mbufs[i * 2 + 1];
+ op->sym->m_src->data_off = 0;
+ op->sym->m_dst->data_off = 0;
+
+ if (unlikely(vhost_crypto_process_one_req(vcrypto, vq,
+ op, head, desc_idx)) < 0)
+ break;
+ }
+
+ if (unlikely(i < count))
+ rte_mempool_put_bulk(vcrypto->mbuf_pool,
+ (void **)&mbufs[i * 2],
+ (count - i) * 2);
+
+ break;
+
+ case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE:
+ if (unlikely(rte_mempool_get_bulk(vcrypto->mbuf_pool,
+ (void **)mbufs, count) < 0)) {
+ VC_LOG_ERR("Insufficient memory");
+ return -ENOMEM;
+ }
+
+ for (i = 0; i < count; i++) {
+ uint16_t used_idx = (start_idx + i) & (vq->size - 1);
+ uint16_t desc_idx = vq->avail->ring[used_idx];
+ struct vring_desc *head = &vq->desc[desc_idx];
+ struct rte_crypto_op *op = ops[i];
+
+ op->sym->m_src = mbufs[i];
+ op->sym->m_dst = NULL;
+ op->sym->m_src->data_off = 0;
+
+ if (unlikely(vhost_crypto_process_one_req(vcrypto, vq,
+ op, head, desc_idx)) < 0)
+ break;
+ }
+
+ if (unlikely(i < count))
+ rte_mempool_put_bulk(vcrypto->mbuf_pool,
+ (void **)&mbufs[i],
+ count - i);
+
+ break;
- if (unlikely(vhost_crypto_process_one_req(vcrypto, vq, op, head,
- desc_idx)) < 0)
- break;
}
vq->last_used_idx += i;
if (ret <= 0)
return ret;
- if (msg && msg->size) {
+ if (msg->size) {
if (msg->size > sizeof(msg->payload)) {
RTE_LOG(ERR, VHOST_CONFIG,
"invalid msg size: %d\n", msg->size);
avail_idx, &desc_count,
buf_vec, &vec_idx,
&buf_id, &len,
- VHOST_ACCESS_RO) < 0))
+ VHOST_ACCESS_RW) < 0))
return -1;
len = RTE_MIN(len, size);
vq->last_avail_idx, &desc_count,
buf_vec, &nr_vec,
&buf_id, &dummy_len,
- VHOST_ACCESS_RW) < 0))
+ VHOST_ACCESS_RO) < 0))
break;
if (likely(dev->dequeue_zero_copy == 0))
# Copyright(c) 2017 Intel Corporation
project('DPDK', 'C',
- version: '18.11.0-rc2',
+ version: '18.11.0-rc3',
license: 'BSD',
default_options: ['buildtype=release', 'default_library=static'],
meson_version: '>= 0.41'
_LDLIBS-$(CONFIG_RTE_LIBRTE_ACL) += --whole-archive
_LDLIBS-$(CONFIG_RTE_LIBRTE_ACL) += -lrte_acl
_LDLIBS-$(CONFIG_RTE_LIBRTE_ACL) += --no-whole-archive
+_LDLIBS-$(CONFIG_RTE_LIBRTE_TELEMETRY) += --no-as-needed
_LDLIBS-$(CONFIG_RTE_LIBRTE_TELEMETRY) += --whole-archive
_LDLIBS-$(CONFIG_RTE_LIBRTE_TELEMETRY) += -lrte_telemetry -ljansson
_LDLIBS-$(CONFIG_RTE_LIBRTE_TELEMETRY) += --no-whole-archive
+_LDLIBS-$(CONFIG_RTE_LIBRTE_TELEMETRY) += --as-needed
_LDLIBS-$(CONFIG_RTE_LIBRTE_JOBSTATS) += -lrte_jobstats
_LDLIBS-$(CONFIG_RTE_LIBRTE_METRICS) += -lrte_metrics
_LDLIBS-$(CONFIG_RTE_LIBRTE_BITRATE) += -lrte_bitratestats
OUTPUT_RELPATH=$(shell $(RTE_SDK)/buildtools/relpath.sh $(abspath $(RTE_OUTPUT)) \
$(abspath $(RTE_SRCDIR)))
$(RTE_OUTPUT)/Makefile: | $(RTE_OUTPUT)
- $(Q)$(RTE_SDK)/buildtools/gen-build-mk.sh $(SDK_RELPATH) $(OUTPUT_RELPATH) \
- > $(RTE_OUTPUT)/Makefile
+ $(Q)$(RTE_SDK)/buildtools/gen-build-mk.sh $(SDK_RELPATH) > $@
# clean installed files, and generate a new config header file
# if NODOTCONF variable is defined, don't try to rebuild .config
* (011) ret #1
* (012) ret #0
*
- * To compile:
- * clang -O2 -target bpf -c t1.c
+ * To compile on x86:
+ * clang -O2 -U __GNUC__ -target bpf -c t1.c
+ *
+ * To compile on ARM:
+ * clang -O2 -I/usr/include/aarch64-linux-gnu/ -target bpf -c t1.c
*/
#include <stdint.h>
#include <net/ethernet.h>
#include <netinet/ip.h>
#include <netinet/udp.h>
+#include <arpa/inet.h>
uint64_t
entry(void *pkt)
{
struct ether_header *ether_header = (void *)pkt;
- if (ether_header->ether_type != __builtin_bswap16(0x0800))
+ if (ether_header->ether_type != htons(0x0800))
return 0;
struct iphdr *iphdr = (void *)(ether_header + 1);
if (iphdr->protocol != 17 || (iphdr->frag_off & 0x1ffff) != 0 ||
- iphdr->daddr != __builtin_bswap32(0x1020304))
+ iphdr->daddr != htonl(0x1020304))
return 0;
int hlen = iphdr->ihl * 4;
struct udphdr *udphdr = (void *)iphdr + hlen;
- if (udphdr->dest != __builtin_bswap16(5000))
+ if (udphdr->dest != htons(5000))
return 0;
return 1;
* eBPF program sample.
* Accepts pointer to struct rte_mbuf as an input parameter.
* Dump the mbuf into stdout if it is an ARP packet (aka tcpdump 'arp').
- * To compile:
- * clang -O2 -I${RTE_SDK}/${RTE_TARGET}/include \
+ *
+ * To compile on x86:
+ * clang -O2 -U __GNUC__ -I${RTE_SDK}/${RTE_TARGET}/include \
* -target bpf -Wno-int-to-void-pointer-cast -c t3.c
+ *
+ * To compile on ARM:
+ * clang -O2 -I/usr/include/aarch64-linux-gnu \
+ * -I${RTE_SDK}/${RTE_TARGET}/include -target bpf \
+ * -Wno-int-to-void-pointer-cast -c t3.c
*/
#include <stdint.h>
#include <net/ethernet.h>
#include <rte_config.h>
#include "mbuf.h"
+#include <arpa/inet.h>
extern void rte_pktmbuf_dump(FILE *, const struct rte_mbuf *, unsigned int);
mb = pkt;
eth = rte_pktmbuf_mtod(mb, const struct ether_header *);
- if (eth->ether_type == __builtin_bswap16(ETHERTYPE_ARP))
+ if (eth->ether_type == htons(ETHERTYPE_ARP))
rte_pktmbuf_dump(stdout, mb, 64);
return 1;
/* merge argc/argv and the environment args */
all_argc = argc + eargc;
all_argv = malloc(sizeof(*all_argv) * (all_argc + 1));
- if (all_argv == NULL)
- return -1;
+ if (all_argv == NULL) {
+ ret = -1;
+ goto out;
+ }
for (i = 0; i < argc; i++)
all_argv[i] = argv[i];
#define TEST_JCC_3 5678
#define TEST_JCC_4 TEST_FILL_1
+#define TEST_IMM_1 UINT64_MAX
+#define TEST_IMM_2 ((uint64_t)INT64_MIN)
+#define TEST_IMM_3 ((uint64_t)INT64_MAX + INT32_MAX)
+#define TEST_IMM_4 ((uint64_t)UINT32_MAX)
+#define TEST_IMM_5 ((uint64_t)UINT32_MAX + 1)
+
struct bpf_test {
const char *name;
size_t arg_sz;
return cmp_res(__func__, v, rc, dft, dft, sizeof(*dft));
}
+/* load immediate test-cases */
+static const struct ebpf_insn test_ldimm1_prog[] = {
+
+ {
+ .code = (BPF_LD | BPF_IMM | EBPF_DW),
+ .dst_reg = EBPF_REG_0,
+ .imm = (uint32_t)TEST_IMM_1,
+ },
+ {
+ .imm = TEST_IMM_1 >> 32,
+ },
+ {
+ .code = (BPF_LD | BPF_IMM | EBPF_DW),
+ .dst_reg = EBPF_REG_3,
+ .imm = (uint32_t)TEST_IMM_2,
+ },
+ {
+ .imm = TEST_IMM_2 >> 32,
+ },
+ {
+ .code = (BPF_LD | BPF_IMM | EBPF_DW),
+ .dst_reg = EBPF_REG_5,
+ .imm = (uint32_t)TEST_IMM_3,
+ },
+ {
+ .imm = TEST_IMM_3 >> 32,
+ },
+ {
+ .code = (BPF_LD | BPF_IMM | EBPF_DW),
+ .dst_reg = EBPF_REG_7,
+ .imm = (uint32_t)TEST_IMM_4,
+ },
+ {
+ .imm = TEST_IMM_4 >> 32,
+ },
+ {
+ .code = (BPF_LD | BPF_IMM | EBPF_DW),
+ .dst_reg = EBPF_REG_9,
+ .imm = (uint32_t)TEST_IMM_5,
+ },
+ {
+ .imm = TEST_IMM_5 >> 32,
+ },
+ /* return sum */
+ {
+ .code = (EBPF_ALU64 | BPF_ADD | BPF_X),
+ .dst_reg = EBPF_REG_0,
+ .src_reg = EBPF_REG_3,
+ },
+ {
+ .code = (EBPF_ALU64 | BPF_ADD | BPF_X),
+ .dst_reg = EBPF_REG_0,
+ .src_reg = EBPF_REG_5,
+ },
+ {
+ .code = (EBPF_ALU64 | BPF_ADD | BPF_X),
+ .dst_reg = EBPF_REG_0,
+ .src_reg = EBPF_REG_7,
+ },
+ {
+ .code = (EBPF_ALU64 | BPF_ADD | BPF_X),
+ .dst_reg = EBPF_REG_0,
+ .src_reg = EBPF_REG_9,
+ },
+ {
+ .code = (BPF_JMP | EBPF_EXIT),
+ },
+};
+
+static int
+test_ldimm1_check(uint64_t rc, const void *arg)
+{
+ uint64_t v1, v2;
+
+ v1 = TEST_IMM_1;
+ v2 = TEST_IMM_2;
+ v1 += v2;
+ v2 = TEST_IMM_3;
+ v1 += v2;
+ v2 = TEST_IMM_4;
+ v1 += v2;
+ v2 = TEST_IMM_5;
+ v1 += v2;
+
+ return cmp_res(__func__, v1, rc, arg, arg, 0);
+}
+
+
/* alu mul test-cases */
static const struct ebpf_insn test_mul1_prog[] = {
.prepare = test_load1_prepare,
.check_result = test_load1_check,
},
+ {
+ .name = "test_ldimm1",
+ .arg_sz = sizeof(struct dummy_offset),
+ .prm = {
+ .ins = test_ldimm1_prog,
+ .nb_ins = RTE_DIM(test_ldimm1_prog),
+ .prog_arg = {
+ .type = RTE_BPF_ARG_PTR,
+ .size = sizeof(struct dummy_offset),
+ },
+ },
+ .prepare = test_store1_prepare,
+ .check_result = test_ldimm1_check,
+ },
{
.name = "test_mul1",
.arg_sz = sizeof(struct dummy_vect8),
return 0;
}
+static int
+test_fls(void)
+{
+ struct fls_test_vector {
+ uint32_t arg;
+ int rc;
+ };
+ int expected, rc;
+ uint32_t i, arg;
+
+ const struct fls_test_vector test[] = {
+ {0x0, 0},
+ {0x1, 1},
+ {0x4000, 15},
+ {0x80000000, 32},
+ };
+
+ for (i = 0; i < RTE_DIM(test); i++) {
+ arg = test[i].arg;
+ rc = rte_fls_u32(arg);
+ expected = test[i].rc;
+ if (rc != expected) {
+ printf("Wrong rte_fls_u32(0x%x) rc=%d, expected=%d\n",
+ arg, rc, expected);
+ return TEST_FAILED;
+ }
+ }
+
+ return 0;
+}
+
static int
test_common(void)
{
ret |= test_macros(0);
ret |= test_misc();
ret |= test_log2();
+ ret |= test_fls();
return ret;
}
reader_faster) < 0)
return -1;
+ printf("================\n");
printf("Results summary:\n");
+ printf("================\n");
printf("single read: %u\n", htm_results.single_read);
printf("single write: %u\n", htm_results.single_write);
for (i = 0; i < NUM_TEST; i++) {
- printf("core_cnt: %u\n", core_cnt[i]);
+ printf("+++ core_cnt: %u +++\n", core_cnt[i]);
printf("HTM:\n");
- printf("read only: %u\n", htm_results.read_only[i]);
- printf("write only: %u\n", htm_results.write_only[i]);
- printf("read-write read: %u\n", htm_results.read_write_r[i]);
- printf("read-write write: %u\n", htm_results.read_write_w[i]);
+ printf(" read only: %u\n", htm_results.read_only[i]);
+ printf(" write only: %u\n", htm_results.write_only[i]);
+ printf(" read-write read: %u\n", htm_results.read_write_r[i]);
+ printf(" read-write write: %u\n", htm_results.read_write_w[i]);
printf("non HTM:\n");
- printf("read only: %u\n", non_htm_results.read_only[i]);
- printf("write only: %u\n", non_htm_results.write_only[i]);
- printf("read-write read: %u\n",
+ printf(" read only: %u\n", non_htm_results.read_only[i]);
+ printf(" write only: %u\n", non_htm_results.write_only[i]);
+ printf(" read-write read: %u\n",
non_htm_results.read_write_r[i]);
- printf("read-write write: %u\n",
+ printf(" read-write write: %u\n",
non_htm_results.read_write_w[i]);
}
if (!dir) {
if (errno == ENOENT) {
printf("Cannot run UT due to missing rte_kni module\n");
- return -1;
+ return TEST_SKIPPED;
}
printf("opendir: %s", strerror(errno));
return -1;
"correctly(APCI cpufreq) or operating in another valid "
"Power management environment\n", TEST_POWER_LCORE_ID);
rte_power_unset_env();
- return -1;
+ return TEST_SKIPPED;
}
/**
goto exit;
}
if (robufs[0] != NULL)
- rte_pktmbuf_free(robufs[i]);
+ rte_pktmbuf_free(robufs[0]);
/* Insert more packets
* RB[] = {NULL, NULL, NULL, NULL}
Code Review / deb_dpdk.git / commitdiff