drivers/net/i40e/i40e_ethdev_vf.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <string.h>
  39 #include <unistd.h>
  40 #include <stdarg.h>
  41 #include <inttypes.h>
  42 #include <rte_byteorder.h>
  43 #include <rte_common.h>
  44 #include <rte_cycles.h>
  45
  46 #include <rte_interrupts.h>
  47 #include <rte_log.h>
  48 #include <rte_debug.h>
  49 #include <rte_pci.h>
  50 #include <rte_atomic.h>
  51 #include <rte_branch_prediction.h>
  52 #include <rte_memory.h>
  53 #include <rte_memzone.h>
  54 #include <rte_eal.h>
  55 #include <rte_alarm.h>
  56 #include <rte_ether.h>
  57 #include <rte_ethdev.h>
  58 #include <rte_atomic.h>
  59 #include <rte_malloc.h>
  60 #include <rte_dev.h>
  61
  62 #include "i40e_logs.h"
  63 #include "base/i40e_prototype.h"
  64 #include "base/i40e_adminq_cmd.h"
  65 #include "base/i40e_type.h"
  66
  67 #include "i40e_rxtx.h"
  68 #include "i40e_ethdev.h"
  69 #include "i40e_pf.h"
  70 #define I40EVF_VSI_DEFAULT_MSIX_INTR     1
  71 #define I40EVF_VSI_DEFAULT_MSIX_INTR_LNX 0
  72
  73 /* busy wait delay in msec */
  74 #define I40EVF_BUSY_WAIT_DELAY 10
  75 #define I40EVF_BUSY_WAIT_COUNT 50
  76 #define MAX_RESET_WAIT_CNT     20
  77
  78 struct i40evf_arq_msg_info {
  79         enum i40e_virtchnl_ops ops;
  80         enum i40e_status_code result;
  81         uint16_t buf_len;
  82         uint16_t msg_len;
  83         uint8_t *msg;
  84 };
  85
  86 struct vf_cmd_info {
  87         enum i40e_virtchnl_ops ops;
  88         uint8_t *in_args;
  89         uint32_t in_args_size;
  90         uint8_t *out_buffer;
  91         /* Input & output type. pass in buffer size and pass out
  92          * actual return result
  93          */
  94         uint32_t out_size;
  95 };
  96
  97 enum i40evf_aq_result {
  98         I40EVF_MSG_ERR = -1, /* Meet error when accessing admin queue */
  99         I40EVF_MSG_NON,      /* Read nothing from admin queue */
 100         I40EVF_MSG_SYS,      /* Read system msg from admin queue */
 101         I40EVF_MSG_CMD,      /* Read async command result */
 102 };
 103
 104 static int i40evf_dev_configure(struct rte_eth_dev *dev);
 105 static int i40evf_dev_start(struct rte_eth_dev *dev);
 106 static void i40evf_dev_stop(struct rte_eth_dev *dev);
 107 static void i40evf_dev_info_get(struct rte_eth_dev *dev,
 108                                 struct rte_eth_dev_info *dev_info);
 109 static int i40evf_dev_link_update(struct rte_eth_dev *dev,
 110                                   __rte_unused int wait_to_complete);
 111 static void i40evf_dev_stats_get(struct rte_eth_dev *dev,
 112                                 struct rte_eth_stats *stats);
 113 static int i40evf_dev_xstats_get(struct rte_eth_dev *dev,
 114                                  struct rte_eth_xstats *xstats, unsigned n);
 115 static void i40evf_dev_xstats_reset(struct rte_eth_dev *dev);
 116 static int i40evf_vlan_filter_set(struct rte_eth_dev *dev,
 117                                   uint16_t vlan_id, int on);
 118 static void i40evf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 119 static int i40evf_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid,
 120                                 int on);
 121 static void i40evf_dev_close(struct rte_eth_dev *dev);
 122 static void i40evf_dev_promiscuous_enable(struct rte_eth_dev *dev);
 123 static void i40evf_dev_promiscuous_disable(struct rte_eth_dev *dev);
 124 static void i40evf_dev_allmulticast_enable(struct rte_eth_dev *dev);
 125 static void i40evf_dev_allmulticast_disable(struct rte_eth_dev *dev);
 126 static int i40evf_get_link_status(struct rte_eth_dev *dev,
 127                                   struct rte_eth_link *link);
 128 static int i40evf_init_vlan(struct rte_eth_dev *dev);
 129 static int i40evf_dev_rx_queue_start(struct rte_eth_dev *dev,
 130                                      uint16_t rx_queue_id);
 131 static int i40evf_dev_rx_queue_stop(struct rte_eth_dev *dev,
 132                                     uint16_t rx_queue_id);
 133 static int i40evf_dev_tx_queue_start(struct rte_eth_dev *dev,
 134                                      uint16_t tx_queue_id);
 135 static int i40evf_dev_tx_queue_stop(struct rte_eth_dev *dev,
 136                                     uint16_t tx_queue_id);
 137 static void i40evf_add_mac_addr(struct rte_eth_dev *dev,
 138                                 struct ether_addr *addr,
 139                                 uint32_t index,
 140                                 uint32_t pool);
 141 static void i40evf_del_mac_addr(struct rte_eth_dev *dev, uint32_t index);
 142 static int i40evf_dev_rss_reta_update(struct rte_eth_dev *dev,
 143                         struct rte_eth_rss_reta_entry64 *reta_conf,
 144                         uint16_t reta_size);
 145 static int i40evf_dev_rss_reta_query(struct rte_eth_dev *dev,
 146                         struct rte_eth_rss_reta_entry64 *reta_conf,
 147                         uint16_t reta_size);
 148 static int i40evf_config_rss(struct i40e_vf *vf);
 149 static int i40evf_dev_rss_hash_update(struct rte_eth_dev *dev,
 150                                       struct rte_eth_rss_conf *rss_conf);
 151 static int i40evf_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
 152                                         struct rte_eth_rss_conf *rss_conf);
 153 static int
 154 i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id);
 155 static int
 156 i40evf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id);
 157 static void i40evf_handle_pf_event(__rte_unused struct rte_eth_dev *dev,
 158                                    uint8_t *msg,
 159                                    uint16_t msglen);
 160
 161 /* Default hash key buffer for RSS */
 162 static uint32_t rss_key_default[I40E_VFQF_HKEY_MAX_INDEX + 1];
 163
 164 struct rte_i40evf_xstats_name_off {
 165         char name[RTE_ETH_XSTATS_NAME_SIZE];
 166         unsigned offset;
 167 };
 168
 169 static const struct rte_i40evf_xstats_name_off rte_i40evf_stats_strings[] = {
 170         {"rx_bytes", offsetof(struct i40e_eth_stats, rx_bytes)},
 171         {"rx_unicast_packets", offsetof(struct i40e_eth_stats, rx_unicast)},
 172         {"rx_multicast_packets", offsetof(struct i40e_eth_stats, rx_multicast)},
 173         {"rx_broadcast_packets", offsetof(struct i40e_eth_stats, rx_broadcast)},
 174         {"rx_dropped_packets", offsetof(struct i40e_eth_stats, rx_discards)},
 175         {"rx_unknown_protocol_packets", offsetof(struct i40e_eth_stats,
 176                 rx_unknown_protocol)},
 177         {"tx_bytes", offsetof(struct i40e_eth_stats, tx_bytes)},
 178         {"tx_unicast_packets", offsetof(struct i40e_eth_stats, tx_bytes)},
 179         {"tx_multicast_packets", offsetof(struct i40e_eth_stats, tx_bytes)},
 180         {"tx_broadcast_packets", offsetof(struct i40e_eth_stats, tx_bytes)},
 181         {"tx_dropped_packets", offsetof(struct i40e_eth_stats, tx_bytes)},
 182         {"tx_error_packets", offsetof(struct i40e_eth_stats, tx_bytes)},
 183 };
 184
 185 #define I40EVF_NB_XSTATS (sizeof(rte_i40evf_stats_strings) / \
 186                 sizeof(rte_i40evf_stats_strings[0]))
 187
 188 static const struct eth_dev_ops i40evf_eth_dev_ops = {
 189         .dev_configure        = i40evf_dev_configure,
 190         .dev_start            = i40evf_dev_start,
 191         .dev_stop             = i40evf_dev_stop,
 192         .promiscuous_enable   = i40evf_dev_promiscuous_enable,
 193         .promiscuous_disable  = i40evf_dev_promiscuous_disable,
 194         .allmulticast_enable  = i40evf_dev_allmulticast_enable,
 195         .allmulticast_disable = i40evf_dev_allmulticast_disable,
 196         .link_update          = i40evf_dev_link_update,
 197         .stats_get            = i40evf_dev_stats_get,
 198         .xstats_get           = i40evf_dev_xstats_get,
 199         .xstats_reset         = i40evf_dev_xstats_reset,
 200         .dev_close            = i40evf_dev_close,
 201         .dev_infos_get        = i40evf_dev_info_get,
 202         .dev_supported_ptypes_get = i40e_dev_supported_ptypes_get,
 203         .vlan_filter_set      = i40evf_vlan_filter_set,
 204         .vlan_offload_set     = i40evf_vlan_offload_set,
 205         .vlan_pvid_set        = i40evf_vlan_pvid_set,
 206         .rx_queue_start       = i40evf_dev_rx_queue_start,
 207         .rx_queue_stop        = i40evf_dev_rx_queue_stop,
 208         .tx_queue_start       = i40evf_dev_tx_queue_start,
 209         .tx_queue_stop        = i40evf_dev_tx_queue_stop,
 210         .rx_queue_setup       = i40e_dev_rx_queue_setup,
 211         .rx_queue_release     = i40e_dev_rx_queue_release,
 212         .rx_queue_intr_enable = i40evf_dev_rx_queue_intr_enable,
 213         .rx_queue_intr_disable = i40evf_dev_rx_queue_intr_disable,
 214         .rx_descriptor_done   = i40e_dev_rx_descriptor_done,
 215         .tx_queue_setup       = i40e_dev_tx_queue_setup,
 216         .tx_queue_release     = i40e_dev_tx_queue_release,
 217         .mac_addr_add         = i40evf_add_mac_addr,
 218         .mac_addr_remove      = i40evf_del_mac_addr,
 219         .reta_update          = i40evf_dev_rss_reta_update,
 220         .reta_query           = i40evf_dev_rss_reta_query,
 221         .rss_hash_update      = i40evf_dev_rss_hash_update,
 222         .rss_hash_conf_get    = i40evf_dev_rss_hash_conf_get,
 223 };
 224
 225 /*
 226  * Read data in admin queue to get msg from pf driver
 227  */
 228 static enum i40evf_aq_result
 229 i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info *data)
 230 {
 231         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 232         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 233         struct i40e_arq_event_info event;
 234         enum i40e_virtchnl_ops opcode;
 235         enum i40e_status_code retval;
 236         int ret;
 237         enum i40evf_aq_result result = I40EVF_MSG_NON;
 238
 239         event.buf_len = data->buf_len;
 240         event.msg_buf = data->msg;
 241         ret = i40e_clean_arq_element(hw, &event, NULL);
 242         /* Can't read any msg from adminQ */
 243         if (ret) {
 244                 if (ret != I40E_ERR_ADMIN_QUEUE_NO_WORK)
 245                         result = I40EVF_MSG_ERR;
 246                 return result;
 247         }
 248
 249         opcode = (enum i40e_virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high);
 250         retval = (enum i40e_status_code)rte_le_to_cpu_32(event.desc.cookie_low);
 251         /* pf sys event */
 252         if (opcode == I40E_VIRTCHNL_OP_EVENT) {
 253                 struct i40e_virtchnl_pf_event *vpe =
 254                         (struct i40e_virtchnl_pf_event *)event.msg_buf;
 255
 256                 result = I40EVF_MSG_SYS;
 257                 switch (vpe->event) {
 258                 case I40E_VIRTCHNL_EVENT_LINK_CHANGE:
 259                         vf->link_up =
 260                                 vpe->event_data.link_event.link_status;
 261                         vf->link_speed =
 262                                 vpe->event_data.link_event.link_speed;
 263                         vf->pend_msg |= PFMSG_LINK_CHANGE;
 264                         PMD_DRV_LOG(INFO, "Link status update:%s",
 265                                     vf->link_up ? "up" : "down");
 266                         break;
 267                 case I40E_VIRTCHNL_EVENT_RESET_IMPENDING:
 268                         vf->vf_reset = true;
 269                         vf->pend_msg |= PFMSG_RESET_IMPENDING;
 270                         PMD_DRV_LOG(INFO, "vf is reseting");
 271                         break;
 272                 case I40E_VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
 273                         vf->dev_closed = true;
 274                         vf->pend_msg |= PFMSG_DRIVER_CLOSE;
 275                         PMD_DRV_LOG(INFO, "PF driver closed");
 276                         break;
 277                 default:
 278                         PMD_DRV_LOG(ERR, "%s: Unknown event %d from pf",
 279                                     __func__, vpe->event);
 280                 }
 281         } else {
 282                 /* async reply msg on command issued by vf previously */
 283                 result = I40EVF_MSG_CMD;
 284                 /* Actual data length read from PF */
 285                 data->msg_len = event.msg_len;
 286         }
 287
 288         data->result = retval;
 289         data->ops = opcode;
 290
 291         return result;
 292 }
 293
 294 /**
 295  * clear current command. Only call in case execute
 296  * _atomic_set_cmd successfully.
 297  */
 298 static inline void
 299 _clear_cmd(struct i40e_vf *vf)
 300 {
 301         rte_wmb();
 302         vf->pend_cmd = I40E_VIRTCHNL_OP_UNKNOWN;
 303 }
 304
 305 /*
 306  * Check there is pending cmd in execution. If none, set new command.
 307  */
 308 static inline int
 309 _atomic_set_cmd(struct i40e_vf *vf, enum i40e_virtchnl_ops ops)
 310 {
 311         int ret = rte_atomic32_cmpset(&vf->pend_cmd,
 312                         I40E_VIRTCHNL_OP_UNKNOWN, ops);
 313
 314         if (!ret)
 315                 PMD_DRV_LOG(ERR, "There is incomplete cmd %d", vf->pend_cmd);
 316
 317         return !ret;
 318 }
 319
 320 #define MAX_TRY_TIMES 200
 321 #define ASQ_DELAY_MS  10
 322
 323 static int
 324 i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
 325 {
 326         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 327         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 328         struct i40evf_arq_msg_info info;
 329         enum i40evf_aq_result ret;
 330         int err = -1;
 331         int i = 0;
 332
 333         if (_atomic_set_cmd(vf, args->ops))
 334                 return -1;
 335
 336         info.msg = args->out_buffer;
 337         info.buf_len = args->out_size;
 338         info.ops = I40E_VIRTCHNL_OP_UNKNOWN;
 339         info.result = I40E_SUCCESS;
 340
 341         err = i40e_aq_send_msg_to_pf(hw, args->ops, I40E_SUCCESS,
 342                      args->in_args, args->in_args_size, NULL);
 343         if (err) {
 344                 PMD_DRV_LOG(ERR, "fail to send cmd %d", args->ops);
 345                 _clear_cmd(vf);
 346                 return err;
 347         }
 348
 349         switch (args->ops) {
 350         case I40E_VIRTCHNL_OP_RESET_VF:
 351                 /*no need to process in this function */
 352                 break;
 353         case I40E_VIRTCHNL_OP_VERSION:
 354         case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
 355                 /* for init adminq commands, need to poll the response */
 356                 do {
 357                         ret = i40evf_read_pfmsg(dev, &info);
 358                         if (ret == I40EVF_MSG_CMD) {
 359                                 err = 0;
 360                                 break;
 361                         } else if (ret == I40EVF_MSG_ERR) {
 362                                 err = -1;
 363                                 break;
 364                         }
 365                         rte_delay_ms(ASQ_DELAY_MS);
 366                         /* If don't read msg or read sys event, continue */
 367                 } while (i++ < MAX_TRY_TIMES);
 368                 _clear_cmd(vf);
 369                 break;
 370
 371         default:
 372                 /* for other adminq in running time, waiting the cmd done flag */
 373                 do {
 374                         if (vf->pend_cmd == I40E_VIRTCHNL_OP_UNKNOWN) {
 375                                 err = 0;
 376                                 break;
 377                         }
 378                         rte_delay_ms(ASQ_DELAY_MS);
 379                         /* If don't read msg or read sys event, continue */
 380                 } while (i++ < MAX_TRY_TIMES);
 381                 break;
 382         }
 383
 384         return err | vf->cmd_retval;
 385 }
 386
 387 /*
 388  * Check API version with sync wait until version read or fail from admin queue
 389  */
 390 static int
 391 i40evf_check_api_version(struct rte_eth_dev *dev)
 392 {
 393         struct i40e_virtchnl_version_info version, *pver;
 394         int err;
 395         struct vf_cmd_info args;
 396         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 397
 398         version.major = I40E_VIRTCHNL_VERSION_MAJOR;
 399         version.minor = I40E_VIRTCHNL_VERSION_MINOR;
 400
 401         args.ops = I40E_VIRTCHNL_OP_VERSION;
 402         args.in_args = (uint8_t *)&version;
 403         args.in_args_size = sizeof(version);
 404         args.out_buffer = vf->aq_resp;
 405         args.out_size = I40E_AQ_BUF_SZ;
 406
 407         err = i40evf_execute_vf_cmd(dev, &args);
 408         if (err) {
 409                 PMD_INIT_LOG(ERR, "fail to execute command OP_VERSION");
 410                 return err;
 411         }
 412
 413         pver = (struct i40e_virtchnl_version_info *)args.out_buffer;
 414         vf->version_major = pver->major;
 415         vf->version_minor = pver->minor;
 416         if (vf->version_major == I40E_DPDK_VERSION_MAJOR)
 417                 PMD_DRV_LOG(INFO, "Peer is DPDK PF host");
 418         else if ((vf->version_major == I40E_VIRTCHNL_VERSION_MAJOR) &&
 419                 (vf->version_minor <= I40E_VIRTCHNL_VERSION_MINOR))
 420                 PMD_DRV_LOG(INFO, "Peer is Linux PF host");
 421         else {
 422                 PMD_INIT_LOG(ERR, "PF/VF API version mismatch:(%u.%u)-(%u.%u)",
 423                                         vf->version_major, vf->version_minor,
 424                                                 I40E_VIRTCHNL_VERSION_MAJOR,
 425                                                 I40E_VIRTCHNL_VERSION_MINOR);
 426                 return -1;
 427         }
 428
 429         return 0;
 430 }
 431
 432 static int
 433 i40evf_get_vf_resource(struct rte_eth_dev *dev)
 434 {
 435         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 436         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 437         int err;
 438         struct vf_cmd_info args;
 439         uint32_t caps, len;
 440
 441         args.ops = I40E_VIRTCHNL_OP_GET_VF_RESOURCES;
 442         args.out_buffer = vf->aq_resp;
 443         args.out_size = I40E_AQ_BUF_SZ;
 444         if (PF_IS_V11(vf)) {
 445                 caps = I40E_VIRTCHNL_VF_OFFLOAD_L2 |
 446                        I40E_VIRTCHNL_VF_OFFLOAD_RSS_AQ |
 447                        I40E_VIRTCHNL_VF_OFFLOAD_RSS_REG |
 448                        I40E_VIRTCHNL_VF_OFFLOAD_VLAN |
 449                        I40E_VIRTCHNL_VF_OFFLOAD_RX_POLLING;
 450                 args.in_args = (uint8_t *)&caps;
 451                 args.in_args_size = sizeof(caps);
 452         } else {
 453                 args.in_args = NULL;
 454                 args.in_args_size = 0;
 455         }
 456         err = i40evf_execute_vf_cmd(dev, &args);
 457
 458         if (err) {
 459                 PMD_DRV_LOG(ERR, "fail to execute command OP_GET_VF_RESOURCE");
 460                 return err;
 461         }
 462
 463         len =  sizeof(struct i40e_virtchnl_vf_resource) +
 464                 I40E_MAX_VF_VSI * sizeof(struct i40e_virtchnl_vsi_resource);
 465
 466         (void)rte_memcpy(vf->vf_res, args.out_buffer,
 467                         RTE_MIN(args.out_size, len));
 468         i40e_vf_parse_hw_config(hw, vf->vf_res);
 469
 470         return 0;
 471 }
 472
 473 static int
 474 i40evf_config_promisc(struct rte_eth_dev *dev,
 475                       bool enable_unicast,
 476                       bool enable_multicast)
 477 {
 478         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 479         int err;
 480         struct vf_cmd_info args;
 481         struct i40e_virtchnl_promisc_info promisc;
 482
 483         promisc.flags = 0;
 484         promisc.vsi_id = vf->vsi_res->vsi_id;
 485
 486         if (enable_unicast)
 487                 promisc.flags |= I40E_FLAG_VF_UNICAST_PROMISC;
 488
 489         if (enable_multicast)
 490                 promisc.flags |= I40E_FLAG_VF_MULTICAST_PROMISC;
 491
 492         args.ops = I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE;
 493         args.in_args = (uint8_t *)&promisc;
 494         args.in_args_size = sizeof(promisc);
 495         args.out_buffer = vf->aq_resp;
 496         args.out_size = I40E_AQ_BUF_SZ;
 497
 498         err = i40evf_execute_vf_cmd(dev, &args);
 499
 500         if (err)
 501                 PMD_DRV_LOG(ERR, "fail to execute command "
 502                             "CONFIG_PROMISCUOUS_MODE");
 503         return err;
 504 }
 505
 506 /* Configure vlan and double vlan offload. Use flag to specify which part to configure */
 507 static int
 508 i40evf_config_vlan_offload(struct rte_eth_dev *dev,
 509                                 bool enable_vlan_strip)
 510 {
 511         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 512         int err;
 513         struct vf_cmd_info args;
 514         struct i40e_virtchnl_vlan_offload_info offload;
 515
 516         offload.vsi_id = vf->vsi_res->vsi_id;
 517         offload.enable_vlan_strip = enable_vlan_strip;
 518
 519         args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_OFFLOAD;
 520         args.in_args = (uint8_t *)&offload;
 521         args.in_args_size = sizeof(offload);
 522         args.out_buffer = vf->aq_resp;
 523         args.out_size = I40E_AQ_BUF_SZ;
 524
 525         err = i40evf_execute_vf_cmd(dev, &args);
 526         if (err)
 527                 PMD_DRV_LOG(ERR, "fail to execute command CFG_VLAN_OFFLOAD");
 528
 529         return err;
 530 }
 531
 532 static int
 533 i40evf_config_vlan_pvid(struct rte_eth_dev *dev,
 534                                 struct i40e_vsi_vlan_pvid_info *info)
 535 {
 536         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 537         int err;
 538         struct vf_cmd_info args;
 539         struct i40e_virtchnl_pvid_info tpid_info;
 540
 541         if (dev == NULL || info == NULL) {
 542                 PMD_DRV_LOG(ERR, "invalid parameters");
 543                 return I40E_ERR_PARAM;
 544         }
 545
 546         memset(&tpid_info, 0, sizeof(tpid_info));
 547         tpid_info.vsi_id = vf->vsi_res->vsi_id;
 548         (void)rte_memcpy(&tpid_info.info, info, sizeof(*info));
 549
 550         args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_PVID;
 551         args.in_args = (uint8_t *)&tpid_info;
 552         args.in_args_size = sizeof(tpid_info);
 553         args.out_buffer = vf->aq_resp;
 554         args.out_size = I40E_AQ_BUF_SZ;
 555
 556         err = i40evf_execute_vf_cmd(dev, &args);
 557         if (err)
 558                 PMD_DRV_LOG(ERR, "fail to execute command CFG_VLAN_PVID");
 559
 560         return err;
 561 }
 562
 563 static void
 564 i40evf_fill_virtchnl_vsi_txq_info(struct i40e_virtchnl_txq_info *txq_info,
 565                                   uint16_t vsi_id,
 566                                   uint16_t queue_id,
 567                                   uint16_t nb_txq,
 568                                   struct i40e_tx_queue *txq)
 569 {
 570         txq_info->vsi_id = vsi_id;
 571         txq_info->queue_id = queue_id;
 572         if (queue_id < nb_txq) {
 573                 txq_info->ring_len = txq->nb_tx_desc;
 574                 txq_info->dma_ring_addr = txq->tx_ring_phys_addr;
 575         }
 576 }
 577
 578 static void
 579 i40evf_fill_virtchnl_vsi_rxq_info(struct i40e_virtchnl_rxq_info *rxq_info,
 580                                   uint16_t vsi_id,
 581                                   uint16_t queue_id,
 582                                   uint16_t nb_rxq,
 583                                   uint32_t max_pkt_size,
 584                                   struct i40e_rx_queue *rxq)
 585 {
 586         rxq_info->vsi_id = vsi_id;
 587         rxq_info->queue_id = queue_id;
 588         rxq_info->max_pkt_size = max_pkt_size;
 589         if (queue_id < nb_rxq) {
 590                 rxq_info->ring_len = rxq->nb_rx_desc;
 591                 rxq_info->dma_ring_addr = rxq->rx_ring_phys_addr;
 592                 rxq_info->databuffer_size =
 593                         (rte_pktmbuf_data_room_size(rxq->mp) -
 594                                 RTE_PKTMBUF_HEADROOM);
 595         }
 596 }
 597
 598 /* It configures VSI queues to co-work with Linux PF host */
 599 static int
 600 i40evf_configure_vsi_queues(struct rte_eth_dev *dev)
 601 {
 602         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 603         struct i40e_rx_queue **rxq =
 604                 (struct i40e_rx_queue **)dev->data->rx_queues;
 605         struct i40e_tx_queue **txq =
 606                 (struct i40e_tx_queue **)dev->data->tx_queues;
 607         struct i40e_virtchnl_vsi_queue_config_info *vc_vqci;
 608         struct i40e_virtchnl_queue_pair_info *vc_qpi;
 609         struct vf_cmd_info args;
 610         uint16_t i, nb_qp = vf->num_queue_pairs;
 611         const uint32_t size =
 612                 I40E_VIRTCHNL_CONFIG_VSI_QUEUES_SIZE(vc_vqci, nb_qp);
 613         uint8_t buff[size];
 614         int ret;
 615
 616         memset(buff, 0, sizeof(buff));
 617         vc_vqci = (struct i40e_virtchnl_vsi_queue_config_info *)buff;
 618         vc_vqci->vsi_id = vf->vsi_res->vsi_id;
 619         vc_vqci->num_queue_pairs = nb_qp;
 620
 621         for (i = 0, vc_qpi = vc_vqci->qpair; i < nb_qp; i++, vc_qpi++) {
 622                 i40evf_fill_virtchnl_vsi_txq_info(&vc_qpi->txq,
 623                         vc_vqci->vsi_id, i, dev->data->nb_tx_queues, txq[i]);
 624                 i40evf_fill_virtchnl_vsi_rxq_info(&vc_qpi->rxq,
 625                         vc_vqci->vsi_id, i, dev->data->nb_rx_queues,
 626                                         vf->max_pkt_len, rxq[i]);
 627         }
 628         memset(&args, 0, sizeof(args));
 629         args.ops = I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES;
 630         args.in_args = (uint8_t *)vc_vqci;
 631         args.in_args_size = size;
 632         args.out_buffer = vf->aq_resp;
 633         args.out_size = I40E_AQ_BUF_SZ;
 634         ret = i40evf_execute_vf_cmd(dev, &args);
 635         if (ret)
 636                 PMD_DRV_LOG(ERR, "Failed to execute command of "
 637                         "I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES\n");
 638
 639         return ret;
 640 }
 641
 642 /* It configures VSI queues to co-work with DPDK PF host */
 643 static int
 644 i40evf_configure_vsi_queues_ext(struct rte_eth_dev *dev)
 645 {
 646         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 647         struct i40e_rx_queue **rxq =
 648                 (struct i40e_rx_queue **)dev->data->rx_queues;
 649         struct i40e_tx_queue **txq =
 650                 (struct i40e_tx_queue **)dev->data->tx_queues;
 651         struct i40e_virtchnl_vsi_queue_config_ext_info *vc_vqcei;
 652         struct i40e_virtchnl_queue_pair_ext_info *vc_qpei;
 653         struct vf_cmd_info args;
 654         uint16_t i, nb_qp = vf->num_queue_pairs;
 655         const uint32_t size =
 656                 I40E_VIRTCHNL_CONFIG_VSI_QUEUES_SIZE(vc_vqcei, nb_qp);
 657         uint8_t buff[size];
 658         int ret;
 659
 660         memset(buff, 0, sizeof(buff));
 661         vc_vqcei = (struct i40e_virtchnl_vsi_queue_config_ext_info *)buff;
 662         vc_vqcei->vsi_id = vf->vsi_res->vsi_id;
 663         vc_vqcei->num_queue_pairs = nb_qp;
 664         vc_qpei = vc_vqcei->qpair;
 665         for (i = 0; i < nb_qp; i++, vc_qpei++) {
 666                 i40evf_fill_virtchnl_vsi_txq_info(&vc_qpei->txq,
 667                         vc_vqcei->vsi_id, i, dev->data->nb_tx_queues, txq[i]);
 668                 i40evf_fill_virtchnl_vsi_rxq_info(&vc_qpei->rxq,
 669                         vc_vqcei->vsi_id, i, dev->data->nb_rx_queues,
 670                                         vf->max_pkt_len, rxq[i]);
 671                 if (i < dev->data->nb_rx_queues)
 672                         /*
 673                          * It adds extra info for configuring VSI queues, which
 674                          * is needed to enable the configurable crc stripping
 675                          * in VF.
 676                          */
 677                         vc_qpei->rxq_ext.crcstrip =
 678                                 dev->data->dev_conf.rxmode.hw_strip_crc;
 679         }
 680         memset(&args, 0, sizeof(args));
 681         args.ops =
 682                 (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT;
 683         args.in_args = (uint8_t *)vc_vqcei;
 684         args.in_args_size = size;
 685         args.out_buffer = vf->aq_resp;
 686         args.out_size = I40E_AQ_BUF_SZ;
 687         ret = i40evf_execute_vf_cmd(dev, &args);
 688         if (ret)
 689                 PMD_DRV_LOG(ERR, "Failed to execute command of "
 690                         "I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT\n");
 691
 692         return ret;
 693 }
 694
 695 static int
 696 i40evf_configure_queues(struct rte_eth_dev *dev)
 697 {
 698         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 699
 700         if (vf->version_major == I40E_DPDK_VERSION_MAJOR)
 701                 /* To support DPDK PF host */
 702                 return i40evf_configure_vsi_queues_ext(dev);
 703         else
 704                 /* To support Linux PF host */
 705                 return i40evf_configure_vsi_queues(dev);
 706 }
 707
 708 static int
 709 i40evf_config_irq_map(struct rte_eth_dev *dev)
 710 {
 711         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 712         struct vf_cmd_info args;
 713         uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_irq_map_info) + \
 714                 sizeof(struct i40e_virtchnl_vector_map)];
 715         struct i40e_virtchnl_irq_map_info *map_info;
 716         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
 717         uint32_t vector_id;
 718         int i, err;
 719
 720         if (rte_intr_allow_others(intr_handle)) {
 721                 if (vf->version_major == I40E_DPDK_VERSION_MAJOR)
 722                         vector_id = I40EVF_VSI_DEFAULT_MSIX_INTR;
 723                 else
 724                         vector_id = I40EVF_VSI_DEFAULT_MSIX_INTR_LNX;
 725         } else {
 726                 vector_id = I40E_MISC_VEC_ID;
 727         }
 728
 729         map_info = (struct i40e_virtchnl_irq_map_info *)cmd_buffer;
 730         map_info->num_vectors = 1;
 731         map_info->vecmap[0].rxitr_idx = I40E_ITR_INDEX_DEFAULT;
 732         map_info->vecmap[0].vsi_id = vf->vsi_res->vsi_id;
 733         /* Alway use default dynamic MSIX interrupt */
 734         map_info->vecmap[0].vector_id = vector_id;
 735         /* Don't map any tx queue */
 736         map_info->vecmap[0].txq_map = 0;
 737         map_info->vecmap[0].rxq_map = 0;
 738         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 739                 map_info->vecmap[0].rxq_map |= 1 << i;
 740                 if (rte_intr_dp_is_en(intr_handle))
 741                         intr_handle->intr_vec[i] = vector_id;
 742         }
 743
 744         args.ops = I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP;
 745         args.in_args = (u8 *)cmd_buffer;
 746         args.in_args_size = sizeof(cmd_buffer);
 747         args.out_buffer = vf->aq_resp;
 748         args.out_size = I40E_AQ_BUF_SZ;
 749         err = i40evf_execute_vf_cmd(dev, &args);
 750         if (err)
 751                 PMD_DRV_LOG(ERR, "fail to execute command OP_ENABLE_QUEUES");
 752
 753         return err;
 754 }
 755
 756 static int
 757 i40evf_switch_queue(struct rte_eth_dev *dev, bool isrx, uint16_t qid,
 758                                 bool on)
 759 {
 760         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 761         struct i40e_virtchnl_queue_select queue_select;
 762         int err;
 763         struct vf_cmd_info args;
 764         memset(&queue_select, 0, sizeof(queue_select));
 765         queue_select.vsi_id = vf->vsi_res->vsi_id;
 766
 767         if (isrx)
 768                 queue_select.rx_queues |= 1 << qid;
 769         else
 770                 queue_select.tx_queues |= 1 << qid;
 771
 772         if (on)
 773                 args.ops = I40E_VIRTCHNL_OP_ENABLE_QUEUES;
 774         else
 775                 args.ops = I40E_VIRTCHNL_OP_DISABLE_QUEUES;
 776         args.in_args = (u8 *)&queue_select;
 777         args.in_args_size = sizeof(queue_select);
 778         args.out_buffer = vf->aq_resp;
 779         args.out_size = I40E_AQ_BUF_SZ;
 780         err = i40evf_execute_vf_cmd(dev, &args);
 781         if (err)
 782                 PMD_DRV_LOG(ERR, "fail to switch %s %u %s",
 783                             isrx ? "RX" : "TX", qid, on ? "on" : "off");
 784
 785         return err;
 786 }
 787
 788 static int
 789 i40evf_start_queues(struct rte_eth_dev *dev)
 790 {
 791         struct rte_eth_dev_data *dev_data = dev->data;
 792         int i;
 793         struct i40e_rx_queue *rxq;
 794         struct i40e_tx_queue *txq;
 795
 796         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 797                 rxq = dev_data->rx_queues[i];
 798                 if (rxq->rx_deferred_start)
 799                         continue;
 800                 if (i40evf_dev_rx_queue_start(dev, i) != 0) {
 801                         PMD_DRV_LOG(ERR, "Fail to start queue %u", i);
 802                         return -1;
 803                 }
 804         }
 805
 806         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 807                 txq = dev_data->tx_queues[i];
 808                 if (txq->tx_deferred_start)
 809                         continue;
 810                 if (i40evf_dev_tx_queue_start(dev, i) != 0) {
 811                         PMD_DRV_LOG(ERR, "Fail to start queue %u", i);
 812                         return -1;
 813                 }
 814         }
 815
 816         return 0;
 817 }
 818
 819 static int
 820 i40evf_stop_queues(struct rte_eth_dev *dev)
 821 {
 822         int i;
 823
 824         /* Stop TX queues first */
 825         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 826                 if (i40evf_dev_tx_queue_stop(dev, i) != 0) {
 827                         PMD_DRV_LOG(ERR, "Fail to stop queue %u", i);
 828                         return -1;
 829                 }
 830         }
 831
 832         /* Then stop RX queues */
 833         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 834                 if (i40evf_dev_rx_queue_stop(dev, i) != 0) {
 835                         PMD_DRV_LOG(ERR, "Fail to stop queue %u", i);
 836                         return -1;
 837                 }
 838         }
 839
 840         return 0;
 841 }
 842
 843 static void
 844 i40evf_add_mac_addr(struct rte_eth_dev *dev,
 845                     struct ether_addr *addr,
 846                     __rte_unused uint32_t index,
 847                     __rte_unused uint32_t pool)
 848 {
 849         struct i40e_virtchnl_ether_addr_list *list;
 850         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 851         uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_ether_addr_list) + \
 852                         sizeof(struct i40e_virtchnl_ether_addr)];
 853         int err;
 854         struct vf_cmd_info args;
 855
 856         if (i40e_validate_mac_addr(addr->addr_bytes) != I40E_SUCCESS) {
 857                 PMD_DRV_LOG(ERR, "Invalid mac:%x:%x:%x:%x:%x:%x",
 858                             addr->addr_bytes[0], addr->addr_bytes[1],
 859                             addr->addr_bytes[2], addr->addr_bytes[3],
 860                             addr->addr_bytes[4], addr->addr_bytes[5]);
 861                 return;
 862         }
 863
 864         list = (struct i40e_virtchnl_ether_addr_list *)cmd_buffer;
 865         list->vsi_id = vf->vsi_res->vsi_id;
 866         list->num_elements = 1;
 867         (void)rte_memcpy(list->list[0].addr, addr->addr_bytes,
 868                                         sizeof(addr->addr_bytes));
 869
 870         args.ops = I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS;
 871         args.in_args = cmd_buffer;
 872         args.in_args_size = sizeof(cmd_buffer);
 873         args.out_buffer = vf->aq_resp;
 874         args.out_size = I40E_AQ_BUF_SZ;
 875         err = i40evf_execute_vf_cmd(dev, &args);
 876         if (err)
 877                 PMD_DRV_LOG(ERR, "fail to execute command "
 878                             "OP_ADD_ETHER_ADDRESS");
 879
 880         return;
 881 }
 882
 883 static void
 884 i40evf_del_mac_addr(struct rte_eth_dev *dev, uint32_t index)
 885 {
 886         struct i40e_virtchnl_ether_addr_list *list;
 887         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 888         struct rte_eth_dev_data *data = dev->data;
 889         struct ether_addr *addr;
 890         uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_ether_addr_list) + \
 891                         sizeof(struct i40e_virtchnl_ether_addr)];
 892         int err;
 893         struct vf_cmd_info args;
 894
 895         addr = &(data->mac_addrs[index]);
 896
 897         if (i40e_validate_mac_addr(addr->addr_bytes) != I40E_SUCCESS) {
 898                 PMD_DRV_LOG(ERR, "Invalid mac:%x-%x-%x-%x-%x-%x",
 899                             addr->addr_bytes[0], addr->addr_bytes[1],
 900                             addr->addr_bytes[2], addr->addr_bytes[3],
 901                             addr->addr_bytes[4], addr->addr_bytes[5]);
 902                 return;
 903         }
 904
 905         list = (struct i40e_virtchnl_ether_addr_list *)cmd_buffer;
 906         list->vsi_id = vf->vsi_res->vsi_id;
 907         list->num_elements = 1;
 908         (void)rte_memcpy(list->list[0].addr, addr->addr_bytes,
 909                         sizeof(addr->addr_bytes));
 910
 911         args.ops = I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS;
 912         args.in_args = cmd_buffer;
 913         args.in_args_size = sizeof(cmd_buffer);
 914         args.out_buffer = vf->aq_resp;
 915         args.out_size = I40E_AQ_BUF_SZ;
 916         err = i40evf_execute_vf_cmd(dev, &args);
 917         if (err)
 918                 PMD_DRV_LOG(ERR, "fail to execute command "
 919                             "OP_DEL_ETHER_ADDRESS");
 920         return;
 921 }
 922
 923 static int
 924 i40evf_update_stats(struct rte_eth_dev *dev, struct i40e_eth_stats **pstats)
 925 {
 926         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 927         struct i40e_virtchnl_queue_select q_stats;
 928         int err;
 929         struct vf_cmd_info args;
 930
 931         memset(&q_stats, 0, sizeof(q_stats));
 932         q_stats.vsi_id = vf->vsi_res->vsi_id;
 933         args.ops = I40E_VIRTCHNL_OP_GET_STATS;
 934         args.in_args = (u8 *)&q_stats;
 935         args.in_args_size = sizeof(q_stats);
 936         args.out_buffer = vf->aq_resp;
 937         args.out_size = I40E_AQ_BUF_SZ;
 938
 939         err = i40evf_execute_vf_cmd(dev, &args);
 940         if (err) {
 941                 PMD_DRV_LOG(ERR, "fail to execute command OP_GET_STATS");
 942                 *pstats = NULL;
 943                 return err;
 944         }
 945         *pstats = (struct i40e_eth_stats *)args.out_buffer;
 946         return 0;
 947 }
 948
 949 static int
 950 i40evf_get_statics(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 951 {
 952         int ret;
 953         struct i40e_eth_stats *pstats = NULL;
 954
 955         ret = i40evf_update_stats(dev, &pstats);
 956         if (ret != 0)
 957                 return 0;
 958
 959         stats->ipackets = pstats->rx_unicast + pstats->rx_multicast +
 960                                                 pstats->rx_broadcast;
 961         stats->opackets = pstats->tx_broadcast + pstats->tx_multicast +
 962                                                 pstats->tx_unicast;
 963         stats->ierrors = pstats->rx_discards;
 964         stats->oerrors = pstats->tx_errors + pstats->tx_discards;
 965         stats->ibytes = pstats->rx_bytes;
 966         stats->obytes = pstats->tx_bytes;
 967
 968         return 0;
 969 }
 970
 971 static void
 972 i40evf_dev_xstats_reset(struct rte_eth_dev *dev)
 973 {
 974         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 975         struct i40e_eth_stats *pstats = NULL;
 976
 977         /* read stat values to clear hardware registers */
 978         i40evf_update_stats(dev, &pstats);
 979
 980         /* set stats offset base on current values */
 981         vf->vsi.eth_stats_offset = vf->vsi.eth_stats;
 982 }
 983
 984 static int i40evf_dev_xstats_get(struct rte_eth_dev *dev,
 985                                  struct rte_eth_xstats *xstats, unsigned n)
 986 {
 987         int ret;
 988         unsigned i;
 989         struct i40e_eth_stats *pstats = NULL;
 990
 991         if (n < I40EVF_NB_XSTATS)
 992                 return I40EVF_NB_XSTATS;
 993
 994         ret = i40evf_update_stats(dev, &pstats);
 995         if (ret != 0)
 996                 return 0;
 997
 998         if (!xstats)
 999                 return 0;
1000
1001         /* loop over xstats array and values from pstats */
1002         for (i = 0; i < I40EVF_NB_XSTATS; i++) {
1003                 snprintf(xstats[i].name, sizeof(xstats[i].name),
1004                          "%s", rte_i40evf_stats_strings[i].name);
1005                 xstats[i].value = *(uint64_t *)(((char *)pstats) +
1006                         rte_i40evf_stats_strings[i].offset);
1007         }
1008
1009         return I40EVF_NB_XSTATS;
1010 }
1011
1012 static int
1013 i40evf_add_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
1014 {
1015         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1016         struct i40e_virtchnl_vlan_filter_list *vlan_list;
1017         uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_vlan_filter_list) +
1018                                                         sizeof(uint16_t)];
1019         int err;
1020         struct vf_cmd_info args;
1021
1022         vlan_list = (struct i40e_virtchnl_vlan_filter_list *)cmd_buffer;
1023         vlan_list->vsi_id = vf->vsi_res->vsi_id;
1024         vlan_list->num_elements = 1;
1025         vlan_list->vlan_id[0] = vlanid;
1026
1027         args.ops = I40E_VIRTCHNL_OP_ADD_VLAN;
1028         args.in_args = (u8 *)&cmd_buffer;
1029         args.in_args_size = sizeof(cmd_buffer);
1030         args.out_buffer = vf->aq_resp;
1031         args.out_size = I40E_AQ_BUF_SZ;
1032         err = i40evf_execute_vf_cmd(dev, &args);
1033         if (err)
1034                 PMD_DRV_LOG(ERR, "fail to execute command OP_ADD_VLAN");
1035
1036         return err;
1037 }
1038
1039 static int
1040 i40evf_del_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
1041 {
1042         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1043         struct i40e_virtchnl_vlan_filter_list *vlan_list;
1044         uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_vlan_filter_list) +
1045                                                         sizeof(uint16_t)];
1046         int err;
1047         struct vf_cmd_info args;
1048
1049         vlan_list = (struct i40e_virtchnl_vlan_filter_list *)cmd_buffer;
1050         vlan_list->vsi_id = vf->vsi_res->vsi_id;
1051         vlan_list->num_elements = 1;
1052         vlan_list->vlan_id[0] = vlanid;
1053
1054         args.ops = I40E_VIRTCHNL_OP_DEL_VLAN;
1055         args.in_args = (u8 *)&cmd_buffer;
1056         args.in_args_size = sizeof(cmd_buffer);
1057         args.out_buffer = vf->aq_resp;
1058         args.out_size = I40E_AQ_BUF_SZ;
1059         err = i40evf_execute_vf_cmd(dev, &args);
1060         if (err)
1061                 PMD_DRV_LOG(ERR, "fail to execute command OP_DEL_VLAN");
1062
1063         return err;
1064 }
1065
1066 static int
1067 i40evf_get_link_status(struct rte_eth_dev *dev, struct rte_eth_link *link)
1068 {
1069         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1070         int err;
1071         struct vf_cmd_info args;
1072         struct rte_eth_link *new_link;
1073
1074         args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_GET_LINK_STAT;
1075         args.in_args = NULL;
1076         args.in_args_size = 0;
1077         args.out_buffer = vf->aq_resp;
1078         args.out_size = I40E_AQ_BUF_SZ;
1079         err = i40evf_execute_vf_cmd(dev, &args);
1080         if (err) {
1081                 PMD_DRV_LOG(ERR, "fail to execute command OP_GET_LINK_STAT");
1082                 return err;
1083         }
1084
1085         new_link = (struct rte_eth_link *)args.out_buffer;
1086         (void)rte_memcpy(link, new_link, sizeof(*link));
1087
1088         return 0;
1089 }
1090
1091 static const struct rte_pci_id pci_id_i40evf_map[] = {
1092 #define RTE_PCI_DEV_ID_DECL_I40EVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
1093 #include "rte_pci_dev_ids.h"
1094 { .vendor_id = 0, /* sentinel */ },
1095 };
1096
1097 static inline int
1098 i40evf_dev_atomic_write_link_status(struct rte_eth_dev *dev,
1099                                     struct rte_eth_link *link)
1100 {
1101         struct rte_eth_link *dst = &(dev->data->dev_link);
1102         struct rte_eth_link *src = link;
1103
1104         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
1105                                         *(uint64_t *)src) == 0)
1106                 return -1;
1107
1108         return 0;
1109 }
1110
1111 /* Disable IRQ0 */
1112 static inline void
1113 i40evf_disable_irq0(struct i40e_hw *hw)
1114 {
1115         /* Disable all interrupt types */
1116         I40E_WRITE_REG(hw, I40E_VFINT_ICR0_ENA1, 0);
1117         I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
1118                        I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
1119         I40EVF_WRITE_FLUSH(hw);
1120 }
1121
1122 /* Enable IRQ0 */
1123 static inline void
1124 i40evf_enable_irq0(struct i40e_hw *hw)
1125 {
1126         /* Enable admin queue interrupt trigger */
1127         uint32_t val;
1128
1129         i40evf_disable_irq0(hw);
1130         val = I40E_READ_REG(hw, I40E_VFINT_ICR0_ENA1);
1131         val |= I40E_VFINT_ICR0_ENA1_ADMINQ_MASK |
1132                 I40E_VFINT_ICR0_ENA1_LINK_STAT_CHANGE_MASK;
1133         I40E_WRITE_REG(hw, I40E_VFINT_ICR0_ENA1, val);
1134
1135         I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
1136                 I40E_VFINT_DYN_CTL01_INTENA_MASK |
1137                 I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
1138                 I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
1139
1140         I40EVF_WRITE_FLUSH(hw);
1141 }
1142
1143 static int
1144 i40evf_reset_vf(struct i40e_hw *hw)
1145 {
1146         int i, reset;
1147
1148         if (i40e_vf_reset(hw) != I40E_SUCCESS) {
1149                 PMD_INIT_LOG(ERR, "Reset VF NIC failed");
1150                 return -1;
1151         }
1152         /**
1153           * After issuing vf reset command to pf, pf won't necessarily
1154           * reset vf, it depends on what state it exactly is. If it's not
1155           * initialized yet, it won't have vf reset since it's in a certain
1156           * state. If not, it will try to reset. Even vf is reset, pf will
1157           * set I40E_VFGEN_RSTAT to COMPLETE first, then wait 10ms and set
1158           * it to ACTIVE. In this duration, vf may not catch the moment that
1159           * COMPLETE is set. So, for vf, we'll try to wait a long time.
1160           */
1161         rte_delay_ms(200);
1162
1163         for (i = 0; i < MAX_RESET_WAIT_CNT; i++) {
1164                 reset = rd32(hw, I40E_VFGEN_RSTAT) &
1165                         I40E_VFGEN_RSTAT_VFR_STATE_MASK;
1166                 reset = reset >> I40E_VFGEN_RSTAT_VFR_STATE_SHIFT;
1167                 if (I40E_VFR_COMPLETED == reset || I40E_VFR_VFACTIVE == reset)
1168                         break;
1169                 else
1170                         rte_delay_ms(50);
1171         }
1172
1173         if (i >= MAX_RESET_WAIT_CNT) {
1174                 PMD_INIT_LOG(ERR, "Reset VF NIC failed");
1175                 return -1;
1176         }
1177
1178         return 0;
1179 }
1180
1181 static int
1182 i40evf_init_vf(struct rte_eth_dev *dev)
1183 {
1184         int i, err, bufsz;
1185         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1186         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1187         struct ether_addr *p_mac_addr;
1188         uint16_t interval =
1189                 i40e_calc_itr_interval(I40E_QUEUE_ITR_INTERVAL_MAX);
1190
1191         vf->adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1192         vf->dev_data = dev->data;
1193         err = i40e_set_mac_type(hw);
1194         if (err) {
1195                 PMD_INIT_LOG(ERR, "set_mac_type failed: %d", err);
1196                 goto err;
1197         }
1198
1199         i40e_init_adminq_parameter(hw);
1200         err = i40e_init_adminq(hw);
1201         if (err) {
1202                 PMD_INIT_LOG(ERR, "init_adminq failed: %d", err);
1203                 goto err;
1204         }
1205
1206         /* Reset VF and wait until it's complete */
1207         if (i40evf_reset_vf(hw)) {
1208                 PMD_INIT_LOG(ERR, "reset NIC failed");
1209                 goto err_aq;
1210         }
1211
1212         /* VF reset, shutdown admin queue and initialize again */
1213         if (i40e_shutdown_adminq(hw) != I40E_SUCCESS) {
1214                 PMD_INIT_LOG(ERR, "i40e_shutdown_adminq failed");
1215                 return -1;
1216         }
1217
1218         i40e_init_adminq_parameter(hw);
1219         if (i40e_init_adminq(hw) != I40E_SUCCESS) {
1220                 PMD_INIT_LOG(ERR, "init_adminq failed");
1221                 return -1;
1222         }
1223         vf->aq_resp = rte_zmalloc("vf_aq_resp", I40E_AQ_BUF_SZ, 0);
1224         if (!vf->aq_resp) {
1225                 PMD_INIT_LOG(ERR, "unable to allocate vf_aq_resp memory");
1226                         goto err_aq;
1227         }
1228         if (i40evf_check_api_version(dev) != 0) {
1229                 PMD_INIT_LOG(ERR, "check_api version failed");
1230                 goto err_aq;
1231         }
1232         bufsz = sizeof(struct i40e_virtchnl_vf_resource) +
1233                 (I40E_MAX_VF_VSI * sizeof(struct i40e_virtchnl_vsi_resource));
1234         vf->vf_res = rte_zmalloc("vf_res", bufsz, 0);
1235         if (!vf->vf_res) {
1236                 PMD_INIT_LOG(ERR, "unable to allocate vf_res memory");
1237                         goto err_aq;
1238         }
1239
1240         if (i40evf_get_vf_resource(dev) != 0) {
1241                 PMD_INIT_LOG(ERR, "i40evf_get_vf_config failed");
1242                 goto err_alloc;
1243         }
1244
1245         /* got VF config message back from PF, now we can parse it */
1246         for (i = 0; i < vf->vf_res->num_vsis; i++) {
1247                 if (vf->vf_res->vsi_res[i].vsi_type == I40E_VSI_SRIOV)
1248                         vf->vsi_res = &vf->vf_res->vsi_res[i];
1249         }
1250
1251         if (!vf->vsi_res) {
1252                 PMD_INIT_LOG(ERR, "no LAN VSI found");
1253                 goto err_alloc;
1254         }
1255
1256         if (hw->mac.type == I40E_MAC_X722_VF)
1257                 vf->flags = I40E_FLAG_RSS_AQ_CAPABLE;
1258         vf->vsi.vsi_id = vf->vsi_res->vsi_id;
1259         vf->vsi.type = vf->vsi_res->vsi_type;
1260         vf->vsi.nb_qps = vf->vsi_res->num_queue_pairs;
1261         vf->vsi.adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1262
1263         /* Store the MAC address configured by host, or generate random one */
1264         p_mac_addr = (struct ether_addr *)(vf->vsi_res->default_mac_addr);
1265         if (is_valid_assigned_ether_addr(p_mac_addr)) /* Configured by host */
1266                 ether_addr_copy(p_mac_addr, (struct ether_addr *)hw->mac.addr);
1267         else
1268                 eth_random_addr(hw->mac.addr); /* Generate a random one */
1269
1270         /* If the PF host is not DPDK, set the interval of ITR0 to max*/
1271         if (vf->version_major != I40E_DPDK_VERSION_MAJOR) {
1272                 I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
1273                                (I40E_ITR_INDEX_DEFAULT <<
1274                                 I40E_VFINT_DYN_CTL0_ITR_INDX_SHIFT) |
1275                                (interval <<
1276                                 I40E_VFINT_DYN_CTL0_INTERVAL_SHIFT));
1277                 I40EVF_WRITE_FLUSH(hw);
1278         }
1279
1280         return 0;
1281
1282 err_alloc:
1283         rte_free(vf->vf_res);
1284 err_aq:
1285         i40e_shutdown_adminq(hw); /* ignore error */
1286 err:
1287         return -1;
1288 }
1289
1290 static int
1291 i40evf_uninit_vf(struct rte_eth_dev *dev)
1292 {
1293         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1294         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1295
1296         PMD_INIT_FUNC_TRACE();
1297
1298         if (hw->adapter_stopped == 0)
1299                 i40evf_dev_close(dev);
1300         rte_free(vf->vf_res);
1301         vf->vf_res = NULL;
1302         rte_free(vf->aq_resp);
1303         vf->aq_resp = NULL;
1304
1305         return 0;
1306 }
1307
1308 static void
1309 i40evf_handle_pf_event(__rte_unused struct rte_eth_dev *dev,
1310                            uint8_t *msg,
1311                            __rte_unused uint16_t msglen)
1312 {
1313         struct i40e_virtchnl_pf_event *pf_msg =
1314                         (struct i40e_virtchnl_pf_event *)msg;
1315         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1316
1317         switch (pf_msg->event) {
1318         case I40E_VIRTCHNL_EVENT_RESET_IMPENDING:
1319                 PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_RESET_IMPENDING event\n");
1320                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET);
1321                 break;
1322         case I40E_VIRTCHNL_EVENT_LINK_CHANGE:
1323                 PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_LINK_CHANGE event\n");
1324                 vf->link_up = pf_msg->event_data.link_event.link_status;
1325                 vf->link_speed = pf_msg->event_data.link_event.link_speed;
1326                 break;
1327         case I40E_VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
1328                 PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_PF_DRIVER_CLOSE event\n");
1329                 break;
1330         default:
1331                 PMD_DRV_LOG(ERR, " unknown event received %u", pf_msg->event);
1332                 break;
1333         }
1334 }
1335
1336 static void
1337 i40evf_handle_aq_msg(struct rte_eth_dev *dev)
1338 {
1339         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1340         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1341         struct i40e_arq_event_info info;
1342         struct i40e_virtchnl_msg *v_msg;
1343         uint16_t pending, opcode;
1344         int ret;
1345
1346         info.buf_len = I40E_AQ_BUF_SZ;
1347         if (!vf->aq_resp) {
1348                 PMD_DRV_LOG(ERR, "Buffer for adminq resp should not be NULL");
1349                 return;
1350         }
1351         info.msg_buf = vf->aq_resp;
1352         v_msg = (struct i40e_virtchnl_msg *)&info.desc;
1353
1354         pending = 1;
1355         while (pending) {
1356                 ret = i40e_clean_arq_element(hw, &info, &pending);
1357
1358                 if (ret != I40E_SUCCESS) {
1359                         PMD_DRV_LOG(INFO, "Failed to read msg from AdminQ,"
1360                                     "ret: %d", ret);
1361                         break;
1362                 }
1363                 opcode = rte_le_to_cpu_16(info.desc.opcode);
1364
1365                 switch (opcode) {
1366                 case i40e_aqc_opc_send_msg_to_vf:
1367                         if (v_msg->v_opcode == I40E_VIRTCHNL_OP_EVENT)
1368                                 /* process event*/
1369                                 i40evf_handle_pf_event(dev, info.msg_buf,
1370                                                        info.msg_len);
1371                         else {
1372                                 /* read message and it's expected one */
1373                                 if (v_msg->v_opcode == vf->pend_cmd) {
1374                                         vf->cmd_retval = v_msg->v_retval;
1375                                         /* prevent compiler reordering */
1376                                         rte_compiler_barrier();
1377                                         _clear_cmd(vf);
1378                                 } else
1379                                         PMD_DRV_LOG(ERR, "command mismatch,"
1380                                                 "expect %u, get %u",
1381                                                 vf->pend_cmd, v_msg->v_opcode);
1382                                 PMD_DRV_LOG(DEBUG, "adminq response is received,"
1383                                              " opcode = %d\n", v_msg->v_opcode);
1384                         }
1385                         break;
1386                 default:
1387                         PMD_DRV_LOG(ERR, "Request %u is not supported yet",
1388                                     opcode);
1389                         break;
1390                 }
1391         }
1392 }
1393
1394 /**
1395  * Interrupt handler triggered by NIC  for handling
1396  * specific interrupt. Only adminq interrupt is processed in VF.
1397  *
1398  * @param handle
1399  *  Pointer to interrupt handle.
1400  * @param param
1401  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1402  *
1403  * @return
1404  *  void
1405  */
1406 static void
1407 i40evf_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
1408                              void *param)
1409 {
1410         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1411         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1412         uint32_t icr0;
1413
1414         i40evf_disable_irq0(hw);
1415
1416         /* read out interrupt causes */
1417         icr0 = I40E_READ_REG(hw, I40E_VFINT_ICR01);
1418
1419         /* No interrupt event indicated */
1420         if (!(icr0 & I40E_VFINT_ICR01_INTEVENT_MASK)) {
1421                 PMD_DRV_LOG(DEBUG, "No interrupt event, nothing to do\n");
1422                 goto done;
1423         }
1424
1425         if (icr0 & I40E_VFINT_ICR01_ADMINQ_MASK) {
1426                 PMD_DRV_LOG(DEBUG, "ICR01_ADMINQ is reported\n");
1427                 i40evf_handle_aq_msg(dev);
1428         }
1429
1430         /* Link Status Change interrupt */
1431         if (icr0 & I40E_VFINT_ICR01_LINK_STAT_CHANGE_MASK)
1432                 PMD_DRV_LOG(DEBUG, "LINK_STAT_CHANGE is reported,"
1433                                    " do nothing\n");
1434
1435 done:
1436         i40evf_enable_irq0(hw);
1437         rte_intr_enable(&dev->pci_dev->intr_handle);
1438 }
1439
1440 static int
1441 i40evf_dev_init(struct rte_eth_dev *eth_dev)
1442 {
1443         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(\
1444                         eth_dev->data->dev_private);
1445         struct rte_pci_device *pci_dev = eth_dev->pci_dev;
1446
1447         PMD_INIT_FUNC_TRACE();
1448
1449         /* assign ops func pointer */
1450         eth_dev->dev_ops = &i40evf_eth_dev_ops;
1451         eth_dev->rx_pkt_burst = &i40e_recv_pkts;
1452         eth_dev->tx_pkt_burst = &i40e_xmit_pkts;
1453
1454         /*
1455          * For secondary processes, we don't initialise any further as primary
1456          * has already done this work.
1457          */
1458         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
1459                 i40e_set_rx_function(eth_dev);
1460                 i40e_set_tx_function(eth_dev);
1461                 return 0;
1462         }
1463
1464         rte_eth_copy_pci_info(eth_dev, eth_dev->pci_dev);
1465
1466         hw->vendor_id = eth_dev->pci_dev->id.vendor_id;
1467         hw->device_id = eth_dev->pci_dev->id.device_id;
1468         hw->subsystem_vendor_id = eth_dev->pci_dev->id.subsystem_vendor_id;
1469         hw->subsystem_device_id = eth_dev->pci_dev->id.subsystem_device_id;
1470         hw->bus.device = eth_dev->pci_dev->addr.devid;
1471         hw->bus.func = eth_dev->pci_dev->addr.function;
1472         hw->hw_addr = (void *)eth_dev->pci_dev->mem_resource[0].addr;
1473         hw->adapter_stopped = 0;
1474
1475         if(i40evf_init_vf(eth_dev) != 0) {
1476                 PMD_INIT_LOG(ERR, "Init vf failed");
1477                 return -1;
1478         }
1479
1480         /* register callback func to eal lib */
1481         rte_intr_callback_register(&pci_dev->intr_handle,
1482                 i40evf_dev_interrupt_handler, (void *)eth_dev);
1483
1484         /* enable uio intr after callback register */
1485         rte_intr_enable(&pci_dev->intr_handle);
1486
1487         /* configure and enable device interrupt */
1488         i40evf_enable_irq0(hw);
1489
1490         /* copy mac addr */
1491         eth_dev->data->mac_addrs = rte_zmalloc("i40evf_mac",
1492                                         ETHER_ADDR_LEN * I40E_NUM_MACADDR_MAX,
1493                                         0);
1494         if (eth_dev->data->mac_addrs == NULL) {
1495                 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to"
1496                                 " store MAC addresses",
1497                                 ETHER_ADDR_LEN * I40E_NUM_MACADDR_MAX);
1498                 return -ENOMEM;
1499         }
1500         ether_addr_copy((struct ether_addr *)hw->mac.addr,
1501                         &eth_dev->data->mac_addrs[0]);
1502
1503         return 0;
1504 }
1505
1506 static int
1507 i40evf_dev_uninit(struct rte_eth_dev *eth_dev)
1508 {
1509         PMD_INIT_FUNC_TRACE();
1510
1511         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1512                 return -EPERM;
1513
1514         eth_dev->dev_ops = NULL;
1515         eth_dev->rx_pkt_burst = NULL;
1516         eth_dev->tx_pkt_burst = NULL;
1517
1518         if (i40evf_uninit_vf(eth_dev) != 0) {
1519                 PMD_INIT_LOG(ERR, "i40evf_uninit_vf failed");
1520                 return -1;
1521         }
1522
1523         rte_free(eth_dev->data->mac_addrs);
1524         eth_dev->data->mac_addrs = NULL;
1525
1526         return 0;
1527 }
1528 /*
1529  * virtual function driver struct
1530  */
1531 static struct eth_driver rte_i40evf_pmd = {
1532         .pci_drv = {
1533                 .name = "rte_i40evf_pmd",
1534                 .id_table = pci_id_i40evf_map,
1535                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
1536         },
1537         .eth_dev_init = i40evf_dev_init,
1538         .eth_dev_uninit = i40evf_dev_uninit,
1539         .dev_private_size = sizeof(struct i40e_adapter),
1540 };
1541
1542 /*
1543  * VF Driver initialization routine.
1544  * Invoked one at EAL init time.
1545  * Register itself as the [Virtual Poll Mode] Driver of PCI Fortville devices.
1546  */
1547 static int
1548 rte_i40evf_pmd_init(const char *name __rte_unused,
1549                     const char *params __rte_unused)
1550 {
1551         PMD_INIT_FUNC_TRACE();
1552
1553         rte_eth_driver_register(&rte_i40evf_pmd);
1554
1555         return 0;
1556 }
1557
1558 static struct rte_driver rte_i40evf_driver = {
1559         .type = PMD_PDEV,
1560         .init = rte_i40evf_pmd_init,
1561 };
1562
1563 PMD_REGISTER_DRIVER(rte_i40evf_driver);
1564
1565 static int
1566 i40evf_dev_configure(struct rte_eth_dev *dev)
1567 {
1568         struct i40e_adapter *ad =
1569                 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1570
1571         /* Initialize to TRUE. If any of Rx queues doesn't meet the bulk
1572          * allocation or vector Rx preconditions we will reset it.
1573          */
1574         ad->rx_bulk_alloc_allowed = true;
1575         ad->rx_vec_allowed = true;
1576         ad->tx_simple_allowed = true;
1577         ad->tx_vec_allowed = true;
1578
1579         return i40evf_init_vlan(dev);
1580 }
1581
1582 static int
1583 i40evf_init_vlan(struct rte_eth_dev *dev)
1584 {
1585         struct rte_eth_dev_data *data = dev->data;
1586         int ret;
1587
1588         /* Apply vlan offload setting */
1589         i40evf_vlan_offload_set(dev, ETH_VLAN_STRIP_MASK);
1590
1591         /* Apply pvid setting */
1592         ret = i40evf_vlan_pvid_set(dev, data->dev_conf.txmode.pvid,
1593                                 data->dev_conf.txmode.hw_vlan_insert_pvid);
1594         return ret;
1595 }
1596
1597 static void
1598 i40evf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1599 {
1600         bool enable_vlan_strip = 0;
1601         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
1602         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1603
1604         /* Linux pf host doesn't support vlan offload yet */
1605         if (vf->version_major == I40E_DPDK_VERSION_MAJOR) {
1606                 /* Vlan stripping setting */
1607                 if (mask & ETH_VLAN_STRIP_MASK) {
1608                         /* Enable or disable VLAN stripping */
1609                         if (dev_conf->rxmode.hw_vlan_strip)
1610                                 enable_vlan_strip = 1;
1611                         else
1612                                 enable_vlan_strip = 0;
1613
1614                         i40evf_config_vlan_offload(dev, enable_vlan_strip);
1615                 }
1616         }
1617 }
1618
1619 static int
1620 i40evf_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid, int on)
1621 {
1622         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
1623         struct i40e_vsi_vlan_pvid_info info;
1624         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1625
1626         memset(&info, 0, sizeof(info));
1627         info.on = on;
1628
1629         /* Linux pf host don't support vlan offload yet */
1630         if (vf->version_major == I40E_DPDK_VERSION_MAJOR) {
1631                 if (info.on)
1632                         info.config.pvid = pvid;
1633                 else {
1634                         info.config.reject.tagged =
1635                                 dev_conf->txmode.hw_vlan_reject_tagged;
1636                         info.config.reject.untagged =
1637                                 dev_conf->txmode.hw_vlan_reject_untagged;
1638                 }
1639                 return i40evf_config_vlan_pvid(dev, &info);
1640         }
1641
1642         return 0;
1643 }
1644
1645 static int
1646 i40evf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1647 {
1648         struct i40e_rx_queue *rxq;
1649         int err = 0;
1650         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1651
1652         PMD_INIT_FUNC_TRACE();
1653
1654         if (rx_queue_id < dev->data->nb_rx_queues) {
1655                 rxq = dev->data->rx_queues[rx_queue_id];
1656
1657                 err = i40e_alloc_rx_queue_mbufs(rxq);
1658                 if (err) {
1659                         PMD_DRV_LOG(ERR, "Failed to allocate RX queue mbuf");
1660                         return err;
1661                 }
1662
1663                 rte_wmb();
1664
1665                 /* Init the RX tail register. */
1666                 I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
1667                 I40EVF_WRITE_FLUSH(hw);
1668
1669                 /* Ready to switch the queue on */
1670                 err = i40evf_switch_queue(dev, TRUE, rx_queue_id, TRUE);
1671
1672                 if (err)
1673                         PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on",
1674                                     rx_queue_id);
1675                 else
1676                         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
1677         }
1678
1679         return err;
1680 }
1681
1682 static int
1683 i40evf_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1684 {
1685         struct i40e_rx_queue *rxq;
1686         int err;
1687
1688         if (rx_queue_id < dev->data->nb_rx_queues) {
1689                 rxq = dev->data->rx_queues[rx_queue_id];
1690
1691                 err = i40evf_switch_queue(dev, TRUE, rx_queue_id, FALSE);
1692
1693                 if (err) {
1694                         PMD_DRV_LOG(ERR, "Failed to switch RX queue %u off",
1695                                     rx_queue_id);
1696                         return err;
1697                 }
1698
1699                 i40e_rx_queue_release_mbufs(rxq);
1700                 i40e_reset_rx_queue(rxq);
1701                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
1702         }
1703
1704         return 0;
1705 }
1706
1707 static int
1708 i40evf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
1709 {
1710         int err = 0;
1711
1712         PMD_INIT_FUNC_TRACE();
1713
1714         if (tx_queue_id < dev->data->nb_tx_queues) {
1715
1716                 /* Ready to switch the queue on */
1717                 err = i40evf_switch_queue(dev, FALSE, tx_queue_id, TRUE);
1718
1719                 if (err)
1720                         PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on",
1721                                     tx_queue_id);
1722                 else
1723                         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
1724         }
1725
1726         return err;
1727 }
1728
1729 static int
1730 i40evf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
1731 {
1732         struct i40e_tx_queue *txq;
1733         int err;
1734
1735         if (tx_queue_id < dev->data->nb_tx_queues) {
1736                 txq = dev->data->tx_queues[tx_queue_id];
1737
1738                 err = i40evf_switch_queue(dev, FALSE, tx_queue_id, FALSE);
1739
1740                 if (err) {
1741                         PMD_DRV_LOG(ERR, "Failed to switch TX queue %u off",
1742                                     tx_queue_id);
1743                         return err;
1744                 }
1745
1746                 i40e_tx_queue_release_mbufs(txq);
1747                 i40e_reset_tx_queue(txq);
1748                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
1749         }
1750
1751         return 0;
1752 }
1753
1754 static int
1755 i40evf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1756 {
1757         int ret;
1758
1759         if (on)
1760                 ret = i40evf_add_vlan(dev, vlan_id);
1761         else
1762                 ret = i40evf_del_vlan(dev,vlan_id);
1763
1764         return ret;
1765 }
1766
1767 static int
1768 i40evf_rxq_init(struct rte_eth_dev *dev, struct i40e_rx_queue *rxq)
1769 {
1770         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1771         struct rte_eth_dev_data *dev_data = dev->data;
1772         struct rte_pktmbuf_pool_private *mbp_priv;
1773         uint16_t buf_size, len;
1774
1775         rxq->qrx_tail = hw->hw_addr + I40E_QRX_TAIL1(rxq->queue_id);
1776         I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
1777         I40EVF_WRITE_FLUSH(hw);
1778
1779         /* Calculate the maximum packet length allowed */
1780         mbp_priv = rte_mempool_get_priv(rxq->mp);
1781         buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
1782                                         RTE_PKTMBUF_HEADROOM);
1783         rxq->hs_mode = i40e_header_split_none;
1784         rxq->rx_hdr_len = 0;
1785         rxq->rx_buf_len = RTE_ALIGN(buf_size, (1 << I40E_RXQ_CTX_DBUFF_SHIFT));
1786         len = rxq->rx_buf_len * I40E_MAX_CHAINED_RX_BUFFERS;
1787         rxq->max_pkt_len = RTE_MIN(len,
1788                 dev_data->dev_conf.rxmode.max_rx_pkt_len);
1789
1790         /**
1791          * Check if the jumbo frame and maximum packet length are set correctly
1792          */
1793         if (dev_data->dev_conf.rxmode.jumbo_frame == 1) {
1794                 if (rxq->max_pkt_len <= ETHER_MAX_LEN ||
1795                     rxq->max_pkt_len > I40E_FRAME_SIZE_MAX) {
1796                         PMD_DRV_LOG(ERR, "maximum packet length must be "
1797                                 "larger than %u and smaller than %u, as jumbo "
1798                                 "frame is enabled", (uint32_t)ETHER_MAX_LEN,
1799                                         (uint32_t)I40E_FRAME_SIZE_MAX);
1800                         return I40E_ERR_CONFIG;
1801                 }
1802         } else {
1803                 if (rxq->max_pkt_len < ETHER_MIN_LEN ||
1804                     rxq->max_pkt_len > ETHER_MAX_LEN) {
1805                         PMD_DRV_LOG(ERR, "maximum packet length must be "
1806                                 "larger than %u and smaller than %u, as jumbo "
1807                                 "frame is disabled", (uint32_t)ETHER_MIN_LEN,
1808                                                 (uint32_t)ETHER_MAX_LEN);
1809                         return I40E_ERR_CONFIG;
1810                 }
1811         }
1812
1813         if (dev_data->dev_conf.rxmode.enable_scatter ||
1814             (rxq->max_pkt_len + 2 * I40E_VLAN_TAG_SIZE) > buf_size) {
1815                 dev_data->scattered_rx = 1;
1816         }
1817
1818         return 0;
1819 }
1820
1821 static int
1822 i40evf_rx_init(struct rte_eth_dev *dev)
1823 {
1824         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1825         uint16_t i;
1826         int ret = I40E_SUCCESS;
1827         struct i40e_rx_queue **rxq =
1828                 (struct i40e_rx_queue **)dev->data->rx_queues;
1829
1830         i40evf_config_rss(vf);
1831         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1832                 if (!rxq[i] || !rxq[i]->q_set)
1833                         continue;
1834                 ret = i40evf_rxq_init(dev, rxq[i]);
1835                 if (ret != I40E_SUCCESS)
1836                         break;
1837         }
1838         if (ret == I40E_SUCCESS)
1839                 i40e_set_rx_function(dev);
1840
1841         return ret;
1842 }
1843
1844 static void
1845 i40evf_tx_init(struct rte_eth_dev *dev)
1846 {
1847         uint16_t i;
1848         struct i40e_tx_queue **txq =
1849                 (struct i40e_tx_queue **)dev->data->tx_queues;
1850         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1851
1852         for (i = 0; i < dev->data->nb_tx_queues; i++)
1853                 txq[i]->qtx_tail = hw->hw_addr + I40E_QTX_TAIL1(i);
1854
1855         i40e_set_tx_function(dev);
1856 }
1857
1858 static inline void
1859 i40evf_enable_queues_intr(struct rte_eth_dev *dev)
1860 {
1861         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1862         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1863         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1864
1865         if (!rte_intr_allow_others(intr_handle)) {
1866                 I40E_WRITE_REG(hw,
1867                                I40E_VFINT_DYN_CTL01,
1868                                I40E_VFINT_DYN_CTL01_INTENA_MASK |
1869                                I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
1870                                I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
1871                 I40EVF_WRITE_FLUSH(hw);
1872                 return;
1873         }
1874
1875         if (vf->version_major == I40E_DPDK_VERSION_MAJOR)
1876                 /* To support DPDK PF host */
1877                 I40E_WRITE_REG(hw,
1878                         I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR - 1),
1879                         I40E_VFINT_DYN_CTLN1_INTENA_MASK |
1880                         I40E_VFINT_DYN_CTLN_CLEARPBA_MASK);
1881         /* If host driver is kernel driver, do nothing.
1882          * Interrupt 0 is used for rx packets, but don't set
1883          * I40E_VFINT_DYN_CTL01,
1884          * because it is already done in i40evf_enable_irq0.
1885          */
1886
1887         I40EVF_WRITE_FLUSH(hw);
1888 }
1889
1890 static inline void
1891 i40evf_disable_queues_intr(struct rte_eth_dev *dev)
1892 {
1893         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1894         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1895         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1896
1897         if (!rte_intr_allow_others(intr_handle)) {
1898                 I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
1899                                I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
1900                 I40EVF_WRITE_FLUSH(hw);
1901                 return;
1902         }
1903
1904         if (vf->version_major == I40E_DPDK_VERSION_MAJOR)
1905                 I40E_WRITE_REG(hw,
1906                                I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR
1907                                                     - 1),
1908                                0);
1909         /* If host driver is kernel driver, do nothing.
1910          * Interrupt 0 is used for rx packets, but don't zero
1911          * I40E_VFINT_DYN_CTL01,
1912          * because interrupt 0 is also used for adminq processing.
1913          */
1914
1915         I40EVF_WRITE_FLUSH(hw);
1916 }
1917
1918 static int
1919 i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
1920 {
1921         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1922         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1923         uint16_t interval =
1924                 i40e_calc_itr_interval(RTE_LIBRTE_I40E_ITR_INTERVAL);
1925         uint16_t msix_intr;
1926
1927         msix_intr = intr_handle->intr_vec[queue_id];
1928         if (msix_intr == I40E_MISC_VEC_ID)
1929                 I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
1930                                I40E_VFINT_DYN_CTL01_INTENA_MASK |
1931                                I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
1932                                (0 << I40E_VFINT_DYN_CTL01_ITR_INDX_SHIFT) |
1933                                (interval <<
1934                                 I40E_VFINT_DYN_CTL01_INTERVAL_SHIFT));
1935         else
1936                 I40E_WRITE_REG(hw,
1937                                I40E_VFINT_DYN_CTLN1(msix_intr -
1938                                                     I40E_RX_VEC_START),
1939                                I40E_VFINT_DYN_CTLN1_INTENA_MASK |
1940                                I40E_VFINT_DYN_CTLN1_CLEARPBA_MASK |
1941                                (0 << I40E_VFINT_DYN_CTLN1_ITR_INDX_SHIFT) |
1942                                (interval <<
1943                                 I40E_VFINT_DYN_CTLN1_INTERVAL_SHIFT));
1944
1945         I40EVF_WRITE_FLUSH(hw);
1946
1947         rte_intr_enable(&dev->pci_dev->intr_handle);
1948
1949         return 0;
1950 }
1951
1952 static int
1953 i40evf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
1954 {
1955         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1956         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1957         uint16_t msix_intr;
1958
1959         msix_intr = intr_handle->intr_vec[queue_id];
1960         if (msix_intr == I40E_MISC_VEC_ID)
1961                 I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 0);
1962         else
1963                 I40E_WRITE_REG(hw,
1964                                I40E_VFINT_DYN_CTLN1(msix_intr -
1965                                                     I40E_RX_VEC_START),
1966                                0);
1967
1968         I40EVF_WRITE_FLUSH(hw);
1969
1970         return 0;
1971 }
1972
1973 static void
1974 i40evf_add_del_all_mac_addr(struct rte_eth_dev *dev, bool add)
1975 {
1976         struct i40e_virtchnl_ether_addr_list *list;
1977         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1978         int err, i, j;
1979         int next_begin = 0;
1980         int begin = 0;
1981         uint32_t len;
1982         struct ether_addr *addr;
1983         struct vf_cmd_info args;
1984
1985         do {
1986                 j = 0;
1987                 len = sizeof(struct i40e_virtchnl_ether_addr_list);
1988                 for (i = begin; i < I40E_NUM_MACADDR_MAX; i++, next_begin++) {
1989                         if (is_zero_ether_addr(&dev->data->mac_addrs[i]))
1990                                 continue;
1991                         len += sizeof(struct i40e_virtchnl_ether_addr);
1992                         if (len >= I40E_AQ_BUF_SZ) {
1993                                 next_begin = i + 1;
1994                                 break;
1995                         }
1996                 }
1997
1998                 list = rte_zmalloc("i40evf_del_mac_buffer", len, 0);
1999
2000                 for (i = begin; i < next_begin; i++) {
2001                         addr = &dev->data->mac_addrs[i];
2002                         if (is_zero_ether_addr(addr))
2003                                 continue;
2004                         (void)rte_memcpy(list->list[j].addr, addr->addr_bytes,
2005                                          sizeof(addr->addr_bytes));
2006                         PMD_DRV_LOG(DEBUG, "add/rm mac:%x:%x:%x:%x:%x:%x",
2007                                     addr->addr_bytes[0], addr->addr_bytes[1],
2008                                     addr->addr_bytes[2], addr->addr_bytes[3],
2009                                     addr->addr_bytes[4], addr->addr_bytes[5]);
2010                         j++;
2011                 }
2012                 list->vsi_id = vf->vsi_res->vsi_id;
2013                 list->num_elements = j;
2014                 args.ops = add ? I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS :
2015                            I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS;
2016                 args.in_args = (uint8_t *)list;
2017                 args.in_args_size = len;
2018                 args.out_buffer = vf->aq_resp;
2019                 args.out_size = I40E_AQ_BUF_SZ;
2020                 err = i40evf_execute_vf_cmd(dev, &args);
2021                 if (err)
2022                         PMD_DRV_LOG(ERR, "fail to execute command %s",
2023                                     add ? "OP_ADD_ETHER_ADDRESS" :
2024                                     "OP_DEL_ETHER_ADDRESS");
2025                 rte_free(list);
2026                 begin = next_begin;
2027         } while (begin < I40E_NUM_MACADDR_MAX);
2028 }
2029
2030 static int
2031 i40evf_dev_start(struct rte_eth_dev *dev)
2032 {
2033         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2034         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2035         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
2036         uint32_t intr_vector = 0;
2037
2038         PMD_INIT_FUNC_TRACE();
2039
2040         hw->adapter_stopped = 0;
2041
2042         vf->max_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
2043         vf->num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues,
2044                                         dev->data->nb_tx_queues);
2045
2046         /* check and configure queue intr-vector mapping */
2047         if (dev->data->dev_conf.intr_conf.rxq != 0) {
2048                 intr_vector = dev->data->nb_rx_queues;
2049                 if (rte_intr_efd_enable(intr_handle, intr_vector))
2050                         return -1;
2051         }
2052
2053         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
2054                 intr_handle->intr_vec =
2055                         rte_zmalloc("intr_vec",
2056                                     dev->data->nb_rx_queues * sizeof(int), 0);
2057                 if (!intr_handle->intr_vec) {
2058                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2059                                      " intr_vec\n", dev->data->nb_rx_queues);
2060                         return -ENOMEM;
2061                 }
2062         }
2063
2064         if (i40evf_rx_init(dev) != 0){
2065                 PMD_DRV_LOG(ERR, "failed to do RX init");
2066                 return -1;
2067         }
2068
2069         i40evf_tx_init(dev);
2070
2071         if (i40evf_configure_queues(dev) != 0) {
2072                 PMD_DRV_LOG(ERR, "configure queues failed");
2073                 goto err_queue;
2074         }
2075         if (i40evf_config_irq_map(dev)) {
2076                 PMD_DRV_LOG(ERR, "config_irq_map failed");
2077                 goto err_queue;
2078         }
2079
2080         /* Set all mac addrs */
2081         i40evf_add_del_all_mac_addr(dev, TRUE);
2082
2083         if (i40evf_start_queues(dev) != 0) {
2084                 PMD_DRV_LOG(ERR, "enable queues failed");
2085                 goto err_mac;
2086         }
2087
2088         i40evf_enable_queues_intr(dev);
2089         return 0;
2090
2091 err_mac:
2092         i40evf_add_del_all_mac_addr(dev, FALSE);
2093 err_queue:
2094         return -1;
2095 }
2096
2097 static void
2098 i40evf_dev_stop(struct rte_eth_dev *dev)
2099 {
2100         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
2101
2102         PMD_INIT_FUNC_TRACE();
2103
2104         i40evf_stop_queues(dev);
2105         i40evf_disable_queues_intr(dev);
2106         i40e_dev_clear_queues(dev);
2107
2108         /* Clean datapath event and queue/vec mapping */
2109         rte_intr_efd_disable(intr_handle);
2110         if (intr_handle->intr_vec) {
2111                 rte_free(intr_handle->intr_vec);
2112                 intr_handle->intr_vec = NULL;
2113         }
2114         /* remove all mac addrs */
2115         i40evf_add_del_all_mac_addr(dev, FALSE);
2116
2117 }
2118
2119 static int
2120 i40evf_dev_link_update(struct rte_eth_dev *dev,
2121                        __rte_unused int wait_to_complete)
2122 {
2123         struct rte_eth_link new_link;
2124         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2125         /*
2126          * DPDK pf host provide interfacet to acquire link status
2127          * while Linux driver does not
2128          */
2129         if (vf->version_major == I40E_DPDK_VERSION_MAJOR)
2130                 i40evf_get_link_status(dev, &new_link);
2131         else {
2132                 /* Linux driver PF host */
2133                 switch (vf->link_speed) {
2134                 case I40E_LINK_SPEED_100MB:
2135                         new_link.link_speed = ETH_SPEED_NUM_100M;
2136                         break;
2137                 case I40E_LINK_SPEED_1GB:
2138                         new_link.link_speed = ETH_SPEED_NUM_1G;
2139                         break;
2140                 case I40E_LINK_SPEED_10GB:
2141                         new_link.link_speed = ETH_SPEED_NUM_10G;
2142                         break;
2143                 case I40E_LINK_SPEED_20GB:
2144                         new_link.link_speed = ETH_SPEED_NUM_20G;
2145                         break;
2146                 case I40E_LINK_SPEED_40GB:
2147                         new_link.link_speed = ETH_SPEED_NUM_40G;
2148                         break;
2149                 default:
2150                         new_link.link_speed = ETH_SPEED_NUM_100M;
2151                         break;
2152                 }
2153                 /* full duplex only */
2154                 new_link.link_duplex = ETH_LINK_FULL_DUPLEX;
2155                 new_link.link_status = vf->link_up ? ETH_LINK_UP :
2156                                                      ETH_LINK_DOWN;
2157         }
2158         i40evf_dev_atomic_write_link_status(dev, &new_link);
2159
2160         return 0;
2161 }
2162
2163 static void
2164 i40evf_dev_promiscuous_enable(struct rte_eth_dev *dev)
2165 {
2166         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2167         int ret;
2168
2169         /* If enabled, just return */
2170         if (vf->promisc_unicast_enabled)
2171                 return;
2172
2173         ret = i40evf_config_promisc(dev, 1, vf->promisc_multicast_enabled);
2174         if (ret == 0)
2175                 vf->promisc_unicast_enabled = TRUE;
2176 }
2177
2178 static void
2179 i40evf_dev_promiscuous_disable(struct rte_eth_dev *dev)
2180 {
2181         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2182         int ret;
2183
2184         /* If disabled, just return */
2185         if (!vf->promisc_unicast_enabled)
2186                 return;
2187
2188         ret = i40evf_config_promisc(dev, 0, vf->promisc_multicast_enabled);
2189         if (ret == 0)
2190                 vf->promisc_unicast_enabled = FALSE;
2191 }
2192
2193 static void
2194 i40evf_dev_allmulticast_enable(struct rte_eth_dev *dev)
2195 {
2196         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2197         int ret;
2198
2199         /* If enabled, just return */
2200         if (vf->promisc_multicast_enabled)
2201                 return;
2202
2203         ret = i40evf_config_promisc(dev, vf->promisc_unicast_enabled, 1);
2204         if (ret == 0)
2205                 vf->promisc_multicast_enabled = TRUE;
2206 }
2207
2208 static void
2209 i40evf_dev_allmulticast_disable(struct rte_eth_dev *dev)
2210 {
2211         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2212         int ret;
2213
2214         /* If enabled, just return */
2215         if (!vf->promisc_multicast_enabled)
2216                 return;
2217
2218         ret = i40evf_config_promisc(dev, vf->promisc_unicast_enabled, 0);
2219         if (ret == 0)
2220                 vf->promisc_multicast_enabled = FALSE;
2221 }
2222
2223 static void
2224 i40evf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
2225 {
2226         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2227
2228         memset(dev_info, 0, sizeof(*dev_info));
2229         dev_info->max_rx_queues = vf->vsi_res->num_queue_pairs;
2230         dev_info->max_tx_queues = vf->vsi_res->num_queue_pairs;
2231         dev_info->min_rx_bufsize = I40E_BUF_SIZE_MIN;
2232         dev_info->max_rx_pktlen = I40E_FRAME_SIZE_MAX;
2233         dev_info->hash_key_size = (I40E_VFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t);
2234         dev_info->reta_size = ETH_RSS_RETA_SIZE_64;
2235         dev_info->flow_type_rss_offloads = I40E_RSS_OFFLOAD_ALL;
2236         dev_info->max_mac_addrs = I40E_NUM_MACADDR_MAX;
2237         dev_info->rx_offload_capa =
2238                 DEV_RX_OFFLOAD_VLAN_STRIP |
2239                 DEV_RX_OFFLOAD_QINQ_STRIP |
2240                 DEV_RX_OFFLOAD_IPV4_CKSUM |
2241                 DEV_RX_OFFLOAD_UDP_CKSUM |
2242                 DEV_RX_OFFLOAD_TCP_CKSUM;
2243         dev_info->tx_offload_capa =
2244                 DEV_TX_OFFLOAD_VLAN_INSERT |
2245                 DEV_TX_OFFLOAD_QINQ_INSERT |
2246                 DEV_TX_OFFLOAD_IPV4_CKSUM |
2247                 DEV_TX_OFFLOAD_UDP_CKSUM |
2248                 DEV_TX_OFFLOAD_TCP_CKSUM |
2249                 DEV_TX_OFFLOAD_SCTP_CKSUM;
2250
2251         dev_info->default_rxconf = (struct rte_eth_rxconf) {
2252                 .rx_thresh = {
2253                         .pthresh = I40E_DEFAULT_RX_PTHRESH,
2254                         .hthresh = I40E_DEFAULT_RX_HTHRESH,
2255                         .wthresh = I40E_DEFAULT_RX_WTHRESH,
2256                 },
2257                 .rx_free_thresh = I40E_DEFAULT_RX_FREE_THRESH,
2258                 .rx_drop_en = 0,
2259         };
2260
2261         dev_info->default_txconf = (struct rte_eth_txconf) {
2262                 .tx_thresh = {
2263                         .pthresh = I40E_DEFAULT_TX_PTHRESH,
2264                         .hthresh = I40E_DEFAULT_TX_HTHRESH,
2265                         .wthresh = I40E_DEFAULT_TX_WTHRESH,
2266                 },
2267                 .tx_free_thresh = I40E_DEFAULT_TX_FREE_THRESH,
2268                 .tx_rs_thresh = I40E_DEFAULT_TX_RSBIT_THRESH,
2269                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
2270                                 ETH_TXQ_FLAGS_NOOFFLOADS,
2271         };
2272
2273         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
2274                 .nb_max = I40E_MAX_RING_DESC,
2275                 .nb_min = I40E_MIN_RING_DESC,
2276                 .nb_align = I40E_ALIGN_RING_DESC,
2277         };
2278
2279         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
2280                 .nb_max = I40E_MAX_RING_DESC,
2281                 .nb_min = I40E_MIN_RING_DESC,
2282                 .nb_align = I40E_ALIGN_RING_DESC,
2283         };
2284 }
2285
2286 static void
2287 i40evf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
2288 {
2289         if (i40evf_get_statics(dev, stats))
2290                 PMD_DRV_LOG(ERR, "Get statics failed");
2291 }
2292
2293 static void
2294 i40evf_dev_close(struct rte_eth_dev *dev)
2295 {
2296         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2297         struct rte_pci_device *pci_dev = dev->pci_dev;
2298
2299         i40evf_dev_stop(dev);
2300         hw->adapter_stopped = 1;
2301         i40e_dev_free_queues(dev);
2302         i40evf_reset_vf(hw);
2303         i40e_shutdown_adminq(hw);
2304         /* disable uio intr before callback unregister */
2305         rte_intr_disable(&pci_dev->intr_handle);
2306
2307         /* unregister callback func from eal lib */
2308         rte_intr_callback_unregister(&pci_dev->intr_handle,
2309                 i40evf_dev_interrupt_handler, (void *)dev);
2310         i40evf_disable_irq0(hw);
2311 }
2312
2313 static int
2314 i40evf_get_rss_lut(struct i40e_vsi *vsi, uint8_t *lut, uint16_t lut_size)
2315 {
2316         struct i40e_vf *vf = I40E_VSI_TO_VF(vsi);
2317         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2318         int ret;
2319
2320         if (!lut)
2321                 return -EINVAL;
2322
2323         if (vf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
2324                 ret = i40e_aq_get_rss_lut(hw, vsi->vsi_id, FALSE,
2325                                           lut, lut_size);
2326                 if (ret) {
2327                         PMD_DRV_LOG(ERR, "Failed to get RSS lookup table");
2328                         return ret;
2329                 }
2330         } else {
2331                 uint32_t *lut_dw = (uint32_t *)lut;
2332                 uint16_t i, lut_size_dw = lut_size / 4;
2333
2334                 for (i = 0; i < lut_size_dw; i++)
2335                         lut_dw[i] = I40E_READ_REG(hw, I40E_VFQF_HLUT(i));
2336         }
2337
2338         return 0;
2339 }
2340
2341 static int
2342 i40evf_set_rss_lut(struct i40e_vsi *vsi, uint8_t *lut, uint16_t lut_size)
2343 {
2344         struct i40e_vf *vf = I40E_VSI_TO_VF(vsi);
2345         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2346         int ret;
2347
2348         if (!vsi || !lut)
2349                 return -EINVAL;
2350
2351         if (vf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
2352                 ret = i40e_aq_set_rss_lut(hw, vsi->vsi_id, FALSE,
2353                                           lut, lut_size);
2354                 if (ret) {
2355                         PMD_DRV_LOG(ERR, "Failed to set RSS lookup table");
2356                         return ret;
2357                 }
2358         } else {
2359                 uint32_t *lut_dw = (uint32_t *)lut;
2360                 uint16_t i, lut_size_dw = lut_size / 4;
2361
2362                 for (i = 0; i < lut_size_dw; i++)
2363                         I40E_WRITE_REG(hw, I40E_VFQF_HLUT(i), lut_dw[i]);
2364                 I40EVF_WRITE_FLUSH(hw);
2365         }
2366
2367         return 0;
2368 }
2369
2370 static int
2371 i40evf_dev_rss_reta_update(struct rte_eth_dev *dev,
2372                            struct rte_eth_rss_reta_entry64 *reta_conf,
2373                            uint16_t reta_size)
2374 {
2375         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2376         uint8_t *lut;
2377         uint16_t i, idx, shift;
2378         int ret;
2379
2380         if (reta_size != ETH_RSS_RETA_SIZE_64) {
2381                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2382                         "(%d) doesn't match the number of hardware can "
2383                         "support (%d)\n", reta_size, ETH_RSS_RETA_SIZE_64);
2384                 return -EINVAL;
2385         }
2386
2387         lut = rte_zmalloc("i40e_rss_lut", reta_size, 0);
2388         if (!lut) {
2389                 PMD_DRV_LOG(ERR, "No memory can be allocated");
2390                 return -ENOMEM;
2391         }
2392         ret = i40evf_get_rss_lut(&vf->vsi, lut, reta_size);
2393         if (ret)
2394                 goto out;
2395         for (i = 0; i < reta_size; i++) {
2396                 idx = i / RTE_RETA_GROUP_SIZE;
2397                 shift = i % RTE_RETA_GROUP_SIZE;
2398                 if (reta_conf[idx].mask & (1ULL << shift))
2399                         lut[i] = reta_conf[idx].reta[shift];
2400         }
2401         ret = i40evf_set_rss_lut(&vf->vsi, lut, reta_size);
2402
2403 out:
2404         rte_free(lut);
2405
2406         return ret;
2407 }
2408
2409 static int
2410 i40evf_dev_rss_reta_query(struct rte_eth_dev *dev,
2411                           struct rte_eth_rss_reta_entry64 *reta_conf,
2412                           uint16_t reta_size)
2413 {
2414         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2415         uint16_t i, idx, shift;
2416         uint8_t *lut;
2417         int ret;
2418
2419         if (reta_size != ETH_RSS_RETA_SIZE_64) {
2420                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2421                         "(%d) doesn't match the number of hardware can "
2422                         "support (%d)\n", reta_size, ETH_RSS_RETA_SIZE_64);
2423                 return -EINVAL;
2424         }
2425
2426         lut = rte_zmalloc("i40e_rss_lut", reta_size, 0);
2427         if (!lut) {
2428                 PMD_DRV_LOG(ERR, "No memory can be allocated");
2429                 return -ENOMEM;
2430         }
2431
2432         ret = i40evf_get_rss_lut(&vf->vsi, lut, reta_size);
2433         if (ret)
2434                 goto out;
2435         for (i = 0; i < reta_size; i++) {
2436                 idx = i / RTE_RETA_GROUP_SIZE;
2437                 shift = i % RTE_RETA_GROUP_SIZE;
2438                 if (reta_conf[idx].mask & (1ULL << shift))
2439                         reta_conf[idx].reta[shift] = lut[i];
2440         }
2441
2442 out:
2443         rte_free(lut);
2444
2445         return ret;
2446 }
2447
2448 static int
2449 i40evf_set_rss_key(struct i40e_vsi *vsi, uint8_t *key, uint8_t key_len)
2450 {
2451         struct i40e_vf *vf = I40E_VSI_TO_VF(vsi);
2452         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2453         int ret = 0;
2454
2455         if (!key || key_len == 0) {
2456                 PMD_DRV_LOG(DEBUG, "No key to be configured");
2457                 return 0;
2458         } else if (key_len != (I40E_VFQF_HKEY_MAX_INDEX + 1) *
2459                 sizeof(uint32_t)) {
2460                 PMD_DRV_LOG(ERR, "Invalid key length %u", key_len);
2461                 return -EINVAL;
2462         }
2463
2464         if (vf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
2465                 struct i40e_aqc_get_set_rss_key_data *key_dw =
2466                         (struct i40e_aqc_get_set_rss_key_data *)key;
2467
2468                 ret = i40e_aq_set_rss_key(hw, vsi->vsi_id, key_dw);
2469                 if (ret)
2470                         PMD_INIT_LOG(ERR, "Failed to configure RSS key "
2471                                      "via AQ");
2472         } else {
2473                 uint32_t *hash_key = (uint32_t *)key;
2474                 uint16_t i;
2475
2476                 for (i = 0; i <= I40E_VFQF_HKEY_MAX_INDEX; i++)
2477                         i40e_write_rx_ctl(hw, I40E_VFQF_HKEY(i), hash_key[i]);
2478                 I40EVF_WRITE_FLUSH(hw);
2479         }
2480
2481         return ret;
2482 }
2483
2484 static int
2485 i40evf_get_rss_key(struct i40e_vsi *vsi, uint8_t *key, uint8_t *key_len)
2486 {
2487         struct i40e_vf *vf = I40E_VSI_TO_VF(vsi);
2488         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2489         int ret;
2490
2491         if (!key || !key_len)
2492                 return -EINVAL;
2493
2494         if (vf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
2495                 ret = i40e_aq_get_rss_key(hw, vsi->vsi_id,
2496                         (struct i40e_aqc_get_set_rss_key_data *)key);
2497                 if (ret) {
2498                         PMD_INIT_LOG(ERR, "Failed to get RSS key via AQ");
2499                         return ret;
2500                 }
2501         } else {
2502                 uint32_t *key_dw = (uint32_t *)key;
2503                 uint16_t i;
2504
2505                 for (i = 0; i <= I40E_VFQF_HKEY_MAX_INDEX; i++)
2506                         key_dw[i] = i40e_read_rx_ctl(hw, I40E_VFQF_HKEY(i));
2507         }
2508         *key_len = (I40E_VFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t);
2509
2510         return 0;
2511 }
2512
2513 static int
2514 i40evf_hw_rss_hash_set(struct i40e_vf *vf, struct rte_eth_rss_conf *rss_conf)
2515 {
2516         struct i40e_hw *hw = I40E_VF_TO_HW(vf);
2517         uint64_t rss_hf, hena;
2518         int ret;
2519
2520         ret = i40evf_set_rss_key(&vf->vsi, rss_conf->rss_key,
2521                                  rss_conf->rss_key_len);
2522         if (ret)
2523                 return ret;
2524
2525         rss_hf = rss_conf->rss_hf;
2526         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(0));
2527         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(1))) << 32;
2528         hena &= ~I40E_RSS_HENA_ALL;
2529         hena |= i40e_config_hena(rss_hf);
2530         i40e_write_rx_ctl(hw, I40E_VFQF_HENA(0), (uint32_t)hena);
2531         i40e_write_rx_ctl(hw, I40E_VFQF_HENA(1), (uint32_t)(hena >> 32));
2532         I40EVF_WRITE_FLUSH(hw);
2533
2534         return 0;
2535 }
2536
2537 static void
2538 i40evf_disable_rss(struct i40e_vf *vf)
2539 {
2540         struct i40e_hw *hw = I40E_VF_TO_HW(vf);
2541         uint64_t hena;
2542
2543         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(0));
2544         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(1))) << 32;
2545         hena &= ~I40E_RSS_HENA_ALL;
2546         i40e_write_rx_ctl(hw, I40E_VFQF_HENA(0), (uint32_t)hena);
2547         i40e_write_rx_ctl(hw, I40E_VFQF_HENA(1), (uint32_t)(hena >> 32));
2548         I40EVF_WRITE_FLUSH(hw);
2549 }
2550
2551 static int
2552 i40evf_config_rss(struct i40e_vf *vf)
2553 {
2554         struct i40e_hw *hw = I40E_VF_TO_HW(vf);
2555         struct rte_eth_rss_conf rss_conf;
2556         uint32_t i, j, lut = 0, nb_q = (I40E_VFQF_HLUT_MAX_INDEX + 1) * 4;
2557         uint16_t num;
2558
2559         if (vf->dev_data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
2560                 i40evf_disable_rss(vf);
2561                 PMD_DRV_LOG(DEBUG, "RSS not configured\n");
2562                 return 0;
2563         }
2564
2565         num = RTE_MIN(vf->dev_data->nb_rx_queues, I40E_MAX_QP_NUM_PER_VF);
2566         /* Fill out the look up table */
2567         for (i = 0, j = 0; i < nb_q; i++, j++) {
2568                 if (j >= num)
2569                         j = 0;
2570                 lut = (lut << 8) | j;
2571                 if ((i & 3) == 3)
2572                         I40E_WRITE_REG(hw, I40E_VFQF_HLUT(i >> 2), lut);
2573         }
2574
2575         rss_conf = vf->dev_data->dev_conf.rx_adv_conf.rss_conf;
2576         if ((rss_conf.rss_hf & I40E_RSS_OFFLOAD_ALL) == 0) {
2577                 i40evf_disable_rss(vf);
2578                 PMD_DRV_LOG(DEBUG, "No hash flag is set\n");
2579                 return 0;
2580         }
2581
2582         if (rss_conf.rss_key == NULL || rss_conf.rss_key_len <
2583                 (I40E_VFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t)) {
2584                 /* Calculate the default hash key */
2585                 for (i = 0; i <= I40E_VFQF_HKEY_MAX_INDEX; i++)
2586                         rss_key_default[i] = (uint32_t)rte_rand();
2587                 rss_conf.rss_key = (uint8_t *)rss_key_default;
2588                 rss_conf.rss_key_len = (I40E_VFQF_HKEY_MAX_INDEX + 1) *
2589                         sizeof(uint32_t);
2590         }
2591
2592         return i40evf_hw_rss_hash_set(vf, &rss_conf);
2593 }
2594
2595 static int
2596 i40evf_dev_rss_hash_update(struct rte_eth_dev *dev,
2597                            struct rte_eth_rss_conf *rss_conf)
2598 {
2599         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2600         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2601         uint64_t rss_hf = rss_conf->rss_hf & I40E_RSS_OFFLOAD_ALL;
2602         uint64_t hena;
2603
2604         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(0));
2605         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(1))) << 32;
2606         if (!(hena & I40E_RSS_HENA_ALL)) { /* RSS disabled */
2607                 if (rss_hf != 0) /* Enable RSS */
2608                         return -EINVAL;
2609                 return 0;
2610         }
2611
2612         /* RSS enabled */
2613         if (rss_hf == 0) /* Disable RSS */
2614                 return -EINVAL;
2615
2616         return i40evf_hw_rss_hash_set(vf, rss_conf);
2617 }
2618
2619 static int
2620 i40evf_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
2621                              struct rte_eth_rss_conf *rss_conf)
2622 {
2623         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2624         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2625         uint64_t hena;
2626
2627         i40evf_get_rss_key(&vf->vsi, rss_conf->rss_key,
2628                            &rss_conf->rss_key_len);
2629
2630         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(0));
2631         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(1))) << 32;
2632         rss_conf->rss_hf = i40e_parse_hena(hena);
2633
2634         return 0;
2635 }