drivers/net/mlx5/mlx5_fdir.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright 2015 6WIND S.A.
   5  *   Copyright 2015 Mellanox.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of 6WIND S.A. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stddef.h>
  35 #include <assert.h>
  36 #include <stdint.h>
  37 #include <string.h>
  38 #include <errno.h>
  39
  40 /* Verbs header. */
  41 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  42 #ifdef PEDANTIC
  43 #pragma GCC diagnostic ignored "-Wpedantic"
  44 #endif
  45 #include <infiniband/verbs.h>
  46 #ifdef PEDANTIC
  47 #pragma GCC diagnostic error "-Wpedantic"
  48 #endif
  49
  50 /* DPDK headers don't like -pedantic. */
  51 #ifdef PEDANTIC
  52 #pragma GCC diagnostic ignored "-Wpedantic"
  53 #endif
  54 #include <rte_ether.h>
  55 #include <rte_malloc.h>
  56 #include <rte_ethdev.h>
  57 #include <rte_common.h>
  58 #ifdef PEDANTIC
  59 #pragma GCC diagnostic error "-Wpedantic"
  60 #endif
  61
  62 #include "mlx5.h"
  63 #include "mlx5_rxtx.h"
  64
  65 struct fdir_flow_desc {
  66         uint16_t dst_port;
  67         uint16_t src_port;
  68         uint32_t src_ip[4];
  69         uint32_t dst_ip[4];
  70         uint8_t mac[6];
  71         uint16_t vlan_tag;
  72         enum hash_rxq_type type;
  73 };
  74
  75 struct mlx5_fdir_filter {
  76         LIST_ENTRY(mlx5_fdir_filter) next;
  77         uint16_t queue; /* Queue assigned to if FDIR match. */
  78         enum rte_eth_fdir_behavior behavior;
  79         struct fdir_flow_desc desc;
  80         struct ibv_exp_flow *flow;
  81 };
  82
  83 LIST_HEAD(fdir_filter_list, mlx5_fdir_filter);
  84
  85 /**
  86  * Convert struct rte_eth_fdir_filter to mlx5 filter descriptor.
  87  *
  88  * @param[in] fdir_filter
  89  *   DPDK filter structure to convert.
  90  * @param[out] desc
  91  *   Resulting mlx5 filter descriptor.
  92  * @param mode
  93  *   Flow director mode.
  94  */
  95 static void
  96 fdir_filter_to_flow_desc(const struct rte_eth_fdir_filter *fdir_filter,
  97                          struct fdir_flow_desc *desc, enum rte_fdir_mode mode)
  98 {
  99         /* Initialize descriptor. */
 100         memset(desc, 0, sizeof(*desc));
 101
 102         /* Set VLAN ID. */
 103         desc->vlan_tag = fdir_filter->input.flow_ext.vlan_tci;
 104
 105         /* Set MAC address. */
 106         if (mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
 107                 rte_memcpy(desc->mac,
 108                            fdir_filter->input.flow.mac_vlan_flow.mac_addr.
 109                                 addr_bytes,
 110                            sizeof(desc->mac));
 111                 desc->type = HASH_RXQ_ETH;
 112                 return;
 113         }
 114
 115         /* Set mode */
 116         switch (fdir_filter->input.flow_type) {
 117         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
 118                 desc->type = HASH_RXQ_UDPV4;
 119                 break;
 120         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
 121                 desc->type = HASH_RXQ_TCPV4;
 122                 break;
 123         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
 124                 desc->type = HASH_RXQ_IPV4;
 125                 break;
 126         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
 127                 desc->type = HASH_RXQ_UDPV6;
 128                 break;
 129         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
 130                 desc->type = HASH_RXQ_TCPV6;
 131                 break;
 132         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
 133                 desc->type = HASH_RXQ_IPV6;
 134                 break;
 135         default:
 136                 break;
 137         }
 138
 139         /* Set flow values */
 140         switch (fdir_filter->input.flow_type) {
 141         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
 142         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
 143                 desc->src_port = fdir_filter->input.flow.udp4_flow.src_port;
 144                 desc->dst_port = fdir_filter->input.flow.udp4_flow.dst_port;
 145                 /* fallthrough */
 146         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
 147                 desc->src_ip[0] = fdir_filter->input.flow.ip4_flow.src_ip;
 148                 desc->dst_ip[0] = fdir_filter->input.flow.ip4_flow.dst_ip;
 149                 break;
 150         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
 151         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
 152                 desc->src_port = fdir_filter->input.flow.udp6_flow.src_port;
 153                 desc->dst_port = fdir_filter->input.flow.udp6_flow.dst_port;
 154                 /* Fall through. */
 155         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
 156                 rte_memcpy(desc->src_ip,
 157                            fdir_filter->input.flow.ipv6_flow.src_ip,
 158                            sizeof(desc->src_ip));
 159                 rte_memcpy(desc->dst_ip,
 160                            fdir_filter->input.flow.ipv6_flow.dst_ip,
 161                            sizeof(desc->dst_ip));
 162                 break;
 163         default:
 164                 break;
 165         }
 166 }
 167
 168 /**
 169  * Check if two flow descriptors overlap according to configured mask.
 170  *
 171  * @param priv
 172  *   Private structure that provides flow director mask.
 173  * @param desc1
 174  *   First flow descriptor to compare.
 175  * @param desc2
 176  *   Second flow descriptor to compare.
 177  *
 178  * @return
 179  *   Nonzero if descriptors overlap.
 180  */
 181 static int
 182 priv_fdir_overlap(const struct priv *priv,
 183                   const struct fdir_flow_desc *desc1,
 184                   const struct fdir_flow_desc *desc2)
 185 {
 186         const struct rte_eth_fdir_masks *mask =
 187                 &priv->dev->data->dev_conf.fdir_conf.mask;
 188         unsigned int i;
 189
 190         if (desc1->type != desc2->type)
 191                 return 0;
 192         /* Ignore non masked bits. */
 193         for (i = 0; i != RTE_DIM(desc1->mac); ++i)
 194                 if ((desc1->mac[i] & mask->mac_addr_byte_mask) !=
 195                     (desc2->mac[i] & mask->mac_addr_byte_mask))
 196                         return 0;
 197         if (((desc1->src_port & mask->src_port_mask) !=
 198              (desc2->src_port & mask->src_port_mask)) ||
 199             ((desc1->dst_port & mask->dst_port_mask) !=
 200              (desc2->dst_port & mask->dst_port_mask)))
 201                 return 0;
 202         switch (desc1->type) {
 203         case HASH_RXQ_IPV4:
 204         case HASH_RXQ_UDPV4:
 205         case HASH_RXQ_TCPV4:
 206                 if (((desc1->src_ip[0] & mask->ipv4_mask.src_ip) !=
 207                      (desc2->src_ip[0] & mask->ipv4_mask.src_ip)) ||
 208                     ((desc1->dst_ip[0] & mask->ipv4_mask.dst_ip) !=
 209                      (desc2->dst_ip[0] & mask->ipv4_mask.dst_ip)))
 210                         return 0;
 211                 break;
 212         case HASH_RXQ_IPV6:
 213         case HASH_RXQ_UDPV6:
 214         case HASH_RXQ_TCPV6:
 215                 for (i = 0; i != RTE_DIM(desc1->src_ip); ++i)
 216                         if (((desc1->src_ip[i] & mask->ipv6_mask.src_ip[i]) !=
 217                              (desc2->src_ip[i] & mask->ipv6_mask.src_ip[i])) ||
 218                             ((desc1->dst_ip[i] & mask->ipv6_mask.dst_ip[i]) !=
 219                              (desc2->dst_ip[i] & mask->ipv6_mask.dst_ip[i])))
 220                                 return 0;
 221                 break;
 222         default:
 223                 break;
 224         }
 225         return 1;
 226 }
 227
 228 /**
 229  * Create flow director steering rule for a specific filter.
 230  *
 231  * @param priv
 232  *   Private structure.
 233  * @param mlx5_fdir_filter
 234  *   Filter to create a steering rule for.
 235  * @param fdir_queue
 236  *   Flow director queue for matching packets.
 237  *
 238  * @return
 239  *   0 on success, errno value on failure.
 240  */
 241 static int
 242 priv_fdir_flow_add(struct priv *priv,
 243                    struct mlx5_fdir_filter *mlx5_fdir_filter,
 244                    struct fdir_queue *fdir_queue)
 245 {
 246         struct ibv_exp_flow *flow;
 247         struct fdir_flow_desc *desc = &mlx5_fdir_filter->desc;
 248         enum rte_fdir_mode fdir_mode =
 249                 priv->dev->data->dev_conf.fdir_conf.mode;
 250         struct rte_eth_fdir_masks *mask =
 251                 &priv->dev->data->dev_conf.fdir_conf.mask;
 252         FLOW_ATTR_SPEC_ETH(data, priv_flow_attr(priv, NULL, 0, desc->type));
 253         struct ibv_exp_flow_attr *attr = &data->attr;
 254         uintptr_t spec_offset = (uintptr_t)&data->spec;
 255         struct ibv_exp_flow_spec_eth *spec_eth;
 256         struct ibv_exp_flow_spec_ipv4 *spec_ipv4;
 257         struct ibv_exp_flow_spec_ipv6 *spec_ipv6;
 258         struct ibv_exp_flow_spec_tcp_udp *spec_tcp_udp;
 259         struct mlx5_fdir_filter *iter_fdir_filter;
 260         unsigned int i;
 261
 262         /* Abort if an existing flow overlaps this one to avoid packet
 263          * duplication, even if it targets another queue. */
 264         LIST_FOREACH(iter_fdir_filter, priv->fdir_filter_list, next)
 265                 if ((iter_fdir_filter != mlx5_fdir_filter) &&
 266                     (iter_fdir_filter->flow != NULL) &&
 267                     (priv_fdir_overlap(priv,
 268                                        &mlx5_fdir_filter->desc,
 269                                        &iter_fdir_filter->desc)))
 270                         return EEXIST;
 271
 272         /*
 273          * No padding must be inserted by the compiler between attr and spec.
 274          * This layout is expected by libibverbs.
 275          */
 276         assert(((uint8_t *)attr + sizeof(*attr)) == (uint8_t *)spec_offset);
 277         priv_flow_attr(priv, attr, sizeof(data), desc->type);
 278
 279         /* Set Ethernet spec */
 280         spec_eth = (struct ibv_exp_flow_spec_eth *)spec_offset;
 281
 282         /* The first specification must be Ethernet. */
 283         assert(spec_eth->type == IBV_EXP_FLOW_SPEC_ETH);
 284         assert(spec_eth->size == sizeof(*spec_eth));
 285
 286         /* VLAN ID */
 287         spec_eth->val.vlan_tag = desc->vlan_tag & mask->vlan_tci_mask;
 288         spec_eth->mask.vlan_tag = mask->vlan_tci_mask;
 289
 290         /* Update priority */
 291         attr->priority = 2;
 292
 293         if (fdir_mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
 294                 /* MAC Address */
 295                 for (i = 0; i != RTE_DIM(spec_eth->mask.dst_mac); ++i) {
 296                         spec_eth->val.dst_mac[i] =
 297                                 desc->mac[i] & mask->mac_addr_byte_mask;
 298                         spec_eth->mask.dst_mac[i] = mask->mac_addr_byte_mask;
 299                 }
 300                 goto create_flow;
 301         }
 302
 303         switch (desc->type) {
 304         case HASH_RXQ_IPV4:
 305         case HASH_RXQ_UDPV4:
 306         case HASH_RXQ_TCPV4:
 307                 spec_offset += spec_eth->size;
 308
 309                 /* Set IP spec */
 310                 spec_ipv4 = (struct ibv_exp_flow_spec_ipv4 *)spec_offset;
 311
 312                 /* The second specification must be IP. */
 313                 assert(spec_ipv4->type == IBV_EXP_FLOW_SPEC_IPV4);
 314                 assert(spec_ipv4->size == sizeof(*spec_ipv4));
 315
 316                 spec_ipv4->val.src_ip =
 317                         desc->src_ip[0] & mask->ipv4_mask.src_ip;
 318                 spec_ipv4->val.dst_ip =
 319                         desc->dst_ip[0] & mask->ipv4_mask.dst_ip;
 320                 spec_ipv4->mask.src_ip = mask->ipv4_mask.src_ip;
 321                 spec_ipv4->mask.dst_ip = mask->ipv4_mask.dst_ip;
 322
 323                 /* Update priority */
 324                 attr->priority = 1;
 325
 326                 if (desc->type == HASH_RXQ_IPV4)
 327                         goto create_flow;
 328
 329                 spec_offset += spec_ipv4->size;
 330                 break;
 331         case HASH_RXQ_IPV6:
 332         case HASH_RXQ_UDPV6:
 333         case HASH_RXQ_TCPV6:
 334                 spec_offset += spec_eth->size;
 335
 336                 /* Set IP spec */
 337                 spec_ipv6 = (struct ibv_exp_flow_spec_ipv6 *)spec_offset;
 338
 339                 /* The second specification must be IP. */
 340                 assert(spec_ipv6->type == IBV_EXP_FLOW_SPEC_IPV6);
 341                 assert(spec_ipv6->size == sizeof(*spec_ipv6));
 342
 343                 for (i = 0; i != RTE_DIM(desc->src_ip); ++i) {
 344                         ((uint32_t *)spec_ipv6->val.src_ip)[i] =
 345                                 desc->src_ip[i] & mask->ipv6_mask.src_ip[i];
 346                         ((uint32_t *)spec_ipv6->val.dst_ip)[i] =
 347                                 desc->dst_ip[i] & mask->ipv6_mask.dst_ip[i];
 348                 }
 349                 rte_memcpy(spec_ipv6->mask.src_ip,
 350                            mask->ipv6_mask.src_ip,
 351                            sizeof(spec_ipv6->mask.src_ip));
 352                 rte_memcpy(spec_ipv6->mask.dst_ip,
 353                            mask->ipv6_mask.dst_ip,
 354                            sizeof(spec_ipv6->mask.dst_ip));
 355
 356                 /* Update priority */
 357                 attr->priority = 1;
 358
 359                 if (desc->type == HASH_RXQ_IPV6)
 360                         goto create_flow;
 361
 362                 spec_offset += spec_ipv6->size;
 363                 break;
 364         default:
 365                 ERROR("invalid flow attribute type");
 366                 return EINVAL;
 367         }
 368
 369         /* Set TCP/UDP flow specification. */
 370         spec_tcp_udp = (struct ibv_exp_flow_spec_tcp_udp *)spec_offset;
 371
 372         /* The third specification must be TCP/UDP. */
 373         assert(spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_TCP ||
 374                spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_UDP);
 375         assert(spec_tcp_udp->size == sizeof(*spec_tcp_udp));
 376
 377         spec_tcp_udp->val.src_port = desc->src_port & mask->src_port_mask;
 378         spec_tcp_udp->val.dst_port = desc->dst_port & mask->dst_port_mask;
 379         spec_tcp_udp->mask.src_port = mask->src_port_mask;
 380         spec_tcp_udp->mask.dst_port = mask->dst_port_mask;
 381
 382         /* Update priority */
 383         attr->priority = 0;
 384
 385 create_flow:
 386
 387         errno = 0;
 388         flow = ibv_exp_create_flow(fdir_queue->qp, attr);
 389         if (flow == NULL) {
 390                 /* It's not clear whether errno is always set in this case. */
 391                 ERROR("%p: flow director configuration failed, errno=%d: %s",
 392                       (void *)priv, errno,
 393                       (errno ? strerror(errno) : "Unknown error"));
 394                 if (errno)
 395                         return errno;
 396                 return EINVAL;
 397         }
 398
 399         DEBUG("%p: added flow director rule (%p)", (void *)priv, (void *)flow);
 400         mlx5_fdir_filter->flow = flow;
 401         return 0;
 402 }
 403
 404 /**
 405  * Destroy a flow director queue.
 406  *
 407  * @param fdir_queue
 408  *   Flow director queue to be destroyed.
 409  */
 410 void
 411 priv_fdir_queue_destroy(struct priv *priv, struct fdir_queue *fdir_queue)
 412 {
 413         struct mlx5_fdir_filter *fdir_filter;
 414
 415         /* Disable filter flows still applying to this queue. */
 416         LIST_FOREACH(fdir_filter, priv->fdir_filter_list, next) {
 417                 unsigned int idx = fdir_filter->queue;
 418                 struct rxq_ctrl *rxq_ctrl =
 419                         container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq);
 420
 421                 assert(idx < priv->rxqs_n);
 422                 if (fdir_queue == rxq_ctrl->fdir_queue &&
 423                     fdir_filter->flow != NULL) {
 424                         claim_zero(ibv_exp_destroy_flow(fdir_filter->flow));
 425                         fdir_filter->flow = NULL;
 426                 }
 427         }
 428         assert(fdir_queue->qp);
 429         claim_zero(ibv_destroy_qp(fdir_queue->qp));
 430         assert(fdir_queue->ind_table);
 431         claim_zero(ibv_exp_destroy_rwq_ind_table(fdir_queue->ind_table));
 432         if (fdir_queue->wq)
 433                 claim_zero(ibv_exp_destroy_wq(fdir_queue->wq));
 434         if (fdir_queue->cq)
 435                 claim_zero(ibv_destroy_cq(fdir_queue->cq));
 436 #ifndef NDEBUG
 437         memset(fdir_queue, 0x2a, sizeof(*fdir_queue));
 438 #endif
 439         rte_free(fdir_queue);
 440 }
 441
 442 /**
 443  * Create a flow director queue.
 444  *
 445  * @param priv
 446  *   Private structure.
 447  * @param wq
 448  *   Work queue to route matched packets to, NULL if one needs to
 449  *   be created.
 450  *
 451  * @return
 452  *   Related flow director queue on success, NULL otherwise.
 453  */
 454 static struct fdir_queue *
 455 priv_fdir_queue_create(struct priv *priv, struct ibv_exp_wq *wq,
 456                        unsigned int socket)
 457 {
 458         struct fdir_queue *fdir_queue;
 459
 460         fdir_queue = rte_calloc_socket(__func__, 1, sizeof(*fdir_queue),
 461                                        0, socket);
 462         if (!fdir_queue) {
 463                 ERROR("cannot allocate flow director queue");
 464                 return NULL;
 465         }
 466         assert(priv->pd);
 467         assert(priv->ctx);
 468         if (!wq) {
 469                 fdir_queue->cq = ibv_exp_create_cq(
 470                         priv->ctx, 1, NULL, NULL, 0,
 471                         &(struct ibv_exp_cq_init_attr){
 472                                 .comp_mask = 0,
 473                         });
 474                 if (!fdir_queue->cq) {
 475                         ERROR("cannot create flow director CQ");
 476                         goto error;
 477                 }
 478                 fdir_queue->wq = ibv_exp_create_wq(
 479                         priv->ctx,
 480                         &(struct ibv_exp_wq_init_attr){
 481                                 .wq_type = IBV_EXP_WQT_RQ,
 482                                 .max_recv_wr = 1,
 483                                 .max_recv_sge = 1,
 484                                 .pd = priv->pd,
 485                                 .cq = fdir_queue->cq,
 486                         });
 487                 if (!fdir_queue->wq) {
 488                         ERROR("cannot create flow director WQ");
 489                         goto error;
 490                 }
 491                 wq = fdir_queue->wq;
 492         }
 493         fdir_queue->ind_table = ibv_exp_create_rwq_ind_table(
 494                 priv->ctx,
 495                 &(struct ibv_exp_rwq_ind_table_init_attr){
 496                         .pd = priv->pd,
 497                         .log_ind_tbl_size = 0,
 498                         .ind_tbl = &wq,
 499                         .comp_mask = 0,
 500                 });
 501         if (!fdir_queue->ind_table) {
 502                 ERROR("cannot create flow director indirection table");
 503                 goto error;
 504         }
 505         fdir_queue->qp = ibv_exp_create_qp(
 506                 priv->ctx,
 507                 &(struct ibv_exp_qp_init_attr){
 508                         .qp_type = IBV_QPT_RAW_PACKET,
 509                         .comp_mask =
 510                                 IBV_EXP_QP_INIT_ATTR_PD |
 511                                 IBV_EXP_QP_INIT_ATTR_PORT |
 512                                 IBV_EXP_QP_INIT_ATTR_RX_HASH,
 513                         .pd = priv->pd,
 514                         .rx_hash_conf = &(struct ibv_exp_rx_hash_conf){
 515                                 .rx_hash_function =
 516                                         IBV_EXP_RX_HASH_FUNC_TOEPLITZ,
 517                                 .rx_hash_key_len = rss_hash_default_key_len,
 518                                 .rx_hash_key = rss_hash_default_key,
 519                                 .rx_hash_fields_mask = 0,
 520                                 .rwq_ind_tbl = fdir_queue->ind_table,
 521                         },
 522                         .port_num = priv->port,
 523                 });
 524         if (!fdir_queue->qp) {
 525                 ERROR("cannot create flow director hash RX QP");
 526                 goto error;
 527         }
 528         return fdir_queue;
 529 error:
 530         assert(fdir_queue);
 531         assert(!fdir_queue->qp);
 532         if (fdir_queue->ind_table)
 533                 claim_zero(ibv_exp_destroy_rwq_ind_table
 534                            (fdir_queue->ind_table));
 535         if (fdir_queue->wq)
 536                 claim_zero(ibv_exp_destroy_wq(fdir_queue->wq));
 537         if (fdir_queue->cq)
 538                 claim_zero(ibv_destroy_cq(fdir_queue->cq));
 539         rte_free(fdir_queue);
 540         return NULL;
 541 }
 542
 543 /**
 544  * Get flow director queue for a specific RX queue, create it in case
 545  * it does not exist.
 546  *
 547  * @param priv
 548  *   Private structure.
 549  * @param idx
 550  *   RX queue index.
 551  *
 552  * @return
 553  *   Related flow director queue on success, NULL otherwise.
 554  */
 555 static struct fdir_queue *
 556 priv_get_fdir_queue(struct priv *priv, uint16_t idx)
 557 {
 558         struct rxq_ctrl *rxq_ctrl =
 559                 container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq);
 560         struct fdir_queue *fdir_queue = rxq_ctrl->fdir_queue;
 561
 562         assert(rxq_ctrl->wq);
 563         if (fdir_queue == NULL) {
 564                 fdir_queue = priv_fdir_queue_create(priv, rxq_ctrl->wq,
 565                                                     rxq_ctrl->socket);
 566                 rxq_ctrl->fdir_queue = fdir_queue;
 567         }
 568         return fdir_queue;
 569 }
 570
 571 /**
 572  * Get or flow director drop queue. Create it if it does not exist.
 573  *
 574  * @param priv
 575  *   Private structure.
 576  *
 577  * @return
 578  *   Flow director drop queue on success, NULL otherwise.
 579  */
 580 static struct fdir_queue *
 581 priv_get_fdir_drop_queue(struct priv *priv)
 582 {
 583         struct fdir_queue *fdir_queue = priv->fdir_drop_queue;
 584
 585         if (fdir_queue == NULL) {
 586                 unsigned int socket = SOCKET_ID_ANY;
 587
 588                 /* Select a known NUMA socket if possible. */
 589                 if (priv->rxqs_n && (*priv->rxqs)[0])
 590                         socket = container_of((*priv->rxqs)[0],
 591                                               struct rxq_ctrl, rxq)->socket;
 592                 fdir_queue = priv_fdir_queue_create(priv, NULL, socket);
 593                 priv->fdir_drop_queue = fdir_queue;
 594         }
 595         return fdir_queue;
 596 }
 597
 598 /**
 599  * Enable flow director filter and create steering rules.
 600  *
 601  * @param priv
 602  *   Private structure.
 603  * @param mlx5_fdir_filter
 604  *   Filter to create steering rule for.
 605  *
 606  * @return
 607  *   0 on success, errno value on failure.
 608  */
 609 static int
 610 priv_fdir_filter_enable(struct priv *priv,
 611                         struct mlx5_fdir_filter *mlx5_fdir_filter)
 612 {
 613         struct fdir_queue *fdir_queue;
 614
 615         /* Check if flow already exists. */
 616         if (mlx5_fdir_filter->flow != NULL)
 617                 return 0;
 618
 619         /* Get fdir_queue for specific queue. */
 620         if (mlx5_fdir_filter->behavior == RTE_ETH_FDIR_REJECT)
 621                 fdir_queue = priv_get_fdir_drop_queue(priv);
 622         else
 623                 fdir_queue = priv_get_fdir_queue(priv,
 624                                                  mlx5_fdir_filter->queue);
 625
 626         if (fdir_queue == NULL) {
 627                 ERROR("failed to create flow director rxq for queue %d",
 628                       mlx5_fdir_filter->queue);
 629                 return EINVAL;
 630         }
 631
 632         /* Create flow */
 633         return priv_fdir_flow_add(priv, mlx5_fdir_filter, fdir_queue);
 634 }
 635
 636 /**
 637  * Initialize flow director filters list.
 638  *
 639  * @param priv
 640  *   Private structure.
 641  *
 642  * @return
 643  *   0 on success, errno value on failure.
 644  */
 645 int
 646 fdir_init_filters_list(struct priv *priv)
 647 {
 648         /* Filter list initialization should be done only once. */
 649         if (priv->fdir_filter_list)
 650                 return 0;
 651
 652         /* Create filters list. */
 653         priv->fdir_filter_list =
 654                 rte_calloc(__func__, 1, sizeof(*priv->fdir_filter_list), 0);
 655
 656         if (priv->fdir_filter_list == NULL) {
 657                 int err = ENOMEM;
 658
 659                 ERROR("cannot allocate flow director filter list: %s",
 660                       strerror(err));
 661                 return err;
 662         }
 663
 664         LIST_INIT(priv->fdir_filter_list);
 665
 666         return 0;
 667 }
 668
 669 /**
 670  * Flush all filters.
 671  *
 672  * @param priv
 673  *   Private structure.
 674  */
 675 static void
 676 priv_fdir_filter_flush(struct priv *priv)
 677 {
 678         struct mlx5_fdir_filter *mlx5_fdir_filter;
 679
 680         while ((mlx5_fdir_filter = LIST_FIRST(priv->fdir_filter_list))) {
 681                 struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
 682
 683                 DEBUG("%p: flushing flow director filter %p",
 684                       (void *)priv, (void *)mlx5_fdir_filter);
 685                 LIST_REMOVE(mlx5_fdir_filter, next);
 686                 if (flow != NULL)
 687                         claim_zero(ibv_exp_destroy_flow(flow));
 688                 rte_free(mlx5_fdir_filter);
 689         }
 690 }
 691
 692 /**
 693  * Remove all flow director filters and delete list.
 694  *
 695  * @param priv
 696  *   Private structure.
 697  */
 698 void
 699 priv_fdir_delete_filters_list(struct priv *priv)
 700 {
 701         priv_fdir_filter_flush(priv);
 702         rte_free(priv->fdir_filter_list);
 703         priv->fdir_filter_list = NULL;
 704 }
 705
 706 /**
 707  * Disable flow director, remove all steering rules.
 708  *
 709  * @param priv
 710  *   Private structure.
 711  */
 712 void
 713 priv_fdir_disable(struct priv *priv)
 714 {
 715         unsigned int i;
 716         struct mlx5_fdir_filter *mlx5_fdir_filter;
 717
 718         /* Run on every flow director filter and destroy flow handle. */
 719         LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
 720                 struct ibv_exp_flow *flow;
 721
 722                 /* Only valid elements should be in the list */
 723                 assert(mlx5_fdir_filter != NULL);
 724                 flow = mlx5_fdir_filter->flow;
 725
 726                 /* Destroy flow handle */
 727                 if (flow != NULL) {
 728                         claim_zero(ibv_exp_destroy_flow(flow));
 729                         mlx5_fdir_filter->flow = NULL;
 730                 }
 731         }
 732
 733         /* Destroy flow director context in each RX queue. */
 734         for (i = 0; (i != priv->rxqs_n); i++) {
 735                 struct rxq_ctrl *rxq_ctrl;
 736
 737                 if (!(*priv->rxqs)[i])
 738                         continue;
 739                 rxq_ctrl = container_of((*priv->rxqs)[i], struct rxq_ctrl, rxq);
 740                 if (!rxq_ctrl->fdir_queue)
 741                         continue;
 742                 priv_fdir_queue_destroy(priv, rxq_ctrl->fdir_queue);
 743                 rxq_ctrl->fdir_queue = NULL;
 744         }
 745         if (priv->fdir_drop_queue) {
 746                 priv_fdir_queue_destroy(priv, priv->fdir_drop_queue);
 747                 priv->fdir_drop_queue = NULL;
 748         }
 749 }
 750
 751 /**
 752  * Enable flow director, create steering rules.
 753  *
 754  * @param priv
 755  *   Private structure.
 756  */
 757 void
 758 priv_fdir_enable(struct priv *priv)
 759 {
 760         struct mlx5_fdir_filter *mlx5_fdir_filter;
 761
 762         /* Run on every fdir filter and create flow handle */
 763         LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
 764                 /* Only valid elements should be in the list */
 765                 assert(mlx5_fdir_filter != NULL);
 766
 767                 priv_fdir_filter_enable(priv, mlx5_fdir_filter);
 768         }
 769 }
 770
 771 /**
 772  * Find specific filter in list.
 773  *
 774  * @param priv
 775  *   Private structure.
 776  * @param fdir_filter
 777  *   Flow director filter to find.
 778  *
 779  * @return
 780  *   Filter element if found, otherwise NULL.
 781  */
 782 static struct mlx5_fdir_filter *
 783 priv_find_filter_in_list(struct priv *priv,
 784                          const struct rte_eth_fdir_filter *fdir_filter)
 785 {
 786         struct fdir_flow_desc desc;
 787         struct mlx5_fdir_filter *mlx5_fdir_filter;
 788         enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode;
 789
 790         /* Get flow director filter to look for. */
 791         fdir_filter_to_flow_desc(fdir_filter, &desc, fdir_mode);
 792
 793         /* Look for the requested element. */
 794         LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
 795                 /* Only valid elements should be in the list. */
 796                 assert(mlx5_fdir_filter != NULL);
 797
 798                 /* Return matching filter. */
 799                 if (!memcmp(&desc, &mlx5_fdir_filter->desc, sizeof(desc)))
 800                         return mlx5_fdir_filter;
 801         }
 802
 803         /* Filter not found */
 804         return NULL;
 805 }
 806
 807 /**
 808  * Add new flow director filter and store it in list.
 809  *
 810  * @param priv
 811  *   Private structure.
 812  * @param fdir_filter
 813  *   Flow director filter to add.
 814  *
 815  * @return
 816  *   0 on success, errno value on failure.
 817  */
 818 static int
 819 priv_fdir_filter_add(struct priv *priv,
 820                      const struct rte_eth_fdir_filter *fdir_filter)
 821 {
 822         struct mlx5_fdir_filter *mlx5_fdir_filter;
 823         enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode;
 824         int err = 0;
 825
 826         /* Validate queue number. */
 827         if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
 828                 ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
 829                 return EINVAL;
 830         }
 831
 832         /* Duplicate filters are currently unsupported. */
 833         mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
 834         if (mlx5_fdir_filter != NULL) {
 835                 ERROR("filter already exists");
 836                 return EINVAL;
 837         }
 838
 839         /* Create new flow director filter. */
 840         mlx5_fdir_filter =
 841                 rte_calloc(__func__, 1, sizeof(*mlx5_fdir_filter), 0);
 842         if (mlx5_fdir_filter == NULL) {
 843                 err = ENOMEM;
 844                 ERROR("cannot allocate flow director filter: %s",
 845                       strerror(err));
 846                 return err;
 847         }
 848
 849         /* Set action parameters. */
 850         mlx5_fdir_filter->queue = fdir_filter->action.rx_queue;
 851         mlx5_fdir_filter->behavior = fdir_filter->action.behavior;
 852
 853         /* Convert to mlx5 filter descriptor. */
 854         fdir_filter_to_flow_desc(fdir_filter,
 855                                  &mlx5_fdir_filter->desc, fdir_mode);
 856
 857         /* Insert new filter into list. */
 858         LIST_INSERT_HEAD(priv->fdir_filter_list, mlx5_fdir_filter, next);
 859
 860         DEBUG("%p: flow director filter %p added",
 861               (void *)priv, (void *)mlx5_fdir_filter);
 862
 863         /* Enable filter immediately if device is started. */
 864         if (priv->started)
 865                 err = priv_fdir_filter_enable(priv, mlx5_fdir_filter);
 866
 867         return err;
 868 }
 869
 870 /**
 871  * Update queue for specific filter.
 872  *
 873  * @param priv
 874  *   Private structure.
 875  * @param fdir_filter
 876  *   Filter to be updated.
 877  *
 878  * @return
 879  *   0 on success, errno value on failure.
 880  */
 881 static int
 882 priv_fdir_filter_update(struct priv *priv,
 883                         const struct rte_eth_fdir_filter *fdir_filter)
 884 {
 885         struct mlx5_fdir_filter *mlx5_fdir_filter;
 886
 887         /* Validate queue number. */
 888         if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
 889                 ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
 890                 return EINVAL;
 891         }
 892
 893         mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
 894         if (mlx5_fdir_filter != NULL) {
 895                 struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
 896                 int err = 0;
 897
 898                 /* Update queue number. */
 899                 mlx5_fdir_filter->queue = fdir_filter->action.rx_queue;
 900
 901                 /* Destroy flow handle. */
 902                 if (flow != NULL) {
 903                         claim_zero(ibv_exp_destroy_flow(flow));
 904                         mlx5_fdir_filter->flow = NULL;
 905                 }
 906                 DEBUG("%p: flow director filter %p updated",
 907                       (void *)priv, (void *)mlx5_fdir_filter);
 908
 909                 /* Enable filter if device is started. */
 910                 if (priv->started)
 911                         err = priv_fdir_filter_enable(priv, mlx5_fdir_filter);
 912
 913                 return err;
 914         }
 915
 916         /* Filter not found, create it. */
 917         DEBUG("%p: filter not found for update, creating new filter",
 918               (void *)priv);
 919         return priv_fdir_filter_add(priv, fdir_filter);
 920 }
 921
 922 /**
 923  * Delete specific filter.
 924  *
 925  * @param priv
 926  *   Private structure.
 927  * @param fdir_filter
 928  *   Filter to be deleted.
 929  *
 930  * @return
 931  *   0 on success, errno value on failure.
 932  */
 933 static int
 934 priv_fdir_filter_delete(struct priv *priv,
 935                         const struct rte_eth_fdir_filter *fdir_filter)
 936 {
 937         struct mlx5_fdir_filter *mlx5_fdir_filter;
 938
 939         mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
 940         if (mlx5_fdir_filter != NULL) {
 941                 struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
 942
 943                 /* Remove element from list. */
 944                 LIST_REMOVE(mlx5_fdir_filter, next);
 945
 946                 /* Destroy flow handle. */
 947                 if (flow != NULL) {
 948                         claim_zero(ibv_exp_destroy_flow(flow));
 949                         mlx5_fdir_filter->flow = NULL;
 950                 }
 951
 952                 DEBUG("%p: flow director filter %p deleted",
 953                       (void *)priv, (void *)mlx5_fdir_filter);
 954
 955                 /* Delete filter. */
 956                 rte_free(mlx5_fdir_filter);
 957
 958                 return 0;
 959         }
 960
 961         ERROR("%p: flow director delete failed, cannot find filter",
 962               (void *)priv);
 963         return EINVAL;
 964 }
 965
 966 /**
 967  * Get flow director information.
 968  *
 969  * @param priv
 970  *   Private structure.
 971  * @param[out] fdir_info
 972  *   Resulting flow director information.
 973  */
 974 static void
 975 priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
 976 {
 977         struct rte_eth_fdir_masks *mask =
 978                 &priv->dev->data->dev_conf.fdir_conf.mask;
 979
 980         fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode;
 981         fdir_info->guarant_spc = 0;
 982
 983         rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
 984
 985         fdir_info->max_flexpayload = 0;
 986         fdir_info->flow_types_mask[0] = 0;
 987
 988         fdir_info->flex_payload_unit = 0;
 989         fdir_info->max_flex_payload_segment_num = 0;
 990         fdir_info->flex_payload_limit = 0;
 991         memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
 992 }
 993
 994 /**
 995  * Deal with flow director operations.
 996  *
 997  * @param priv
 998  *   Pointer to private structure.
 999  * @param filter_op
1000  *   Operation to perform.
1001  * @param arg
1002  *   Pointer to operation-specific structure.
1003  *
1004  * @return
1005  *   0 on success, errno value on failure.
1006  */
1007 static int
1008 priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg)
1009 {
1010         enum rte_fdir_mode fdir_mode =
1011                 priv->dev->data->dev_conf.fdir_conf.mode;
1012         int ret = 0;
1013
1014         if (filter_op == RTE_ETH_FILTER_NOP)
1015                 return 0;
1016
1017         if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
1018             fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
1019                 ERROR("%p: flow director mode %d not supported",
1020                       (void *)priv, fdir_mode);
1021                 return EINVAL;
1022         }
1023
1024         switch (filter_op) {
1025         case RTE_ETH_FILTER_ADD:
1026                 ret = priv_fdir_filter_add(priv, arg);
1027                 break;
1028         case RTE_ETH_FILTER_UPDATE:
1029                 ret = priv_fdir_filter_update(priv, arg);
1030                 break;
1031         case RTE_ETH_FILTER_DELETE:
1032                 ret = priv_fdir_filter_delete(priv, arg);
1033                 break;
1034         case RTE_ETH_FILTER_FLUSH:
1035                 priv_fdir_filter_flush(priv);
1036                 break;
1037         case RTE_ETH_FILTER_INFO:
1038                 priv_fdir_info_get(priv, arg);
1039                 break;
1040         default:
1041                 DEBUG("%p: unknown operation %u", (void *)priv, filter_op);
1042                 ret = EINVAL;
1043                 break;
1044         }
1045         return ret;
1046 }
1047
1048 /**
1049  * Manage filter operations.
1050  *
1051  * @param dev
1052  *   Pointer to Ethernet device structure.
1053  * @param filter_type
1054  *   Filter type.
1055  * @param filter_op
1056  *   Operation to perform.
1057  * @param arg
1058  *   Pointer to operation-specific structure.
1059  *
1060  * @return
1061  *   0 on success, negative errno value on failure.
1062  */
1063 int
1064 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
1065                      enum rte_filter_type filter_type,
1066                      enum rte_filter_op filter_op,
1067                      void *arg)
1068 {
1069         int ret = EINVAL;
1070         struct priv *priv = dev->data->dev_private;
1071
1072         switch (filter_type) {
1073         case RTE_ETH_FILTER_FDIR:
1074                 priv_lock(priv);
1075                 ret = priv_fdir_ctrl_func(priv, filter_op, arg);
1076                 priv_unlock(priv);
1077                 break;
1078         default:
1079                 ERROR("%p: filter type (%d) not supported",
1080                       (void *)dev, filter_type);
1081                 break;
1082         }
1083
1084         return -ret;
1085 }