drivers/net/mlx5/mlx5_fdir.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright 2015 6WIND S.A.
   5  *   Copyright 2015 Mellanox.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of 6WIND S.A. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stddef.h>
  35 #include <assert.h>
  36 #include <stdint.h>
  37 #include <string.h>
  38 #include <errno.h>
  39
  40 /* Verbs header. */
  41 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  42 #ifdef PEDANTIC
  43 #pragma GCC diagnostic ignored "-Wpedantic"
  44 #endif
  45 #include <infiniband/verbs.h>
  46 #ifdef PEDANTIC
  47 #pragma GCC diagnostic error "-Wpedantic"
  48 #endif
  49
  50 /* DPDK headers don't like -pedantic. */
  51 #ifdef PEDANTIC
  52 #pragma GCC diagnostic ignored "-Wpedantic"
  53 #endif
  54 #include <rte_ether.h>
  55 #include <rte_malloc.h>
  56 #include <rte_ethdev.h>
  57 #include <rte_common.h>
  58 #include <rte_flow.h>
  59 #include <rte_flow_driver.h>
  60 #ifdef PEDANTIC
  61 #pragma GCC diagnostic error "-Wpedantic"
  62 #endif
  63
  64 #include "mlx5.h"
  65 #include "mlx5_rxtx.h"
  66
  67 struct fdir_flow_desc {
  68         uint16_t dst_port;
  69         uint16_t src_port;
  70         uint32_t src_ip[4];
  71         uint32_t dst_ip[4];
  72         uint8_t mac[6];
  73         uint16_t vlan_tag;
  74         enum hash_rxq_type type;
  75 };
  76
  77 struct mlx5_fdir_filter {
  78         LIST_ENTRY(mlx5_fdir_filter) next;
  79         uint16_t queue; /* Queue assigned to if FDIR match. */
  80         enum rte_eth_fdir_behavior behavior;
  81         struct fdir_flow_desc desc;
  82         struct ibv_exp_flow *flow;
  83 };
  84
  85 LIST_HEAD(fdir_filter_list, mlx5_fdir_filter);
  86
  87 /**
  88  * Convert struct rte_eth_fdir_filter to mlx5 filter descriptor.
  89  *
  90  * @param[in] fdir_filter
  91  *   DPDK filter structure to convert.
  92  * @param[out] desc
  93  *   Resulting mlx5 filter descriptor.
  94  * @param mode
  95  *   Flow director mode.
  96  */
  97 static void
  98 fdir_filter_to_flow_desc(const struct rte_eth_fdir_filter *fdir_filter,
  99                          struct fdir_flow_desc *desc, enum rte_fdir_mode mode)
 100 {
 101         /* Initialize descriptor. */
 102         memset(desc, 0, sizeof(*desc));
 103
 104         /* Set VLAN ID. */
 105         desc->vlan_tag = fdir_filter->input.flow_ext.vlan_tci;
 106
 107         /* Set MAC address. */
 108         if (mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
 109                 rte_memcpy(desc->mac,
 110                            fdir_filter->input.flow.mac_vlan_flow.mac_addr.
 111                                 addr_bytes,
 112                            sizeof(desc->mac));
 113                 desc->type = HASH_RXQ_ETH;
 114                 return;
 115         }
 116
 117         /* Set mode */
 118         switch (fdir_filter->input.flow_type) {
 119         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
 120                 desc->type = HASH_RXQ_UDPV4;
 121                 break;
 122         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
 123                 desc->type = HASH_RXQ_TCPV4;
 124                 break;
 125         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
 126                 desc->type = HASH_RXQ_IPV4;
 127                 break;
 128         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
 129                 desc->type = HASH_RXQ_UDPV6;
 130                 break;
 131         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
 132                 desc->type = HASH_RXQ_TCPV6;
 133                 break;
 134         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
 135                 desc->type = HASH_RXQ_IPV6;
 136                 break;
 137         default:
 138                 break;
 139         }
 140
 141         /* Set flow values */
 142         switch (fdir_filter->input.flow_type) {
 143         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
 144         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
 145                 desc->src_port = fdir_filter->input.flow.udp4_flow.src_port;
 146                 desc->dst_port = fdir_filter->input.flow.udp4_flow.dst_port;
 147                 /* fallthrough */
 148         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
 149                 desc->src_ip[0] = fdir_filter->input.flow.ip4_flow.src_ip;
 150                 desc->dst_ip[0] = fdir_filter->input.flow.ip4_flow.dst_ip;
 151                 break;
 152         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
 153         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
 154                 desc->src_port = fdir_filter->input.flow.udp6_flow.src_port;
 155                 desc->dst_port = fdir_filter->input.flow.udp6_flow.dst_port;
 156                 /* Fall through. */
 157         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
 158                 rte_memcpy(desc->src_ip,
 159                            fdir_filter->input.flow.ipv6_flow.src_ip,
 160                            sizeof(desc->src_ip));
 161                 rte_memcpy(desc->dst_ip,
 162                            fdir_filter->input.flow.ipv6_flow.dst_ip,
 163                            sizeof(desc->dst_ip));
 164                 break;
 165         default:
 166                 break;
 167         }
 168 }
 169
 170 /**
 171  * Check if two flow descriptors overlap according to configured mask.
 172  *
 173  * @param priv
 174  *   Private structure that provides flow director mask.
 175  * @param desc1
 176  *   First flow descriptor to compare.
 177  * @param desc2
 178  *   Second flow descriptor to compare.
 179  *
 180  * @return
 181  *   Nonzero if descriptors overlap.
 182  */
 183 static int
 184 priv_fdir_overlap(const struct priv *priv,
 185                   const struct fdir_flow_desc *desc1,
 186                   const struct fdir_flow_desc *desc2)
 187 {
 188         const struct rte_eth_fdir_masks *mask =
 189                 &priv->dev->data->dev_conf.fdir_conf.mask;
 190         unsigned int i;
 191
 192         if (desc1->type != desc2->type)
 193                 return 0;
 194         /* Ignore non masked bits. */
 195         for (i = 0; i != RTE_DIM(desc1->mac); ++i)
 196                 if ((desc1->mac[i] & mask->mac_addr_byte_mask) !=
 197                     (desc2->mac[i] & mask->mac_addr_byte_mask))
 198                         return 0;
 199         if (((desc1->src_port & mask->src_port_mask) !=
 200              (desc2->src_port & mask->src_port_mask)) ||
 201             ((desc1->dst_port & mask->dst_port_mask) !=
 202              (desc2->dst_port & mask->dst_port_mask)))
 203                 return 0;
 204         switch (desc1->type) {
 205         case HASH_RXQ_IPV4:
 206         case HASH_RXQ_UDPV4:
 207         case HASH_RXQ_TCPV4:
 208                 if (((desc1->src_ip[0] & mask->ipv4_mask.src_ip) !=
 209                      (desc2->src_ip[0] & mask->ipv4_mask.src_ip)) ||
 210                     ((desc1->dst_ip[0] & mask->ipv4_mask.dst_ip) !=
 211                      (desc2->dst_ip[0] & mask->ipv4_mask.dst_ip)))
 212                         return 0;
 213                 break;
 214         case HASH_RXQ_IPV6:
 215         case HASH_RXQ_UDPV6:
 216         case HASH_RXQ_TCPV6:
 217                 for (i = 0; i != RTE_DIM(desc1->src_ip); ++i)
 218                         if (((desc1->src_ip[i] & mask->ipv6_mask.src_ip[i]) !=
 219                              (desc2->src_ip[i] & mask->ipv6_mask.src_ip[i])) ||
 220                             ((desc1->dst_ip[i] & mask->ipv6_mask.dst_ip[i]) !=
 221                              (desc2->dst_ip[i] & mask->ipv6_mask.dst_ip[i])))
 222                                 return 0;
 223                 break;
 224         default:
 225                 break;
 226         }
 227         return 1;
 228 }
 229
 230 /**
 231  * Create flow director steering rule for a specific filter.
 232  *
 233  * @param priv
 234  *   Private structure.
 235  * @param mlx5_fdir_filter
 236  *   Filter to create a steering rule for.
 237  * @param fdir_queue
 238  *   Flow director queue for matching packets.
 239  *
 240  * @return
 241  *   0 on success, errno value on failure.
 242  */
 243 static int
 244 priv_fdir_flow_add(struct priv *priv,
 245                    struct mlx5_fdir_filter *mlx5_fdir_filter,
 246                    struct fdir_queue *fdir_queue)
 247 {
 248         struct ibv_exp_flow *flow;
 249         struct fdir_flow_desc *desc = &mlx5_fdir_filter->desc;
 250         enum rte_fdir_mode fdir_mode =
 251                 priv->dev->data->dev_conf.fdir_conf.mode;
 252         struct rte_eth_fdir_masks *mask =
 253                 &priv->dev->data->dev_conf.fdir_conf.mask;
 254         FLOW_ATTR_SPEC_ETH(data, priv_flow_attr(priv, NULL, 0, desc->type));
 255         struct ibv_exp_flow_attr *attr = &data->attr;
 256         uintptr_t spec_offset = (uintptr_t)&data->spec;
 257         struct ibv_exp_flow_spec_eth *spec_eth;
 258         struct ibv_exp_flow_spec_ipv4 *spec_ipv4;
 259         struct ibv_exp_flow_spec_ipv6 *spec_ipv6;
 260         struct ibv_exp_flow_spec_tcp_udp *spec_tcp_udp;
 261         struct mlx5_fdir_filter *iter_fdir_filter;
 262         unsigned int i;
 263
 264         /* Abort if an existing flow overlaps this one to avoid packet
 265          * duplication, even if it targets another queue. */
 266         LIST_FOREACH(iter_fdir_filter, priv->fdir_filter_list, next)
 267                 if ((iter_fdir_filter != mlx5_fdir_filter) &&
 268                     (iter_fdir_filter->flow != NULL) &&
 269                     (priv_fdir_overlap(priv,
 270                                        &mlx5_fdir_filter->desc,
 271                                        &iter_fdir_filter->desc)))
 272                         return EEXIST;
 273
 274         /*
 275          * No padding must be inserted by the compiler between attr and spec.
 276          * This layout is expected by libibverbs.
 277          */
 278         assert(((uint8_t *)attr + sizeof(*attr)) == (uint8_t *)spec_offset);
 279         priv_flow_attr(priv, attr, sizeof(data), desc->type);
 280
 281         /* Set Ethernet spec */
 282         spec_eth = (struct ibv_exp_flow_spec_eth *)spec_offset;
 283
 284         /* The first specification must be Ethernet. */
 285         assert(spec_eth->type == IBV_EXP_FLOW_SPEC_ETH);
 286         assert(spec_eth->size == sizeof(*spec_eth));
 287
 288         /* VLAN ID */
 289         spec_eth->val.vlan_tag = desc->vlan_tag & mask->vlan_tci_mask;
 290         spec_eth->mask.vlan_tag = mask->vlan_tci_mask;
 291
 292         /* Update priority */
 293         attr->priority = 2;
 294
 295         if (fdir_mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
 296                 /* MAC Address */
 297                 for (i = 0; i != RTE_DIM(spec_eth->mask.dst_mac); ++i) {
 298                         spec_eth->val.dst_mac[i] =
 299                                 desc->mac[i] & mask->mac_addr_byte_mask;
 300                         spec_eth->mask.dst_mac[i] = mask->mac_addr_byte_mask;
 301                 }
 302                 goto create_flow;
 303         }
 304
 305         switch (desc->type) {
 306         case HASH_RXQ_IPV4:
 307         case HASH_RXQ_UDPV4:
 308         case HASH_RXQ_TCPV4:
 309                 spec_offset += spec_eth->size;
 310
 311                 /* Set IP spec */
 312                 spec_ipv4 = (struct ibv_exp_flow_spec_ipv4 *)spec_offset;
 313
 314                 /* The second specification must be IP. */
 315                 assert(spec_ipv4->type == IBV_EXP_FLOW_SPEC_IPV4);
 316                 assert(spec_ipv4->size == sizeof(*spec_ipv4));
 317
 318                 spec_ipv4->val.src_ip =
 319                         desc->src_ip[0] & mask->ipv4_mask.src_ip;
 320                 spec_ipv4->val.dst_ip =
 321                         desc->dst_ip[0] & mask->ipv4_mask.dst_ip;
 322                 spec_ipv4->mask.src_ip = mask->ipv4_mask.src_ip;
 323                 spec_ipv4->mask.dst_ip = mask->ipv4_mask.dst_ip;
 324
 325                 /* Update priority */
 326                 attr->priority = 1;
 327
 328                 if (desc->type == HASH_RXQ_IPV4)
 329                         goto create_flow;
 330
 331                 spec_offset += spec_ipv4->size;
 332                 break;
 333         case HASH_RXQ_IPV6:
 334         case HASH_RXQ_UDPV6:
 335         case HASH_RXQ_TCPV6:
 336                 spec_offset += spec_eth->size;
 337
 338                 /* Set IP spec */
 339                 spec_ipv6 = (struct ibv_exp_flow_spec_ipv6 *)spec_offset;
 340
 341                 /* The second specification must be IP. */
 342                 assert(spec_ipv6->type == IBV_EXP_FLOW_SPEC_IPV6);
 343                 assert(spec_ipv6->size == sizeof(*spec_ipv6));
 344
 345                 for (i = 0; i != RTE_DIM(desc->src_ip); ++i) {
 346                         ((uint32_t *)spec_ipv6->val.src_ip)[i] =
 347                                 desc->src_ip[i] & mask->ipv6_mask.src_ip[i];
 348                         ((uint32_t *)spec_ipv6->val.dst_ip)[i] =
 349                                 desc->dst_ip[i] & mask->ipv6_mask.dst_ip[i];
 350                 }
 351                 rte_memcpy(spec_ipv6->mask.src_ip,
 352                            mask->ipv6_mask.src_ip,
 353                            sizeof(spec_ipv6->mask.src_ip));
 354                 rte_memcpy(spec_ipv6->mask.dst_ip,
 355                            mask->ipv6_mask.dst_ip,
 356                            sizeof(spec_ipv6->mask.dst_ip));
 357
 358                 /* Update priority */
 359                 attr->priority = 1;
 360
 361                 if (desc->type == HASH_RXQ_IPV6)
 362                         goto create_flow;
 363
 364                 spec_offset += spec_ipv6->size;
 365                 break;
 366         default:
 367                 ERROR("invalid flow attribute type");
 368                 return EINVAL;
 369         }
 370
 371         /* Set TCP/UDP flow specification. */
 372         spec_tcp_udp = (struct ibv_exp_flow_spec_tcp_udp *)spec_offset;
 373
 374         /* The third specification must be TCP/UDP. */
 375         assert(spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_TCP ||
 376                spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_UDP);
 377         assert(spec_tcp_udp->size == sizeof(*spec_tcp_udp));
 378
 379         spec_tcp_udp->val.src_port = desc->src_port & mask->src_port_mask;
 380         spec_tcp_udp->val.dst_port = desc->dst_port & mask->dst_port_mask;
 381         spec_tcp_udp->mask.src_port = mask->src_port_mask;
 382         spec_tcp_udp->mask.dst_port = mask->dst_port_mask;
 383
 384         /* Update priority */
 385         attr->priority = 0;
 386
 387 create_flow:
 388
 389         errno = 0;
 390         flow = ibv_exp_create_flow(fdir_queue->qp, attr);
 391         if (flow == NULL) {
 392                 /* It's not clear whether errno is always set in this case. */
 393                 ERROR("%p: flow director configuration failed, errno=%d: %s",
 394                       (void *)priv, errno,
 395                       (errno ? strerror(errno) : "Unknown error"));
 396                 if (errno)
 397                         return errno;
 398                 return EINVAL;
 399         }
 400
 401         DEBUG("%p: added flow director rule (%p)", (void *)priv, (void *)flow);
 402         mlx5_fdir_filter->flow = flow;
 403         return 0;
 404 }
 405
 406 /**
 407  * Destroy a flow director queue.
 408  *
 409  * @param fdir_queue
 410  *   Flow director queue to be destroyed.
 411  */
 412 void
 413 priv_fdir_queue_destroy(struct priv *priv, struct fdir_queue *fdir_queue)
 414 {
 415         struct mlx5_fdir_filter *fdir_filter;
 416
 417         /* Disable filter flows still applying to this queue. */
 418         LIST_FOREACH(fdir_filter, priv->fdir_filter_list, next) {
 419                 unsigned int idx = fdir_filter->queue;
 420                 struct rxq_ctrl *rxq_ctrl =
 421                         container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq);
 422
 423                 assert(idx < priv->rxqs_n);
 424                 if (fdir_queue == rxq_ctrl->fdir_queue &&
 425                     fdir_filter->flow != NULL) {
 426                         claim_zero(ibv_exp_destroy_flow(fdir_filter->flow));
 427                         fdir_filter->flow = NULL;
 428                 }
 429         }
 430         assert(fdir_queue->qp);
 431         claim_zero(ibv_destroy_qp(fdir_queue->qp));
 432         assert(fdir_queue->ind_table);
 433         claim_zero(ibv_exp_destroy_rwq_ind_table(fdir_queue->ind_table));
 434         if (fdir_queue->wq)
 435                 claim_zero(ibv_exp_destroy_wq(fdir_queue->wq));
 436         if (fdir_queue->cq)
 437                 claim_zero(ibv_destroy_cq(fdir_queue->cq));
 438 #ifndef NDEBUG
 439         memset(fdir_queue, 0x2a, sizeof(*fdir_queue));
 440 #endif
 441         rte_free(fdir_queue);
 442 }
 443
 444 /**
 445  * Create a flow director queue.
 446  *
 447  * @param priv
 448  *   Private structure.
 449  * @param wq
 450  *   Work queue to route matched packets to, NULL if one needs to
 451  *   be created.
 452  *
 453  * @return
 454  *   Related flow director queue on success, NULL otherwise.
 455  */
 456 static struct fdir_queue *
 457 priv_fdir_queue_create(struct priv *priv, struct ibv_exp_wq *wq,
 458                        unsigned int socket)
 459 {
 460         struct fdir_queue *fdir_queue;
 461
 462         fdir_queue = rte_calloc_socket(__func__, 1, sizeof(*fdir_queue),
 463                                        0, socket);
 464         if (!fdir_queue) {
 465                 ERROR("cannot allocate flow director queue");
 466                 return NULL;
 467         }
 468         assert(priv->pd);
 469         assert(priv->ctx);
 470         if (!wq) {
 471                 fdir_queue->cq = ibv_exp_create_cq(
 472                         priv->ctx, 1, NULL, NULL, 0,
 473                         &(struct ibv_exp_cq_init_attr){
 474                                 .comp_mask = 0,
 475                         });
 476                 if (!fdir_queue->cq) {
 477                         ERROR("cannot create flow director CQ");
 478                         goto error;
 479                 }
 480                 fdir_queue->wq = ibv_exp_create_wq(
 481                         priv->ctx,
 482                         &(struct ibv_exp_wq_init_attr){
 483                                 .wq_type = IBV_EXP_WQT_RQ,
 484                                 .max_recv_wr = 1,
 485                                 .max_recv_sge = 1,
 486                                 .pd = priv->pd,
 487                                 .cq = fdir_queue->cq,
 488                         });
 489                 if (!fdir_queue->wq) {
 490                         ERROR("cannot create flow director WQ");
 491                         goto error;
 492                 }
 493                 wq = fdir_queue->wq;
 494         }
 495         fdir_queue->ind_table = ibv_exp_create_rwq_ind_table(
 496                 priv->ctx,
 497                 &(struct ibv_exp_rwq_ind_table_init_attr){
 498                         .pd = priv->pd,
 499                         .log_ind_tbl_size = 0,
 500                         .ind_tbl = &wq,
 501                         .comp_mask = 0,
 502                 });
 503         if (!fdir_queue->ind_table) {
 504                 ERROR("cannot create flow director indirection table");
 505                 goto error;
 506         }
 507         fdir_queue->qp = ibv_exp_create_qp(
 508                 priv->ctx,
 509                 &(struct ibv_exp_qp_init_attr){
 510                         .qp_type = IBV_QPT_RAW_PACKET,
 511                         .comp_mask =
 512                                 IBV_EXP_QP_INIT_ATTR_PD |
 513                                 IBV_EXP_QP_INIT_ATTR_PORT |
 514                                 IBV_EXP_QP_INIT_ATTR_RX_HASH,
 515                         .pd = priv->pd,
 516                         .rx_hash_conf = &(struct ibv_exp_rx_hash_conf){
 517                                 .rx_hash_function =
 518                                         IBV_EXP_RX_HASH_FUNC_TOEPLITZ,
 519                                 .rx_hash_key_len = rss_hash_default_key_len,
 520                                 .rx_hash_key = rss_hash_default_key,
 521                                 .rx_hash_fields_mask = 0,
 522                                 .rwq_ind_tbl = fdir_queue->ind_table,
 523                         },
 524                         .port_num = priv->port,
 525                 });
 526         if (!fdir_queue->qp) {
 527                 ERROR("cannot create flow director hash RX QP");
 528                 goto error;
 529         }
 530         return fdir_queue;
 531 error:
 532         assert(fdir_queue);
 533         assert(!fdir_queue->qp);
 534         if (fdir_queue->ind_table)
 535                 claim_zero(ibv_exp_destroy_rwq_ind_table
 536                            (fdir_queue->ind_table));
 537         if (fdir_queue->wq)
 538                 claim_zero(ibv_exp_destroy_wq(fdir_queue->wq));
 539         if (fdir_queue->cq)
 540                 claim_zero(ibv_destroy_cq(fdir_queue->cq));
 541         rte_free(fdir_queue);
 542         return NULL;
 543 }
 544
 545 /**
 546  * Get flow director queue for a specific RX queue, create it in case
 547  * it does not exist.
 548  *
 549  * @param priv
 550  *   Private structure.
 551  * @param idx
 552  *   RX queue index.
 553  *
 554  * @return
 555  *   Related flow director queue on success, NULL otherwise.
 556  */
 557 static struct fdir_queue *
 558 priv_get_fdir_queue(struct priv *priv, uint16_t idx)
 559 {
 560         struct rxq_ctrl *rxq_ctrl =
 561                 container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq);
 562         struct fdir_queue *fdir_queue = rxq_ctrl->fdir_queue;
 563
 564         assert(rxq_ctrl->wq);
 565         if (fdir_queue == NULL) {
 566                 fdir_queue = priv_fdir_queue_create(priv, rxq_ctrl->wq,
 567                                                     rxq_ctrl->socket);
 568                 rxq_ctrl->fdir_queue = fdir_queue;
 569         }
 570         return fdir_queue;
 571 }
 572
 573 /**
 574  * Get or flow director drop queue. Create it if it does not exist.
 575  *
 576  * @param priv
 577  *   Private structure.
 578  *
 579  * @return
 580  *   Flow director drop queue on success, NULL otherwise.
 581  */
 582 static struct fdir_queue *
 583 priv_get_fdir_drop_queue(struct priv *priv)
 584 {
 585         struct fdir_queue *fdir_queue = priv->fdir_drop_queue;
 586
 587         if (fdir_queue == NULL) {
 588                 unsigned int socket = SOCKET_ID_ANY;
 589
 590                 /* Select a known NUMA socket if possible. */
 591                 if (priv->rxqs_n && (*priv->rxqs)[0])
 592                         socket = container_of((*priv->rxqs)[0],
 593                                               struct rxq_ctrl, rxq)->socket;
 594                 fdir_queue = priv_fdir_queue_create(priv, NULL, socket);
 595                 priv->fdir_drop_queue = fdir_queue;
 596         }
 597         return fdir_queue;
 598 }
 599
 600 /**
 601  * Enable flow director filter and create steering rules.
 602  *
 603  * @param priv
 604  *   Private structure.
 605  * @param mlx5_fdir_filter
 606  *   Filter to create steering rule for.
 607  *
 608  * @return
 609  *   0 on success, errno value on failure.
 610  */
 611 static int
 612 priv_fdir_filter_enable(struct priv *priv,
 613                         struct mlx5_fdir_filter *mlx5_fdir_filter)
 614 {
 615         struct fdir_queue *fdir_queue;
 616
 617         /* Check if flow already exists. */
 618         if (mlx5_fdir_filter->flow != NULL)
 619                 return 0;
 620
 621         /* Get fdir_queue for specific queue. */
 622         if (mlx5_fdir_filter->behavior == RTE_ETH_FDIR_REJECT)
 623                 fdir_queue = priv_get_fdir_drop_queue(priv);
 624         else
 625                 fdir_queue = priv_get_fdir_queue(priv,
 626                                                  mlx5_fdir_filter->queue);
 627
 628         if (fdir_queue == NULL) {
 629                 ERROR("failed to create flow director rxq for queue %d",
 630                       mlx5_fdir_filter->queue);
 631                 return EINVAL;
 632         }
 633
 634         /* Create flow */
 635         return priv_fdir_flow_add(priv, mlx5_fdir_filter, fdir_queue);
 636 }
 637
 638 /**
 639  * Initialize flow director filters list.
 640  *
 641  * @param priv
 642  *   Private structure.
 643  *
 644  * @return
 645  *   0 on success, errno value on failure.
 646  */
 647 int
 648 fdir_init_filters_list(struct priv *priv)
 649 {
 650         /* Filter list initialization should be done only once. */
 651         if (priv->fdir_filter_list)
 652                 return 0;
 653
 654         /* Create filters list. */
 655         priv->fdir_filter_list =
 656                 rte_calloc(__func__, 1, sizeof(*priv->fdir_filter_list), 0);
 657
 658         if (priv->fdir_filter_list == NULL) {
 659                 int err = ENOMEM;
 660
 661                 ERROR("cannot allocate flow director filter list: %s",
 662                       strerror(err));
 663                 return err;
 664         }
 665
 666         LIST_INIT(priv->fdir_filter_list);
 667
 668         return 0;
 669 }
 670
 671 /**
 672  * Flush all filters.
 673  *
 674  * @param priv
 675  *   Private structure.
 676  */
 677 static void
 678 priv_fdir_filter_flush(struct priv *priv)
 679 {
 680         struct mlx5_fdir_filter *mlx5_fdir_filter;
 681
 682         while ((mlx5_fdir_filter = LIST_FIRST(priv->fdir_filter_list))) {
 683                 struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
 684
 685                 DEBUG("%p: flushing flow director filter %p",
 686                       (void *)priv, (void *)mlx5_fdir_filter);
 687                 LIST_REMOVE(mlx5_fdir_filter, next);
 688                 if (flow != NULL)
 689                         claim_zero(ibv_exp_destroy_flow(flow));
 690                 rte_free(mlx5_fdir_filter);
 691         }
 692 }
 693
 694 /**
 695  * Remove all flow director filters and delete list.
 696  *
 697  * @param priv
 698  *   Private structure.
 699  */
 700 void
 701 priv_fdir_delete_filters_list(struct priv *priv)
 702 {
 703         priv_fdir_filter_flush(priv);
 704         rte_free(priv->fdir_filter_list);
 705         priv->fdir_filter_list = NULL;
 706 }
 707
 708 /**
 709  * Disable flow director, remove all steering rules.
 710  *
 711  * @param priv
 712  *   Private structure.
 713  */
 714 void
 715 priv_fdir_disable(struct priv *priv)
 716 {
 717         unsigned int i;
 718         struct mlx5_fdir_filter *mlx5_fdir_filter;
 719
 720         /* Run on every flow director filter and destroy flow handle. */
 721         LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
 722                 struct ibv_exp_flow *flow;
 723
 724                 /* Only valid elements should be in the list */
 725                 assert(mlx5_fdir_filter != NULL);
 726                 flow = mlx5_fdir_filter->flow;
 727
 728                 /* Destroy flow handle */
 729                 if (flow != NULL) {
 730                         claim_zero(ibv_exp_destroy_flow(flow));
 731                         mlx5_fdir_filter->flow = NULL;
 732                 }
 733         }
 734
 735         /* Destroy flow director context in each RX queue. */
 736         for (i = 0; (i != priv->rxqs_n); i++) {
 737                 struct rxq_ctrl *rxq_ctrl;
 738
 739                 if (!(*priv->rxqs)[i])
 740                         continue;
 741                 rxq_ctrl = container_of((*priv->rxqs)[i], struct rxq_ctrl, rxq);
 742                 if (!rxq_ctrl->fdir_queue)
 743                         continue;
 744                 priv_fdir_queue_destroy(priv, rxq_ctrl->fdir_queue);
 745                 rxq_ctrl->fdir_queue = NULL;
 746         }
 747         if (priv->fdir_drop_queue) {
 748                 priv_fdir_queue_destroy(priv, priv->fdir_drop_queue);
 749                 priv->fdir_drop_queue = NULL;
 750         }
 751 }
 752
 753 /**
 754  * Enable flow director, create steering rules.
 755  *
 756  * @param priv
 757  *   Private structure.
 758  */
 759 void
 760 priv_fdir_enable(struct priv *priv)
 761 {
 762         struct mlx5_fdir_filter *mlx5_fdir_filter;
 763
 764         /* Run on every fdir filter and create flow handle */
 765         LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
 766                 /* Only valid elements should be in the list */
 767                 assert(mlx5_fdir_filter != NULL);
 768
 769                 priv_fdir_filter_enable(priv, mlx5_fdir_filter);
 770         }
 771 }
 772
 773 /**
 774  * Find specific filter in list.
 775  *
 776  * @param priv
 777  *   Private structure.
 778  * @param fdir_filter
 779  *   Flow director filter to find.
 780  *
 781  * @return
 782  *   Filter element if found, otherwise NULL.
 783  */
 784 static struct mlx5_fdir_filter *
 785 priv_find_filter_in_list(struct priv *priv,
 786                          const struct rte_eth_fdir_filter *fdir_filter)
 787 {
 788         struct fdir_flow_desc desc;
 789         struct mlx5_fdir_filter *mlx5_fdir_filter;
 790         enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode;
 791
 792         /* Get flow director filter to look for. */
 793         fdir_filter_to_flow_desc(fdir_filter, &desc, fdir_mode);
 794
 795         /* Look for the requested element. */
 796         LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
 797                 /* Only valid elements should be in the list. */
 798                 assert(mlx5_fdir_filter != NULL);
 799
 800                 /* Return matching filter. */
 801                 if (!memcmp(&desc, &mlx5_fdir_filter->desc, sizeof(desc)))
 802                         return mlx5_fdir_filter;
 803         }
 804
 805         /* Filter not found */
 806         return NULL;
 807 }
 808
 809 /**
 810  * Add new flow director filter and store it in list.
 811  *
 812  * @param priv
 813  *   Private structure.
 814  * @param fdir_filter
 815  *   Flow director filter to add.
 816  *
 817  * @return
 818  *   0 on success, errno value on failure.
 819  */
 820 static int
 821 priv_fdir_filter_add(struct priv *priv,
 822                      const struct rte_eth_fdir_filter *fdir_filter)
 823 {
 824         struct mlx5_fdir_filter *mlx5_fdir_filter;
 825         enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode;
 826         int err = 0;
 827
 828         /* Validate queue number. */
 829         if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
 830                 ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
 831                 return EINVAL;
 832         }
 833
 834         /* Duplicate filters are currently unsupported. */
 835         mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
 836         if (mlx5_fdir_filter != NULL) {
 837                 ERROR("filter already exists");
 838                 return EINVAL;
 839         }
 840
 841         /* Create new flow director filter. */
 842         mlx5_fdir_filter =
 843                 rte_calloc(__func__, 1, sizeof(*mlx5_fdir_filter), 0);
 844         if (mlx5_fdir_filter == NULL) {
 845                 err = ENOMEM;
 846                 ERROR("cannot allocate flow director filter: %s",
 847                       strerror(err));
 848                 return err;
 849         }
 850
 851         /* Set action parameters. */
 852         mlx5_fdir_filter->queue = fdir_filter->action.rx_queue;
 853         mlx5_fdir_filter->behavior = fdir_filter->action.behavior;
 854
 855         /* Convert to mlx5 filter descriptor. */
 856         fdir_filter_to_flow_desc(fdir_filter,
 857                                  &mlx5_fdir_filter->desc, fdir_mode);
 858
 859         /* Insert new filter into list. */
 860         LIST_INSERT_HEAD(priv->fdir_filter_list, mlx5_fdir_filter, next);
 861
 862         DEBUG("%p: flow director filter %p added",
 863               (void *)priv, (void *)mlx5_fdir_filter);
 864
 865         /* Enable filter immediately if device is started. */
 866         if (priv->started)
 867                 err = priv_fdir_filter_enable(priv, mlx5_fdir_filter);
 868
 869         return err;
 870 }
 871
 872 /**
 873  * Update queue for specific filter.
 874  *
 875  * @param priv
 876  *   Private structure.
 877  * @param fdir_filter
 878  *   Filter to be updated.
 879  *
 880  * @return
 881  *   0 on success, errno value on failure.
 882  */
 883 static int
 884 priv_fdir_filter_update(struct priv *priv,
 885                         const struct rte_eth_fdir_filter *fdir_filter)
 886 {
 887         struct mlx5_fdir_filter *mlx5_fdir_filter;
 888
 889         /* Validate queue number. */
 890         if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
 891                 ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
 892                 return EINVAL;
 893         }
 894
 895         mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
 896         if (mlx5_fdir_filter != NULL) {
 897                 struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
 898                 int err = 0;
 899
 900                 /* Update queue number. */
 901                 mlx5_fdir_filter->queue = fdir_filter->action.rx_queue;
 902
 903                 /* Destroy flow handle. */
 904                 if (flow != NULL) {
 905                         claim_zero(ibv_exp_destroy_flow(flow));
 906                         mlx5_fdir_filter->flow = NULL;
 907                 }
 908                 DEBUG("%p: flow director filter %p updated",
 909                       (void *)priv, (void *)mlx5_fdir_filter);
 910
 911                 /* Enable filter if device is started. */
 912                 if (priv->started)
 913                         err = priv_fdir_filter_enable(priv, mlx5_fdir_filter);
 914
 915                 return err;
 916         }
 917
 918         /* Filter not found, create it. */
 919         DEBUG("%p: filter not found for update, creating new filter",
 920               (void *)priv);
 921         return priv_fdir_filter_add(priv, fdir_filter);
 922 }
 923
 924 /**
 925  * Delete specific filter.
 926  *
 927  * @param priv
 928  *   Private structure.
 929  * @param fdir_filter
 930  *   Filter to be deleted.
 931  *
 932  * @return
 933  *   0 on success, errno value on failure.
 934  */
 935 static int
 936 priv_fdir_filter_delete(struct priv *priv,
 937                         const struct rte_eth_fdir_filter *fdir_filter)
 938 {
 939         struct mlx5_fdir_filter *mlx5_fdir_filter;
 940
 941         mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
 942         if (mlx5_fdir_filter != NULL) {
 943                 struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
 944
 945                 /* Remove element from list. */
 946                 LIST_REMOVE(mlx5_fdir_filter, next);
 947
 948                 /* Destroy flow handle. */
 949                 if (flow != NULL) {
 950                         claim_zero(ibv_exp_destroy_flow(flow));
 951                         mlx5_fdir_filter->flow = NULL;
 952                 }
 953
 954                 DEBUG("%p: flow director filter %p deleted",
 955                       (void *)priv, (void *)mlx5_fdir_filter);
 956
 957                 /* Delete filter. */
 958                 rte_free(mlx5_fdir_filter);
 959
 960                 return 0;
 961         }
 962
 963         ERROR("%p: flow director delete failed, cannot find filter",
 964               (void *)priv);
 965         return EINVAL;
 966 }
 967
 968 /**
 969  * Get flow director information.
 970  *
 971  * @param priv
 972  *   Private structure.
 973  * @param[out] fdir_info
 974  *   Resulting flow director information.
 975  */
 976 static void
 977 priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
 978 {
 979         struct rte_eth_fdir_masks *mask =
 980                 &priv->dev->data->dev_conf.fdir_conf.mask;
 981
 982         fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode;
 983         fdir_info->guarant_spc = 0;
 984
 985         rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
 986
 987         fdir_info->max_flexpayload = 0;
 988         fdir_info->flow_types_mask[0] = 0;
 989
 990         fdir_info->flex_payload_unit = 0;
 991         fdir_info->max_flex_payload_segment_num = 0;
 992         fdir_info->flex_payload_limit = 0;
 993         memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
 994 }
 995
 996 /**
 997  * Deal with flow director operations.
 998  *
 999  * @param priv
1000  *   Pointer to private structure.
1001  * @param filter_op
1002  *   Operation to perform.
1003  * @param arg
1004  *   Pointer to operation-specific structure.
1005  *
1006  * @return
1007  *   0 on success, errno value on failure.
1008  */
1009 static int
1010 priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg)
1011 {
1012         enum rte_fdir_mode fdir_mode =
1013                 priv->dev->data->dev_conf.fdir_conf.mode;
1014         int ret = 0;
1015
1016         if (filter_op == RTE_ETH_FILTER_NOP)
1017                 return 0;
1018
1019         if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
1020             fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
1021                 ERROR("%p: flow director mode %d not supported",
1022                       (void *)priv, fdir_mode);
1023                 return EINVAL;
1024         }
1025
1026         switch (filter_op) {
1027         case RTE_ETH_FILTER_ADD:
1028                 ret = priv_fdir_filter_add(priv, arg);
1029                 break;
1030         case RTE_ETH_FILTER_UPDATE:
1031                 ret = priv_fdir_filter_update(priv, arg);
1032                 break;
1033         case RTE_ETH_FILTER_DELETE:
1034                 ret = priv_fdir_filter_delete(priv, arg);
1035                 break;
1036         case RTE_ETH_FILTER_FLUSH:
1037                 priv_fdir_filter_flush(priv);
1038                 break;
1039         case RTE_ETH_FILTER_INFO:
1040                 priv_fdir_info_get(priv, arg);
1041                 break;
1042         default:
1043                 DEBUG("%p: unknown operation %u", (void *)priv, filter_op);
1044                 ret = EINVAL;
1045                 break;
1046         }
1047         return ret;
1048 }
1049
1050 static const struct rte_flow_ops mlx5_flow_ops = {
1051         .validate = mlx5_flow_validate,
1052         .create = mlx5_flow_create,
1053         .destroy = mlx5_flow_destroy,
1054         .flush = mlx5_flow_flush,
1055         .query = NULL,
1056         .isolate = mlx5_flow_isolate,
1057 };
1058
1059 /**
1060  * Manage filter operations.
1061  *
1062  * @param dev
1063  *   Pointer to Ethernet device structure.
1064  * @param filter_type
1065  *   Filter type.
1066  * @param filter_op
1067  *   Operation to perform.
1068  * @param arg
1069  *   Pointer to operation-specific structure.
1070  *
1071  * @return
1072  *   0 on success, negative errno value on failure.
1073  */
1074 int
1075 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
1076                      enum rte_filter_type filter_type,
1077                      enum rte_filter_op filter_op,
1078                      void *arg)
1079 {
1080         int ret = EINVAL;
1081         struct priv *priv = dev->data->dev_private;
1082
1083         switch (filter_type) {
1084         case RTE_ETH_FILTER_GENERIC:
1085                 if (filter_op != RTE_ETH_FILTER_GET)
1086                         return -EINVAL;
1087                 *(const void **)arg = &mlx5_flow_ops;
1088                 return 0;
1089         case RTE_ETH_FILTER_FDIR:
1090                 priv_lock(priv);
1091                 ret = priv_fdir_ctrl_func(priv, filter_op, arg);
1092                 priv_unlock(priv);
1093                 break;
1094         default:
1095                 ERROR("%p: filter type (%d) not supported",
1096                       (void *)dev, filter_type);
1097                 break;
1098         }
1099
1100         return -ret;
1101 }