drivers/net/mlx5/mlx5_fdir.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright 2015 6WIND S.A.
   5  *   Copyright 2015 Mellanox.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of 6WIND S.A. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stddef.h>
  35 #include <assert.h>
  36 #include <stdint.h>
  37 #include <string.h>
  38 #include <errno.h>
  39
  40 /* Verbs header. */
  41 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  42 #ifdef PEDANTIC
  43 #pragma GCC diagnostic ignored "-Wpedantic"
  44 #endif
  45 #include <infiniband/verbs.h>
  46 #ifdef PEDANTIC
  47 #pragma GCC diagnostic error "-Wpedantic"
  48 #endif
  49
  50 /* DPDK headers don't like -pedantic. */
  51 #ifdef PEDANTIC
  52 #pragma GCC diagnostic ignored "-Wpedantic"
  53 #endif
  54 #include <rte_ether.h>
  55 #include <rte_malloc.h>
  56 #include <rte_ethdev.h>
  57 #include <rte_common.h>
  58 #include <rte_flow.h>
  59 #include <rte_flow_driver.h>
  60 #ifdef PEDANTIC
  61 #pragma GCC diagnostic error "-Wpedantic"
  62 #endif
  63
  64 #include "mlx5.h"
  65 #include "mlx5_rxtx.h"
  66
  67 struct fdir_flow_desc {
  68         uint16_t dst_port;
  69         uint16_t src_port;
  70         uint32_t src_ip[4];
  71         uint32_t dst_ip[4];
  72         uint8_t mac[6];
  73         uint16_t vlan_tag;
  74         enum hash_rxq_type type;
  75 };
  76
  77 struct mlx5_fdir_filter {
  78         LIST_ENTRY(mlx5_fdir_filter) next;
  79         uint16_t queue; /* Queue assigned to if FDIR match. */
  80         enum rte_eth_fdir_behavior behavior;
  81         struct fdir_flow_desc desc;
  82         struct ibv_exp_flow *flow;
  83 };
  84
  85 LIST_HEAD(fdir_filter_list, mlx5_fdir_filter);
  86
  87 /**
  88  * Convert struct rte_eth_fdir_filter to mlx5 filter descriptor.
  89  *
  90  * @param[in] fdir_filter
  91  *   DPDK filter structure to convert.
  92  * @param[out] desc
  93  *   Resulting mlx5 filter descriptor.
  94  * @param mode
  95  *   Flow director mode.
  96  */
  97 static void
  98 fdir_filter_to_flow_desc(const struct rte_eth_fdir_filter *fdir_filter,
  99                          struct fdir_flow_desc *desc, enum rte_fdir_mode mode)
 100 {
 101         /* Initialize descriptor. */
 102         memset(desc, 0, sizeof(*desc));
 103
 104         /* Set VLAN ID. */
 105         desc->vlan_tag = fdir_filter->input.flow_ext.vlan_tci;
 106
 107         /* Set MAC address. */
 108         if (mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
 109                 rte_memcpy(desc->mac,
 110                            fdir_filter->input.flow.mac_vlan_flow.mac_addr.
 111                                 addr_bytes,
 112                            sizeof(desc->mac));
 113                 desc->type = HASH_RXQ_ETH;
 114                 return;
 115         }
 116
 117         /* Set mode */
 118         switch (fdir_filter->input.flow_type) {
 119         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
 120                 desc->type = HASH_RXQ_UDPV4;
 121                 break;
 122         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
 123                 desc->type = HASH_RXQ_TCPV4;
 124                 break;
 125         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
 126                 desc->type = HASH_RXQ_IPV4;
 127                 break;
 128         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
 129                 desc->type = HASH_RXQ_UDPV6;
 130                 break;
 131         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
 132                 desc->type = HASH_RXQ_TCPV6;
 133                 break;
 134         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
 135                 desc->type = HASH_RXQ_IPV6;
 136                 break;
 137         default:
 138                 break;
 139         }
 140
 141         /* Set flow values */
 142         switch (fdir_filter->input.flow_type) {
 143         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
 144         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
 145                 desc->src_port = fdir_filter->input.flow.udp4_flow.src_port;
 146                 desc->dst_port = fdir_filter->input.flow.udp4_flow.dst_port;
 147         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
 148                 desc->src_ip[0] = fdir_filter->input.flow.ip4_flow.src_ip;
 149                 desc->dst_ip[0] = fdir_filter->input.flow.ip4_flow.dst_ip;
 150                 break;
 151         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
 152         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
 153                 desc->src_port = fdir_filter->input.flow.udp6_flow.src_port;
 154                 desc->dst_port = fdir_filter->input.flow.udp6_flow.dst_port;
 155                 /* Fall through. */
 156         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
 157                 rte_memcpy(desc->src_ip,
 158                            fdir_filter->input.flow.ipv6_flow.src_ip,
 159                            sizeof(desc->src_ip));
 160                 rte_memcpy(desc->dst_ip,
 161                            fdir_filter->input.flow.ipv6_flow.dst_ip,
 162                            sizeof(desc->dst_ip));
 163                 break;
 164         default:
 165                 break;
 166         }
 167 }
 168
 169 /**
 170  * Check if two flow descriptors overlap according to configured mask.
 171  *
 172  * @param priv
 173  *   Private structure that provides flow director mask.
 174  * @param desc1
 175  *   First flow descriptor to compare.
 176  * @param desc2
 177  *   Second flow descriptor to compare.
 178  *
 179  * @return
 180  *   Nonzero if descriptors overlap.
 181  */
 182 static int
 183 priv_fdir_overlap(const struct priv *priv,
 184                   const struct fdir_flow_desc *desc1,
 185                   const struct fdir_flow_desc *desc2)
 186 {
 187         const struct rte_eth_fdir_masks *mask =
 188                 &priv->dev->data->dev_conf.fdir_conf.mask;
 189         unsigned int i;
 190
 191         if (desc1->type != desc2->type)
 192                 return 0;
 193         /* Ignore non masked bits. */
 194         for (i = 0; i != RTE_DIM(desc1->mac); ++i)
 195                 if ((desc1->mac[i] & mask->mac_addr_byte_mask) !=
 196                     (desc2->mac[i] & mask->mac_addr_byte_mask))
 197                         return 0;
 198         if (((desc1->src_port & mask->src_port_mask) !=
 199              (desc2->src_port & mask->src_port_mask)) ||
 200             ((desc1->dst_port & mask->dst_port_mask) !=
 201              (desc2->dst_port & mask->dst_port_mask)))
 202                 return 0;
 203         switch (desc1->type) {
 204         case HASH_RXQ_IPV4:
 205         case HASH_RXQ_UDPV4:
 206         case HASH_RXQ_TCPV4:
 207                 if (((desc1->src_ip[0] & mask->ipv4_mask.src_ip) !=
 208                      (desc2->src_ip[0] & mask->ipv4_mask.src_ip)) ||
 209                     ((desc1->dst_ip[0] & mask->ipv4_mask.dst_ip) !=
 210                      (desc2->dst_ip[0] & mask->ipv4_mask.dst_ip)))
 211                         return 0;
 212                 break;
 213         case HASH_RXQ_IPV6:
 214         case HASH_RXQ_UDPV6:
 215         case HASH_RXQ_TCPV6:
 216                 for (i = 0; i != RTE_DIM(desc1->src_ip); ++i)
 217                         if (((desc1->src_ip[i] & mask->ipv6_mask.src_ip[i]) !=
 218                              (desc2->src_ip[i] & mask->ipv6_mask.src_ip[i])) ||
 219                             ((desc1->dst_ip[i] & mask->ipv6_mask.dst_ip[i]) !=
 220                              (desc2->dst_ip[i] & mask->ipv6_mask.dst_ip[i])))
 221                                 return 0;
 222                 break;
 223         default:
 224                 break;
 225         }
 226         return 1;
 227 }
 228
 229 /**
 230  * Create flow director steering rule for a specific filter.
 231  *
 232  * @param priv
 233  *   Private structure.
 234  * @param mlx5_fdir_filter
 235  *   Filter to create a steering rule for.
 236  * @param fdir_queue
 237  *   Flow director queue for matching packets.
 238  *
 239  * @return
 240  *   0 on success, errno value on failure.
 241  */
 242 static int
 243 priv_fdir_flow_add(struct priv *priv,
 244                    struct mlx5_fdir_filter *mlx5_fdir_filter,
 245                    struct fdir_queue *fdir_queue)
 246 {
 247         struct ibv_exp_flow *flow;
 248         struct fdir_flow_desc *desc = &mlx5_fdir_filter->desc;
 249         enum rte_fdir_mode fdir_mode =
 250                 priv->dev->data->dev_conf.fdir_conf.mode;
 251         struct rte_eth_fdir_masks *mask =
 252                 &priv->dev->data->dev_conf.fdir_conf.mask;
 253         FLOW_ATTR_SPEC_ETH(data, priv_flow_attr(priv, NULL, 0, desc->type));
 254         struct ibv_exp_flow_attr *attr = &data->attr;
 255         uintptr_t spec_offset = (uintptr_t)&data->spec;
 256         struct ibv_exp_flow_spec_eth *spec_eth;
 257         struct ibv_exp_flow_spec_ipv4 *spec_ipv4;
 258         struct ibv_exp_flow_spec_ipv6 *spec_ipv6;
 259         struct ibv_exp_flow_spec_tcp_udp *spec_tcp_udp;
 260         struct mlx5_fdir_filter *iter_fdir_filter;
 261         unsigned int i;
 262
 263         /* Abort if an existing flow overlaps this one to avoid packet
 264          * duplication, even if it targets another queue. */
 265         LIST_FOREACH(iter_fdir_filter, priv->fdir_filter_list, next)
 266                 if ((iter_fdir_filter != mlx5_fdir_filter) &&
 267                     (iter_fdir_filter->flow != NULL) &&
 268                     (priv_fdir_overlap(priv,
 269                                        &mlx5_fdir_filter->desc,
 270                                        &iter_fdir_filter->desc)))
 271                         return EEXIST;
 272
 273         /*
 274          * No padding must be inserted by the compiler between attr and spec.
 275          * This layout is expected by libibverbs.
 276          */
 277         assert(((uint8_t *)attr + sizeof(*attr)) == (uint8_t *)spec_offset);
 278         priv_flow_attr(priv, attr, sizeof(data), desc->type);
 279
 280         /* Set Ethernet spec */
 281         spec_eth = (struct ibv_exp_flow_spec_eth *)spec_offset;
 282
 283         /* The first specification must be Ethernet. */
 284         assert(spec_eth->type == IBV_EXP_FLOW_SPEC_ETH);
 285         assert(spec_eth->size == sizeof(*spec_eth));
 286
 287         /* VLAN ID */
 288         spec_eth->val.vlan_tag = desc->vlan_tag & mask->vlan_tci_mask;
 289         spec_eth->mask.vlan_tag = mask->vlan_tci_mask;
 290
 291         /* Update priority */
 292         attr->priority = 2;
 293
 294         if (fdir_mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
 295                 /* MAC Address */
 296                 for (i = 0; i != RTE_DIM(spec_eth->mask.dst_mac); ++i) {
 297                         spec_eth->val.dst_mac[i] =
 298                                 desc->mac[i] & mask->mac_addr_byte_mask;
 299                         spec_eth->mask.dst_mac[i] = mask->mac_addr_byte_mask;
 300                 }
 301                 goto create_flow;
 302         }
 303
 304         switch (desc->type) {
 305         case HASH_RXQ_IPV4:
 306         case HASH_RXQ_UDPV4:
 307         case HASH_RXQ_TCPV4:
 308                 spec_offset += spec_eth->size;
 309
 310                 /* Set IP spec */
 311                 spec_ipv4 = (struct ibv_exp_flow_spec_ipv4 *)spec_offset;
 312
 313                 /* The second specification must be IP. */
 314                 assert(spec_ipv4->type == IBV_EXP_FLOW_SPEC_IPV4);
 315                 assert(spec_ipv4->size == sizeof(*spec_ipv4));
 316
 317                 spec_ipv4->val.src_ip =
 318                         desc->src_ip[0] & mask->ipv4_mask.src_ip;
 319                 spec_ipv4->val.dst_ip =
 320                         desc->dst_ip[0] & mask->ipv4_mask.dst_ip;
 321                 spec_ipv4->mask.src_ip = mask->ipv4_mask.src_ip;
 322                 spec_ipv4->mask.dst_ip = mask->ipv4_mask.dst_ip;
 323
 324                 /* Update priority */
 325                 attr->priority = 1;
 326
 327                 if (desc->type == HASH_RXQ_IPV4)
 328                         goto create_flow;
 329
 330                 spec_offset += spec_ipv4->size;
 331                 break;
 332         case HASH_RXQ_IPV6:
 333         case HASH_RXQ_UDPV6:
 334         case HASH_RXQ_TCPV6:
 335                 spec_offset += spec_eth->size;
 336
 337                 /* Set IP spec */
 338                 spec_ipv6 = (struct ibv_exp_flow_spec_ipv6 *)spec_offset;
 339
 340                 /* The second specification must be IP. */
 341                 assert(spec_ipv6->type == IBV_EXP_FLOW_SPEC_IPV6);
 342                 assert(spec_ipv6->size == sizeof(*spec_ipv6));
 343
 344                 for (i = 0; i != RTE_DIM(desc->src_ip); ++i) {
 345                         ((uint32_t *)spec_ipv6->val.src_ip)[i] =
 346                                 desc->src_ip[i] & mask->ipv6_mask.src_ip[i];
 347                         ((uint32_t *)spec_ipv6->val.dst_ip)[i] =
 348                                 desc->dst_ip[i] & mask->ipv6_mask.dst_ip[i];
 349                 }
 350                 rte_memcpy(spec_ipv6->mask.src_ip,
 351                            mask->ipv6_mask.src_ip,
 352                            sizeof(spec_ipv6->mask.src_ip));
 353                 rte_memcpy(spec_ipv6->mask.dst_ip,
 354                            mask->ipv6_mask.dst_ip,
 355                            sizeof(spec_ipv6->mask.dst_ip));
 356
 357                 /* Update priority */
 358                 attr->priority = 1;
 359
 360                 if (desc->type == HASH_RXQ_IPV6)
 361                         goto create_flow;
 362
 363                 spec_offset += spec_ipv6->size;
 364                 break;
 365         default:
 366                 ERROR("invalid flow attribute type");
 367                 return EINVAL;
 368         }
 369
 370         /* Set TCP/UDP flow specification. */
 371         spec_tcp_udp = (struct ibv_exp_flow_spec_tcp_udp *)spec_offset;
 372
 373         /* The third specification must be TCP/UDP. */
 374         assert(spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_TCP ||
 375                spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_UDP);
 376         assert(spec_tcp_udp->size == sizeof(*spec_tcp_udp));
 377
 378         spec_tcp_udp->val.src_port = desc->src_port & mask->src_port_mask;
 379         spec_tcp_udp->val.dst_port = desc->dst_port & mask->dst_port_mask;
 380         spec_tcp_udp->mask.src_port = mask->src_port_mask;
 381         spec_tcp_udp->mask.dst_port = mask->dst_port_mask;
 382
 383         /* Update priority */
 384         attr->priority = 0;
 385
 386 create_flow:
 387
 388         errno = 0;
 389         flow = ibv_exp_create_flow(fdir_queue->qp, attr);
 390         if (flow == NULL) {
 391                 /* It's not clear whether errno is always set in this case. */
 392                 ERROR("%p: flow director configuration failed, errno=%d: %s",
 393                       (void *)priv, errno,
 394                       (errno ? strerror(errno) : "Unknown error"));
 395                 if (errno)
 396                         return errno;
 397                 return EINVAL;
 398         }
 399
 400         DEBUG("%p: added flow director rule (%p)", (void *)priv, (void *)flow);
 401         mlx5_fdir_filter->flow = flow;
 402         return 0;
 403 }
 404
 405 /**
 406  * Destroy a flow director queue.
 407  *
 408  * @param fdir_queue
 409  *   Flow director queue to be destroyed.
 410  */
 411 void
 412 priv_fdir_queue_destroy(struct priv *priv, struct fdir_queue *fdir_queue)
 413 {
 414         struct mlx5_fdir_filter *fdir_filter;
 415
 416         /* Disable filter flows still applying to this queue. */
 417         LIST_FOREACH(fdir_filter, priv->fdir_filter_list, next) {
 418                 unsigned int idx = fdir_filter->queue;
 419                 struct rxq_ctrl *rxq_ctrl =
 420                         container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq);
 421
 422                 assert(idx < priv->rxqs_n);
 423                 if (fdir_queue == rxq_ctrl->fdir_queue &&
 424                     fdir_filter->flow != NULL) {
 425                         claim_zero(ibv_exp_destroy_flow(fdir_filter->flow));
 426                         fdir_filter->flow = NULL;
 427                 }
 428         }
 429         assert(fdir_queue->qp);
 430         claim_zero(ibv_destroy_qp(fdir_queue->qp));
 431         assert(fdir_queue->ind_table);
 432         claim_zero(ibv_exp_destroy_rwq_ind_table(fdir_queue->ind_table));
 433         if (fdir_queue->wq)
 434                 claim_zero(ibv_exp_destroy_wq(fdir_queue->wq));
 435         if (fdir_queue->cq)
 436                 claim_zero(ibv_destroy_cq(fdir_queue->cq));
 437 #ifndef NDEBUG
 438         memset(fdir_queue, 0x2a, sizeof(*fdir_queue));
 439 #endif
 440         rte_free(fdir_queue);
 441 }
 442
 443 /**
 444  * Create a flow director queue.
 445  *
 446  * @param priv
 447  *   Private structure.
 448  * @param wq
 449  *   Work queue to route matched packets to, NULL if one needs to
 450  *   be created.
 451  *
 452  * @return
 453  *   Related flow director queue on success, NULL otherwise.
 454  */
 455 static struct fdir_queue *
 456 priv_fdir_queue_create(struct priv *priv, struct ibv_exp_wq *wq,
 457                        unsigned int socket)
 458 {
 459         struct fdir_queue *fdir_queue;
 460
 461         fdir_queue = rte_calloc_socket(__func__, 1, sizeof(*fdir_queue),
 462                                        0, socket);
 463         if (!fdir_queue) {
 464                 ERROR("cannot allocate flow director queue");
 465                 return NULL;
 466         }
 467         assert(priv->pd);
 468         assert(priv->ctx);
 469         if (!wq) {
 470                 fdir_queue->cq = ibv_exp_create_cq(
 471                         priv->ctx, 1, NULL, NULL, 0,
 472                         &(struct ibv_exp_cq_init_attr){
 473                                 .comp_mask = 0,
 474                         });
 475                 if (!fdir_queue->cq) {
 476                         ERROR("cannot create flow director CQ");
 477                         goto error;
 478                 }
 479                 fdir_queue->wq = ibv_exp_create_wq(
 480                         priv->ctx,
 481                         &(struct ibv_exp_wq_init_attr){
 482                                 .wq_type = IBV_EXP_WQT_RQ,
 483                                 .max_recv_wr = 1,
 484                                 .max_recv_sge = 1,
 485                                 .pd = priv->pd,
 486                                 .cq = fdir_queue->cq,
 487                         });
 488                 if (!fdir_queue->wq) {
 489                         ERROR("cannot create flow director WQ");
 490                         goto error;
 491                 }
 492                 wq = fdir_queue->wq;
 493         }
 494         fdir_queue->ind_table = ibv_exp_create_rwq_ind_table(
 495                 priv->ctx,
 496                 &(struct ibv_exp_rwq_ind_table_init_attr){
 497                         .pd = priv->pd,
 498                         .log_ind_tbl_size = 0,
 499                         .ind_tbl = &wq,
 500                         .comp_mask = 0,
 501                 });
 502         if (!fdir_queue->ind_table) {
 503                 ERROR("cannot create flow director indirection table");
 504                 goto error;
 505         }
 506         fdir_queue->qp = ibv_exp_create_qp(
 507                 priv->ctx,
 508                 &(struct ibv_exp_qp_init_attr){
 509                         .qp_type = IBV_QPT_RAW_PACKET,
 510                         .comp_mask =
 511                                 IBV_EXP_QP_INIT_ATTR_PD |
 512                                 IBV_EXP_QP_INIT_ATTR_PORT |
 513                                 IBV_EXP_QP_INIT_ATTR_RX_HASH,
 514                         .pd = priv->pd,
 515                         .rx_hash_conf = &(struct ibv_exp_rx_hash_conf){
 516                                 .rx_hash_function =
 517                                         IBV_EXP_RX_HASH_FUNC_TOEPLITZ,
 518                                 .rx_hash_key_len = rss_hash_default_key_len,
 519                                 .rx_hash_key = rss_hash_default_key,
 520                                 .rx_hash_fields_mask = 0,
 521                                 .rwq_ind_tbl = fdir_queue->ind_table,
 522                         },
 523                         .port_num = priv->port,
 524                 });
 525         if (!fdir_queue->qp) {
 526                 ERROR("cannot create flow director hash RX QP");
 527                 goto error;
 528         }
 529         return fdir_queue;
 530 error:
 531         assert(fdir_queue);
 532         assert(!fdir_queue->qp);
 533         if (fdir_queue->ind_table)
 534                 claim_zero(ibv_exp_destroy_rwq_ind_table
 535                            (fdir_queue->ind_table));
 536         if (fdir_queue->wq)
 537                 claim_zero(ibv_exp_destroy_wq(fdir_queue->wq));
 538         if (fdir_queue->cq)
 539                 claim_zero(ibv_destroy_cq(fdir_queue->cq));
 540         rte_free(fdir_queue);
 541         return NULL;
 542 }
 543
 544 /**
 545  * Get flow director queue for a specific RX queue, create it in case
 546  * it does not exist.
 547  *
 548  * @param priv
 549  *   Private structure.
 550  * @param idx
 551  *   RX queue index.
 552  *
 553  * @return
 554  *   Related flow director queue on success, NULL otherwise.
 555  */
 556 static struct fdir_queue *
 557 priv_get_fdir_queue(struct priv *priv, uint16_t idx)
 558 {
 559         struct rxq_ctrl *rxq_ctrl =
 560                 container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq);
 561         struct fdir_queue *fdir_queue = rxq_ctrl->fdir_queue;
 562
 563         assert(rxq_ctrl->wq);
 564         if (fdir_queue == NULL) {
 565                 fdir_queue = priv_fdir_queue_create(priv, rxq_ctrl->wq,
 566                                                     rxq_ctrl->socket);
 567                 rxq_ctrl->fdir_queue = fdir_queue;
 568         }
 569         return fdir_queue;
 570 }
 571
 572 /**
 573  * Get or flow director drop queue. Create it if it does not exist.
 574  *
 575  * @param priv
 576  *   Private structure.
 577  *
 578  * @return
 579  *   Flow director drop queue on success, NULL otherwise.
 580  */
 581 static struct fdir_queue *
 582 priv_get_fdir_drop_queue(struct priv *priv)
 583 {
 584         struct fdir_queue *fdir_queue = priv->fdir_drop_queue;
 585
 586         if (fdir_queue == NULL) {
 587                 unsigned int socket = SOCKET_ID_ANY;
 588
 589                 /* Select a known NUMA socket if possible. */
 590                 if (priv->rxqs_n && (*priv->rxqs)[0])
 591                         socket = container_of((*priv->rxqs)[0],
 592                                               struct rxq_ctrl, rxq)->socket;
 593                 fdir_queue = priv_fdir_queue_create(priv, NULL, socket);
 594                 priv->fdir_drop_queue = fdir_queue;
 595         }
 596         return fdir_queue;
 597 }
 598
 599 /**
 600  * Enable flow director filter and create steering rules.
 601  *
 602  * @param priv
 603  *   Private structure.
 604  * @param mlx5_fdir_filter
 605  *   Filter to create steering rule for.
 606  *
 607  * @return
 608  *   0 on success, errno value on failure.
 609  */
 610 static int
 611 priv_fdir_filter_enable(struct priv *priv,
 612                         struct mlx5_fdir_filter *mlx5_fdir_filter)
 613 {
 614         struct fdir_queue *fdir_queue;
 615
 616         /* Check if flow already exists. */
 617         if (mlx5_fdir_filter->flow != NULL)
 618                 return 0;
 619
 620         /* Get fdir_queue for specific queue. */
 621         if (mlx5_fdir_filter->behavior == RTE_ETH_FDIR_REJECT)
 622                 fdir_queue = priv_get_fdir_drop_queue(priv);
 623         else
 624                 fdir_queue = priv_get_fdir_queue(priv,
 625                                                  mlx5_fdir_filter->queue);
 626
 627         if (fdir_queue == NULL) {
 628                 ERROR("failed to create flow director rxq for queue %d",
 629                       mlx5_fdir_filter->queue);
 630                 return EINVAL;
 631         }
 632
 633         /* Create flow */
 634         return priv_fdir_flow_add(priv, mlx5_fdir_filter, fdir_queue);
 635 }
 636
 637 /**
 638  * Initialize flow director filters list.
 639  *
 640  * @param priv
 641  *   Private structure.
 642  *
 643  * @return
 644  *   0 on success, errno value on failure.
 645  */
 646 int
 647 fdir_init_filters_list(struct priv *priv)
 648 {
 649         /* Filter list initialization should be done only once. */
 650         if (priv->fdir_filter_list)
 651                 return 0;
 652
 653         /* Create filters list. */
 654         priv->fdir_filter_list =
 655                 rte_calloc(__func__, 1, sizeof(*priv->fdir_filter_list), 0);
 656
 657         if (priv->fdir_filter_list == NULL) {
 658                 int err = ENOMEM;
 659
 660                 ERROR("cannot allocate flow director filter list: %s",
 661                       strerror(err));
 662                 return err;
 663         }
 664
 665         LIST_INIT(priv->fdir_filter_list);
 666
 667         return 0;
 668 }
 669
 670 /**
 671  * Flush all filters.
 672  *
 673  * @param priv
 674  *   Private structure.
 675  */
 676 static void
 677 priv_fdir_filter_flush(struct priv *priv)
 678 {
 679         struct mlx5_fdir_filter *mlx5_fdir_filter;
 680
 681         while ((mlx5_fdir_filter = LIST_FIRST(priv->fdir_filter_list))) {
 682                 struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
 683
 684                 DEBUG("%p: flushing flow director filter %p",
 685                       (void *)priv, (void *)mlx5_fdir_filter);
 686                 LIST_REMOVE(mlx5_fdir_filter, next);
 687                 if (flow != NULL)
 688                         claim_zero(ibv_exp_destroy_flow(flow));
 689                 rte_free(mlx5_fdir_filter);
 690         }
 691 }
 692
 693 /**
 694  * Remove all flow director filters and delete list.
 695  *
 696  * @param priv
 697  *   Private structure.
 698  */
 699 void
 700 priv_fdir_delete_filters_list(struct priv *priv)
 701 {
 702         priv_fdir_filter_flush(priv);
 703         rte_free(priv->fdir_filter_list);
 704         priv->fdir_filter_list = NULL;
 705 }
 706
 707 /**
 708  * Disable flow director, remove all steering rules.
 709  *
 710  * @param priv
 711  *   Private structure.
 712  */
 713 void
 714 priv_fdir_disable(struct priv *priv)
 715 {
 716         unsigned int i;
 717         struct mlx5_fdir_filter *mlx5_fdir_filter;
 718
 719         /* Run on every flow director filter and destroy flow handle. */
 720         LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
 721                 struct ibv_exp_flow *flow;
 722
 723                 /* Only valid elements should be in the list */
 724                 assert(mlx5_fdir_filter != NULL);
 725                 flow = mlx5_fdir_filter->flow;
 726
 727                 /* Destroy flow handle */
 728                 if (flow != NULL) {
 729                         claim_zero(ibv_exp_destroy_flow(flow));
 730                         mlx5_fdir_filter->flow = NULL;
 731                 }
 732         }
 733
 734         /* Destroy flow director context in each RX queue. */
 735         for (i = 0; (i != priv->rxqs_n); i++) {
 736                 struct rxq_ctrl *rxq_ctrl =
 737                         container_of((*priv->rxqs)[i], struct rxq_ctrl, rxq);
 738
 739                 if (!rxq_ctrl->fdir_queue)
 740                         continue;
 741                 priv_fdir_queue_destroy(priv, rxq_ctrl->fdir_queue);
 742                 rxq_ctrl->fdir_queue = NULL;
 743         }
 744         if (priv->fdir_drop_queue) {
 745                 priv_fdir_queue_destroy(priv, priv->fdir_drop_queue);
 746                 priv->fdir_drop_queue = NULL;
 747         }
 748 }
 749
 750 /**
 751  * Enable flow director, create steering rules.
 752  *
 753  * @param priv
 754  *   Private structure.
 755  */
 756 void
 757 priv_fdir_enable(struct priv *priv)
 758 {
 759         struct mlx5_fdir_filter *mlx5_fdir_filter;
 760
 761         /* Run on every fdir filter and create flow handle */
 762         LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
 763                 /* Only valid elements should be in the list */
 764                 assert(mlx5_fdir_filter != NULL);
 765
 766                 priv_fdir_filter_enable(priv, mlx5_fdir_filter);
 767         }
 768 }
 769
 770 /**
 771  * Find specific filter in list.
 772  *
 773  * @param priv
 774  *   Private structure.
 775  * @param fdir_filter
 776  *   Flow director filter to find.
 777  *
 778  * @return
 779  *   Filter element if found, otherwise NULL.
 780  */
 781 static struct mlx5_fdir_filter *
 782 priv_find_filter_in_list(struct priv *priv,
 783                          const struct rte_eth_fdir_filter *fdir_filter)
 784 {
 785         struct fdir_flow_desc desc;
 786         struct mlx5_fdir_filter *mlx5_fdir_filter;
 787         enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode;
 788
 789         /* Get flow director filter to look for. */
 790         fdir_filter_to_flow_desc(fdir_filter, &desc, fdir_mode);
 791
 792         /* Look for the requested element. */
 793         LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
 794                 /* Only valid elements should be in the list. */
 795                 assert(mlx5_fdir_filter != NULL);
 796
 797                 /* Return matching filter. */
 798                 if (!memcmp(&desc, &mlx5_fdir_filter->desc, sizeof(desc)))
 799                         return mlx5_fdir_filter;
 800         }
 801
 802         /* Filter not found */
 803         return NULL;
 804 }
 805
 806 /**
 807  * Add new flow director filter and store it in list.
 808  *
 809  * @param priv
 810  *   Private structure.
 811  * @param fdir_filter
 812  *   Flow director filter to add.
 813  *
 814  * @return
 815  *   0 on success, errno value on failure.
 816  */
 817 static int
 818 priv_fdir_filter_add(struct priv *priv,
 819                      const struct rte_eth_fdir_filter *fdir_filter)
 820 {
 821         struct mlx5_fdir_filter *mlx5_fdir_filter;
 822         enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode;
 823         int err = 0;
 824
 825         /* Validate queue number. */
 826         if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
 827                 ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
 828                 return EINVAL;
 829         }
 830
 831         /* Duplicate filters are currently unsupported. */
 832         mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
 833         if (mlx5_fdir_filter != NULL) {
 834                 ERROR("filter already exists");
 835                 return EINVAL;
 836         }
 837
 838         /* Create new flow director filter. */
 839         mlx5_fdir_filter =
 840                 rte_calloc(__func__, 1, sizeof(*mlx5_fdir_filter), 0);
 841         if (mlx5_fdir_filter == NULL) {
 842                 err = ENOMEM;
 843                 ERROR("cannot allocate flow director filter: %s",
 844                       strerror(err));
 845                 return err;
 846         }
 847
 848         /* Set action parameters. */
 849         mlx5_fdir_filter->queue = fdir_filter->action.rx_queue;
 850         mlx5_fdir_filter->behavior = fdir_filter->action.behavior;
 851
 852         /* Convert to mlx5 filter descriptor. */
 853         fdir_filter_to_flow_desc(fdir_filter,
 854                                  &mlx5_fdir_filter->desc, fdir_mode);
 855
 856         /* Insert new filter into list. */
 857         LIST_INSERT_HEAD(priv->fdir_filter_list, mlx5_fdir_filter, next);
 858
 859         DEBUG("%p: flow director filter %p added",
 860               (void *)priv, (void *)mlx5_fdir_filter);
 861
 862         /* Enable filter immediately if device is started. */
 863         if (priv->started)
 864                 err = priv_fdir_filter_enable(priv, mlx5_fdir_filter);
 865
 866         return err;
 867 }
 868
 869 /**
 870  * Update queue for specific filter.
 871  *
 872  * @param priv
 873  *   Private structure.
 874  * @param fdir_filter
 875  *   Filter to be updated.
 876  *
 877  * @return
 878  *   0 on success, errno value on failure.
 879  */
 880 static int
 881 priv_fdir_filter_update(struct priv *priv,
 882                         const struct rte_eth_fdir_filter *fdir_filter)
 883 {
 884         struct mlx5_fdir_filter *mlx5_fdir_filter;
 885
 886         /* Validate queue number. */
 887         if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
 888                 ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
 889                 return EINVAL;
 890         }
 891
 892         mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
 893         if (mlx5_fdir_filter != NULL) {
 894                 struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
 895                 int err = 0;
 896
 897                 /* Update queue number. */
 898                 mlx5_fdir_filter->queue = fdir_filter->action.rx_queue;
 899
 900                 /* Destroy flow handle. */
 901                 if (flow != NULL) {
 902                         claim_zero(ibv_exp_destroy_flow(flow));
 903                         mlx5_fdir_filter->flow = NULL;
 904                 }
 905                 DEBUG("%p: flow director filter %p updated",
 906                       (void *)priv, (void *)mlx5_fdir_filter);
 907
 908                 /* Enable filter if device is started. */
 909                 if (priv->started)
 910                         err = priv_fdir_filter_enable(priv, mlx5_fdir_filter);
 911
 912                 return err;
 913         }
 914
 915         /* Filter not found, create it. */
 916         DEBUG("%p: filter not found for update, creating new filter",
 917               (void *)priv);
 918         return priv_fdir_filter_add(priv, fdir_filter);
 919 }
 920
 921 /**
 922  * Delete specific filter.
 923  *
 924  * @param priv
 925  *   Private structure.
 926  * @param fdir_filter
 927  *   Filter to be deleted.
 928  *
 929  * @return
 930  *   0 on success, errno value on failure.
 931  */
 932 static int
 933 priv_fdir_filter_delete(struct priv *priv,
 934                         const struct rte_eth_fdir_filter *fdir_filter)
 935 {
 936         struct mlx5_fdir_filter *mlx5_fdir_filter;
 937
 938         mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
 939         if (mlx5_fdir_filter != NULL) {
 940                 struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
 941
 942                 /* Remove element from list. */
 943                 LIST_REMOVE(mlx5_fdir_filter, next);
 944
 945                 /* Destroy flow handle. */
 946                 if (flow != NULL) {
 947                         claim_zero(ibv_exp_destroy_flow(flow));
 948                         mlx5_fdir_filter->flow = NULL;
 949                 }
 950
 951                 DEBUG("%p: flow director filter %p deleted",
 952                       (void *)priv, (void *)mlx5_fdir_filter);
 953
 954                 /* Delete filter. */
 955                 rte_free(mlx5_fdir_filter);
 956
 957                 return 0;
 958         }
 959
 960         ERROR("%p: flow director delete failed, cannot find filter",
 961               (void *)priv);
 962         return EINVAL;
 963 }
 964
 965 /**
 966  * Get flow director information.
 967  *
 968  * @param priv
 969  *   Private structure.
 970  * @param[out] fdir_info
 971  *   Resulting flow director information.
 972  */
 973 static void
 974 priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
 975 {
 976         struct rte_eth_fdir_masks *mask =
 977                 &priv->dev->data->dev_conf.fdir_conf.mask;
 978
 979         fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode;
 980         fdir_info->guarant_spc = 0;
 981
 982         rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
 983
 984         fdir_info->max_flexpayload = 0;
 985         fdir_info->flow_types_mask[0] = 0;
 986
 987         fdir_info->flex_payload_unit = 0;
 988         fdir_info->max_flex_payload_segment_num = 0;
 989         fdir_info->flex_payload_limit = 0;
 990         memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
 991 }
 992
 993 /**
 994  * Deal with flow director operations.
 995  *
 996  * @param priv
 997  *   Pointer to private structure.
 998  * @param filter_op
 999  *   Operation to perform.
1000  * @param arg
1001  *   Pointer to operation-specific structure.
1002  *
1003  * @return
1004  *   0 on success, errno value on failure.
1005  */
1006 static int
1007 priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg)
1008 {
1009         enum rte_fdir_mode fdir_mode =
1010                 priv->dev->data->dev_conf.fdir_conf.mode;
1011         int ret = 0;
1012
1013         if (filter_op == RTE_ETH_FILTER_NOP)
1014                 return 0;
1015
1016         if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
1017             fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
1018                 ERROR("%p: flow director mode %d not supported",
1019                       (void *)priv, fdir_mode);
1020                 return EINVAL;
1021         }
1022
1023         switch (filter_op) {
1024         case RTE_ETH_FILTER_ADD:
1025                 ret = priv_fdir_filter_add(priv, arg);
1026                 break;
1027         case RTE_ETH_FILTER_UPDATE:
1028                 ret = priv_fdir_filter_update(priv, arg);
1029                 break;
1030         case RTE_ETH_FILTER_DELETE:
1031                 ret = priv_fdir_filter_delete(priv, arg);
1032                 break;
1033         case RTE_ETH_FILTER_FLUSH:
1034                 priv_fdir_filter_flush(priv);
1035                 break;
1036         case RTE_ETH_FILTER_INFO:
1037                 priv_fdir_info_get(priv, arg);
1038                 break;
1039         default:
1040                 DEBUG("%p: unknown operation %u", (void *)priv, filter_op);
1041                 ret = EINVAL;
1042                 break;
1043         }
1044         return ret;
1045 }
1046
1047 static const struct rte_flow_ops mlx5_flow_ops = {
1048         .validate = mlx5_flow_validate,
1049         .create = mlx5_flow_create,
1050         .destroy = mlx5_flow_destroy,
1051         .flush = mlx5_flow_flush,
1052         .query = NULL,
1053 };
1054
1055 /**
1056  * Manage filter operations.
1057  *
1058  * @param dev
1059  *   Pointer to Ethernet device structure.
1060  * @param filter_type
1061  *   Filter type.
1062  * @param filter_op
1063  *   Operation to perform.
1064  * @param arg
1065  *   Pointer to operation-specific structure.
1066  *
1067  * @return
1068  *   0 on success, negative errno value on failure.
1069  */
1070 int
1071 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
1072                      enum rte_filter_type filter_type,
1073                      enum rte_filter_op filter_op,
1074                      void *arg)
1075 {
1076         int ret = EINVAL;
1077         struct priv *priv = dev->data->dev_private;
1078
1079         switch (filter_type) {
1080         case RTE_ETH_FILTER_GENERIC:
1081                 if (filter_op != RTE_ETH_FILTER_GET)
1082                         return -EINVAL;
1083                 *(const void **)arg = &mlx5_flow_ops;
1084                 return 0;
1085         case RTE_ETH_FILTER_FDIR:
1086                 priv_lock(priv);
1087                 ret = priv_fdir_ctrl_func(priv, filter_op, arg);
1088                 priv_unlock(priv);
1089                 break;
1090         default:
1091                 ERROR("%p: filter type (%d) not supported",
1092                       (void *)dev, filter_type);
1093                 break;
1094         }
1095
1096         return -ret;
1097 }