drivers/net/mlx5/mlx5_fdir.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright 2015 6WIND S.A.
   5  *   Copyright 2015 Mellanox.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of 6WIND S.A. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stddef.h>
  35 #include <assert.h>
  36 #include <stdint.h>
  37 #include <string.h>
  38 #include <errno.h>
  39
  40 /* Verbs header. */
  41 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  42 #ifdef PEDANTIC
  43 #pragma GCC diagnostic ignored "-pedantic"
  44 #endif
  45 #include <infiniband/verbs.h>
  46 #ifdef PEDANTIC
  47 #pragma GCC diagnostic error "-pedantic"
  48 #endif
  49
  50 /* DPDK headers don't like -pedantic. */
  51 #ifdef PEDANTIC
  52 #pragma GCC diagnostic ignored "-pedantic"
  53 #endif
  54 #include <rte_ether.h>
  55 #include <rte_malloc.h>
  56 #include <rte_ethdev.h>
  57 #include <rte_common.h>
  58 #ifdef PEDANTIC
  59 #pragma GCC diagnostic error "-pedantic"
  60 #endif
  61
  62 #include "mlx5.h"
  63 #include "mlx5_rxtx.h"
  64
  65 struct fdir_flow_desc {
  66         uint16_t dst_port;
  67         uint16_t src_port;
  68         uint32_t src_ip[4];
  69         uint32_t dst_ip[4];
  70         uint8_t mac[6];
  71         uint16_t vlan_tag;
  72         enum hash_rxq_type type;
  73 };
  74
  75 struct mlx5_fdir_filter {
  76         LIST_ENTRY(mlx5_fdir_filter) next;
  77         uint16_t queue; /* Queue assigned to if FDIR match. */
  78         struct fdir_flow_desc desc;
  79         struct ibv_exp_flow *flow;
  80 };
  81
  82 LIST_HEAD(fdir_filter_list, mlx5_fdir_filter);
  83
  84 /**
  85  * Convert struct rte_eth_fdir_filter to mlx5 filter descriptor.
  86  *
  87  * @param[in] fdir_filter
  88  *   DPDK filter structure to convert.
  89  * @param[out] desc
  90  *   Resulting mlx5 filter descriptor.
  91  * @param mode
  92  *   Flow director mode.
  93  */
  94 static void
  95 fdir_filter_to_flow_desc(const struct rte_eth_fdir_filter *fdir_filter,
  96                          struct fdir_flow_desc *desc, enum rte_fdir_mode mode)
  97 {
  98         /* Initialize descriptor. */
  99         memset(desc, 0, sizeof(*desc));
 100
 101         /* Set VLAN ID. */
 102         desc->vlan_tag = fdir_filter->input.flow_ext.vlan_tci;
 103
 104         /* Set MAC address. */
 105         if (mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
 106                 rte_memcpy(desc->mac,
 107                            fdir_filter->input.flow.mac_vlan_flow.mac_addr.
 108                                 addr_bytes,
 109                            sizeof(desc->mac));
 110                 desc->type = HASH_RXQ_ETH;
 111                 return;
 112         }
 113
 114         /* Set mode */
 115         switch (fdir_filter->input.flow_type) {
 116         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
 117                 desc->type = HASH_RXQ_UDPV4;
 118                 break;
 119         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
 120                 desc->type = HASH_RXQ_TCPV4;
 121                 break;
 122         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
 123                 desc->type = HASH_RXQ_IPV4;
 124                 break;
 125         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
 126                 desc->type = HASH_RXQ_UDPV6;
 127                 break;
 128         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
 129                 desc->type = HASH_RXQ_TCPV6;
 130                 break;
 131         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
 132                 desc->type = HASH_RXQ_IPV6;
 133                 break;
 134         default:
 135                 break;
 136         }
 137
 138         /* Set flow values */
 139         switch (fdir_filter->input.flow_type) {
 140         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
 141         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
 142                 desc->src_port = fdir_filter->input.flow.udp4_flow.src_port;
 143                 desc->dst_port = fdir_filter->input.flow.udp4_flow.dst_port;
 144         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
 145                 desc->src_ip[0] = fdir_filter->input.flow.ip4_flow.src_ip;
 146                 desc->dst_ip[0] = fdir_filter->input.flow.ip4_flow.dst_ip;
 147                 break;
 148         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
 149         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
 150                 desc->src_port = fdir_filter->input.flow.udp6_flow.src_port;
 151                 desc->dst_port = fdir_filter->input.flow.udp6_flow.dst_port;
 152                 /* Fall through. */
 153         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
 154                 rte_memcpy(desc->src_ip,
 155                            fdir_filter->input.flow.ipv6_flow.src_ip,
 156                            sizeof(desc->src_ip));
 157                 rte_memcpy(desc->dst_ip,
 158                            fdir_filter->input.flow.ipv6_flow.dst_ip,
 159                            sizeof(desc->dst_ip));
 160                 break;
 161         default:
 162                 break;
 163         }
 164 }
 165
 166 /**
 167  * Check if two flow descriptors overlap according to configured mask.
 168  *
 169  * @param priv
 170  *   Private structure that provides flow director mask.
 171  * @param desc1
 172  *   First flow descriptor to compare.
 173  * @param desc2
 174  *   Second flow descriptor to compare.
 175  *
 176  * @return
 177  *   Nonzero if descriptors overlap.
 178  */
 179 static int
 180 priv_fdir_overlap(const struct priv *priv,
 181                   const struct fdir_flow_desc *desc1,
 182                   const struct fdir_flow_desc *desc2)
 183 {
 184         const struct rte_eth_fdir_masks *mask =
 185                 &priv->dev->data->dev_conf.fdir_conf.mask;
 186         unsigned int i;
 187
 188         if (desc1->type != desc2->type)
 189                 return 0;
 190         /* Ignore non masked bits. */
 191         for (i = 0; i != RTE_DIM(desc1->mac); ++i)
 192                 if ((desc1->mac[i] & mask->mac_addr_byte_mask) !=
 193                     (desc2->mac[i] & mask->mac_addr_byte_mask))
 194                         return 0;
 195         if (((desc1->src_port & mask->src_port_mask) !=
 196              (desc2->src_port & mask->src_port_mask)) ||
 197             ((desc1->dst_port & mask->dst_port_mask) !=
 198              (desc2->dst_port & mask->dst_port_mask)))
 199                 return 0;
 200         switch (desc1->type) {
 201         case HASH_RXQ_IPV4:
 202         case HASH_RXQ_UDPV4:
 203         case HASH_RXQ_TCPV4:
 204                 if (((desc1->src_ip[0] & mask->ipv4_mask.src_ip) !=
 205                      (desc2->src_ip[0] & mask->ipv4_mask.src_ip)) ||
 206                     ((desc1->dst_ip[0] & mask->ipv4_mask.dst_ip) !=
 207                      (desc2->dst_ip[0] & mask->ipv4_mask.dst_ip)))
 208                         return 0;
 209                 break;
 210         case HASH_RXQ_IPV6:
 211         case HASH_RXQ_UDPV6:
 212         case HASH_RXQ_TCPV6:
 213                 for (i = 0; i != RTE_DIM(desc1->src_ip); ++i)
 214                         if (((desc1->src_ip[i] & mask->ipv6_mask.src_ip[i]) !=
 215                              (desc2->src_ip[i] & mask->ipv6_mask.src_ip[i])) ||
 216                             ((desc1->dst_ip[i] & mask->ipv6_mask.dst_ip[i]) !=
 217                              (desc2->dst_ip[i] & mask->ipv6_mask.dst_ip[i])))
 218                                 return 0;
 219                 break;
 220         default:
 221                 break;
 222         }
 223         return 1;
 224 }
 225
 226 /**
 227  * Create flow director steering rule for a specific filter.
 228  *
 229  * @param priv
 230  *   Private structure.
 231  * @param mlx5_fdir_filter
 232  *   Filter to create a steering rule for.
 233  * @param fdir_queue
 234  *   Flow director queue for matching packets.
 235  *
 236  * @return
 237  *   0 on success, errno value on failure.
 238  */
 239 static int
 240 priv_fdir_flow_add(struct priv *priv,
 241                    struct mlx5_fdir_filter *mlx5_fdir_filter,
 242                    struct fdir_queue *fdir_queue)
 243 {
 244         struct ibv_exp_flow *flow;
 245         struct fdir_flow_desc *desc = &mlx5_fdir_filter->desc;
 246         enum rte_fdir_mode fdir_mode =
 247                 priv->dev->data->dev_conf.fdir_conf.mode;
 248         struct rte_eth_fdir_masks *mask =
 249                 &priv->dev->data->dev_conf.fdir_conf.mask;
 250         FLOW_ATTR_SPEC_ETH(data, priv_flow_attr(priv, NULL, 0, desc->type));
 251         struct ibv_exp_flow_attr *attr = &data->attr;
 252         uintptr_t spec_offset = (uintptr_t)&data->spec;
 253         struct ibv_exp_flow_spec_eth *spec_eth;
 254         struct ibv_exp_flow_spec_ipv4 *spec_ipv4;
 255         struct ibv_exp_flow_spec_ipv6 *spec_ipv6;
 256         struct ibv_exp_flow_spec_tcp_udp *spec_tcp_udp;
 257         struct mlx5_fdir_filter *iter_fdir_filter;
 258         unsigned int i;
 259
 260         /* Abort if an existing flow overlaps this one to avoid packet
 261          * duplication, even if it targets another queue. */
 262         LIST_FOREACH(iter_fdir_filter, priv->fdir_filter_list, next)
 263                 if ((iter_fdir_filter != mlx5_fdir_filter) &&
 264                     (iter_fdir_filter->flow != NULL) &&
 265                     (priv_fdir_overlap(priv,
 266                                        &mlx5_fdir_filter->desc,
 267                                        &iter_fdir_filter->desc)))
 268                         return EEXIST;
 269
 270         /*
 271          * No padding must be inserted by the compiler between attr and spec.
 272          * This layout is expected by libibverbs.
 273          */
 274         assert(((uint8_t *)attr + sizeof(*attr)) == (uint8_t *)spec_offset);
 275         priv_flow_attr(priv, attr, sizeof(data), desc->type);
 276
 277         /* Set Ethernet spec */
 278         spec_eth = (struct ibv_exp_flow_spec_eth *)spec_offset;
 279
 280         /* The first specification must be Ethernet. */
 281         assert(spec_eth->type == IBV_EXP_FLOW_SPEC_ETH);
 282         assert(spec_eth->size == sizeof(*spec_eth));
 283
 284         /* VLAN ID */
 285         spec_eth->val.vlan_tag = desc->vlan_tag & mask->vlan_tci_mask;
 286         spec_eth->mask.vlan_tag = mask->vlan_tci_mask;
 287
 288         /* Update priority */
 289         attr->priority = 2;
 290
 291         if (fdir_mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
 292                 /* MAC Address */
 293                 for (i = 0; i != RTE_DIM(spec_eth->mask.dst_mac); ++i) {
 294                         spec_eth->val.dst_mac[i] =
 295                                 desc->mac[i] & mask->mac_addr_byte_mask;
 296                         spec_eth->mask.dst_mac[i] = mask->mac_addr_byte_mask;
 297                 }
 298                 goto create_flow;
 299         }
 300
 301         switch (desc->type) {
 302         case HASH_RXQ_IPV4:
 303         case HASH_RXQ_UDPV4:
 304         case HASH_RXQ_TCPV4:
 305                 spec_offset += spec_eth->size;
 306
 307                 /* Set IP spec */
 308                 spec_ipv4 = (struct ibv_exp_flow_spec_ipv4 *)spec_offset;
 309
 310                 /* The second specification must be IP. */
 311                 assert(spec_ipv4->type == IBV_EXP_FLOW_SPEC_IPV4);
 312                 assert(spec_ipv4->size == sizeof(*spec_ipv4));
 313
 314                 spec_ipv4->val.src_ip =
 315                         desc->src_ip[0] & mask->ipv4_mask.src_ip;
 316                 spec_ipv4->val.dst_ip =
 317                         desc->dst_ip[0] & mask->ipv4_mask.dst_ip;
 318                 spec_ipv4->mask.src_ip = mask->ipv4_mask.src_ip;
 319                 spec_ipv4->mask.dst_ip = mask->ipv4_mask.dst_ip;
 320
 321                 /* Update priority */
 322                 attr->priority = 1;
 323
 324                 if (desc->type == HASH_RXQ_IPV4)
 325                         goto create_flow;
 326
 327                 spec_offset += spec_ipv4->size;
 328                 break;
 329         case HASH_RXQ_IPV6:
 330         case HASH_RXQ_UDPV6:
 331         case HASH_RXQ_TCPV6:
 332                 spec_offset += spec_eth->size;
 333
 334                 /* Set IP spec */
 335                 spec_ipv6 = (struct ibv_exp_flow_spec_ipv6 *)spec_offset;
 336
 337                 /* The second specification must be IP. */
 338                 assert(spec_ipv6->type == IBV_EXP_FLOW_SPEC_IPV6);
 339                 assert(spec_ipv6->size == sizeof(*spec_ipv6));
 340
 341                 for (i = 0; i != RTE_DIM(desc->src_ip); ++i) {
 342                         ((uint32_t *)spec_ipv6->val.src_ip)[i] =
 343                                 desc->src_ip[i] & mask->ipv6_mask.src_ip[i];
 344                         ((uint32_t *)spec_ipv6->val.dst_ip)[i] =
 345                                 desc->dst_ip[i] & mask->ipv6_mask.dst_ip[i];
 346                 }
 347                 rte_memcpy(spec_ipv6->mask.src_ip,
 348                            mask->ipv6_mask.src_ip,
 349                            sizeof(spec_ipv6->mask.src_ip));
 350                 rte_memcpy(spec_ipv6->mask.dst_ip,
 351                            mask->ipv6_mask.dst_ip,
 352                            sizeof(spec_ipv6->mask.dst_ip));
 353
 354                 /* Update priority */
 355                 attr->priority = 1;
 356
 357                 if (desc->type == HASH_RXQ_IPV6)
 358                         goto create_flow;
 359
 360                 spec_offset += spec_ipv6->size;
 361                 break;
 362         default:
 363                 ERROR("invalid flow attribute type");
 364                 return EINVAL;
 365         }
 366
 367         /* Set TCP/UDP flow specification. */
 368         spec_tcp_udp = (struct ibv_exp_flow_spec_tcp_udp *)spec_offset;
 369
 370         /* The third specification must be TCP/UDP. */
 371         assert(spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_TCP ||
 372                spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_UDP);
 373         assert(spec_tcp_udp->size == sizeof(*spec_tcp_udp));
 374
 375         spec_tcp_udp->val.src_port = desc->src_port & mask->src_port_mask;
 376         spec_tcp_udp->val.dst_port = desc->dst_port & mask->dst_port_mask;
 377         spec_tcp_udp->mask.src_port = mask->src_port_mask;
 378         spec_tcp_udp->mask.dst_port = mask->dst_port_mask;
 379
 380         /* Update priority */
 381         attr->priority = 0;
 382
 383 create_flow:
 384
 385         errno = 0;
 386         flow = ibv_exp_create_flow(fdir_queue->qp, attr);
 387         if (flow == NULL) {
 388                 /* It's not clear whether errno is always set in this case. */
 389                 ERROR("%p: flow director configuration failed, errno=%d: %s",
 390                       (void *)priv, errno,
 391                       (errno ? strerror(errno) : "Unknown error"));
 392                 if (errno)
 393                         return errno;
 394                 return EINVAL;
 395         }
 396
 397         DEBUG("%p: added flow director rule (%p)", (void *)priv, (void *)flow);
 398         mlx5_fdir_filter->flow = flow;
 399         return 0;
 400 }
 401
 402 /**
 403  * Get flow director queue for a specific RX queue, create it in case
 404  * it does not exist.
 405  *
 406  * @param priv
 407  *   Private structure.
 408  * @param idx
 409  *   RX queue index.
 410  *
 411  * @return
 412  *   Related flow director queue on success, NULL otherwise.
 413  */
 414 static struct fdir_queue *
 415 priv_get_fdir_queue(struct priv *priv, uint16_t idx)
 416 {
 417         struct rxq_ctrl *rxq_ctrl =
 418                 container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq);
 419         struct fdir_queue *fdir_queue = &rxq_ctrl->fdir_queue;
 420         struct ibv_exp_rwq_ind_table *ind_table = NULL;
 421         struct ibv_qp *qp = NULL;
 422         struct ibv_exp_rwq_ind_table_init_attr ind_init_attr;
 423         struct ibv_exp_rx_hash_conf hash_conf;
 424         struct ibv_exp_qp_init_attr qp_init_attr;
 425         int err = 0;
 426
 427         /* Return immediately if it has already been created. */
 428         if (fdir_queue->qp != NULL)
 429                 return fdir_queue;
 430
 431         ind_init_attr = (struct ibv_exp_rwq_ind_table_init_attr){
 432                 .pd = priv->pd,
 433                 .log_ind_tbl_size = 0,
 434                 .ind_tbl = &rxq_ctrl->wq,
 435                 .comp_mask = 0,
 436         };
 437
 438         errno = 0;
 439         ind_table = ibv_exp_create_rwq_ind_table(priv->ctx,
 440                                                  &ind_init_attr);
 441         if (ind_table == NULL) {
 442                 /* Not clear whether errno is set. */
 443                 err = (errno ? errno : EINVAL);
 444                 ERROR("RX indirection table creation failed with error %d: %s",
 445                       err, strerror(err));
 446                 goto error;
 447         }
 448
 449         /* Create fdir_queue qp. */
 450         hash_conf = (struct ibv_exp_rx_hash_conf){
 451                 .rx_hash_function = IBV_EXP_RX_HASH_FUNC_TOEPLITZ,
 452                 .rx_hash_key_len = rss_hash_default_key_len,
 453                 .rx_hash_key = rss_hash_default_key,
 454                 .rx_hash_fields_mask = 0,
 455                 .rwq_ind_tbl = ind_table,
 456         };
 457         qp_init_attr = (struct ibv_exp_qp_init_attr){
 458                 .max_inl_recv = 0, /* Currently not supported. */
 459                 .qp_type = IBV_QPT_RAW_PACKET,
 460                 .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD |
 461                               IBV_EXP_QP_INIT_ATTR_RX_HASH),
 462                 .pd = priv->pd,
 463                 .rx_hash_conf = &hash_conf,
 464                 .port_num = priv->port,
 465         };
 466
 467         qp = ibv_exp_create_qp(priv->ctx, &qp_init_attr);
 468         if (qp == NULL) {
 469                 err = (errno ? errno : EINVAL);
 470                 ERROR("hash RX QP creation failure: %s", strerror(err));
 471                 goto error;
 472         }
 473
 474         fdir_queue->ind_table = ind_table;
 475         fdir_queue->qp = qp;
 476
 477         return fdir_queue;
 478
 479 error:
 480         if (qp != NULL)
 481                 claim_zero(ibv_destroy_qp(qp));
 482
 483         if (ind_table != NULL)
 484                 claim_zero(ibv_exp_destroy_rwq_ind_table(ind_table));
 485
 486         return NULL;
 487 }
 488
 489 /**
 490  * Enable flow director filter and create steering rules.
 491  *
 492  * @param priv
 493  *   Private structure.
 494  * @param mlx5_fdir_filter
 495  *   Filter to create steering rule for.
 496  *
 497  * @return
 498  *   0 on success, errno value on failure.
 499  */
 500 static int
 501 priv_fdir_filter_enable(struct priv *priv,
 502                         struct mlx5_fdir_filter *mlx5_fdir_filter)
 503 {
 504         struct fdir_queue *fdir_queue;
 505
 506         /* Check if flow already exists. */
 507         if (mlx5_fdir_filter->flow != NULL)
 508                 return 0;
 509
 510         /* Get fdir_queue for specific queue. */
 511         fdir_queue = priv_get_fdir_queue(priv, mlx5_fdir_filter->queue);
 512
 513         if (fdir_queue == NULL) {
 514                 ERROR("failed to create flow director rxq for queue %d",
 515                       mlx5_fdir_filter->queue);
 516                 return EINVAL;
 517         }
 518
 519         /* Create flow */
 520         return priv_fdir_flow_add(priv, mlx5_fdir_filter, fdir_queue);
 521 }
 522
 523 /**
 524  * Initialize flow director filters list.
 525  *
 526  * @param priv
 527  *   Private structure.
 528  *
 529  * @return
 530  *   0 on success, errno value on failure.
 531  */
 532 int
 533 fdir_init_filters_list(struct priv *priv)
 534 {
 535         /* Filter list initialization should be done only once. */
 536         if (priv->fdir_filter_list)
 537                 return 0;
 538
 539         /* Create filters list. */
 540         priv->fdir_filter_list =
 541                 rte_calloc(__func__, 1, sizeof(*priv->fdir_filter_list), 0);
 542
 543         if (priv->fdir_filter_list == NULL) {
 544                 int err = ENOMEM;
 545
 546                 ERROR("cannot allocate flow director filter list: %s",
 547                       strerror(err));
 548                 return err;
 549         }
 550
 551         LIST_INIT(priv->fdir_filter_list);
 552
 553         return 0;
 554 }
 555
 556 /**
 557  * Flush all filters.
 558  *
 559  * @param priv
 560  *   Private structure.
 561  */
 562 static void
 563 priv_fdir_filter_flush(struct priv *priv)
 564 {
 565         struct mlx5_fdir_filter *mlx5_fdir_filter;
 566
 567         while ((mlx5_fdir_filter = LIST_FIRST(priv->fdir_filter_list))) {
 568                 struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
 569
 570                 DEBUG("%p: flushing flow director filter %p",
 571                       (void *)priv, (void *)mlx5_fdir_filter);
 572                 LIST_REMOVE(mlx5_fdir_filter, next);
 573                 if (flow != NULL)
 574                         claim_zero(ibv_exp_destroy_flow(flow));
 575                 rte_free(mlx5_fdir_filter);
 576         }
 577 }
 578
 579 /**
 580  * Remove all flow director filters and delete list.
 581  *
 582  * @param priv
 583  *   Private structure.
 584  */
 585 void
 586 priv_fdir_delete_filters_list(struct priv *priv)
 587 {
 588         priv_fdir_filter_flush(priv);
 589         rte_free(priv->fdir_filter_list);
 590         priv->fdir_filter_list = NULL;
 591 }
 592
 593 /**
 594  * Disable flow director, remove all steering rules.
 595  *
 596  * @param priv
 597  *   Private structure.
 598  */
 599 void
 600 priv_fdir_disable(struct priv *priv)
 601 {
 602         unsigned int i;
 603         struct mlx5_fdir_filter *mlx5_fdir_filter;
 604         struct fdir_queue *fdir_queue;
 605
 606         /* Run on every flow director filter and destroy flow handle. */
 607         LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
 608                 struct ibv_exp_flow *flow;
 609
 610                 /* Only valid elements should be in the list */
 611                 assert(mlx5_fdir_filter != NULL);
 612                 flow = mlx5_fdir_filter->flow;
 613
 614                 /* Destroy flow handle */
 615                 if (flow != NULL) {
 616                         claim_zero(ibv_exp_destroy_flow(flow));
 617                         mlx5_fdir_filter->flow = NULL;
 618                 }
 619         }
 620
 621         /* Run on every RX queue to destroy related flow director QP and
 622          * indirection table. */
 623         for (i = 0; (i != priv->rxqs_n); i++) {
 624                 struct rxq_ctrl *rxq_ctrl =
 625                         container_of((*priv->rxqs)[i], struct rxq_ctrl, rxq);
 626
 627                 fdir_queue = &rxq_ctrl->fdir_queue;
 628                 if (fdir_queue->qp != NULL) {
 629                         claim_zero(ibv_destroy_qp(fdir_queue->qp));
 630                         fdir_queue->qp = NULL;
 631                 }
 632
 633                 if (fdir_queue->ind_table != NULL) {
 634                         claim_zero(ibv_exp_destroy_rwq_ind_table
 635                                    (fdir_queue->ind_table));
 636                         fdir_queue->ind_table = NULL;
 637                 }
 638         }
 639 }
 640
 641 /**
 642  * Enable flow director, create steering rules.
 643  *
 644  * @param priv
 645  *   Private structure.
 646  */
 647 void
 648 priv_fdir_enable(struct priv *priv)
 649 {
 650         struct mlx5_fdir_filter *mlx5_fdir_filter;
 651
 652         /* Run on every fdir filter and create flow handle */
 653         LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
 654                 /* Only valid elements should be in the list */
 655                 assert(mlx5_fdir_filter != NULL);
 656
 657                 priv_fdir_filter_enable(priv, mlx5_fdir_filter);
 658         }
 659 }
 660
 661 /**
 662  * Find specific filter in list.
 663  *
 664  * @param priv
 665  *   Private structure.
 666  * @param fdir_filter
 667  *   Flow director filter to find.
 668  *
 669  * @return
 670  *   Filter element if found, otherwise NULL.
 671  */
 672 static struct mlx5_fdir_filter *
 673 priv_find_filter_in_list(struct priv *priv,
 674                          const struct rte_eth_fdir_filter *fdir_filter)
 675 {
 676         struct fdir_flow_desc desc;
 677         struct mlx5_fdir_filter *mlx5_fdir_filter;
 678         enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode;
 679
 680         /* Get flow director filter to look for. */
 681         fdir_filter_to_flow_desc(fdir_filter, &desc, fdir_mode);
 682
 683         /* Look for the requested element. */
 684         LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
 685                 /* Only valid elements should be in the list. */
 686                 assert(mlx5_fdir_filter != NULL);
 687
 688                 /* Return matching filter. */
 689                 if (!memcmp(&desc, &mlx5_fdir_filter->desc, sizeof(desc)))
 690                         return mlx5_fdir_filter;
 691         }
 692
 693         /* Filter not found */
 694         return NULL;
 695 }
 696
 697 /**
 698  * Add new flow director filter and store it in list.
 699  *
 700  * @param priv
 701  *   Private structure.
 702  * @param fdir_filter
 703  *   Flow director filter to add.
 704  *
 705  * @return
 706  *   0 on success, errno value on failure.
 707  */
 708 static int
 709 priv_fdir_filter_add(struct priv *priv,
 710                      const struct rte_eth_fdir_filter *fdir_filter)
 711 {
 712         struct mlx5_fdir_filter *mlx5_fdir_filter;
 713         enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode;
 714         int err = 0;
 715
 716         /* Validate queue number. */
 717         if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
 718                 ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
 719                 return EINVAL;
 720         }
 721
 722         /* Duplicate filters are currently unsupported. */
 723         mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
 724         if (mlx5_fdir_filter != NULL) {
 725                 ERROR("filter already exists");
 726                 return EINVAL;
 727         }
 728
 729         /* Create new flow director filter. */
 730         mlx5_fdir_filter =
 731                 rte_calloc(__func__, 1, sizeof(*mlx5_fdir_filter), 0);
 732         if (mlx5_fdir_filter == NULL) {
 733                 err = ENOMEM;
 734                 ERROR("cannot allocate flow director filter: %s",
 735                       strerror(err));
 736                 return err;
 737         }
 738
 739         /* Set queue. */
 740         mlx5_fdir_filter->queue = fdir_filter->action.rx_queue;
 741
 742         /* Convert to mlx5 filter descriptor. */
 743         fdir_filter_to_flow_desc(fdir_filter,
 744                                  &mlx5_fdir_filter->desc, fdir_mode);
 745
 746         /* Insert new filter into list. */
 747         LIST_INSERT_HEAD(priv->fdir_filter_list, mlx5_fdir_filter, next);
 748
 749         DEBUG("%p: flow director filter %p added",
 750               (void *)priv, (void *)mlx5_fdir_filter);
 751
 752         /* Enable filter immediately if device is started. */
 753         if (priv->started)
 754                 err = priv_fdir_filter_enable(priv, mlx5_fdir_filter);
 755
 756         return err;
 757 }
 758
 759 /**
 760  * Update queue for specific filter.
 761  *
 762  * @param priv
 763  *   Private structure.
 764  * @param fdir_filter
 765  *   Filter to be updated.
 766  *
 767  * @return
 768  *   0 on success, errno value on failure.
 769  */
 770 static int
 771 priv_fdir_filter_update(struct priv *priv,
 772                         const struct rte_eth_fdir_filter *fdir_filter)
 773 {
 774         struct mlx5_fdir_filter *mlx5_fdir_filter;
 775
 776         /* Validate queue number. */
 777         if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
 778                 ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
 779                 return EINVAL;
 780         }
 781
 782         mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
 783         if (mlx5_fdir_filter != NULL) {
 784                 struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
 785                 int err = 0;
 786
 787                 /* Update queue number. */
 788                 mlx5_fdir_filter->queue = fdir_filter->action.rx_queue;
 789
 790                 /* Destroy flow handle. */
 791                 if (flow != NULL) {
 792                         claim_zero(ibv_exp_destroy_flow(flow));
 793                         mlx5_fdir_filter->flow = NULL;
 794                 }
 795                 DEBUG("%p: flow director filter %p updated",
 796                       (void *)priv, (void *)mlx5_fdir_filter);
 797
 798                 /* Enable filter if device is started. */
 799                 if (priv->started)
 800                         err = priv_fdir_filter_enable(priv, mlx5_fdir_filter);
 801
 802                 return err;
 803         }
 804
 805         /* Filter not found, create it. */
 806         DEBUG("%p: filter not found for update, creating new filter",
 807               (void *)priv);
 808         return priv_fdir_filter_add(priv, fdir_filter);
 809 }
 810
 811 /**
 812  * Delete specific filter.
 813  *
 814  * @param priv
 815  *   Private structure.
 816  * @param fdir_filter
 817  *   Filter to be deleted.
 818  *
 819  * @return
 820  *   0 on success, errno value on failure.
 821  */
 822 static int
 823 priv_fdir_filter_delete(struct priv *priv,
 824                         const struct rte_eth_fdir_filter *fdir_filter)
 825 {
 826         struct mlx5_fdir_filter *mlx5_fdir_filter;
 827
 828         mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
 829         if (mlx5_fdir_filter != NULL) {
 830                 struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
 831
 832                 /* Remove element from list. */
 833                 LIST_REMOVE(mlx5_fdir_filter, next);
 834
 835                 /* Destroy flow handle. */
 836                 if (flow != NULL) {
 837                         claim_zero(ibv_exp_destroy_flow(flow));
 838                         mlx5_fdir_filter->flow = NULL;
 839                 }
 840
 841                 DEBUG("%p: flow director filter %p deleted",
 842                       (void *)priv, (void *)mlx5_fdir_filter);
 843
 844                 /* Delete filter. */
 845                 rte_free(mlx5_fdir_filter);
 846
 847                 return 0;
 848         }
 849
 850         ERROR("%p: flow director delete failed, cannot find filter",
 851               (void *)priv);
 852         return EINVAL;
 853 }
 854
 855 /**
 856  * Get flow director information.
 857  *
 858  * @param priv
 859  *   Private structure.
 860  * @param[out] fdir_info
 861  *   Resulting flow director information.
 862  */
 863 static void
 864 priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
 865 {
 866         struct rte_eth_fdir_masks *mask =
 867                 &priv->dev->data->dev_conf.fdir_conf.mask;
 868
 869         fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode;
 870         fdir_info->guarant_spc = 0;
 871
 872         rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
 873
 874         fdir_info->max_flexpayload = 0;
 875         fdir_info->flow_types_mask[0] = 0;
 876
 877         fdir_info->flex_payload_unit = 0;
 878         fdir_info->max_flex_payload_segment_num = 0;
 879         fdir_info->flex_payload_limit = 0;
 880         memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
 881 }
 882
 883 /**
 884  * Deal with flow director operations.
 885  *
 886  * @param priv
 887  *   Pointer to private structure.
 888  * @param filter_op
 889  *   Operation to perform.
 890  * @param arg
 891  *   Pointer to operation-specific structure.
 892  *
 893  * @return
 894  *   0 on success, errno value on failure.
 895  */
 896 static int
 897 priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg)
 898 {
 899         enum rte_fdir_mode fdir_mode =
 900                 priv->dev->data->dev_conf.fdir_conf.mode;
 901         int ret = 0;
 902
 903         if (filter_op == RTE_ETH_FILTER_NOP)
 904                 return 0;
 905
 906         if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
 907             fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
 908                 ERROR("%p: flow director mode %d not supported",
 909                       (void *)priv, fdir_mode);
 910                 return EINVAL;
 911         }
 912
 913         switch (filter_op) {
 914         case RTE_ETH_FILTER_ADD:
 915                 ret = priv_fdir_filter_add(priv, arg);
 916                 break;
 917         case RTE_ETH_FILTER_UPDATE:
 918                 ret = priv_fdir_filter_update(priv, arg);
 919                 break;
 920         case RTE_ETH_FILTER_DELETE:
 921                 ret = priv_fdir_filter_delete(priv, arg);
 922                 break;
 923         case RTE_ETH_FILTER_FLUSH:
 924                 priv_fdir_filter_flush(priv);
 925                 break;
 926         case RTE_ETH_FILTER_INFO:
 927                 priv_fdir_info_get(priv, arg);
 928                 break;
 929         default:
 930                 DEBUG("%p: unknown operation %u", (void *)priv, filter_op);
 931                 ret = EINVAL;
 932                 break;
 933         }
 934         return ret;
 935 }
 936
 937 /**
 938  * Manage filter operations.
 939  *
 940  * @param dev
 941  *   Pointer to Ethernet device structure.
 942  * @param filter_type
 943  *   Filter type.
 944  * @param filter_op
 945  *   Operation to perform.
 946  * @param arg
 947  *   Pointer to operation-specific structure.
 948  *
 949  * @return
 950  *   0 on success, negative errno value on failure.
 951  */
 952 int
 953 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
 954                      enum rte_filter_type filter_type,
 955                      enum rte_filter_op filter_op,
 956                      void *arg)
 957 {
 958         int ret = -EINVAL;
 959         struct priv *priv = dev->data->dev_private;
 960
 961         switch (filter_type) {
 962         case RTE_ETH_FILTER_FDIR:
 963                 priv_lock(priv);
 964                 ret = priv_fdir_ctrl_func(priv, filter_op, arg);
 965                 priv_unlock(priv);
 966                 break;
 967         default:
 968                 ERROR("%p: filter type (%d) not supported",
 969                       (void *)dev, filter_type);
 970                 break;
 971         }
 972
 973         return ret;
 974 }