drivers/net/mlx4/mlx4_flow.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright 2017 6WIND S.A.
   5  *   Copyright 2017 Mellanox.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of 6WIND S.A. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <assert.h>
  35
  36 #include <rte_flow.h>
  37 #include <rte_flow_driver.h>
  38 #include <rte_malloc.h>
  39
  40 /* Generated configuration header. */
  41 #include "mlx4_autoconf.h"
  42
  43 /* PMD headers. */
  44 #include "mlx4.h"
  45 #include "mlx4_flow.h"
  46
  47 /** Static initializer for items. */
  48 #define ITEMS(...) \
  49         (const enum rte_flow_item_type []){ \
  50                 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
  51         }
  52
  53 /** Structure to generate a simple graph of layers supported by the NIC. */
  54 struct mlx4_flow_items {
  55         /** List of possible actions for these items. */
  56         const enum rte_flow_action_type *const actions;
  57         /** Bit-masks corresponding to the possibilities for the item. */
  58         const void *mask;
  59         /**
  60          * Default bit-masks to use when item->mask is not provided. When
  61          * \default_mask is also NULL, the full supported bit-mask (\mask) is
  62          * used instead.
  63          */
  64         const void *default_mask;
  65         /** Bit-masks size in bytes. */
  66         const unsigned int mask_sz;
  67         /**
  68          * Check support for a given item.
  69          *
  70          * @param item[in]
  71          *   Item specification.
  72          * @param mask[in]
  73          *   Bit-masks covering supported fields to compare with spec,
  74          *   last and mask in
  75          *   \item.
  76          * @param size
  77          *   Bit-Mask size in bytes.
  78          *
  79          * @return
  80          *   0 on success, negative value otherwise.
  81          */
  82         int (*validate)(const struct rte_flow_item *item,
  83                         const uint8_t *mask, unsigned int size);
  84         /**
  85          * Conversion function from rte_flow to NIC specific flow.
  86          *
  87          * @param item
  88          *   rte_flow item to convert.
  89          * @param default_mask
  90          *   Default bit-masks to use when item->mask is not provided.
  91          * @param data
  92          *   Internal structure to store the conversion.
  93          *
  94          * @return
  95          *   0 on success, negative value otherwise.
  96          */
  97         int (*convert)(const struct rte_flow_item *item,
  98                        const void *default_mask,
  99                        void *data);
 100         /** Size in bytes of the destination structure. */
 101         const unsigned int dst_sz;
 102         /** List of possible following items.  */
 103         const enum rte_flow_item_type *const items;
 104 };
 105
 106 struct rte_flow_drop {
 107         struct ibv_qp *qp; /**< Verbs queue pair. */
 108         struct ibv_cq *cq; /**< Verbs completion queue. */
 109 };
 110
 111 /** Valid action for this PMD. */
 112 static const enum rte_flow_action_type valid_actions[] = {
 113         RTE_FLOW_ACTION_TYPE_DROP,
 114         RTE_FLOW_ACTION_TYPE_QUEUE,
 115         RTE_FLOW_ACTION_TYPE_RSS,
 116         RTE_FLOW_ACTION_TYPE_END,
 117 };
 118
 119 /**
 120  * Convert Ethernet item to Verbs specification.
 121  *
 122  * @param item[in]
 123  *   Item specification.
 124  * @param default_mask[in]
 125  *   Default bit-masks to use when item->mask is not provided.
 126  * @param data[in, out]
 127  *   User structure.
 128  */
 129 static int
 130 mlx4_flow_create_eth(const struct rte_flow_item *item,
 131                      const void *default_mask,
 132                      void *data)
 133 {
 134         const struct rte_flow_item_eth *spec = item->spec;
 135         const struct rte_flow_item_eth *mask = item->mask;
 136         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 137         struct ibv_flow_spec_eth *eth;
 138         const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
 139         unsigned int i;
 140
 141         ++flow->ibv_attr->num_of_specs;
 142         flow->ibv_attr->priority = 2;
 143         eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 144         *eth = (struct ibv_flow_spec_eth) {
 145                 .type = IBV_FLOW_SPEC_ETH,
 146                 .size = eth_size,
 147         };
 148         if (!spec) {
 149                 flow->ibv_attr->type = IBV_FLOW_ATTR_ALL_DEFAULT;
 150                 return 0;
 151         }
 152         if (!mask)
 153                 mask = default_mask;
 154         memcpy(eth->val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
 155         memcpy(eth->val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
 156         memcpy(eth->mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
 157         memcpy(eth->mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
 158         /* Remove unwanted bits from values. */
 159         for (i = 0; i < ETHER_ADDR_LEN; ++i) {
 160                 eth->val.dst_mac[i] &= eth->mask.dst_mac[i];
 161                 eth->val.src_mac[i] &= eth->mask.src_mac[i];
 162         }
 163         return 0;
 164 }
 165
 166 /**
 167  * Convert VLAN item to Verbs specification.
 168  *
 169  * @param item[in]
 170  *   Item specification.
 171  * @param default_mask[in]
 172  *   Default bit-masks to use when item->mask is not provided.
 173  * @param data[in, out]
 174  *   User structure.
 175  */
 176 static int
 177 mlx4_flow_create_vlan(const struct rte_flow_item *item,
 178                       const void *default_mask,
 179                       void *data)
 180 {
 181         const struct rte_flow_item_vlan *spec = item->spec;
 182         const struct rte_flow_item_vlan *mask = item->mask;
 183         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 184         struct ibv_flow_spec_eth *eth;
 185         const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
 186
 187         eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset - eth_size);
 188         if (!spec)
 189                 return 0;
 190         if (!mask)
 191                 mask = default_mask;
 192         eth->val.vlan_tag = spec->tci;
 193         eth->mask.vlan_tag = mask->tci;
 194         eth->val.vlan_tag &= eth->mask.vlan_tag;
 195         return 0;
 196 }
 197
 198 /**
 199  * Convert IPv4 item to Verbs specification.
 200  *
 201  * @param item[in]
 202  *   Item specification.
 203  * @param default_mask[in]
 204  *   Default bit-masks to use when item->mask is not provided.
 205  * @param data[in, out]
 206  *   User structure.
 207  */
 208 static int
 209 mlx4_flow_create_ipv4(const struct rte_flow_item *item,
 210                       const void *default_mask,
 211                       void *data)
 212 {
 213         const struct rte_flow_item_ipv4 *spec = item->spec;
 214         const struct rte_flow_item_ipv4 *mask = item->mask;
 215         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 216         struct ibv_flow_spec_ipv4 *ipv4;
 217         unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4);
 218
 219         ++flow->ibv_attr->num_of_specs;
 220         flow->ibv_attr->priority = 1;
 221         ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 222         *ipv4 = (struct ibv_flow_spec_ipv4) {
 223                 .type = IBV_FLOW_SPEC_IPV4,
 224                 .size = ipv4_size,
 225         };
 226         if (!spec)
 227                 return 0;
 228         ipv4->val = (struct ibv_flow_ipv4_filter) {
 229                 .src_ip = spec->hdr.src_addr,
 230                 .dst_ip = spec->hdr.dst_addr,
 231         };
 232         if (!mask)
 233                 mask = default_mask;
 234         ipv4->mask = (struct ibv_flow_ipv4_filter) {
 235                 .src_ip = mask->hdr.src_addr,
 236                 .dst_ip = mask->hdr.dst_addr,
 237         };
 238         /* Remove unwanted bits from values. */
 239         ipv4->val.src_ip &= ipv4->mask.src_ip;
 240         ipv4->val.dst_ip &= ipv4->mask.dst_ip;
 241         return 0;
 242 }
 243
 244 /**
 245  * Convert UDP item to Verbs specification.
 246  *
 247  * @param item[in]
 248  *   Item specification.
 249  * @param default_mask[in]
 250  *   Default bit-masks to use when item->mask is not provided.
 251  * @param data[in, out]
 252  *   User structure.
 253  */
 254 static int
 255 mlx4_flow_create_udp(const struct rte_flow_item *item,
 256                      const void *default_mask,
 257                      void *data)
 258 {
 259         const struct rte_flow_item_udp *spec = item->spec;
 260         const struct rte_flow_item_udp *mask = item->mask;
 261         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 262         struct ibv_flow_spec_tcp_udp *udp;
 263         unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
 264
 265         ++flow->ibv_attr->num_of_specs;
 266         flow->ibv_attr->priority = 0;
 267         udp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 268         *udp = (struct ibv_flow_spec_tcp_udp) {
 269                 .type = IBV_FLOW_SPEC_UDP,
 270                 .size = udp_size,
 271         };
 272         if (!spec)
 273                 return 0;
 274         udp->val.dst_port = spec->hdr.dst_port;
 275         udp->val.src_port = spec->hdr.src_port;
 276         if (!mask)
 277                 mask = default_mask;
 278         udp->mask.dst_port = mask->hdr.dst_port;
 279         udp->mask.src_port = mask->hdr.src_port;
 280         /* Remove unwanted bits from values. */
 281         udp->val.src_port &= udp->mask.src_port;
 282         udp->val.dst_port &= udp->mask.dst_port;
 283         return 0;
 284 }
 285
 286 /**
 287  * Convert TCP item to Verbs specification.
 288  *
 289  * @param item[in]
 290  *   Item specification.
 291  * @param default_mask[in]
 292  *   Default bit-masks to use when item->mask is not provided.
 293  * @param data[in, out]
 294  *   User structure.
 295  */
 296 static int
 297 mlx4_flow_create_tcp(const struct rte_flow_item *item,
 298                      const void *default_mask,
 299                      void *data)
 300 {
 301         const struct rte_flow_item_tcp *spec = item->spec;
 302         const struct rte_flow_item_tcp *mask = item->mask;
 303         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 304         struct ibv_flow_spec_tcp_udp *tcp;
 305         unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
 306
 307         ++flow->ibv_attr->num_of_specs;
 308         flow->ibv_attr->priority = 0;
 309         tcp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 310         *tcp = (struct ibv_flow_spec_tcp_udp) {
 311                 .type = IBV_FLOW_SPEC_TCP,
 312                 .size = tcp_size,
 313         };
 314         if (!spec)
 315                 return 0;
 316         tcp->val.dst_port = spec->hdr.dst_port;
 317         tcp->val.src_port = spec->hdr.src_port;
 318         if (!mask)
 319                 mask = default_mask;
 320         tcp->mask.dst_port = mask->hdr.dst_port;
 321         tcp->mask.src_port = mask->hdr.src_port;
 322         /* Remove unwanted bits from values. */
 323         tcp->val.src_port &= tcp->mask.src_port;
 324         tcp->val.dst_port &= tcp->mask.dst_port;
 325         return 0;
 326 }
 327
 328 /**
 329  * Check support for a given item.
 330  *
 331  * @param item[in]
 332  *   Item specification.
 333  * @param mask[in]
 334  *   Bit-masks covering supported fields to compare with spec, last and mask in
 335  *   \item.
 336  * @param size
 337  *   Bit-Mask size in bytes.
 338  *
 339  * @return
 340  *   0 on success, negative value otherwise.
 341  */
 342 static int
 343 mlx4_flow_item_validate(const struct rte_flow_item *item,
 344                         const uint8_t *mask, unsigned int size)
 345 {
 346         int ret = 0;
 347
 348         if (!item->spec && (item->mask || item->last))
 349                 return -1;
 350         if (item->spec && !item->mask) {
 351                 unsigned int i;
 352                 const uint8_t *spec = item->spec;
 353
 354                 for (i = 0; i < size; ++i)
 355                         if ((spec[i] | mask[i]) != mask[i])
 356                                 return -1;
 357         }
 358         if (item->last && !item->mask) {
 359                 unsigned int i;
 360                 const uint8_t *spec = item->last;
 361
 362                 for (i = 0; i < size; ++i)
 363                         if ((spec[i] | mask[i]) != mask[i])
 364                                 return -1;
 365         }
 366         if (item->spec && item->last) {
 367                 uint8_t spec[size];
 368                 uint8_t last[size];
 369                 const uint8_t *apply = mask;
 370                 unsigned int i;
 371
 372                 if (item->mask)
 373                         apply = item->mask;
 374                 for (i = 0; i < size; ++i) {
 375                         spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
 376                         last[i] = ((const uint8_t *)item->last)[i] & apply[i];
 377                 }
 378                 ret = memcmp(spec, last, size);
 379         }
 380         return ret;
 381 }
 382
 383 static int
 384 mlx4_flow_validate_eth(const struct rte_flow_item *item,
 385                        const uint8_t *mask, unsigned int size)
 386 {
 387         if (item->mask) {
 388                 const struct rte_flow_item_eth *mask = item->mask;
 389
 390                 if (mask->dst.addr_bytes[0] != 0xff ||
 391                                 mask->dst.addr_bytes[1] != 0xff ||
 392                                 mask->dst.addr_bytes[2] != 0xff ||
 393                                 mask->dst.addr_bytes[3] != 0xff ||
 394                                 mask->dst.addr_bytes[4] != 0xff ||
 395                                 mask->dst.addr_bytes[5] != 0xff)
 396                         return -1;
 397         }
 398         return mlx4_flow_item_validate(item, mask, size);
 399 }
 400
 401 static int
 402 mlx4_flow_validate_vlan(const struct rte_flow_item *item,
 403                         const uint8_t *mask, unsigned int size)
 404 {
 405         if (item->mask) {
 406                 const struct rte_flow_item_vlan *mask = item->mask;
 407
 408                 if (mask->tci != 0 &&
 409                     ntohs(mask->tci) != 0x0fff)
 410                         return -1;
 411         }
 412         return mlx4_flow_item_validate(item, mask, size);
 413 }
 414
 415 static int
 416 mlx4_flow_validate_ipv4(const struct rte_flow_item *item,
 417                         const uint8_t *mask, unsigned int size)
 418 {
 419         if (item->mask) {
 420                 const struct rte_flow_item_ipv4 *mask = item->mask;
 421
 422                 if (mask->hdr.src_addr != 0 &&
 423                     mask->hdr.src_addr != 0xffffffff)
 424                         return -1;
 425                 if (mask->hdr.dst_addr != 0 &&
 426                     mask->hdr.dst_addr != 0xffffffff)
 427                         return -1;
 428         }
 429         return mlx4_flow_item_validate(item, mask, size);
 430 }
 431
 432 static int
 433 mlx4_flow_validate_udp(const struct rte_flow_item *item,
 434                        const uint8_t *mask, unsigned int size)
 435 {
 436         if (item->mask) {
 437                 const struct rte_flow_item_udp *mask = item->mask;
 438
 439                 if (mask->hdr.src_port != 0 &&
 440                     mask->hdr.src_port != 0xffff)
 441                         return -1;
 442                 if (mask->hdr.dst_port != 0 &&
 443                     mask->hdr.dst_port != 0xffff)
 444                         return -1;
 445         }
 446         return mlx4_flow_item_validate(item, mask, size);
 447 }
 448
 449 static int
 450 mlx4_flow_validate_tcp(const struct rte_flow_item *item,
 451                        const uint8_t *mask, unsigned int size)
 452 {
 453         if (item->mask) {
 454                 const struct rte_flow_item_tcp *mask = item->mask;
 455
 456                 if (mask->hdr.src_port != 0 &&
 457                     mask->hdr.src_port != 0xffff)
 458                         return -1;
 459                 if (mask->hdr.dst_port != 0 &&
 460                     mask->hdr.dst_port != 0xffff)
 461                         return -1;
 462         }
 463         return mlx4_flow_item_validate(item, mask, size);
 464 }
 465
 466 /** Graph of supported items and associated actions. */
 467 static const struct mlx4_flow_items mlx4_flow_items[] = {
 468         [RTE_FLOW_ITEM_TYPE_END] = {
 469                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
 470         },
 471         [RTE_FLOW_ITEM_TYPE_ETH] = {
 472                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
 473                                RTE_FLOW_ITEM_TYPE_IPV4),
 474                 .actions = valid_actions,
 475                 .mask = &(const struct rte_flow_item_eth){
 476                         .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 477                         .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 478                 },
 479                 .default_mask = &rte_flow_item_eth_mask,
 480                 .mask_sz = sizeof(struct rte_flow_item_eth),
 481                 .validate = mlx4_flow_validate_eth,
 482                 .convert = mlx4_flow_create_eth,
 483                 .dst_sz = sizeof(struct ibv_flow_spec_eth),
 484         },
 485         [RTE_FLOW_ITEM_TYPE_VLAN] = {
 486                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4),
 487                 .actions = valid_actions,
 488                 .mask = &(const struct rte_flow_item_vlan){
 489                 /* rte_flow_item_vlan_mask is invalid for mlx4. */
 490 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
 491                         .tci = 0x0fff,
 492 #else
 493                         .tci = 0xff0f,
 494 #endif
 495                 },
 496                 .mask_sz = sizeof(struct rte_flow_item_vlan),
 497                 .validate = mlx4_flow_validate_vlan,
 498                 .convert = mlx4_flow_create_vlan,
 499                 .dst_sz = 0,
 500         },
 501         [RTE_FLOW_ITEM_TYPE_IPV4] = {
 502                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
 503                                RTE_FLOW_ITEM_TYPE_TCP),
 504                 .actions = valid_actions,
 505                 .mask = &(const struct rte_flow_item_ipv4){
 506                         .hdr = {
 507                                 .src_addr = -1,
 508                                 .dst_addr = -1,
 509                         },
 510                 },
 511                 .default_mask = &rte_flow_item_ipv4_mask,
 512                 .mask_sz = sizeof(struct rte_flow_item_ipv4),
 513                 .validate = mlx4_flow_validate_ipv4,
 514                 .convert = mlx4_flow_create_ipv4,
 515                 .dst_sz = sizeof(struct ibv_flow_spec_ipv4),
 516         },
 517         [RTE_FLOW_ITEM_TYPE_UDP] = {
 518                 .actions = valid_actions,
 519                 .mask = &(const struct rte_flow_item_udp){
 520                         .hdr = {
 521                                 .src_port = -1,
 522                                 .dst_port = -1,
 523                         },
 524                 },
 525                 .default_mask = &rte_flow_item_udp_mask,
 526                 .mask_sz = sizeof(struct rte_flow_item_udp),
 527                 .validate = mlx4_flow_validate_udp,
 528                 .convert = mlx4_flow_create_udp,
 529                 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
 530         },
 531         [RTE_FLOW_ITEM_TYPE_TCP] = {
 532                 .actions = valid_actions,
 533                 .mask = &(const struct rte_flow_item_tcp){
 534                         .hdr = {
 535                                 .src_port = -1,
 536                                 .dst_port = -1,
 537                         },
 538                 },
 539                 .default_mask = &rte_flow_item_tcp_mask,
 540                 .mask_sz = sizeof(struct rte_flow_item_tcp),
 541                 .validate = mlx4_flow_validate_tcp,
 542                 .convert = mlx4_flow_create_tcp,
 543                 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
 544         },
 545 };
 546
 547 /**
 548  * Validate a flow supported by the NIC.
 549  *
 550  * @param priv
 551  *   Pointer to private structure.
 552  * @param[in] attr
 553  *   Flow rule attributes.
 554  * @param[in] items
 555  *   Pattern specification (list terminated by the END pattern item).
 556  * @param[in] actions
 557  *   Associated actions (list terminated by the END action).
 558  * @param[out] error
 559  *   Perform verbose error reporting if not NULL.
 560  * @param[in, out] flow
 561  *   Flow structure to update.
 562  *
 563  * @return
 564  *   0 on success, a negative errno value otherwise and rte_errno is set.
 565  */
 566 static int
 567 priv_flow_validate(struct priv *priv,
 568                    const struct rte_flow_attr *attr,
 569                    const struct rte_flow_item items[],
 570                    const struct rte_flow_action actions[],
 571                    struct rte_flow_error *error,
 572                    struct mlx4_flow *flow)
 573 {
 574         const struct mlx4_flow_items *cur_item = mlx4_flow_items;
 575         struct mlx4_flow_action action = {
 576                 .queue = 0,
 577                 .drop = 0,
 578         };
 579
 580         (void)priv;
 581         if (attr->group) {
 582                 rte_flow_error_set(error, ENOTSUP,
 583                                    RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
 584                                    NULL,
 585                                    "groups are not supported");
 586                 return -rte_errno;
 587         }
 588         if (attr->priority) {
 589                 rte_flow_error_set(error, ENOTSUP,
 590                                    RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
 591                                    NULL,
 592                                    "priorities are not supported");
 593                 return -rte_errno;
 594         }
 595         if (attr->egress) {
 596                 rte_flow_error_set(error, ENOTSUP,
 597                                    RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
 598                                    NULL,
 599                                    "egress is not supported");
 600                 return -rte_errno;
 601         }
 602         if (!attr->ingress) {
 603                 rte_flow_error_set(error, ENOTSUP,
 604                                    RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 605                                    NULL,
 606                                    "only ingress is supported");
 607                 return -rte_errno;
 608         }
 609         /* Go over items list. */
 610         for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
 611                 const struct mlx4_flow_items *token = NULL;
 612                 unsigned int i;
 613                 int err;
 614
 615                 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
 616                         continue;
 617                 /*
 618                  * The nic can support patterns with NULL eth spec only
 619                  * if eth is a single item in a rule.
 620                  */
 621                 if (!items->spec &&
 622                         items->type == RTE_FLOW_ITEM_TYPE_ETH) {
 623                         const struct rte_flow_item *next = items + 1;
 624
 625                         if (next->type != RTE_FLOW_ITEM_TYPE_END) {
 626                                 rte_flow_error_set(error, ENOTSUP,
 627                                                    RTE_FLOW_ERROR_TYPE_ITEM,
 628                                                    items,
 629                                                    "the rule requires"
 630                                                    " an Ethernet spec");
 631                                 return -rte_errno;
 632                         }
 633                 }
 634                 for (i = 0;
 635                      cur_item->items &&
 636                      cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
 637                      ++i) {
 638                         if (cur_item->items[i] == items->type) {
 639                                 token = &mlx4_flow_items[items->type];
 640                                 break;
 641                         }
 642                 }
 643                 if (!token)
 644                         goto exit_item_not_supported;
 645                 cur_item = token;
 646                 err = cur_item->validate(items,
 647                                         (const uint8_t *)cur_item->mask,
 648                                          cur_item->mask_sz);
 649                 if (err)
 650                         goto exit_item_not_supported;
 651                 if (flow->ibv_attr && cur_item->convert) {
 652                         err = cur_item->convert(items,
 653                                                 (cur_item->default_mask ?
 654                                                  cur_item->default_mask :
 655                                                  cur_item->mask),
 656                                                  flow);
 657                         if (err)
 658                                 goto exit_item_not_supported;
 659                 }
 660                 flow->offset += cur_item->dst_sz;
 661         }
 662         /* Go over actions list */
 663         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
 664                 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
 665                         continue;
 666                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
 667                         action.drop = 1;
 668                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
 669                         const struct rte_flow_action_queue *queue =
 670                                 (const struct rte_flow_action_queue *)
 671                                 actions->conf;
 672
 673                         if (!queue || (queue->index > (priv->rxqs_n - 1)))
 674                                 goto exit_action_not_supported;
 675                         action.queue = 1;
 676                         action.queues_n = 1;
 677                         action.queues[0] = queue->index;
 678                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
 679                         int i;
 680                         int ierr;
 681                         const struct rte_flow_action_rss *rss =
 682                                 (const struct rte_flow_action_rss *)
 683                                 actions->conf;
 684
 685                         if (!priv->hw_rss) {
 686                                 rte_flow_error_set(error, ENOTSUP,
 687                                            RTE_FLOW_ERROR_TYPE_ACTION,
 688                                            actions,
 689                                            "RSS cannot be used with "
 690                                            "the current configuration");
 691                                 return -rte_errno;
 692                         }
 693                         if (!priv->isolated) {
 694                                 rte_flow_error_set(error, ENOTSUP,
 695                                            RTE_FLOW_ERROR_TYPE_ACTION,
 696                                            actions,
 697                                            "RSS cannot be used without "
 698                                            "isolated mode");
 699                                 return -rte_errno;
 700                         }
 701                         if (!rte_is_power_of_2(rss->num)) {
 702                                 rte_flow_error_set(error, ENOTSUP,
 703                                            RTE_FLOW_ERROR_TYPE_ACTION,
 704                                            actions,
 705                                            "the number of queues "
 706                                            "should be power of two");
 707                                 return -rte_errno;
 708                         }
 709                         if (priv->max_rss_tbl_sz < rss->num) {
 710                                 rte_flow_error_set(error, ENOTSUP,
 711                                            RTE_FLOW_ERROR_TYPE_ACTION,
 712                                            actions,
 713                                            "the number of queues "
 714                                            "is too large");
 715                                 return -rte_errno;
 716                         }
 717                         /* checking indexes array */
 718                         ierr = 0;
 719                         for (i = 0; i < rss->num; ++i) {
 720                                 int j;
 721                                 if (rss->queue[i] >= priv->rxqs_n)
 722                                         ierr = 1;
 723                                 /*
 724                                  * Prevent the user from specifying
 725                                  * the same queue twice in the RSS array.
 726                                  */
 727                                 for (j = i + 1; j < rss->num && !ierr; ++j)
 728                                         if (rss->queue[j] == rss->queue[i])
 729                                                 ierr = 1;
 730                                 if (ierr) {
 731                                         rte_flow_error_set(
 732                                                 error,
 733                                                 ENOTSUP,
 734                                                 RTE_FLOW_ERROR_TYPE_HANDLE,
 735                                                 NULL,
 736                                                 "RSS action only supports "
 737                                                 "unique queue indices "
 738                                                 "in a list");
 739                                         return -rte_errno;
 740                                 }
 741                         }
 742                         action.queue = 1;
 743                         action.queues_n = rss->num;
 744                         for (i = 0; i < rss->num; ++i)
 745                                 action.queues[i] = rss->queue[i];
 746                 } else {
 747                         goto exit_action_not_supported;
 748                 }
 749         }
 750         if (!action.queue && !action.drop) {
 751                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
 752                                    NULL, "no valid action");
 753                 return -rte_errno;
 754         }
 755         return 0;
 756 exit_item_not_supported:
 757         rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 758                            items, "item not supported");
 759         return -rte_errno;
 760 exit_action_not_supported:
 761         rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
 762                            actions, "action not supported");
 763         return -rte_errno;
 764 }
 765
 766 /**
 767  * Validate a flow supported by the NIC.
 768  *
 769  * @see rte_flow_validate()
 770  * @see rte_flow_ops
 771  */
 772 int
 773 mlx4_flow_validate(struct rte_eth_dev *dev,
 774                    const struct rte_flow_attr *attr,
 775                    const struct rte_flow_item items[],
 776                    const struct rte_flow_action actions[],
 777                    struct rte_flow_error *error)
 778 {
 779         struct priv *priv = dev->data->dev_private;
 780         int ret;
 781         struct mlx4_flow flow = { .offset = sizeof(struct ibv_flow_attr) };
 782
 783         priv_lock(priv);
 784         ret = priv_flow_validate(priv, attr, items, actions, error, &flow);
 785         priv_unlock(priv);
 786         return ret;
 787 }
 788
 789 /**
 790  * Destroy a drop queue.
 791  *
 792  * @param priv
 793  *   Pointer to private structure.
 794  */
 795 static void
 796 mlx4_flow_destroy_drop_queue(struct priv *priv)
 797 {
 798         if (priv->flow_drop_queue) {
 799                 struct rte_flow_drop *fdq = priv->flow_drop_queue;
 800
 801                 priv->flow_drop_queue = NULL;
 802                 claim_zero(ibv_destroy_qp(fdq->qp));
 803                 claim_zero(ibv_destroy_cq(fdq->cq));
 804                 rte_free(fdq);
 805         }
 806 }
 807
 808 /**
 809  * Create a single drop queue for all drop flows.
 810  *
 811  * @param priv
 812  *   Pointer to private structure.
 813  *
 814  * @return
 815  *   0 on success, negative value otherwise.
 816  */
 817 static int
 818 mlx4_flow_create_drop_queue(struct priv *priv)
 819 {
 820         struct ibv_qp *qp;
 821         struct ibv_cq *cq;
 822         struct rte_flow_drop *fdq;
 823
 824         fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
 825         if (!fdq) {
 826                 ERROR("Cannot allocate memory for drop struct");
 827                 goto err;
 828         }
 829         cq = ibv_exp_create_cq(priv->ctx, 1, NULL, NULL, 0,
 830                               &(struct ibv_exp_cq_init_attr){
 831                                         .comp_mask = 0,
 832                               });
 833         if (!cq) {
 834                 ERROR("Cannot create drop CQ");
 835                 goto err_create_cq;
 836         }
 837         qp = ibv_exp_create_qp(priv->ctx,
 838                               &(struct ibv_exp_qp_init_attr){
 839                                         .send_cq = cq,
 840                                         .recv_cq = cq,
 841                                         .cap = {
 842                                                 .max_recv_wr = 1,
 843                                                 .max_recv_sge = 1,
 844                                         },
 845                                         .qp_type = IBV_QPT_RAW_PACKET,
 846                                         .comp_mask =
 847                                                 IBV_EXP_QP_INIT_ATTR_PD |
 848                                                 IBV_EXP_QP_INIT_ATTR_PORT,
 849                                         .pd = priv->pd,
 850                                         .port_num = priv->port,
 851                               });
 852         if (!qp) {
 853                 ERROR("Cannot create drop QP");
 854                 goto err_create_qp;
 855         }
 856         *fdq = (struct rte_flow_drop){
 857                 .qp = qp,
 858                 .cq = cq,
 859         };
 860         priv->flow_drop_queue = fdq;
 861         return 0;
 862 err_create_qp:
 863         claim_zero(ibv_destroy_cq(cq));
 864 err_create_cq:
 865         rte_free(fdq);
 866 err:
 867         return -1;
 868 }
 869
 870 /**
 871  * Get RSS parent rxq structure for given queues.
 872  *
 873  * Creates a new or returns an existed one.
 874  *
 875  * @param priv
 876  *   Pointer to private structure.
 877  * @param queues
 878  *   queues indices array, NULL in default RSS case.
 879  * @param children_n
 880  *   the size of queues array.
 881  *
 882  * @return
 883  *   Pointer to a parent rxq structure, NULL on failure.
 884  */
 885 static struct rxq *
 886 priv_parent_get(struct priv *priv,
 887                 uint16_t queues[],
 888                 uint16_t children_n,
 889                 struct rte_flow_error *error)
 890 {
 891         unsigned int i;
 892         struct rxq *parent;
 893
 894         for (parent = LIST_FIRST(&priv->parents);
 895              parent;
 896              parent = LIST_NEXT(parent, next)) {
 897                 unsigned int same = 0;
 898                 unsigned int overlap = 0;
 899
 900                 /*
 901                  * Find out whether an appropriate parent queue already exists
 902                  * and can be reused, otherwise make sure there are no overlaps.
 903                  */
 904                 for (i = 0; i < children_n; ++i) {
 905                         unsigned int j;
 906
 907                         for (j = 0; j < parent->rss.queues_n; ++j) {
 908                                 if (parent->rss.queues[j] != queues[i])
 909                                         continue;
 910                                 ++overlap;
 911                                 if (i == j)
 912                                         ++same;
 913                         }
 914                 }
 915                 if (same == children_n &&
 916                         children_n == parent->rss.queues_n)
 917                         return parent;
 918                 else if (overlap)
 919                         goto error;
 920         }
 921         /* Exclude the cases when some QPs were created without RSS */
 922         for (i = 0; i < children_n; ++i) {
 923                 struct rxq *rxq = (*priv->rxqs)[queues[i]];
 924                 if (rxq->qp)
 925                         goto error;
 926         }
 927         parent = priv_parent_create(priv, queues, children_n);
 928         if (!parent) {
 929                 rte_flow_error_set(error,
 930                                    ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 931                                    NULL, "flow rule creation failure");
 932                 return NULL;
 933         }
 934         return parent;
 935
 936 error:
 937         rte_flow_error_set(error,
 938                            EEXIST,
 939                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 940                            NULL,
 941                            "sharing a queue between several"
 942                            " RSS groups is not supported");
 943         return NULL;
 944 }
 945
 946 /**
 947  * Complete flow rule creation.
 948  *
 949  * @param priv
 950  *   Pointer to private structure.
 951  * @param ibv_attr
 952  *   Verbs flow attributes.
 953  * @param action
 954  *   Target action structure.
 955  * @param[out] error
 956  *   Perform verbose error reporting if not NULL.
 957  *
 958  * @return
 959  *   A flow if the rule could be created.
 960  */
 961 static struct rte_flow *
 962 priv_flow_create_action_queue(struct priv *priv,
 963                               struct ibv_flow_attr *ibv_attr,
 964                               struct mlx4_flow_action *action,
 965                               struct rte_flow_error *error)
 966 {
 967         struct ibv_qp *qp;
 968         struct rte_flow *rte_flow;
 969         struct rxq *rxq_parent = NULL;
 970
 971         assert(priv->pd);
 972         assert(priv->ctx);
 973         rte_flow = rte_calloc(__func__, 1, sizeof(*rte_flow), 0);
 974         if (!rte_flow) {
 975                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
 976                                    NULL, "cannot allocate flow memory");
 977                 return NULL;
 978         }
 979         if (action->drop) {
 980                 qp = priv->flow_drop_queue ? priv->flow_drop_queue->qp : NULL;
 981         } else {
 982                 int ret;
 983                 unsigned int i;
 984                 struct rxq *rxq = NULL;
 985
 986                 if (action->queues_n > 1) {
 987                         rxq_parent = priv_parent_get(priv, action->queues,
 988                                                      action->queues_n, error);
 989                         if (!rxq_parent)
 990                                 goto error;
 991                 }
 992                 for (i = 0; i < action->queues_n; ++i) {
 993                         rxq = (*priv->rxqs)[action->queues[i]];
 994                         /*
 995                          * In case of isolated mode we postpone
 996                          * ibv receive queue creation till the first
 997                          * rte_flow rule will be applied on that queue.
 998                          */
 999                         if (!rxq->qp) {
1000                                 assert(priv->isolated);
1001                                 ret = rxq_create_qp(rxq, rxq->elts_n,
1002                                                     0, 0, rxq_parent);
1003                                 if (ret) {
1004                                         rte_flow_error_set(
1005                                                 error,
1006                                                 ENOMEM,
1007                                                 RTE_FLOW_ERROR_TYPE_HANDLE,
1008                                                 NULL,
1009                                                 "flow rule creation failure");
1010                                         goto error;
1011                                 }
1012                         }
1013                 }
1014                 qp = action->queues_n > 1 ? rxq_parent->qp : rxq->qp;
1015                 rte_flow->qp = qp;
1016         }
1017         rte_flow->ibv_attr = ibv_attr;
1018         if (!priv->started)
1019                 return rte_flow;
1020         rte_flow->ibv_flow = ibv_create_flow(qp, rte_flow->ibv_attr);
1021         if (!rte_flow->ibv_flow) {
1022                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1023                                    NULL, "flow rule creation failure");
1024                 goto error;
1025         }
1026         return rte_flow;
1027
1028 error:
1029         if (rxq_parent)
1030                 rxq_parent_cleanup(rxq_parent);
1031         rte_free(rte_flow);
1032         return NULL;
1033 }
1034
1035 /**
1036  * Convert a flow.
1037  *
1038  * @param priv
1039  *   Pointer to private structure.
1040  * @param[in] attr
1041  *   Flow rule attributes.
1042  * @param[in] items
1043  *   Pattern specification (list terminated by the END pattern item).
1044  * @param[in] actions
1045  *   Associated actions (list terminated by the END action).
1046  * @param[out] error
1047  *   Perform verbose error reporting if not NULL.
1048  *
1049  * @return
1050  *   A flow on success, NULL otherwise.
1051  */
1052 static struct rte_flow *
1053 priv_flow_create(struct priv *priv,
1054                  const struct rte_flow_attr *attr,
1055                  const struct rte_flow_item items[],
1056                  const struct rte_flow_action actions[],
1057                  struct rte_flow_error *error)
1058 {
1059         struct rte_flow *rte_flow;
1060         struct mlx4_flow_action action;
1061         struct mlx4_flow flow = { .offset = sizeof(struct ibv_flow_attr), };
1062         int err;
1063
1064         err = priv_flow_validate(priv, attr, items, actions, error, &flow);
1065         if (err)
1066                 return NULL;
1067         flow.ibv_attr = rte_malloc(__func__, flow.offset, 0);
1068         if (!flow.ibv_attr) {
1069                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1070                                    NULL, "cannot allocate ibv_attr memory");
1071                 return NULL;
1072         }
1073         flow.offset = sizeof(struct ibv_flow_attr);
1074         *flow.ibv_attr = (struct ibv_flow_attr){
1075                 .comp_mask = 0,
1076                 .type = IBV_FLOW_ATTR_NORMAL,
1077                 .size = sizeof(struct ibv_flow_attr),
1078                 .priority = attr->priority,
1079                 .num_of_specs = 0,
1080                 .port = priv->port,
1081                 .flags = 0,
1082         };
1083         claim_zero(priv_flow_validate(priv, attr, items, actions,
1084                                       error, &flow));
1085         action = (struct mlx4_flow_action){
1086                 .queue = 0,
1087                 .drop = 0,
1088         };
1089         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
1090                 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
1091                         continue;
1092                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
1093                         action.queue = 1;
1094                         action.queues_n = 1;
1095                         action.queues[0] =
1096                                 ((const struct rte_flow_action_queue *)
1097                                  actions->conf)->index;
1098                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
1099                         action.drop = 1;
1100                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
1101                         unsigned int i;
1102                         const struct rte_flow_action_rss *rss =
1103                                 (const struct rte_flow_action_rss *)
1104                                  actions->conf;
1105
1106                         action.queue = 1;
1107                         action.queues_n = rss->num;
1108                         for (i = 0; i < rss->num; ++i)
1109                                 action.queues[i] = rss->queue[i];
1110                 } else {
1111                         rte_flow_error_set(error, ENOTSUP,
1112                                            RTE_FLOW_ERROR_TYPE_ACTION,
1113                                            actions, "unsupported action");
1114                         goto exit;
1115                 }
1116         }
1117         rte_flow = priv_flow_create_action_queue(priv, flow.ibv_attr,
1118                                                  &action, error);
1119         if (rte_flow)
1120                 return rte_flow;
1121 exit:
1122         rte_free(flow.ibv_attr);
1123         return NULL;
1124 }
1125
1126 /**
1127  * Create a flow.
1128  *
1129  * @see rte_flow_create()
1130  * @see rte_flow_ops
1131  */
1132 struct rte_flow *
1133 mlx4_flow_create(struct rte_eth_dev *dev,
1134                  const struct rte_flow_attr *attr,
1135                  const struct rte_flow_item items[],
1136                  const struct rte_flow_action actions[],
1137                  struct rte_flow_error *error)
1138 {
1139         struct priv *priv = dev->data->dev_private;
1140         struct rte_flow *flow;
1141
1142         priv_lock(priv);
1143         flow = priv_flow_create(priv, attr, items, actions, error);
1144         if (flow) {
1145                 LIST_INSERT_HEAD(&priv->flows, flow, next);
1146                 DEBUG("Flow created %p", (void *)flow);
1147         }
1148         priv_unlock(priv);
1149         return flow;
1150 }
1151
1152 /**
1153  * @see rte_flow_isolate()
1154  *
1155  * Must be done before calling dev_configure().
1156  *
1157  * @param dev
1158  *   Pointer to the ethernet device structure.
1159  * @param enable
1160  *   Nonzero to enter isolated mode, attempt to leave it otherwise.
1161  * @param[out] error
1162  *   Perform verbose error reporting if not NULL. PMDs initialize this
1163  *   structure in case of error only.
1164  *
1165  * @return
1166  *   0 on success, a negative value on error.
1167  */
1168 int
1169 mlx4_flow_isolate(struct rte_eth_dev *dev,
1170                   int enable,
1171                   struct rte_flow_error *error)
1172 {
1173         struct priv *priv = dev->data->dev_private;
1174
1175         priv_lock(priv);
1176         if (priv->rxqs) {
1177                 rte_flow_error_set(error, ENOTSUP,
1178                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1179                                    NULL, "isolated mode must be set"
1180                                    " before configuring the device");
1181                 priv_unlock(priv);
1182                 return -rte_errno;
1183         }
1184         priv->isolated = !!enable;
1185         priv_unlock(priv);
1186         return 0;
1187 }
1188
1189 /**
1190  * Destroy a flow.
1191  *
1192  * @param priv
1193  *   Pointer to private structure.
1194  * @param[in] flow
1195  *   Flow to destroy.
1196  */
1197 static void
1198 priv_flow_destroy(struct priv *priv, struct rte_flow *flow)
1199 {
1200         (void)priv;
1201         LIST_REMOVE(flow, next);
1202         if (flow->ibv_flow)
1203                 claim_zero(ibv_destroy_flow(flow->ibv_flow));
1204         rte_free(flow->ibv_attr);
1205         DEBUG("Flow destroyed %p", (void *)flow);
1206         rte_free(flow);
1207 }
1208
1209 /**
1210  * Destroy a flow.
1211  *
1212  * @see rte_flow_destroy()
1213  * @see rte_flow_ops
1214  */
1215 int
1216 mlx4_flow_destroy(struct rte_eth_dev *dev,
1217                   struct rte_flow *flow,
1218                   struct rte_flow_error *error)
1219 {
1220         struct priv *priv = dev->data->dev_private;
1221
1222         (void)error;
1223         priv_lock(priv);
1224         priv_flow_destroy(priv, flow);
1225         priv_unlock(priv);
1226         return 0;
1227 }
1228
1229 /**
1230  * Destroy all flows.
1231  *
1232  * @param priv
1233  *   Pointer to private structure.
1234  */
1235 static void
1236 priv_flow_flush(struct priv *priv)
1237 {
1238         while (!LIST_EMPTY(&priv->flows)) {
1239                 struct rte_flow *flow;
1240
1241                 flow = LIST_FIRST(&priv->flows);
1242                 priv_flow_destroy(priv, flow);
1243         }
1244 }
1245
1246 /**
1247  * Destroy all flows.
1248  *
1249  * @see rte_flow_flush()
1250  * @see rte_flow_ops
1251  */
1252 int
1253 mlx4_flow_flush(struct rte_eth_dev *dev,
1254                 struct rte_flow_error *error)
1255 {
1256         struct priv *priv = dev->data->dev_private;
1257
1258         (void)error;
1259         priv_lock(priv);
1260         priv_flow_flush(priv);
1261         priv_unlock(priv);
1262         return 0;
1263 }
1264
1265 /**
1266  * Remove all flows.
1267  *
1268  * Called by dev_stop() to remove all flows.
1269  *
1270  * @param priv
1271  *   Pointer to private structure.
1272  */
1273 void
1274 mlx4_priv_flow_stop(struct priv *priv)
1275 {
1276         struct rte_flow *flow;
1277
1278         for (flow = LIST_FIRST(&priv->flows);
1279              flow;
1280              flow = LIST_NEXT(flow, next)) {
1281                 claim_zero(ibv_destroy_flow(flow->ibv_flow));
1282                 flow->ibv_flow = NULL;
1283                 DEBUG("Flow %p removed", (void *)flow);
1284         }
1285         mlx4_flow_destroy_drop_queue(priv);
1286 }
1287
1288 /**
1289  * Add all flows.
1290  *
1291  * @param priv
1292  *   Pointer to private structure.
1293  *
1294  * @return
1295  *   0 on success, a errno value otherwise and rte_errno is set.
1296  */
1297 int
1298 mlx4_priv_flow_start(struct priv *priv)
1299 {
1300         int ret;
1301         struct ibv_qp *qp;
1302         struct rte_flow *flow;
1303
1304         ret = mlx4_flow_create_drop_queue(priv);
1305         if (ret)
1306                 return -1;
1307         for (flow = LIST_FIRST(&priv->flows);
1308              flow;
1309              flow = LIST_NEXT(flow, next)) {
1310                 qp = flow->qp ? flow->qp : priv->flow_drop_queue->qp;
1311                 flow->ibv_flow = ibv_create_flow(qp, flow->ibv_attr);
1312                 if (!flow->ibv_flow) {
1313                         DEBUG("Flow %p cannot be applied", (void *)flow);
1314                         rte_errno = EINVAL;
1315                         return rte_errno;
1316                 }
1317                 DEBUG("Flow %p applied", (void *)flow);
1318         }
1319         return 0;
1320 }