drivers/net/mlx4/mlx4_flow.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright 2017 6WIND S.A.
   5  *   Copyright 2017 Mellanox.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of 6WIND S.A. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <assert.h>
  35
  36 #include <rte_flow.h>
  37 #include <rte_flow_driver.h>
  38 #include <rte_malloc.h>
  39
  40 /* Generated configuration header. */
  41 #include "mlx4_autoconf.h"
  42
  43 /* PMD headers. */
  44 #include "mlx4.h"
  45 #include "mlx4_flow.h"
  46
  47 /** Static initializer for items. */
  48 #define ITEMS(...) \
  49         (const enum rte_flow_item_type []){ \
  50                 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
  51         }
  52
  53 /** Structure to generate a simple graph of layers supported by the NIC. */
  54 struct mlx4_flow_items {
  55         /** List of possible actions for these items. */
  56         const enum rte_flow_action_type *const actions;
  57         /** Bit-masks corresponding to the possibilities for the item. */
  58         const void *mask;
  59         /**
  60          * Default bit-masks to use when item->mask is not provided. When
  61          * \default_mask is also NULL, the full supported bit-mask (\mask) is
  62          * used instead.
  63          */
  64         const void *default_mask;
  65         /** Bit-masks size in bytes. */
  66         const unsigned int mask_sz;
  67         /**
  68          * Check support for a given item.
  69          *
  70          * @param item[in]
  71          *   Item specification.
  72          * @param mask[in]
  73          *   Bit-masks covering supported fields to compare with spec,
  74          *   last and mask in
  75          *   \item.
  76          * @param size
  77          *   Bit-Mask size in bytes.
  78          *
  79          * @return
  80          *   0 on success, negative value otherwise.
  81          */
  82         int (*validate)(const struct rte_flow_item *item,
  83                         const uint8_t *mask, unsigned int size);
  84         /**
  85          * Conversion function from rte_flow to NIC specific flow.
  86          *
  87          * @param item
  88          *   rte_flow item to convert.
  89          * @param default_mask
  90          *   Default bit-masks to use when item->mask is not provided.
  91          * @param data
  92          *   Internal structure to store the conversion.
  93          *
  94          * @return
  95          *   0 on success, negative value otherwise.
  96          */
  97         int (*convert)(const struct rte_flow_item *item,
  98                        const void *default_mask,
  99                        void *data);
 100         /** Size in bytes of the destination structure. */
 101         const unsigned int dst_sz;
 102         /** List of possible following items.  */
 103         const enum rte_flow_item_type *const items;
 104 };
 105
 106 struct rte_flow_drop {
 107         struct ibv_qp *qp; /**< Verbs queue pair. */
 108         struct ibv_cq *cq; /**< Verbs completion queue. */
 109 };
 110
 111 /** Valid action for this PMD. */
 112 static const enum rte_flow_action_type valid_actions[] = {
 113         RTE_FLOW_ACTION_TYPE_DROP,
 114         RTE_FLOW_ACTION_TYPE_QUEUE,
 115         RTE_FLOW_ACTION_TYPE_END,
 116 };
 117
 118 /**
 119  * Convert Ethernet item to Verbs specification.
 120  *
 121  * @param item[in]
 122  *   Item specification.
 123  * @param default_mask[in]
 124  *   Default bit-masks to use when item->mask is not provided.
 125  * @param data[in, out]
 126  *   User structure.
 127  */
 128 static int
 129 mlx4_flow_create_eth(const struct rte_flow_item *item,
 130                      const void *default_mask,
 131                      void *data)
 132 {
 133         const struct rte_flow_item_eth *spec = item->spec;
 134         const struct rte_flow_item_eth *mask = item->mask;
 135         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 136         struct ibv_flow_spec_eth *eth;
 137         const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
 138         unsigned int i;
 139
 140         ++flow->ibv_attr->num_of_specs;
 141         flow->ibv_attr->priority = 2;
 142         eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 143         *eth = (struct ibv_flow_spec_eth) {
 144                 .type = IBV_FLOW_SPEC_ETH,
 145                 .size = eth_size,
 146         };
 147         if (!spec) {
 148                 flow->ibv_attr->type = IBV_FLOW_ATTR_ALL_DEFAULT;
 149                 return 0;
 150         }
 151         if (!mask)
 152                 mask = default_mask;
 153         memcpy(eth->val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
 154         memcpy(eth->val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
 155         memcpy(eth->mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
 156         memcpy(eth->mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
 157         /* Remove unwanted bits from values. */
 158         for (i = 0; i < ETHER_ADDR_LEN; ++i) {
 159                 eth->val.dst_mac[i] &= eth->mask.dst_mac[i];
 160                 eth->val.src_mac[i] &= eth->mask.src_mac[i];
 161         }
 162         return 0;
 163 }
 164
 165 /**
 166  * Convert VLAN item to Verbs specification.
 167  *
 168  * @param item[in]
 169  *   Item specification.
 170  * @param default_mask[in]
 171  *   Default bit-masks to use when item->mask is not provided.
 172  * @param data[in, out]
 173  *   User structure.
 174  */
 175 static int
 176 mlx4_flow_create_vlan(const struct rte_flow_item *item,
 177                       const void *default_mask,
 178                       void *data)
 179 {
 180         const struct rte_flow_item_vlan *spec = item->spec;
 181         const struct rte_flow_item_vlan *mask = item->mask;
 182         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 183         struct ibv_flow_spec_eth *eth;
 184         const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
 185
 186         eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset - eth_size);
 187         if (!spec)
 188                 return 0;
 189         if (!mask)
 190                 mask = default_mask;
 191         eth->val.vlan_tag = spec->tci;
 192         eth->mask.vlan_tag = mask->tci;
 193         eth->val.vlan_tag &= eth->mask.vlan_tag;
 194         return 0;
 195 }
 196
 197 /**
 198  * Convert IPv4 item to Verbs specification.
 199  *
 200  * @param item[in]
 201  *   Item specification.
 202  * @param default_mask[in]
 203  *   Default bit-masks to use when item->mask is not provided.
 204  * @param data[in, out]
 205  *   User structure.
 206  */
 207 static int
 208 mlx4_flow_create_ipv4(const struct rte_flow_item *item,
 209                       const void *default_mask,
 210                       void *data)
 211 {
 212         const struct rte_flow_item_ipv4 *spec = item->spec;
 213         const struct rte_flow_item_ipv4 *mask = item->mask;
 214         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 215         struct ibv_flow_spec_ipv4 *ipv4;
 216         unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4);
 217
 218         ++flow->ibv_attr->num_of_specs;
 219         flow->ibv_attr->priority = 1;
 220         ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 221         *ipv4 = (struct ibv_flow_spec_ipv4) {
 222                 .type = IBV_FLOW_SPEC_IPV4,
 223                 .size = ipv4_size,
 224         };
 225         if (!spec)
 226                 return 0;
 227         ipv4->val = (struct ibv_flow_ipv4_filter) {
 228                 .src_ip = spec->hdr.src_addr,
 229                 .dst_ip = spec->hdr.dst_addr,
 230         };
 231         if (!mask)
 232                 mask = default_mask;
 233         ipv4->mask = (struct ibv_flow_ipv4_filter) {
 234                 .src_ip = mask->hdr.src_addr,
 235                 .dst_ip = mask->hdr.dst_addr,
 236         };
 237         /* Remove unwanted bits from values. */
 238         ipv4->val.src_ip &= ipv4->mask.src_ip;
 239         ipv4->val.dst_ip &= ipv4->mask.dst_ip;
 240         return 0;
 241 }
 242
 243 /**
 244  * Convert UDP item to Verbs specification.
 245  *
 246  * @param item[in]
 247  *   Item specification.
 248  * @param default_mask[in]
 249  *   Default bit-masks to use when item->mask is not provided.
 250  * @param data[in, out]
 251  *   User structure.
 252  */
 253 static int
 254 mlx4_flow_create_udp(const struct rte_flow_item *item,
 255                      const void *default_mask,
 256                      void *data)
 257 {
 258         const struct rte_flow_item_udp *spec = item->spec;
 259         const struct rte_flow_item_udp *mask = item->mask;
 260         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 261         struct ibv_flow_spec_tcp_udp *udp;
 262         unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
 263
 264         ++flow->ibv_attr->num_of_specs;
 265         flow->ibv_attr->priority = 0;
 266         udp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 267         *udp = (struct ibv_flow_spec_tcp_udp) {
 268                 .type = IBV_FLOW_SPEC_UDP,
 269                 .size = udp_size,
 270         };
 271         if (!spec)
 272                 return 0;
 273         udp->val.dst_port = spec->hdr.dst_port;
 274         udp->val.src_port = spec->hdr.src_port;
 275         if (!mask)
 276                 mask = default_mask;
 277         udp->mask.dst_port = mask->hdr.dst_port;
 278         udp->mask.src_port = mask->hdr.src_port;
 279         /* Remove unwanted bits from values. */
 280         udp->val.src_port &= udp->mask.src_port;
 281         udp->val.dst_port &= udp->mask.dst_port;
 282         return 0;
 283 }
 284
 285 /**
 286  * Convert TCP item to Verbs specification.
 287  *
 288  * @param item[in]
 289  *   Item specification.
 290  * @param default_mask[in]
 291  *   Default bit-masks to use when item->mask is not provided.
 292  * @param data[in, out]
 293  *   User structure.
 294  */
 295 static int
 296 mlx4_flow_create_tcp(const struct rte_flow_item *item,
 297                      const void *default_mask,
 298                      void *data)
 299 {
 300         const struct rte_flow_item_tcp *spec = item->spec;
 301         const struct rte_flow_item_tcp *mask = item->mask;
 302         struct mlx4_flow *flow = (struct mlx4_flow *)data;
 303         struct ibv_flow_spec_tcp_udp *tcp;
 304         unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
 305
 306         ++flow->ibv_attr->num_of_specs;
 307         flow->ibv_attr->priority = 0;
 308         tcp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
 309         *tcp = (struct ibv_flow_spec_tcp_udp) {
 310                 .type = IBV_FLOW_SPEC_TCP,
 311                 .size = tcp_size,
 312         };
 313         if (!spec)
 314                 return 0;
 315         tcp->val.dst_port = spec->hdr.dst_port;
 316         tcp->val.src_port = spec->hdr.src_port;
 317         if (!mask)
 318                 mask = default_mask;
 319         tcp->mask.dst_port = mask->hdr.dst_port;
 320         tcp->mask.src_port = mask->hdr.src_port;
 321         /* Remove unwanted bits from values. */
 322         tcp->val.src_port &= tcp->mask.src_port;
 323         tcp->val.dst_port &= tcp->mask.dst_port;
 324         return 0;
 325 }
 326
 327 /**
 328  * Check support for a given item.
 329  *
 330  * @param item[in]
 331  *   Item specification.
 332  * @param mask[in]
 333  *   Bit-masks covering supported fields to compare with spec, last and mask in
 334  *   \item.
 335  * @param size
 336  *   Bit-Mask size in bytes.
 337  *
 338  * @return
 339  *   0 on success, negative value otherwise.
 340  */
 341 static int
 342 mlx4_flow_item_validate(const struct rte_flow_item *item,
 343                         const uint8_t *mask, unsigned int size)
 344 {
 345         int ret = 0;
 346
 347         if (!item->spec && (item->mask || item->last))
 348                 return -1;
 349         if (item->spec && !item->mask) {
 350                 unsigned int i;
 351                 const uint8_t *spec = item->spec;
 352
 353                 for (i = 0; i < size; ++i)
 354                         if ((spec[i] | mask[i]) != mask[i])
 355                                 return -1;
 356         }
 357         if (item->last && !item->mask) {
 358                 unsigned int i;
 359                 const uint8_t *spec = item->last;
 360
 361                 for (i = 0; i < size; ++i)
 362                         if ((spec[i] | mask[i]) != mask[i])
 363                                 return -1;
 364         }
 365         if (item->spec && item->last) {
 366                 uint8_t spec[size];
 367                 uint8_t last[size];
 368                 const uint8_t *apply = mask;
 369                 unsigned int i;
 370
 371                 if (item->mask)
 372                         apply = item->mask;
 373                 for (i = 0; i < size; ++i) {
 374                         spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
 375                         last[i] = ((const uint8_t *)item->last)[i] & apply[i];
 376                 }
 377                 ret = memcmp(spec, last, size);
 378         }
 379         return ret;
 380 }
 381
 382 static int
 383 mlx4_flow_validate_eth(const struct rte_flow_item *item,
 384                        const uint8_t *mask, unsigned int size)
 385 {
 386         if (item->mask) {
 387                 const struct rte_flow_item_eth *mask = item->mask;
 388
 389                 if (mask->dst.addr_bytes[0] != 0xff ||
 390                                 mask->dst.addr_bytes[1] != 0xff ||
 391                                 mask->dst.addr_bytes[2] != 0xff ||
 392                                 mask->dst.addr_bytes[3] != 0xff ||
 393                                 mask->dst.addr_bytes[4] != 0xff ||
 394                                 mask->dst.addr_bytes[5] != 0xff)
 395                         return -1;
 396         }
 397         return mlx4_flow_item_validate(item, mask, size);
 398 }
 399
 400 static int
 401 mlx4_flow_validate_vlan(const struct rte_flow_item *item,
 402                         const uint8_t *mask, unsigned int size)
 403 {
 404         if (item->mask) {
 405                 const struct rte_flow_item_vlan *mask = item->mask;
 406
 407                 if (mask->tci != 0 &&
 408                     ntohs(mask->tci) != 0x0fff)
 409                         return -1;
 410         }
 411         return mlx4_flow_item_validate(item, mask, size);
 412 }
 413
 414 static int
 415 mlx4_flow_validate_ipv4(const struct rte_flow_item *item,
 416                         const uint8_t *mask, unsigned int size)
 417 {
 418         if (item->mask) {
 419                 const struct rte_flow_item_ipv4 *mask = item->mask;
 420
 421                 if (mask->hdr.src_addr != 0 &&
 422                     mask->hdr.src_addr != 0xffffffff)
 423                         return -1;
 424                 if (mask->hdr.dst_addr != 0 &&
 425                     mask->hdr.dst_addr != 0xffffffff)
 426                         return -1;
 427         }
 428         return mlx4_flow_item_validate(item, mask, size);
 429 }
 430
 431 static int
 432 mlx4_flow_validate_udp(const struct rte_flow_item *item,
 433                        const uint8_t *mask, unsigned int size)
 434 {
 435         if (item->mask) {
 436                 const struct rte_flow_item_udp *mask = item->mask;
 437
 438                 if (mask->hdr.src_port != 0 &&
 439                     mask->hdr.src_port != 0xffff)
 440                         return -1;
 441                 if (mask->hdr.dst_port != 0 &&
 442                     mask->hdr.dst_port != 0xffff)
 443                         return -1;
 444         }
 445         return mlx4_flow_item_validate(item, mask, size);
 446 }
 447
 448 static int
 449 mlx4_flow_validate_tcp(const struct rte_flow_item *item,
 450                        const uint8_t *mask, unsigned int size)
 451 {
 452         if (item->mask) {
 453                 const struct rte_flow_item_tcp *mask = item->mask;
 454
 455                 if (mask->hdr.src_port != 0 &&
 456                     mask->hdr.src_port != 0xffff)
 457                         return -1;
 458                 if (mask->hdr.dst_port != 0 &&
 459                     mask->hdr.dst_port != 0xffff)
 460                         return -1;
 461         }
 462         return mlx4_flow_item_validate(item, mask, size);
 463 }
 464
 465 /** Graph of supported items and associated actions. */
 466 static const struct mlx4_flow_items mlx4_flow_items[] = {
 467         [RTE_FLOW_ITEM_TYPE_END] = {
 468                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
 469         },
 470         [RTE_FLOW_ITEM_TYPE_ETH] = {
 471                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
 472                                RTE_FLOW_ITEM_TYPE_IPV4),
 473                 .actions = valid_actions,
 474                 .mask = &(const struct rte_flow_item_eth){
 475                         .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 476                         .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 477                 },
 478                 .default_mask = &rte_flow_item_eth_mask,
 479                 .mask_sz = sizeof(struct rte_flow_item_eth),
 480                 .validate = mlx4_flow_validate_eth,
 481                 .convert = mlx4_flow_create_eth,
 482                 .dst_sz = sizeof(struct ibv_flow_spec_eth),
 483         },
 484         [RTE_FLOW_ITEM_TYPE_VLAN] = {
 485                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4),
 486                 .actions = valid_actions,
 487                 .mask = &(const struct rte_flow_item_vlan){
 488                 /* rte_flow_item_vlan_mask is invalid for mlx4. */
 489 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
 490                         .tci = 0x0fff,
 491 #else
 492                         .tci = 0xff0f,
 493 #endif
 494                 },
 495                 .mask_sz = sizeof(struct rte_flow_item_vlan),
 496                 .validate = mlx4_flow_validate_vlan,
 497                 .convert = mlx4_flow_create_vlan,
 498                 .dst_sz = 0,
 499         },
 500         [RTE_FLOW_ITEM_TYPE_IPV4] = {
 501                 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
 502                                RTE_FLOW_ITEM_TYPE_TCP),
 503                 .actions = valid_actions,
 504                 .mask = &(const struct rte_flow_item_ipv4){
 505                         .hdr = {
 506                                 .src_addr = -1,
 507                                 .dst_addr = -1,
 508                         },
 509                 },
 510                 .default_mask = &rte_flow_item_ipv4_mask,
 511                 .mask_sz = sizeof(struct rte_flow_item_ipv4),
 512                 .validate = mlx4_flow_validate_ipv4,
 513                 .convert = mlx4_flow_create_ipv4,
 514                 .dst_sz = sizeof(struct ibv_flow_spec_ipv4),
 515         },
 516         [RTE_FLOW_ITEM_TYPE_UDP] = {
 517                 .actions = valid_actions,
 518                 .mask = &(const struct rte_flow_item_udp){
 519                         .hdr = {
 520                                 .src_port = -1,
 521                                 .dst_port = -1,
 522                         },
 523                 },
 524                 .default_mask = &rte_flow_item_udp_mask,
 525                 .mask_sz = sizeof(struct rte_flow_item_udp),
 526                 .validate = mlx4_flow_validate_udp,
 527                 .convert = mlx4_flow_create_udp,
 528                 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
 529         },
 530         [RTE_FLOW_ITEM_TYPE_TCP] = {
 531                 .actions = valid_actions,
 532                 .mask = &(const struct rte_flow_item_tcp){
 533                         .hdr = {
 534                                 .src_port = -1,
 535                                 .dst_port = -1,
 536                         },
 537                 },
 538                 .default_mask = &rte_flow_item_tcp_mask,
 539                 .mask_sz = sizeof(struct rte_flow_item_tcp),
 540                 .validate = mlx4_flow_validate_tcp,
 541                 .convert = mlx4_flow_create_tcp,
 542                 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
 543         },
 544 };
 545
 546 /**
 547  * Validate a flow supported by the NIC.
 548  *
 549  * @param priv
 550  *   Pointer to private structure.
 551  * @param[in] attr
 552  *   Flow rule attributes.
 553  * @param[in] items
 554  *   Pattern specification (list terminated by the END pattern item).
 555  * @param[in] actions
 556  *   Associated actions (list terminated by the END action).
 557  * @param[out] error
 558  *   Perform verbose error reporting if not NULL.
 559  * @param[in, out] flow
 560  *   Flow structure to update.
 561  *
 562  * @return
 563  *   0 on success, a negative errno value otherwise and rte_errno is set.
 564  */
 565 static int
 566 priv_flow_validate(struct priv *priv,
 567                    const struct rte_flow_attr *attr,
 568                    const struct rte_flow_item items[],
 569                    const struct rte_flow_action actions[],
 570                    struct rte_flow_error *error,
 571                    struct mlx4_flow *flow)
 572 {
 573         const struct mlx4_flow_items *cur_item = mlx4_flow_items;
 574         struct mlx4_flow_action action = {
 575                 .queue = 0,
 576                 .drop = 0,
 577         };
 578
 579         (void)priv;
 580         if (attr->group) {
 581                 rte_flow_error_set(error, ENOTSUP,
 582                                    RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
 583                                    NULL,
 584                                    "groups are not supported");
 585                 return -rte_errno;
 586         }
 587         if (attr->priority) {
 588                 rte_flow_error_set(error, ENOTSUP,
 589                                    RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
 590                                    NULL,
 591                                    "priorities are not supported");
 592                 return -rte_errno;
 593         }
 594         if (attr->egress) {
 595                 rte_flow_error_set(error, ENOTSUP,
 596                                    RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
 597                                    NULL,
 598                                    "egress is not supported");
 599                 return -rte_errno;
 600         }
 601         if (!attr->ingress) {
 602                 rte_flow_error_set(error, ENOTSUP,
 603                                    RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
 604                                    NULL,
 605                                    "only ingress is supported");
 606                 return -rte_errno;
 607         }
 608         /* Go over items list. */
 609         for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
 610                 const struct mlx4_flow_items *token = NULL;
 611                 unsigned int i;
 612                 int err;
 613
 614                 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
 615                         continue;
 616                 /*
 617                  * The nic can support patterns with NULL eth spec only
 618                  * if eth is a single item in a rule.
 619                  */
 620                 if (!items->spec &&
 621                         items->type == RTE_FLOW_ITEM_TYPE_ETH) {
 622                         const struct rte_flow_item *next = items + 1;
 623
 624                         if (next->type != RTE_FLOW_ITEM_TYPE_END) {
 625                                 rte_flow_error_set(error, ENOTSUP,
 626                                                    RTE_FLOW_ERROR_TYPE_ITEM,
 627                                                    items,
 628                                                    "the rule requires"
 629                                                    " an Ethernet spec");
 630                                 return -rte_errno;
 631                         }
 632                 }
 633                 for (i = 0;
 634                      cur_item->items &&
 635                      cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
 636                      ++i) {
 637                         if (cur_item->items[i] == items->type) {
 638                                 token = &mlx4_flow_items[items->type];
 639                                 break;
 640                         }
 641                 }
 642                 if (!token)
 643                         goto exit_item_not_supported;
 644                 cur_item = token;
 645                 err = cur_item->validate(items,
 646                                         (const uint8_t *)cur_item->mask,
 647                                          cur_item->mask_sz);
 648                 if (err)
 649                         goto exit_item_not_supported;
 650                 if (flow->ibv_attr && cur_item->convert) {
 651                         err = cur_item->convert(items,
 652                                                 (cur_item->default_mask ?
 653                                                  cur_item->default_mask :
 654                                                  cur_item->mask),
 655                                                  flow);
 656                         if (err)
 657                                 goto exit_item_not_supported;
 658                 }
 659                 flow->offset += cur_item->dst_sz;
 660         }
 661         /* Go over actions list */
 662         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
 663                 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
 664                         continue;
 665                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
 666                         action.drop = 1;
 667                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
 668                         const struct rte_flow_action_queue *queue =
 669                                 (const struct rte_flow_action_queue *)
 670                                 actions->conf;
 671
 672                         if (!queue || (queue->index > (priv->rxqs_n - 1)))
 673                                 goto exit_action_not_supported;
 674                         action.queue = 1;
 675                 } else {
 676                         goto exit_action_not_supported;
 677                 }
 678         }
 679         if (!action.queue && !action.drop) {
 680                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
 681                                    NULL, "no valid action");
 682                 return -rte_errno;
 683         }
 684         return 0;
 685 exit_item_not_supported:
 686         rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 687                            items, "item not supported");
 688         return -rte_errno;
 689 exit_action_not_supported:
 690         rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
 691                            actions, "action not supported");
 692         return -rte_errno;
 693 }
 694
 695 /**
 696  * Validate a flow supported by the NIC.
 697  *
 698  * @see rte_flow_validate()
 699  * @see rte_flow_ops
 700  */
 701 int
 702 mlx4_flow_validate(struct rte_eth_dev *dev,
 703                    const struct rte_flow_attr *attr,
 704                    const struct rte_flow_item items[],
 705                    const struct rte_flow_action actions[],
 706                    struct rte_flow_error *error)
 707 {
 708         struct priv *priv = dev->data->dev_private;
 709         int ret;
 710         struct mlx4_flow flow = { .offset = sizeof(struct ibv_flow_attr) };
 711
 712         priv_lock(priv);
 713         ret = priv_flow_validate(priv, attr, items, actions, error, &flow);
 714         priv_unlock(priv);
 715         return ret;
 716 }
 717
 718 /**
 719  * Destroy a drop queue.
 720  *
 721  * @param priv
 722  *   Pointer to private structure.
 723  */
 724 static void
 725 mlx4_flow_destroy_drop_queue(struct priv *priv)
 726 {
 727         if (priv->flow_drop_queue) {
 728                 struct rte_flow_drop *fdq = priv->flow_drop_queue;
 729
 730                 priv->flow_drop_queue = NULL;
 731                 claim_zero(ibv_destroy_qp(fdq->qp));
 732                 claim_zero(ibv_destroy_cq(fdq->cq));
 733                 rte_free(fdq);
 734         }
 735 }
 736
 737 /**
 738  * Create a single drop queue for all drop flows.
 739  *
 740  * @param priv
 741  *   Pointer to private structure.
 742  *
 743  * @return
 744  *   0 on success, negative value otherwise.
 745  */
 746 static int
 747 mlx4_flow_create_drop_queue(struct priv *priv)
 748 {
 749         struct ibv_qp *qp;
 750         struct ibv_cq *cq;
 751         struct rte_flow_drop *fdq;
 752
 753         fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
 754         if (!fdq) {
 755                 ERROR("Cannot allocate memory for drop struct");
 756                 goto err;
 757         }
 758         cq = ibv_exp_create_cq(priv->ctx, 1, NULL, NULL, 0,
 759                               &(struct ibv_exp_cq_init_attr){
 760                                         .comp_mask = 0,
 761                               });
 762         if (!cq) {
 763                 ERROR("Cannot create drop CQ");
 764                 goto err_create_cq;
 765         }
 766         qp = ibv_exp_create_qp(priv->ctx,
 767                               &(struct ibv_exp_qp_init_attr){
 768                                         .send_cq = cq,
 769                                         .recv_cq = cq,
 770                                         .cap = {
 771                                                 .max_recv_wr = 1,
 772                                                 .max_recv_sge = 1,
 773                                         },
 774                                         .qp_type = IBV_QPT_RAW_PACKET,
 775                                         .comp_mask =
 776                                                 IBV_EXP_QP_INIT_ATTR_PD |
 777                                                 IBV_EXP_QP_INIT_ATTR_PORT,
 778                                         .pd = priv->pd,
 779                                         .port_num = priv->port,
 780                               });
 781         if (!qp) {
 782                 ERROR("Cannot create drop QP");
 783                 goto err_create_qp;
 784         }
 785         *fdq = (struct rte_flow_drop){
 786                 .qp = qp,
 787                 .cq = cq,
 788         };
 789         priv->flow_drop_queue = fdq;
 790         return 0;
 791 err_create_qp:
 792         claim_zero(ibv_destroy_cq(cq));
 793 err_create_cq:
 794         rte_free(fdq);
 795 err:
 796         return -1;
 797 }
 798
 799 /**
 800  * Complete flow rule creation.
 801  *
 802  * @param priv
 803  *   Pointer to private structure.
 804  * @param ibv_attr
 805  *   Verbs flow attributes.
 806  * @param action
 807  *   Target action structure.
 808  * @param[out] error
 809  *   Perform verbose error reporting if not NULL.
 810  *
 811  * @return
 812  *   A flow if the rule could be created.
 813  */
 814 static struct rte_flow *
 815 priv_flow_create_action_queue(struct priv *priv,
 816                               struct ibv_flow_attr *ibv_attr,
 817                               struct mlx4_flow_action *action,
 818                               struct rte_flow_error *error)
 819 {
 820         struct ibv_qp *qp;
 821         struct rte_flow *rte_flow;
 822
 823         assert(priv->pd);
 824         assert(priv->ctx);
 825         rte_flow = rte_calloc(__func__, 1, sizeof(*rte_flow), 0);
 826         if (!rte_flow) {
 827                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
 828                                    NULL, "cannot allocate flow memory");
 829                 return NULL;
 830         }
 831         if (action->drop) {
 832                 qp = priv->flow_drop_queue ? priv->flow_drop_queue->qp : NULL;
 833         } else {
 834                 struct rxq *rxq = (*priv->rxqs)[action->queue_id];
 835
 836                 qp = rxq->qp;
 837                 rte_flow->qp = qp;
 838         }
 839         rte_flow->ibv_attr = ibv_attr;
 840         if (!priv->started)
 841                 return rte_flow;
 842         rte_flow->ibv_flow = ibv_create_flow(qp, rte_flow->ibv_attr);
 843         if (!rte_flow->ibv_flow) {
 844                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
 845                                    NULL, "flow rule creation failure");
 846                 goto error;
 847         }
 848         return rte_flow;
 849
 850 error:
 851         rte_free(rte_flow);
 852         return NULL;
 853 }
 854
 855 /**
 856  * Convert a flow.
 857  *
 858  * @param priv
 859  *   Pointer to private structure.
 860  * @param[in] attr
 861  *   Flow rule attributes.
 862  * @param[in] items
 863  *   Pattern specification (list terminated by the END pattern item).
 864  * @param[in] actions
 865  *   Associated actions (list terminated by the END action).
 866  * @param[out] error
 867  *   Perform verbose error reporting if not NULL.
 868  *
 869  * @return
 870  *   A flow on success, NULL otherwise.
 871  */
 872 static struct rte_flow *
 873 priv_flow_create(struct priv *priv,
 874                  const struct rte_flow_attr *attr,
 875                  const struct rte_flow_item items[],
 876                  const struct rte_flow_action actions[],
 877                  struct rte_flow_error *error)
 878 {
 879         struct rte_flow *rte_flow;
 880         struct mlx4_flow_action action;
 881         struct mlx4_flow flow = { .offset = sizeof(struct ibv_flow_attr), };
 882         int err;
 883
 884         err = priv_flow_validate(priv, attr, items, actions, error, &flow);
 885         if (err)
 886                 return NULL;
 887         flow.ibv_attr = rte_malloc(__func__, flow.offset, 0);
 888         if (!flow.ibv_attr) {
 889                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
 890                                    NULL, "cannot allocate ibv_attr memory");
 891                 return NULL;
 892         }
 893         flow.offset = sizeof(struct ibv_flow_attr);
 894         *flow.ibv_attr = (struct ibv_flow_attr){
 895                 .comp_mask = 0,
 896                 .type = IBV_FLOW_ATTR_NORMAL,
 897                 .size = sizeof(struct ibv_flow_attr),
 898                 .priority = attr->priority,
 899                 .num_of_specs = 0,
 900                 .port = priv->port,
 901                 .flags = 0,
 902         };
 903         claim_zero(priv_flow_validate(priv, attr, items, actions,
 904                                       error, &flow));
 905         action = (struct mlx4_flow_action){
 906                 .queue = 0,
 907                 .drop = 0,
 908         };
 909         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
 910                 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
 911                         continue;
 912                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
 913                         action.queue = 1;
 914                         action.queue_id =
 915                                 ((const struct rte_flow_action_queue *)
 916                                  actions->conf)->index;
 917                 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
 918                         action.drop = 1;
 919                 } else {
 920                         rte_flow_error_set(error, ENOTSUP,
 921                                            RTE_FLOW_ERROR_TYPE_ACTION,
 922                                            actions, "unsupported action");
 923                         goto exit;
 924                 }
 925         }
 926         rte_flow = priv_flow_create_action_queue(priv, flow.ibv_attr,
 927                                                  &action, error);
 928         if (rte_flow)
 929                 return rte_flow;
 930 exit:
 931         rte_free(flow.ibv_attr);
 932         return NULL;
 933 }
 934
 935 /**
 936  * Create a flow.
 937  *
 938  * @see rte_flow_create()
 939  * @see rte_flow_ops
 940  */
 941 struct rte_flow *
 942 mlx4_flow_create(struct rte_eth_dev *dev,
 943                  const struct rte_flow_attr *attr,
 944                  const struct rte_flow_item items[],
 945                  const struct rte_flow_action actions[],
 946                  struct rte_flow_error *error)
 947 {
 948         struct priv *priv = dev->data->dev_private;
 949         struct rte_flow *flow;
 950
 951         priv_lock(priv);
 952         flow = priv_flow_create(priv, attr, items, actions, error);
 953         if (flow) {
 954                 LIST_INSERT_HEAD(&priv->flows, flow, next);
 955                 DEBUG("Flow created %p", (void *)flow);
 956         }
 957         priv_unlock(priv);
 958         return flow;
 959 }
 960
 961 /**
 962  * Destroy a flow.
 963  *
 964  * @param priv
 965  *   Pointer to private structure.
 966  * @param[in] flow
 967  *   Flow to destroy.
 968  */
 969 static void
 970 priv_flow_destroy(struct priv *priv, struct rte_flow *flow)
 971 {
 972         (void)priv;
 973         LIST_REMOVE(flow, next);
 974         if (flow->ibv_flow)
 975                 claim_zero(ibv_destroy_flow(flow->ibv_flow));
 976         rte_free(flow->ibv_attr);
 977         DEBUG("Flow destroyed %p", (void *)flow);
 978         rte_free(flow);
 979 }
 980
 981 /**
 982  * Destroy a flow.
 983  *
 984  * @see rte_flow_destroy()
 985  * @see rte_flow_ops
 986  */
 987 int
 988 mlx4_flow_destroy(struct rte_eth_dev *dev,
 989                   struct rte_flow *flow,
 990                   struct rte_flow_error *error)
 991 {
 992         struct priv *priv = dev->data->dev_private;
 993
 994         (void)error;
 995         priv_lock(priv);
 996         priv_flow_destroy(priv, flow);
 997         priv_unlock(priv);
 998         return 0;
 999 }
1000
1001 /**
1002  * Destroy all flows.
1003  *
1004  * @param priv
1005  *   Pointer to private structure.
1006  */
1007 static void
1008 priv_flow_flush(struct priv *priv)
1009 {
1010         while (!LIST_EMPTY(&priv->flows)) {
1011                 struct rte_flow *flow;
1012
1013                 flow = LIST_FIRST(&priv->flows);
1014                 priv_flow_destroy(priv, flow);
1015         }
1016 }
1017
1018 /**
1019  * Destroy all flows.
1020  *
1021  * @see rte_flow_flush()
1022  * @see rte_flow_ops
1023  */
1024 int
1025 mlx4_flow_flush(struct rte_eth_dev *dev,
1026                 struct rte_flow_error *error)
1027 {
1028         struct priv *priv = dev->data->dev_private;
1029
1030         (void)error;
1031         priv_lock(priv);
1032         priv_flow_flush(priv);
1033         priv_unlock(priv);
1034         return 0;
1035 }
1036
1037 /**
1038  * Remove all flows.
1039  *
1040  * Called by dev_stop() to remove all flows.
1041  *
1042  * @param priv
1043  *   Pointer to private structure.
1044  */
1045 void
1046 mlx4_priv_flow_stop(struct priv *priv)
1047 {
1048         struct rte_flow *flow;
1049
1050         for (flow = LIST_FIRST(&priv->flows);
1051              flow;
1052              flow = LIST_NEXT(flow, next)) {
1053                 claim_zero(ibv_destroy_flow(flow->ibv_flow));
1054                 flow->ibv_flow = NULL;
1055                 DEBUG("Flow %p removed", (void *)flow);
1056         }
1057         mlx4_flow_destroy_drop_queue(priv);
1058 }
1059
1060 /**
1061  * Add all flows.
1062  *
1063  * @param priv
1064  *   Pointer to private structure.
1065  *
1066  * @return
1067  *   0 on success, a errno value otherwise and rte_errno is set.
1068  */
1069 int
1070 mlx4_priv_flow_start(struct priv *priv)
1071 {
1072         int ret;
1073         struct ibv_qp *qp;
1074         struct rte_flow *flow;
1075
1076         ret = mlx4_flow_create_drop_queue(priv);
1077         if (ret)
1078                 return -1;
1079         for (flow = LIST_FIRST(&priv->flows);
1080              flow;
1081              flow = LIST_NEXT(flow, next)) {
1082                 qp = flow->qp ? flow->qp : priv->flow_drop_queue->qp;
1083                 flow->ibv_flow = ibv_create_flow(qp, flow->ibv_attr);
1084                 if (!flow->ibv_flow) {
1085                         DEBUG("Flow %p cannot be applied", (void *)flow);
1086                         rte_errno = EINVAL;
1087                         return rte_errno;
1088                 }
1089                 DEBUG("Flow %p applied", (void *)flow);
1090         }
1091         return 0;
1092 }