drivers/net/mlx5/mlx5_flow_verbs.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <netinet/in.h>
   6 #include <sys/queue.h>
   7 #include <stdalign.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10
  11 /* Verbs header. */
  12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  13 #ifdef PEDANTIC
  14 #pragma GCC diagnostic ignored "-Wpedantic"
  15 #endif
  16 #include <infiniband/verbs.h>
  17 #ifdef PEDANTIC
  18 #pragma GCC diagnostic error "-Wpedantic"
  19 #endif
  20
  21 #include <rte_common.h>
  22 #include <rte_ether.h>
  23 #include <rte_eth_ctrl.h>
  24 #include <rte_ethdev_driver.h>
  25 #include <rte_flow.h>
  26 #include <rte_flow_driver.h>
  27 #include <rte_malloc.h>
  28 #include <rte_ip.h>
  29
  30 #include "mlx5.h"
  31 #include "mlx5_defs.h"
  32 #include "mlx5_prm.h"
  33 #include "mlx5_glue.h"
  34 #include "mlx5_flow.h"
  35
  36 /**
  37  * Create Verbs flow counter with Verbs library.
  38  *
  39  * @param[in] dev
  40  *   Pointer to the Ethernet device structure.
  41  * @param[in, out] counter
  42  *   mlx5 flow counter object, contains the counter id,
  43  *   handle of created Verbs flow counter is returned
  44  *   in cs field (if counters are supported).
  45  *
  46  * @return
  47  *   0 On success else a negative errno value is returned
  48  *   and rte_errno is set.
  49  */
  50 static int
  51 flow_verbs_counter_create(struct rte_eth_dev *dev,
  52                           struct mlx5_flow_counter *counter)
  53 {
  54 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
  55         struct priv *priv = dev->data->dev_private;
  56         struct ibv_counter_set_init_attr init = {
  57                          .counter_set_id = counter->id};
  58
  59         counter->cs = mlx5_glue->create_counter_set(priv->ctx, &init);
  60         if (!counter->cs) {
  61                 rte_errno = ENOTSUP;
  62                 return -ENOTSUP;
  63         }
  64         return 0;
  65 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
  66         struct priv *priv = dev->data->dev_private;
  67         struct ibv_counters_init_attr init = {0};
  68         struct ibv_counter_attach_attr attach = {0};
  69         int ret;
  70
  71         counter->cs = mlx5_glue->create_counters(priv->ctx, &init);
  72         if (!counter->cs) {
  73                 rte_errno = ENOTSUP;
  74                 return -ENOTSUP;
  75         }
  76         attach.counter_desc = IBV_COUNTER_PACKETS;
  77         attach.index = 0;
  78         ret = mlx5_glue->attach_counters(counter->cs, &attach, NULL);
  79         if (!ret) {
  80                 attach.counter_desc = IBV_COUNTER_BYTES;
  81                 attach.index = 1;
  82                 ret = mlx5_glue->attach_counters
  83                                         (counter->cs, &attach, NULL);
  84         }
  85         if (ret) {
  86                 claim_zero(mlx5_glue->destroy_counters(counter->cs));
  87                 counter->cs = NULL;
  88                 rte_errno = ret;
  89                 return -ret;
  90         }
  91         return 0;
  92 #else
  93         (void)dev;
  94         (void)counter;
  95         rte_errno = ENOTSUP;
  96         return -ENOTSUP;
  97 #endif
  98 }
  99
 100 /**
 101  * Get a flow counter.
 102  *
 103  * @param[in] dev
 104  *   Pointer to the Ethernet device structure.
 105  * @param[in] shared
 106  *   Indicate if this counter is shared with other flows.
 107  * @param[in] id
 108  *   Counter identifier.
 109  *
 110  * @return
 111  *   A pointer to the counter, NULL otherwise and rte_errno is set.
 112  */
 113 static struct mlx5_flow_counter *
 114 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
 115 {
 116         struct priv *priv = dev->data->dev_private;
 117         struct mlx5_flow_counter *cnt;
 118         int ret;
 119
 120         LIST_FOREACH(cnt, &priv->flow_counters, next) {
 121                 if (!cnt->shared || cnt->shared != shared)
 122                         continue;
 123                 if (cnt->id != id)
 124                         continue;
 125                 cnt->ref_cnt++;
 126                 return cnt;
 127         }
 128         cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
 129         if (!cnt) {
 130                 rte_errno = ENOMEM;
 131                 return NULL;
 132         }
 133         cnt->id = id;
 134         cnt->shared = shared;
 135         cnt->ref_cnt = 1;
 136         cnt->hits = 0;
 137         cnt->bytes = 0;
 138         /* Create counter with Verbs. */
 139         ret = flow_verbs_counter_create(dev, cnt);
 140         if (!ret) {
 141                 LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
 142                 return cnt;
 143         }
 144         /* Some error occurred in Verbs library. */
 145         rte_free(cnt);
 146         rte_errno = -ret;
 147         return NULL;
 148 }
 149
 150 /**
 151  * Release a flow counter.
 152  *
 153  * @param[in] counter
 154  *   Pointer to the counter handler.
 155  */
 156 static void
 157 flow_verbs_counter_release(struct mlx5_flow_counter *counter)
 158 {
 159         if (--counter->ref_cnt == 0) {
 160 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 161                 claim_zero(mlx5_glue->destroy_counter_set(counter->cs));
 162 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 163                 claim_zero(mlx5_glue->destroy_counters(counter->cs));
 164 #endif
 165                 LIST_REMOVE(counter, next);
 166                 rte_free(counter);
 167         }
 168 }
 169
 170 /**
 171  * Query a flow counter via Verbs library call.
 172  *
 173  * @see rte_flow_query()
 174  * @see rte_flow_ops
 175  */
 176 static int
 177 flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
 178                          struct rte_flow *flow, void *data,
 179                          struct rte_flow_error *error)
 180 {
 181 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
 182         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 183         if (flow->actions & MLX5_FLOW_ACTION_COUNT) {
 184                 struct rte_flow_query_count *qc = data;
 185                 uint64_t counters[2] = {0, 0};
 186 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 187                 struct ibv_query_counter_set_attr query_cs_attr = {
 188                         .cs = flow->counter->cs,
 189                         .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
 190                 };
 191                 struct ibv_counter_set_data query_out = {
 192                         .out = counters,
 193                         .outlen = 2 * sizeof(uint64_t),
 194                 };
 195                 int err = mlx5_glue->query_counter_set(&query_cs_attr,
 196                                                        &query_out);
 197 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 198                 int err = mlx5_glue->query_counters
 199                                (flow->counter->cs, counters,
 200                                 RTE_DIM(counters),
 201                                 IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
 202 #endif
 203                 if (err)
 204                         return rte_flow_error_set
 205                                 (error, err,
 206                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 207                                  NULL,
 208                                  "cannot read counter");
 209                 qc->hits_set = 1;
 210                 qc->bytes_set = 1;
 211                 qc->hits = counters[0] - flow->counter->hits;
 212                 qc->bytes = counters[1] - flow->counter->bytes;
 213                 if (qc->reset) {
 214                         flow->counter->hits = counters[0];
 215                         flow->counter->bytes = counters[1];
 216                 }
 217                 return 0;
 218         }
 219         return rte_flow_error_set(error, EINVAL,
 220                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 221                                   NULL,
 222                                   "flow does not have counter");
 223 #else
 224         (void)flow;
 225         (void)data;
 226         return rte_flow_error_set(error, ENOTSUP,
 227                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 228                                   NULL,
 229                                   "counters are not available");
 230 #endif
 231 }
 232
 233 /**
 234  * Add a verbs item specification into @p flow.
 235  *
 236  * @param[in, out] flow
 237  *   Pointer to flow structure.
 238  * @param[in] src
 239  *   Create specification.
 240  * @param[in] size
 241  *   Size in bytes of the specification to copy.
 242  */
 243 static void
 244 flow_verbs_spec_add(struct mlx5_flow *flow, void *src, unsigned int size)
 245 {
 246         struct mlx5_flow_verbs *verbs = &flow->verbs;
 247
 248         if (verbs->specs) {
 249                 void *dst;
 250
 251                 dst = (void *)(verbs->specs + verbs->size);
 252                 memcpy(dst, src, size);
 253                 ++verbs->attr->num_of_specs;
 254         }
 255         verbs->size += size;
 256 }
 257
 258 /**
 259  * Convert the @p item into a Verbs specification. This function assumes that
 260  * the input is valid and that there is space to insert the requested item
 261  * into the flow.
 262  *
 263  * @param[in] item
 264  *   Item specification.
 265  * @param[in] item_flags
 266  *   Bit field with all detected items.
 267  * @param[in, out] dev_flow
 268  *   Pointer to dev_flow structure.
 269  */
 270 static void
 271 flow_verbs_translate_item_eth(const struct rte_flow_item *item,
 272                               uint64_t *item_flags,
 273                               struct mlx5_flow *dev_flow)
 274 {
 275         const struct rte_flow_item_eth *spec = item->spec;
 276         const struct rte_flow_item_eth *mask = item->mask;
 277         const int tunnel = !!(*item_flags & MLX5_FLOW_LAYER_TUNNEL);
 278         const unsigned int size = sizeof(struct ibv_flow_spec_eth);
 279         struct ibv_flow_spec_eth eth = {
 280                 .type = IBV_FLOW_SPEC_ETH | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 281                 .size = size,
 282         };
 283
 284         if (!mask)
 285                 mask = &rte_flow_item_eth_mask;
 286         if (spec) {
 287                 unsigned int i;
 288
 289                 memcpy(&eth.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
 290                 memcpy(&eth.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
 291                 eth.val.ether_type = spec->type;
 292                 memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
 293                 memcpy(&eth.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
 294                 eth.mask.ether_type = mask->type;
 295                 /* Remove unwanted bits from values. */
 296                 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
 297                         eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
 298                         eth.val.src_mac[i] &= eth.mask.src_mac[i];
 299                 }
 300                 eth.val.ether_type &= eth.mask.ether_type;
 301                 dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L2;
 302         }
 303         flow_verbs_spec_add(dev_flow, &eth, size);
 304         *item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
 305                                 MLX5_FLOW_LAYER_OUTER_L2;
 306 }
 307
 308 /**
 309  * Update the VLAN tag in the Verbs Ethernet specification.
 310  * This function assumes that the input is valid and there is space to add
 311  * the requested item.
 312  *
 313  * @param[in, out] attr
 314  *   Pointer to Verbs attributes structure.
 315  * @param[in] eth
 316  *   Verbs structure containing the VLAN information to copy.
 317  */
 318 static void
 319 flow_verbs_item_vlan_update(struct ibv_flow_attr *attr,
 320                             struct ibv_flow_spec_eth *eth)
 321 {
 322         unsigned int i;
 323         const enum ibv_flow_spec_type search = eth->type;
 324         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 325                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 326
 327         for (i = 0; i != attr->num_of_specs; ++i) {
 328                 if (hdr->type == search) {
 329                         struct ibv_flow_spec_eth *e =
 330                                 (struct ibv_flow_spec_eth *)hdr;
 331
 332                         e->val.vlan_tag = eth->val.vlan_tag;
 333                         e->mask.vlan_tag = eth->mask.vlan_tag;
 334                         e->val.ether_type = eth->val.ether_type;
 335                         e->mask.ether_type = eth->mask.ether_type;
 336                         break;
 337                 }
 338                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 339         }
 340 }
 341
 342 /**
 343  * Convert the @p item into a Verbs specification. This function assumes that
 344  * the input is valid and that there is space to insert the requested item
 345  * into the flow.
 346  *
 347  * @param[in] item
 348  *   Item specification.
 349  * @param[in, out] item_flags
 350  *   Bit mask that holds all detected items.
 351  * @param[in, out] dev_flow
 352  *   Pointer to dev_flow structure.
 353  */
 354 static void
 355 flow_verbs_translate_item_vlan(const struct rte_flow_item *item,
 356                                uint64_t *item_flags,
 357                                struct mlx5_flow *dev_flow)
 358 {
 359         const struct rte_flow_item_vlan *spec = item->spec;
 360         const struct rte_flow_item_vlan *mask = item->mask;
 361         unsigned int size = sizeof(struct ibv_flow_spec_eth);
 362         const int tunnel = !!(*item_flags & MLX5_FLOW_LAYER_TUNNEL);
 363         struct ibv_flow_spec_eth eth = {
 364                 .type = IBV_FLOW_SPEC_ETH | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 365                 .size = size,
 366         };
 367         const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
 368                                       MLX5_FLOW_LAYER_OUTER_L2;
 369
 370         if (!mask)
 371                 mask = &rte_flow_item_vlan_mask;
 372         if (spec) {
 373                 eth.val.vlan_tag = spec->tci;
 374                 eth.mask.vlan_tag = mask->tci;
 375                 eth.val.vlan_tag &= eth.mask.vlan_tag;
 376                 eth.val.ether_type = spec->inner_type;
 377                 eth.mask.ether_type = mask->inner_type;
 378                 eth.val.ether_type &= eth.mask.ether_type;
 379         }
 380         if (!(*item_flags & l2m)) {
 381                 dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L2;
 382                 flow_verbs_spec_add(dev_flow, &eth, size);
 383         } else {
 384                 flow_verbs_item_vlan_update(dev_flow->verbs.attr, &eth);
 385                 size = 0; /* Only an update is done in eth specification. */
 386         }
 387         *item_flags |= tunnel ?
 388                        (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_VLAN) :
 389                        (MLX5_FLOW_LAYER_OUTER_L2 | MLX5_FLOW_LAYER_OUTER_VLAN);
 390 }
 391
 392 /**
 393  * Convert the @p item into a Verbs specification. This function assumes that
 394  * the input is valid and that there is space to insert the requested item
 395  * into the flow.
 396  *
 397  * @param[in] item
 398  *   Item specification.
 399  * @param[in, out] item_flags
 400  *   Bit mask that marks all detected items.
 401  * @param[in, out] dev_flow
 402  *   Pointer to sepacific flow structure.
 403  */
 404 static void
 405 flow_verbs_translate_item_ipv4(const struct rte_flow_item *item,
 406                                uint64_t *item_flags,
 407                                struct mlx5_flow *dev_flow)
 408 {
 409         const struct rte_flow_item_ipv4 *spec = item->spec;
 410         const struct rte_flow_item_ipv4 *mask = item->mask;
 411         const int tunnel = !!(*item_flags & MLX5_FLOW_LAYER_TUNNEL);
 412         unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
 413         struct ibv_flow_spec_ipv4_ext ipv4 = {
 414                 .type = IBV_FLOW_SPEC_IPV4_EXT |
 415                         (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 416                 .size = size,
 417         };
 418
 419         if (!mask)
 420                 mask = &rte_flow_item_ipv4_mask;
 421         *item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
 422                                 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
 423         if (spec) {
 424                 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
 425                         .src_ip = spec->hdr.src_addr,
 426                         .dst_ip = spec->hdr.dst_addr,
 427                         .proto = spec->hdr.next_proto_id,
 428                         .tos = spec->hdr.type_of_service,
 429                 };
 430                 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
 431                         .src_ip = mask->hdr.src_addr,
 432                         .dst_ip = mask->hdr.dst_addr,
 433                         .proto = mask->hdr.next_proto_id,
 434                         .tos = mask->hdr.type_of_service,
 435                 };
 436                 /* Remove unwanted bits from values. */
 437                 ipv4.val.src_ip &= ipv4.mask.src_ip;
 438                 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
 439                 ipv4.val.proto &= ipv4.mask.proto;
 440                 ipv4.val.tos &= ipv4.mask.tos;
 441         }
 442         dev_flow->verbs.hash_fields |=
 443                 mlx5_flow_hashfields_adjust(dev_flow, tunnel,
 444                                             MLX5_IPV4_LAYER_TYPES,
 445                                             MLX5_IPV4_IBV_RX_HASH);
 446         dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L3;
 447         flow_verbs_spec_add(dev_flow, &ipv4, size);
 448 }
 449
 450 /**
 451  * Convert the @p item into a Verbs specification. This function assumes that
 452  * the input is valid and that there is space to insert the requested item
 453  * into the flow.
 454  *
 455  * @param[in] item
 456  *   Item specification.
 457  * @param[in, out] item_flags
 458  *   Bit mask that marks all detected items.
 459  * @param[in, out] dev_flow
 460  *   Pointer to sepacific flow structure.
 461  */
 462 static void
 463 flow_verbs_translate_item_ipv6(const struct rte_flow_item *item,
 464                                uint64_t *item_flags,
 465                                struct mlx5_flow *dev_flow)
 466 {
 467         const struct rte_flow_item_ipv6 *spec = item->spec;
 468         const struct rte_flow_item_ipv6 *mask = item->mask;
 469         const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 470         unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
 471         struct ibv_flow_spec_ipv6 ipv6 = {
 472                 .type = IBV_FLOW_SPEC_IPV6 | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 473                 .size = size,
 474         };
 475
 476         if (!mask)
 477                 mask = &rte_flow_item_ipv6_mask;
 478          *item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
 479                                  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
 480         if (spec) {
 481                 unsigned int i;
 482                 uint32_t vtc_flow_val;
 483                 uint32_t vtc_flow_mask;
 484
 485                 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
 486                        RTE_DIM(ipv6.val.src_ip));
 487                 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
 488                        RTE_DIM(ipv6.val.dst_ip));
 489                 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
 490                        RTE_DIM(ipv6.mask.src_ip));
 491                 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
 492                        RTE_DIM(ipv6.mask.dst_ip));
 493                 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
 494                 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
 495                 ipv6.val.flow_label =
 496                         rte_cpu_to_be_32((vtc_flow_val & IPV6_HDR_FL_MASK) >>
 497                                          IPV6_HDR_FL_SHIFT);
 498                 ipv6.val.traffic_class = (vtc_flow_val & IPV6_HDR_TC_MASK) >>
 499                                          IPV6_HDR_TC_SHIFT;
 500                 ipv6.val.next_hdr = spec->hdr.proto;
 501                 ipv6.val.hop_limit = spec->hdr.hop_limits;
 502                 ipv6.mask.flow_label =
 503                         rte_cpu_to_be_32((vtc_flow_mask & IPV6_HDR_FL_MASK) >>
 504                                          IPV6_HDR_FL_SHIFT);
 505                 ipv6.mask.traffic_class = (vtc_flow_mask & IPV6_HDR_TC_MASK) >>
 506                                           IPV6_HDR_TC_SHIFT;
 507                 ipv6.mask.next_hdr = mask->hdr.proto;
 508                 ipv6.mask.hop_limit = mask->hdr.hop_limits;
 509                 /* Remove unwanted bits from values. */
 510                 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
 511                         ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
 512                         ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
 513                 }
 514                 ipv6.val.flow_label &= ipv6.mask.flow_label;
 515                 ipv6.val.traffic_class &= ipv6.mask.traffic_class;
 516                 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
 517                 ipv6.val.hop_limit &= ipv6.mask.hop_limit;
 518         }
 519         dev_flow->verbs.hash_fields |=
 520                 mlx5_flow_hashfields_adjust(dev_flow, tunnel,
 521                                             MLX5_IPV6_LAYER_TYPES,
 522                                             MLX5_IPV6_IBV_RX_HASH);
 523         dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L3;
 524         flow_verbs_spec_add(dev_flow, &ipv6, size);
 525 }
 526
 527 /**
 528  * Convert the @p item into a Verbs specification. This function assumes that
 529  * the input is valid and that there is space to insert the requested item
 530  * into the flow.
 531  *
 532  * @param[in] item
 533  *   Item specification.
 534  * @param[in, out] item_flags
 535  *   Bit mask that marks all detected items.
 536  * @param[in, out] dev_flow
 537  *   Pointer to sepacific flow structure.
 538  */
 539 static void
 540 flow_verbs_translate_item_udp(const struct rte_flow_item *item,
 541                               uint64_t *item_flags,
 542                               struct mlx5_flow *dev_flow)
 543 {
 544         const struct rte_flow_item_udp *spec = item->spec;
 545         const struct rte_flow_item_udp *mask = item->mask;
 546         const int tunnel = !!(*item_flags & MLX5_FLOW_LAYER_TUNNEL);
 547         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 548         struct ibv_flow_spec_tcp_udp udp = {
 549                 .type = IBV_FLOW_SPEC_UDP | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 550                 .size = size,
 551         };
 552
 553         if (!mask)
 554                 mask = &rte_flow_item_udp_mask;
 555         *item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
 556                                 MLX5_FLOW_LAYER_OUTER_L4_UDP;
 557         if (spec) {
 558                 udp.val.dst_port = spec->hdr.dst_port;
 559                 udp.val.src_port = spec->hdr.src_port;
 560                 udp.mask.dst_port = mask->hdr.dst_port;
 561                 udp.mask.src_port = mask->hdr.src_port;
 562                 /* Remove unwanted bits from values. */
 563                 udp.val.src_port &= udp.mask.src_port;
 564                 udp.val.dst_port &= udp.mask.dst_port;
 565         }
 566         dev_flow->verbs.hash_fields |=
 567                 mlx5_flow_hashfields_adjust(dev_flow, tunnel, ETH_RSS_UDP,
 568                                             (IBV_RX_HASH_SRC_PORT_UDP |
 569                                              IBV_RX_HASH_DST_PORT_UDP));
 570         dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L4;
 571         flow_verbs_spec_add(dev_flow, &udp, size);
 572 }
 573
 574 /**
 575  * Convert the @p item into a Verbs specification. This function assumes that
 576  * the input is valid and that there is space to insert the requested item
 577  * into the flow.
 578  *
 579  * @param[in] item
 580  *   Item specification.
 581  * @param[in, out] item_flags
 582  *   Bit mask that marks all detected items.
 583  * @param[in, out] dev_flow
 584  *   Pointer to sepacific flow structure.
 585  */
 586 static void
 587 flow_verbs_translate_item_tcp(const struct rte_flow_item *item,
 588                               uint64_t *item_flags,
 589                               struct mlx5_flow *dev_flow)
 590 {
 591         const struct rte_flow_item_tcp *spec = item->spec;
 592         const struct rte_flow_item_tcp *mask = item->mask;
 593         const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 594         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 595         struct ibv_flow_spec_tcp_udp tcp = {
 596                 .type = IBV_FLOW_SPEC_TCP | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 597                 .size = size,
 598         };
 599
 600         if (!mask)
 601                 mask = &rte_flow_item_tcp_mask;
 602         *item_flags |=  tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
 603                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
 604         if (spec) {
 605                 tcp.val.dst_port = spec->hdr.dst_port;
 606                 tcp.val.src_port = spec->hdr.src_port;
 607                 tcp.mask.dst_port = mask->hdr.dst_port;
 608                 tcp.mask.src_port = mask->hdr.src_port;
 609                 /* Remove unwanted bits from values. */
 610                 tcp.val.src_port &= tcp.mask.src_port;
 611                 tcp.val.dst_port &= tcp.mask.dst_port;
 612         }
 613         dev_flow->verbs.hash_fields |=
 614                 mlx5_flow_hashfields_adjust(dev_flow, tunnel, ETH_RSS_TCP,
 615                                             (IBV_RX_HASH_SRC_PORT_TCP |
 616                                              IBV_RX_HASH_DST_PORT_TCP));
 617         dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L4;
 618         flow_verbs_spec_add(dev_flow, &tcp, size);
 619 }
 620
 621 /**
 622  * Convert the @p item into a Verbs specification. This function assumes that
 623  * the input is valid and that there is space to insert the requested item
 624  * into the flow.
 625  *
 626  * @param[in] item
 627  *   Item specification.
 628  * @param[in, out] item_flags
 629  *   Bit mask that marks all detected items.
 630  * @param[in, out] dev_flow
 631  *   Pointer to sepacific flow structure.
 632  */
 633 static void
 634 flow_verbs_translate_item_vxlan(const struct rte_flow_item *item,
 635                                 uint64_t *item_flags,
 636                                 struct mlx5_flow *dev_flow)
 637 {
 638         const struct rte_flow_item_vxlan *spec = item->spec;
 639         const struct rte_flow_item_vxlan *mask = item->mask;
 640         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 641         struct ibv_flow_spec_tunnel vxlan = {
 642                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 643                 .size = size,
 644         };
 645         union vni {
 646                 uint32_t vlan_id;
 647                 uint8_t vni[4];
 648         } id = { .vlan_id = 0, };
 649
 650         if (!mask)
 651                 mask = &rte_flow_item_vxlan_mask;
 652         if (spec) {
 653                 memcpy(&id.vni[1], spec->vni, 3);
 654                 vxlan.val.tunnel_id = id.vlan_id;
 655                 memcpy(&id.vni[1], mask->vni, 3);
 656                 vxlan.mask.tunnel_id = id.vlan_id;
 657                 /* Remove unwanted bits from values. */
 658                 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
 659         }
 660         flow_verbs_spec_add(dev_flow, &vxlan, size);
 661         dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L2;
 662         *item_flags |= MLX5_FLOW_LAYER_VXLAN;
 663 }
 664
 665 /**
 666  * Convert the @p item into a Verbs specification. This function assumes that
 667  * the input is valid and that there is space to insert the requested item
 668  * into the flow.
 669  *
 670  * @param[in] item
 671  *   Item specification.
 672  * @param[in, out] item_flags
 673  *   Bit mask that marks all detected items.
 674  * @param[in, out] dev_flow
 675  *   Pointer to sepacific flow structure.
 676  */
 677 static void
 678 flow_verbs_translate_item_vxlan_gpe(const struct rte_flow_item *item,
 679                                     uint64_t *item_flags,
 680                                     struct mlx5_flow *dev_flow)
 681 {
 682         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
 683         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
 684         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 685         struct ibv_flow_spec_tunnel vxlan_gpe = {
 686                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 687                 .size = size,
 688         };
 689         union vni {
 690                 uint32_t vlan_id;
 691                 uint8_t vni[4];
 692         } id = { .vlan_id = 0, };
 693
 694         if (!mask)
 695                 mask = &rte_flow_item_vxlan_gpe_mask;
 696         if (spec) {
 697                 memcpy(&id.vni[1], spec->vni, 3);
 698                 vxlan_gpe.val.tunnel_id = id.vlan_id;
 699                 memcpy(&id.vni[1], mask->vni, 3);
 700                 vxlan_gpe.mask.tunnel_id = id.vlan_id;
 701                 /* Remove unwanted bits from values. */
 702                 vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
 703         }
 704         flow_verbs_spec_add(dev_flow, &vxlan_gpe, size);
 705         dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L2;
 706         *item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
 707 }
 708
 709 /**
 710  * Update the protocol in Verbs IPv4/IPv6 spec.
 711  *
 712  * @param[in, out] attr
 713  *   Pointer to Verbs attributes structure.
 714  * @param[in] search
 715  *   Specification type to search in order to update the IP protocol.
 716  * @param[in] protocol
 717  *   Protocol value to set if none is present in the specification.
 718  */
 719 static void
 720 flow_verbs_item_gre_ip_protocol_update(struct ibv_flow_attr *attr,
 721                                        enum ibv_flow_spec_type search,
 722                                        uint8_t protocol)
 723 {
 724         unsigned int i;
 725         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 726                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 727
 728         if (!attr)
 729                 return;
 730         for (i = 0; i != attr->num_of_specs; ++i) {
 731                 if (hdr->type == search) {
 732                         union {
 733                                 struct ibv_flow_spec_ipv4_ext *ipv4;
 734                                 struct ibv_flow_spec_ipv6 *ipv6;
 735                         } ip;
 736
 737                         switch (search) {
 738                         case IBV_FLOW_SPEC_IPV4_EXT:
 739                                 ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr;
 740                                 if (!ip.ipv4->val.proto) {
 741                                         ip.ipv4->val.proto = protocol;
 742                                         ip.ipv4->mask.proto = 0xff;
 743                                 }
 744                                 break;
 745                         case IBV_FLOW_SPEC_IPV6:
 746                                 ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr;
 747                                 if (!ip.ipv6->val.next_hdr) {
 748                                         ip.ipv6->val.next_hdr = protocol;
 749                                         ip.ipv6->mask.next_hdr = 0xff;
 750                                 }
 751                                 break;
 752                         default:
 753                                 break;
 754                         }
 755                         break;
 756                 }
 757                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 758         }
 759 }
 760
 761 /**
 762  * Convert the @p item into a Verbs specification. This function assumes that
 763  * the input is valid and that there is space to insert the requested item
 764  * into the flow.
 765  *
 766  * @param[in] item
 767  *   Item specification.
 768  * @param[in, out] item_flags
 769  *   Bit mask that marks all detected items.
 770  * @param[in, out] dev_flow
 771  *   Pointer to sepacific flow structure.
 772  */
 773 static void
 774 flow_verbs_translate_item_gre(const struct rte_flow_item *item __rte_unused,
 775                               uint64_t *item_flags,
 776                               struct mlx5_flow *dev_flow)
 777 {
 778         struct mlx5_flow_verbs *verbs = &dev_flow->verbs;
 779 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
 780         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 781         struct ibv_flow_spec_tunnel tunnel = {
 782                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 783                 .size = size,
 784         };
 785 #else
 786         const struct rte_flow_item_gre *spec = item->spec;
 787         const struct rte_flow_item_gre *mask = item->mask;
 788         unsigned int size = sizeof(struct ibv_flow_spec_gre);
 789         struct ibv_flow_spec_gre tunnel = {
 790                 .type = IBV_FLOW_SPEC_GRE,
 791                 .size = size,
 792         };
 793
 794         if (!mask)
 795                 mask = &rte_flow_item_gre_mask;
 796         if (spec) {
 797                 tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
 798                 tunnel.val.protocol = spec->protocol;
 799                 tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
 800                 tunnel.mask.protocol = mask->protocol;
 801                 /* Remove unwanted bits from values. */
 802                 tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
 803                 tunnel.val.protocol &= tunnel.mask.protocol;
 804                 tunnel.val.key &= tunnel.mask.key;
 805         }
 806 #endif
 807         if (*item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 808                 flow_verbs_item_gre_ip_protocol_update(verbs->attr,
 809                                                        IBV_FLOW_SPEC_IPV4_EXT,
 810                                                        IPPROTO_GRE);
 811         else
 812                 flow_verbs_item_gre_ip_protocol_update(verbs->attr,
 813                                                        IBV_FLOW_SPEC_IPV6,
 814                                                        IPPROTO_GRE);
 815         flow_verbs_spec_add(dev_flow, &tunnel, size);
 816         verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
 817         *item_flags |= MLX5_FLOW_LAYER_GRE;
 818 }
 819
 820 /**
 821  * Convert the @p action into a Verbs specification. This function assumes that
 822  * the input is valid and that there is space to insert the requested action
 823  * into the flow. This function also return the action that was added.
 824  *
 825  * @param[in] item
 826  *   Item specification.
 827  * @param[in, out] item_flags
 828  *   Bit mask that marks all detected items.
 829  * @param[in, out] dev_flow
 830  *   Pointer to sepacific flow structure.
 831  */
 832 static void
 833 flow_verbs_translate_item_mpls(const struct rte_flow_item *item __rte_unused,
 834                                uint64_t *action_flags __rte_unused,
 835                                struct mlx5_flow *dev_flow __rte_unused)
 836 {
 837 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
 838         const struct rte_flow_item_mpls *spec = item->spec;
 839         const struct rte_flow_item_mpls *mask = item->mask;
 840         unsigned int size = sizeof(struct ibv_flow_spec_mpls);
 841         struct ibv_flow_spec_mpls mpls = {
 842                 .type = IBV_FLOW_SPEC_MPLS,
 843                 .size = size,
 844         };
 845
 846         if (!mask)
 847                 mask = &rte_flow_item_mpls_mask;
 848         if (spec) {
 849                 memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
 850                 memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
 851                 /* Remove unwanted bits from values.  */
 852                 mpls.val.label &= mpls.mask.label;
 853         }
 854         flow_verbs_spec_add(dev_flow, &mpls, size);
 855         dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L2;
 856         *action_flags |= MLX5_FLOW_LAYER_MPLS;
 857 #endif
 858 }
 859
 860 /**
 861  * Convert the @p action into a Verbs specification. This function assumes that
 862  * the input is valid and that there is space to insert the requested action
 863  * into the flow. This function also return the action that was added.
 864  *
 865  * @param[in, out] action_flags
 866  *   Pointer to the detected actions.
 867  * @param[in] dev_flow
 868  *   Pointer to mlx5_flow.
 869  */
 870 static void
 871 flow_verbs_translate_action_drop(uint64_t *action_flags,
 872                                  struct mlx5_flow *dev_flow)
 873 {
 874         unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
 875         struct ibv_flow_spec_action_drop drop = {
 876                         .type = IBV_FLOW_SPEC_ACTION_DROP,
 877                         .size = size,
 878         };
 879
 880         flow_verbs_spec_add(dev_flow, &drop, size);
 881         *action_flags |= MLX5_FLOW_ACTION_DROP;
 882 }
 883
 884 /**
 885  * Convert the @p action into a Verbs specification. This function assumes that
 886  * the input is valid and that there is space to insert the requested action
 887  * into the flow. This function also return the action that was added.
 888  *
 889  * @param[in] action
 890  *   Action configuration.
 891  * @param[in, out] action_flags
 892  *   Pointer to the detected actions.
 893  * @param[in] dev_flow
 894  *   Pointer to mlx5_flow.
 895  */
 896 static void
 897 flow_verbs_translate_action_queue(const struct rte_flow_action *action,
 898                                   uint64_t *action_flags,
 899                                   struct mlx5_flow *dev_flow)
 900 {
 901         const struct rte_flow_action_queue *queue = action->conf;
 902         struct rte_flow *flow = dev_flow->flow;
 903
 904         if (flow->queue)
 905                 (*flow->queue)[0] = queue->index;
 906         flow->rss.queue_num = 1;
 907         *action_flags |= MLX5_FLOW_ACTION_QUEUE;
 908 }
 909
 910 /**
 911  * Convert the @p action into a Verbs specification. This function assumes that
 912  * the input is valid and that there is space to insert the requested action
 913  * into the flow. This function also return the action that was added.
 914  *
 915  * @param[in] action
 916  *   Action configuration.
 917  * @param[in, out] action_flags
 918  *   Pointer to the detected actions.
 919  * @param[in] dev_flow
 920  *   Pointer to mlx5_flow.
 921  */
 922 static void
 923 flow_verbs_translate_action_rss(const struct rte_flow_action *action,
 924                                 uint64_t *action_flags,
 925                                 struct mlx5_flow *dev_flow)
 926 {
 927         const struct rte_flow_action_rss *rss = action->conf;
 928         struct rte_flow *flow = dev_flow->flow;
 929
 930         if (flow->queue)
 931                 memcpy((*flow->queue), rss->queue,
 932                        rss->queue_num * sizeof(uint16_t));
 933         flow->rss.queue_num = rss->queue_num;
 934         memcpy(flow->key, rss->key, MLX5_RSS_HASH_KEY_LEN);
 935         flow->rss.types = rss->types;
 936         flow->rss.level = rss->level;
 937         *action_flags |= MLX5_FLOW_ACTION_RSS;
 938 }
 939
 940 /**
 941  * Convert the @p action into a Verbs specification. This function assumes that
 942  * the input is valid and that there is space to insert the requested action
 943  * into the flow. This function also return the action that was added.
 944  *
 945  * @param[in] action
 946  *   Action configuration.
 947  * @param[in, out] action_flags
 948  *   Pointer to the detected actions.
 949  * @param[in] dev_flow
 950  *   Pointer to mlx5_flow.
 951  */
 952 static void
 953 flow_verbs_translate_action_flag
 954                         (const struct rte_flow_action *action __rte_unused,
 955                          uint64_t *action_flags,
 956                          struct mlx5_flow *dev_flow)
 957 {
 958         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
 959         struct ibv_flow_spec_action_tag tag = {
 960                 .type = IBV_FLOW_SPEC_ACTION_TAG,
 961                 .size = size,
 962                 .tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
 963         };
 964         *action_flags |= MLX5_FLOW_ACTION_MARK;
 965         flow_verbs_spec_add(dev_flow, &tag, size);
 966 }
 967
 968 /**
 969  * Update verbs specification to modify the flag to mark.
 970  *
 971  * @param[in, out] verbs
 972  *   Pointer to the mlx5_flow_verbs structure.
 973  * @param[in] mark_id
 974  *   Mark identifier to replace the flag.
 975  */
 976 static void
 977 flow_verbs_mark_update(struct mlx5_flow_verbs *verbs, uint32_t mark_id)
 978 {
 979         struct ibv_spec_header *hdr;
 980         int i;
 981
 982         if (!verbs)
 983                 return;
 984         /* Update Verbs specification. */
 985         hdr = (struct ibv_spec_header *)verbs->specs;
 986         if (!hdr)
 987                 return;
 988         for (i = 0; i != verbs->attr->num_of_specs; ++i) {
 989                 if (hdr->type == IBV_FLOW_SPEC_ACTION_TAG) {
 990                         struct ibv_flow_spec_action_tag *t =
 991                                 (struct ibv_flow_spec_action_tag *)hdr;
 992
 993                         t->tag_id = mlx5_flow_mark_set(mark_id);
 994                 }
 995                 hdr = (struct ibv_spec_header *)((uintptr_t)hdr + hdr->size);
 996         }
 997 }
 998
 999 /**
1000  * Convert the @p action into a Verbs specification. This function assumes that
1001  * the input is valid and that there is space to insert the requested action
1002  * into the flow. This function also return the action that was added.
1003  *
1004  * @param[in] action
1005  *   Action configuration.
1006  * @param[in, out] action_flags
1007  *   Pointer to the detected actions.
1008  * @param[in] dev_flow
1009  *   Pointer to mlx5_flow.
1010  */
1011 static void
1012 flow_verbs_translate_action_mark(const struct rte_flow_action *action,
1013                                  uint64_t *action_flags,
1014                                  struct mlx5_flow *dev_flow)
1015 {
1016         const struct rte_flow_action_mark *mark = action->conf;
1017         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1018         struct ibv_flow_spec_action_tag tag = {
1019                 .type = IBV_FLOW_SPEC_ACTION_TAG,
1020                 .size = size,
1021         };
1022         struct mlx5_flow_verbs *verbs = &dev_flow->verbs;
1023
1024         if (*action_flags & MLX5_FLOW_ACTION_FLAG) {
1025                 flow_verbs_mark_update(verbs, mark->id);
1026                 size = 0;
1027         } else {
1028                 tag.tag_id = mlx5_flow_mark_set(mark->id);
1029                 flow_verbs_spec_add(dev_flow, &tag, size);
1030         }
1031         *action_flags |= MLX5_FLOW_ACTION_MARK;
1032 }
1033
1034 /**
1035  * Convert the @p action into a Verbs specification. This function assumes that
1036  * the input is valid and that there is space to insert the requested action
1037  * into the flow. This function also return the action that was added.
1038  *
1039  * @param[in] dev
1040  *   Pointer to the Ethernet device structure.
1041  * @param[in] action
1042  *   Action configuration.
1043  * @param[in, out] action_flags
1044  *   Pointer to the detected actions.
1045  * @param[in] dev_flow
1046  *   Pointer to mlx5_flow.
1047  * @param[out] error
1048  *   Pointer to error structure.
1049  *
1050  * @return
1051  *   0 On success else a negative errno value is returned and rte_errno is set.
1052  */
1053 static int
1054 flow_verbs_translate_action_count(struct rte_eth_dev *dev,
1055                                   const struct rte_flow_action *action,
1056                                   uint64_t *action_flags,
1057                                   struct mlx5_flow *dev_flow,
1058                                   struct rte_flow_error *error)
1059 {
1060         const struct rte_flow_action_count *count = action->conf;
1061         struct rte_flow *flow = dev_flow->flow;
1062 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1063         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1064         unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
1065         struct ibv_flow_spec_counter_action counter = {
1066                 .type = IBV_FLOW_SPEC_ACTION_COUNT,
1067                 .size = size,
1068         };
1069 #endif
1070
1071         if (!flow->counter) {
1072                 flow->counter = flow_verbs_counter_new(dev, count->shared,
1073                                                        count->id);
1074                 if (!flow->counter)
1075                         return rte_flow_error_set(error, rte_errno,
1076                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1077                                                   action,
1078                                                   "cannot get counter"
1079                                                   " context.");
1080         }
1081         *action_flags |= MLX5_FLOW_ACTION_COUNT;
1082 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
1083         counter.counter_set_handle = flow->counter->cs->handle;
1084         flow_verbs_spec_add(dev_flow, &counter, size);
1085 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1086         counter.counters = flow->counter->cs;
1087         flow_verbs_spec_add(dev_flow, &counter, size);
1088 #endif
1089         return 0;
1090 }
1091
1092 /**
1093  * Internal validation function. For validating both actions and items.
1094  *
1095  * @param[in] dev
1096  *   Pointer to the Ethernet device structure.
1097  * @param[in] attr
1098  *   Pointer to the flow attributes.
1099  * @param[in] items
1100  *   Pointer to the list of items.
1101  * @param[in] actions
1102  *   Pointer to the list of actions.
1103  * @param[out] error
1104  *   Pointer to the error structure.
1105  *
1106  * @return
1107  *   0 on success, a negative errno value otherwise and rte_errno is set.
1108  */
1109 static int
1110 flow_verbs_validate(struct rte_eth_dev *dev,
1111                     const struct rte_flow_attr *attr,
1112                     const struct rte_flow_item items[],
1113                     const struct rte_flow_action actions[],
1114                     struct rte_flow_error *error)
1115 {
1116         int ret;
1117         uint64_t action_flags = 0;
1118         uint64_t item_flags = 0;
1119         int tunnel = 0;
1120         uint8_t next_protocol = 0xff;
1121
1122         if (items == NULL)
1123                 return -1;
1124         ret = mlx5_flow_validate_attributes(dev, attr, error);
1125         if (ret < 0)
1126                 return ret;
1127         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1128                 int ret = 0;
1129
1130                 tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1131                 switch (items->type) {
1132                 case RTE_FLOW_ITEM_TYPE_VOID:
1133                         break;
1134                 case RTE_FLOW_ITEM_TYPE_ETH:
1135                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1136                                                           error);
1137                         if (ret < 0)
1138                                 return ret;
1139                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1140                                                MLX5_FLOW_LAYER_OUTER_L2;
1141                         break;
1142                 case RTE_FLOW_ITEM_TYPE_VLAN:
1143                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
1144                                                            error);
1145                         if (ret < 0)
1146                                 return ret;
1147                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1148                                                MLX5_FLOW_LAYER_OUTER_VLAN;
1149                         break;
1150                 case RTE_FLOW_ITEM_TYPE_IPV4:
1151                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
1152                                                            error);
1153                         if (ret < 0)
1154                                 return ret;
1155                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1156                                                MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1157                         if (items->mask != NULL &&
1158                             ((const struct rte_flow_item_ipv4 *)
1159                              items->mask)->hdr.next_proto_id)
1160                                 next_protocol =
1161                                         ((const struct rte_flow_item_ipv4 *)
1162                                          (items->spec))->hdr.next_proto_id;
1163                         break;
1164                 case RTE_FLOW_ITEM_TYPE_IPV6:
1165                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1166                                                            error);
1167                         if (ret < 0)
1168                                 return ret;
1169                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1170                                                MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1171                         if (items->mask != NULL &&
1172                             ((const struct rte_flow_item_ipv6 *)
1173                              items->mask)->hdr.proto)
1174                                 next_protocol =
1175                                         ((const struct rte_flow_item_ipv6 *)
1176                                          items->spec)->hdr.proto;
1177                         break;
1178                 case RTE_FLOW_ITEM_TYPE_UDP:
1179                         ret = mlx5_flow_validate_item_udp(items, item_flags,
1180                                                           next_protocol,
1181                                                           error);
1182                         if (ret < 0)
1183                                 return ret;
1184                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1185                                                MLX5_FLOW_LAYER_OUTER_L4_UDP;
1186                         break;
1187                 case RTE_FLOW_ITEM_TYPE_TCP:
1188                         ret = mlx5_flow_validate_item_tcp
1189                                                 (items, item_flags,
1190                                                  next_protocol,
1191                                                  &rte_flow_item_tcp_mask,
1192                                                  error);
1193                         if (ret < 0)
1194                                 return ret;
1195                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1196                                                MLX5_FLOW_LAYER_OUTER_L4_TCP;
1197                         break;
1198                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1199                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
1200                                                             error);
1201                         if (ret < 0)
1202                                 return ret;
1203                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
1204                         break;
1205                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1206                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
1207                                                                 item_flags,
1208                                                                 dev, error);
1209                         if (ret < 0)
1210                                 return ret;
1211                         item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
1212                         break;
1213                 case RTE_FLOW_ITEM_TYPE_GRE:
1214                         ret = mlx5_flow_validate_item_gre(items, item_flags,
1215                                                           next_protocol, error);
1216                         if (ret < 0)
1217                                 return ret;
1218                         item_flags |= MLX5_FLOW_LAYER_GRE;
1219                         break;
1220                 case RTE_FLOW_ITEM_TYPE_MPLS:
1221                         ret = mlx5_flow_validate_item_mpls(items, item_flags,
1222                                                            next_protocol,
1223                                                            error);
1224                         if (ret < 0)
1225                                 return ret;
1226                         if (next_protocol != 0xff &&
1227                             next_protocol != IPPROTO_MPLS)
1228                                 return rte_flow_error_set
1229                                         (error, EINVAL,
1230                                          RTE_FLOW_ERROR_TYPE_ITEM, items,
1231                                          "protocol filtering not compatible"
1232                                          " with MPLS layer");
1233                         item_flags |= MLX5_FLOW_LAYER_MPLS;
1234                         break;
1235                 default:
1236                         return rte_flow_error_set(error, ENOTSUP,
1237                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1238                                                   NULL, "item not supported");
1239                 }
1240         }
1241         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1242                 switch (actions->type) {
1243                 case RTE_FLOW_ACTION_TYPE_VOID:
1244                         break;
1245                 case RTE_FLOW_ACTION_TYPE_FLAG:
1246                         ret = mlx5_flow_validate_action_flag(action_flags,
1247                                                              attr,
1248                                                              error);
1249                         if (ret < 0)
1250                                 return ret;
1251                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1252                         break;
1253                 case RTE_FLOW_ACTION_TYPE_MARK:
1254                         ret = mlx5_flow_validate_action_mark(actions,
1255                                                              action_flags,
1256                                                              attr,
1257                                                              error);
1258                         if (ret < 0)
1259                                 return ret;
1260                         action_flags |= MLX5_FLOW_ACTION_MARK;
1261                         break;
1262                 case RTE_FLOW_ACTION_TYPE_DROP:
1263                         ret = mlx5_flow_validate_action_drop(action_flags,
1264                                                              attr,
1265                                                              error);
1266                         if (ret < 0)
1267                                 return ret;
1268                         action_flags |= MLX5_FLOW_ACTION_DROP;
1269                         break;
1270                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1271                         ret = mlx5_flow_validate_action_queue(actions,
1272                                                               action_flags, dev,
1273                                                               attr,
1274                                                               error);
1275                         if (ret < 0)
1276                                 return ret;
1277                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1278                         break;
1279                 case RTE_FLOW_ACTION_TYPE_RSS:
1280                         ret = mlx5_flow_validate_action_rss(actions,
1281                                                             action_flags, dev,
1282                                                             attr,
1283                                                             error);
1284                         if (ret < 0)
1285                                 return ret;
1286                         action_flags |= MLX5_FLOW_ACTION_RSS;
1287                         break;
1288                 case RTE_FLOW_ACTION_TYPE_COUNT:
1289                         ret = mlx5_flow_validate_action_count(dev, attr, error);
1290                         if (ret < 0)
1291                                 return ret;
1292                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1293                         break;
1294                 default:
1295                         return rte_flow_error_set(error, ENOTSUP,
1296                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1297                                                   actions,
1298                                                   "action not supported");
1299                 }
1300         }
1301         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS))
1302                 return rte_flow_error_set(error, EINVAL,
1303                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
1304                                           "no fate action is found");
1305         return 0;
1306 }
1307
1308 /**
1309  * Calculate the required bytes that are needed for the action part of the verbs
1310  * flow, in addtion returns bit-fields with all the detected action, in order to
1311  * avoid another interation over the actions.
1312  *
1313  * @param[in] actions
1314  *   Pointer to the list of actions.
1315  * @param[out] action_flags
1316  *   Pointer to the detected actions.
1317  *
1318  * @return
1319  *   The size of the memory needed for all actions.
1320  */
1321 static int
1322 flow_verbs_get_actions_and_size(const struct rte_flow_action actions[],
1323                                 uint64_t *action_flags)
1324 {
1325         int size = 0;
1326         uint64_t detected_actions = 0;
1327
1328         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1329                 switch (actions->type) {
1330                 case RTE_FLOW_ACTION_TYPE_VOID:
1331                         break;
1332                 case RTE_FLOW_ACTION_TYPE_FLAG:
1333                         size += sizeof(struct ibv_flow_spec_action_tag);
1334                         detected_actions |= MLX5_FLOW_ACTION_FLAG;
1335                         break;
1336                 case RTE_FLOW_ACTION_TYPE_MARK:
1337                         size += sizeof(struct ibv_flow_spec_action_tag);
1338                         detected_actions |= MLX5_FLOW_ACTION_MARK;
1339                         break;
1340                 case RTE_FLOW_ACTION_TYPE_DROP:
1341                         size += sizeof(struct ibv_flow_spec_action_drop);
1342                         detected_actions |= MLX5_FLOW_ACTION_DROP;
1343                         break;
1344                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1345                         detected_actions |= MLX5_FLOW_ACTION_QUEUE;
1346                         break;
1347                 case RTE_FLOW_ACTION_TYPE_RSS:
1348                         detected_actions |= MLX5_FLOW_ACTION_RSS;
1349                         break;
1350                 case RTE_FLOW_ACTION_TYPE_COUNT:
1351 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1352         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1353                         size += sizeof(struct ibv_flow_spec_counter_action);
1354 #endif
1355                         detected_actions |= MLX5_FLOW_ACTION_COUNT;
1356                         break;
1357                 default:
1358                         break;
1359                 }
1360         }
1361         *action_flags = detected_actions;
1362         return size;
1363 }
1364
1365 /**
1366  * Calculate the required bytes that are needed for the item part of the verbs
1367  * flow, in addtion returns bit-fields with all the detected action, in order to
1368  * avoid another interation over the actions.
1369  *
1370  * @param[in] actions
1371  *   Pointer to the list of items.
1372  * @param[in, out] item_flags
1373  *   Pointer to the detected items.
1374  *
1375  * @return
1376  *   The size of the memory needed for all items.
1377  */
1378 static int
1379 flow_verbs_get_items_and_size(const struct rte_flow_item items[],
1380                               uint64_t *item_flags)
1381 {
1382         int size = 0;
1383         uint64_t detected_items = 0;
1384
1385         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1386                 int tunnel = !!(detected_items & MLX5_FLOW_LAYER_TUNNEL);
1387
1388                 switch (items->type) {
1389                 case RTE_FLOW_ITEM_TYPE_VOID:
1390                         break;
1391                 case RTE_FLOW_ITEM_TYPE_ETH:
1392                         size += sizeof(struct ibv_flow_spec_eth);
1393                         detected_items |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1394                                                    MLX5_FLOW_LAYER_OUTER_L2;
1395                         break;
1396                 case RTE_FLOW_ITEM_TYPE_VLAN:
1397                         size += sizeof(struct ibv_flow_spec_eth);
1398                         detected_items |= tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1399                                                    MLX5_FLOW_LAYER_OUTER_VLAN;
1400                         break;
1401                 case RTE_FLOW_ITEM_TYPE_IPV4:
1402                         size += sizeof(struct ibv_flow_spec_ipv4_ext);
1403                         detected_items |= tunnel ?
1404                                           MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1405                                           MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1406                         break;
1407                 case RTE_FLOW_ITEM_TYPE_IPV6:
1408                         size += sizeof(struct ibv_flow_spec_ipv6);
1409                         detected_items |= tunnel ?
1410                                           MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1411                                           MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1412                         break;
1413                 case RTE_FLOW_ITEM_TYPE_UDP:
1414                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1415                         detected_items |= tunnel ?
1416                                           MLX5_FLOW_LAYER_INNER_L4_UDP :
1417                                           MLX5_FLOW_LAYER_OUTER_L4_UDP;
1418                         break;
1419                 case RTE_FLOW_ITEM_TYPE_TCP:
1420                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1421                         detected_items |= tunnel ?
1422                                           MLX5_FLOW_LAYER_INNER_L4_TCP :
1423                                           MLX5_FLOW_LAYER_OUTER_L4_TCP;
1424                         break;
1425                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1426                         size += sizeof(struct ibv_flow_spec_tunnel);
1427                         detected_items |= MLX5_FLOW_LAYER_VXLAN;
1428                         break;
1429                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1430                         size += sizeof(struct ibv_flow_spec_tunnel);
1431                         detected_items |= MLX5_FLOW_LAYER_VXLAN_GPE;
1432                         break;
1433 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1434                 case RTE_FLOW_ITEM_TYPE_GRE:
1435                         size += sizeof(struct ibv_flow_spec_gre);
1436                         detected_items |= MLX5_FLOW_LAYER_GRE;
1437                         break;
1438                 case RTE_FLOW_ITEM_TYPE_MPLS:
1439                         size += sizeof(struct ibv_flow_spec_mpls);
1440                         detected_items |= MLX5_FLOW_LAYER_MPLS;
1441                         break;
1442 #else
1443                 case RTE_FLOW_ITEM_TYPE_GRE:
1444                         size += sizeof(struct ibv_flow_spec_tunnel);
1445                         detected_items |= MLX5_FLOW_LAYER_TUNNEL;
1446                         break;
1447 #endif
1448                 default:
1449                         break;
1450                 }
1451         }
1452         *item_flags = detected_items;
1453         return size;
1454 }
1455
1456 /**
1457  * Internal preparation function. Allocate mlx5_flow with the required size.
1458  * The required size is calculate based on the actions and items. This function
1459  * also returns the detected actions and items for later use.
1460  *
1461  * @param[in] attr
1462  *   Pointer to the flow attributes.
1463  * @param[in] items
1464  *   Pointer to the list of items.
1465  * @param[in] actions
1466  *   Pointer to the list of actions.
1467  * @param[out] item_flags
1468  *   Pointer to bit mask of all items detected.
1469  * @param[out] action_flags
1470  *   Pointer to bit mask of all actions detected.
1471  * @param[out] error
1472  *   Pointer to the error structure.
1473  *
1474  * @return
1475  *   Pointer to mlx5_flow object on success, otherwise NULL and rte_errno
1476  *   is set.
1477  */
1478 static struct mlx5_flow *
1479 flow_verbs_prepare(const struct rte_flow_attr *attr __rte_unused,
1480                    const struct rte_flow_item items[],
1481                    const struct rte_flow_action actions[],
1482                    uint64_t *item_flags,
1483                    uint64_t *action_flags,
1484                    struct rte_flow_error *error)
1485 {
1486         uint32_t size = sizeof(struct mlx5_flow) + sizeof(struct ibv_flow_attr);
1487         struct mlx5_flow *flow;
1488
1489         size += flow_verbs_get_actions_and_size(actions, action_flags);
1490         size += flow_verbs_get_items_and_size(items, item_flags);
1491         flow = rte_calloc(__func__, 1, size, 0);
1492         if (!flow) {
1493                 rte_flow_error_set(error, ENOMEM,
1494                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1495                                    "not enough memory to create flow");
1496                 return NULL;
1497         }
1498         flow->verbs.attr = (void *)(flow + 1);
1499         flow->verbs.specs =
1500                 (uint8_t *)(flow + 1) + sizeof(struct ibv_flow_attr);
1501         return flow;
1502 }
1503
1504 /**
1505  * Fill the flow with verb spec.
1506  *
1507  * @param[in] dev
1508  *   Pointer to Ethernet device.
1509  * @param[in, out] dev_flow
1510  *   Pointer to the mlx5 flow.
1511  * @param[in] attr
1512  *   Pointer to the flow attributes.
1513  * @param[in] items
1514  *   Pointer to the list of items.
1515  * @param[in] actions
1516  *   Pointer to the list of actions.
1517  * @param[out] error
1518  *   Pointer to the error structure.
1519  *
1520  * @return
1521  *   0 on success, else a negative errno value otherwise and rte_ernno is set.
1522  */
1523 static int
1524 flow_verbs_translate(struct rte_eth_dev *dev,
1525                      struct mlx5_flow *dev_flow,
1526                      const struct rte_flow_attr *attr,
1527                      const struct rte_flow_item items[],
1528                      const struct rte_flow_action actions[],
1529                      struct rte_flow_error *error)
1530 {
1531         uint64_t action_flags = 0;
1532         uint64_t item_flags = 0;
1533         uint64_t priority = attr->priority;
1534         struct priv *priv = dev->data->dev_private;
1535
1536         if (priority == MLX5_FLOW_PRIO_RSVD)
1537                 priority = priv->config.flow_prio - 1;
1538         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1539                 int ret;
1540                 switch (actions->type) {
1541                 case RTE_FLOW_ACTION_TYPE_VOID:
1542                         break;
1543                 case RTE_FLOW_ACTION_TYPE_FLAG:
1544                         flow_verbs_translate_action_flag(actions,
1545                                                          &action_flags,
1546                                                          dev_flow);
1547                         break;
1548                 case RTE_FLOW_ACTION_TYPE_MARK:
1549                         flow_verbs_translate_action_mark(actions,
1550                                                          &action_flags,
1551                                                          dev_flow);
1552                         break;
1553                 case RTE_FLOW_ACTION_TYPE_DROP:
1554                         flow_verbs_translate_action_drop(&action_flags,
1555                                                          dev_flow);
1556                         break;
1557                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1558                         flow_verbs_translate_action_queue(actions,
1559                                                           &action_flags,
1560                                                           dev_flow);
1561                         break;
1562                 case RTE_FLOW_ACTION_TYPE_RSS:
1563                         flow_verbs_translate_action_rss(actions,
1564                                                         &action_flags,
1565                                                         dev_flow);
1566                         break;
1567                 case RTE_FLOW_ACTION_TYPE_COUNT:
1568                         ret = flow_verbs_translate_action_count(dev,
1569                                                                 actions,
1570                                                                 &action_flags,
1571                                                                 dev_flow,
1572                                                                 error);
1573                         if (ret < 0)
1574                                 return ret;
1575                         break;
1576                 default:
1577                         return rte_flow_error_set(error, ENOTSUP,
1578                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1579                                                   actions,
1580                                                   "action not supported");
1581                 }
1582         }
1583         /* Device flow should have action flags by flow_drv_prepare(). */
1584         assert(dev_flow->flow->actions == action_flags);
1585         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1586                 switch (items->type) {
1587                 case RTE_FLOW_ITEM_TYPE_VOID:
1588                         break;
1589                 case RTE_FLOW_ITEM_TYPE_ETH:
1590                         flow_verbs_translate_item_eth(items, &item_flags,
1591                                                       dev_flow);
1592                         break;
1593                 case RTE_FLOW_ITEM_TYPE_VLAN:
1594                         flow_verbs_translate_item_vlan(items, &item_flags,
1595                                                        dev_flow);
1596                         break;
1597                 case RTE_FLOW_ITEM_TYPE_IPV4:
1598                         flow_verbs_translate_item_ipv4(items, &item_flags,
1599                                                        dev_flow);
1600                         break;
1601                 case RTE_FLOW_ITEM_TYPE_IPV6:
1602                         flow_verbs_translate_item_ipv6(items, &item_flags,
1603                                                        dev_flow);
1604                         break;
1605                 case RTE_FLOW_ITEM_TYPE_UDP:
1606                         flow_verbs_translate_item_udp(items, &item_flags,
1607                                                       dev_flow);
1608                         break;
1609                 case RTE_FLOW_ITEM_TYPE_TCP:
1610                         flow_verbs_translate_item_tcp(items, &item_flags,
1611                                                       dev_flow);
1612                         break;
1613                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1614                         flow_verbs_translate_item_vxlan(items, &item_flags,
1615                                                         dev_flow);
1616                         break;
1617                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1618                         flow_verbs_translate_item_vxlan_gpe(items, &item_flags,
1619                                                             dev_flow);
1620                         break;
1621                 case RTE_FLOW_ITEM_TYPE_GRE:
1622                         flow_verbs_translate_item_gre(items, &item_flags,
1623                                                       dev_flow);
1624                         break;
1625                 case RTE_FLOW_ITEM_TYPE_MPLS:
1626                         flow_verbs_translate_item_mpls(items, &item_flags,
1627                                                        dev_flow);
1628                         break;
1629                 default:
1630                         return rte_flow_error_set(error, ENOTSUP,
1631                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1632                                                   NULL,
1633                                                   "item not supported");
1634                 }
1635         }
1636         dev_flow->verbs.attr->priority =
1637                 mlx5_flow_adjust_priority(dev, priority,
1638                                           dev_flow->verbs.attr->priority);
1639         return 0;
1640 }
1641
1642 /**
1643  * Remove the flow from the NIC but keeps it in memory.
1644  *
1645  * @param[in] dev
1646  *   Pointer to the Ethernet device structure.
1647  * @param[in, out] flow
1648  *   Pointer to flow structure.
1649  */
1650 static void
1651 flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1652 {
1653         struct mlx5_flow_verbs *verbs;
1654         struct mlx5_flow *dev_flow;
1655
1656         if (!flow)
1657                 return;
1658         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1659                 verbs = &dev_flow->verbs;
1660                 if (verbs->flow) {
1661                         claim_zero(mlx5_glue->destroy_flow(verbs->flow));
1662                         verbs->flow = NULL;
1663                 }
1664                 if (verbs->hrxq) {
1665                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
1666                                 mlx5_hrxq_drop_release(dev);
1667                         else
1668                                 mlx5_hrxq_release(dev, verbs->hrxq);
1669                         verbs->hrxq = NULL;
1670                 }
1671         }
1672         if (flow->counter) {
1673                 flow_verbs_counter_release(flow->counter);
1674                 flow->counter = NULL;
1675         }
1676 }
1677
1678 /**
1679  * Remove the flow from the NIC and the memory.
1680  *
1681  * @param[in] dev
1682  *   Pointer to the Ethernet device structure.
1683  * @param[in, out] flow
1684  *   Pointer to flow structure.
1685  */
1686 static void
1687 flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1688 {
1689         struct mlx5_flow *dev_flow;
1690
1691         if (!flow)
1692                 return;
1693         flow_verbs_remove(dev, flow);
1694         while (!LIST_EMPTY(&flow->dev_flows)) {
1695                 dev_flow = LIST_FIRST(&flow->dev_flows);
1696                 LIST_REMOVE(dev_flow, next);
1697                 rte_free(dev_flow);
1698         }
1699 }
1700
1701 /**
1702  * Apply the flow to the NIC.
1703  *
1704  * @param[in] dev
1705  *   Pointer to the Ethernet device structure.
1706  * @param[in, out] flow
1707  *   Pointer to flow structure.
1708  * @param[out] error
1709  *   Pointer to error structure.
1710  *
1711  * @return
1712  *   0 on success, a negative errno value otherwise and rte_errno is set.
1713  */
1714 static int
1715 flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1716                  struct rte_flow_error *error)
1717 {
1718         struct mlx5_flow_verbs *verbs;
1719         struct mlx5_flow *dev_flow;
1720         int err;
1721
1722         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1723                 verbs = &dev_flow->verbs;
1724                 if (flow->actions & MLX5_FLOW_ACTION_DROP) {
1725                         verbs->hrxq = mlx5_hrxq_drop_new(dev);
1726                         if (!verbs->hrxq) {
1727                                 rte_flow_error_set
1728                                         (error, errno,
1729                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1730                                          "cannot get drop hash queue");
1731                                 goto error;
1732                         }
1733                 } else {
1734                         struct mlx5_hrxq *hrxq;
1735
1736                         hrxq = mlx5_hrxq_get(dev, flow->key,
1737                                              MLX5_RSS_HASH_KEY_LEN,
1738                                              verbs->hash_fields,
1739                                              (*flow->queue),
1740                                              flow->rss.queue_num);
1741                         if (!hrxq)
1742                                 hrxq = mlx5_hrxq_new(dev, flow->key,
1743                                                      MLX5_RSS_HASH_KEY_LEN,
1744                                                      verbs->hash_fields,
1745                                                      (*flow->queue),
1746                                                      flow->rss.queue_num,
1747                                                      !!(dev_flow->layers &
1748                                                       MLX5_FLOW_LAYER_TUNNEL));
1749                         if (!hrxq) {
1750                                 rte_flow_error_set
1751                                         (error, rte_errno,
1752                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1753                                          "cannot get hash queue");
1754                                 goto error;
1755                         }
1756                         verbs->hrxq = hrxq;
1757                 }
1758                 verbs->flow = mlx5_glue->create_flow(verbs->hrxq->qp,
1759                                                      verbs->attr);
1760                 if (!verbs->flow) {
1761                         rte_flow_error_set(error, errno,
1762                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1763                                            NULL,
1764                                            "hardware refuses to create flow");
1765                         goto error;
1766                 }
1767         }
1768         return 0;
1769 error:
1770         err = rte_errno; /* Save rte_errno before cleanup. */
1771         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1772                 verbs = &dev_flow->verbs;
1773                 if (verbs->hrxq) {
1774                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
1775                                 mlx5_hrxq_drop_release(dev);
1776                         else
1777                                 mlx5_hrxq_release(dev, verbs->hrxq);
1778                         verbs->hrxq = NULL;
1779                 }
1780         }
1781         rte_errno = err; /* Restore rte_errno. */
1782         return -rte_errno;
1783 }
1784
1785 /**
1786  * Query a flow.
1787  *
1788  * @see rte_flow_query()
1789  * @see rte_flow_ops
1790  */
1791 static int
1792 flow_verbs_query(struct rte_eth_dev *dev,
1793                  struct rte_flow *flow,
1794                  const struct rte_flow_action *actions,
1795                  void *data,
1796                  struct rte_flow_error *error)
1797 {
1798         int ret = -EINVAL;
1799
1800         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1801                 switch (actions->type) {
1802                 case RTE_FLOW_ACTION_TYPE_VOID:
1803                         break;
1804                 case RTE_FLOW_ACTION_TYPE_COUNT:
1805                         ret = flow_verbs_counter_query(dev, flow, data, error);
1806                         break;
1807                 default:
1808                         return rte_flow_error_set(error, ENOTSUP,
1809                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1810                                                   actions,
1811                                                   "action not supported");
1812                 }
1813         }
1814         return ret;
1815 }
1816
1817 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = {
1818         .validate = flow_verbs_validate,
1819         .prepare = flow_verbs_prepare,
1820         .translate = flow_verbs_translate,
1821         .apply = flow_verbs_apply,
1822         .remove = flow_verbs_remove,
1823         .destroy = flow_verbs_destroy,
1824         .query = flow_verbs_query,
1825 };