drivers/net/sfc/sfc_flow.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  * Copyright (c) 2017 Solarflare Communications Inc.
   5  * All rights reserved.
   6  *
   7  * This software was jointly developed between OKTET Labs (under contract
   8  * for Solarflare) and Solarflare Communications, Inc.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions are met:
  12  *
  13  * 1. Redistributions of source code must retain the above copyright notice,
  14  *    this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright notice,
  16  *    this list of conditions and the following disclaimer in the documentation
  17  *    and/or other materials provided with the distribution.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  20  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  21  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  22  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  23  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  24  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  25  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  26  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  27  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  28  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  29  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30  */
  31
  32 #include <rte_tailq.h>
  33 #include <rte_common.h>
  34 #include <rte_ethdev.h>
  35 #include <rte_eth_ctrl.h>
  36 #include <rte_ether.h>
  37 #include <rte_flow.h>
  38 #include <rte_flow_driver.h>
  39
  40 #include "efx.h"
  41
  42 #include "sfc.h"
  43 #include "sfc_rx.h"
  44 #include "sfc_filter.h"
  45 #include "sfc_flow.h"
  46 #include "sfc_log.h"
  47
  48 /*
  49  * At now flow API is implemented in such a manner that each
  50  * flow rule is converted to a hardware filter.
  51  * All elements of flow rule (attributes, pattern items, actions)
  52  * correspond to one or more fields in the efx_filter_spec_s structure
  53  * that is responsible for the hardware filter.
  54  */
  55
  56 enum sfc_flow_item_layers {
  57         SFC_FLOW_ITEM_ANY_LAYER,
  58         SFC_FLOW_ITEM_START_LAYER,
  59         SFC_FLOW_ITEM_L2,
  60         SFC_FLOW_ITEM_L3,
  61         SFC_FLOW_ITEM_L4,
  62 };
  63
  64 typedef int (sfc_flow_item_parse)(const struct rte_flow_item *item,
  65                                   efx_filter_spec_t *spec,
  66                                   struct rte_flow_error *error);
  67
  68 struct sfc_flow_item {
  69         enum rte_flow_item_type type;           /* Type of item */
  70         enum sfc_flow_item_layers layer;        /* Layer of item */
  71         enum sfc_flow_item_layers prev_layer;   /* Previous layer of item */
  72         sfc_flow_item_parse *parse;             /* Parsing function */
  73 };
  74
  75 static sfc_flow_item_parse sfc_flow_parse_void;
  76 static sfc_flow_item_parse sfc_flow_parse_eth;
  77 static sfc_flow_item_parse sfc_flow_parse_vlan;
  78 static sfc_flow_item_parse sfc_flow_parse_ipv4;
  79 static sfc_flow_item_parse sfc_flow_parse_ipv6;
  80 static sfc_flow_item_parse sfc_flow_parse_tcp;
  81 static sfc_flow_item_parse sfc_flow_parse_udp;
  82
  83 static boolean_t
  84 sfc_flow_is_zero(const uint8_t *buf, unsigned int size)
  85 {
  86         uint8_t sum = 0;
  87         unsigned int i;
  88
  89         for (i = 0; i < size; i++)
  90                 sum |= buf[i];
  91
  92         return (sum == 0) ? B_TRUE : B_FALSE;
  93 }
  94
  95 /*
  96  * Validate item and prepare structures spec and mask for parsing
  97  */
  98 static int
  99 sfc_flow_parse_init(const struct rte_flow_item *item,
 100                     const void **spec_ptr,
 101                     const void **mask_ptr,
 102                     const void *supp_mask,
 103                     const void *def_mask,
 104                     unsigned int size,
 105                     struct rte_flow_error *error)
 106 {
 107         const uint8_t *spec;
 108         const uint8_t *mask;
 109         const uint8_t *last;
 110         uint8_t match;
 111         uint8_t supp;
 112         unsigned int i;
 113
 114         if (item == NULL) {
 115                 rte_flow_error_set(error, EINVAL,
 116                                    RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 117                                    "NULL item");
 118                 return -rte_errno;
 119         }
 120
 121         if ((item->last != NULL || item->mask != NULL) && item->spec == NULL) {
 122                 rte_flow_error_set(error, EINVAL,
 123                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 124                                    "Mask or last is set without spec");
 125                 return -rte_errno;
 126         }
 127
 128         /*
 129          * If "mask" is not set, default mask is used,
 130          * but if default mask is NULL, "mask" should be set
 131          */
 132         if (item->mask == NULL) {
 133                 if (def_mask == NULL) {
 134                         rte_flow_error_set(error, EINVAL,
 135                                 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 136                                 "Mask should be specified");
 137                         return -rte_errno;
 138                 }
 139
 140                 mask = (const uint8_t *)def_mask;
 141         } else {
 142                 mask = (const uint8_t *)item->mask;
 143         }
 144
 145         spec = (const uint8_t *)item->spec;
 146         last = (const uint8_t *)item->last;
 147
 148         if (spec == NULL)
 149                 goto exit;
 150
 151         /*
 152          * If field values in "last" are either 0 or equal to the corresponding
 153          * values in "spec" then they are ignored
 154          */
 155         if (last != NULL &&
 156             !sfc_flow_is_zero(last, size) &&
 157             memcmp(last, spec, size) != 0) {
 158                 rte_flow_error_set(error, ENOTSUP,
 159                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 160                                    "Ranging is not supported");
 161                 return -rte_errno;
 162         }
 163
 164         if (supp_mask == NULL) {
 165                 rte_flow_error_set(error, EINVAL,
 166                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 167                         "Supported mask for item should be specified");
 168                 return -rte_errno;
 169         }
 170
 171         /* Check that mask and spec not asks for more match than supp_mask */
 172         for (i = 0; i < size; i++) {
 173                 match = spec[i] | mask[i];
 174                 supp = ((const uint8_t *)supp_mask)[i];
 175
 176                 if ((match | supp) != supp) {
 177                         rte_flow_error_set(error, ENOTSUP,
 178                                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 179                                            "Item's field is not supported");
 180                         return -rte_errno;
 181                 }
 182         }
 183
 184 exit:
 185         *spec_ptr = spec;
 186         *mask_ptr = mask;
 187         return 0;
 188 }
 189
 190 /*
 191  * Protocol parsers.
 192  * Masking is not supported, so masks in items should be either
 193  * full or empty (zeroed) and set only for supported fields which
 194  * are specified in the supp_mask.
 195  */
 196
 197 static int
 198 sfc_flow_parse_void(__rte_unused const struct rte_flow_item *item,
 199                     __rte_unused efx_filter_spec_t *efx_spec,
 200                     __rte_unused struct rte_flow_error *error)
 201 {
 202         return 0;
 203 }
 204
 205 /**
 206  * Convert Ethernet item to EFX filter specification.
 207  *
 208  * @param item[in]
 209  *   Item specification. Only source and destination addresses and
 210  *   Ethernet type fields are supported. In addition to full and
 211  *   empty masks of destination address, individual/group mask is
 212  *   also supported. If the mask is NULL, default mask will be used.
 213  *   Ranging is not supported.
 214  * @param efx_spec[in, out]
 215  *   EFX filter specification to update.
 216  * @param[out] error
 217  *   Perform verbose error reporting if not NULL.
 218  */
 219 static int
 220 sfc_flow_parse_eth(const struct rte_flow_item *item,
 221                    efx_filter_spec_t *efx_spec,
 222                    struct rte_flow_error *error)
 223 {
 224         int rc;
 225         const struct rte_flow_item_eth *spec = NULL;
 226         const struct rte_flow_item_eth *mask = NULL;
 227         const struct rte_flow_item_eth supp_mask = {
 228                 .dst.addr_bytes = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
 229                 .src.addr_bytes = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
 230                 .type = 0xffff,
 231         };
 232         const uint8_t ig_mask[EFX_MAC_ADDR_LEN] = {
 233                 0x01, 0x00, 0x00, 0x00, 0x00, 0x00
 234         };
 235
 236         rc = sfc_flow_parse_init(item,
 237                                  (const void **)&spec,
 238                                  (const void **)&mask,
 239                                  &supp_mask,
 240                                  &rte_flow_item_eth_mask,
 241                                  sizeof(struct rte_flow_item_eth),
 242                                  error);
 243         if (rc != 0)
 244                 return rc;
 245
 246         /* If "spec" is not set, could be any Ethernet */
 247         if (spec == NULL)
 248                 return 0;
 249
 250         if (is_same_ether_addr(&mask->dst, &supp_mask.dst)) {
 251                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC;
 252                 rte_memcpy(efx_spec->efs_loc_mac, spec->dst.addr_bytes,
 253                            EFX_MAC_ADDR_LEN);
 254         } else if (memcmp(mask->dst.addr_bytes, ig_mask,
 255                           EFX_MAC_ADDR_LEN) == 0) {
 256                 if (is_unicast_ether_addr(&spec->dst))
 257                         efx_spec->efs_match_flags |=
 258                                 EFX_FILTER_MATCH_UNKNOWN_UCAST_DST;
 259                 else
 260                         efx_spec->efs_match_flags |=
 261                                 EFX_FILTER_MATCH_UNKNOWN_MCAST_DST;
 262         } else if (!is_zero_ether_addr(&mask->dst)) {
 263                 goto fail_bad_mask;
 264         }
 265
 266         if (is_same_ether_addr(&mask->src, &supp_mask.src)) {
 267                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_MAC;
 268                 rte_memcpy(efx_spec->efs_rem_mac, spec->src.addr_bytes,
 269                            EFX_MAC_ADDR_LEN);
 270         } else if (!is_zero_ether_addr(&mask->src)) {
 271                 goto fail_bad_mask;
 272         }
 273
 274         /*
 275          * Ether type is in big-endian byte order in item and
 276          * in little-endian in efx_spec, so byte swap is used
 277          */
 278         if (mask->type == supp_mask.type) {
 279                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 280                 efx_spec->efs_ether_type = rte_bswap16(spec->type);
 281         } else if (mask->type != 0) {
 282                 goto fail_bad_mask;
 283         }
 284
 285         return 0;
 286
 287 fail_bad_mask:
 288         rte_flow_error_set(error, EINVAL,
 289                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 290                            "Bad mask in the ETH pattern item");
 291         return -rte_errno;
 292 }
 293
 294 /**
 295  * Convert VLAN item to EFX filter specification.
 296  *
 297  * @param item[in]
 298  *   Item specification. Only VID field is supported.
 299  *   The mask can not be NULL. Ranging is not supported.
 300  * @param efx_spec[in, out]
 301  *   EFX filter specification to update.
 302  * @param[out] error
 303  *   Perform verbose error reporting if not NULL.
 304  */
 305 static int
 306 sfc_flow_parse_vlan(const struct rte_flow_item *item,
 307                     efx_filter_spec_t *efx_spec,
 308                     struct rte_flow_error *error)
 309 {
 310         int rc;
 311         uint16_t vid;
 312         const struct rte_flow_item_vlan *spec = NULL;
 313         const struct rte_flow_item_vlan *mask = NULL;
 314         const struct rte_flow_item_vlan supp_mask = {
 315                 .tci = rte_cpu_to_be_16(ETH_VLAN_ID_MAX),
 316         };
 317
 318         rc = sfc_flow_parse_init(item,
 319                                  (const void **)&spec,
 320                                  (const void **)&mask,
 321                                  &supp_mask,
 322                                  NULL,
 323                                  sizeof(struct rte_flow_item_vlan),
 324                                  error);
 325         if (rc != 0)
 326                 return rc;
 327
 328         /*
 329          * VID is in big-endian byte order in item and
 330          * in little-endian in efx_spec, so byte swap is used.
 331          * If two VLAN items are included, the first matches
 332          * the outer tag and the next matches the inner tag.
 333          */
 334         if (mask->tci == supp_mask.tci) {
 335                 vid = rte_bswap16(spec->tci);
 336
 337                 if (!(efx_spec->efs_match_flags &
 338                       EFX_FILTER_MATCH_OUTER_VID)) {
 339                         efx_spec->efs_match_flags |= EFX_FILTER_MATCH_OUTER_VID;
 340                         efx_spec->efs_outer_vid = vid;
 341                 } else if (!(efx_spec->efs_match_flags &
 342                              EFX_FILTER_MATCH_INNER_VID)) {
 343                         efx_spec->efs_match_flags |= EFX_FILTER_MATCH_INNER_VID;
 344                         efx_spec->efs_inner_vid = vid;
 345                 } else {
 346                         rte_flow_error_set(error, EINVAL,
 347                                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 348                                            "More than two VLAN items");
 349                         return -rte_errno;
 350                 }
 351         } else {
 352                 rte_flow_error_set(error, EINVAL,
 353                                    RTE_FLOW_ERROR_TYPE_ITEM, item,
 354                                    "VLAN ID in TCI match is required");
 355                 return -rte_errno;
 356         }
 357
 358         return 0;
 359 }
 360
 361 /**
 362  * Convert IPv4 item to EFX filter specification.
 363  *
 364  * @param item[in]
 365  *   Item specification. Only source and destination addresses and
 366  *   protocol fields are supported. If the mask is NULL, default
 367  *   mask will be used. Ranging is not supported.
 368  * @param efx_spec[in, out]
 369  *   EFX filter specification to update.
 370  * @param[out] error
 371  *   Perform verbose error reporting if not NULL.
 372  */
 373 static int
 374 sfc_flow_parse_ipv4(const struct rte_flow_item *item,
 375                     efx_filter_spec_t *efx_spec,
 376                     struct rte_flow_error *error)
 377 {
 378         int rc;
 379         const struct rte_flow_item_ipv4 *spec = NULL;
 380         const struct rte_flow_item_ipv4 *mask = NULL;
 381         const uint16_t ether_type_ipv4 = rte_cpu_to_le_16(EFX_ETHER_TYPE_IPV4);
 382         const struct rte_flow_item_ipv4 supp_mask = {
 383                 .hdr = {
 384                         .src_addr = 0xffffffff,
 385                         .dst_addr = 0xffffffff,
 386                         .next_proto_id = 0xff,
 387                 }
 388         };
 389
 390         rc = sfc_flow_parse_init(item,
 391                                  (const void **)&spec,
 392                                  (const void **)&mask,
 393                                  &supp_mask,
 394                                  &rte_flow_item_ipv4_mask,
 395                                  sizeof(struct rte_flow_item_ipv4),
 396                                  error);
 397         if (rc != 0)
 398                 return rc;
 399
 400         /*
 401          * Filtering by IPv4 source and destination addresses requires
 402          * the appropriate ETHER_TYPE in hardware filters
 403          */
 404         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_ETHER_TYPE)) {
 405                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 406                 efx_spec->efs_ether_type = ether_type_ipv4;
 407         } else if (efx_spec->efs_ether_type != ether_type_ipv4) {
 408                 rte_flow_error_set(error, EINVAL,
 409                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 410                         "Ethertype in pattern with IPV4 item should be appropriate");
 411                 return -rte_errno;
 412         }
 413
 414         if (spec == NULL)
 415                 return 0;
 416
 417         /*
 418          * IPv4 addresses are in big-endian byte order in item and in
 419          * efx_spec
 420          */
 421         if (mask->hdr.src_addr == supp_mask.hdr.src_addr) {
 422                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_HOST;
 423                 efx_spec->efs_rem_host.eo_u32[0] = spec->hdr.src_addr;
 424         } else if (mask->hdr.src_addr != 0) {
 425                 goto fail_bad_mask;
 426         }
 427
 428         if (mask->hdr.dst_addr == supp_mask.hdr.dst_addr) {
 429                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_HOST;
 430                 efx_spec->efs_loc_host.eo_u32[0] = spec->hdr.dst_addr;
 431         } else if (mask->hdr.dst_addr != 0) {
 432                 goto fail_bad_mask;
 433         }
 434
 435         if (mask->hdr.next_proto_id == supp_mask.hdr.next_proto_id) {
 436                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 437                 efx_spec->efs_ip_proto = spec->hdr.next_proto_id;
 438         } else if (mask->hdr.next_proto_id != 0) {
 439                 goto fail_bad_mask;
 440         }
 441
 442         return 0;
 443
 444 fail_bad_mask:
 445         rte_flow_error_set(error, EINVAL,
 446                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 447                            "Bad mask in the IPV4 pattern item");
 448         return -rte_errno;
 449 }
 450
 451 /**
 452  * Convert IPv6 item to EFX filter specification.
 453  *
 454  * @param item[in]
 455  *   Item specification. Only source and destination addresses and
 456  *   next header fields are supported. If the mask is NULL, default
 457  *   mask will be used. Ranging is not supported.
 458  * @param efx_spec[in, out]
 459  *   EFX filter specification to update.
 460  * @param[out] error
 461  *   Perform verbose error reporting if not NULL.
 462  */
 463 static int
 464 sfc_flow_parse_ipv6(const struct rte_flow_item *item,
 465                     efx_filter_spec_t *efx_spec,
 466                     struct rte_flow_error *error)
 467 {
 468         int rc;
 469         const struct rte_flow_item_ipv6 *spec = NULL;
 470         const struct rte_flow_item_ipv6 *mask = NULL;
 471         const uint16_t ether_type_ipv6 = rte_cpu_to_le_16(EFX_ETHER_TYPE_IPV6);
 472         const struct rte_flow_item_ipv6 supp_mask = {
 473                 .hdr = {
 474                         .src_addr = { 0xff, 0xff, 0xff, 0xff,
 475                                       0xff, 0xff, 0xff, 0xff,
 476                                       0xff, 0xff, 0xff, 0xff,
 477                                       0xff, 0xff, 0xff, 0xff },
 478                         .dst_addr = { 0xff, 0xff, 0xff, 0xff,
 479                                       0xff, 0xff, 0xff, 0xff,
 480                                       0xff, 0xff, 0xff, 0xff,
 481                                       0xff, 0xff, 0xff, 0xff },
 482                         .proto = 0xff,
 483                 }
 484         };
 485
 486         rc = sfc_flow_parse_init(item,
 487                                  (const void **)&spec,
 488                                  (const void **)&mask,
 489                                  &supp_mask,
 490                                  &rte_flow_item_ipv6_mask,
 491                                  sizeof(struct rte_flow_item_ipv6),
 492                                  error);
 493         if (rc != 0)
 494                 return rc;
 495
 496         /*
 497          * Filtering by IPv6 source and destination addresses requires
 498          * the appropriate ETHER_TYPE in hardware filters
 499          */
 500         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_ETHER_TYPE)) {
 501                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 502                 efx_spec->efs_ether_type = ether_type_ipv6;
 503         } else if (efx_spec->efs_ether_type != ether_type_ipv6) {
 504                 rte_flow_error_set(error, EINVAL,
 505                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 506                         "Ethertype in pattern with IPV6 item should be appropriate");
 507                 return -rte_errno;
 508         }
 509
 510         if (spec == NULL)
 511                 return 0;
 512
 513         /*
 514          * IPv6 addresses are in big-endian byte order in item and in
 515          * efx_spec
 516          */
 517         if (memcmp(mask->hdr.src_addr, supp_mask.hdr.src_addr,
 518                    sizeof(mask->hdr.src_addr)) == 0) {
 519                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_HOST;
 520
 521                 RTE_BUILD_BUG_ON(sizeof(efx_spec->efs_rem_host) !=
 522                                  sizeof(spec->hdr.src_addr));
 523                 rte_memcpy(&efx_spec->efs_rem_host, spec->hdr.src_addr,
 524                            sizeof(efx_spec->efs_rem_host));
 525         } else if (!sfc_flow_is_zero(mask->hdr.src_addr,
 526                                      sizeof(mask->hdr.src_addr))) {
 527                 goto fail_bad_mask;
 528         }
 529
 530         if (memcmp(mask->hdr.dst_addr, supp_mask.hdr.dst_addr,
 531                    sizeof(mask->hdr.dst_addr)) == 0) {
 532                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_HOST;
 533
 534                 RTE_BUILD_BUG_ON(sizeof(efx_spec->efs_loc_host) !=
 535                                  sizeof(spec->hdr.dst_addr));
 536                 rte_memcpy(&efx_spec->efs_loc_host, spec->hdr.dst_addr,
 537                            sizeof(efx_spec->efs_loc_host));
 538         } else if (!sfc_flow_is_zero(mask->hdr.dst_addr,
 539                                      sizeof(mask->hdr.dst_addr))) {
 540                 goto fail_bad_mask;
 541         }
 542
 543         if (mask->hdr.proto == supp_mask.hdr.proto) {
 544                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 545                 efx_spec->efs_ip_proto = spec->hdr.proto;
 546         } else if (mask->hdr.proto != 0) {
 547                 goto fail_bad_mask;
 548         }
 549
 550         return 0;
 551
 552 fail_bad_mask:
 553         rte_flow_error_set(error, EINVAL,
 554                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 555                            "Bad mask in the IPV6 pattern item");
 556         return -rte_errno;
 557 }
 558
 559 /**
 560  * Convert TCP item to EFX filter specification.
 561  *
 562  * @param item[in]
 563  *   Item specification. Only source and destination ports fields
 564  *   are supported. If the mask is NULL, default mask will be used.
 565  *   Ranging is not supported.
 566  * @param efx_spec[in, out]
 567  *   EFX filter specification to update.
 568  * @param[out] error
 569  *   Perform verbose error reporting if not NULL.
 570  */
 571 static int
 572 sfc_flow_parse_tcp(const struct rte_flow_item *item,
 573                    efx_filter_spec_t *efx_spec,
 574                    struct rte_flow_error *error)
 575 {
 576         int rc;
 577         const struct rte_flow_item_tcp *spec = NULL;
 578         const struct rte_flow_item_tcp *mask = NULL;
 579         const struct rte_flow_item_tcp supp_mask = {
 580                 .hdr = {
 581                         .src_port = 0xffff,
 582                         .dst_port = 0xffff,
 583                 }
 584         };
 585
 586         rc = sfc_flow_parse_init(item,
 587                                  (const void **)&spec,
 588                                  (const void **)&mask,
 589                                  &supp_mask,
 590                                  &rte_flow_item_tcp_mask,
 591                                  sizeof(struct rte_flow_item_tcp),
 592                                  error);
 593         if (rc != 0)
 594                 return rc;
 595
 596         /*
 597          * Filtering by TCP source and destination ports requires
 598          * the appropriate IP_PROTO in hardware filters
 599          */
 600         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_IP_PROTO)) {
 601                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 602                 efx_spec->efs_ip_proto = EFX_IPPROTO_TCP;
 603         } else if (efx_spec->efs_ip_proto != EFX_IPPROTO_TCP) {
 604                 rte_flow_error_set(error, EINVAL,
 605                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 606                         "IP proto in pattern with TCP item should be appropriate");
 607                 return -rte_errno;
 608         }
 609
 610         if (spec == NULL)
 611                 return 0;
 612
 613         /*
 614          * Source and destination ports are in big-endian byte order in item and
 615          * in little-endian in efx_spec, so byte swap is used
 616          */
 617         if (mask->hdr.src_port == supp_mask.hdr.src_port) {
 618                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_PORT;
 619                 efx_spec->efs_rem_port = rte_bswap16(spec->hdr.src_port);
 620         } else if (mask->hdr.src_port != 0) {
 621                 goto fail_bad_mask;
 622         }
 623
 624         if (mask->hdr.dst_port == supp_mask.hdr.dst_port) {
 625                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_PORT;
 626                 efx_spec->efs_loc_port = rte_bswap16(spec->hdr.dst_port);
 627         } else if (mask->hdr.dst_port != 0) {
 628                 goto fail_bad_mask;
 629         }
 630
 631         return 0;
 632
 633 fail_bad_mask:
 634         rte_flow_error_set(error, EINVAL,
 635                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 636                            "Bad mask in the TCP pattern item");
 637         return -rte_errno;
 638 }
 639
 640 /**
 641  * Convert UDP item to EFX filter specification.
 642  *
 643  * @param item[in]
 644  *   Item specification. Only source and destination ports fields
 645  *   are supported. If the mask is NULL, default mask will be used.
 646  *   Ranging is not supported.
 647  * @param efx_spec[in, out]
 648  *   EFX filter specification to update.
 649  * @param[out] error
 650  *   Perform verbose error reporting if not NULL.
 651  */
 652 static int
 653 sfc_flow_parse_udp(const struct rte_flow_item *item,
 654                    efx_filter_spec_t *efx_spec,
 655                    struct rte_flow_error *error)
 656 {
 657         int rc;
 658         const struct rte_flow_item_udp *spec = NULL;
 659         const struct rte_flow_item_udp *mask = NULL;
 660         const struct rte_flow_item_udp supp_mask = {
 661                 .hdr = {
 662                         .src_port = 0xffff,
 663                         .dst_port = 0xffff,
 664                 }
 665         };
 666
 667         rc = sfc_flow_parse_init(item,
 668                                  (const void **)&spec,
 669                                  (const void **)&mask,
 670                                  &supp_mask,
 671                                  &rte_flow_item_udp_mask,
 672                                  sizeof(struct rte_flow_item_udp),
 673                                  error);
 674         if (rc != 0)
 675                 return rc;
 676
 677         /*
 678          * Filtering by UDP source and destination ports requires
 679          * the appropriate IP_PROTO in hardware filters
 680          */
 681         if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_IP_PROTO)) {
 682                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO;
 683                 efx_spec->efs_ip_proto = EFX_IPPROTO_UDP;
 684         } else if (efx_spec->efs_ip_proto != EFX_IPPROTO_UDP) {
 685                 rte_flow_error_set(error, EINVAL,
 686                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 687                         "IP proto in pattern with UDP item should be appropriate");
 688                 return -rte_errno;
 689         }
 690
 691         if (spec == NULL)
 692                 return 0;
 693
 694         /*
 695          * Source and destination ports are in big-endian byte order in item and
 696          * in little-endian in efx_spec, so byte swap is used
 697          */
 698         if (mask->hdr.src_port == supp_mask.hdr.src_port) {
 699                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_PORT;
 700                 efx_spec->efs_rem_port = rte_bswap16(spec->hdr.src_port);
 701         } else if (mask->hdr.src_port != 0) {
 702                 goto fail_bad_mask;
 703         }
 704
 705         if (mask->hdr.dst_port == supp_mask.hdr.dst_port) {
 706                 efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_PORT;
 707                 efx_spec->efs_loc_port = rte_bswap16(spec->hdr.dst_port);
 708         } else if (mask->hdr.dst_port != 0) {
 709                 goto fail_bad_mask;
 710         }
 711
 712         return 0;
 713
 714 fail_bad_mask:
 715         rte_flow_error_set(error, EINVAL,
 716                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 717                            "Bad mask in the UDP pattern item");
 718         return -rte_errno;
 719 }
 720
 721 static const struct sfc_flow_item sfc_flow_items[] = {
 722         {
 723                 .type = RTE_FLOW_ITEM_TYPE_VOID,
 724                 .prev_layer = SFC_FLOW_ITEM_ANY_LAYER,
 725                 .layer = SFC_FLOW_ITEM_ANY_LAYER,
 726                 .parse = sfc_flow_parse_void,
 727         },
 728         {
 729                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 730                 .prev_layer = SFC_FLOW_ITEM_START_LAYER,
 731                 .layer = SFC_FLOW_ITEM_L2,
 732                 .parse = sfc_flow_parse_eth,
 733         },
 734         {
 735                 .type = RTE_FLOW_ITEM_TYPE_VLAN,
 736                 .prev_layer = SFC_FLOW_ITEM_L2,
 737                 .layer = SFC_FLOW_ITEM_L2,
 738                 .parse = sfc_flow_parse_vlan,
 739         },
 740         {
 741                 .type = RTE_FLOW_ITEM_TYPE_IPV4,
 742                 .prev_layer = SFC_FLOW_ITEM_L2,
 743                 .layer = SFC_FLOW_ITEM_L3,
 744                 .parse = sfc_flow_parse_ipv4,
 745         },
 746         {
 747                 .type = RTE_FLOW_ITEM_TYPE_IPV6,
 748                 .prev_layer = SFC_FLOW_ITEM_L2,
 749                 .layer = SFC_FLOW_ITEM_L3,
 750                 .parse = sfc_flow_parse_ipv6,
 751         },
 752         {
 753                 .type = RTE_FLOW_ITEM_TYPE_TCP,
 754                 .prev_layer = SFC_FLOW_ITEM_L3,
 755                 .layer = SFC_FLOW_ITEM_L4,
 756                 .parse = sfc_flow_parse_tcp,
 757         },
 758         {
 759                 .type = RTE_FLOW_ITEM_TYPE_UDP,
 760                 .prev_layer = SFC_FLOW_ITEM_L3,
 761                 .layer = SFC_FLOW_ITEM_L4,
 762                 .parse = sfc_flow_parse_udp,
 763         },
 764 };
 765
 766 /*
 767  * Protocol-independent flow API support
 768  */
 769 static int
 770 sfc_flow_parse_attr(const struct rte_flow_attr *attr,
 771                     struct rte_flow *flow,
 772                     struct rte_flow_error *error)
 773 {
 774         if (attr == NULL) {
 775                 rte_flow_error_set(error, EINVAL,
 776                                    RTE_FLOW_ERROR_TYPE_ATTR, NULL,
 777                                    "NULL attribute");
 778                 return -rte_errno;
 779         }
 780         if (attr->group != 0) {
 781                 rte_flow_error_set(error, ENOTSUP,
 782                                    RTE_FLOW_ERROR_TYPE_ATTR_GROUP, attr,
 783                                    "Groups are not supported");
 784                 return -rte_errno;
 785         }
 786         if (attr->priority != 0) {
 787                 rte_flow_error_set(error, ENOTSUP,
 788                                    RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, attr,
 789                                    "Priorities are not supported");
 790                 return -rte_errno;
 791         }
 792         if (attr->egress != 0) {
 793                 rte_flow_error_set(error, ENOTSUP,
 794                                    RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attr,
 795                                    "Egress is not supported");
 796                 return -rte_errno;
 797         }
 798         if (attr->ingress == 0) {
 799                 rte_flow_error_set(error, ENOTSUP,
 800                                    RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, attr,
 801                                    "Only ingress is supported");
 802                 return -rte_errno;
 803         }
 804
 805         flow->spec.efs_flags |= EFX_FILTER_FLAG_RX;
 806         flow->spec.efs_rss_context = EFX_FILTER_SPEC_RSS_CONTEXT_DEFAULT;
 807
 808         return 0;
 809 }
 810
 811 /* Get item from array sfc_flow_items */
 812 static const struct sfc_flow_item *
 813 sfc_flow_get_item(enum rte_flow_item_type type)
 814 {
 815         unsigned int i;
 816
 817         for (i = 0; i < RTE_DIM(sfc_flow_items); i++)
 818                 if (sfc_flow_items[i].type == type)
 819                         return &sfc_flow_items[i];
 820
 821         return NULL;
 822 }
 823
 824 static int
 825 sfc_flow_parse_pattern(const struct rte_flow_item pattern[],
 826                        struct rte_flow *flow,
 827                        struct rte_flow_error *error)
 828 {
 829         int rc;
 830         unsigned int prev_layer = SFC_FLOW_ITEM_ANY_LAYER;
 831         const struct sfc_flow_item *item;
 832
 833         if (pattern == NULL) {
 834                 rte_flow_error_set(error, EINVAL,
 835                                    RTE_FLOW_ERROR_TYPE_ITEM_NUM, NULL,
 836                                    "NULL pattern");
 837                 return -rte_errno;
 838         }
 839
 840         for (; pattern->type != RTE_FLOW_ITEM_TYPE_END; pattern++) {
 841                 item = sfc_flow_get_item(pattern->type);
 842                 if (item == NULL) {
 843                         rte_flow_error_set(error, ENOTSUP,
 844                                            RTE_FLOW_ERROR_TYPE_ITEM, pattern,
 845                                            "Unsupported pattern item");
 846                         return -rte_errno;
 847                 }
 848
 849                 /*
 850                  * Omitting one or several protocol layers at the beginning
 851                  * of pattern is supported
 852                  */
 853                 if (item->prev_layer != SFC_FLOW_ITEM_ANY_LAYER &&
 854                     prev_layer != SFC_FLOW_ITEM_ANY_LAYER &&
 855                     item->prev_layer != prev_layer) {
 856                         rte_flow_error_set(error, ENOTSUP,
 857                                            RTE_FLOW_ERROR_TYPE_ITEM, pattern,
 858                                            "Unexpected sequence of pattern items");
 859                         return -rte_errno;
 860                 }
 861
 862                 rc = item->parse(pattern, &flow->spec, error);
 863                 if (rc != 0)
 864                         return rc;
 865
 866                 if (item->layer != SFC_FLOW_ITEM_ANY_LAYER)
 867                         prev_layer = item->layer;
 868         }
 869
 870         return 0;
 871 }
 872
 873 static int
 874 sfc_flow_parse_queue(struct sfc_adapter *sa,
 875                      const struct rte_flow_action_queue *queue,
 876                      struct rte_flow *flow)
 877 {
 878         struct sfc_rxq *rxq;
 879
 880         if (queue->index >= sa->rxq_count)
 881                 return -EINVAL;
 882
 883         rxq = sa->rxq_info[queue->index].rxq;
 884         flow->spec.efs_dmaq_id = (uint16_t)rxq->hw_index;
 885
 886         return 0;
 887 }
 888
 889 static int
 890 sfc_flow_parse_actions(struct sfc_adapter *sa,
 891                        const struct rte_flow_action actions[],
 892                        struct rte_flow *flow,
 893                        struct rte_flow_error *error)
 894 {
 895         int rc;
 896         boolean_t is_specified = B_FALSE;
 897
 898         if (actions == NULL) {
 899                 rte_flow_error_set(error, EINVAL,
 900                                    RTE_FLOW_ERROR_TYPE_ACTION_NUM, NULL,
 901                                    "NULL actions");
 902                 return -rte_errno;
 903         }
 904
 905         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
 906                 switch (actions->type) {
 907                 case RTE_FLOW_ACTION_TYPE_VOID:
 908                         break;
 909
 910                 case RTE_FLOW_ACTION_TYPE_QUEUE:
 911                         rc = sfc_flow_parse_queue(sa, actions->conf, flow);
 912                         if (rc != 0) {
 913                                 rte_flow_error_set(error, EINVAL,
 914                                         RTE_FLOW_ERROR_TYPE_ACTION, actions,
 915                                         "Bad QUEUE action");
 916                                 return -rte_errno;
 917                         }
 918
 919                         is_specified = B_TRUE;
 920                         break;
 921
 922                 default:
 923                         rte_flow_error_set(error, ENOTSUP,
 924                                            RTE_FLOW_ERROR_TYPE_ACTION, actions,
 925                                            "Action is not supported");
 926                         return -rte_errno;
 927                 }
 928         }
 929
 930         if (!is_specified) {
 931                 rte_flow_error_set(error, EINVAL,
 932                                    RTE_FLOW_ERROR_TYPE_ACTION_NUM, actions,
 933                                    "Action is unspecified");
 934                 return -rte_errno;
 935         }
 936
 937         return 0;
 938 }
 939
 940 static int
 941 sfc_flow_parse(struct rte_eth_dev *dev,
 942                const struct rte_flow_attr *attr,
 943                const struct rte_flow_item pattern[],
 944                const struct rte_flow_action actions[],
 945                struct rte_flow *flow,
 946                struct rte_flow_error *error)
 947 {
 948         struct sfc_adapter *sa = dev->data->dev_private;
 949         int rc;
 950
 951         memset(&flow->spec, 0, sizeof(flow->spec));
 952
 953         rc = sfc_flow_parse_attr(attr, flow, error);
 954         if (rc != 0)
 955                 goto fail_bad_value;
 956
 957         rc = sfc_flow_parse_pattern(pattern, flow, error);
 958         if (rc != 0)
 959                 goto fail_bad_value;
 960
 961         rc = sfc_flow_parse_actions(sa, actions, flow, error);
 962         if (rc != 0)
 963                 goto fail_bad_value;
 964
 965         if (!sfc_filter_is_match_supported(sa, flow->spec.efs_match_flags)) {
 966                 rte_flow_error_set(error, ENOTSUP,
 967                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 968                                    "Flow rule pattern is not supported");
 969                 return -rte_errno;
 970         }
 971
 972 fail_bad_value:
 973         return rc;
 974 }
 975
 976 static int
 977 sfc_flow_validate(struct rte_eth_dev *dev,
 978                   const struct rte_flow_attr *attr,
 979                   const struct rte_flow_item pattern[],
 980                   const struct rte_flow_action actions[],
 981                   struct rte_flow_error *error)
 982 {
 983         struct rte_flow flow;
 984
 985         return sfc_flow_parse(dev, attr, pattern, actions, &flow, error);
 986 }
 987
 988 static struct rte_flow *
 989 sfc_flow_create(struct rte_eth_dev *dev,
 990                 const struct rte_flow_attr *attr,
 991                 const struct rte_flow_item pattern[],
 992                 const struct rte_flow_action actions[],
 993                 struct rte_flow_error *error)
 994 {
 995         struct sfc_adapter *sa = dev->data->dev_private;
 996         struct rte_flow *flow = NULL;
 997         int rc;
 998
 999         flow = rte_zmalloc("sfc_rte_flow", sizeof(*flow), 0);
1000         if (flow == NULL) {
1001                 rte_flow_error_set(error, ENOMEM,
1002                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1003                                    "Failed to allocate memory");
1004                 goto fail_no_mem;
1005         }
1006
1007         rc = sfc_flow_parse(dev, attr, pattern, actions, flow, error);
1008         if (rc != 0)
1009                 goto fail_bad_value;
1010
1011         TAILQ_INSERT_TAIL(&sa->filter.flow_list, flow, entries);
1012
1013         sfc_adapter_lock(sa);
1014
1015         if (sa->state == SFC_ADAPTER_STARTED) {
1016                 rc = efx_filter_insert(sa->nic, &flow->spec);
1017                 if (rc != 0) {
1018                         rte_flow_error_set(error, rc,
1019                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1020                                 "Failed to insert filter");
1021                         goto fail_filter_insert;
1022                 }
1023         }
1024
1025         sfc_adapter_unlock(sa);
1026
1027         return flow;
1028
1029 fail_filter_insert:
1030         TAILQ_REMOVE(&sa->filter.flow_list, flow, entries);
1031
1032 fail_bad_value:
1033         rte_free(flow);
1034         sfc_adapter_unlock(sa);
1035
1036 fail_no_mem:
1037         return NULL;
1038 }
1039
1040 static int
1041 sfc_flow_remove(struct sfc_adapter *sa,
1042                 struct rte_flow *flow,
1043                 struct rte_flow_error *error)
1044 {
1045         int rc = 0;
1046
1047         SFC_ASSERT(sfc_adapter_is_locked(sa));
1048
1049         if (sa->state == SFC_ADAPTER_STARTED) {
1050                 rc = efx_filter_remove(sa->nic, &flow->spec);
1051                 if (rc != 0)
1052                         rte_flow_error_set(error, rc,
1053                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1054                                 "Failed to destroy flow rule");
1055         }
1056
1057         TAILQ_REMOVE(&sa->filter.flow_list, flow, entries);
1058         rte_free(flow);
1059
1060         return rc;
1061 }
1062
1063 static int
1064 sfc_flow_destroy(struct rte_eth_dev *dev,
1065                  struct rte_flow *flow,
1066                  struct rte_flow_error *error)
1067 {
1068         struct sfc_adapter *sa = dev->data->dev_private;
1069         struct rte_flow *flow_ptr;
1070         int rc = EINVAL;
1071
1072         sfc_adapter_lock(sa);
1073
1074         TAILQ_FOREACH(flow_ptr, &sa->filter.flow_list, entries) {
1075                 if (flow_ptr == flow)
1076                         rc = 0;
1077         }
1078         if (rc != 0) {
1079                 rte_flow_error_set(error, rc,
1080                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1081                                    "Failed to find flow rule to destroy");
1082                 goto fail_bad_value;
1083         }
1084
1085         rc = sfc_flow_remove(sa, flow, error);
1086
1087 fail_bad_value:
1088         sfc_adapter_unlock(sa);
1089
1090         return -rc;
1091 }
1092
1093 static int
1094 sfc_flow_flush(struct rte_eth_dev *dev,
1095                struct rte_flow_error *error)
1096 {
1097         struct sfc_adapter *sa = dev->data->dev_private;
1098         struct rte_flow *flow;
1099         int rc = 0;
1100         int ret = 0;
1101
1102         sfc_adapter_lock(sa);
1103
1104         while ((flow = TAILQ_FIRST(&sa->filter.flow_list)) != NULL) {
1105                 rc = sfc_flow_remove(sa, flow, error);
1106                 if (rc != 0)
1107                         ret = rc;
1108         }
1109
1110         sfc_adapter_unlock(sa);
1111
1112         return -ret;
1113 }
1114
1115 static int
1116 sfc_flow_isolate(struct rte_eth_dev *dev, int enable,
1117                  struct rte_flow_error *error)
1118 {
1119         struct sfc_adapter *sa = dev->data->dev_private;
1120         struct sfc_port *port = &sa->port;
1121         int ret = 0;
1122
1123         sfc_adapter_lock(sa);
1124         if (sa->state != SFC_ADAPTER_INITIALIZED) {
1125                 rte_flow_error_set(error, EBUSY,
1126                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1127                                    NULL, "please close the port first");
1128                 ret = -rte_errno;
1129         } else {
1130                 port->isolated = (enable) ? B_TRUE : B_FALSE;
1131         }
1132         sfc_adapter_unlock(sa);
1133
1134         return ret;
1135 }
1136
1137 const struct rte_flow_ops sfc_flow_ops = {
1138         .validate = sfc_flow_validate,
1139         .create = sfc_flow_create,
1140         .destroy = sfc_flow_destroy,
1141         .flush = sfc_flow_flush,
1142         .query = NULL,
1143         .isolate = sfc_flow_isolate,
1144 };
1145
1146 void
1147 sfc_flow_init(struct sfc_adapter *sa)
1148 {
1149         SFC_ASSERT(sfc_adapter_is_locked(sa));
1150
1151         TAILQ_INIT(&sa->filter.flow_list);
1152 }
1153
1154 void
1155 sfc_flow_fini(struct sfc_adapter *sa)
1156 {
1157         struct rte_flow *flow;
1158
1159         SFC_ASSERT(sfc_adapter_is_locked(sa));
1160
1161         while ((flow = TAILQ_FIRST(&sa->filter.flow_list)) != NULL) {
1162                 TAILQ_REMOVE(&sa->filter.flow_list, flow, entries);
1163                 rte_free(flow);
1164         }
1165 }
1166
1167 void
1168 sfc_flow_stop(struct sfc_adapter *sa)
1169 {
1170         struct rte_flow *flow;
1171
1172         SFC_ASSERT(sfc_adapter_is_locked(sa));
1173
1174         TAILQ_FOREACH(flow, &sa->filter.flow_list, entries)
1175                 efx_filter_remove(sa->nic, &flow->spec);
1176 }
1177
1178 int
1179 sfc_flow_start(struct sfc_adapter *sa)
1180 {
1181         struct rte_flow *flow;
1182         int rc = 0;
1183
1184         sfc_log_init(sa, "entry");
1185
1186         SFC_ASSERT(sfc_adapter_is_locked(sa));
1187
1188         TAILQ_FOREACH(flow, &sa->filter.flow_list, entries) {
1189                 rc = efx_filter_insert(sa->nic, &flow->spec);
1190                 if (rc != 0)
1191                         goto fail_bad_flow;
1192         }
1193
1194         sfc_log_init(sa, "done");
1195
1196 fail_bad_flow:
1197         return rc;
1198 }