drivers/net/sfc/base/efx_filter.c

   1 /*
   2  * Copyright (c) 2007-2016 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright notice,
  11  *    this list of conditions and the following disclaimer in the documentation
  12  *    and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * The views and conclusions contained in the software and documentation are
  27  * those of the authors and should not be interpreted as representing official
  28  * policies, either expressed or implied, of the FreeBSD Project.
  29  */
  30
  31 #include "efx.h"
  32 #include "efx_impl.h"
  33
  34
  35 #if EFSYS_OPT_FILTER
  36
  37 #if EFSYS_OPT_SIENA
  38
  39 static  __checkReturn   efx_rc_t
  40 siena_filter_init(
  41         __in            efx_nic_t *enp);
  42
  43 static                  void
  44 siena_filter_fini(
  45         __in            efx_nic_t *enp);
  46
  47 static  __checkReturn   efx_rc_t
  48 siena_filter_restore(
  49         __in            efx_nic_t *enp);
  50
  51 static  __checkReturn   efx_rc_t
  52 siena_filter_add(
  53         __in            efx_nic_t *enp,
  54         __inout         efx_filter_spec_t *spec,
  55         __in            boolean_t may_replace);
  56
  57 static  __checkReturn   efx_rc_t
  58 siena_filter_delete(
  59         __in            efx_nic_t *enp,
  60         __inout         efx_filter_spec_t *spec);
  61
  62 static  __checkReturn   efx_rc_t
  63 siena_filter_supported_filters(
  64         __in                            efx_nic_t *enp,
  65         __out_ecount(buffer_length)     uint32_t *buffer,
  66         __in                            size_t buffer_length,
  67         __out                           size_t *list_lengthp);
  68
  69 #endif /* EFSYS_OPT_SIENA */
  70
  71 #if EFSYS_OPT_SIENA
  72 static const efx_filter_ops_t   __efx_filter_siena_ops = {
  73         siena_filter_init,              /* efo_init */
  74         siena_filter_fini,              /* efo_fini */
  75         siena_filter_restore,           /* efo_restore */
  76         siena_filter_add,               /* efo_add */
  77         siena_filter_delete,            /* efo_delete */
  78         siena_filter_supported_filters, /* efo_supported_filters */
  79         NULL,                           /* efo_reconfigure */
  80 };
  81 #endif /* EFSYS_OPT_SIENA */
  82
  83 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
  84 static const efx_filter_ops_t   __efx_filter_ef10_ops = {
  85         ef10_filter_init,               /* efo_init */
  86         ef10_filter_fini,               /* efo_fini */
  87         ef10_filter_restore,            /* efo_restore */
  88         ef10_filter_add,                /* efo_add */
  89         ef10_filter_delete,             /* efo_delete */
  90         ef10_filter_supported_filters,  /* efo_supported_filters */
  91         ef10_filter_reconfigure,        /* efo_reconfigure */
  92 };
  93 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */
  94
  95         __checkReturn   efx_rc_t
  96 efx_filter_insert(
  97         __in            efx_nic_t *enp,
  98         __inout         efx_filter_spec_t *spec)
  99 {
 100         const efx_filter_ops_t *efop = enp->en_efop;
 101
 102         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 103         EFSYS_ASSERT3P(spec, !=, NULL);
 104         EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 105
 106         return (efop->efo_add(enp, spec, B_FALSE));
 107 }
 108
 109         __checkReturn   efx_rc_t
 110 efx_filter_remove(
 111         __in            efx_nic_t *enp,
 112         __inout         efx_filter_spec_t *spec)
 113 {
 114         const efx_filter_ops_t *efop = enp->en_efop;
 115
 116         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 117         EFSYS_ASSERT3P(spec, !=, NULL);
 118         EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 119
 120         return (efop->efo_delete(enp, spec));
 121 }
 122
 123         __checkReturn   efx_rc_t
 124 efx_filter_restore(
 125         __in            efx_nic_t *enp)
 126 {
 127         efx_rc_t rc;
 128
 129         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 130
 131         if ((rc = enp->en_efop->efo_restore(enp)) != 0)
 132                 goto fail1;
 133
 134         return (0);
 135
 136 fail1:
 137         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 138
 139         return (rc);
 140 }
 141
 142         __checkReturn   efx_rc_t
 143 efx_filter_init(
 144         __in            efx_nic_t *enp)
 145 {
 146         const efx_filter_ops_t *efop;
 147         efx_rc_t rc;
 148
 149         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 150         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 151         EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_FILTER));
 152
 153         switch (enp->en_family) {
 154 #if EFSYS_OPT_SIENA
 155         case EFX_FAMILY_SIENA:
 156                 efop = &__efx_filter_siena_ops;
 157                 break;
 158 #endif /* EFSYS_OPT_SIENA */
 159
 160 #if EFSYS_OPT_HUNTINGTON
 161         case EFX_FAMILY_HUNTINGTON:
 162                 efop = &__efx_filter_ef10_ops;
 163                 break;
 164 #endif /* EFSYS_OPT_HUNTINGTON */
 165
 166 #if EFSYS_OPT_MEDFORD
 167         case EFX_FAMILY_MEDFORD:
 168                 efop = &__efx_filter_ef10_ops;
 169                 break;
 170 #endif /* EFSYS_OPT_MEDFORD */
 171
 172         default:
 173                 EFSYS_ASSERT(0);
 174                 rc = ENOTSUP;
 175                 goto fail1;
 176         }
 177
 178         if ((rc = efop->efo_init(enp)) != 0)
 179                 goto fail2;
 180
 181         enp->en_efop = efop;
 182         enp->en_mod_flags |= EFX_MOD_FILTER;
 183         return (0);
 184
 185 fail2:
 186         EFSYS_PROBE(fail2);
 187 fail1:
 188         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 189
 190         enp->en_efop = NULL;
 191         enp->en_mod_flags &= ~EFX_MOD_FILTER;
 192         return (rc);
 193 }
 194
 195                         void
 196 efx_filter_fini(
 197         __in            efx_nic_t *enp)
 198 {
 199         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 200         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 201         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 202
 203         enp->en_efop->efo_fini(enp);
 204
 205         enp->en_efop = NULL;
 206         enp->en_mod_flags &= ~EFX_MOD_FILTER;
 207 }
 208
 209 /*
 210  * Query the possible combinations of match flags which can be filtered on.
 211  * These are returned as a list, of which each 32 bit element is a bitmask
 212  * formed of EFX_FILTER_MATCH flags.
 213  *
 214  * The combinations are ordered in priority from highest to lowest.
 215  *
 216  * If the provided buffer is too short to hold the list, the call with fail with
 217  * ENOSPC and *list_lengthp will be set to the buffer length required.
 218  */
 219         __checkReturn   efx_rc_t
 220 efx_filter_supported_filters(
 221         __in                            efx_nic_t *enp,
 222         __out_ecount(buffer_length)     uint32_t *buffer,
 223         __in                            size_t buffer_length,
 224         __out                           size_t *list_lengthp)
 225 {
 226         efx_rc_t rc;
 227
 228         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 229         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 230         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 231         EFSYS_ASSERT(enp->en_efop->efo_supported_filters != NULL);
 232
 233         if (buffer == NULL) {
 234                 rc = EINVAL;
 235                 goto fail1;
 236         }
 237
 238         rc = enp->en_efop->efo_supported_filters(enp, buffer, buffer_length,
 239                                                     list_lengthp);
 240         if (rc != 0)
 241                 goto fail2;
 242
 243         return (0);
 244
 245 fail2:
 246         EFSYS_PROBE(fail2);
 247 fail1:
 248         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 249
 250         return (rc);
 251 }
 252
 253         __checkReturn   efx_rc_t
 254 efx_filter_reconfigure(
 255         __in                            efx_nic_t *enp,
 256         __in_ecount(6)                  uint8_t const *mac_addr,
 257         __in                            boolean_t all_unicst,
 258         __in                            boolean_t mulcst,
 259         __in                            boolean_t all_mulcst,
 260         __in                            boolean_t brdcst,
 261         __in_ecount(6*count)            uint8_t const *addrs,
 262         __in                            uint32_t count)
 263 {
 264         efx_rc_t rc;
 265
 266         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 267         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 268         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 269
 270         if (enp->en_efop->efo_reconfigure != NULL) {
 271                 if ((rc = enp->en_efop->efo_reconfigure(enp, mac_addr,
 272                                                         all_unicst, mulcst,
 273                                                         all_mulcst, brdcst,
 274                                                         addrs, count)) != 0)
 275                         goto fail1;
 276         }
 277
 278         return (0);
 279
 280 fail1:
 281         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 282
 283         return (rc);
 284 }
 285
 286                 void
 287 efx_filter_spec_init_rx(
 288         __out           efx_filter_spec_t *spec,
 289         __in            efx_filter_priority_t priority,
 290         __in            efx_filter_flags_t flags,
 291         __in            efx_rxq_t *erp)
 292 {
 293         EFSYS_ASSERT3P(spec, !=, NULL);
 294         EFSYS_ASSERT3P(erp, !=, NULL);
 295         EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
 296                                 EFX_FILTER_FLAG_RX_SCATTER)) == 0);
 297
 298         memset(spec, 0, sizeof (*spec));
 299         spec->efs_priority = priority;
 300         spec->efs_flags = EFX_FILTER_FLAG_RX | flags;
 301         spec->efs_rss_context = EFX_RSS_CONTEXT_DEFAULT;
 302         spec->efs_dmaq_id = (uint16_t)erp->er_index;
 303 }
 304
 305                 void
 306 efx_filter_spec_init_tx(
 307         __out           efx_filter_spec_t *spec,
 308         __in            efx_txq_t *etp)
 309 {
 310         EFSYS_ASSERT3P(spec, !=, NULL);
 311         EFSYS_ASSERT3P(etp, !=, NULL);
 312
 313         memset(spec, 0, sizeof (*spec));
 314         spec->efs_priority = EFX_FILTER_PRI_REQUIRED;
 315         spec->efs_flags = EFX_FILTER_FLAG_TX;
 316         spec->efs_dmaq_id = (uint16_t)etp->et_index;
 317 }
 318
 319
 320 /*
 321  *  Specify IPv4 host, transport protocol and port in a filter specification
 322  */
 323 __checkReturn           efx_rc_t
 324 efx_filter_spec_set_ipv4_local(
 325         __inout         efx_filter_spec_t *spec,
 326         __in            uint8_t proto,
 327         __in            uint32_t host,
 328         __in            uint16_t port)
 329 {
 330         EFSYS_ASSERT3P(spec, !=, NULL);
 331
 332         spec->efs_match_flags |=
 333                 EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 334                 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT;
 335         spec->efs_ether_type = EFX_ETHER_TYPE_IPV4;
 336         spec->efs_ip_proto = proto;
 337         spec->efs_loc_host.eo_u32[0] = host;
 338         spec->efs_loc_port = port;
 339         return (0);
 340 }
 341
 342 /*
 343  * Specify IPv4 hosts, transport protocol and ports in a filter specification
 344  */
 345 __checkReturn           efx_rc_t
 346 efx_filter_spec_set_ipv4_full(
 347         __inout         efx_filter_spec_t *spec,
 348         __in            uint8_t proto,
 349         __in            uint32_t lhost,
 350         __in            uint16_t lport,
 351         __in            uint32_t rhost,
 352         __in            uint16_t rport)
 353 {
 354         EFSYS_ASSERT3P(spec, !=, NULL);
 355
 356         spec->efs_match_flags |=
 357                 EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 358                 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
 359                 EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
 360         spec->efs_ether_type = EFX_ETHER_TYPE_IPV4;
 361         spec->efs_ip_proto = proto;
 362         spec->efs_loc_host.eo_u32[0] = lhost;
 363         spec->efs_loc_port = lport;
 364         spec->efs_rem_host.eo_u32[0] = rhost;
 365         spec->efs_rem_port = rport;
 366         return (0);
 367 }
 368
 369 /*
 370  * Specify local Ethernet address and/or VID in filter specification
 371  */
 372 __checkReturn           efx_rc_t
 373 efx_filter_spec_set_eth_local(
 374         __inout         efx_filter_spec_t *spec,
 375         __in            uint16_t vid,
 376         __in            const uint8_t *addr)
 377 {
 378         EFSYS_ASSERT3P(spec, !=, NULL);
 379         EFSYS_ASSERT3P(addr, !=, NULL);
 380
 381         if (vid == EFX_FILTER_SPEC_VID_UNSPEC && addr == NULL)
 382                 return (EINVAL);
 383
 384         if (vid != EFX_FILTER_SPEC_VID_UNSPEC) {
 385                 spec->efs_match_flags |= EFX_FILTER_MATCH_OUTER_VID;
 386                 spec->efs_outer_vid = vid;
 387         }
 388         if (addr != NULL) {
 389                 spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC;
 390                 memcpy(spec->efs_loc_mac, addr, EFX_MAC_ADDR_LEN);
 391         }
 392         return (0);
 393 }
 394
 395                         void
 396 efx_filter_spec_set_ether_type(
 397         __inout         efx_filter_spec_t *spec,
 398         __in            uint16_t ether_type)
 399 {
 400         EFSYS_ASSERT3P(spec, !=, NULL);
 401
 402         spec->efs_ether_type = ether_type;
 403         spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 404 }
 405
 406 /*
 407  * Specify matching otherwise-unmatched unicast in a filter specification
 408  */
 409 __checkReturn           efx_rc_t
 410 efx_filter_spec_set_uc_def(
 411         __inout         efx_filter_spec_t *spec)
 412 {
 413         EFSYS_ASSERT3P(spec, !=, NULL);
 414
 415         spec->efs_match_flags |= EFX_FILTER_MATCH_UNKNOWN_UCAST_DST;
 416         return (0);
 417 }
 418
 419 /*
 420  * Specify matching otherwise-unmatched multicast in a filter specification
 421  */
 422 __checkReturn           efx_rc_t
 423 efx_filter_spec_set_mc_def(
 424         __inout         efx_filter_spec_t *spec)
 425 {
 426         EFSYS_ASSERT3P(spec, !=, NULL);
 427
 428         spec->efs_match_flags |= EFX_FILTER_MATCH_UNKNOWN_MCAST_DST;
 429         return (0);
 430 }
 431
 432
 433 __checkReturn           efx_rc_t
 434 efx_filter_spec_set_encap_type(
 435         __inout         efx_filter_spec_t *spec,
 436         __in            efx_tunnel_protocol_t encap_type,
 437         __in            efx_filter_inner_frame_match_t inner_frame_match)
 438 {
 439         uint32_t match_flags = 0;
 440         uint8_t ip_proto;
 441         efx_rc_t rc;
 442
 443         EFSYS_ASSERT3P(spec, !=, NULL);
 444
 445         switch (encap_type) {
 446         case EFX_TUNNEL_PROTOCOL_VXLAN:
 447         case EFX_TUNNEL_PROTOCOL_GENEVE:
 448                 ip_proto = EFX_IPPROTO_UDP;
 449                 break;
 450         case EFX_TUNNEL_PROTOCOL_NVGRE:
 451                 ip_proto = EFX_IPPROTO_GRE;
 452                 break;
 453         default:
 454                 EFSYS_ASSERT(0);
 455                 rc = EINVAL;
 456                 goto fail1;
 457         }
 458
 459         switch (inner_frame_match) {
 460         case EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_MCAST_DST:
 461                 match_flags |= EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST;
 462                 break;
 463         case EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_UCAST_DST:
 464                 match_flags |= EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST;
 465                 break;
 466         case EFX_FILTER_INNER_FRAME_MATCH_OTHER:
 467                 /* This is for when specific inner frames are to be matched. */
 468                 break;
 469         default:
 470                 EFSYS_ASSERT(0);
 471                 rc = EINVAL;
 472                 goto fail2;
 473         }
 474
 475         spec->efs_encap_type = encap_type;
 476         spec->efs_ip_proto = ip_proto;
 477         spec->efs_match_flags |= (match_flags | EFX_FILTER_MATCH_IP_PROTO);
 478
 479         return (0);
 480
 481 fail2:
 482         EFSYS_PROBE(fail2);
 483 fail1:
 484         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 485
 486         return (rc);
 487 }
 488
 489 #if EFSYS_OPT_RX_SCALE
 490         __checkReturn   efx_rc_t
 491 efx_filter_spec_set_rss_context(
 492         __inout         efx_filter_spec_t *spec,
 493         __in            uint32_t rss_context)
 494 {
 495         efx_rc_t rc;
 496
 497         EFSYS_ASSERT3P(spec, !=, NULL);
 498
 499         /* The filter must have been created with EFX_FILTER_FLAG_RX_RSS. */
 500         if ((spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) == 0) {
 501                 rc = EINVAL;
 502                 goto fail1;
 503         }
 504
 505         spec->efs_rss_context = rss_context;
 506
 507         return (0);
 508
 509 fail1:
 510         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 511
 512         return (rc);
 513 }
 514 #endif
 515
 516 #if EFSYS_OPT_SIENA
 517
 518 /*
 519  * "Fudge factors" - difference between programmed value and actual depth.
 520  * Due to pipelined implementation we need to program H/W with a value that
 521  * is larger than the hop limit we want.
 522  */
 523 #define FILTER_CTL_SRCH_FUDGE_WILD 3
 524 #define FILTER_CTL_SRCH_FUDGE_FULL 1
 525
 526 /*
 527  * Hard maximum hop limit.  Hardware will time-out beyond 200-something.
 528  * We also need to avoid infinite loops in efx_filter_search() when the
 529  * table is full.
 530  */
 531 #define FILTER_CTL_SRCH_MAX 200
 532
 533 static  __checkReturn   efx_rc_t
 534 siena_filter_spec_from_gen_spec(
 535         __out           siena_filter_spec_t *sf_spec,
 536         __in            efx_filter_spec_t *gen_spec)
 537 {
 538         efx_rc_t rc;
 539         boolean_t is_full = B_FALSE;
 540
 541         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX)
 542                 EFSYS_ASSERT3U(gen_spec->efs_flags, ==, EFX_FILTER_FLAG_TX);
 543         else
 544                 EFSYS_ASSERT3U(gen_spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 545
 546         /* Siena only has one RSS context */
 547         if ((gen_spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) &&
 548             gen_spec->efs_rss_context != EFX_RSS_CONTEXT_DEFAULT) {
 549                 rc = EINVAL;
 550                 goto fail1;
 551         }
 552
 553         sf_spec->sfs_flags = gen_spec->efs_flags;
 554         sf_spec->sfs_dmaq_id = gen_spec->efs_dmaq_id;
 555
 556         switch (gen_spec->efs_match_flags) {
 557         case EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 558             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
 559             EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT:
 560                 is_full = B_TRUE;
 561                 /* Fall through */
 562         case EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 563             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT: {
 564                 uint32_t rhost, host1, host2;
 565                 uint16_t rport, port1, port2;
 566
 567                 if (gen_spec->efs_ether_type != EFX_ETHER_TYPE_IPV4) {
 568                         rc = ENOTSUP;
 569                         goto fail2;
 570                 }
 571                 if (gen_spec->efs_loc_port == 0 ||
 572                     (is_full && gen_spec->efs_rem_port == 0)) {
 573                         rc = EINVAL;
 574                         goto fail3;
 575                 }
 576                 switch (gen_spec->efs_ip_proto) {
 577                 case EFX_IPPROTO_TCP:
 578                         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 579                                 sf_spec->sfs_type = (is_full ?
 580                                     EFX_SIENA_FILTER_TX_TCP_FULL :
 581                                     EFX_SIENA_FILTER_TX_TCP_WILD);
 582                         } else {
 583                                 sf_spec->sfs_type = (is_full ?
 584                                     EFX_SIENA_FILTER_RX_TCP_FULL :
 585                                     EFX_SIENA_FILTER_RX_TCP_WILD);
 586                         }
 587                         break;
 588                 case EFX_IPPROTO_UDP:
 589                         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 590                                 sf_spec->sfs_type = (is_full ?
 591                                     EFX_SIENA_FILTER_TX_UDP_FULL :
 592                                     EFX_SIENA_FILTER_TX_UDP_WILD);
 593                         } else {
 594                                 sf_spec->sfs_type = (is_full ?
 595                                     EFX_SIENA_FILTER_RX_UDP_FULL :
 596                                     EFX_SIENA_FILTER_RX_UDP_WILD);
 597                         }
 598                         break;
 599                 default:
 600                         rc = ENOTSUP;
 601                         goto fail4;
 602                 }
 603                 /*
 604                  * The filter is constructed in terms of source and destination,
 605                  * with the odd wrinkle that the ports are swapped in a UDP
 606                  * wildcard filter. We need to convert from local and remote
 607                  * addresses (zero for a wildcard).
 608                  */
 609                 rhost = is_full ? gen_spec->efs_rem_host.eo_u32[0] : 0;
 610                 rport = is_full ? gen_spec->efs_rem_port : 0;
 611                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 612                         host1 = gen_spec->efs_loc_host.eo_u32[0];
 613                         host2 = rhost;
 614                 } else {
 615                         host1 = rhost;
 616                         host2 = gen_spec->efs_loc_host.eo_u32[0];
 617                 }
 618                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 619                         if (sf_spec->sfs_type ==
 620                             EFX_SIENA_FILTER_TX_UDP_WILD) {
 621                                 port1 = rport;
 622                                 port2 = gen_spec->efs_loc_port;
 623                         } else {
 624                                 port1 = gen_spec->efs_loc_port;
 625                                 port2 = rport;
 626                         }
 627                 } else {
 628                         if (sf_spec->sfs_type ==
 629                             EFX_SIENA_FILTER_RX_UDP_WILD) {
 630                                 port1 = gen_spec->efs_loc_port;
 631                                 port2 = rport;
 632                         } else {
 633                                 port1 = rport;
 634                                 port2 = gen_spec->efs_loc_port;
 635                         }
 636                 }
 637                 sf_spec->sfs_dword[0] = (host1 << 16) | port1;
 638                 sf_spec->sfs_dword[1] = (port2 << 16) | (host1 >> 16);
 639                 sf_spec->sfs_dword[2] = host2;
 640                 break;
 641         }
 642
 643         case EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_OUTER_VID:
 644                 is_full = B_TRUE;
 645                 /* Fall through */
 646         case EFX_FILTER_MATCH_LOC_MAC:
 647                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 648                         sf_spec->sfs_type = (is_full ?
 649                             EFX_SIENA_FILTER_TX_MAC_FULL :
 650                             EFX_SIENA_FILTER_TX_MAC_WILD);
 651                 } else {
 652                         sf_spec->sfs_type = (is_full ?
 653                             EFX_SIENA_FILTER_RX_MAC_FULL :
 654                             EFX_SIENA_FILTER_RX_MAC_WILD);
 655                 }
 656                 sf_spec->sfs_dword[0] = is_full ? gen_spec->efs_outer_vid : 0;
 657                 sf_spec->sfs_dword[1] =
 658                     gen_spec->efs_loc_mac[2] << 24 |
 659                     gen_spec->efs_loc_mac[3] << 16 |
 660                     gen_spec->efs_loc_mac[4] <<  8 |
 661                     gen_spec->efs_loc_mac[5];
 662                 sf_spec->sfs_dword[2] =
 663                     gen_spec->efs_loc_mac[0] << 8 |
 664                     gen_spec->efs_loc_mac[1];
 665                 break;
 666
 667         default:
 668                 EFSYS_ASSERT(B_FALSE);
 669                 rc = ENOTSUP;
 670                 goto fail5;
 671         }
 672
 673         return (0);
 674
 675 fail5:
 676         EFSYS_PROBE(fail5);
 677 fail4:
 678         EFSYS_PROBE(fail4);
 679 fail3:
 680         EFSYS_PROBE(fail3);
 681 fail2:
 682         EFSYS_PROBE(fail2);
 683 fail1:
 684         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 685
 686         return (rc);
 687 }
 688
 689 /*
 690  * The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
 691  * key derived from the n-tuple.
 692  */
 693 static                  uint16_t
 694 siena_filter_tbl_hash(
 695         __in            uint32_t key)
 696 {
 697         uint16_t tmp;
 698
 699         /* First 16 rounds */
 700         tmp = 0x1fff ^ (uint16_t)(key >> 16);
 701         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
 702         tmp = tmp ^ tmp >> 9;
 703
 704         /* Last 16 rounds */
 705         tmp = tmp ^ tmp << 13 ^ (uint16_t)(key & 0xffff);
 706         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
 707         tmp = tmp ^ tmp >> 9;
 708
 709         return (tmp);
 710 }
 711
 712 /*
 713  * To allow for hash collisions, filter search continues at these
 714  * increments from the first possible entry selected by the hash.
 715  */
 716 static                  uint16_t
 717 siena_filter_tbl_increment(
 718         __in            uint32_t key)
 719 {
 720         return ((uint16_t)(key * 2 - 1));
 721 }
 722
 723 static  __checkReturn   boolean_t
 724 siena_filter_test_used(
 725         __in            siena_filter_tbl_t *sftp,
 726         __in            unsigned int index)
 727 {
 728         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 729         return ((sftp->sft_bitmap[index / 32] & (1 << (index % 32))) != 0);
 730 }
 731
 732 static                  void
 733 siena_filter_set_used(
 734         __in            siena_filter_tbl_t *sftp,
 735         __in            unsigned int index)
 736 {
 737         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 738         sftp->sft_bitmap[index / 32] |= (1 << (index % 32));
 739         ++sftp->sft_used;
 740 }
 741
 742 static                  void
 743 siena_filter_clear_used(
 744         __in            siena_filter_tbl_t *sftp,
 745         __in            unsigned int index)
 746 {
 747         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 748         sftp->sft_bitmap[index / 32] &= ~(1 << (index % 32));
 749
 750         --sftp->sft_used;
 751         EFSYS_ASSERT3U(sftp->sft_used, >=, 0);
 752 }
 753
 754
 755 static                  siena_filter_tbl_id_t
 756 siena_filter_tbl_id(
 757         __in            siena_filter_type_t type)
 758 {
 759         siena_filter_tbl_id_t tbl_id;
 760
 761         switch (type) {
 762         case EFX_SIENA_FILTER_RX_TCP_FULL:
 763         case EFX_SIENA_FILTER_RX_TCP_WILD:
 764         case EFX_SIENA_FILTER_RX_UDP_FULL:
 765         case EFX_SIENA_FILTER_RX_UDP_WILD:
 766                 tbl_id = EFX_SIENA_FILTER_TBL_RX_IP;
 767                 break;
 768
 769         case EFX_SIENA_FILTER_RX_MAC_FULL:
 770         case EFX_SIENA_FILTER_RX_MAC_WILD:
 771                 tbl_id = EFX_SIENA_FILTER_TBL_RX_MAC;
 772                 break;
 773
 774         case EFX_SIENA_FILTER_TX_TCP_FULL:
 775         case EFX_SIENA_FILTER_TX_TCP_WILD:
 776         case EFX_SIENA_FILTER_TX_UDP_FULL:
 777         case EFX_SIENA_FILTER_TX_UDP_WILD:
 778                 tbl_id = EFX_SIENA_FILTER_TBL_TX_IP;
 779                 break;
 780
 781         case EFX_SIENA_FILTER_TX_MAC_FULL:
 782         case EFX_SIENA_FILTER_TX_MAC_WILD:
 783                 tbl_id = EFX_SIENA_FILTER_TBL_TX_MAC;
 784                 break;
 785
 786         default:
 787                 EFSYS_ASSERT(B_FALSE);
 788                 tbl_id = EFX_SIENA_FILTER_NTBLS;
 789                 break;
 790         }
 791         return (tbl_id);
 792 }
 793
 794 static                  void
 795 siena_filter_reset_search_depth(
 796         __inout         siena_filter_t *sfp,
 797         __in            siena_filter_tbl_id_t tbl_id)
 798 {
 799         switch (tbl_id) {
 800         case EFX_SIENA_FILTER_TBL_RX_IP:
 801                 sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_FULL] = 0;
 802                 sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_WILD] = 0;
 803                 sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_FULL] = 0;
 804                 sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_WILD] = 0;
 805                 break;
 806
 807         case EFX_SIENA_FILTER_TBL_RX_MAC:
 808                 sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_FULL] = 0;
 809                 sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_WILD] = 0;
 810                 break;
 811
 812         case EFX_SIENA_FILTER_TBL_TX_IP:
 813                 sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_FULL] = 0;
 814                 sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_WILD] = 0;
 815                 sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_FULL] = 0;
 816                 sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_WILD] = 0;
 817                 break;
 818
 819         case EFX_SIENA_FILTER_TBL_TX_MAC:
 820                 sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_FULL] = 0;
 821                 sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_WILD] = 0;
 822                 break;
 823
 824         default:
 825                 EFSYS_ASSERT(B_FALSE);
 826                 break;
 827         }
 828 }
 829
 830 static                  void
 831 siena_filter_push_rx_limits(
 832         __in            efx_nic_t *enp)
 833 {
 834         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
 835         efx_oword_t oword;
 836
 837         EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 838
 839         EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_FULL_SRCH_LIMIT,
 840             sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_FULL] +
 841             FILTER_CTL_SRCH_FUDGE_FULL);
 842         EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_WILD_SRCH_LIMIT,
 843             sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_WILD] +
 844             FILTER_CTL_SRCH_FUDGE_WILD);
 845         EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_FULL_SRCH_LIMIT,
 846             sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_FULL] +
 847             FILTER_CTL_SRCH_FUDGE_FULL);
 848         EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_WILD_SRCH_LIMIT,
 849             sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_WILD] +
 850             FILTER_CTL_SRCH_FUDGE_WILD);
 851
 852         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_MAC].sft_size) {
 853                 EFX_SET_OWORD_FIELD(oword,
 854                     FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
 855                     sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_FULL] +
 856                     FILTER_CTL_SRCH_FUDGE_FULL);
 857                 EFX_SET_OWORD_FIELD(oword,
 858                     FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
 859                     sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_WILD] +
 860                     FILTER_CTL_SRCH_FUDGE_WILD);
 861         }
 862
 863         EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 864 }
 865
 866 static                  void
 867 siena_filter_push_tx_limits(
 868         __in            efx_nic_t *enp)
 869 {
 870         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
 871         efx_oword_t oword;
 872
 873         EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
 874
 875         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_IP].sft_size != 0) {
 876                 EFX_SET_OWORD_FIELD(oword,
 877                     FRF_CZ_TX_TCPIP_FILTER_FULL_SEARCH_RANGE,
 878                     sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_FULL] +
 879                     FILTER_CTL_SRCH_FUDGE_FULL);
 880                 EFX_SET_OWORD_FIELD(oword,
 881                     FRF_CZ_TX_TCPIP_FILTER_WILD_SEARCH_RANGE,
 882                     sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_WILD] +
 883                     FILTER_CTL_SRCH_FUDGE_WILD);
 884                 EFX_SET_OWORD_FIELD(oword,
 885                     FRF_CZ_TX_UDPIP_FILTER_FULL_SEARCH_RANGE,
 886                     sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_FULL] +
 887                     FILTER_CTL_SRCH_FUDGE_FULL);
 888                 EFX_SET_OWORD_FIELD(oword,
 889                     FRF_CZ_TX_UDPIP_FILTER_WILD_SEARCH_RANGE,
 890                     sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_WILD] +
 891                     FILTER_CTL_SRCH_FUDGE_WILD);
 892         }
 893
 894         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_MAC].sft_size != 0) {
 895                 EFX_SET_OWORD_FIELD(
 896                         oword, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE,
 897                         sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_FULL] +
 898                         FILTER_CTL_SRCH_FUDGE_FULL);
 899                 EFX_SET_OWORD_FIELD(
 900                         oword, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE,
 901                         sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_WILD] +
 902                         FILTER_CTL_SRCH_FUDGE_WILD);
 903         }
 904
 905         EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
 906 }
 907
 908 /* Build a filter entry and return its n-tuple key. */
 909 static  __checkReturn   uint32_t
 910 siena_filter_build(
 911         __out           efx_oword_t *filter,
 912         __in            siena_filter_spec_t *spec)
 913 {
 914         uint32_t dword3;
 915         uint32_t key;
 916         uint8_t  type  = spec->sfs_type;
 917         uint32_t flags = spec->sfs_flags;
 918
 919         switch (siena_filter_tbl_id(type)) {
 920         case EFX_SIENA_FILTER_TBL_RX_IP: {
 921                 boolean_t is_udp = (type == EFX_SIENA_FILTER_RX_UDP_FULL ||
 922                     type == EFX_SIENA_FILTER_RX_UDP_WILD);
 923                 EFX_POPULATE_OWORD_7(*filter,
 924                     FRF_BZ_RSS_EN,
 925                     (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
 926                     FRF_BZ_SCATTER_EN,
 927                     (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
 928                     FRF_AZ_TCP_UDP, is_udp,
 929                     FRF_AZ_RXQ_ID, spec->sfs_dmaq_id,
 930                     EFX_DWORD_2, spec->sfs_dword[2],
 931                     EFX_DWORD_1, spec->sfs_dword[1],
 932                     EFX_DWORD_0, spec->sfs_dword[0]);
 933                 dword3 = is_udp;
 934                 break;
 935         }
 936
 937         case EFX_SIENA_FILTER_TBL_RX_MAC: {
 938                 boolean_t is_wild = (type == EFX_SIENA_FILTER_RX_MAC_WILD);
 939                 EFX_POPULATE_OWORD_7(*filter,
 940                     FRF_CZ_RMFT_RSS_EN,
 941                     (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
 942                     FRF_CZ_RMFT_SCATTER_EN,
 943                     (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
 944                     FRF_CZ_RMFT_RXQ_ID, spec->sfs_dmaq_id,
 945                     FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
 946                     FRF_CZ_RMFT_DEST_MAC_DW1, spec->sfs_dword[2],
 947                     FRF_CZ_RMFT_DEST_MAC_DW0, spec->sfs_dword[1],
 948                     FRF_CZ_RMFT_VLAN_ID, spec->sfs_dword[0]);
 949                 dword3 = is_wild;
 950                 break;
 951         }
 952
 953         case EFX_SIENA_FILTER_TBL_TX_IP: {
 954                 boolean_t is_udp = (type == EFX_SIENA_FILTER_TX_UDP_FULL ||
 955                     type == EFX_SIENA_FILTER_TX_UDP_WILD);
 956                 EFX_POPULATE_OWORD_5(*filter,
 957                     FRF_CZ_TIFT_TCP_UDP, is_udp,
 958                     FRF_CZ_TIFT_TXQ_ID, spec->sfs_dmaq_id,
 959                     EFX_DWORD_2, spec->sfs_dword[2],
 960                     EFX_DWORD_1, spec->sfs_dword[1],
 961                     EFX_DWORD_0, spec->sfs_dword[0]);
 962                 dword3 = is_udp | spec->sfs_dmaq_id << 1;
 963                 break;
 964         }
 965
 966         case EFX_SIENA_FILTER_TBL_TX_MAC: {
 967                 boolean_t is_wild = (type == EFX_SIENA_FILTER_TX_MAC_WILD);
 968                 EFX_POPULATE_OWORD_5(*filter,
 969                     FRF_CZ_TMFT_TXQ_ID, spec->sfs_dmaq_id,
 970                     FRF_CZ_TMFT_WILDCARD_MATCH, is_wild,
 971                     FRF_CZ_TMFT_SRC_MAC_DW1, spec->sfs_dword[2],
 972                     FRF_CZ_TMFT_SRC_MAC_DW0, spec->sfs_dword[1],
 973                     FRF_CZ_TMFT_VLAN_ID, spec->sfs_dword[0]);
 974                 dword3 = is_wild | spec->sfs_dmaq_id << 1;
 975                 break;
 976         }
 977
 978         default:
 979                 EFSYS_ASSERT(B_FALSE);
 980                 return (0);
 981         }
 982
 983         key =
 984             spec->sfs_dword[0] ^
 985             spec->sfs_dword[1] ^
 986             spec->sfs_dword[2] ^
 987             dword3;
 988
 989         return (key);
 990 }
 991
 992 static  __checkReturn           efx_rc_t
 993 siena_filter_push_entry(
 994         __inout                 efx_nic_t *enp,
 995         __in                    siena_filter_type_t type,
 996         __in                    int index,
 997         __in                    efx_oword_t *eop)
 998 {
 999         efx_rc_t rc;
1000
1001         switch (type) {
1002         case EFX_SIENA_FILTER_RX_TCP_FULL:
1003         case EFX_SIENA_FILTER_RX_TCP_WILD:
1004         case EFX_SIENA_FILTER_RX_UDP_FULL:
1005         case EFX_SIENA_FILTER_RX_UDP_WILD:
1006                 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_FILTER_TBL0, index,
1007                     eop, B_TRUE);
1008                 break;
1009
1010         case EFX_SIENA_FILTER_RX_MAC_FULL:
1011         case EFX_SIENA_FILTER_RX_MAC_WILD:
1012                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_RX_MAC_FILTER_TBL0, index,
1013                     eop, B_TRUE);
1014                 break;
1015
1016         case EFX_SIENA_FILTER_TX_TCP_FULL:
1017         case EFX_SIENA_FILTER_TX_TCP_WILD:
1018         case EFX_SIENA_FILTER_TX_UDP_FULL:
1019         case EFX_SIENA_FILTER_TX_UDP_WILD:
1020                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_FILTER_TBL0, index,
1021                     eop, B_TRUE);
1022                 break;
1023
1024         case EFX_SIENA_FILTER_TX_MAC_FULL:
1025         case EFX_SIENA_FILTER_TX_MAC_WILD:
1026                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_MAC_FILTER_TBL0, index,
1027                     eop, B_TRUE);
1028                 break;
1029
1030         default:
1031                 EFSYS_ASSERT(B_FALSE);
1032                 rc = ENOTSUP;
1033                 goto fail1;
1034         }
1035         return (0);
1036
1037 fail1:
1038         return (rc);
1039 }
1040
1041
1042 static  __checkReturn   boolean_t
1043 siena_filter_equal(
1044         __in            const siena_filter_spec_t *left,
1045         __in            const siena_filter_spec_t *right)
1046 {
1047         siena_filter_tbl_id_t tbl_id;
1048
1049         tbl_id = siena_filter_tbl_id(left->sfs_type);
1050
1051
1052         if (left->sfs_type != right->sfs_type)
1053                 return (B_FALSE);
1054
1055         if (memcmp(left->sfs_dword, right->sfs_dword,
1056                 sizeof (left->sfs_dword)))
1057                 return (B_FALSE);
1058
1059         if ((tbl_id == EFX_SIENA_FILTER_TBL_TX_IP ||
1060                 tbl_id == EFX_SIENA_FILTER_TBL_TX_MAC) &&
1061             left->sfs_dmaq_id != right->sfs_dmaq_id)
1062                 return (B_FALSE);
1063
1064         return (B_TRUE);
1065 }
1066
1067 static  __checkReturn   efx_rc_t
1068 siena_filter_search(
1069         __in            siena_filter_tbl_t *sftp,
1070         __in            siena_filter_spec_t *spec,
1071         __in            uint32_t key,
1072         __in            boolean_t for_insert,
1073         __out           int *filter_index,
1074         __out           unsigned int *depth_required)
1075 {
1076         unsigned int hash, incr, filter_idx, depth;
1077
1078         hash = siena_filter_tbl_hash(key);
1079         incr = siena_filter_tbl_increment(key);
1080
1081         filter_idx = hash & (sftp->sft_size - 1);
1082         depth = 1;
1083
1084         for (;;) {
1085                 /*
1086                  * Return success if entry is used and matches this spec
1087                  * or entry is unused and we are trying to insert.
1088                  */
1089                 if (siena_filter_test_used(sftp, filter_idx) ?
1090                     siena_filter_equal(spec,
1091                     &sftp->sft_spec[filter_idx]) :
1092                     for_insert) {
1093                         *filter_index = filter_idx;
1094                         *depth_required = depth;
1095                         return (0);
1096                 }
1097
1098                 /* Return failure if we reached the maximum search depth */
1099                 if (depth == FILTER_CTL_SRCH_MAX)
1100                         return (for_insert ? EBUSY : ENOENT);
1101
1102                 filter_idx = (filter_idx + incr) & (sftp->sft_size - 1);
1103                 ++depth;
1104         }
1105 }
1106
1107 static                  void
1108 siena_filter_clear_entry(
1109         __in            efx_nic_t *enp,
1110         __in            siena_filter_tbl_t *sftp,
1111         __in            int index)
1112 {
1113         efx_oword_t filter;
1114
1115         if (siena_filter_test_used(sftp, index)) {
1116                 siena_filter_clear_used(sftp, index);
1117
1118                 EFX_ZERO_OWORD(filter);
1119                 siena_filter_push_entry(enp,
1120                     sftp->sft_spec[index].sfs_type,
1121                     index, &filter);
1122
1123                 memset(&sftp->sft_spec[index],
1124                     0, sizeof (sftp->sft_spec[0]));
1125         }
1126 }
1127
1128                         void
1129 siena_filter_tbl_clear(
1130         __in            efx_nic_t *enp,
1131         __in            siena_filter_tbl_id_t tbl_id)
1132 {
1133         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1134         siena_filter_tbl_t *sftp = &sfp->sf_tbl[tbl_id];
1135         int index;
1136         efsys_lock_state_t state;
1137
1138         EFSYS_LOCK(enp->en_eslp, state);
1139
1140         for (index = 0; index < sftp->sft_size; ++index) {
1141                 siena_filter_clear_entry(enp, sftp, index);
1142         }
1143
1144         if (sftp->sft_used == 0)
1145                 siena_filter_reset_search_depth(sfp, tbl_id);
1146
1147         EFSYS_UNLOCK(enp->en_eslp, state);
1148 }
1149
1150 static  __checkReturn   efx_rc_t
1151 siena_filter_init(
1152         __in            efx_nic_t *enp)
1153 {
1154         siena_filter_t *sfp;
1155         siena_filter_tbl_t *sftp;
1156         int tbl_id;
1157         efx_rc_t rc;
1158
1159         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (siena_filter_t), sfp);
1160
1161         if (!sfp) {
1162                 rc = ENOMEM;
1163                 goto fail1;
1164         }
1165
1166         enp->en_filter.ef_siena_filter = sfp;
1167
1168         switch (enp->en_family) {
1169         case EFX_FAMILY_SIENA:
1170                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_IP];
1171                 sftp->sft_size = FR_AZ_RX_FILTER_TBL0_ROWS;
1172
1173                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_MAC];
1174                 sftp->sft_size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
1175
1176                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_IP];
1177                 sftp->sft_size = FR_CZ_TX_FILTER_TBL0_ROWS;
1178
1179                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_MAC];
1180                 sftp->sft_size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS;
1181                 break;
1182
1183         default:
1184                 rc = ENOTSUP;
1185                 goto fail2;
1186         }
1187
1188         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1189                 unsigned int bitmap_size;
1190
1191                 sftp = &sfp->sf_tbl[tbl_id];
1192                 if (sftp->sft_size == 0)
1193                         continue;
1194
1195                 EFX_STATIC_ASSERT(sizeof (sftp->sft_bitmap[0]) ==
1196                     sizeof (uint32_t));
1197                 bitmap_size =
1198                     (sftp->sft_size + (sizeof (uint32_t) * 8) - 1) / 8;
1199
1200                 EFSYS_KMEM_ALLOC(enp->en_esip, bitmap_size, sftp->sft_bitmap);
1201                 if (!sftp->sft_bitmap) {
1202                         rc = ENOMEM;
1203                         goto fail3;
1204                 }
1205
1206                 EFSYS_KMEM_ALLOC(enp->en_esip,
1207                     sftp->sft_size * sizeof (*sftp->sft_spec),
1208                     sftp->sft_spec);
1209                 if (!sftp->sft_spec) {
1210                         rc = ENOMEM;
1211                         goto fail4;
1212                 }
1213                 memset(sftp->sft_spec, 0,
1214                     sftp->sft_size * sizeof (*sftp->sft_spec));
1215         }
1216
1217         return (0);
1218
1219 fail4:
1220         EFSYS_PROBE(fail4);
1221
1222 fail3:
1223         EFSYS_PROBE(fail3);
1224
1225 fail2:
1226         EFSYS_PROBE(fail2);
1227         siena_filter_fini(enp);
1228
1229 fail1:
1230         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1231         return (rc);
1232 }
1233
1234 static                  void
1235 siena_filter_fini(
1236         __in            efx_nic_t *enp)
1237 {
1238         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1239         siena_filter_tbl_id_t tbl_id;
1240
1241         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1242         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
1243
1244         if (sfp == NULL)
1245                 return;
1246
1247         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1248                 siena_filter_tbl_t *sftp = &sfp->sf_tbl[tbl_id];
1249                 unsigned int bitmap_size;
1250
1251                 EFX_STATIC_ASSERT(sizeof (sftp->sft_bitmap[0]) ==
1252                     sizeof (uint32_t));
1253                 bitmap_size =
1254                     (sftp->sft_size + (sizeof (uint32_t) * 8) - 1) / 8;
1255
1256                 if (sftp->sft_bitmap != NULL) {
1257                         EFSYS_KMEM_FREE(enp->en_esip, bitmap_size,
1258                             sftp->sft_bitmap);
1259                         sftp->sft_bitmap = NULL;
1260                 }
1261
1262                 if (sftp->sft_spec != NULL) {
1263                         EFSYS_KMEM_FREE(enp->en_esip, sftp->sft_size *
1264                             sizeof (*sftp->sft_spec), sftp->sft_spec);
1265                         sftp->sft_spec = NULL;
1266                 }
1267         }
1268
1269         EFSYS_KMEM_FREE(enp->en_esip, sizeof (siena_filter_t),
1270             enp->en_filter.ef_siena_filter);
1271 }
1272
1273 /* Restore filter state after a reset */
1274 static  __checkReturn   efx_rc_t
1275 siena_filter_restore(
1276         __in            efx_nic_t *enp)
1277 {
1278         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1279         siena_filter_tbl_id_t tbl_id;
1280         siena_filter_tbl_t *sftp;
1281         siena_filter_spec_t *spec;
1282         efx_oword_t filter;
1283         int filter_idx;
1284         efsys_lock_state_t state;
1285         uint32_t key;
1286         efx_rc_t rc;
1287
1288         EFSYS_LOCK(enp->en_eslp, state);
1289
1290         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1291                 sftp = &sfp->sf_tbl[tbl_id];
1292                 for (filter_idx = 0;
1293                         filter_idx < sftp->sft_size;
1294                         filter_idx++) {
1295                         if (!siena_filter_test_used(sftp, filter_idx))
1296                                 continue;
1297
1298                         spec = &sftp->sft_spec[filter_idx];
1299                         if ((key = siena_filter_build(&filter, spec)) == 0) {
1300                                 rc = EINVAL;
1301                                 goto fail1;
1302                         }
1303                         if ((rc = siena_filter_push_entry(enp,
1304                                     spec->sfs_type, filter_idx, &filter)) != 0)
1305                                 goto fail2;
1306                 }
1307         }
1308
1309         siena_filter_push_rx_limits(enp);
1310         siena_filter_push_tx_limits(enp);
1311
1312         EFSYS_UNLOCK(enp->en_eslp, state);
1313
1314         return (0);
1315
1316 fail2:
1317         EFSYS_PROBE(fail2);
1318
1319 fail1:
1320         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1321
1322         EFSYS_UNLOCK(enp->en_eslp, state);
1323
1324         return (rc);
1325 }
1326
1327 static   __checkReturn  efx_rc_t
1328 siena_filter_add(
1329         __in            efx_nic_t *enp,
1330         __inout         efx_filter_spec_t *spec,
1331         __in            boolean_t may_replace)
1332 {
1333         efx_rc_t rc;
1334         siena_filter_spec_t sf_spec;
1335         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1336         siena_filter_tbl_id_t tbl_id;
1337         siena_filter_tbl_t *sftp;
1338         siena_filter_spec_t *saved_sf_spec;
1339         efx_oword_t filter;
1340         int filter_idx;
1341         unsigned int depth;
1342         efsys_lock_state_t state;
1343         uint32_t key;
1344
1345
1346         EFSYS_ASSERT3P(spec, !=, NULL);
1347
1348         if ((rc = siena_filter_spec_from_gen_spec(&sf_spec, spec)) != 0)
1349                 goto fail1;
1350
1351         tbl_id = siena_filter_tbl_id(sf_spec.sfs_type);
1352         sftp = &sfp->sf_tbl[tbl_id];
1353
1354         if (sftp->sft_size == 0) {
1355                 rc = EINVAL;
1356                 goto fail2;
1357         }
1358
1359         key = siena_filter_build(&filter, &sf_spec);
1360
1361         EFSYS_LOCK(enp->en_eslp, state);
1362
1363         rc = siena_filter_search(sftp, &sf_spec, key, B_TRUE,
1364             &filter_idx, &depth);
1365         if (rc != 0)
1366                 goto fail3;
1367
1368         EFSYS_ASSERT3U(filter_idx, <, sftp->sft_size);
1369         saved_sf_spec = &sftp->sft_spec[filter_idx];
1370
1371         if (siena_filter_test_used(sftp, filter_idx)) {
1372                 if (may_replace == B_FALSE) {
1373                         rc = EEXIST;
1374                         goto fail4;
1375                 }
1376         }
1377         siena_filter_set_used(sftp, filter_idx);
1378         *saved_sf_spec = sf_spec;
1379
1380         if (sfp->sf_depth[sf_spec.sfs_type] < depth) {
1381                 sfp->sf_depth[sf_spec.sfs_type] = depth;
1382                 if (tbl_id == EFX_SIENA_FILTER_TBL_TX_IP ||
1383                     tbl_id == EFX_SIENA_FILTER_TBL_TX_MAC)
1384                         siena_filter_push_tx_limits(enp);
1385                 else
1386                         siena_filter_push_rx_limits(enp);
1387         }
1388
1389         siena_filter_push_entry(enp, sf_spec.sfs_type,
1390             filter_idx, &filter);
1391
1392         EFSYS_UNLOCK(enp->en_eslp, state);
1393         return (0);
1394
1395 fail4:
1396         EFSYS_PROBE(fail4);
1397
1398 fail3:
1399         EFSYS_UNLOCK(enp->en_eslp, state);
1400         EFSYS_PROBE(fail3);
1401
1402 fail2:
1403         EFSYS_PROBE(fail2);
1404
1405 fail1:
1406         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1407         return (rc);
1408 }
1409
1410 static   __checkReturn  efx_rc_t
1411 siena_filter_delete(
1412         __in            efx_nic_t *enp,
1413         __inout         efx_filter_spec_t *spec)
1414 {
1415         efx_rc_t rc;
1416         siena_filter_spec_t sf_spec;
1417         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1418         siena_filter_tbl_id_t tbl_id;
1419         siena_filter_tbl_t *sftp;
1420         efx_oword_t filter;
1421         int filter_idx;
1422         unsigned int depth;
1423         efsys_lock_state_t state;
1424         uint32_t key;
1425
1426         EFSYS_ASSERT3P(spec, !=, NULL);
1427
1428         if ((rc = siena_filter_spec_from_gen_spec(&sf_spec, spec)) != 0)
1429                 goto fail1;
1430
1431         tbl_id = siena_filter_tbl_id(sf_spec.sfs_type);
1432         sftp = &sfp->sf_tbl[tbl_id];
1433
1434         key = siena_filter_build(&filter, &sf_spec);
1435
1436         EFSYS_LOCK(enp->en_eslp, state);
1437
1438         rc = siena_filter_search(sftp, &sf_spec, key, B_FALSE,
1439             &filter_idx, &depth);
1440         if (rc != 0)
1441                 goto fail2;
1442
1443         siena_filter_clear_entry(enp, sftp, filter_idx);
1444         if (sftp->sft_used == 0)
1445                 siena_filter_reset_search_depth(sfp, tbl_id);
1446
1447         EFSYS_UNLOCK(enp->en_eslp, state);
1448         return (0);
1449
1450 fail2:
1451         EFSYS_UNLOCK(enp->en_eslp, state);
1452         EFSYS_PROBE(fail2);
1453
1454 fail1:
1455         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1456         return (rc);
1457 }
1458
1459 #define SIENA_MAX_SUPPORTED_MATCHES 4
1460
1461 static  __checkReturn   efx_rc_t
1462 siena_filter_supported_filters(
1463         __in                            efx_nic_t *enp,
1464         __out_ecount(buffer_length)     uint32_t *buffer,
1465         __in                            size_t buffer_length,
1466         __out                           size_t *list_lengthp)
1467 {
1468         uint32_t index = 0;
1469         uint32_t rx_matches[SIENA_MAX_SUPPORTED_MATCHES];
1470         size_t list_length;
1471         efx_rc_t rc;
1472
1473         rx_matches[index++] =
1474             EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
1475             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
1476             EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
1477
1478         rx_matches[index++] =
1479             EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
1480             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT;
1481
1482         if (enp->en_features & EFX_FEATURE_MAC_HEADER_FILTERS) {
1483                 rx_matches[index++] =
1484                     EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC;
1485
1486                 rx_matches[index++] = EFX_FILTER_MATCH_LOC_MAC;
1487         }
1488
1489         EFSYS_ASSERT3U(index, <=, SIENA_MAX_SUPPORTED_MATCHES);
1490         list_length = index;
1491
1492         *list_lengthp = list_length;
1493
1494         if (buffer_length < list_length) {
1495                 rc = ENOSPC;
1496                 goto fail1;
1497         }
1498
1499         memcpy(buffer, rx_matches, list_length * sizeof (rx_matches[0]));
1500
1501         return (0);
1502
1503 fail1:
1504         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1505
1506         return (rc);
1507 }
1508
1509 #undef MAX_SUPPORTED
1510
1511 #endif /* EFSYS_OPT_SIENA */
1512
1513 #endif /* EFSYS_OPT_FILTER */