drivers/net/sfc/base/efx_filter.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright (c) 2007-2018 Solarflare Communications Inc.
   4  * All rights reserved.
   5  */
   6
   7 #include "efx.h"
   8 #include "efx_impl.h"
   9
  10
  11 #if EFSYS_OPT_FILTER
  12
  13 #if EFSYS_OPT_SIENA
  14
  15 static  __checkReturn   efx_rc_t
  16 siena_filter_init(
  17         __in            efx_nic_t *enp);
  18
  19 static                  void
  20 siena_filter_fini(
  21         __in            efx_nic_t *enp);
  22
  23 static  __checkReturn   efx_rc_t
  24 siena_filter_restore(
  25         __in            efx_nic_t *enp);
  26
  27 static  __checkReturn   efx_rc_t
  28 siena_filter_add(
  29         __in            efx_nic_t *enp,
  30         __inout         efx_filter_spec_t *spec,
  31         __in            boolean_t may_replace);
  32
  33 static  __checkReturn   efx_rc_t
  34 siena_filter_delete(
  35         __in            efx_nic_t *enp,
  36         __inout         efx_filter_spec_t *spec);
  37
  38 static  __checkReturn   efx_rc_t
  39 siena_filter_supported_filters(
  40         __in                            efx_nic_t *enp,
  41         __out_ecount(buffer_length)     uint32_t *buffer,
  42         __in                            size_t buffer_length,
  43         __out                           size_t *list_lengthp);
  44
  45 #endif /* EFSYS_OPT_SIENA */
  46
  47 #if EFSYS_OPT_SIENA
  48 static const efx_filter_ops_t   __efx_filter_siena_ops = {
  49         siena_filter_init,              /* efo_init */
  50         siena_filter_fini,              /* efo_fini */
  51         siena_filter_restore,           /* efo_restore */
  52         siena_filter_add,               /* efo_add */
  53         siena_filter_delete,            /* efo_delete */
  54         siena_filter_supported_filters, /* efo_supported_filters */
  55         NULL,                           /* efo_reconfigure */
  56 };
  57 #endif /* EFSYS_OPT_SIENA */
  58
  59 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2
  60 static const efx_filter_ops_t   __efx_filter_ef10_ops = {
  61         ef10_filter_init,               /* efo_init */
  62         ef10_filter_fini,               /* efo_fini */
  63         ef10_filter_restore,            /* efo_restore */
  64         ef10_filter_add,                /* efo_add */
  65         ef10_filter_delete,             /* efo_delete */
  66         ef10_filter_supported_filters,  /* efo_supported_filters */
  67         ef10_filter_reconfigure,        /* efo_reconfigure */
  68 };
  69 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 */
  70
  71         __checkReturn   efx_rc_t
  72 efx_filter_insert(
  73         __in            efx_nic_t *enp,
  74         __inout         efx_filter_spec_t *spec)
  75 {
  76         const efx_filter_ops_t *efop = enp->en_efop;
  77         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
  78         efx_rc_t rc;
  79
  80         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
  81         EFSYS_ASSERT3P(spec, !=, NULL);
  82         EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX);
  83
  84         if ((spec->efs_flags & EFX_FILTER_FLAG_ACTION_MARK) &&
  85             !encp->enc_filter_action_mark_supported) {
  86                 rc = ENOTSUP;
  87                 goto fail1;
  88         }
  89
  90         if ((spec->efs_flags & EFX_FILTER_FLAG_ACTION_FLAG) &&
  91             !encp->enc_filter_action_flag_supported) {
  92                 rc = ENOTSUP;
  93                 goto fail2;
  94         }
  95
  96         return (efop->efo_add(enp, spec, B_FALSE));
  97
  98 fail2:
  99         EFSYS_PROBE(fail2);
 100 fail1:
 101         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 102
 103         return (rc);
 104 }
 105
 106         __checkReturn   efx_rc_t
 107 efx_filter_remove(
 108         __in            efx_nic_t *enp,
 109         __inout         efx_filter_spec_t *spec)
 110 {
 111         const efx_filter_ops_t *efop = enp->en_efop;
 112
 113         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 114         EFSYS_ASSERT3P(spec, !=, NULL);
 115         EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 116
 117         return (efop->efo_delete(enp, spec));
 118 }
 119
 120         __checkReturn   efx_rc_t
 121 efx_filter_restore(
 122         __in            efx_nic_t *enp)
 123 {
 124         efx_rc_t rc;
 125
 126         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 127
 128         if ((rc = enp->en_efop->efo_restore(enp)) != 0)
 129                 goto fail1;
 130
 131         return (0);
 132
 133 fail1:
 134         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 135
 136         return (rc);
 137 }
 138
 139         __checkReturn   efx_rc_t
 140 efx_filter_init(
 141         __in            efx_nic_t *enp)
 142 {
 143         const efx_filter_ops_t *efop;
 144         efx_rc_t rc;
 145
 146         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 147         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 148         EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_FILTER));
 149
 150         switch (enp->en_family) {
 151 #if EFSYS_OPT_SIENA
 152         case EFX_FAMILY_SIENA:
 153                 efop = &__efx_filter_siena_ops;
 154                 break;
 155 #endif /* EFSYS_OPT_SIENA */
 156
 157 #if EFSYS_OPT_HUNTINGTON
 158         case EFX_FAMILY_HUNTINGTON:
 159                 efop = &__efx_filter_ef10_ops;
 160                 break;
 161 #endif /* EFSYS_OPT_HUNTINGTON */
 162
 163 #if EFSYS_OPT_MEDFORD
 164         case EFX_FAMILY_MEDFORD:
 165                 efop = &__efx_filter_ef10_ops;
 166                 break;
 167 #endif /* EFSYS_OPT_MEDFORD */
 168
 169 #if EFSYS_OPT_MEDFORD2
 170         case EFX_FAMILY_MEDFORD2:
 171                 efop = &__efx_filter_ef10_ops;
 172                 break;
 173 #endif /* EFSYS_OPT_MEDFORD2 */
 174
 175         default:
 176                 EFSYS_ASSERT(0);
 177                 rc = ENOTSUP;
 178                 goto fail1;
 179         }
 180
 181         if ((rc = efop->efo_init(enp)) != 0)
 182                 goto fail2;
 183
 184         enp->en_efop = efop;
 185         enp->en_mod_flags |= EFX_MOD_FILTER;
 186         return (0);
 187
 188 fail2:
 189         EFSYS_PROBE(fail2);
 190 fail1:
 191         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 192
 193         enp->en_efop = NULL;
 194         enp->en_mod_flags &= ~EFX_MOD_FILTER;
 195         return (rc);
 196 }
 197
 198                         void
 199 efx_filter_fini(
 200         __in            efx_nic_t *enp)
 201 {
 202         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 203         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 204         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 205
 206         enp->en_efop->efo_fini(enp);
 207
 208         enp->en_efop = NULL;
 209         enp->en_mod_flags &= ~EFX_MOD_FILTER;
 210 }
 211
 212 /*
 213  * Query the possible combinations of match flags which can be filtered on.
 214  * These are returned as a list, of which each 32 bit element is a bitmask
 215  * formed of EFX_FILTER_MATCH flags.
 216  *
 217  * The combinations are ordered in priority from highest to lowest.
 218  *
 219  * If the provided buffer is too short to hold the list, the call with fail with
 220  * ENOSPC and *list_lengthp will be set to the buffer length required.
 221  */
 222         __checkReturn   efx_rc_t
 223 efx_filter_supported_filters(
 224         __in                            efx_nic_t *enp,
 225         __out_ecount(buffer_length)     uint32_t *buffer,
 226         __in                            size_t buffer_length,
 227         __out                           size_t *list_lengthp)
 228 {
 229         efx_rc_t rc;
 230
 231         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 232         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 233         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 234         EFSYS_ASSERT(enp->en_efop->efo_supported_filters != NULL);
 235
 236         if (buffer == NULL) {
 237                 rc = EINVAL;
 238                 goto fail1;
 239         }
 240
 241         rc = enp->en_efop->efo_supported_filters(enp, buffer, buffer_length,
 242                                                     list_lengthp);
 243         if (rc != 0)
 244                 goto fail2;
 245
 246         return (0);
 247
 248 fail2:
 249         EFSYS_PROBE(fail2);
 250 fail1:
 251         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 252
 253         return (rc);
 254 }
 255
 256         __checkReturn   efx_rc_t
 257 efx_filter_reconfigure(
 258         __in                            efx_nic_t *enp,
 259         __in_ecount(6)                  uint8_t const *mac_addr,
 260         __in                            boolean_t all_unicst,
 261         __in                            boolean_t mulcst,
 262         __in                            boolean_t all_mulcst,
 263         __in                            boolean_t brdcst,
 264         __in_ecount(6*count)            uint8_t const *addrs,
 265         __in                            uint32_t count)
 266 {
 267         efx_rc_t rc;
 268
 269         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 270         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 271         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 272
 273         if (enp->en_efop->efo_reconfigure != NULL) {
 274                 if ((rc = enp->en_efop->efo_reconfigure(enp, mac_addr,
 275                                                         all_unicst, mulcst,
 276                                                         all_mulcst, brdcst,
 277                                                         addrs, count)) != 0)
 278                         goto fail1;
 279         }
 280
 281         return (0);
 282
 283 fail1:
 284         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 285
 286         return (rc);
 287 }
 288
 289                 void
 290 efx_filter_spec_init_rx(
 291         __out           efx_filter_spec_t *spec,
 292         __in            efx_filter_priority_t priority,
 293         __in            efx_filter_flags_t flags,
 294         __in            efx_rxq_t *erp)
 295 {
 296         EFSYS_ASSERT3P(spec, !=, NULL);
 297         EFSYS_ASSERT3P(erp, !=, NULL);
 298         EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
 299                                 EFX_FILTER_FLAG_RX_SCATTER)) == 0);
 300
 301         memset(spec, 0, sizeof (*spec));
 302         spec->efs_priority = priority;
 303         spec->efs_flags = EFX_FILTER_FLAG_RX | flags;
 304         spec->efs_rss_context = EFX_RSS_CONTEXT_DEFAULT;
 305         spec->efs_dmaq_id = (uint16_t)erp->er_index;
 306 }
 307
 308                 void
 309 efx_filter_spec_init_tx(
 310         __out           efx_filter_spec_t *spec,
 311         __in            efx_txq_t *etp)
 312 {
 313         EFSYS_ASSERT3P(spec, !=, NULL);
 314         EFSYS_ASSERT3P(etp, !=, NULL);
 315
 316         memset(spec, 0, sizeof (*spec));
 317         spec->efs_priority = EFX_FILTER_PRI_REQUIRED;
 318         spec->efs_flags = EFX_FILTER_FLAG_TX;
 319         spec->efs_dmaq_id = (uint16_t)etp->et_index;
 320 }
 321
 322
 323 /*
 324  *  Specify IPv4 host, transport protocol and port in a filter specification
 325  */
 326 __checkReturn           efx_rc_t
 327 efx_filter_spec_set_ipv4_local(
 328         __inout         efx_filter_spec_t *spec,
 329         __in            uint8_t proto,
 330         __in            uint32_t host,
 331         __in            uint16_t port)
 332 {
 333         EFSYS_ASSERT3P(spec, !=, NULL);
 334
 335         spec->efs_match_flags |=
 336                 EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 337                 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT;
 338         spec->efs_ether_type = EFX_ETHER_TYPE_IPV4;
 339         spec->efs_ip_proto = proto;
 340         spec->efs_loc_host.eo_u32[0] = host;
 341         spec->efs_loc_port = port;
 342         return (0);
 343 }
 344
 345 /*
 346  * Specify IPv4 hosts, transport protocol and ports in a filter specification
 347  */
 348 __checkReturn           efx_rc_t
 349 efx_filter_spec_set_ipv4_full(
 350         __inout         efx_filter_spec_t *spec,
 351         __in            uint8_t proto,
 352         __in            uint32_t lhost,
 353         __in            uint16_t lport,
 354         __in            uint32_t rhost,
 355         __in            uint16_t rport)
 356 {
 357         EFSYS_ASSERT3P(spec, !=, NULL);
 358
 359         spec->efs_match_flags |=
 360                 EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 361                 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
 362                 EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
 363         spec->efs_ether_type = EFX_ETHER_TYPE_IPV4;
 364         spec->efs_ip_proto = proto;
 365         spec->efs_loc_host.eo_u32[0] = lhost;
 366         spec->efs_loc_port = lport;
 367         spec->efs_rem_host.eo_u32[0] = rhost;
 368         spec->efs_rem_port = rport;
 369         return (0);
 370 }
 371
 372 /*
 373  * Specify local Ethernet address and/or VID in filter specification
 374  */
 375 __checkReturn           efx_rc_t
 376 efx_filter_spec_set_eth_local(
 377         __inout         efx_filter_spec_t *spec,
 378         __in            uint16_t vid,
 379         __in            const uint8_t *addr)
 380 {
 381         EFSYS_ASSERT3P(spec, !=, NULL);
 382         EFSYS_ASSERT3P(addr, !=, NULL);
 383
 384         if (vid == EFX_FILTER_SPEC_VID_UNSPEC && addr == NULL)
 385                 return (EINVAL);
 386
 387         if (vid != EFX_FILTER_SPEC_VID_UNSPEC) {
 388                 spec->efs_match_flags |= EFX_FILTER_MATCH_OUTER_VID;
 389                 spec->efs_outer_vid = vid;
 390         }
 391         if (addr != NULL) {
 392                 spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC;
 393                 memcpy(spec->efs_loc_mac, addr, EFX_MAC_ADDR_LEN);
 394         }
 395         return (0);
 396 }
 397
 398                         void
 399 efx_filter_spec_set_ether_type(
 400         __inout         efx_filter_spec_t *spec,
 401         __in            uint16_t ether_type)
 402 {
 403         EFSYS_ASSERT3P(spec, !=, NULL);
 404
 405         spec->efs_ether_type = ether_type;
 406         spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 407 }
 408
 409 /*
 410  * Specify matching otherwise-unmatched unicast in a filter specification
 411  */
 412 __checkReturn           efx_rc_t
 413 efx_filter_spec_set_uc_def(
 414         __inout         efx_filter_spec_t *spec)
 415 {
 416         EFSYS_ASSERT3P(spec, !=, NULL);
 417
 418         spec->efs_match_flags |= EFX_FILTER_MATCH_UNKNOWN_UCAST_DST;
 419         return (0);
 420 }
 421
 422 /*
 423  * Specify matching otherwise-unmatched multicast in a filter specification
 424  */
 425 __checkReturn           efx_rc_t
 426 efx_filter_spec_set_mc_def(
 427         __inout         efx_filter_spec_t *spec)
 428 {
 429         EFSYS_ASSERT3P(spec, !=, NULL);
 430
 431         spec->efs_match_flags |= EFX_FILTER_MATCH_UNKNOWN_MCAST_DST;
 432         return (0);
 433 }
 434
 435
 436 __checkReturn           efx_rc_t
 437 efx_filter_spec_set_encap_type(
 438         __inout         efx_filter_spec_t *spec,
 439         __in            efx_tunnel_protocol_t encap_type,
 440         __in            efx_filter_inner_frame_match_t inner_frame_match)
 441 {
 442         uint32_t match_flags = EFX_FILTER_MATCH_ENCAP_TYPE;
 443         uint8_t ip_proto;
 444         efx_rc_t rc;
 445
 446         EFSYS_ASSERT3P(spec, !=, NULL);
 447
 448         switch (encap_type) {
 449         case EFX_TUNNEL_PROTOCOL_VXLAN:
 450         case EFX_TUNNEL_PROTOCOL_GENEVE:
 451                 ip_proto = EFX_IPPROTO_UDP;
 452                 break;
 453         case EFX_TUNNEL_PROTOCOL_NVGRE:
 454                 ip_proto = EFX_IPPROTO_GRE;
 455                 break;
 456         default:
 457                 EFSYS_ASSERT(0);
 458                 rc = EINVAL;
 459                 goto fail1;
 460         }
 461
 462         switch (inner_frame_match) {
 463         case EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_MCAST_DST:
 464                 match_flags |= EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST;
 465                 break;
 466         case EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_UCAST_DST:
 467                 match_flags |= EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST;
 468                 break;
 469         case EFX_FILTER_INNER_FRAME_MATCH_OTHER:
 470                 /* This is for when specific inner frames are to be matched. */
 471                 break;
 472         default:
 473                 EFSYS_ASSERT(0);
 474                 rc = EINVAL;
 475                 goto fail2;
 476         }
 477
 478         spec->efs_encap_type = encap_type;
 479         spec->efs_ip_proto = ip_proto;
 480         spec->efs_match_flags |= (match_flags | EFX_FILTER_MATCH_IP_PROTO);
 481
 482         return (0);
 483
 484 fail2:
 485         EFSYS_PROBE(fail2);
 486 fail1:
 487         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 488
 489         return (rc);
 490 }
 491
 492 /*
 493  * Specify inner and outer Ethernet address and VNI or VSID in tunnel filter
 494  * specification.
 495  */
 496 static  __checkReturn   efx_rc_t
 497 efx_filter_spec_set_tunnel(
 498         __inout efx_filter_spec_t *spec,
 499         __in            efx_tunnel_protocol_t encap_type,
 500         __in            const uint8_t *vni_or_vsid,
 501         __in            const uint8_t *inner_addr,
 502         __in            const uint8_t *outer_addr)
 503 {
 504         efx_rc_t rc;
 505
 506         EFSYS_ASSERT3P(spec, !=, NULL);
 507         EFSYS_ASSERT3P(vni_or_vsid, !=, NULL);
 508         EFSYS_ASSERT3P(inner_addr, !=, NULL);
 509         EFSYS_ASSERT3P(outer_addr, !=, NULL);
 510
 511         switch (encap_type) {
 512         case EFX_TUNNEL_PROTOCOL_VXLAN:
 513         case EFX_TUNNEL_PROTOCOL_GENEVE:
 514         case EFX_TUNNEL_PROTOCOL_NVGRE:
 515                 break;
 516         default:
 517                 rc = EINVAL;
 518                 goto fail1;
 519         }
 520
 521         if ((inner_addr == NULL) && (outer_addr == NULL)) {
 522                 rc = EINVAL;
 523                 goto fail2;
 524         }
 525
 526         if (vni_or_vsid != NULL) {
 527                 spec->efs_match_flags |= EFX_FILTER_MATCH_VNI_OR_VSID;
 528                 memcpy(spec->efs_vni_or_vsid, vni_or_vsid, EFX_VNI_OR_VSID_LEN);
 529         }
 530         if (outer_addr != NULL) {
 531                 spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC;
 532                 memcpy(spec->efs_loc_mac, outer_addr, EFX_MAC_ADDR_LEN);
 533         }
 534         if (inner_addr != NULL) {
 535                 spec->efs_match_flags |= EFX_FILTER_MATCH_IFRM_LOC_MAC;
 536                 memcpy(spec->efs_ifrm_loc_mac, inner_addr, EFX_MAC_ADDR_LEN);
 537         }
 538
 539         spec->efs_match_flags |= EFX_FILTER_MATCH_ENCAP_TYPE;
 540         spec->efs_encap_type = encap_type;
 541
 542         return (0);
 543
 544 fail2:
 545         EFSYS_PROBE(fail2);
 546 fail1:
 547         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 548
 549         return (rc);
 550 }
 551
 552 /*
 553  * Specify inner and outer Ethernet address and VNI in VXLAN filter
 554  * specification.
 555  */
 556 __checkReturn           efx_rc_t
 557 efx_filter_spec_set_vxlan(
 558         __inout         efx_filter_spec_t *spec,
 559         __in            const uint8_t *vni,
 560         __in            const uint8_t *inner_addr,
 561         __in            const uint8_t *outer_addr)
 562 {
 563         return efx_filter_spec_set_tunnel(spec, EFX_TUNNEL_PROTOCOL_VXLAN,
 564             vni, inner_addr, outer_addr);
 565 }
 566
 567 /*
 568  * Specify inner and outer Ethernet address and VNI in Geneve filter
 569  * specification.
 570  */
 571 __checkReturn           efx_rc_t
 572 efx_filter_spec_set_geneve(
 573         __inout         efx_filter_spec_t *spec,
 574         __in            const uint8_t *vni,
 575         __in            const uint8_t *inner_addr,
 576         __in            const uint8_t *outer_addr)
 577 {
 578         return efx_filter_spec_set_tunnel(spec, EFX_TUNNEL_PROTOCOL_GENEVE,
 579             vni, inner_addr, outer_addr);
 580 }
 581
 582 /*
 583  * Specify inner and outer Ethernet address and vsid in NVGRE filter
 584  * specification.
 585  */
 586 __checkReturn           efx_rc_t
 587 efx_filter_spec_set_nvgre(
 588         __inout         efx_filter_spec_t *spec,
 589         __in            const uint8_t *vsid,
 590         __in            const uint8_t *inner_addr,
 591         __in            const uint8_t *outer_addr)
 592 {
 593         return efx_filter_spec_set_tunnel(spec, EFX_TUNNEL_PROTOCOL_NVGRE,
 594             vsid, inner_addr, outer_addr);
 595 }
 596
 597 #if EFSYS_OPT_RX_SCALE
 598         __checkReturn   efx_rc_t
 599 efx_filter_spec_set_rss_context(
 600         __inout         efx_filter_spec_t *spec,
 601         __in            uint32_t rss_context)
 602 {
 603         efx_rc_t rc;
 604
 605         EFSYS_ASSERT3P(spec, !=, NULL);
 606
 607         /* The filter must have been created with EFX_FILTER_FLAG_RX_RSS. */
 608         if ((spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) == 0) {
 609                 rc = EINVAL;
 610                 goto fail1;
 611         }
 612
 613         spec->efs_rss_context = rss_context;
 614
 615         return (0);
 616
 617 fail1:
 618         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 619
 620         return (rc);
 621 }
 622 #endif
 623
 624 #if EFSYS_OPT_SIENA
 625
 626 /*
 627  * "Fudge factors" - difference between programmed value and actual depth.
 628  * Due to pipelined implementation we need to program H/W with a value that
 629  * is larger than the hop limit we want.
 630  */
 631 #define FILTER_CTL_SRCH_FUDGE_WILD 3
 632 #define FILTER_CTL_SRCH_FUDGE_FULL 1
 633
 634 /*
 635  * Hard maximum hop limit.  Hardware will time-out beyond 200-something.
 636  * We also need to avoid infinite loops in efx_filter_search() when the
 637  * table is full.
 638  */
 639 #define FILTER_CTL_SRCH_MAX 200
 640
 641 static  __checkReturn   efx_rc_t
 642 siena_filter_spec_from_gen_spec(
 643         __out           siena_filter_spec_t *sf_spec,
 644         __in            efx_filter_spec_t *gen_spec)
 645 {
 646         efx_rc_t rc;
 647         boolean_t is_full = B_FALSE;
 648
 649         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX)
 650                 EFSYS_ASSERT3U(gen_spec->efs_flags, ==, EFX_FILTER_FLAG_TX);
 651         else
 652                 EFSYS_ASSERT3U(gen_spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 653
 654         /* Siena only has one RSS context */
 655         if ((gen_spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) &&
 656             gen_spec->efs_rss_context != EFX_RSS_CONTEXT_DEFAULT) {
 657                 rc = EINVAL;
 658                 goto fail1;
 659         }
 660
 661         sf_spec->sfs_flags = gen_spec->efs_flags;
 662         sf_spec->sfs_dmaq_id = gen_spec->efs_dmaq_id;
 663
 664         switch (gen_spec->efs_match_flags) {
 665         case EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 666             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
 667             EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT:
 668                 is_full = B_TRUE;
 669                 /* Fall through */
 670         case EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 671             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT: {
 672                 uint32_t rhost, host1, host2;
 673                 uint16_t rport, port1, port2;
 674
 675                 if (gen_spec->efs_ether_type != EFX_ETHER_TYPE_IPV4) {
 676                         rc = ENOTSUP;
 677                         goto fail2;
 678                 }
 679                 if (gen_spec->efs_loc_port == 0 ||
 680                     (is_full && gen_spec->efs_rem_port == 0)) {
 681                         rc = EINVAL;
 682                         goto fail3;
 683                 }
 684                 switch (gen_spec->efs_ip_proto) {
 685                 case EFX_IPPROTO_TCP:
 686                         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 687                                 sf_spec->sfs_type = (is_full ?
 688                                     EFX_SIENA_FILTER_TX_TCP_FULL :
 689                                     EFX_SIENA_FILTER_TX_TCP_WILD);
 690                         } else {
 691                                 sf_spec->sfs_type = (is_full ?
 692                                     EFX_SIENA_FILTER_RX_TCP_FULL :
 693                                     EFX_SIENA_FILTER_RX_TCP_WILD);
 694                         }
 695                         break;
 696                 case EFX_IPPROTO_UDP:
 697                         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 698                                 sf_spec->sfs_type = (is_full ?
 699                                     EFX_SIENA_FILTER_TX_UDP_FULL :
 700                                     EFX_SIENA_FILTER_TX_UDP_WILD);
 701                         } else {
 702                                 sf_spec->sfs_type = (is_full ?
 703                                     EFX_SIENA_FILTER_RX_UDP_FULL :
 704                                     EFX_SIENA_FILTER_RX_UDP_WILD);
 705                         }
 706                         break;
 707                 default:
 708                         rc = ENOTSUP;
 709                         goto fail4;
 710                 }
 711                 /*
 712                  * The filter is constructed in terms of source and destination,
 713                  * with the odd wrinkle that the ports are swapped in a UDP
 714                  * wildcard filter. We need to convert from local and remote
 715                  * addresses (zero for a wildcard).
 716                  */
 717                 rhost = is_full ? gen_spec->efs_rem_host.eo_u32[0] : 0;
 718                 rport = is_full ? gen_spec->efs_rem_port : 0;
 719                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 720                         host1 = gen_spec->efs_loc_host.eo_u32[0];
 721                         host2 = rhost;
 722                 } else {
 723                         host1 = rhost;
 724                         host2 = gen_spec->efs_loc_host.eo_u32[0];
 725                 }
 726                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 727                         if (sf_spec->sfs_type ==
 728                             EFX_SIENA_FILTER_TX_UDP_WILD) {
 729                                 port1 = rport;
 730                                 port2 = gen_spec->efs_loc_port;
 731                         } else {
 732                                 port1 = gen_spec->efs_loc_port;
 733                                 port2 = rport;
 734                         }
 735                 } else {
 736                         if (sf_spec->sfs_type ==
 737                             EFX_SIENA_FILTER_RX_UDP_WILD) {
 738                                 port1 = gen_spec->efs_loc_port;
 739                                 port2 = rport;
 740                         } else {
 741                                 port1 = rport;
 742                                 port2 = gen_spec->efs_loc_port;
 743                         }
 744                 }
 745                 sf_spec->sfs_dword[0] = (host1 << 16) | port1;
 746                 sf_spec->sfs_dword[1] = (port2 << 16) | (host1 >> 16);
 747                 sf_spec->sfs_dword[2] = host2;
 748                 break;
 749         }
 750
 751         case EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_OUTER_VID:
 752                 is_full = B_TRUE;
 753                 /* Fall through */
 754         case EFX_FILTER_MATCH_LOC_MAC:
 755                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 756                         sf_spec->sfs_type = (is_full ?
 757                             EFX_SIENA_FILTER_TX_MAC_FULL :
 758                             EFX_SIENA_FILTER_TX_MAC_WILD);
 759                 } else {
 760                         sf_spec->sfs_type = (is_full ?
 761                             EFX_SIENA_FILTER_RX_MAC_FULL :
 762                             EFX_SIENA_FILTER_RX_MAC_WILD);
 763                 }
 764                 sf_spec->sfs_dword[0] = is_full ? gen_spec->efs_outer_vid : 0;
 765                 sf_spec->sfs_dword[1] =
 766                     gen_spec->efs_loc_mac[2] << 24 |
 767                     gen_spec->efs_loc_mac[3] << 16 |
 768                     gen_spec->efs_loc_mac[4] <<  8 |
 769                     gen_spec->efs_loc_mac[5];
 770                 sf_spec->sfs_dword[2] =
 771                     gen_spec->efs_loc_mac[0] << 8 |
 772                     gen_spec->efs_loc_mac[1];
 773                 break;
 774
 775         default:
 776                 EFSYS_ASSERT(B_FALSE);
 777                 rc = ENOTSUP;
 778                 goto fail5;
 779         }
 780
 781         return (0);
 782
 783 fail5:
 784         EFSYS_PROBE(fail5);
 785 fail4:
 786         EFSYS_PROBE(fail4);
 787 fail3:
 788         EFSYS_PROBE(fail3);
 789 fail2:
 790         EFSYS_PROBE(fail2);
 791 fail1:
 792         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 793
 794         return (rc);
 795 }
 796
 797 /*
 798  * The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
 799  * key derived from the n-tuple.
 800  */
 801 static                  uint16_t
 802 siena_filter_tbl_hash(
 803         __in            uint32_t key)
 804 {
 805         uint16_t tmp;
 806
 807         /* First 16 rounds */
 808         tmp = 0x1fff ^ (uint16_t)(key >> 16);
 809         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
 810         tmp = tmp ^ tmp >> 9;
 811
 812         /* Last 16 rounds */
 813         tmp = tmp ^ tmp << 13 ^ (uint16_t)(key & 0xffff);
 814         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
 815         tmp = tmp ^ tmp >> 9;
 816
 817         return (tmp);
 818 }
 819
 820 /*
 821  * To allow for hash collisions, filter search continues at these
 822  * increments from the first possible entry selected by the hash.
 823  */
 824 static                  uint16_t
 825 siena_filter_tbl_increment(
 826         __in            uint32_t key)
 827 {
 828         return ((uint16_t)(key * 2 - 1));
 829 }
 830
 831 static  __checkReturn   boolean_t
 832 siena_filter_test_used(
 833         __in            siena_filter_tbl_t *sftp,
 834         __in            unsigned int index)
 835 {
 836         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 837         return ((sftp->sft_bitmap[index / 32] & (1 << (index % 32))) != 0);
 838 }
 839
 840 static                  void
 841 siena_filter_set_used(
 842         __in            siena_filter_tbl_t *sftp,
 843         __in            unsigned int index)
 844 {
 845         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 846         sftp->sft_bitmap[index / 32] |= (1 << (index % 32));
 847         ++sftp->sft_used;
 848 }
 849
 850 static                  void
 851 siena_filter_clear_used(
 852         __in            siena_filter_tbl_t *sftp,
 853         __in            unsigned int index)
 854 {
 855         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 856         sftp->sft_bitmap[index / 32] &= ~(1 << (index % 32));
 857
 858         --sftp->sft_used;
 859         EFSYS_ASSERT3U(sftp->sft_used, >=, 0);
 860 }
 861
 862
 863 static                  siena_filter_tbl_id_t
 864 siena_filter_tbl_id(
 865         __in            siena_filter_type_t type)
 866 {
 867         siena_filter_tbl_id_t tbl_id;
 868
 869         switch (type) {
 870         case EFX_SIENA_FILTER_RX_TCP_FULL:
 871         case EFX_SIENA_FILTER_RX_TCP_WILD:
 872         case EFX_SIENA_FILTER_RX_UDP_FULL:
 873         case EFX_SIENA_FILTER_RX_UDP_WILD:
 874                 tbl_id = EFX_SIENA_FILTER_TBL_RX_IP;
 875                 break;
 876
 877         case EFX_SIENA_FILTER_RX_MAC_FULL:
 878         case EFX_SIENA_FILTER_RX_MAC_WILD:
 879                 tbl_id = EFX_SIENA_FILTER_TBL_RX_MAC;
 880                 break;
 881
 882         case EFX_SIENA_FILTER_TX_TCP_FULL:
 883         case EFX_SIENA_FILTER_TX_TCP_WILD:
 884         case EFX_SIENA_FILTER_TX_UDP_FULL:
 885         case EFX_SIENA_FILTER_TX_UDP_WILD:
 886                 tbl_id = EFX_SIENA_FILTER_TBL_TX_IP;
 887                 break;
 888
 889         case EFX_SIENA_FILTER_TX_MAC_FULL:
 890         case EFX_SIENA_FILTER_TX_MAC_WILD:
 891                 tbl_id = EFX_SIENA_FILTER_TBL_TX_MAC;
 892                 break;
 893
 894         default:
 895                 EFSYS_ASSERT(B_FALSE);
 896                 tbl_id = EFX_SIENA_FILTER_NTBLS;
 897                 break;
 898         }
 899         return (tbl_id);
 900 }
 901
 902 static                  void
 903 siena_filter_reset_search_depth(
 904         __inout         siena_filter_t *sfp,
 905         __in            siena_filter_tbl_id_t tbl_id)
 906 {
 907         switch (tbl_id) {
 908         case EFX_SIENA_FILTER_TBL_RX_IP:
 909                 sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_FULL] = 0;
 910                 sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_WILD] = 0;
 911                 sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_FULL] = 0;
 912                 sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_WILD] = 0;
 913                 break;
 914
 915         case EFX_SIENA_FILTER_TBL_RX_MAC:
 916                 sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_FULL] = 0;
 917                 sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_WILD] = 0;
 918                 break;
 919
 920         case EFX_SIENA_FILTER_TBL_TX_IP:
 921                 sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_FULL] = 0;
 922                 sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_WILD] = 0;
 923                 sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_FULL] = 0;
 924                 sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_WILD] = 0;
 925                 break;
 926
 927         case EFX_SIENA_FILTER_TBL_TX_MAC:
 928                 sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_FULL] = 0;
 929                 sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_WILD] = 0;
 930                 break;
 931
 932         default:
 933                 EFSYS_ASSERT(B_FALSE);
 934                 break;
 935         }
 936 }
 937
 938 static                  void
 939 siena_filter_push_rx_limits(
 940         __in            efx_nic_t *enp)
 941 {
 942         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
 943         efx_oword_t oword;
 944
 945         EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 946
 947         EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_FULL_SRCH_LIMIT,
 948             sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_FULL] +
 949             FILTER_CTL_SRCH_FUDGE_FULL);
 950         EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_WILD_SRCH_LIMIT,
 951             sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_WILD] +
 952             FILTER_CTL_SRCH_FUDGE_WILD);
 953         EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_FULL_SRCH_LIMIT,
 954             sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_FULL] +
 955             FILTER_CTL_SRCH_FUDGE_FULL);
 956         EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_WILD_SRCH_LIMIT,
 957             sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_WILD] +
 958             FILTER_CTL_SRCH_FUDGE_WILD);
 959
 960         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_MAC].sft_size) {
 961                 EFX_SET_OWORD_FIELD(oword,
 962                     FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
 963                     sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_FULL] +
 964                     FILTER_CTL_SRCH_FUDGE_FULL);
 965                 EFX_SET_OWORD_FIELD(oword,
 966                     FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
 967                     sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_WILD] +
 968                     FILTER_CTL_SRCH_FUDGE_WILD);
 969         }
 970
 971         EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 972 }
 973
 974 static                  void
 975 siena_filter_push_tx_limits(
 976         __in            efx_nic_t *enp)
 977 {
 978         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
 979         efx_oword_t oword;
 980
 981         EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
 982
 983         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_IP].sft_size != 0) {
 984                 EFX_SET_OWORD_FIELD(oword,
 985                     FRF_CZ_TX_TCPIP_FILTER_FULL_SEARCH_RANGE,
 986                     sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_FULL] +
 987                     FILTER_CTL_SRCH_FUDGE_FULL);
 988                 EFX_SET_OWORD_FIELD(oword,
 989                     FRF_CZ_TX_TCPIP_FILTER_WILD_SEARCH_RANGE,
 990                     sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_WILD] +
 991                     FILTER_CTL_SRCH_FUDGE_WILD);
 992                 EFX_SET_OWORD_FIELD(oword,
 993                     FRF_CZ_TX_UDPIP_FILTER_FULL_SEARCH_RANGE,
 994                     sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_FULL] +
 995                     FILTER_CTL_SRCH_FUDGE_FULL);
 996                 EFX_SET_OWORD_FIELD(oword,
 997                     FRF_CZ_TX_UDPIP_FILTER_WILD_SEARCH_RANGE,
 998                     sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_WILD] +
 999                     FILTER_CTL_SRCH_FUDGE_WILD);
1000         }
1001
1002         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_MAC].sft_size != 0) {
1003                 EFX_SET_OWORD_FIELD(
1004                         oword, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE,
1005                         sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_FULL] +
1006                         FILTER_CTL_SRCH_FUDGE_FULL);
1007                 EFX_SET_OWORD_FIELD(
1008                         oword, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE,
1009                         sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_WILD] +
1010                         FILTER_CTL_SRCH_FUDGE_WILD);
1011         }
1012
1013         EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
1014 }
1015
1016 /* Build a filter entry and return its n-tuple key. */
1017 static  __checkReturn   uint32_t
1018 siena_filter_build(
1019         __out           efx_oword_t *filter,
1020         __in            siena_filter_spec_t *spec)
1021 {
1022         uint32_t dword3;
1023         uint32_t key;
1024         uint8_t  type  = spec->sfs_type;
1025         uint32_t flags = spec->sfs_flags;
1026
1027         switch (siena_filter_tbl_id(type)) {
1028         case EFX_SIENA_FILTER_TBL_RX_IP: {
1029                 boolean_t is_udp = (type == EFX_SIENA_FILTER_RX_UDP_FULL ||
1030                     type == EFX_SIENA_FILTER_RX_UDP_WILD);
1031                 EFX_POPULATE_OWORD_7(*filter,
1032                     FRF_BZ_RSS_EN,
1033                     (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
1034                     FRF_BZ_SCATTER_EN,
1035                     (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
1036                     FRF_AZ_TCP_UDP, is_udp,
1037                     FRF_AZ_RXQ_ID, spec->sfs_dmaq_id,
1038                     EFX_DWORD_2, spec->sfs_dword[2],
1039                     EFX_DWORD_1, spec->sfs_dword[1],
1040                     EFX_DWORD_0, spec->sfs_dword[0]);
1041                 dword3 = is_udp;
1042                 break;
1043         }
1044
1045         case EFX_SIENA_FILTER_TBL_RX_MAC: {
1046                 boolean_t is_wild = (type == EFX_SIENA_FILTER_RX_MAC_WILD);
1047                 EFX_POPULATE_OWORD_7(*filter,
1048                     FRF_CZ_RMFT_RSS_EN,
1049                     (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
1050                     FRF_CZ_RMFT_SCATTER_EN,
1051                     (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
1052                     FRF_CZ_RMFT_RXQ_ID, spec->sfs_dmaq_id,
1053                     FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
1054                     FRF_CZ_RMFT_DEST_MAC_DW1, spec->sfs_dword[2],
1055                     FRF_CZ_RMFT_DEST_MAC_DW0, spec->sfs_dword[1],
1056                     FRF_CZ_RMFT_VLAN_ID, spec->sfs_dword[0]);
1057                 dword3 = is_wild;
1058                 break;
1059         }
1060
1061         case EFX_SIENA_FILTER_TBL_TX_IP: {
1062                 boolean_t is_udp = (type == EFX_SIENA_FILTER_TX_UDP_FULL ||
1063                     type == EFX_SIENA_FILTER_TX_UDP_WILD);
1064                 EFX_POPULATE_OWORD_5(*filter,
1065                     FRF_CZ_TIFT_TCP_UDP, is_udp,
1066                     FRF_CZ_TIFT_TXQ_ID, spec->sfs_dmaq_id,
1067                     EFX_DWORD_2, spec->sfs_dword[2],
1068                     EFX_DWORD_1, spec->sfs_dword[1],
1069                     EFX_DWORD_0, spec->sfs_dword[0]);
1070                 dword3 = is_udp | spec->sfs_dmaq_id << 1;
1071                 break;
1072         }
1073
1074         case EFX_SIENA_FILTER_TBL_TX_MAC: {
1075                 boolean_t is_wild = (type == EFX_SIENA_FILTER_TX_MAC_WILD);
1076                 EFX_POPULATE_OWORD_5(*filter,
1077                     FRF_CZ_TMFT_TXQ_ID, spec->sfs_dmaq_id,
1078                     FRF_CZ_TMFT_WILDCARD_MATCH, is_wild,
1079                     FRF_CZ_TMFT_SRC_MAC_DW1, spec->sfs_dword[2],
1080                     FRF_CZ_TMFT_SRC_MAC_DW0, spec->sfs_dword[1],
1081                     FRF_CZ_TMFT_VLAN_ID, spec->sfs_dword[0]);
1082                 dword3 = is_wild | spec->sfs_dmaq_id << 1;
1083                 break;
1084         }
1085
1086         default:
1087                 EFSYS_ASSERT(B_FALSE);
1088                 EFX_ZERO_OWORD(*filter);
1089                 return (0);
1090         }
1091
1092         key =
1093             spec->sfs_dword[0] ^
1094             spec->sfs_dword[1] ^
1095             spec->sfs_dword[2] ^
1096             dword3;
1097
1098         return (key);
1099 }
1100
1101 static  __checkReturn           efx_rc_t
1102 siena_filter_push_entry(
1103         __inout                 efx_nic_t *enp,
1104         __in                    siena_filter_type_t type,
1105         __in                    int index,
1106         __in                    efx_oword_t *eop)
1107 {
1108         efx_rc_t rc;
1109
1110         switch (type) {
1111         case EFX_SIENA_FILTER_RX_TCP_FULL:
1112         case EFX_SIENA_FILTER_RX_TCP_WILD:
1113         case EFX_SIENA_FILTER_RX_UDP_FULL:
1114         case EFX_SIENA_FILTER_RX_UDP_WILD:
1115                 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_FILTER_TBL0, index,
1116                     eop, B_TRUE);
1117                 break;
1118
1119         case EFX_SIENA_FILTER_RX_MAC_FULL:
1120         case EFX_SIENA_FILTER_RX_MAC_WILD:
1121                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_RX_MAC_FILTER_TBL0, index,
1122                     eop, B_TRUE);
1123                 break;
1124
1125         case EFX_SIENA_FILTER_TX_TCP_FULL:
1126         case EFX_SIENA_FILTER_TX_TCP_WILD:
1127         case EFX_SIENA_FILTER_TX_UDP_FULL:
1128         case EFX_SIENA_FILTER_TX_UDP_WILD:
1129                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_FILTER_TBL0, index,
1130                     eop, B_TRUE);
1131                 break;
1132
1133         case EFX_SIENA_FILTER_TX_MAC_FULL:
1134         case EFX_SIENA_FILTER_TX_MAC_WILD:
1135                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_MAC_FILTER_TBL0, index,
1136                     eop, B_TRUE);
1137                 break;
1138
1139         default:
1140                 EFSYS_ASSERT(B_FALSE);
1141                 rc = ENOTSUP;
1142                 goto fail1;
1143         }
1144         return (0);
1145
1146 fail1:
1147         return (rc);
1148 }
1149
1150
1151 static  __checkReturn   boolean_t
1152 siena_filter_equal(
1153         __in            const siena_filter_spec_t *left,
1154         __in            const siena_filter_spec_t *right)
1155 {
1156         siena_filter_tbl_id_t tbl_id;
1157
1158         tbl_id = siena_filter_tbl_id(left->sfs_type);
1159
1160
1161         if (left->sfs_type != right->sfs_type)
1162                 return (B_FALSE);
1163
1164         if (memcmp(left->sfs_dword, right->sfs_dword,
1165                 sizeof (left->sfs_dword)))
1166                 return (B_FALSE);
1167
1168         if ((tbl_id == EFX_SIENA_FILTER_TBL_TX_IP ||
1169                 tbl_id == EFX_SIENA_FILTER_TBL_TX_MAC) &&
1170             left->sfs_dmaq_id != right->sfs_dmaq_id)
1171                 return (B_FALSE);
1172
1173         return (B_TRUE);
1174 }
1175
1176 static  __checkReturn   efx_rc_t
1177 siena_filter_search(
1178         __in            siena_filter_tbl_t *sftp,
1179         __in            siena_filter_spec_t *spec,
1180         __in            uint32_t key,
1181         __in            boolean_t for_insert,
1182         __out           int *filter_index,
1183         __out           unsigned int *depth_required)
1184 {
1185         unsigned int hash, incr, filter_idx, depth;
1186
1187         hash = siena_filter_tbl_hash(key);
1188         incr = siena_filter_tbl_increment(key);
1189
1190         filter_idx = hash & (sftp->sft_size - 1);
1191         depth = 1;
1192
1193         for (;;) {
1194                 /*
1195                  * Return success if entry is used and matches this spec
1196                  * or entry is unused and we are trying to insert.
1197                  */
1198                 if (siena_filter_test_used(sftp, filter_idx) ?
1199                     siena_filter_equal(spec,
1200                     &sftp->sft_spec[filter_idx]) :
1201                     for_insert) {
1202                         *filter_index = filter_idx;
1203                         *depth_required = depth;
1204                         return (0);
1205                 }
1206
1207                 /* Return failure if we reached the maximum search depth */
1208                 if (depth == FILTER_CTL_SRCH_MAX)
1209                         return (for_insert ? EBUSY : ENOENT);
1210
1211                 filter_idx = (filter_idx + incr) & (sftp->sft_size - 1);
1212                 ++depth;
1213         }
1214 }
1215
1216 static                  void
1217 siena_filter_clear_entry(
1218         __in            efx_nic_t *enp,
1219         __in            siena_filter_tbl_t *sftp,
1220         __in            int index)
1221 {
1222         efx_oword_t filter;
1223
1224         if (siena_filter_test_used(sftp, index)) {
1225                 siena_filter_clear_used(sftp, index);
1226
1227                 EFX_ZERO_OWORD(filter);
1228                 siena_filter_push_entry(enp,
1229                     sftp->sft_spec[index].sfs_type,
1230                     index, &filter);
1231
1232                 memset(&sftp->sft_spec[index],
1233                     0, sizeof (sftp->sft_spec[0]));
1234         }
1235 }
1236
1237                         void
1238 siena_filter_tbl_clear(
1239         __in            efx_nic_t *enp,
1240         __in            siena_filter_tbl_id_t tbl_id)
1241 {
1242         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1243         siena_filter_tbl_t *sftp = &sfp->sf_tbl[tbl_id];
1244         int index;
1245         efsys_lock_state_t state;
1246
1247         EFSYS_LOCK(enp->en_eslp, state);
1248
1249         for (index = 0; index < sftp->sft_size; ++index) {
1250                 siena_filter_clear_entry(enp, sftp, index);
1251         }
1252
1253         if (sftp->sft_used == 0)
1254                 siena_filter_reset_search_depth(sfp, tbl_id);
1255
1256         EFSYS_UNLOCK(enp->en_eslp, state);
1257 }
1258
1259 static  __checkReturn   efx_rc_t
1260 siena_filter_init(
1261         __in            efx_nic_t *enp)
1262 {
1263         siena_filter_t *sfp;
1264         siena_filter_tbl_t *sftp;
1265         int tbl_id;
1266         efx_rc_t rc;
1267
1268         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (siena_filter_t), sfp);
1269
1270         if (!sfp) {
1271                 rc = ENOMEM;
1272                 goto fail1;
1273         }
1274
1275         enp->en_filter.ef_siena_filter = sfp;
1276
1277         switch (enp->en_family) {
1278         case EFX_FAMILY_SIENA:
1279                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_IP];
1280                 sftp->sft_size = FR_AZ_RX_FILTER_TBL0_ROWS;
1281
1282                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_MAC];
1283                 sftp->sft_size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
1284
1285                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_IP];
1286                 sftp->sft_size = FR_CZ_TX_FILTER_TBL0_ROWS;
1287
1288                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_MAC];
1289                 sftp->sft_size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS;
1290                 break;
1291
1292         default:
1293                 rc = ENOTSUP;
1294                 goto fail2;
1295         }
1296
1297         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1298                 unsigned int bitmap_size;
1299
1300                 sftp = &sfp->sf_tbl[tbl_id];
1301                 if (sftp->sft_size == 0)
1302                         continue;
1303
1304                 EFX_STATIC_ASSERT(sizeof (sftp->sft_bitmap[0]) ==
1305                     sizeof (uint32_t));
1306                 bitmap_size =
1307                     (sftp->sft_size + (sizeof (uint32_t) * 8) - 1) / 8;
1308
1309                 EFSYS_KMEM_ALLOC(enp->en_esip, bitmap_size, sftp->sft_bitmap);
1310                 if (!sftp->sft_bitmap) {
1311                         rc = ENOMEM;
1312                         goto fail3;
1313                 }
1314
1315                 EFSYS_KMEM_ALLOC(enp->en_esip,
1316                     sftp->sft_size * sizeof (*sftp->sft_spec),
1317                     sftp->sft_spec);
1318                 if (!sftp->sft_spec) {
1319                         rc = ENOMEM;
1320                         goto fail4;
1321                 }
1322                 memset(sftp->sft_spec, 0,
1323                     sftp->sft_size * sizeof (*sftp->sft_spec));
1324         }
1325
1326         return (0);
1327
1328 fail4:
1329         EFSYS_PROBE(fail4);
1330
1331 fail3:
1332         EFSYS_PROBE(fail3);
1333
1334 fail2:
1335         EFSYS_PROBE(fail2);
1336         siena_filter_fini(enp);
1337
1338 fail1:
1339         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1340         return (rc);
1341 }
1342
1343 static                  void
1344 siena_filter_fini(
1345         __in            efx_nic_t *enp)
1346 {
1347         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1348         siena_filter_tbl_id_t tbl_id;
1349
1350         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1351         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
1352
1353         if (sfp == NULL)
1354                 return;
1355
1356         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1357                 siena_filter_tbl_t *sftp = &sfp->sf_tbl[tbl_id];
1358                 unsigned int bitmap_size;
1359
1360                 EFX_STATIC_ASSERT(sizeof (sftp->sft_bitmap[0]) ==
1361                     sizeof (uint32_t));
1362                 bitmap_size =
1363                     (sftp->sft_size + (sizeof (uint32_t) * 8) - 1) / 8;
1364
1365                 if (sftp->sft_bitmap != NULL) {
1366                         EFSYS_KMEM_FREE(enp->en_esip, bitmap_size,
1367                             sftp->sft_bitmap);
1368                         sftp->sft_bitmap = NULL;
1369                 }
1370
1371                 if (sftp->sft_spec != NULL) {
1372                         EFSYS_KMEM_FREE(enp->en_esip, sftp->sft_size *
1373                             sizeof (*sftp->sft_spec), sftp->sft_spec);
1374                         sftp->sft_spec = NULL;
1375                 }
1376         }
1377
1378         EFSYS_KMEM_FREE(enp->en_esip, sizeof (siena_filter_t),
1379             enp->en_filter.ef_siena_filter);
1380 }
1381
1382 /* Restore filter state after a reset */
1383 static  __checkReturn   efx_rc_t
1384 siena_filter_restore(
1385         __in            efx_nic_t *enp)
1386 {
1387         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1388         siena_filter_tbl_id_t tbl_id;
1389         siena_filter_tbl_t *sftp;
1390         siena_filter_spec_t *spec;
1391         efx_oword_t filter;
1392         int filter_idx;
1393         efsys_lock_state_t state;
1394         uint32_t key;
1395         efx_rc_t rc;
1396
1397         EFSYS_LOCK(enp->en_eslp, state);
1398
1399         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1400                 sftp = &sfp->sf_tbl[tbl_id];
1401                 for (filter_idx = 0;
1402                         filter_idx < sftp->sft_size;
1403                         filter_idx++) {
1404                         if (!siena_filter_test_used(sftp, filter_idx))
1405                                 continue;
1406
1407                         spec = &sftp->sft_spec[filter_idx];
1408                         if ((key = siena_filter_build(&filter, spec)) == 0) {
1409                                 rc = EINVAL;
1410                                 goto fail1;
1411                         }
1412                         if ((rc = siena_filter_push_entry(enp,
1413                                     spec->sfs_type, filter_idx, &filter)) != 0)
1414                                 goto fail2;
1415                 }
1416         }
1417
1418         siena_filter_push_rx_limits(enp);
1419         siena_filter_push_tx_limits(enp);
1420
1421         EFSYS_UNLOCK(enp->en_eslp, state);
1422
1423         return (0);
1424
1425 fail2:
1426         EFSYS_PROBE(fail2);
1427
1428 fail1:
1429         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1430
1431         EFSYS_UNLOCK(enp->en_eslp, state);
1432
1433         return (rc);
1434 }
1435
1436 static   __checkReturn  efx_rc_t
1437 siena_filter_add(
1438         __in            efx_nic_t *enp,
1439         __inout         efx_filter_spec_t *spec,
1440         __in            boolean_t may_replace)
1441 {
1442         efx_rc_t rc;
1443         siena_filter_spec_t sf_spec;
1444         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1445         siena_filter_tbl_id_t tbl_id;
1446         siena_filter_tbl_t *sftp;
1447         siena_filter_spec_t *saved_sf_spec;
1448         efx_oword_t filter;
1449         int filter_idx;
1450         unsigned int depth;
1451         efsys_lock_state_t state;
1452         uint32_t key;
1453
1454
1455         EFSYS_ASSERT3P(spec, !=, NULL);
1456
1457         if ((rc = siena_filter_spec_from_gen_spec(&sf_spec, spec)) != 0)
1458                 goto fail1;
1459
1460         tbl_id = siena_filter_tbl_id(sf_spec.sfs_type);
1461         sftp = &sfp->sf_tbl[tbl_id];
1462
1463         if (sftp->sft_size == 0) {
1464                 rc = EINVAL;
1465                 goto fail2;
1466         }
1467
1468         key = siena_filter_build(&filter, &sf_spec);
1469
1470         EFSYS_LOCK(enp->en_eslp, state);
1471
1472         rc = siena_filter_search(sftp, &sf_spec, key, B_TRUE,
1473             &filter_idx, &depth);
1474         if (rc != 0)
1475                 goto fail3;
1476
1477         EFSYS_ASSERT3U(filter_idx, <, sftp->sft_size);
1478         saved_sf_spec = &sftp->sft_spec[filter_idx];
1479
1480         if (siena_filter_test_used(sftp, filter_idx)) {
1481                 if (may_replace == B_FALSE) {
1482                         rc = EEXIST;
1483                         goto fail4;
1484                 }
1485         }
1486         siena_filter_set_used(sftp, filter_idx);
1487         *saved_sf_spec = sf_spec;
1488
1489         if (sfp->sf_depth[sf_spec.sfs_type] < depth) {
1490                 sfp->sf_depth[sf_spec.sfs_type] = depth;
1491                 if (tbl_id == EFX_SIENA_FILTER_TBL_TX_IP ||
1492                     tbl_id == EFX_SIENA_FILTER_TBL_TX_MAC)
1493                         siena_filter_push_tx_limits(enp);
1494                 else
1495                         siena_filter_push_rx_limits(enp);
1496         }
1497
1498         siena_filter_push_entry(enp, sf_spec.sfs_type,
1499             filter_idx, &filter);
1500
1501         EFSYS_UNLOCK(enp->en_eslp, state);
1502         return (0);
1503
1504 fail4:
1505         EFSYS_PROBE(fail4);
1506
1507 fail3:
1508         EFSYS_UNLOCK(enp->en_eslp, state);
1509         EFSYS_PROBE(fail3);
1510
1511 fail2:
1512         EFSYS_PROBE(fail2);
1513
1514 fail1:
1515         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1516         return (rc);
1517 }
1518
1519 static   __checkReturn  efx_rc_t
1520 siena_filter_delete(
1521         __in            efx_nic_t *enp,
1522         __inout         efx_filter_spec_t *spec)
1523 {
1524         efx_rc_t rc;
1525         siena_filter_spec_t sf_spec;
1526         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1527         siena_filter_tbl_id_t tbl_id;
1528         siena_filter_tbl_t *sftp;
1529         efx_oword_t filter;
1530         int filter_idx;
1531         unsigned int depth;
1532         efsys_lock_state_t state;
1533         uint32_t key;
1534
1535         EFSYS_ASSERT3P(spec, !=, NULL);
1536
1537         if ((rc = siena_filter_spec_from_gen_spec(&sf_spec, spec)) != 0)
1538                 goto fail1;
1539
1540         tbl_id = siena_filter_tbl_id(sf_spec.sfs_type);
1541         sftp = &sfp->sf_tbl[tbl_id];
1542
1543         key = siena_filter_build(&filter, &sf_spec);
1544
1545         EFSYS_LOCK(enp->en_eslp, state);
1546
1547         rc = siena_filter_search(sftp, &sf_spec, key, B_FALSE,
1548             &filter_idx, &depth);
1549         if (rc != 0)
1550                 goto fail2;
1551
1552         siena_filter_clear_entry(enp, sftp, filter_idx);
1553         if (sftp->sft_used == 0)
1554                 siena_filter_reset_search_depth(sfp, tbl_id);
1555
1556         EFSYS_UNLOCK(enp->en_eslp, state);
1557         return (0);
1558
1559 fail2:
1560         EFSYS_UNLOCK(enp->en_eslp, state);
1561         EFSYS_PROBE(fail2);
1562
1563 fail1:
1564         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1565         return (rc);
1566 }
1567
1568 #define SIENA_MAX_SUPPORTED_MATCHES 4
1569
1570 static  __checkReturn   efx_rc_t
1571 siena_filter_supported_filters(
1572         __in                            efx_nic_t *enp,
1573         __out_ecount(buffer_length)     uint32_t *buffer,
1574         __in                            size_t buffer_length,
1575         __out                           size_t *list_lengthp)
1576 {
1577         uint32_t index = 0;
1578         uint32_t rx_matches[SIENA_MAX_SUPPORTED_MATCHES];
1579         size_t list_length;
1580         efx_rc_t rc;
1581
1582         rx_matches[index++] =
1583             EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
1584             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
1585             EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
1586
1587         rx_matches[index++] =
1588             EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
1589             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT;
1590
1591         if (enp->en_features & EFX_FEATURE_MAC_HEADER_FILTERS) {
1592                 rx_matches[index++] =
1593                     EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC;
1594
1595                 rx_matches[index++] = EFX_FILTER_MATCH_LOC_MAC;
1596         }
1597
1598         EFSYS_ASSERT3U(index, <=, SIENA_MAX_SUPPORTED_MATCHES);
1599         list_length = index;
1600
1601         *list_lengthp = list_length;
1602
1603         if (buffer_length < list_length) {
1604                 rc = ENOSPC;
1605                 goto fail1;
1606         }
1607
1608         memcpy(buffer, rx_matches, list_length * sizeof (rx_matches[0]));
1609
1610         return (0);
1611
1612 fail1:
1613         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1614
1615         return (rc);
1616 }
1617
1618 #undef MAX_SUPPORTED
1619
1620 #endif /* EFSYS_OPT_SIENA */
1621
1622 #endif /* EFSYS_OPT_FILTER */