drivers/net/sfc/base/efx_rx.c

   1 /*
   2  * Copyright (c) 2007-2016 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright notice,
  11  *    this list of conditions and the following disclaimer in the documentation
  12  *    and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * The views and conclusions contained in the software and documentation are
  27  * those of the authors and should not be interpreted as representing official
  28  * policies, either expressed or implied, of the FreeBSD Project.
  29  */
  30
  31 #include "efx.h"
  32 #include "efx_impl.h"
  33
  34
  35 #if EFSYS_OPT_SIENA
  36
  37 static  __checkReturn   efx_rc_t
  38 siena_rx_init(
  39         __in            efx_nic_t *enp);
  40
  41 static                  void
  42 siena_rx_fini(
  43         __in            efx_nic_t *enp);
  44
  45 #if EFSYS_OPT_RX_SCATTER
  46 static  __checkReturn   efx_rc_t
  47 siena_rx_scatter_enable(
  48         __in            efx_nic_t *enp,
  49         __in            unsigned int buf_size);
  50 #endif /* EFSYS_OPT_RX_SCATTER */
  51
  52 #if EFSYS_OPT_RX_SCALE
  53 static  __checkReturn   efx_rc_t
  54 siena_rx_scale_mode_set(
  55         __in            efx_nic_t *enp,
  56         __in            efx_rx_hash_alg_t alg,
  57         __in            efx_rx_hash_type_t type,
  58         __in            boolean_t insert);
  59
  60 static  __checkReturn   efx_rc_t
  61 siena_rx_scale_key_set(
  62         __in            efx_nic_t *enp,
  63         __in_ecount(n)  uint8_t *key,
  64         __in            size_t n);
  65
  66 static  __checkReturn   efx_rc_t
  67 siena_rx_scale_tbl_set(
  68         __in            efx_nic_t *enp,
  69         __in_ecount(n)  unsigned int *table,
  70         __in            size_t n);
  71
  72 static  __checkReturn   uint32_t
  73 siena_rx_prefix_hash(
  74         __in            efx_nic_t *enp,
  75         __in            efx_rx_hash_alg_t func,
  76         __in            uint8_t *buffer);
  77
  78 #endif /* EFSYS_OPT_RX_SCALE */
  79
  80 static  __checkReturn   efx_rc_t
  81 siena_rx_prefix_pktlen(
  82         __in            efx_nic_t *enp,
  83         __in            uint8_t *buffer,
  84         __out           uint16_t *lengthp);
  85
  86 static                  void
  87 siena_rx_qpost(
  88         __in            efx_rxq_t *erp,
  89         __in_ecount(n)  efsys_dma_addr_t *addrp,
  90         __in            size_t size,
  91         __in            unsigned int n,
  92         __in            unsigned int completed,
  93         __in            unsigned int added);
  94
  95 static                  void
  96 siena_rx_qpush(
  97         __in            efx_rxq_t *erp,
  98         __in            unsigned int added,
  99         __inout         unsigned int *pushedp);
 100
 101 #if EFSYS_OPT_RX_PACKED_STREAM
 102 static          void
 103 siena_rx_qps_update_credits(
 104         __in            efx_rxq_t *erp);
 105
 106 static  __checkReturn   uint8_t *
 107 siena_rx_qps_packet_info(
 108         __in            efx_rxq_t *erp,
 109         __in            uint8_t *buffer,
 110         __in            uint32_t buffer_length,
 111         __in            uint32_t current_offset,
 112         __out           uint16_t *lengthp,
 113         __out           uint32_t *next_offsetp,
 114         __out           uint32_t *timestamp);
 115 #endif
 116
 117 static  __checkReturn   efx_rc_t
 118 siena_rx_qflush(
 119         __in            efx_rxq_t *erp);
 120
 121 static                  void
 122 siena_rx_qenable(
 123         __in            efx_rxq_t *erp);
 124
 125 static  __checkReturn   efx_rc_t
 126 siena_rx_qcreate(
 127         __in            efx_nic_t *enp,
 128         __in            unsigned int index,
 129         __in            unsigned int label,
 130         __in            efx_rxq_type_t type,
 131         __in            efsys_mem_t *esmp,
 132         __in            size_t n,
 133         __in            uint32_t id,
 134         __in            efx_evq_t *eep,
 135         __in            efx_rxq_t *erp);
 136
 137 static                  void
 138 siena_rx_qdestroy(
 139         __in            efx_rxq_t *erp);
 140
 141 #endif /* EFSYS_OPT_SIENA */
 142
 143
 144 #if EFSYS_OPT_SIENA
 145 static const efx_rx_ops_t __efx_rx_siena_ops = {
 146         siena_rx_init,                          /* erxo_init */
 147         siena_rx_fini,                          /* erxo_fini */
 148 #if EFSYS_OPT_RX_SCATTER
 149         siena_rx_scatter_enable,                /* erxo_scatter_enable */
 150 #endif
 151 #if EFSYS_OPT_RX_SCALE
 152         siena_rx_scale_mode_set,                /* erxo_scale_mode_set */
 153         siena_rx_scale_key_set,                 /* erxo_scale_key_set */
 154         siena_rx_scale_tbl_set,                 /* erxo_scale_tbl_set */
 155         siena_rx_prefix_hash,                   /* erxo_prefix_hash */
 156 #endif
 157         siena_rx_prefix_pktlen,                 /* erxo_prefix_pktlen */
 158         siena_rx_qpost,                         /* erxo_qpost */
 159         siena_rx_qpush,                         /* erxo_qpush */
 160 #if EFSYS_OPT_RX_PACKED_STREAM
 161         siena_rx_qps_update_credits,            /* erxo_qps_update_credits */
 162         siena_rx_qps_packet_info,               /* erxo_qps_packet_info */
 163 #endif
 164         siena_rx_qflush,                        /* erxo_qflush */
 165         siena_rx_qenable,                       /* erxo_qenable */
 166         siena_rx_qcreate,                       /* erxo_qcreate */
 167         siena_rx_qdestroy,                      /* erxo_qdestroy */
 168 };
 169 #endif  /* EFSYS_OPT_SIENA */
 170
 171 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
 172 static const efx_rx_ops_t __efx_rx_ef10_ops = {
 173         ef10_rx_init,                           /* erxo_init */
 174         ef10_rx_fini,                           /* erxo_fini */
 175 #if EFSYS_OPT_RX_SCATTER
 176         ef10_rx_scatter_enable,                 /* erxo_scatter_enable */
 177 #endif
 178 #if EFSYS_OPT_RX_SCALE
 179         ef10_rx_scale_mode_set,                 /* erxo_scale_mode_set */
 180         ef10_rx_scale_key_set,                  /* erxo_scale_key_set */
 181         ef10_rx_scale_tbl_set,                  /* erxo_scale_tbl_set */
 182         ef10_rx_prefix_hash,                    /* erxo_prefix_hash */
 183 #endif
 184         ef10_rx_prefix_pktlen,                  /* erxo_prefix_pktlen */
 185         ef10_rx_qpost,                          /* erxo_qpost */
 186         ef10_rx_qpush,                          /* erxo_qpush */
 187 #if EFSYS_OPT_RX_PACKED_STREAM
 188         ef10_rx_qps_update_credits,             /* erxo_qps_update_credits */
 189         ef10_rx_qps_packet_info,                /* erxo_qps_packet_info */
 190 #endif
 191         ef10_rx_qflush,                         /* erxo_qflush */
 192         ef10_rx_qenable,                        /* erxo_qenable */
 193         ef10_rx_qcreate,                        /* erxo_qcreate */
 194         ef10_rx_qdestroy,                       /* erxo_qdestroy */
 195 };
 196 #endif  /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */
 197
 198
 199         __checkReturn   efx_rc_t
 200 efx_rx_init(
 201         __inout         efx_nic_t *enp)
 202 {
 203         const efx_rx_ops_t *erxop;
 204         efx_rc_t rc;
 205
 206         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 207         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
 208
 209         if (!(enp->en_mod_flags & EFX_MOD_EV)) {
 210                 rc = EINVAL;
 211                 goto fail1;
 212         }
 213
 214         if (enp->en_mod_flags & EFX_MOD_RX) {
 215                 rc = EINVAL;
 216                 goto fail2;
 217         }
 218
 219         switch (enp->en_family) {
 220 #if EFSYS_OPT_SIENA
 221         case EFX_FAMILY_SIENA:
 222                 erxop = &__efx_rx_siena_ops;
 223                 break;
 224 #endif /* EFSYS_OPT_SIENA */
 225
 226 #if EFSYS_OPT_HUNTINGTON
 227         case EFX_FAMILY_HUNTINGTON:
 228                 erxop = &__efx_rx_ef10_ops;
 229                 break;
 230 #endif /* EFSYS_OPT_HUNTINGTON */
 231
 232 #if EFSYS_OPT_MEDFORD
 233         case EFX_FAMILY_MEDFORD:
 234                 erxop = &__efx_rx_ef10_ops;
 235                 break;
 236 #endif /* EFSYS_OPT_MEDFORD */
 237
 238         default:
 239                 EFSYS_ASSERT(0);
 240                 rc = ENOTSUP;
 241                 goto fail3;
 242         }
 243
 244         if ((rc = erxop->erxo_init(enp)) != 0)
 245                 goto fail4;
 246
 247         enp->en_erxop = erxop;
 248         enp->en_mod_flags |= EFX_MOD_RX;
 249         return (0);
 250
 251 fail4:
 252         EFSYS_PROBE(fail4);
 253 fail3:
 254         EFSYS_PROBE(fail3);
 255 fail2:
 256         EFSYS_PROBE(fail2);
 257 fail1:
 258         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 259
 260         enp->en_erxop = NULL;
 261         enp->en_mod_flags &= ~EFX_MOD_RX;
 262         return (rc);
 263 }
 264
 265                         void
 266 efx_rx_fini(
 267         __in            efx_nic_t *enp)
 268 {
 269         const efx_rx_ops_t *erxop = enp->en_erxop;
 270
 271         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 272         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
 273         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 274         EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0);
 275
 276         erxop->erxo_fini(enp);
 277
 278         enp->en_erxop = NULL;
 279         enp->en_mod_flags &= ~EFX_MOD_RX;
 280 }
 281
 282 #if EFSYS_OPT_RX_SCATTER
 283         __checkReturn   efx_rc_t
 284 efx_rx_scatter_enable(
 285         __in            efx_nic_t *enp,
 286         __in            unsigned int buf_size)
 287 {
 288         const efx_rx_ops_t *erxop = enp->en_erxop;
 289         efx_rc_t rc;
 290
 291         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 292         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 293
 294         if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0)
 295                 goto fail1;
 296
 297         return (0);
 298
 299 fail1:
 300         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 301         return (rc);
 302 }
 303 #endif  /* EFSYS_OPT_RX_SCATTER */
 304
 305 #if EFSYS_OPT_RX_SCALE
 306         __checkReturn   efx_rc_t
 307 efx_rx_hash_support_get(
 308         __in            efx_nic_t *enp,
 309         __out           efx_rx_hash_support_t *supportp)
 310 {
 311         efx_rc_t rc;
 312
 313         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 314         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 315
 316         if (supportp == NULL) {
 317                 rc = EINVAL;
 318                 goto fail1;
 319         }
 320
 321         /* Report if resources are available to insert RX hash value */
 322         *supportp = enp->en_hash_support;
 323
 324         return (0);
 325
 326 fail1:
 327         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 328
 329         return (rc);
 330 }
 331
 332         __checkReturn   efx_rc_t
 333 efx_rx_scale_support_get(
 334         __in            efx_nic_t *enp,
 335         __out           efx_rx_scale_support_t *supportp)
 336 {
 337         efx_rc_t rc;
 338
 339         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 340         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 341
 342         if (supportp == NULL) {
 343                 rc = EINVAL;
 344                 goto fail1;
 345         }
 346
 347         /* Report if resources are available to support RSS */
 348         *supportp = enp->en_rss_support;
 349
 350         return (0);
 351
 352 fail1:
 353         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 354
 355         return (rc);
 356 }
 357
 358         __checkReturn   efx_rc_t
 359 efx_rx_scale_mode_set(
 360         __in            efx_nic_t *enp,
 361         __in            efx_rx_hash_alg_t alg,
 362         __in            efx_rx_hash_type_t type,
 363         __in            boolean_t insert)
 364 {
 365         const efx_rx_ops_t *erxop = enp->en_erxop;
 366         efx_rc_t rc;
 367
 368         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 369         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 370
 371         if (erxop->erxo_scale_mode_set != NULL) {
 372                 if ((rc = erxop->erxo_scale_mode_set(enp, alg,
 373                             type, insert)) != 0)
 374                         goto fail1;
 375         }
 376
 377         return (0);
 378
 379 fail1:
 380         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 381         return (rc);
 382 }
 383 #endif  /* EFSYS_OPT_RX_SCALE */
 384
 385 #if EFSYS_OPT_RX_SCALE
 386         __checkReturn   efx_rc_t
 387 efx_rx_scale_key_set(
 388         __in            efx_nic_t *enp,
 389         __in_ecount(n)  uint8_t *key,
 390         __in            size_t n)
 391 {
 392         const efx_rx_ops_t *erxop = enp->en_erxop;
 393         efx_rc_t rc;
 394
 395         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 396         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 397
 398         if ((rc = erxop->erxo_scale_key_set(enp, key, n)) != 0)
 399                 goto fail1;
 400
 401         return (0);
 402
 403 fail1:
 404         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 405
 406         return (rc);
 407 }
 408 #endif  /* EFSYS_OPT_RX_SCALE */
 409
 410 #if EFSYS_OPT_RX_SCALE
 411         __checkReturn   efx_rc_t
 412 efx_rx_scale_tbl_set(
 413         __in            efx_nic_t *enp,
 414         __in_ecount(n)  unsigned int *table,
 415         __in            size_t n)
 416 {
 417         const efx_rx_ops_t *erxop = enp->en_erxop;
 418         efx_rc_t rc;
 419
 420         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 421         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 422
 423         if ((rc = erxop->erxo_scale_tbl_set(enp, table, n)) != 0)
 424                 goto fail1;
 425
 426         return (0);
 427
 428 fail1:
 429         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 430
 431         return (rc);
 432 }
 433 #endif  /* EFSYS_OPT_RX_SCALE */
 434
 435                         void
 436 efx_rx_qpost(
 437         __in            efx_rxq_t *erp,
 438         __in_ecount(n)  efsys_dma_addr_t *addrp,
 439         __in            size_t size,
 440         __in            unsigned int n,
 441         __in            unsigned int completed,
 442         __in            unsigned int added)
 443 {
 444         efx_nic_t *enp = erp->er_enp;
 445         const efx_rx_ops_t *erxop = enp->en_erxop;
 446
 447         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 448
 449         erxop->erxo_qpost(erp, addrp, size, n, completed, added);
 450 }
 451
 452 #if EFSYS_OPT_RX_PACKED_STREAM
 453
 454                         void
 455 efx_rx_qps_update_credits(
 456         __in            efx_rxq_t *erp)
 457 {
 458         efx_nic_t *enp = erp->er_enp;
 459         const efx_rx_ops_t *erxop = enp->en_erxop;
 460
 461         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 462
 463         erxop->erxo_qps_update_credits(erp);
 464 }
 465
 466         __checkReturn   uint8_t *
 467 efx_rx_qps_packet_info(
 468         __in            efx_rxq_t *erp,
 469         __in            uint8_t *buffer,
 470         __in            uint32_t buffer_length,
 471         __in            uint32_t current_offset,
 472         __out           uint16_t *lengthp,
 473         __out           uint32_t *next_offsetp,
 474         __out           uint32_t *timestamp)
 475 {
 476         efx_nic_t *enp = erp->er_enp;
 477         const efx_rx_ops_t *erxop = enp->en_erxop;
 478
 479         return (erxop->erxo_qps_packet_info(erp, buffer,
 480                 buffer_length, current_offset, lengthp,
 481                 next_offsetp, timestamp));
 482 }
 483
 484 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
 485
 486                         void
 487 efx_rx_qpush(
 488         __in            efx_rxq_t *erp,
 489         __in            unsigned int added,
 490         __inout         unsigned int *pushedp)
 491 {
 492         efx_nic_t *enp = erp->er_enp;
 493         const efx_rx_ops_t *erxop = enp->en_erxop;
 494
 495         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 496
 497         erxop->erxo_qpush(erp, added, pushedp);
 498 }
 499
 500         __checkReturn   efx_rc_t
 501 efx_rx_qflush(
 502         __in            efx_rxq_t *erp)
 503 {
 504         efx_nic_t *enp = erp->er_enp;
 505         const efx_rx_ops_t *erxop = enp->en_erxop;
 506         efx_rc_t rc;
 507
 508         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 509
 510         if ((rc = erxop->erxo_qflush(erp)) != 0)
 511                 goto fail1;
 512
 513         return (0);
 514
 515 fail1:
 516         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 517
 518         return (rc);
 519 }
 520
 521                         void
 522 efx_rx_qenable(
 523         __in            efx_rxq_t *erp)
 524 {
 525         efx_nic_t *enp = erp->er_enp;
 526         const efx_rx_ops_t *erxop = enp->en_erxop;
 527
 528         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 529
 530         erxop->erxo_qenable(erp);
 531 }
 532
 533         __checkReturn   efx_rc_t
 534 efx_rx_qcreate(
 535         __in            efx_nic_t *enp,
 536         __in            unsigned int index,
 537         __in            unsigned int label,
 538         __in            efx_rxq_type_t type,
 539         __in            efsys_mem_t *esmp,
 540         __in            size_t n,
 541         __in            uint32_t id,
 542         __in            efx_evq_t *eep,
 543         __deref_out     efx_rxq_t **erpp)
 544 {
 545         const efx_rx_ops_t *erxop = enp->en_erxop;
 546         efx_rxq_t *erp;
 547         efx_rc_t rc;
 548
 549         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 550         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 551
 552         /* Allocate an RXQ object */
 553         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp);
 554
 555         if (erp == NULL) {
 556                 rc = ENOMEM;
 557                 goto fail1;
 558         }
 559
 560         erp->er_magic = EFX_RXQ_MAGIC;
 561         erp->er_enp = enp;
 562         erp->er_index = index;
 563         erp->er_mask = n - 1;
 564         erp->er_esmp = esmp;
 565
 566         if ((rc = erxop->erxo_qcreate(enp, index, label, type, esmp, n, id,
 567             eep, erp)) != 0)
 568                 goto fail2;
 569
 570         enp->en_rx_qcount++;
 571         *erpp = erp;
 572
 573         return (0);
 574
 575 fail2:
 576         EFSYS_PROBE(fail2);
 577
 578         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
 579 fail1:
 580         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 581
 582         return (rc);
 583 }
 584
 585                         void
 586 efx_rx_qdestroy(
 587         __in            efx_rxq_t *erp)
 588 {
 589         efx_nic_t *enp = erp->er_enp;
 590         const efx_rx_ops_t *erxop = enp->en_erxop;
 591
 592         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 593
 594         erxop->erxo_qdestroy(erp);
 595 }
 596
 597         __checkReturn   efx_rc_t
 598 efx_pseudo_hdr_pkt_length_get(
 599         __in            efx_rxq_t *erp,
 600         __in            uint8_t *buffer,
 601         __out           uint16_t *lengthp)
 602 {
 603         efx_nic_t *enp = erp->er_enp;
 604         const efx_rx_ops_t *erxop = enp->en_erxop;
 605
 606         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 607
 608         return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp));
 609 }
 610
 611 #if EFSYS_OPT_RX_SCALE
 612         __checkReturn   uint32_t
 613 efx_pseudo_hdr_hash_get(
 614         __in            efx_rxq_t *erp,
 615         __in            efx_rx_hash_alg_t func,
 616         __in            uint8_t *buffer)
 617 {
 618         efx_nic_t *enp = erp->er_enp;
 619         const efx_rx_ops_t *erxop = enp->en_erxop;
 620
 621         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 622
 623         EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE);
 624         return (erxop->erxo_prefix_hash(enp, func, buffer));
 625 }
 626 #endif  /* EFSYS_OPT_RX_SCALE */
 627
 628 #if EFSYS_OPT_SIENA
 629
 630 static  __checkReturn   efx_rc_t
 631 siena_rx_init(
 632         __in            efx_nic_t *enp)
 633 {
 634         efx_oword_t oword;
 635         unsigned int index;
 636
 637         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 638
 639         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0);
 640         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);
 641         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);
 642         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);
 643         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0);
 644         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32);
 645         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 646
 647         /* Zero the RSS table */
 648         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS;
 649             index++) {
 650                 EFX_ZERO_OWORD(oword);
 651                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
 652                                     index, &oword, B_TRUE);
 653         }
 654
 655 #if EFSYS_OPT_RX_SCALE
 656         /* The RSS key and indirection table are writable. */
 657         enp->en_rss_support = EFX_RX_SCALE_EXCLUSIVE;
 658
 659         /* Hardware can insert RX hash with/without RSS */
 660         enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
 661 #endif  /* EFSYS_OPT_RX_SCALE */
 662
 663         return (0);
 664 }
 665
 666 #if EFSYS_OPT_RX_SCATTER
 667 static  __checkReturn   efx_rc_t
 668 siena_rx_scatter_enable(
 669         __in            efx_nic_t *enp,
 670         __in            unsigned int buf_size)
 671 {
 672         unsigned int nbuf32;
 673         efx_oword_t oword;
 674         efx_rc_t rc;
 675
 676         nbuf32 = buf_size / 32;
 677         if ((nbuf32 == 0) ||
 678             (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) ||
 679             ((buf_size % 32) != 0)) {
 680                 rc = EINVAL;
 681                 goto fail1;
 682         }
 683
 684         if (enp->en_rx_qcount > 0) {
 685                 rc = EBUSY;
 686                 goto fail2;
 687         }
 688
 689         /* Set scatter buffer size */
 690         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 691         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32);
 692         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 693
 694         /* Enable scatter for packets not matching a filter */
 695         EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 696         EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1);
 697         EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 698
 699         return (0);
 700
 701 fail2:
 702         EFSYS_PROBE(fail2);
 703 fail1:
 704         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 705
 706         return (rc);
 707 }
 708 #endif  /* EFSYS_OPT_RX_SCATTER */
 709
 710
 711 #define EFX_RX_LFSR_HASH(_enp, _insert)                                 \
 712         do {                                                            \
 713                 efx_oword_t oword;                                      \
 714                                                                         \
 715                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);        \
 716                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);      \
 717                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);       \
 718                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);       \
 719                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 720                     (_insert) ? 1 : 0);                                 \
 721                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);       \
 722                                                                         \
 723                 if ((_enp)->en_family == EFX_FAMILY_SIENA) {            \
 724                         EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3,   \
 725                             &oword);                                    \
 726                         EFX_SET_OWORD_FIELD(oword,                      \
 727                             FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0);        \
 728                         EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3,  \
 729                             &oword);                                    \
 730                 }                                                       \
 731                                                                         \
 732                 _NOTE(CONSTANTCONDITION)                                \
 733         } while (B_FALSE)
 734
 735 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp)             \
 736         do {                                                            \
 737                 efx_oword_t oword;                                      \
 738                                                                         \
 739                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);        \
 740                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1);      \
 741                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH,           \
 742                     (_ip) ? 1 : 0);                                     \
 743                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP,           \
 744                     (_tcp) ? 0 : 1);                                    \
 745                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 746                     (_insert) ? 1 : 0);                                 \
 747                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);       \
 748                                                                         \
 749                 _NOTE(CONSTANTCONDITION)                                \
 750         } while (B_FALSE)
 751
 752 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc)                 \
 753         do {                                                            \
 754                 efx_oword_t oword;                                      \
 755                                                                         \
 756                 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword);  \
 757                 EFX_SET_OWORD_FIELD(oword,                              \
 758                     FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1);                \
 759                 EFX_SET_OWORD_FIELD(oword,                              \
 760                     FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \
 761                 EFX_SET_OWORD_FIELD(oword,                              \
 762                     FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1);   \
 763                 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
 764                                                                         \
 765                 (_rc) = 0;                                              \
 766                                                                         \
 767                 _NOTE(CONSTANTCONDITION)                                \
 768         } while (B_FALSE)
 769
 770
 771 #if EFSYS_OPT_RX_SCALE
 772
 773 static  __checkReturn   efx_rc_t
 774 siena_rx_scale_mode_set(
 775         __in            efx_nic_t *enp,
 776         __in            efx_rx_hash_alg_t alg,
 777         __in            efx_rx_hash_type_t type,
 778         __in            boolean_t insert)
 779 {
 780         efx_rc_t rc;
 781
 782         switch (alg) {
 783         case EFX_RX_HASHALG_LFSR:
 784                 EFX_RX_LFSR_HASH(enp, insert);
 785                 break;
 786
 787         case EFX_RX_HASHALG_TOEPLITZ:
 788                 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert,
 789                     type & EFX_RX_HASH_IPV4,
 790                     type & EFX_RX_HASH_TCPIPV4);
 791
 792                 EFX_RX_TOEPLITZ_IPV6_HASH(enp,
 793                     type & EFX_RX_HASH_IPV6,
 794                     type & EFX_RX_HASH_TCPIPV6,
 795                     rc);
 796                 if (rc != 0)
 797                         goto fail1;
 798
 799                 break;
 800
 801         default:
 802                 rc = EINVAL;
 803                 goto fail2;
 804         }
 805
 806         return (0);
 807
 808 fail2:
 809         EFSYS_PROBE(fail2);
 810 fail1:
 811         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 812
 813         EFX_RX_LFSR_HASH(enp, B_FALSE);
 814
 815         return (rc);
 816 }
 817 #endif
 818
 819 #if EFSYS_OPT_RX_SCALE
 820 static  __checkReturn   efx_rc_t
 821 siena_rx_scale_key_set(
 822         __in            efx_nic_t *enp,
 823         __in_ecount(n)  uint8_t *key,
 824         __in            size_t n)
 825 {
 826         efx_oword_t oword;
 827         unsigned int byte;
 828         unsigned int offset;
 829         efx_rc_t rc;
 830
 831         byte = 0;
 832
 833         /* Write Toeplitz IPv4 hash key */
 834         EFX_ZERO_OWORD(oword);
 835         for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
 836             offset > 0 && byte < n;
 837             --offset)
 838                 oword.eo_u8[offset - 1] = key[byte++];
 839
 840         EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
 841
 842         byte = 0;
 843
 844         /* Verify Toeplitz IPv4 hash key */
 845         EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
 846         for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
 847             offset > 0 && byte < n;
 848             --offset) {
 849                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 850                         rc = EFAULT;
 851                         goto fail1;
 852                 }
 853         }
 854
 855         if ((enp->en_features & EFX_FEATURE_IPV6) == 0)
 856                 goto done;
 857
 858         byte = 0;
 859
 860         /* Write Toeplitz IPv6 hash key 3 */
 861         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 862         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
 863             FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
 864             offset > 0 && byte < n;
 865             --offset)
 866                 oword.eo_u8[offset - 1] = key[byte++];
 867
 868         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 869
 870         /* Write Toeplitz IPv6 hash key 2 */
 871         EFX_ZERO_OWORD(oword);
 872         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
 873             FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
 874             offset > 0 && byte < n;
 875             --offset)
 876                 oword.eo_u8[offset - 1] = key[byte++];
 877
 878         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
 879
 880         /* Write Toeplitz IPv6 hash key 1 */
 881         EFX_ZERO_OWORD(oword);
 882         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
 883             FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
 884             offset > 0 && byte < n;
 885             --offset)
 886                 oword.eo_u8[offset - 1] = key[byte++];
 887
 888         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
 889
 890         byte = 0;
 891
 892         /* Verify Toeplitz IPv6 hash key 3 */
 893         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 894         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
 895             FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
 896             offset > 0 && byte < n;
 897             --offset) {
 898                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 899                         rc = EFAULT;
 900                         goto fail2;
 901                 }
 902         }
 903
 904         /* Verify Toeplitz IPv6 hash key 2 */
 905         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
 906         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
 907             FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
 908             offset > 0 && byte < n;
 909             --offset) {
 910                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 911                         rc = EFAULT;
 912                         goto fail3;
 913                 }
 914         }
 915
 916         /* Verify Toeplitz IPv6 hash key 1 */
 917         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
 918         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
 919             FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
 920             offset > 0 && byte < n;
 921             --offset) {
 922                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 923                         rc = EFAULT;
 924                         goto fail4;
 925                 }
 926         }
 927
 928 done:
 929         return (0);
 930
 931 fail4:
 932         EFSYS_PROBE(fail4);
 933 fail3:
 934         EFSYS_PROBE(fail3);
 935 fail2:
 936         EFSYS_PROBE(fail2);
 937 fail1:
 938         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 939
 940         return (rc);
 941 }
 942 #endif
 943
 944 #if EFSYS_OPT_RX_SCALE
 945 static  __checkReturn   efx_rc_t
 946 siena_rx_scale_tbl_set(
 947         __in            efx_nic_t *enp,
 948         __in_ecount(n)  unsigned int *table,
 949         __in            size_t n)
 950 {
 951         efx_oword_t oword;
 952         int index;
 953         efx_rc_t rc;
 954
 955         EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS);
 956         EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH));
 957
 958         if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) {
 959                 rc = EINVAL;
 960                 goto fail1;
 961         }
 962
 963         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) {
 964                 uint32_t byte;
 965
 966                 /* Calculate the entry to place in the table */
 967                 byte = (n > 0) ? (uint32_t)table[index % n] : 0;
 968
 969                 EFSYS_PROBE2(table, int, index, uint32_t, byte);
 970
 971                 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte);
 972
 973                 /* Write the table */
 974                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
 975                                     index, &oword, B_TRUE);
 976         }
 977
 978         for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) {
 979                 uint32_t byte;
 980
 981                 /* Determine if we're starting a new batch */
 982                 byte = (n > 0) ? (uint32_t)table[index % n] : 0;
 983
 984                 /* Read the table */
 985                 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL,
 986                                     index, &oword, B_TRUE);
 987
 988                 /* Verify the entry */
 989                 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) {
 990                         rc = EFAULT;
 991                         goto fail2;
 992                 }
 993         }
 994
 995         return (0);
 996
 997 fail2:
 998         EFSYS_PROBE(fail2);
 999 fail1:
1000         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1001
1002         return (rc);
1003 }
1004 #endif
1005
1006 /*
1007  * Falcon/Siena pseudo-header
1008  * --------------------------
1009  *
1010  * Receive packets are prefixed by an optional 16 byte pseudo-header.
1011  * The pseudo-header is a byte array of one of the forms:
1012  *
1013  *  0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15
1014  * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT
1015  * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL
1016  *
1017  * where:
1018  *   TT.TT.TT.TT   Toeplitz hash (32-bit big-endian)
1019  *   LL.LL         LFSR hash     (16-bit big-endian)
1020  */
1021
1022 #if EFSYS_OPT_RX_SCALE
1023 static  __checkReturn   uint32_t
1024 siena_rx_prefix_hash(
1025         __in            efx_nic_t *enp,
1026         __in            efx_rx_hash_alg_t func,
1027         __in            uint8_t *buffer)
1028 {
1029         _NOTE(ARGUNUSED(enp))
1030
1031         switch (func) {
1032         case EFX_RX_HASHALG_TOEPLITZ:
1033                 return ((buffer[12] << 24) |
1034                     (buffer[13] << 16) |
1035                     (buffer[14] <<  8) |
1036                     buffer[15]);
1037
1038         case EFX_RX_HASHALG_LFSR:
1039                 return ((buffer[14] << 8) | buffer[15]);
1040
1041         default:
1042                 EFSYS_ASSERT(0);
1043                 return (0);
1044         }
1045 }
1046 #endif /* EFSYS_OPT_RX_SCALE */
1047
1048 static  __checkReturn   efx_rc_t
1049 siena_rx_prefix_pktlen(
1050         __in            efx_nic_t *enp,
1051         __in            uint8_t *buffer,
1052         __out           uint16_t *lengthp)
1053 {
1054         _NOTE(ARGUNUSED(enp, buffer, lengthp))
1055
1056         /* Not supported by Falcon/Siena hardware */
1057         EFSYS_ASSERT(0);
1058         return (ENOTSUP);
1059 }
1060
1061
1062 static                  void
1063 siena_rx_qpost(
1064         __in            efx_rxq_t *erp,
1065         __in_ecount(n)  efsys_dma_addr_t *addrp,
1066         __in            size_t size,
1067         __in            unsigned int n,
1068         __in            unsigned int completed,
1069         __in            unsigned int added)
1070 {
1071         efx_qword_t qword;
1072         unsigned int i;
1073         unsigned int offset;
1074         unsigned int id;
1075
1076         /* The client driver must not overfill the queue */
1077         EFSYS_ASSERT3U(added - completed + n, <=,
1078             EFX_RXQ_LIMIT(erp->er_mask + 1));
1079
1080         id = added & (erp->er_mask);
1081         for (i = 0; i < n; i++) {
1082                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
1083                     unsigned int, id, efsys_dma_addr_t, addrp[i],
1084                     size_t, size);
1085
1086                 EFX_POPULATE_QWORD_3(qword,
1087                     FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size),
1088                     FSF_AZ_RX_KER_BUF_ADDR_DW0,
1089                     (uint32_t)(addrp[i] & 0xffffffff),
1090                     FSF_AZ_RX_KER_BUF_ADDR_DW1,
1091                     (uint32_t)(addrp[i] >> 32));
1092
1093                 offset = id * sizeof (efx_qword_t);
1094                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
1095
1096                 id = (id + 1) & (erp->er_mask);
1097         }
1098 }
1099
1100 static                  void
1101 siena_rx_qpush(
1102         __in    efx_rxq_t *erp,
1103         __in    unsigned int added,
1104         __inout unsigned int *pushedp)
1105 {
1106         efx_nic_t *enp = erp->er_enp;
1107         unsigned int pushed = *pushedp;
1108         uint32_t wptr;
1109         efx_oword_t oword;
1110         efx_dword_t dword;
1111
1112         /* All descriptors are pushed */
1113         *pushedp = added;
1114
1115         /* Push the populated descriptors out */
1116         wptr = added & erp->er_mask;
1117
1118         EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr);
1119
1120         /* Only write the third DWORD */
1121         EFX_POPULATE_DWORD_1(dword,
1122             EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
1123
1124         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
1125         EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
1126             wptr, pushed & erp->er_mask);
1127         EFSYS_PIO_WRITE_BARRIER();
1128         EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0,
1129                             erp->er_index, &dword, B_FALSE);
1130 }
1131
1132 #if EFSYS_OPT_RX_PACKED_STREAM
1133 static          void
1134 siena_rx_qps_update_credits(
1135         __in            efx_rxq_t *erp)
1136 {
1137         /* Not supported by Siena hardware */
1138         EFSYS_ASSERT(0);
1139 }
1140
1141 static          uint8_t *
1142 siena_rx_qps_packet_info(
1143         __in            efx_rxq_t *erp,
1144         __in            uint8_t *buffer,
1145         __in            uint32_t buffer_length,
1146         __in            uint32_t current_offset,
1147         __out           uint16_t *lengthp,
1148         __out           uint32_t *next_offsetp,
1149         __out           uint32_t *timestamp)
1150 {
1151         /* Not supported by Siena hardware */
1152         EFSYS_ASSERT(0);
1153
1154         return (NULL);
1155 }
1156 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1157
1158 static  __checkReturn   efx_rc_t
1159 siena_rx_qflush(
1160         __in    efx_rxq_t *erp)
1161 {
1162         efx_nic_t *enp = erp->er_enp;
1163         efx_oword_t oword;
1164         uint32_t label;
1165
1166         label = erp->er_index;
1167
1168         /* Flush the queue */
1169         EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
1170             FRF_AZ_RX_FLUSH_DESCQ, label);
1171         EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword);
1172
1173         return (0);
1174 }
1175
1176 static          void
1177 siena_rx_qenable(
1178         __in    efx_rxq_t *erp)
1179 {
1180         efx_nic_t *enp = erp->er_enp;
1181         efx_oword_t oword;
1182
1183         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1184
1185         EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL,
1186                             erp->er_index, &oword, B_TRUE);
1187
1188         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0);
1189         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0);
1190         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1);
1191
1192         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1193                             erp->er_index, &oword, B_TRUE);
1194 }
1195
1196 static  __checkReturn   efx_rc_t
1197 siena_rx_qcreate(
1198         __in            efx_nic_t *enp,
1199         __in            unsigned int index,
1200         __in            unsigned int label,
1201         __in            efx_rxq_type_t type,
1202         __in            efsys_mem_t *esmp,
1203         __in            size_t n,
1204         __in            uint32_t id,
1205         __in            efx_evq_t *eep,
1206         __in            efx_rxq_t *erp)
1207 {
1208         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1209         efx_oword_t oword;
1210         uint32_t size;
1211         boolean_t jumbo;
1212         efx_rc_t rc;
1213
1214         _NOTE(ARGUNUSED(esmp))
1215
1216         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS ==
1217             (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH));
1218         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
1219         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
1220
1221         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
1222         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
1223
1224         if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) {
1225                 rc = EINVAL;
1226                 goto fail1;
1227         }
1228         if (index >= encp->enc_rxq_limit) {
1229                 rc = EINVAL;
1230                 goto fail2;
1231         }
1232         for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS);
1233             size++)
1234                 if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS))
1235                         break;
1236         if (id + (1 << size) >= encp->enc_buftbl_limit) {
1237                 rc = EINVAL;
1238                 goto fail3;
1239         }
1240
1241         switch (type) {
1242         case EFX_RXQ_TYPE_DEFAULT:
1243                 jumbo = B_FALSE;
1244                 break;
1245
1246 #if EFSYS_OPT_RX_SCATTER
1247         case EFX_RXQ_TYPE_SCATTER:
1248                 if (enp->en_family < EFX_FAMILY_SIENA) {
1249                         rc = EINVAL;
1250                         goto fail4;
1251                 }
1252                 jumbo = B_TRUE;
1253                 break;
1254 #endif  /* EFSYS_OPT_RX_SCATTER */
1255
1256         default:
1257                 rc = EINVAL;
1258                 goto fail4;
1259         }
1260
1261         /* Set up the new descriptor queue */
1262         EFX_POPULATE_OWORD_7(oword,
1263             FRF_AZ_RX_DESCQ_BUF_BASE_ID, id,
1264             FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index,
1265             FRF_AZ_RX_DESCQ_OWNER_ID, 0,
1266             FRF_AZ_RX_DESCQ_LABEL, label,
1267             FRF_AZ_RX_DESCQ_SIZE, size,
1268             FRF_AZ_RX_DESCQ_TYPE, 0,
1269             FRF_AZ_RX_DESCQ_JUMBO, jumbo);
1270
1271         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1272                             erp->er_index, &oword, B_TRUE);
1273
1274         return (0);
1275
1276 fail4:
1277         EFSYS_PROBE(fail4);
1278 fail3:
1279         EFSYS_PROBE(fail3);
1280 fail2:
1281         EFSYS_PROBE(fail2);
1282 fail1:
1283         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1284
1285         return (rc);
1286 }
1287
1288 static          void
1289 siena_rx_qdestroy(
1290         __in    efx_rxq_t *erp)
1291 {
1292         efx_nic_t *enp = erp->er_enp;
1293         efx_oword_t oword;
1294
1295         EFSYS_ASSERT(enp->en_rx_qcount != 0);
1296         --enp->en_rx_qcount;
1297
1298         /* Purge descriptor queue */
1299         EFX_ZERO_OWORD(oword);
1300
1301         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1302                             erp->er_index, &oword, B_TRUE);
1303
1304         /* Free the RXQ object */
1305         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1306 }
1307
1308 static          void
1309 siena_rx_fini(
1310         __in    efx_nic_t *enp)
1311 {
1312         _NOTE(ARGUNUSED(enp))
1313 }
1314
1315 #endif /* EFSYS_OPT_SIENA */