drivers/net/sfc/base/efx_rx.c

   1 /*
   2  * Copyright (c) 2007-2016 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright notice,
  11  *    this list of conditions and the following disclaimer in the documentation
  12  *    and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * The views and conclusions contained in the software and documentation are
  27  * those of the authors and should not be interpreted as representing official
  28  * policies, either expressed or implied, of the FreeBSD Project.
  29  */
  30
  31 #include "efx.h"
  32 #include "efx_impl.h"
  33
  34
  35 #if EFSYS_OPT_SIENA
  36
  37 static  __checkReturn   efx_rc_t
  38 siena_rx_init(
  39         __in            efx_nic_t *enp);
  40
  41 static                  void
  42 siena_rx_fini(
  43         __in            efx_nic_t *enp);
  44
  45 #if EFSYS_OPT_RX_SCATTER
  46 static  __checkReturn   efx_rc_t
  47 siena_rx_scatter_enable(
  48         __in            efx_nic_t *enp,
  49         __in            unsigned int buf_size);
  50 #endif /* EFSYS_OPT_RX_SCATTER */
  51
  52 #if EFSYS_OPT_RX_SCALE
  53 static  __checkReturn   efx_rc_t
  54 siena_rx_scale_mode_set(
  55         __in            efx_nic_t *enp,
  56         __in            uint32_t rss_context,
  57         __in            efx_rx_hash_alg_t alg,
  58         __in            efx_rx_hash_type_t type,
  59         __in            boolean_t insert);
  60
  61 static  __checkReturn   efx_rc_t
  62 siena_rx_scale_key_set(
  63         __in            efx_nic_t *enp,
  64         __in            uint32_t rss_context,
  65         __in_ecount(n)  uint8_t *key,
  66         __in            size_t n);
  67
  68 static  __checkReturn   efx_rc_t
  69 siena_rx_scale_tbl_set(
  70         __in            efx_nic_t *enp,
  71         __in            uint32_t rss_context,
  72         __in_ecount(n)  unsigned int *table,
  73         __in            size_t n);
  74
  75 static  __checkReturn   uint32_t
  76 siena_rx_prefix_hash(
  77         __in            efx_nic_t *enp,
  78         __in            efx_rx_hash_alg_t func,
  79         __in            uint8_t *buffer);
  80
  81 #endif /* EFSYS_OPT_RX_SCALE */
  82
  83 static  __checkReturn   efx_rc_t
  84 siena_rx_prefix_pktlen(
  85         __in            efx_nic_t *enp,
  86         __in            uint8_t *buffer,
  87         __out           uint16_t *lengthp);
  88
  89 static                  void
  90 siena_rx_qpost(
  91         __in            efx_rxq_t *erp,
  92         __in_ecount(n)  efsys_dma_addr_t *addrp,
  93         __in            size_t size,
  94         __in            unsigned int n,
  95         __in            unsigned int completed,
  96         __in            unsigned int added);
  97
  98 static                  void
  99 siena_rx_qpush(
 100         __in            efx_rxq_t *erp,
 101         __in            unsigned int added,
 102         __inout         unsigned int *pushedp);
 103
 104 #if EFSYS_OPT_RX_PACKED_STREAM
 105 static          void
 106 siena_rx_qps_update_credits(
 107         __in            efx_rxq_t *erp);
 108
 109 static  __checkReturn   uint8_t *
 110 siena_rx_qps_packet_info(
 111         __in            efx_rxq_t *erp,
 112         __in            uint8_t *buffer,
 113         __in            uint32_t buffer_length,
 114         __in            uint32_t current_offset,
 115         __out           uint16_t *lengthp,
 116         __out           uint32_t *next_offsetp,
 117         __out           uint32_t *timestamp);
 118 #endif
 119
 120 static  __checkReturn   efx_rc_t
 121 siena_rx_qflush(
 122         __in            efx_rxq_t *erp);
 123
 124 static                  void
 125 siena_rx_qenable(
 126         __in            efx_rxq_t *erp);
 127
 128 static  __checkReturn   efx_rc_t
 129 siena_rx_qcreate(
 130         __in            efx_nic_t *enp,
 131         __in            unsigned int index,
 132         __in            unsigned int label,
 133         __in            efx_rxq_type_t type,
 134         __in            efsys_mem_t *esmp,
 135         __in            size_t n,
 136         __in            uint32_t id,
 137         __in            efx_evq_t *eep,
 138         __in            efx_rxq_t *erp);
 139
 140 static                  void
 141 siena_rx_qdestroy(
 142         __in            efx_rxq_t *erp);
 143
 144 #endif /* EFSYS_OPT_SIENA */
 145
 146
 147 #if EFSYS_OPT_SIENA
 148 static const efx_rx_ops_t __efx_rx_siena_ops = {
 149         siena_rx_init,                          /* erxo_init */
 150         siena_rx_fini,                          /* erxo_fini */
 151 #if EFSYS_OPT_RX_SCATTER
 152         siena_rx_scatter_enable,                /* erxo_scatter_enable */
 153 #endif
 154 #if EFSYS_OPT_RX_SCALE
 155         NULL,                                   /* erxo_scale_context_alloc */
 156         NULL,                                   /* erxo_scale_context_free */
 157         siena_rx_scale_mode_set,                /* erxo_scale_mode_set */
 158         siena_rx_scale_key_set,                 /* erxo_scale_key_set */
 159         siena_rx_scale_tbl_set,                 /* erxo_scale_tbl_set */
 160         siena_rx_prefix_hash,                   /* erxo_prefix_hash */
 161 #endif
 162         siena_rx_prefix_pktlen,                 /* erxo_prefix_pktlen */
 163         siena_rx_qpost,                         /* erxo_qpost */
 164         siena_rx_qpush,                         /* erxo_qpush */
 165 #if EFSYS_OPT_RX_PACKED_STREAM
 166         siena_rx_qps_update_credits,            /* erxo_qps_update_credits */
 167         siena_rx_qps_packet_info,               /* erxo_qps_packet_info */
 168 #endif
 169         siena_rx_qflush,                        /* erxo_qflush */
 170         siena_rx_qenable,                       /* erxo_qenable */
 171         siena_rx_qcreate,                       /* erxo_qcreate */
 172         siena_rx_qdestroy,                      /* erxo_qdestroy */
 173 };
 174 #endif  /* EFSYS_OPT_SIENA */
 175
 176 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
 177 static const efx_rx_ops_t __efx_rx_ef10_ops = {
 178         ef10_rx_init,                           /* erxo_init */
 179         ef10_rx_fini,                           /* erxo_fini */
 180 #if EFSYS_OPT_RX_SCATTER
 181         ef10_rx_scatter_enable,                 /* erxo_scatter_enable */
 182 #endif
 183 #if EFSYS_OPT_RX_SCALE
 184         ef10_rx_scale_context_alloc,            /* erxo_scale_context_alloc */
 185         ef10_rx_scale_context_free,             /* erxo_scale_context_free */
 186         ef10_rx_scale_mode_set,                 /* erxo_scale_mode_set */
 187         ef10_rx_scale_key_set,                  /* erxo_scale_key_set */
 188         ef10_rx_scale_tbl_set,                  /* erxo_scale_tbl_set */
 189         ef10_rx_prefix_hash,                    /* erxo_prefix_hash */
 190 #endif
 191         ef10_rx_prefix_pktlen,                  /* erxo_prefix_pktlen */
 192         ef10_rx_qpost,                          /* erxo_qpost */
 193         ef10_rx_qpush,                          /* erxo_qpush */
 194 #if EFSYS_OPT_RX_PACKED_STREAM
 195         ef10_rx_qps_update_credits,             /* erxo_qps_update_credits */
 196         ef10_rx_qps_packet_info,                /* erxo_qps_packet_info */
 197 #endif
 198         ef10_rx_qflush,                         /* erxo_qflush */
 199         ef10_rx_qenable,                        /* erxo_qenable */
 200         ef10_rx_qcreate,                        /* erxo_qcreate */
 201         ef10_rx_qdestroy,                       /* erxo_qdestroy */
 202 };
 203 #endif  /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */
 204
 205
 206         __checkReturn   efx_rc_t
 207 efx_rx_init(
 208         __inout         efx_nic_t *enp)
 209 {
 210         const efx_rx_ops_t *erxop;
 211         efx_rc_t rc;
 212
 213         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 214         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
 215
 216         if (!(enp->en_mod_flags & EFX_MOD_EV)) {
 217                 rc = EINVAL;
 218                 goto fail1;
 219         }
 220
 221         if (enp->en_mod_flags & EFX_MOD_RX) {
 222                 rc = EINVAL;
 223                 goto fail2;
 224         }
 225
 226         switch (enp->en_family) {
 227 #if EFSYS_OPT_SIENA
 228         case EFX_FAMILY_SIENA:
 229                 erxop = &__efx_rx_siena_ops;
 230                 break;
 231 #endif /* EFSYS_OPT_SIENA */
 232
 233 #if EFSYS_OPT_HUNTINGTON
 234         case EFX_FAMILY_HUNTINGTON:
 235                 erxop = &__efx_rx_ef10_ops;
 236                 break;
 237 #endif /* EFSYS_OPT_HUNTINGTON */
 238
 239 #if EFSYS_OPT_MEDFORD
 240         case EFX_FAMILY_MEDFORD:
 241                 erxop = &__efx_rx_ef10_ops;
 242                 break;
 243 #endif /* EFSYS_OPT_MEDFORD */
 244
 245         default:
 246                 EFSYS_ASSERT(0);
 247                 rc = ENOTSUP;
 248                 goto fail3;
 249         }
 250
 251         if ((rc = erxop->erxo_init(enp)) != 0)
 252                 goto fail4;
 253
 254         enp->en_erxop = erxop;
 255         enp->en_mod_flags |= EFX_MOD_RX;
 256         return (0);
 257
 258 fail4:
 259         EFSYS_PROBE(fail4);
 260 fail3:
 261         EFSYS_PROBE(fail3);
 262 fail2:
 263         EFSYS_PROBE(fail2);
 264 fail1:
 265         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 266
 267         enp->en_erxop = NULL;
 268         enp->en_mod_flags &= ~EFX_MOD_RX;
 269         return (rc);
 270 }
 271
 272                         void
 273 efx_rx_fini(
 274         __in            efx_nic_t *enp)
 275 {
 276         const efx_rx_ops_t *erxop = enp->en_erxop;
 277
 278         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 279         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
 280         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 281         EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0);
 282
 283         erxop->erxo_fini(enp);
 284
 285         enp->en_erxop = NULL;
 286         enp->en_mod_flags &= ~EFX_MOD_RX;
 287 }
 288
 289 #if EFSYS_OPT_RX_SCATTER
 290         __checkReturn   efx_rc_t
 291 efx_rx_scatter_enable(
 292         __in            efx_nic_t *enp,
 293         __in            unsigned int buf_size)
 294 {
 295         const efx_rx_ops_t *erxop = enp->en_erxop;
 296         efx_rc_t rc;
 297
 298         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 299         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 300
 301         if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0)
 302                 goto fail1;
 303
 304         return (0);
 305
 306 fail1:
 307         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 308         return (rc);
 309 }
 310 #endif  /* EFSYS_OPT_RX_SCATTER */
 311
 312 #if EFSYS_OPT_RX_SCALE
 313         __checkReturn   efx_rc_t
 314 efx_rx_hash_default_support_get(
 315         __in            efx_nic_t *enp,
 316         __out           efx_rx_hash_support_t *supportp)
 317 {
 318         efx_rc_t rc;
 319
 320         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 321         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 322
 323         if (supportp == NULL) {
 324                 rc = EINVAL;
 325                 goto fail1;
 326         }
 327
 328         /*
 329          * Report the hashing support the client gets by default if it
 330          * does not allocate an RSS context itself.
 331          */
 332         *supportp = enp->en_hash_support;
 333
 334         return (0);
 335
 336 fail1:
 337         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 338
 339         return (rc);
 340 }
 341
 342         __checkReturn   efx_rc_t
 343 efx_rx_scale_default_support_get(
 344         __in            efx_nic_t *enp,
 345         __out           efx_rx_scale_context_type_t *typep)
 346 {
 347         efx_rc_t rc;
 348
 349         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 350         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 351
 352         if (typep == NULL) {
 353                 rc = EINVAL;
 354                 goto fail1;
 355         }
 356
 357         /*
 358          * Report the RSS support the client gets by default if it
 359          * does not allocate an RSS context itself.
 360          */
 361         *typep = enp->en_rss_context_type;
 362
 363         return (0);
 364
 365 fail1:
 366         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 367
 368         return (rc);
 369 }
 370 #endif  /* EFSYS_OPT_RX_SCALE */
 371
 372 #if EFSYS_OPT_RX_SCALE
 373         __checkReturn   efx_rc_t
 374 efx_rx_scale_context_alloc(
 375         __in            efx_nic_t *enp,
 376         __in            efx_rx_scale_context_type_t type,
 377         __in            uint32_t num_queues,
 378         __out           uint32_t *rss_contextp)
 379 {
 380         const efx_rx_ops_t *erxop = enp->en_erxop;
 381         efx_rc_t rc;
 382
 383         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 384         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 385
 386         if (erxop->erxo_scale_context_alloc == NULL) {
 387                 rc = ENOTSUP;
 388                 goto fail1;
 389         }
 390         if ((rc = erxop->erxo_scale_context_alloc(enp, type,
 391                             num_queues, rss_contextp)) != 0) {
 392                 goto fail2;
 393         }
 394
 395         return (0);
 396
 397 fail2:
 398         EFSYS_PROBE(fail2);
 399 fail1:
 400         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 401         return (rc);
 402 }
 403 #endif  /* EFSYS_OPT_RX_SCALE */
 404
 405 #if EFSYS_OPT_RX_SCALE
 406         __checkReturn   efx_rc_t
 407 efx_rx_scale_context_free(
 408         __in            efx_nic_t *enp,
 409         __in            uint32_t rss_context)
 410 {
 411         const efx_rx_ops_t *erxop = enp->en_erxop;
 412         efx_rc_t rc;
 413
 414         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 415         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 416
 417         if (erxop->erxo_scale_context_free == NULL) {
 418                 rc = ENOTSUP;
 419                 goto fail1;
 420         }
 421         if ((rc = erxop->erxo_scale_context_free(enp, rss_context)) != 0)
 422                 goto fail2;
 423
 424         return (0);
 425
 426 fail2:
 427         EFSYS_PROBE(fail2);
 428 fail1:
 429         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 430         return (rc);
 431 }
 432 #endif  /* EFSYS_OPT_RX_SCALE */
 433
 434 #if EFSYS_OPT_RX_SCALE
 435         __checkReturn   efx_rc_t
 436 efx_rx_scale_mode_set(
 437         __in            efx_nic_t *enp,
 438         __in            uint32_t rss_context,
 439         __in            efx_rx_hash_alg_t alg,
 440         __in            efx_rx_hash_type_t type,
 441         __in            boolean_t insert)
 442 {
 443         const efx_rx_ops_t *erxop = enp->en_erxop;
 444         efx_rc_t rc;
 445
 446         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 447         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 448
 449         if (erxop->erxo_scale_mode_set != NULL) {
 450                 if ((rc = erxop->erxo_scale_mode_set(enp, rss_context, alg,
 451                             type, insert)) != 0)
 452                         goto fail1;
 453         }
 454
 455         return (0);
 456
 457 fail1:
 458         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 459         return (rc);
 460 }
 461 #endif  /* EFSYS_OPT_RX_SCALE */
 462
 463 #if EFSYS_OPT_RX_SCALE
 464         __checkReturn   efx_rc_t
 465 efx_rx_scale_key_set(
 466         __in            efx_nic_t *enp,
 467         __in            uint32_t rss_context,
 468         __in_ecount(n)  uint8_t *key,
 469         __in            size_t n)
 470 {
 471         const efx_rx_ops_t *erxop = enp->en_erxop;
 472         efx_rc_t rc;
 473
 474         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 475         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 476
 477         if ((rc = erxop->erxo_scale_key_set(enp, rss_context, key, n)) != 0)
 478                 goto fail1;
 479
 480         return (0);
 481
 482 fail1:
 483         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 484
 485         return (rc);
 486 }
 487 #endif  /* EFSYS_OPT_RX_SCALE */
 488
 489 #if EFSYS_OPT_RX_SCALE
 490         __checkReturn   efx_rc_t
 491 efx_rx_scale_tbl_set(
 492         __in            efx_nic_t *enp,
 493         __in            uint32_t rss_context,
 494         __in_ecount(n)  unsigned int *table,
 495         __in            size_t n)
 496 {
 497         const efx_rx_ops_t *erxop = enp->en_erxop;
 498         efx_rc_t rc;
 499
 500         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 501         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 502
 503         if ((rc = erxop->erxo_scale_tbl_set(enp, rss_context, table, n)) != 0)
 504                 goto fail1;
 505
 506         return (0);
 507
 508 fail1:
 509         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 510
 511         return (rc);
 512 }
 513 #endif  /* EFSYS_OPT_RX_SCALE */
 514
 515                         void
 516 efx_rx_qpost(
 517         __in            efx_rxq_t *erp,
 518         __in_ecount(n)  efsys_dma_addr_t *addrp,
 519         __in            size_t size,
 520         __in            unsigned int n,
 521         __in            unsigned int completed,
 522         __in            unsigned int added)
 523 {
 524         efx_nic_t *enp = erp->er_enp;
 525         const efx_rx_ops_t *erxop = enp->en_erxop;
 526
 527         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 528
 529         erxop->erxo_qpost(erp, addrp, size, n, completed, added);
 530 }
 531
 532 #if EFSYS_OPT_RX_PACKED_STREAM
 533
 534                         void
 535 efx_rx_qps_update_credits(
 536         __in            efx_rxq_t *erp)
 537 {
 538         efx_nic_t *enp = erp->er_enp;
 539         const efx_rx_ops_t *erxop = enp->en_erxop;
 540
 541         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 542
 543         erxop->erxo_qps_update_credits(erp);
 544 }
 545
 546         __checkReturn   uint8_t *
 547 efx_rx_qps_packet_info(
 548         __in            efx_rxq_t *erp,
 549         __in            uint8_t *buffer,
 550         __in            uint32_t buffer_length,
 551         __in            uint32_t current_offset,
 552         __out           uint16_t *lengthp,
 553         __out           uint32_t *next_offsetp,
 554         __out           uint32_t *timestamp)
 555 {
 556         efx_nic_t *enp = erp->er_enp;
 557         const efx_rx_ops_t *erxop = enp->en_erxop;
 558
 559         return (erxop->erxo_qps_packet_info(erp, buffer,
 560                 buffer_length, current_offset, lengthp,
 561                 next_offsetp, timestamp));
 562 }
 563
 564 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
 565
 566                         void
 567 efx_rx_qpush(
 568         __in            efx_rxq_t *erp,
 569         __in            unsigned int added,
 570         __inout         unsigned int *pushedp)
 571 {
 572         efx_nic_t *enp = erp->er_enp;
 573         const efx_rx_ops_t *erxop = enp->en_erxop;
 574
 575         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 576
 577         erxop->erxo_qpush(erp, added, pushedp);
 578 }
 579
 580         __checkReturn   efx_rc_t
 581 efx_rx_qflush(
 582         __in            efx_rxq_t *erp)
 583 {
 584         efx_nic_t *enp = erp->er_enp;
 585         const efx_rx_ops_t *erxop = enp->en_erxop;
 586         efx_rc_t rc;
 587
 588         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 589
 590         if ((rc = erxop->erxo_qflush(erp)) != 0)
 591                 goto fail1;
 592
 593         return (0);
 594
 595 fail1:
 596         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 597
 598         return (rc);
 599 }
 600
 601                         void
 602 efx_rx_qenable(
 603         __in            efx_rxq_t *erp)
 604 {
 605         efx_nic_t *enp = erp->er_enp;
 606         const efx_rx_ops_t *erxop = enp->en_erxop;
 607
 608         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 609
 610         erxop->erxo_qenable(erp);
 611 }
 612
 613         __checkReturn   efx_rc_t
 614 efx_rx_qcreate(
 615         __in            efx_nic_t *enp,
 616         __in            unsigned int index,
 617         __in            unsigned int label,
 618         __in            efx_rxq_type_t type,
 619         __in            efsys_mem_t *esmp,
 620         __in            size_t n,
 621         __in            uint32_t id,
 622         __in            efx_evq_t *eep,
 623         __deref_out     efx_rxq_t **erpp)
 624 {
 625         const efx_rx_ops_t *erxop = enp->en_erxop;
 626         efx_rxq_t *erp;
 627         efx_rc_t rc;
 628
 629         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 630         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 631
 632         /* Allocate an RXQ object */
 633         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp);
 634
 635         if (erp == NULL) {
 636                 rc = ENOMEM;
 637                 goto fail1;
 638         }
 639
 640         erp->er_magic = EFX_RXQ_MAGIC;
 641         erp->er_enp = enp;
 642         erp->er_index = index;
 643         erp->er_mask = n - 1;
 644         erp->er_esmp = esmp;
 645
 646         if ((rc = erxop->erxo_qcreate(enp, index, label, type, esmp, n, id,
 647             eep, erp)) != 0)
 648                 goto fail2;
 649
 650         enp->en_rx_qcount++;
 651         *erpp = erp;
 652
 653         return (0);
 654
 655 fail2:
 656         EFSYS_PROBE(fail2);
 657
 658         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
 659 fail1:
 660         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 661
 662         return (rc);
 663 }
 664
 665                         void
 666 efx_rx_qdestroy(
 667         __in            efx_rxq_t *erp)
 668 {
 669         efx_nic_t *enp = erp->er_enp;
 670         const efx_rx_ops_t *erxop = enp->en_erxop;
 671
 672         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 673
 674         erxop->erxo_qdestroy(erp);
 675 }
 676
 677         __checkReturn   efx_rc_t
 678 efx_pseudo_hdr_pkt_length_get(
 679         __in            efx_rxq_t *erp,
 680         __in            uint8_t *buffer,
 681         __out           uint16_t *lengthp)
 682 {
 683         efx_nic_t *enp = erp->er_enp;
 684         const efx_rx_ops_t *erxop = enp->en_erxop;
 685
 686         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 687
 688         return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp));
 689 }
 690
 691 #if EFSYS_OPT_RX_SCALE
 692         __checkReturn   uint32_t
 693 efx_pseudo_hdr_hash_get(
 694         __in            efx_rxq_t *erp,
 695         __in            efx_rx_hash_alg_t func,
 696         __in            uint8_t *buffer)
 697 {
 698         efx_nic_t *enp = erp->er_enp;
 699         const efx_rx_ops_t *erxop = enp->en_erxop;
 700
 701         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 702
 703         EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE);
 704         return (erxop->erxo_prefix_hash(enp, func, buffer));
 705 }
 706 #endif  /* EFSYS_OPT_RX_SCALE */
 707
 708 #if EFSYS_OPT_SIENA
 709
 710 static  __checkReturn   efx_rc_t
 711 siena_rx_init(
 712         __in            efx_nic_t *enp)
 713 {
 714         efx_oword_t oword;
 715         unsigned int index;
 716
 717         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 718
 719         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0);
 720         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);
 721         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);
 722         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);
 723         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0);
 724         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32);
 725         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 726
 727         /* Zero the RSS table */
 728         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS;
 729             index++) {
 730                 EFX_ZERO_OWORD(oword);
 731                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
 732                                     index, &oword, B_TRUE);
 733         }
 734
 735 #if EFSYS_OPT_RX_SCALE
 736         /* The RSS key and indirection table are writable. */
 737         enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE;
 738
 739         /* Hardware can insert RX hash with/without RSS */
 740         enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
 741 #endif  /* EFSYS_OPT_RX_SCALE */
 742
 743         return (0);
 744 }
 745
 746 #if EFSYS_OPT_RX_SCATTER
 747 static  __checkReturn   efx_rc_t
 748 siena_rx_scatter_enable(
 749         __in            efx_nic_t *enp,
 750         __in            unsigned int buf_size)
 751 {
 752         unsigned int nbuf32;
 753         efx_oword_t oword;
 754         efx_rc_t rc;
 755
 756         nbuf32 = buf_size / 32;
 757         if ((nbuf32 == 0) ||
 758             (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) ||
 759             ((buf_size % 32) != 0)) {
 760                 rc = EINVAL;
 761                 goto fail1;
 762         }
 763
 764         if (enp->en_rx_qcount > 0) {
 765                 rc = EBUSY;
 766                 goto fail2;
 767         }
 768
 769         /* Set scatter buffer size */
 770         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 771         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32);
 772         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 773
 774         /* Enable scatter for packets not matching a filter */
 775         EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 776         EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1);
 777         EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 778
 779         return (0);
 780
 781 fail2:
 782         EFSYS_PROBE(fail2);
 783 fail1:
 784         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 785
 786         return (rc);
 787 }
 788 #endif  /* EFSYS_OPT_RX_SCATTER */
 789
 790
 791 #define EFX_RX_LFSR_HASH(_enp, _insert)                                 \
 792         do {                                                            \
 793                 efx_oword_t oword;                                      \
 794                                                                         \
 795                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);        \
 796                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);      \
 797                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);       \
 798                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);       \
 799                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 800                     (_insert) ? 1 : 0);                                 \
 801                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);       \
 802                                                                         \
 803                 if ((_enp)->en_family == EFX_FAMILY_SIENA) {            \
 804                         EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3,   \
 805                             &oword);                                    \
 806                         EFX_SET_OWORD_FIELD(oword,                      \
 807                             FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0);        \
 808                         EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3,  \
 809                             &oword);                                    \
 810                 }                                                       \
 811                                                                         \
 812                 _NOTE(CONSTANTCONDITION)                                \
 813         } while (B_FALSE)
 814
 815 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp)             \
 816         do {                                                            \
 817                 efx_oword_t oword;                                      \
 818                                                                         \
 819                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);        \
 820                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1);      \
 821                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH,           \
 822                     (_ip) ? 1 : 0);                                     \
 823                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP,           \
 824                     (_tcp) ? 0 : 1);                                    \
 825                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 826                     (_insert) ? 1 : 0);                                 \
 827                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);       \
 828                                                                         \
 829                 _NOTE(CONSTANTCONDITION)                                \
 830         } while (B_FALSE)
 831
 832 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc)                 \
 833         do {                                                            \
 834                 efx_oword_t oword;                                      \
 835                                                                         \
 836                 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword);  \
 837                 EFX_SET_OWORD_FIELD(oword,                              \
 838                     FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1);                \
 839                 EFX_SET_OWORD_FIELD(oword,                              \
 840                     FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \
 841                 EFX_SET_OWORD_FIELD(oword,                              \
 842                     FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1);   \
 843                 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
 844                                                                         \
 845                 (_rc) = 0;                                              \
 846                                                                         \
 847                 _NOTE(CONSTANTCONDITION)                                \
 848         } while (B_FALSE)
 849
 850
 851 #if EFSYS_OPT_RX_SCALE
 852
 853 static  __checkReturn   efx_rc_t
 854 siena_rx_scale_mode_set(
 855         __in            efx_nic_t *enp,
 856         __in            uint32_t rss_context,
 857         __in            efx_rx_hash_alg_t alg,
 858         __in            efx_rx_hash_type_t type,
 859         __in            boolean_t insert)
 860 {
 861         efx_rc_t rc;
 862
 863         if (rss_context != EFX_RSS_CONTEXT_DEFAULT) {
 864                 rc = EINVAL;
 865                 goto fail1;
 866         }
 867
 868         switch (alg) {
 869         case EFX_RX_HASHALG_LFSR:
 870                 EFX_RX_LFSR_HASH(enp, insert);
 871                 break;
 872
 873         case EFX_RX_HASHALG_TOEPLITZ:
 874                 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert,
 875                     type & EFX_RX_HASH_IPV4,
 876                     type & EFX_RX_HASH_TCPIPV4);
 877
 878                 EFX_RX_TOEPLITZ_IPV6_HASH(enp,
 879                     type & EFX_RX_HASH_IPV6,
 880                     type & EFX_RX_HASH_TCPIPV6,
 881                     rc);
 882                 if (rc != 0)
 883                         goto fail2;
 884
 885                 break;
 886
 887         default:
 888                 rc = EINVAL;
 889                 goto fail3;
 890         }
 891
 892         return (0);
 893
 894 fail3:
 895         EFSYS_PROBE(fail3);
 896 fail2:
 897         EFSYS_PROBE(fail2);
 898 fail1:
 899         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 900
 901         EFX_RX_LFSR_HASH(enp, B_FALSE);
 902
 903         return (rc);
 904 }
 905 #endif
 906
 907 #if EFSYS_OPT_RX_SCALE
 908 static  __checkReturn   efx_rc_t
 909 siena_rx_scale_key_set(
 910         __in            efx_nic_t *enp,
 911         __in            uint32_t rss_context,
 912         __in_ecount(n)  uint8_t *key,
 913         __in            size_t n)
 914 {
 915         efx_oword_t oword;
 916         unsigned int byte;
 917         unsigned int offset;
 918         efx_rc_t rc;
 919
 920         if (rss_context != EFX_RSS_CONTEXT_DEFAULT) {
 921                 rc = EINVAL;
 922                 goto fail1;
 923         }
 924
 925         byte = 0;
 926
 927         /* Write Toeplitz IPv4 hash key */
 928         EFX_ZERO_OWORD(oword);
 929         for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
 930             offset > 0 && byte < n;
 931             --offset)
 932                 oword.eo_u8[offset - 1] = key[byte++];
 933
 934         EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
 935
 936         byte = 0;
 937
 938         /* Verify Toeplitz IPv4 hash key */
 939         EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
 940         for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
 941             offset > 0 && byte < n;
 942             --offset) {
 943                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 944                         rc = EFAULT;
 945                         goto fail2;
 946                 }
 947         }
 948
 949         if ((enp->en_features & EFX_FEATURE_IPV6) == 0)
 950                 goto done;
 951
 952         byte = 0;
 953
 954         /* Write Toeplitz IPv6 hash key 3 */
 955         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 956         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
 957             FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
 958             offset > 0 && byte < n;
 959             --offset)
 960                 oword.eo_u8[offset - 1] = key[byte++];
 961
 962         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 963
 964         /* Write Toeplitz IPv6 hash key 2 */
 965         EFX_ZERO_OWORD(oword);
 966         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
 967             FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
 968             offset > 0 && byte < n;
 969             --offset)
 970                 oword.eo_u8[offset - 1] = key[byte++];
 971
 972         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
 973
 974         /* Write Toeplitz IPv6 hash key 1 */
 975         EFX_ZERO_OWORD(oword);
 976         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
 977             FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
 978             offset > 0 && byte < n;
 979             --offset)
 980                 oword.eo_u8[offset - 1] = key[byte++];
 981
 982         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
 983
 984         byte = 0;
 985
 986         /* Verify Toeplitz IPv6 hash key 3 */
 987         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 988         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
 989             FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
 990             offset > 0 && byte < n;
 991             --offset) {
 992                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 993                         rc = EFAULT;
 994                         goto fail3;
 995                 }
 996         }
 997
 998         /* Verify Toeplitz IPv6 hash key 2 */
 999         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
1000         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
1001             FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
1002             offset > 0 && byte < n;
1003             --offset) {
1004                 if (oword.eo_u8[offset - 1] != key[byte++]) {
1005                         rc = EFAULT;
1006                         goto fail4;
1007                 }
1008         }
1009
1010         /* Verify Toeplitz IPv6 hash key 1 */
1011         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
1012         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
1013             FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
1014             offset > 0 && byte < n;
1015             --offset) {
1016                 if (oword.eo_u8[offset - 1] != key[byte++]) {
1017                         rc = EFAULT;
1018                         goto fail5;
1019                 }
1020         }
1021
1022 done:
1023         return (0);
1024
1025 fail5:
1026         EFSYS_PROBE(fail5);
1027 fail4:
1028         EFSYS_PROBE(fail4);
1029 fail3:
1030         EFSYS_PROBE(fail3);
1031 fail2:
1032         EFSYS_PROBE(fail2);
1033 fail1:
1034         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1035
1036         return (rc);
1037 }
1038 #endif
1039
1040 #if EFSYS_OPT_RX_SCALE
1041 static  __checkReturn   efx_rc_t
1042 siena_rx_scale_tbl_set(
1043         __in            efx_nic_t *enp,
1044         __in            uint32_t rss_context,
1045         __in_ecount(n)  unsigned int *table,
1046         __in            size_t n)
1047 {
1048         efx_oword_t oword;
1049         int index;
1050         efx_rc_t rc;
1051
1052         EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS);
1053         EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH));
1054
1055         if (rss_context != EFX_RSS_CONTEXT_DEFAULT) {
1056                 rc = EINVAL;
1057                 goto fail1;
1058         }
1059
1060         if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) {
1061                 rc = EINVAL;
1062                 goto fail2;
1063         }
1064
1065         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) {
1066                 uint32_t byte;
1067
1068                 /* Calculate the entry to place in the table */
1069                 byte = (n > 0) ? (uint32_t)table[index % n] : 0;
1070
1071                 EFSYS_PROBE2(table, int, index, uint32_t, byte);
1072
1073                 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte);
1074
1075                 /* Write the table */
1076                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
1077                                     index, &oword, B_TRUE);
1078         }
1079
1080         for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) {
1081                 uint32_t byte;
1082
1083                 /* Determine if we're starting a new batch */
1084                 byte = (n > 0) ? (uint32_t)table[index % n] : 0;
1085
1086                 /* Read the table */
1087                 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL,
1088                                     index, &oword, B_TRUE);
1089
1090                 /* Verify the entry */
1091                 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) {
1092                         rc = EFAULT;
1093                         goto fail3;
1094                 }
1095         }
1096
1097         return (0);
1098
1099 fail3:
1100         EFSYS_PROBE(fail3);
1101 fail2:
1102         EFSYS_PROBE(fail2);
1103 fail1:
1104         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1105
1106         return (rc);
1107 }
1108 #endif
1109
1110 /*
1111  * Falcon/Siena pseudo-header
1112  * --------------------------
1113  *
1114  * Receive packets are prefixed by an optional 16 byte pseudo-header.
1115  * The pseudo-header is a byte array of one of the forms:
1116  *
1117  *  0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15
1118  * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT
1119  * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL
1120  *
1121  * where:
1122  *   TT.TT.TT.TT   Toeplitz hash (32-bit big-endian)
1123  *   LL.LL         LFSR hash     (16-bit big-endian)
1124  */
1125
1126 #if EFSYS_OPT_RX_SCALE
1127 static  __checkReturn   uint32_t
1128 siena_rx_prefix_hash(
1129         __in            efx_nic_t *enp,
1130         __in            efx_rx_hash_alg_t func,
1131         __in            uint8_t *buffer)
1132 {
1133         _NOTE(ARGUNUSED(enp))
1134
1135         switch (func) {
1136         case EFX_RX_HASHALG_TOEPLITZ:
1137                 return ((buffer[12] << 24) |
1138                     (buffer[13] << 16) |
1139                     (buffer[14] <<  8) |
1140                     buffer[15]);
1141
1142         case EFX_RX_HASHALG_LFSR:
1143                 return ((buffer[14] << 8) | buffer[15]);
1144
1145         default:
1146                 EFSYS_ASSERT(0);
1147                 return (0);
1148         }
1149 }
1150 #endif /* EFSYS_OPT_RX_SCALE */
1151
1152 static  __checkReturn   efx_rc_t
1153 siena_rx_prefix_pktlen(
1154         __in            efx_nic_t *enp,
1155         __in            uint8_t *buffer,
1156         __out           uint16_t *lengthp)
1157 {
1158         _NOTE(ARGUNUSED(enp, buffer, lengthp))
1159
1160         /* Not supported by Falcon/Siena hardware */
1161         EFSYS_ASSERT(0);
1162         return (ENOTSUP);
1163 }
1164
1165
1166 static                  void
1167 siena_rx_qpost(
1168         __in            efx_rxq_t *erp,
1169         __in_ecount(n)  efsys_dma_addr_t *addrp,
1170         __in            size_t size,
1171         __in            unsigned int n,
1172         __in            unsigned int completed,
1173         __in            unsigned int added)
1174 {
1175         efx_qword_t qword;
1176         unsigned int i;
1177         unsigned int offset;
1178         unsigned int id;
1179
1180         /* The client driver must not overfill the queue */
1181         EFSYS_ASSERT3U(added - completed + n, <=,
1182             EFX_RXQ_LIMIT(erp->er_mask + 1));
1183
1184         id = added & (erp->er_mask);
1185         for (i = 0; i < n; i++) {
1186                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
1187                     unsigned int, id, efsys_dma_addr_t, addrp[i],
1188                     size_t, size);
1189
1190                 EFX_POPULATE_QWORD_3(qword,
1191                     FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size),
1192                     FSF_AZ_RX_KER_BUF_ADDR_DW0,
1193                     (uint32_t)(addrp[i] & 0xffffffff),
1194                     FSF_AZ_RX_KER_BUF_ADDR_DW1,
1195                     (uint32_t)(addrp[i] >> 32));
1196
1197                 offset = id * sizeof (efx_qword_t);
1198                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
1199
1200                 id = (id + 1) & (erp->er_mask);
1201         }
1202 }
1203
1204 static                  void
1205 siena_rx_qpush(
1206         __in    efx_rxq_t *erp,
1207         __in    unsigned int added,
1208         __inout unsigned int *pushedp)
1209 {
1210         efx_nic_t *enp = erp->er_enp;
1211         unsigned int pushed = *pushedp;
1212         uint32_t wptr;
1213         efx_oword_t oword;
1214         efx_dword_t dword;
1215
1216         /* All descriptors are pushed */
1217         *pushedp = added;
1218
1219         /* Push the populated descriptors out */
1220         wptr = added & erp->er_mask;
1221
1222         EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr);
1223
1224         /* Only write the third DWORD */
1225         EFX_POPULATE_DWORD_1(dword,
1226             EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
1227
1228         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
1229         EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
1230             wptr, pushed & erp->er_mask);
1231         EFSYS_PIO_WRITE_BARRIER();
1232         EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0,
1233                             erp->er_index, &dword, B_FALSE);
1234 }
1235
1236 #if EFSYS_OPT_RX_PACKED_STREAM
1237 static          void
1238 siena_rx_qps_update_credits(
1239         __in            efx_rxq_t *erp)
1240 {
1241         /* Not supported by Siena hardware */
1242         EFSYS_ASSERT(0);
1243 }
1244
1245 static          uint8_t *
1246 siena_rx_qps_packet_info(
1247         __in            efx_rxq_t *erp,
1248         __in            uint8_t *buffer,
1249         __in            uint32_t buffer_length,
1250         __in            uint32_t current_offset,
1251         __out           uint16_t *lengthp,
1252         __out           uint32_t *next_offsetp,
1253         __out           uint32_t *timestamp)
1254 {
1255         /* Not supported by Siena hardware */
1256         EFSYS_ASSERT(0);
1257
1258         return (NULL);
1259 }
1260 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1261
1262 static  __checkReturn   efx_rc_t
1263 siena_rx_qflush(
1264         __in    efx_rxq_t *erp)
1265 {
1266         efx_nic_t *enp = erp->er_enp;
1267         efx_oword_t oword;
1268         uint32_t label;
1269
1270         label = erp->er_index;
1271
1272         /* Flush the queue */
1273         EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
1274             FRF_AZ_RX_FLUSH_DESCQ, label);
1275         EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword);
1276
1277         return (0);
1278 }
1279
1280 static          void
1281 siena_rx_qenable(
1282         __in    efx_rxq_t *erp)
1283 {
1284         efx_nic_t *enp = erp->er_enp;
1285         efx_oword_t oword;
1286
1287         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1288
1289         EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL,
1290                             erp->er_index, &oword, B_TRUE);
1291
1292         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0);
1293         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0);
1294         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1);
1295
1296         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1297                             erp->er_index, &oword, B_TRUE);
1298 }
1299
1300 static  __checkReturn   efx_rc_t
1301 siena_rx_qcreate(
1302         __in            efx_nic_t *enp,
1303         __in            unsigned int index,
1304         __in            unsigned int label,
1305         __in            efx_rxq_type_t type,
1306         __in            efsys_mem_t *esmp,
1307         __in            size_t n,
1308         __in            uint32_t id,
1309         __in            efx_evq_t *eep,
1310         __in            efx_rxq_t *erp)
1311 {
1312         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1313         efx_oword_t oword;
1314         uint32_t size;
1315         boolean_t jumbo;
1316         efx_rc_t rc;
1317
1318         _NOTE(ARGUNUSED(esmp))
1319
1320         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS ==
1321             (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH));
1322         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
1323         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
1324
1325         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
1326         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
1327
1328         if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) {
1329                 rc = EINVAL;
1330                 goto fail1;
1331         }
1332         if (index >= encp->enc_rxq_limit) {
1333                 rc = EINVAL;
1334                 goto fail2;
1335         }
1336         for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS);
1337             size++)
1338                 if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS))
1339                         break;
1340         if (id + (1 << size) >= encp->enc_buftbl_limit) {
1341                 rc = EINVAL;
1342                 goto fail3;
1343         }
1344
1345         switch (type) {
1346         case EFX_RXQ_TYPE_DEFAULT:
1347                 jumbo = B_FALSE;
1348                 break;
1349
1350 #if EFSYS_OPT_RX_SCATTER
1351         case EFX_RXQ_TYPE_SCATTER:
1352                 if (enp->en_family < EFX_FAMILY_SIENA) {
1353                         rc = EINVAL;
1354                         goto fail4;
1355                 }
1356                 jumbo = B_TRUE;
1357                 break;
1358 #endif  /* EFSYS_OPT_RX_SCATTER */
1359
1360         default:
1361                 rc = EINVAL;
1362                 goto fail4;
1363         }
1364
1365         /* Set up the new descriptor queue */
1366         EFX_POPULATE_OWORD_7(oword,
1367             FRF_AZ_RX_DESCQ_BUF_BASE_ID, id,
1368             FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index,
1369             FRF_AZ_RX_DESCQ_OWNER_ID, 0,
1370             FRF_AZ_RX_DESCQ_LABEL, label,
1371             FRF_AZ_RX_DESCQ_SIZE, size,
1372             FRF_AZ_RX_DESCQ_TYPE, 0,
1373             FRF_AZ_RX_DESCQ_JUMBO, jumbo);
1374
1375         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1376                             erp->er_index, &oword, B_TRUE);
1377
1378         return (0);
1379
1380 fail4:
1381         EFSYS_PROBE(fail4);
1382 fail3:
1383         EFSYS_PROBE(fail3);
1384 fail2:
1385         EFSYS_PROBE(fail2);
1386 fail1:
1387         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1388
1389         return (rc);
1390 }
1391
1392 static          void
1393 siena_rx_qdestroy(
1394         __in    efx_rxq_t *erp)
1395 {
1396         efx_nic_t *enp = erp->er_enp;
1397         efx_oword_t oword;
1398
1399         EFSYS_ASSERT(enp->en_rx_qcount != 0);
1400         --enp->en_rx_qcount;
1401
1402         /* Purge descriptor queue */
1403         EFX_ZERO_OWORD(oword);
1404
1405         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1406                             erp->er_index, &oword, B_TRUE);
1407
1408         /* Free the RXQ object */
1409         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1410 }
1411
1412 static          void
1413 siena_rx_fini(
1414         __in    efx_nic_t *enp)
1415 {
1416         _NOTE(ARGUNUSED(enp))
1417 }
1418
1419 #endif /* EFSYS_OPT_SIENA */