drivers/net/sfc/base/efx_rx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright (c) 2007-2018 Solarflare Communications Inc.
   4  * All rights reserved.
   5  */
   6
   7 #include "efx.h"
   8 #include "efx_impl.h"
   9
  10
  11 #if EFSYS_OPT_SIENA
  12
  13 static  __checkReturn   efx_rc_t
  14 siena_rx_init(
  15         __in            efx_nic_t *enp);
  16
  17 static                  void
  18 siena_rx_fini(
  19         __in            efx_nic_t *enp);
  20
  21 #if EFSYS_OPT_RX_SCATTER
  22 static  __checkReturn   efx_rc_t
  23 siena_rx_scatter_enable(
  24         __in            efx_nic_t *enp,
  25         __in            unsigned int buf_size);
  26 #endif /* EFSYS_OPT_RX_SCATTER */
  27
  28 #if EFSYS_OPT_RX_SCALE
  29 static  __checkReturn   efx_rc_t
  30 siena_rx_scale_mode_set(
  31         __in            efx_nic_t *enp,
  32         __in            uint32_t rss_context,
  33         __in            efx_rx_hash_alg_t alg,
  34         __in            efx_rx_hash_type_t type,
  35         __in            boolean_t insert);
  36
  37 static  __checkReturn   efx_rc_t
  38 siena_rx_scale_key_set(
  39         __in            efx_nic_t *enp,
  40         __in            uint32_t rss_context,
  41         __in_ecount(n)  uint8_t *key,
  42         __in            size_t n);
  43
  44 static  __checkReturn   efx_rc_t
  45 siena_rx_scale_tbl_set(
  46         __in            efx_nic_t *enp,
  47         __in            uint32_t rss_context,
  48         __in_ecount(n)  unsigned int *table,
  49         __in            size_t n);
  50
  51 static  __checkReturn   uint32_t
  52 siena_rx_prefix_hash(
  53         __in            efx_nic_t *enp,
  54         __in            efx_rx_hash_alg_t func,
  55         __in            uint8_t *buffer);
  56
  57 #endif /* EFSYS_OPT_RX_SCALE */
  58
  59 static  __checkReturn   efx_rc_t
  60 siena_rx_prefix_pktlen(
  61         __in            efx_nic_t *enp,
  62         __in            uint8_t *buffer,
  63         __out           uint16_t *lengthp);
  64
  65 static                          void
  66 siena_rx_qpost(
  67         __in                    efx_rxq_t *erp,
  68         __in_ecount(ndescs)     efsys_dma_addr_t *addrp,
  69         __in                    size_t size,
  70         __in                    unsigned int ndescs,
  71         __in                    unsigned int completed,
  72         __in                    unsigned int added);
  73
  74 static                  void
  75 siena_rx_qpush(
  76         __in            efx_rxq_t *erp,
  77         __in            unsigned int added,
  78         __inout         unsigned int *pushedp);
  79
  80 #if EFSYS_OPT_RX_PACKED_STREAM
  81 static          void
  82 siena_rx_qpush_ps_credits(
  83         __in            efx_rxq_t *erp);
  84
  85 static  __checkReturn   uint8_t *
  86 siena_rx_qps_packet_info(
  87         __in            efx_rxq_t *erp,
  88         __in            uint8_t *buffer,
  89         __in            uint32_t buffer_length,
  90         __in            uint32_t current_offset,
  91         __out           uint16_t *lengthp,
  92         __out           uint32_t *next_offsetp,
  93         __out           uint32_t *timestamp);
  94 #endif
  95
  96 static  __checkReturn   efx_rc_t
  97 siena_rx_qflush(
  98         __in            efx_rxq_t *erp);
  99
 100 static                  void
 101 siena_rx_qenable(
 102         __in            efx_rxq_t *erp);
 103
 104 static  __checkReturn   efx_rc_t
 105 siena_rx_qcreate(
 106         __in            efx_nic_t *enp,
 107         __in            unsigned int index,
 108         __in            unsigned int label,
 109         __in            efx_rxq_type_t type,
 110         __in            uint32_t type_data,
 111         __in            efsys_mem_t *esmp,
 112         __in            size_t ndescs,
 113         __in            uint32_t id,
 114         __in            unsigned int flags,
 115         __in            efx_evq_t *eep,
 116         __in            efx_rxq_t *erp);
 117
 118 static                  void
 119 siena_rx_qdestroy(
 120         __in            efx_rxq_t *erp);
 121
 122 #endif /* EFSYS_OPT_SIENA */
 123
 124
 125 #if EFSYS_OPT_SIENA
 126 static const efx_rx_ops_t __efx_rx_siena_ops = {
 127         siena_rx_init,                          /* erxo_init */
 128         siena_rx_fini,                          /* erxo_fini */
 129 #if EFSYS_OPT_RX_SCATTER
 130         siena_rx_scatter_enable,                /* erxo_scatter_enable */
 131 #endif
 132 #if EFSYS_OPT_RX_SCALE
 133         NULL,                                   /* erxo_scale_context_alloc */
 134         NULL,                                   /* erxo_scale_context_free */
 135         siena_rx_scale_mode_set,                /* erxo_scale_mode_set */
 136         siena_rx_scale_key_set,                 /* erxo_scale_key_set */
 137         siena_rx_scale_tbl_set,                 /* erxo_scale_tbl_set */
 138         siena_rx_prefix_hash,                   /* erxo_prefix_hash */
 139 #endif
 140         siena_rx_prefix_pktlen,                 /* erxo_prefix_pktlen */
 141         siena_rx_qpost,                         /* erxo_qpost */
 142         siena_rx_qpush,                         /* erxo_qpush */
 143 #if EFSYS_OPT_RX_PACKED_STREAM
 144         siena_rx_qpush_ps_credits,              /* erxo_qpush_ps_credits */
 145         siena_rx_qps_packet_info,               /* erxo_qps_packet_info */
 146 #endif
 147         siena_rx_qflush,                        /* erxo_qflush */
 148         siena_rx_qenable,                       /* erxo_qenable */
 149         siena_rx_qcreate,                       /* erxo_qcreate */
 150         siena_rx_qdestroy,                      /* erxo_qdestroy */
 151 };
 152 #endif  /* EFSYS_OPT_SIENA */
 153
 154 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
 155 static const efx_rx_ops_t __efx_rx_ef10_ops = {
 156         ef10_rx_init,                           /* erxo_init */
 157         ef10_rx_fini,                           /* erxo_fini */
 158 #if EFSYS_OPT_RX_SCATTER
 159         ef10_rx_scatter_enable,                 /* erxo_scatter_enable */
 160 #endif
 161 #if EFSYS_OPT_RX_SCALE
 162         ef10_rx_scale_context_alloc,            /* erxo_scale_context_alloc */
 163         ef10_rx_scale_context_free,             /* erxo_scale_context_free */
 164         ef10_rx_scale_mode_set,                 /* erxo_scale_mode_set */
 165         ef10_rx_scale_key_set,                  /* erxo_scale_key_set */
 166         ef10_rx_scale_tbl_set,                  /* erxo_scale_tbl_set */
 167         ef10_rx_prefix_hash,                    /* erxo_prefix_hash */
 168 #endif
 169         ef10_rx_prefix_pktlen,                  /* erxo_prefix_pktlen */
 170         ef10_rx_qpost,                          /* erxo_qpost */
 171         ef10_rx_qpush,                          /* erxo_qpush */
 172 #if EFSYS_OPT_RX_PACKED_STREAM
 173         ef10_rx_qpush_ps_credits,               /* erxo_qpush_ps_credits */
 174         ef10_rx_qps_packet_info,                /* erxo_qps_packet_info */
 175 #endif
 176         ef10_rx_qflush,                         /* erxo_qflush */
 177         ef10_rx_qenable,                        /* erxo_qenable */
 178         ef10_rx_qcreate,                        /* erxo_qcreate */
 179         ef10_rx_qdestroy,                       /* erxo_qdestroy */
 180 };
 181 #endif  /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */
 182
 183
 184         __checkReturn   efx_rc_t
 185 efx_rx_init(
 186         __inout         efx_nic_t *enp)
 187 {
 188         const efx_rx_ops_t *erxop;
 189         efx_rc_t rc;
 190
 191         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 192         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
 193
 194         if (!(enp->en_mod_flags & EFX_MOD_EV)) {
 195                 rc = EINVAL;
 196                 goto fail1;
 197         }
 198
 199         if (enp->en_mod_flags & EFX_MOD_RX) {
 200                 rc = EINVAL;
 201                 goto fail2;
 202         }
 203
 204         switch (enp->en_family) {
 205 #if EFSYS_OPT_SIENA
 206         case EFX_FAMILY_SIENA:
 207                 erxop = &__efx_rx_siena_ops;
 208                 break;
 209 #endif /* EFSYS_OPT_SIENA */
 210
 211 #if EFSYS_OPT_HUNTINGTON
 212         case EFX_FAMILY_HUNTINGTON:
 213                 erxop = &__efx_rx_ef10_ops;
 214                 break;
 215 #endif /* EFSYS_OPT_HUNTINGTON */
 216
 217 #if EFSYS_OPT_MEDFORD
 218         case EFX_FAMILY_MEDFORD:
 219                 erxop = &__efx_rx_ef10_ops;
 220                 break;
 221 #endif /* EFSYS_OPT_MEDFORD */
 222
 223         default:
 224                 EFSYS_ASSERT(0);
 225                 rc = ENOTSUP;
 226                 goto fail3;
 227         }
 228
 229         if ((rc = erxop->erxo_init(enp)) != 0)
 230                 goto fail4;
 231
 232         enp->en_erxop = erxop;
 233         enp->en_mod_flags |= EFX_MOD_RX;
 234         return (0);
 235
 236 fail4:
 237         EFSYS_PROBE(fail4);
 238 fail3:
 239         EFSYS_PROBE(fail3);
 240 fail2:
 241         EFSYS_PROBE(fail2);
 242 fail1:
 243         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 244
 245         enp->en_erxop = NULL;
 246         enp->en_mod_flags &= ~EFX_MOD_RX;
 247         return (rc);
 248 }
 249
 250                         void
 251 efx_rx_fini(
 252         __in            efx_nic_t *enp)
 253 {
 254         const efx_rx_ops_t *erxop = enp->en_erxop;
 255
 256         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 257         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
 258         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 259         EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0);
 260
 261         erxop->erxo_fini(enp);
 262
 263         enp->en_erxop = NULL;
 264         enp->en_mod_flags &= ~EFX_MOD_RX;
 265 }
 266
 267 #if EFSYS_OPT_RX_SCATTER
 268         __checkReturn   efx_rc_t
 269 efx_rx_scatter_enable(
 270         __in            efx_nic_t *enp,
 271         __in            unsigned int buf_size)
 272 {
 273         const efx_rx_ops_t *erxop = enp->en_erxop;
 274         efx_rc_t rc;
 275
 276         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 277         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 278
 279         if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0)
 280                 goto fail1;
 281
 282         return (0);
 283
 284 fail1:
 285         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 286         return (rc);
 287 }
 288 #endif  /* EFSYS_OPT_RX_SCATTER */
 289
 290 #if EFSYS_OPT_RX_SCALE
 291         __checkReturn   efx_rc_t
 292 efx_rx_hash_default_support_get(
 293         __in            efx_nic_t *enp,
 294         __out           efx_rx_hash_support_t *supportp)
 295 {
 296         efx_rc_t rc;
 297
 298         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 299         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 300
 301         if (supportp == NULL) {
 302                 rc = EINVAL;
 303                 goto fail1;
 304         }
 305
 306         /*
 307          * Report the hashing support the client gets by default if it
 308          * does not allocate an RSS context itself.
 309          */
 310         *supportp = enp->en_hash_support;
 311
 312         return (0);
 313
 314 fail1:
 315         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 316
 317         return (rc);
 318 }
 319
 320         __checkReturn   efx_rc_t
 321 efx_rx_scale_default_support_get(
 322         __in            efx_nic_t *enp,
 323         __out           efx_rx_scale_context_type_t *typep)
 324 {
 325         efx_rc_t rc;
 326
 327         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 328         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 329
 330         if (typep == NULL) {
 331                 rc = EINVAL;
 332                 goto fail1;
 333         }
 334
 335         /*
 336          * Report the RSS support the client gets by default if it
 337          * does not allocate an RSS context itself.
 338          */
 339         *typep = enp->en_rss_context_type;
 340
 341         return (0);
 342
 343 fail1:
 344         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 345
 346         return (rc);
 347 }
 348 #endif  /* EFSYS_OPT_RX_SCALE */
 349
 350 #if EFSYS_OPT_RX_SCALE
 351         __checkReturn   efx_rc_t
 352 efx_rx_scale_context_alloc(
 353         __in            efx_nic_t *enp,
 354         __in            efx_rx_scale_context_type_t type,
 355         __in            uint32_t num_queues,
 356         __out           uint32_t *rss_contextp)
 357 {
 358         const efx_rx_ops_t *erxop = enp->en_erxop;
 359         efx_rc_t rc;
 360
 361         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 362         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 363
 364         if (erxop->erxo_scale_context_alloc == NULL) {
 365                 rc = ENOTSUP;
 366                 goto fail1;
 367         }
 368         if ((rc = erxop->erxo_scale_context_alloc(enp, type,
 369                             num_queues, rss_contextp)) != 0) {
 370                 goto fail2;
 371         }
 372
 373         return (0);
 374
 375 fail2:
 376         EFSYS_PROBE(fail2);
 377 fail1:
 378         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 379         return (rc);
 380 }
 381 #endif  /* EFSYS_OPT_RX_SCALE */
 382
 383 #if EFSYS_OPT_RX_SCALE
 384         __checkReturn   efx_rc_t
 385 efx_rx_scale_context_free(
 386         __in            efx_nic_t *enp,
 387         __in            uint32_t rss_context)
 388 {
 389         const efx_rx_ops_t *erxop = enp->en_erxop;
 390         efx_rc_t rc;
 391
 392         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 393         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 394
 395         if (erxop->erxo_scale_context_free == NULL) {
 396                 rc = ENOTSUP;
 397                 goto fail1;
 398         }
 399         if ((rc = erxop->erxo_scale_context_free(enp, rss_context)) != 0)
 400                 goto fail2;
 401
 402         return (0);
 403
 404 fail2:
 405         EFSYS_PROBE(fail2);
 406 fail1:
 407         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 408         return (rc);
 409 }
 410 #endif  /* EFSYS_OPT_RX_SCALE */
 411
 412 #if EFSYS_OPT_RX_SCALE
 413         __checkReturn   efx_rc_t
 414 efx_rx_scale_mode_set(
 415         __in            efx_nic_t *enp,
 416         __in            uint32_t rss_context,
 417         __in            efx_rx_hash_alg_t alg,
 418         __in            efx_rx_hash_type_t type,
 419         __in            boolean_t insert)
 420 {
 421         const efx_rx_ops_t *erxop = enp->en_erxop;
 422         efx_rc_t rc;
 423
 424         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 425         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 426
 427         if (erxop->erxo_scale_mode_set != NULL) {
 428                 if ((rc = erxop->erxo_scale_mode_set(enp, rss_context, alg,
 429                             type, insert)) != 0)
 430                         goto fail1;
 431         }
 432
 433         return (0);
 434
 435 fail1:
 436         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 437         return (rc);
 438 }
 439 #endif  /* EFSYS_OPT_RX_SCALE */
 440
 441 #if EFSYS_OPT_RX_SCALE
 442         __checkReturn   efx_rc_t
 443 efx_rx_scale_key_set(
 444         __in            efx_nic_t *enp,
 445         __in            uint32_t rss_context,
 446         __in_ecount(n)  uint8_t *key,
 447         __in            size_t n)
 448 {
 449         const efx_rx_ops_t *erxop = enp->en_erxop;
 450         efx_rc_t rc;
 451
 452         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 453         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 454
 455         if ((rc = erxop->erxo_scale_key_set(enp, rss_context, key, n)) != 0)
 456                 goto fail1;
 457
 458         return (0);
 459
 460 fail1:
 461         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 462
 463         return (rc);
 464 }
 465 #endif  /* EFSYS_OPT_RX_SCALE */
 466
 467 #if EFSYS_OPT_RX_SCALE
 468         __checkReturn   efx_rc_t
 469 efx_rx_scale_tbl_set(
 470         __in            efx_nic_t *enp,
 471         __in            uint32_t rss_context,
 472         __in_ecount(n)  unsigned int *table,
 473         __in            size_t n)
 474 {
 475         const efx_rx_ops_t *erxop = enp->en_erxop;
 476         efx_rc_t rc;
 477
 478         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 479         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 480
 481         if ((rc = erxop->erxo_scale_tbl_set(enp, rss_context, table, n)) != 0)
 482                 goto fail1;
 483
 484         return (0);
 485
 486 fail1:
 487         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 488
 489         return (rc);
 490 }
 491 #endif  /* EFSYS_OPT_RX_SCALE */
 492
 493                                 void
 494 efx_rx_qpost(
 495         __in                    efx_rxq_t *erp,
 496         __in_ecount(ndescs)     efsys_dma_addr_t *addrp,
 497         __in                    size_t size,
 498         __in                    unsigned int ndescs,
 499         __in                    unsigned int completed,
 500         __in                    unsigned int added)
 501 {
 502         efx_nic_t *enp = erp->er_enp;
 503         const efx_rx_ops_t *erxop = enp->en_erxop;
 504
 505         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 506
 507         erxop->erxo_qpost(erp, addrp, size, ndescs, completed, added);
 508 }
 509
 510 #if EFSYS_OPT_RX_PACKED_STREAM
 511
 512                         void
 513 efx_rx_qpush_ps_credits(
 514         __in            efx_rxq_t *erp)
 515 {
 516         efx_nic_t *enp = erp->er_enp;
 517         const efx_rx_ops_t *erxop = enp->en_erxop;
 518
 519         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 520
 521         erxop->erxo_qpush_ps_credits(erp);
 522 }
 523
 524         __checkReturn   uint8_t *
 525 efx_rx_qps_packet_info(
 526         __in            efx_rxq_t *erp,
 527         __in            uint8_t *buffer,
 528         __in            uint32_t buffer_length,
 529         __in            uint32_t current_offset,
 530         __out           uint16_t *lengthp,
 531         __out           uint32_t *next_offsetp,
 532         __out           uint32_t *timestamp)
 533 {
 534         efx_nic_t *enp = erp->er_enp;
 535         const efx_rx_ops_t *erxop = enp->en_erxop;
 536
 537         return (erxop->erxo_qps_packet_info(erp, buffer,
 538                 buffer_length, current_offset, lengthp,
 539                 next_offsetp, timestamp));
 540 }
 541
 542 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
 543
 544                         void
 545 efx_rx_qpush(
 546         __in            efx_rxq_t *erp,
 547         __in            unsigned int added,
 548         __inout         unsigned int *pushedp)
 549 {
 550         efx_nic_t *enp = erp->er_enp;
 551         const efx_rx_ops_t *erxop = enp->en_erxop;
 552
 553         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 554
 555         erxop->erxo_qpush(erp, added, pushedp);
 556 }
 557
 558         __checkReturn   efx_rc_t
 559 efx_rx_qflush(
 560         __in            efx_rxq_t *erp)
 561 {
 562         efx_nic_t *enp = erp->er_enp;
 563         const efx_rx_ops_t *erxop = enp->en_erxop;
 564         efx_rc_t rc;
 565
 566         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 567
 568         if ((rc = erxop->erxo_qflush(erp)) != 0)
 569                 goto fail1;
 570
 571         return (0);
 572
 573 fail1:
 574         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 575
 576         return (rc);
 577 }
 578
 579                         void
 580 efx_rx_qenable(
 581         __in            efx_rxq_t *erp)
 582 {
 583         efx_nic_t *enp = erp->er_enp;
 584         const efx_rx_ops_t *erxop = enp->en_erxop;
 585
 586         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 587
 588         erxop->erxo_qenable(erp);
 589 }
 590
 591 static  __checkReturn   efx_rc_t
 592 efx_rx_qcreate_internal(
 593         __in            efx_nic_t *enp,
 594         __in            unsigned int index,
 595         __in            unsigned int label,
 596         __in            efx_rxq_type_t type,
 597         __in            uint32_t type_data,
 598         __in            efsys_mem_t *esmp,
 599         __in            size_t ndescs,
 600         __in            uint32_t id,
 601         __in            unsigned int flags,
 602         __in            efx_evq_t *eep,
 603         __deref_out     efx_rxq_t **erpp)
 604 {
 605         const efx_rx_ops_t *erxop = enp->en_erxop;
 606         efx_rxq_t *erp;
 607         efx_rc_t rc;
 608
 609         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 610         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 611
 612         /* Allocate an RXQ object */
 613         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp);
 614
 615         if (erp == NULL) {
 616                 rc = ENOMEM;
 617                 goto fail1;
 618         }
 619
 620         erp->er_magic = EFX_RXQ_MAGIC;
 621         erp->er_enp = enp;
 622         erp->er_index = index;
 623         erp->er_mask = ndescs - 1;
 624         erp->er_esmp = esmp;
 625
 626         if ((rc = erxop->erxo_qcreate(enp, index, label, type, type_data, esmp,
 627             ndescs, id, flags, eep, erp)) != 0)
 628                 goto fail2;
 629
 630         enp->en_rx_qcount++;
 631         *erpp = erp;
 632
 633         return (0);
 634
 635 fail2:
 636         EFSYS_PROBE(fail2);
 637
 638         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
 639 fail1:
 640         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 641
 642         return (rc);
 643 }
 644
 645         __checkReturn   efx_rc_t
 646 efx_rx_qcreate(
 647         __in            efx_nic_t *enp,
 648         __in            unsigned int index,
 649         __in            unsigned int label,
 650         __in            efx_rxq_type_t type,
 651         __in            efsys_mem_t *esmp,
 652         __in            size_t ndescs,
 653         __in            uint32_t id,
 654         __in            unsigned int flags,
 655         __in            efx_evq_t *eep,
 656         __deref_out     efx_rxq_t **erpp)
 657 {
 658         return efx_rx_qcreate_internal(enp, index, label, type, 0, esmp, ndescs,
 659             id, flags, eep, erpp);
 660 }
 661
 662 #if EFSYS_OPT_RX_PACKED_STREAM
 663
 664         __checkReturn   efx_rc_t
 665 efx_rx_qcreate_packed_stream(
 666         __in            efx_nic_t *enp,
 667         __in            unsigned int index,
 668         __in            unsigned int label,
 669         __in            uint32_t ps_buf_size,
 670         __in            efsys_mem_t *esmp,
 671         __in            size_t ndescs,
 672         __in            efx_evq_t *eep,
 673         __deref_out     efx_rxq_t **erpp)
 674 {
 675         return efx_rx_qcreate_internal(enp, index, label,
 676             EFX_RXQ_TYPE_PACKED_STREAM, ps_buf_size, esmp, ndescs,
 677             0 /* id unused on EF10 */, EFX_RXQ_FLAG_NONE, eep, erpp);
 678 }
 679
 680 #endif
 681
 682                         void
 683 efx_rx_qdestroy(
 684         __in            efx_rxq_t *erp)
 685 {
 686         efx_nic_t *enp = erp->er_enp;
 687         const efx_rx_ops_t *erxop = enp->en_erxop;
 688
 689         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 690
 691         erxop->erxo_qdestroy(erp);
 692 }
 693
 694         __checkReturn   efx_rc_t
 695 efx_pseudo_hdr_pkt_length_get(
 696         __in            efx_rxq_t *erp,
 697         __in            uint8_t *buffer,
 698         __out           uint16_t *lengthp)
 699 {
 700         efx_nic_t *enp = erp->er_enp;
 701         const efx_rx_ops_t *erxop = enp->en_erxop;
 702
 703         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 704
 705         return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp));
 706 }
 707
 708 #if EFSYS_OPT_RX_SCALE
 709         __checkReturn   uint32_t
 710 efx_pseudo_hdr_hash_get(
 711         __in            efx_rxq_t *erp,
 712         __in            efx_rx_hash_alg_t func,
 713         __in            uint8_t *buffer)
 714 {
 715         efx_nic_t *enp = erp->er_enp;
 716         const efx_rx_ops_t *erxop = enp->en_erxop;
 717
 718         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 719
 720         EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE);
 721         return (erxop->erxo_prefix_hash(enp, func, buffer));
 722 }
 723 #endif  /* EFSYS_OPT_RX_SCALE */
 724
 725 #if EFSYS_OPT_SIENA
 726
 727 static  __checkReturn   efx_rc_t
 728 siena_rx_init(
 729         __in            efx_nic_t *enp)
 730 {
 731         efx_oword_t oword;
 732         unsigned int index;
 733
 734         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 735
 736         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0);
 737         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);
 738         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);
 739         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);
 740         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0);
 741         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32);
 742         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 743
 744         /* Zero the RSS table */
 745         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS;
 746             index++) {
 747                 EFX_ZERO_OWORD(oword);
 748                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
 749                                     index, &oword, B_TRUE);
 750         }
 751
 752 #if EFSYS_OPT_RX_SCALE
 753         /* The RSS key and indirection table are writable. */
 754         enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE;
 755
 756         /* Hardware can insert RX hash with/without RSS */
 757         enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
 758 #endif  /* EFSYS_OPT_RX_SCALE */
 759
 760         return (0);
 761 }
 762
 763 #if EFSYS_OPT_RX_SCATTER
 764 static  __checkReturn   efx_rc_t
 765 siena_rx_scatter_enable(
 766         __in            efx_nic_t *enp,
 767         __in            unsigned int buf_size)
 768 {
 769         unsigned int nbuf32;
 770         efx_oword_t oword;
 771         efx_rc_t rc;
 772
 773         nbuf32 = buf_size / 32;
 774         if ((nbuf32 == 0) ||
 775             (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) ||
 776             ((buf_size % 32) != 0)) {
 777                 rc = EINVAL;
 778                 goto fail1;
 779         }
 780
 781         if (enp->en_rx_qcount > 0) {
 782                 rc = EBUSY;
 783                 goto fail2;
 784         }
 785
 786         /* Set scatter buffer size */
 787         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 788         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32);
 789         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 790
 791         /* Enable scatter for packets not matching a filter */
 792         EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 793         EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1);
 794         EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 795
 796         return (0);
 797
 798 fail2:
 799         EFSYS_PROBE(fail2);
 800 fail1:
 801         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 802
 803         return (rc);
 804 }
 805 #endif  /* EFSYS_OPT_RX_SCATTER */
 806
 807
 808 #define EFX_RX_LFSR_HASH(_enp, _insert)                                 \
 809         do {                                                            \
 810                 efx_oword_t oword;                                      \
 811                                                                         \
 812                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);        \
 813                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);      \
 814                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);       \
 815                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);       \
 816                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 817                     (_insert) ? 1 : 0);                                 \
 818                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);       \
 819                                                                         \
 820                 if ((_enp)->en_family == EFX_FAMILY_SIENA) {            \
 821                         EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3,   \
 822                             &oword);                                    \
 823                         EFX_SET_OWORD_FIELD(oword,                      \
 824                             FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0);        \
 825                         EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3,  \
 826                             &oword);                                    \
 827                 }                                                       \
 828                                                                         \
 829                 _NOTE(CONSTANTCONDITION)                                \
 830         } while (B_FALSE)
 831
 832 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp)             \
 833         do {                                                            \
 834                 efx_oword_t oword;                                      \
 835                                                                         \
 836                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);        \
 837                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1);      \
 838                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH,           \
 839                     (_ip) ? 1 : 0);                                     \
 840                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP,           \
 841                     (_tcp) ? 0 : 1);                                    \
 842                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 843                     (_insert) ? 1 : 0);                                 \
 844                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);       \
 845                                                                         \
 846                 _NOTE(CONSTANTCONDITION)                                \
 847         } while (B_FALSE)
 848
 849 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc)                 \
 850         do {                                                            \
 851                 efx_oword_t oword;                                      \
 852                                                                         \
 853                 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword);  \
 854                 EFX_SET_OWORD_FIELD(oword,                              \
 855                     FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1);                \
 856                 EFX_SET_OWORD_FIELD(oword,                              \
 857                     FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \
 858                 EFX_SET_OWORD_FIELD(oword,                              \
 859                     FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1);   \
 860                 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
 861                                                                         \
 862                 (_rc) = 0;                                              \
 863                                                                         \
 864                 _NOTE(CONSTANTCONDITION)                                \
 865         } while (B_FALSE)
 866
 867
 868 #if EFSYS_OPT_RX_SCALE
 869
 870 static  __checkReturn   efx_rc_t
 871 siena_rx_scale_mode_set(
 872         __in            efx_nic_t *enp,
 873         __in            uint32_t rss_context,
 874         __in            efx_rx_hash_alg_t alg,
 875         __in            efx_rx_hash_type_t type,
 876         __in            boolean_t insert)
 877 {
 878         efx_rc_t rc;
 879
 880         if (rss_context != EFX_RSS_CONTEXT_DEFAULT) {
 881                 rc = EINVAL;
 882                 goto fail1;
 883         }
 884
 885         switch (alg) {
 886         case EFX_RX_HASHALG_LFSR:
 887                 EFX_RX_LFSR_HASH(enp, insert);
 888                 break;
 889
 890         case EFX_RX_HASHALG_TOEPLITZ:
 891                 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert,
 892                     type & EFX_RX_HASH_IPV4,
 893                     type & EFX_RX_HASH_TCPIPV4);
 894
 895                 EFX_RX_TOEPLITZ_IPV6_HASH(enp,
 896                     type & EFX_RX_HASH_IPV6,
 897                     type & EFX_RX_HASH_TCPIPV6,
 898                     rc);
 899                 if (rc != 0)
 900                         goto fail2;
 901
 902                 break;
 903
 904         default:
 905                 rc = EINVAL;
 906                 goto fail3;
 907         }
 908
 909         return (0);
 910
 911 fail3:
 912         EFSYS_PROBE(fail3);
 913 fail2:
 914         EFSYS_PROBE(fail2);
 915 fail1:
 916         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 917
 918         EFX_RX_LFSR_HASH(enp, B_FALSE);
 919
 920         return (rc);
 921 }
 922 #endif
 923
 924 #if EFSYS_OPT_RX_SCALE
 925 static  __checkReturn   efx_rc_t
 926 siena_rx_scale_key_set(
 927         __in            efx_nic_t *enp,
 928         __in            uint32_t rss_context,
 929         __in_ecount(n)  uint8_t *key,
 930         __in            size_t n)
 931 {
 932         efx_oword_t oword;
 933         unsigned int byte;
 934         unsigned int offset;
 935         efx_rc_t rc;
 936
 937         if (rss_context != EFX_RSS_CONTEXT_DEFAULT) {
 938                 rc = EINVAL;
 939                 goto fail1;
 940         }
 941
 942         byte = 0;
 943
 944         /* Write Toeplitz IPv4 hash key */
 945         EFX_ZERO_OWORD(oword);
 946         for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
 947             offset > 0 && byte < n;
 948             --offset)
 949                 oword.eo_u8[offset - 1] = key[byte++];
 950
 951         EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
 952
 953         byte = 0;
 954
 955         /* Verify Toeplitz IPv4 hash key */
 956         EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
 957         for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
 958             offset > 0 && byte < n;
 959             --offset) {
 960                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 961                         rc = EFAULT;
 962                         goto fail2;
 963                 }
 964         }
 965
 966         if ((enp->en_features & EFX_FEATURE_IPV6) == 0)
 967                 goto done;
 968
 969         byte = 0;
 970
 971         /* Write Toeplitz IPv6 hash key 3 */
 972         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 973         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
 974             FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
 975             offset > 0 && byte < n;
 976             --offset)
 977                 oword.eo_u8[offset - 1] = key[byte++];
 978
 979         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 980
 981         /* Write Toeplitz IPv6 hash key 2 */
 982         EFX_ZERO_OWORD(oword);
 983         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
 984             FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
 985             offset > 0 && byte < n;
 986             --offset)
 987                 oword.eo_u8[offset - 1] = key[byte++];
 988
 989         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
 990
 991         /* Write Toeplitz IPv6 hash key 1 */
 992         EFX_ZERO_OWORD(oword);
 993         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
 994             FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
 995             offset > 0 && byte < n;
 996             --offset)
 997                 oword.eo_u8[offset - 1] = key[byte++];
 998
 999         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
1000
1001         byte = 0;
1002
1003         /* Verify Toeplitz IPv6 hash key 3 */
1004         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
1005         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
1006             FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
1007             offset > 0 && byte < n;
1008             --offset) {
1009                 if (oword.eo_u8[offset - 1] != key[byte++]) {
1010                         rc = EFAULT;
1011                         goto fail3;
1012                 }
1013         }
1014
1015         /* Verify Toeplitz IPv6 hash key 2 */
1016         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
1017         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
1018             FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
1019             offset > 0 && byte < n;
1020             --offset) {
1021                 if (oword.eo_u8[offset - 1] != key[byte++]) {
1022                         rc = EFAULT;
1023                         goto fail4;
1024                 }
1025         }
1026
1027         /* Verify Toeplitz IPv6 hash key 1 */
1028         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
1029         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
1030             FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
1031             offset > 0 && byte < n;
1032             --offset) {
1033                 if (oword.eo_u8[offset - 1] != key[byte++]) {
1034                         rc = EFAULT;
1035                         goto fail5;
1036                 }
1037         }
1038
1039 done:
1040         return (0);
1041
1042 fail5:
1043         EFSYS_PROBE(fail5);
1044 fail4:
1045         EFSYS_PROBE(fail4);
1046 fail3:
1047         EFSYS_PROBE(fail3);
1048 fail2:
1049         EFSYS_PROBE(fail2);
1050 fail1:
1051         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1052
1053         return (rc);
1054 }
1055 #endif
1056
1057 #if EFSYS_OPT_RX_SCALE
1058 static  __checkReturn   efx_rc_t
1059 siena_rx_scale_tbl_set(
1060         __in            efx_nic_t *enp,
1061         __in            uint32_t rss_context,
1062         __in_ecount(n)  unsigned int *table,
1063         __in            size_t n)
1064 {
1065         efx_oword_t oword;
1066         int index;
1067         efx_rc_t rc;
1068
1069         EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS);
1070         EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH));
1071
1072         if (rss_context != EFX_RSS_CONTEXT_DEFAULT) {
1073                 rc = EINVAL;
1074                 goto fail1;
1075         }
1076
1077         if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) {
1078                 rc = EINVAL;
1079                 goto fail2;
1080         }
1081
1082         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) {
1083                 uint32_t byte;
1084
1085                 /* Calculate the entry to place in the table */
1086                 byte = (n > 0) ? (uint32_t)table[index % n] : 0;
1087
1088                 EFSYS_PROBE2(table, int, index, uint32_t, byte);
1089
1090                 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte);
1091
1092                 /* Write the table */
1093                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
1094                                     index, &oword, B_TRUE);
1095         }
1096
1097         for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) {
1098                 uint32_t byte;
1099
1100                 /* Determine if we're starting a new batch */
1101                 byte = (n > 0) ? (uint32_t)table[index % n] : 0;
1102
1103                 /* Read the table */
1104                 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL,
1105                                     index, &oword, B_TRUE);
1106
1107                 /* Verify the entry */
1108                 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) {
1109                         rc = EFAULT;
1110                         goto fail3;
1111                 }
1112         }
1113
1114         return (0);
1115
1116 fail3:
1117         EFSYS_PROBE(fail3);
1118 fail2:
1119         EFSYS_PROBE(fail2);
1120 fail1:
1121         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1122
1123         return (rc);
1124 }
1125 #endif
1126
1127 /*
1128  * Falcon/Siena pseudo-header
1129  * --------------------------
1130  *
1131  * Receive packets are prefixed by an optional 16 byte pseudo-header.
1132  * The pseudo-header is a byte array of one of the forms:
1133  *
1134  *  0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15
1135  * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT
1136  * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL
1137  *
1138  * where:
1139  *   TT.TT.TT.TT   Toeplitz hash (32-bit big-endian)
1140  *   LL.LL         LFSR hash     (16-bit big-endian)
1141  */
1142
1143 #if EFSYS_OPT_RX_SCALE
1144 static  __checkReturn   uint32_t
1145 siena_rx_prefix_hash(
1146         __in            efx_nic_t *enp,
1147         __in            efx_rx_hash_alg_t func,
1148         __in            uint8_t *buffer)
1149 {
1150         _NOTE(ARGUNUSED(enp))
1151
1152         switch (func) {
1153         case EFX_RX_HASHALG_TOEPLITZ:
1154                 return ((buffer[12] << 24) |
1155                     (buffer[13] << 16) |
1156                     (buffer[14] <<  8) |
1157                     buffer[15]);
1158
1159         case EFX_RX_HASHALG_LFSR:
1160                 return ((buffer[14] << 8) | buffer[15]);
1161
1162         default:
1163                 EFSYS_ASSERT(0);
1164                 return (0);
1165         }
1166 }
1167 #endif /* EFSYS_OPT_RX_SCALE */
1168
1169 static  __checkReturn   efx_rc_t
1170 siena_rx_prefix_pktlen(
1171         __in            efx_nic_t *enp,
1172         __in            uint8_t *buffer,
1173         __out           uint16_t *lengthp)
1174 {
1175         _NOTE(ARGUNUSED(enp, buffer, lengthp))
1176
1177         /* Not supported by Falcon/Siena hardware */
1178         EFSYS_ASSERT(0);
1179         return (ENOTSUP);
1180 }
1181
1182
1183 static                          void
1184 siena_rx_qpost(
1185         __in                    efx_rxq_t *erp,
1186         __in_ecount(ndescs)     efsys_dma_addr_t *addrp,
1187         __in                    size_t size,
1188         __in                    unsigned int ndescs,
1189         __in                    unsigned int completed,
1190         __in                    unsigned int added)
1191 {
1192         efx_qword_t qword;
1193         unsigned int i;
1194         unsigned int offset;
1195         unsigned int id;
1196
1197         /* The client driver must not overfill the queue */
1198         EFSYS_ASSERT3U(added - completed + ndescs, <=,
1199             EFX_RXQ_LIMIT(erp->er_mask + 1));
1200
1201         id = added & (erp->er_mask);
1202         for (i = 0; i < ndescs; i++) {
1203                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
1204                     unsigned int, id, efsys_dma_addr_t, addrp[i],
1205                     size_t, size);
1206
1207                 EFX_POPULATE_QWORD_3(qword,
1208                     FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size),
1209                     FSF_AZ_RX_KER_BUF_ADDR_DW0,
1210                     (uint32_t)(addrp[i] & 0xffffffff),
1211                     FSF_AZ_RX_KER_BUF_ADDR_DW1,
1212                     (uint32_t)(addrp[i] >> 32));
1213
1214                 offset = id * sizeof (efx_qword_t);
1215                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
1216
1217                 id = (id + 1) & (erp->er_mask);
1218         }
1219 }
1220
1221 static                  void
1222 siena_rx_qpush(
1223         __in    efx_rxq_t *erp,
1224         __in    unsigned int added,
1225         __inout unsigned int *pushedp)
1226 {
1227         efx_nic_t *enp = erp->er_enp;
1228         unsigned int pushed = *pushedp;
1229         uint32_t wptr;
1230         efx_oword_t oword;
1231         efx_dword_t dword;
1232
1233         /* All descriptors are pushed */
1234         *pushedp = added;
1235
1236         /* Push the populated descriptors out */
1237         wptr = added & erp->er_mask;
1238
1239         EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr);
1240
1241         /* Only write the third DWORD */
1242         EFX_POPULATE_DWORD_1(dword,
1243             EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
1244
1245         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
1246         EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
1247             wptr, pushed & erp->er_mask);
1248         EFSYS_PIO_WRITE_BARRIER();
1249         EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0,
1250                             erp->er_index, &dword, B_FALSE);
1251 }
1252
1253 #if EFSYS_OPT_RX_PACKED_STREAM
1254 static          void
1255 siena_rx_qpush_ps_credits(
1256         __in            efx_rxq_t *erp)
1257 {
1258         /* Not supported by Siena hardware */
1259         EFSYS_ASSERT(0);
1260 }
1261
1262 static          uint8_t *
1263 siena_rx_qps_packet_info(
1264         __in            efx_rxq_t *erp,
1265         __in            uint8_t *buffer,
1266         __in            uint32_t buffer_length,
1267         __in            uint32_t current_offset,
1268         __out           uint16_t *lengthp,
1269         __out           uint32_t *next_offsetp,
1270         __out           uint32_t *timestamp)
1271 {
1272         /* Not supported by Siena hardware */
1273         EFSYS_ASSERT(0);
1274
1275         return (NULL);
1276 }
1277 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1278
1279 static  __checkReturn   efx_rc_t
1280 siena_rx_qflush(
1281         __in    efx_rxq_t *erp)
1282 {
1283         efx_nic_t *enp = erp->er_enp;
1284         efx_oword_t oword;
1285         uint32_t label;
1286
1287         label = erp->er_index;
1288
1289         /* Flush the queue */
1290         EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
1291             FRF_AZ_RX_FLUSH_DESCQ, label);
1292         EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword);
1293
1294         return (0);
1295 }
1296
1297 static          void
1298 siena_rx_qenable(
1299         __in    efx_rxq_t *erp)
1300 {
1301         efx_nic_t *enp = erp->er_enp;
1302         efx_oword_t oword;
1303
1304         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1305
1306         EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL,
1307                             erp->er_index, &oword, B_TRUE);
1308
1309         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0);
1310         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0);
1311         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1);
1312
1313         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1314                             erp->er_index, &oword, B_TRUE);
1315 }
1316
1317 static  __checkReturn   efx_rc_t
1318 siena_rx_qcreate(
1319         __in            efx_nic_t *enp,
1320         __in            unsigned int index,
1321         __in            unsigned int label,
1322         __in            efx_rxq_type_t type,
1323         __in            uint32_t type_data,
1324         __in            efsys_mem_t *esmp,
1325         __in            size_t ndescs,
1326         __in            uint32_t id,
1327         __in            unsigned int flags,
1328         __in            efx_evq_t *eep,
1329         __in            efx_rxq_t *erp)
1330 {
1331         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1332         efx_oword_t oword;
1333         uint32_t size;
1334         boolean_t jumbo = B_FALSE;
1335         efx_rc_t rc;
1336
1337         _NOTE(ARGUNUSED(esmp))
1338         _NOTE(ARGUNUSED(type_data))
1339
1340         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS ==
1341             (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH));
1342         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
1343         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
1344
1345         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
1346         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
1347
1348         if (!ISP2(ndescs) ||
1349             (ndescs < EFX_RXQ_MINNDESCS) || (ndescs > EFX_RXQ_MAXNDESCS)) {
1350                 rc = EINVAL;
1351                 goto fail1;
1352         }
1353         if (index >= encp->enc_rxq_limit) {
1354                 rc = EINVAL;
1355                 goto fail2;
1356         }
1357         for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS);
1358             size++)
1359                 if ((1 << size) == (int)(ndescs / EFX_RXQ_MINNDESCS))
1360                         break;
1361         if (id + (1 << size) >= encp->enc_buftbl_limit) {
1362                 rc = EINVAL;
1363                 goto fail3;
1364         }
1365
1366         switch (type) {
1367         case EFX_RXQ_TYPE_DEFAULT:
1368                 break;
1369
1370         default:
1371                 rc = EINVAL;
1372                 goto fail4;
1373         }
1374
1375         if (flags & EFX_RXQ_FLAG_SCATTER) {
1376 #if EFSYS_OPT_RX_SCATTER
1377                 jumbo = B_TRUE;
1378 #else
1379                 rc = EINVAL;
1380                 goto fail5;
1381 #endif  /* EFSYS_OPT_RX_SCATTER */
1382         }
1383
1384         /* Set up the new descriptor queue */
1385         EFX_POPULATE_OWORD_7(oword,
1386             FRF_AZ_RX_DESCQ_BUF_BASE_ID, id,
1387             FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index,
1388             FRF_AZ_RX_DESCQ_OWNER_ID, 0,
1389             FRF_AZ_RX_DESCQ_LABEL, label,
1390             FRF_AZ_RX_DESCQ_SIZE, size,
1391             FRF_AZ_RX_DESCQ_TYPE, 0,
1392             FRF_AZ_RX_DESCQ_JUMBO, jumbo);
1393
1394         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1395                             erp->er_index, &oword, B_TRUE);
1396
1397         return (0);
1398
1399 #if !EFSYS_OPT_RX_SCATTER
1400 fail5:
1401         EFSYS_PROBE(fail5);
1402 #endif
1403 fail4:
1404         EFSYS_PROBE(fail4);
1405 fail3:
1406         EFSYS_PROBE(fail3);
1407 fail2:
1408         EFSYS_PROBE(fail2);
1409 fail1:
1410         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1411
1412         return (rc);
1413 }
1414
1415 static          void
1416 siena_rx_qdestroy(
1417         __in    efx_rxq_t *erp)
1418 {
1419         efx_nic_t *enp = erp->er_enp;
1420         efx_oword_t oword;
1421
1422         EFSYS_ASSERT(enp->en_rx_qcount != 0);
1423         --enp->en_rx_qcount;
1424
1425         /* Purge descriptor queue */
1426         EFX_ZERO_OWORD(oword);
1427
1428         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1429                             erp->er_index, &oword, B_TRUE);
1430
1431         /* Free the RXQ object */
1432         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1433 }
1434
1435 static          void
1436 siena_rx_fini(
1437         __in    efx_nic_t *enp)
1438 {
1439         _NOTE(ARGUNUSED(enp))
1440 }
1441
1442 #endif /* EFSYS_OPT_SIENA */