drivers/net/sfc/base/ef10_rx.c

   1 /*
   2  * Copyright (c) 2012-2016 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright notice,
  11  *    this list of conditions and the following disclaimer in the documentation
  12  *    and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * The views and conclusions contained in the software and documentation are
  27  * those of the authors and should not be interpreted as representing official
  28  * policies, either expressed or implied, of the FreeBSD Project.
  29  */
  30
  31 #include "efx.h"
  32 #include "efx_impl.h"
  33
  34
  35 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
  36
  37
  38 static  __checkReturn   efx_rc_t
  39 efx_mcdi_init_rxq(
  40         __in            efx_nic_t *enp,
  41         __in            uint32_t size,
  42         __in            uint32_t target_evq,
  43         __in            uint32_t label,
  44         __in            uint32_t instance,
  45         __in            efsys_mem_t *esmp,
  46         __in            boolean_t disable_scatter,
  47         __in            uint32_t ps_bufsize)
  48 {
  49         efx_mcdi_req_t req;
  50         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_INIT_RXQ_EXT_IN_LEN,
  51                 MC_CMD_INIT_RXQ_EXT_OUT_LEN);
  52         int npages = EFX_RXQ_NBUFS(size);
  53         int i;
  54         efx_qword_t *dma_addr;
  55         uint64_t addr;
  56         efx_rc_t rc;
  57         uint32_t dma_mode;
  58
  59         /* If this changes, then the payload size might need to change. */
  60         EFSYS_ASSERT3U(MC_CMD_INIT_RXQ_OUT_LEN, ==, 0);
  61         EFSYS_ASSERT3U(size, <=, EFX_RXQ_MAXNDESCS);
  62
  63         if (ps_bufsize > 0)
  64                 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_PACKED_STREAM;
  65         else
  66                 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_SINGLE_PACKET;
  67
  68         req.emr_cmd = MC_CMD_INIT_RXQ;
  69         req.emr_in_buf = payload;
  70         req.emr_in_length = MC_CMD_INIT_RXQ_EXT_IN_LEN;
  71         req.emr_out_buf = payload;
  72         req.emr_out_length = MC_CMD_INIT_RXQ_EXT_OUT_LEN;
  73
  74         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_SIZE, size);
  75         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_TARGET_EVQ, target_evq);
  76         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_LABEL, label);
  77         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_INSTANCE, instance);
  78         MCDI_IN_POPULATE_DWORD_8(req, INIT_RXQ_EXT_IN_FLAGS,
  79             INIT_RXQ_EXT_IN_FLAG_BUFF_MODE, 0,
  80             INIT_RXQ_EXT_IN_FLAG_HDR_SPLIT, 0,
  81             INIT_RXQ_EXT_IN_FLAG_TIMESTAMP, 0,
  82             INIT_RXQ_EXT_IN_CRC_MODE, 0,
  83             INIT_RXQ_EXT_IN_FLAG_PREFIX, 1,
  84             INIT_RXQ_EXT_IN_FLAG_DISABLE_SCATTER, disable_scatter,
  85             INIT_RXQ_EXT_IN_DMA_MODE,
  86             dma_mode,
  87             INIT_RXQ_EXT_IN_PACKED_STREAM_BUFF_SIZE, ps_bufsize);
  88         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_OWNER_ID, 0);
  89         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);
  90
  91         dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR);
  92         addr = EFSYS_MEM_ADDR(esmp);
  93
  94         for (i = 0; i < npages; i++) {
  95                 EFX_POPULATE_QWORD_2(*dma_addr,
  96                     EFX_DWORD_1, (uint32_t)(addr >> 32),
  97                     EFX_DWORD_0, (uint32_t)(addr & 0xffffffff));
  98
  99                 dma_addr++;
 100                 addr += EFX_BUF_SIZE;
 101         }
 102
 103         efx_mcdi_execute(enp, &req);
 104
 105         if (req.emr_rc != 0) {
 106                 rc = req.emr_rc;
 107                 goto fail1;
 108         }
 109
 110         return (0);
 111
 112 fail1:
 113         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 114
 115         return (rc);
 116 }
 117
 118 static  __checkReturn   efx_rc_t
 119 efx_mcdi_fini_rxq(
 120         __in            efx_nic_t *enp,
 121         __in            uint32_t instance)
 122 {
 123         efx_mcdi_req_t req;
 124         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_FINI_RXQ_IN_LEN,
 125                 MC_CMD_FINI_RXQ_OUT_LEN);
 126         efx_rc_t rc;
 127
 128         req.emr_cmd = MC_CMD_FINI_RXQ;
 129         req.emr_in_buf = payload;
 130         req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN;
 131         req.emr_out_buf = payload;
 132         req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN;
 133
 134         MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance);
 135
 136         efx_mcdi_execute_quiet(enp, &req);
 137
 138         if (req.emr_rc != 0) {
 139                 rc = req.emr_rc;
 140                 goto fail1;
 141         }
 142
 143         return (0);
 144
 145 fail1:
 146         /*
 147          * EALREADY is not an error, but indicates that the MC has rebooted and
 148          * that the RXQ has already been destroyed.
 149          */
 150         if (rc != EALREADY)
 151                 EFSYS_PROBE1(fail1, efx_rc_t, rc);
 152
 153         return (rc);
 154 }
 155
 156 #if EFSYS_OPT_RX_SCALE
 157 static  __checkReturn   efx_rc_t
 158 efx_mcdi_rss_context_alloc(
 159         __in            efx_nic_t *enp,
 160         __in            efx_rx_scale_context_type_t type,
 161         __in            uint32_t num_queues,
 162         __out           uint32_t *rss_contextp)
 163 {
 164         efx_mcdi_req_t req;
 165         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN,
 166                 MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN);
 167         uint32_t rss_context;
 168         uint32_t context_type;
 169         efx_rc_t rc;
 170
 171         if (num_queues > EFX_MAXRSS) {
 172                 rc = EINVAL;
 173                 goto fail1;
 174         }
 175
 176         switch (type) {
 177         case EFX_RX_SCALE_EXCLUSIVE:
 178                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE;
 179                 break;
 180         case EFX_RX_SCALE_SHARED:
 181                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED;
 182                 break;
 183         default:
 184                 rc = EINVAL;
 185                 goto fail2;
 186         }
 187
 188         req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC;
 189         req.emr_in_buf = payload;
 190         req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN;
 191         req.emr_out_buf = payload;
 192         req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN;
 193
 194         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
 195             EVB_PORT_ID_ASSIGNED);
 196         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type);
 197         /* NUM_QUEUES is only used to validate indirection table offsets */
 198         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues);
 199
 200         efx_mcdi_execute(enp, &req);
 201
 202         if (req.emr_rc != 0) {
 203                 rc = req.emr_rc;
 204                 goto fail3;
 205         }
 206
 207         if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) {
 208                 rc = EMSGSIZE;
 209                 goto fail4;
 210         }
 211
 212         rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);
 213         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 214                 rc = ENOENT;
 215                 goto fail5;
 216         }
 217
 218         *rss_contextp = rss_context;
 219
 220         return (0);
 221
 222 fail5:
 223         EFSYS_PROBE(fail5);
 224 fail4:
 225         EFSYS_PROBE(fail4);
 226 fail3:
 227         EFSYS_PROBE(fail3);
 228 fail2:
 229         EFSYS_PROBE(fail2);
 230 fail1:
 231         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 232
 233         return (rc);
 234 }
 235 #endif /* EFSYS_OPT_RX_SCALE */
 236
 237 #if EFSYS_OPT_RX_SCALE
 238 static                  efx_rc_t
 239 efx_mcdi_rss_context_free(
 240         __in            efx_nic_t *enp,
 241         __in            uint32_t rss_context)
 242 {
 243         efx_mcdi_req_t req;
 244         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_FREE_IN_LEN,
 245                 MC_CMD_RSS_CONTEXT_FREE_OUT_LEN);
 246         efx_rc_t rc;
 247
 248         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 249                 rc = EINVAL;
 250                 goto fail1;
 251         }
 252
 253         req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE;
 254         req.emr_in_buf = payload;
 255         req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN;
 256         req.emr_out_buf = payload;
 257         req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN;
 258
 259         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context);
 260
 261         efx_mcdi_execute_quiet(enp, &req);
 262
 263         if (req.emr_rc != 0) {
 264                 rc = req.emr_rc;
 265                 goto fail2;
 266         }
 267
 268         return (0);
 269
 270 fail2:
 271         EFSYS_PROBE(fail2);
 272 fail1:
 273         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 274
 275         return (rc);
 276 }
 277 #endif /* EFSYS_OPT_RX_SCALE */
 278
 279 #if EFSYS_OPT_RX_SCALE
 280 static                  efx_rc_t
 281 efx_mcdi_rss_context_set_flags(
 282         __in            efx_nic_t *enp,
 283         __in            uint32_t rss_context,
 284         __in            efx_rx_hash_type_t type)
 285 {
 286         efx_mcdi_req_t req;
 287         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN,
 288                 MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN);
 289         efx_rc_t rc;
 290
 291         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 292                 rc = EINVAL;
 293                 goto fail1;
 294         }
 295
 296         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS;
 297         req.emr_in_buf = payload;
 298         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN;
 299         req.emr_out_buf = payload;
 300         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN;
 301
 302         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
 303             rss_context);
 304
 305         MCDI_IN_POPULATE_DWORD_4(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS,
 306             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN,
 307             (type & EFX_RX_HASH_IPV4) ? 1 : 0,
 308             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN,
 309             (type & EFX_RX_HASH_TCPIPV4) ? 1 : 0,
 310             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN,
 311             (type & EFX_RX_HASH_IPV6) ? 1 : 0,
 312             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN,
 313             (type & EFX_RX_HASH_TCPIPV6) ? 1 : 0);
 314
 315         efx_mcdi_execute(enp, &req);
 316
 317         if (req.emr_rc != 0) {
 318                 rc = req.emr_rc;
 319                 goto fail2;
 320         }
 321
 322         return (0);
 323
 324 fail2:
 325         EFSYS_PROBE(fail2);
 326 fail1:
 327         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 328
 329         return (rc);
 330 }
 331 #endif /* EFSYS_OPT_RX_SCALE */
 332
 333 #if EFSYS_OPT_RX_SCALE
 334 static                  efx_rc_t
 335 efx_mcdi_rss_context_set_key(
 336         __in            efx_nic_t *enp,
 337         __in            uint32_t rss_context,
 338         __in_ecount(n)  uint8_t *key,
 339         __in            size_t n)
 340 {
 341         efx_mcdi_req_t req;
 342         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN,
 343                 MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN);
 344         efx_rc_t rc;
 345
 346         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 347                 rc = EINVAL;
 348                 goto fail1;
 349         }
 350
 351         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY;
 352         req.emr_in_buf = payload;
 353         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN;
 354         req.emr_out_buf = payload;
 355         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN;
 356
 357         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
 358             rss_context);
 359
 360         EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
 361         if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) {
 362                 rc = EINVAL;
 363                 goto fail2;
 364         }
 365
 366         memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY),
 367             key, n);
 368
 369         efx_mcdi_execute(enp, &req);
 370
 371         if (req.emr_rc != 0) {
 372                 rc = req.emr_rc;
 373                 goto fail3;
 374         }
 375
 376         return (0);
 377
 378 fail3:
 379         EFSYS_PROBE(fail3);
 380 fail2:
 381         EFSYS_PROBE(fail2);
 382 fail1:
 383         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 384
 385         return (rc);
 386 }
 387 #endif /* EFSYS_OPT_RX_SCALE */
 388
 389 #if EFSYS_OPT_RX_SCALE
 390 static                  efx_rc_t
 391 efx_mcdi_rss_context_set_table(
 392         __in            efx_nic_t *enp,
 393         __in            uint32_t rss_context,
 394         __in_ecount(n)  unsigned int *table,
 395         __in            size_t n)
 396 {
 397         efx_mcdi_req_t req;
 398         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN,
 399                 MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN);
 400         uint8_t *req_table;
 401         int i, rc;
 402
 403         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 404                 rc = EINVAL;
 405                 goto fail1;
 406         }
 407
 408         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE;
 409         req.emr_in_buf = payload;
 410         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN;
 411         req.emr_out_buf = payload;
 412         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN;
 413
 414         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
 415             rss_context);
 416
 417         req_table =
 418             MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE);
 419
 420         for (i = 0;
 421             i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN;
 422             i++) {
 423                 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0;
 424         }
 425
 426         efx_mcdi_execute(enp, &req);
 427
 428         if (req.emr_rc != 0) {
 429                 rc = req.emr_rc;
 430                 goto fail2;
 431         }
 432
 433         return (0);
 434
 435 fail2:
 436         EFSYS_PROBE(fail2);
 437 fail1:
 438         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 439
 440         return (rc);
 441 }
 442 #endif /* EFSYS_OPT_RX_SCALE */
 443
 444
 445         __checkReturn   efx_rc_t
 446 ef10_rx_init(
 447         __in            efx_nic_t *enp)
 448 {
 449 #if EFSYS_OPT_RX_SCALE
 450
 451         if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS,
 452                 &enp->en_rss_context) == 0) {
 453                 /*
 454                  * Allocated an exclusive RSS context, which allows both the
 455                  * indirection table and key to be modified.
 456                  */
 457                 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE;
 458                 enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
 459         } else {
 460                 /*
 461                  * Failed to allocate an exclusive RSS context. Continue
 462                  * operation without support for RSS. The pseudo-header in
 463                  * received packets will not contain a Toeplitz hash value.
 464                  */
 465                 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
 466                 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE;
 467         }
 468
 469 #endif /* EFSYS_OPT_RX_SCALE */
 470
 471         return (0);
 472 }
 473
 474 #if EFSYS_OPT_RX_SCATTER
 475         __checkReturn   efx_rc_t
 476 ef10_rx_scatter_enable(
 477         __in            efx_nic_t *enp,
 478         __in            unsigned int buf_size)
 479 {
 480         _NOTE(ARGUNUSED(enp, buf_size))
 481         return (0);
 482 }
 483 #endif  /* EFSYS_OPT_RX_SCATTER */
 484
 485 #if EFSYS_OPT_RX_SCALE
 486         __checkReturn   efx_rc_t
 487 ef10_rx_scale_context_alloc(
 488         __in            efx_nic_t *enp,
 489         __in            efx_rx_scale_context_type_t type,
 490         __in            uint32_t num_queues,
 491         __out           uint32_t *rss_contextp)
 492 {
 493         efx_rc_t rc;
 494
 495         rc = efx_mcdi_rss_context_alloc(enp, type, num_queues, rss_contextp);
 496         if (rc != 0)
 497                 goto fail1;
 498
 499         return (0);
 500
 501 fail1:
 502         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 503         return (rc);
 504 }
 505 #endif /* EFSYS_OPT_RX_SCALE */
 506
 507 #if EFSYS_OPT_RX_SCALE
 508         __checkReturn   efx_rc_t
 509 ef10_rx_scale_context_free(
 510         __in            efx_nic_t *enp,
 511         __in            uint32_t rss_context)
 512 {
 513         efx_rc_t rc;
 514
 515         rc = efx_mcdi_rss_context_free(enp, rss_context);
 516         if (rc != 0)
 517                 goto fail1;
 518
 519         return (0);
 520
 521 fail1:
 522         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 523         return (rc);
 524 }
 525 #endif /* EFSYS_OPT_RX_SCALE */
 526
 527 #if EFSYS_OPT_RX_SCALE
 528         __checkReturn   efx_rc_t
 529 ef10_rx_scale_mode_set(
 530         __in            efx_nic_t *enp,
 531         __in            uint32_t rss_context,
 532         __in            efx_rx_hash_alg_t alg,
 533         __in            efx_rx_hash_type_t type,
 534         __in            boolean_t insert)
 535 {
 536         efx_rc_t rc;
 537
 538         EFSYS_ASSERT3U(alg, ==, EFX_RX_HASHALG_TOEPLITZ);
 539         EFSYS_ASSERT3U(insert, ==, B_TRUE);
 540
 541         if ((alg != EFX_RX_HASHALG_TOEPLITZ) || (insert == B_FALSE)) {
 542                 rc = EINVAL;
 543                 goto fail1;
 544         }
 545
 546         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 547                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 548                         rc = ENOTSUP;
 549                         goto fail2;
 550                 }
 551                 rss_context = enp->en_rss_context;
 552         }
 553
 554         if ((rc = efx_mcdi_rss_context_set_flags(enp,
 555                     rss_context, type)) != 0)
 556                 goto fail3;
 557
 558         return (0);
 559
 560 fail3:
 561         EFSYS_PROBE(fail3);
 562 fail2:
 563         EFSYS_PROBE(fail2);
 564 fail1:
 565         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 566
 567         return (rc);
 568 }
 569 #endif /* EFSYS_OPT_RX_SCALE */
 570
 571 #if EFSYS_OPT_RX_SCALE
 572         __checkReturn   efx_rc_t
 573 ef10_rx_scale_key_set(
 574         __in            efx_nic_t *enp,
 575         __in            uint32_t rss_context,
 576         __in_ecount(n)  uint8_t *key,
 577         __in            size_t n)
 578 {
 579         efx_rc_t rc;
 580
 581         EFX_STATIC_ASSERT(EFX_RSS_KEY_SIZE ==
 582             MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
 583
 584         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 585                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 586                         rc = ENOTSUP;
 587                         goto fail1;
 588                 }
 589                 rss_context = enp->en_rss_context;
 590         }
 591
 592         if ((rc = efx_mcdi_rss_context_set_key(enp, rss_context, key, n)) != 0)
 593                 goto fail2;
 594
 595         return (0);
 596
 597 fail2:
 598         EFSYS_PROBE(fail2);
 599 fail1:
 600         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 601
 602         return (rc);
 603 }
 604 #endif /* EFSYS_OPT_RX_SCALE */
 605
 606 #if EFSYS_OPT_RX_SCALE
 607         __checkReturn   efx_rc_t
 608 ef10_rx_scale_tbl_set(
 609         __in            efx_nic_t *enp,
 610         __in            uint32_t rss_context,
 611         __in_ecount(n)  unsigned int *table,
 612         __in            size_t n)
 613 {
 614         efx_rc_t rc;
 615
 616
 617         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 618                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 619                         rc = ENOTSUP;
 620                         goto fail1;
 621                 }
 622                 rss_context = enp->en_rss_context;
 623         }
 624
 625         if ((rc = efx_mcdi_rss_context_set_table(enp,
 626                     rss_context, table, n)) != 0)
 627                 goto fail2;
 628
 629         return (0);
 630
 631 fail2:
 632         EFSYS_PROBE(fail2);
 633 fail1:
 634         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 635
 636         return (rc);
 637 }
 638 #endif /* EFSYS_OPT_RX_SCALE */
 639
 640
 641 /*
 642  * EF10 RX pseudo-header
 643  * ---------------------
 644  *
 645  * Receive packets are prefixed by an (optional) 14 byte pseudo-header:
 646  *
 647  *  +00: Toeplitz hash value.
 648  *       (32bit little-endian)
 649  *  +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag.
 650  *       (16bit big-endian)
 651  *  +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag.
 652  *       (16bit big-endian)
 653  *  +08: Packet Length. Zero if the RX datapath was in cut-through mode.
 654  *       (16bit little-endian)
 655  *  +10: MAC timestamp. Zero if timestamping is not enabled.
 656  *       (32bit little-endian)
 657  *
 658  * See "The RX Pseudo-header" in SF-109306-TC.
 659  */
 660
 661         __checkReturn   efx_rc_t
 662 ef10_rx_prefix_pktlen(
 663         __in            efx_nic_t *enp,
 664         __in            uint8_t *buffer,
 665         __out           uint16_t *lengthp)
 666 {
 667         _NOTE(ARGUNUSED(enp))
 668
 669         /*
 670          * The RX pseudo-header contains the packet length, excluding the
 671          * pseudo-header. If the hardware receive datapath was operating in
 672          * cut-through mode then the length in the RX pseudo-header will be
 673          * zero, and the packet length must be obtained from the DMA length
 674          * reported in the RX event.
 675          */
 676         *lengthp = buffer[8] | (buffer[9] << 8);
 677         return (0);
 678 }
 679
 680 #if EFSYS_OPT_RX_SCALE
 681         __checkReturn   uint32_t
 682 ef10_rx_prefix_hash(
 683         __in            efx_nic_t *enp,
 684         __in            efx_rx_hash_alg_t func,
 685         __in            uint8_t *buffer)
 686 {
 687         _NOTE(ARGUNUSED(enp))
 688
 689         switch (func) {
 690         case EFX_RX_HASHALG_TOEPLITZ:
 691                 return (buffer[0] |
 692                     (buffer[1] << 8) |
 693                     (buffer[2] << 16) |
 694                     (buffer[3] << 24));
 695
 696         default:
 697                 EFSYS_ASSERT(0);
 698                 return (0);
 699         }
 700 }
 701 #endif /* EFSYS_OPT_RX_SCALE */
 702
 703                         void
 704 ef10_rx_qpost(
 705         __in            efx_rxq_t *erp,
 706         __in_ecount(n)  efsys_dma_addr_t *addrp,
 707         __in            size_t size,
 708         __in            unsigned int n,
 709         __in            unsigned int completed,
 710         __in            unsigned int added)
 711 {
 712         efx_qword_t qword;
 713         unsigned int i;
 714         unsigned int offset;
 715         unsigned int id;
 716
 717         /* The client driver must not overfill the queue */
 718         EFSYS_ASSERT3U(added - completed + n, <=,
 719             EFX_RXQ_LIMIT(erp->er_mask + 1));
 720
 721         id = added & (erp->er_mask);
 722         for (i = 0; i < n; i++) {
 723                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
 724                     unsigned int, id, efsys_dma_addr_t, addrp[i],
 725                     size_t, size);
 726
 727                 EFX_POPULATE_QWORD_3(qword,
 728                     ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size),
 729                     ESF_DZ_RX_KER_BUF_ADDR_DW0,
 730                     (uint32_t)(addrp[i] & 0xffffffff),
 731                     ESF_DZ_RX_KER_BUF_ADDR_DW1,
 732                     (uint32_t)(addrp[i] >> 32));
 733
 734                 offset = id * sizeof (efx_qword_t);
 735                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
 736
 737                 id = (id + 1) & (erp->er_mask);
 738         }
 739 }
 740
 741                         void
 742 ef10_rx_qpush(
 743         __in    efx_rxq_t *erp,
 744         __in    unsigned int added,
 745         __inout unsigned int *pushedp)
 746 {
 747         efx_nic_t *enp = erp->er_enp;
 748         unsigned int pushed = *pushedp;
 749         uint32_t wptr;
 750         efx_dword_t dword;
 751
 752         /* Hardware has alignment restriction for WPTR */
 753         wptr = P2ALIGN(added, EF10_RX_WPTR_ALIGN);
 754         if (pushed == wptr)
 755                 return;
 756
 757         *pushedp = wptr;
 758
 759         /* Push the populated descriptors out */
 760         wptr &= erp->er_mask;
 761
 762         EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr);
 763
 764         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
 765         EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
 766             wptr, pushed & erp->er_mask);
 767         EFSYS_PIO_WRITE_BARRIER();
 768         EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
 769                             erp->er_index, &dword, B_FALSE);
 770 }
 771
 772 #if EFSYS_OPT_RX_PACKED_STREAM
 773
 774                         void
 775 ef10_rx_qps_update_credits(
 776         __in    efx_rxq_t *erp)
 777 {
 778         efx_nic_t *enp = erp->er_enp;
 779         efx_dword_t dword;
 780         efx_evq_rxq_state_t *rxq_state =
 781                 &erp->er_eep->ee_rxq_state[erp->er_label];
 782
 783         EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
 784
 785         if (rxq_state->eers_rx_packed_stream_credits == 0)
 786                 return;
 787
 788         EFX_POPULATE_DWORD_3(dword,
 789             ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1,
 790             ERF_DZ_RX_DESC_MAGIC_CMD,
 791             ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS,
 792             ERF_DZ_RX_DESC_MAGIC_DATA,
 793             rxq_state->eers_rx_packed_stream_credits);
 794         EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
 795             erp->er_index, &dword, B_FALSE);
 796
 797         rxq_state->eers_rx_packed_stream_credits = 0;
 798 }
 799
 800         __checkReturn   uint8_t *
 801 ef10_rx_qps_packet_info(
 802         __in            efx_rxq_t *erp,
 803         __in            uint8_t *buffer,
 804         __in            uint32_t buffer_length,
 805         __in            uint32_t current_offset,
 806         __out           uint16_t *lengthp,
 807         __out           uint32_t *next_offsetp,
 808         __out           uint32_t *timestamp)
 809 {
 810         uint16_t buf_len;
 811         uint8_t *pkt_start;
 812         efx_qword_t *qwordp;
 813         efx_evq_rxq_state_t *rxq_state =
 814                 &erp->er_eep->ee_rxq_state[erp->er_label];
 815
 816         EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
 817
 818         buffer += current_offset;
 819         pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE;
 820
 821         qwordp = (efx_qword_t *)buffer;
 822         *timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP);
 823         *lengthp   = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN);
 824         buf_len    = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN);
 825
 826         buf_len = P2ROUNDUP(buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE,
 827                             EFX_RX_PACKED_STREAM_ALIGNMENT);
 828         *next_offsetp =
 829             current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT;
 830
 831         EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length);
 832         EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp);
 833
 834         if ((*next_offsetp ^ current_offset) &
 835             EFX_RX_PACKED_STREAM_MEM_PER_CREDIT) {
 836                 if (rxq_state->eers_rx_packed_stream_credits <
 837                     EFX_RX_PACKED_STREAM_MAX_CREDITS)
 838                         rxq_state->eers_rx_packed_stream_credits++;
 839         }
 840
 841         return (pkt_start);
 842 }
 843
 844
 845 #endif
 846
 847         __checkReturn   efx_rc_t
 848 ef10_rx_qflush(
 849         __in    efx_rxq_t *erp)
 850 {
 851         efx_nic_t *enp = erp->er_enp;
 852         efx_rc_t rc;
 853
 854         if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0)
 855                 goto fail1;
 856
 857         return (0);
 858
 859 fail1:
 860         /*
 861          * EALREADY is not an error, but indicates that the MC has rebooted and
 862          * that the RXQ has already been destroyed. Callers need to know that
 863          * the RXQ flush has completed to avoid waiting until timeout for a
 864          * flush done event that will not be delivered.
 865          */
 866         if (rc != EALREADY)
 867                 EFSYS_PROBE1(fail1, efx_rc_t, rc);
 868
 869         return (rc);
 870 }
 871
 872                 void
 873 ef10_rx_qenable(
 874         __in    efx_rxq_t *erp)
 875 {
 876         /* FIXME */
 877         _NOTE(ARGUNUSED(erp))
 878         /* FIXME */
 879 }
 880
 881         __checkReturn   efx_rc_t
 882 ef10_rx_qcreate(
 883         __in            efx_nic_t *enp,
 884         __in            unsigned int index,
 885         __in            unsigned int label,
 886         __in            efx_rxq_type_t type,
 887         __in            efsys_mem_t *esmp,
 888         __in            size_t n,
 889         __in            uint32_t id,
 890         __in            efx_evq_t *eep,
 891         __in            efx_rxq_t *erp)
 892 {
 893         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 894         efx_rc_t rc;
 895         boolean_t disable_scatter;
 896         unsigned int ps_buf_size;
 897
 898         _NOTE(ARGUNUSED(id, erp))
 899
 900         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH));
 901         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
 902         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
 903
 904         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
 905         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
 906
 907         if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) {
 908                 rc = EINVAL;
 909                 goto fail1;
 910         }
 911         if (index >= encp->enc_rxq_limit) {
 912                 rc = EINVAL;
 913                 goto fail2;
 914         }
 915
 916         switch (type) {
 917         case EFX_RXQ_TYPE_DEFAULT:
 918         case EFX_RXQ_TYPE_SCATTER:
 919                 ps_buf_size = 0;
 920                 break;
 921 #if EFSYS_OPT_RX_PACKED_STREAM
 922         case EFX_RXQ_TYPE_PACKED_STREAM_1M:
 923                 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M;
 924                 break;
 925         case EFX_RXQ_TYPE_PACKED_STREAM_512K:
 926                 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K;
 927                 break;
 928         case EFX_RXQ_TYPE_PACKED_STREAM_256K:
 929                 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K;
 930                 break;
 931         case EFX_RXQ_TYPE_PACKED_STREAM_128K:
 932                 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K;
 933                 break;
 934         case EFX_RXQ_TYPE_PACKED_STREAM_64K:
 935                 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K;
 936                 break;
 937 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
 938         default:
 939                 rc = ENOTSUP;
 940                 goto fail3;
 941         }
 942
 943 #if EFSYS_OPT_RX_PACKED_STREAM
 944         if (ps_buf_size != 0) {
 945                 /* Check if datapath firmware supports packed stream mode */
 946                 if (encp->enc_rx_packed_stream_supported == B_FALSE) {
 947                         rc = ENOTSUP;
 948                         goto fail4;
 949                 }
 950                 /* Check if packed stream allows configurable buffer sizes */
 951                 if ((type != EFX_RXQ_TYPE_PACKED_STREAM_1M) &&
 952                     (encp->enc_rx_var_packed_stream_supported == B_FALSE)) {
 953                         rc = ENOTSUP;
 954                         goto fail5;
 955                 }
 956         }
 957 #else /* EFSYS_OPT_RX_PACKED_STREAM */
 958         EFSYS_ASSERT(ps_buf_size == 0);
 959 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
 960
 961         /* Scatter can only be disabled if the firmware supports doing so */
 962         if (type == EFX_RXQ_TYPE_SCATTER)
 963                 disable_scatter = B_FALSE;
 964         else
 965                 disable_scatter = encp->enc_rx_disable_scatter_supported;
 966
 967         if ((rc = efx_mcdi_init_rxq(enp, n, eep->ee_index, label, index,
 968                     esmp, disable_scatter, ps_buf_size)) != 0)
 969                 goto fail6;
 970
 971         erp->er_eep = eep;
 972         erp->er_label = label;
 973
 974         ef10_ev_rxlabel_init(eep, erp, label, ps_buf_size != 0);
 975
 976         return (0);
 977
 978 fail6:
 979         EFSYS_PROBE(fail6);
 980 #if EFSYS_OPT_RX_PACKED_STREAM
 981 fail5:
 982         EFSYS_PROBE(fail5);
 983 fail4:
 984         EFSYS_PROBE(fail4);
 985 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
 986 fail3:
 987         EFSYS_PROBE(fail3);
 988 fail2:
 989         EFSYS_PROBE(fail2);
 990 fail1:
 991         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 992
 993         return (rc);
 994 }
 995
 996                 void
 997 ef10_rx_qdestroy(
 998         __in    efx_rxq_t *erp)
 999 {
1000         efx_nic_t *enp = erp->er_enp;
1001         efx_evq_t *eep = erp->er_eep;
1002         unsigned int label = erp->er_label;
1003
1004         ef10_ev_rxlabel_fini(eep, label);
1005
1006         EFSYS_ASSERT(enp->en_rx_qcount != 0);
1007         --enp->en_rx_qcount;
1008
1009         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1010 }
1011
1012                 void
1013 ef10_rx_fini(
1014         __in    efx_nic_t *enp)
1015 {
1016 #if EFSYS_OPT_RX_SCALE
1017         if (enp->en_rss_context_type != EFX_RX_SCALE_UNAVAILABLE)
1018                 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context);
1019         enp->en_rss_context = 0;
1020         enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
1021 #else
1022         _NOTE(ARGUNUSED(enp))
1023 #endif /* EFSYS_OPT_RX_SCALE */
1024 }
1025
1026 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */