drivers/net/sfc/base/ef10_rx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright (c) 2012-2018 Solarflare Communications Inc.
   4  * All rights reserved.
   5  */
   6
   7 #include "efx.h"
   8 #include "efx_impl.h"
   9
  10
  11 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2
  12
  13
  14 static  __checkReturn   efx_rc_t
  15 efx_mcdi_init_rxq(
  16         __in            efx_nic_t *enp,
  17         __in            uint32_t ndescs,
  18         __in            uint32_t target_evq,
  19         __in            uint32_t label,
  20         __in            uint32_t instance,
  21         __in            efsys_mem_t *esmp,
  22         __in            boolean_t disable_scatter,
  23         __in            boolean_t want_inner_classes,
  24         __in            uint32_t ps_bufsize,
  25         __in            uint32_t es_bufs_per_desc,
  26         __in            uint32_t es_max_dma_len,
  27         __in            uint32_t es_buf_stride,
  28         __in            uint32_t hol_block_timeout)
  29 {
  30         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
  31         efx_mcdi_req_t req;
  32         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_INIT_RXQ_V3_IN_LEN,
  33                 MC_CMD_INIT_RXQ_V3_OUT_LEN);
  34         int npages = EFX_RXQ_NBUFS(ndescs);
  35         int i;
  36         efx_qword_t *dma_addr;
  37         uint64_t addr;
  38         efx_rc_t rc;
  39         uint32_t dma_mode;
  40         boolean_t want_outer_classes;
  41
  42         EFSYS_ASSERT3U(ndescs, <=, EFX_RXQ_MAXNDESCS);
  43
  44         if ((esmp == NULL) || (EFSYS_MEM_SIZE(esmp) < EFX_RXQ_SIZE(ndescs))) {
  45                 rc = EINVAL;
  46                 goto fail1;
  47         }
  48
  49         if (ps_bufsize > 0)
  50                 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_PACKED_STREAM;
  51         else if (es_bufs_per_desc > 0)
  52                 dma_mode = MC_CMD_INIT_RXQ_V3_IN_EQUAL_STRIDE_SUPER_BUFFER;
  53         else
  54                 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_SINGLE_PACKET;
  55
  56         if (encp->enc_tunnel_encapsulations_supported != 0 &&
  57             !want_inner_classes) {
  58                 /*
  59                  * WANT_OUTER_CLASSES can only be specified on hardware which
  60                  * supports tunnel encapsulation offloads, even though it is
  61                  * effectively the behaviour the hardware gives.
  62                  *
  63                  * Also, on hardware which does support such offloads, older
  64                  * firmware rejects the flag if the offloads are not supported
  65                  * by the current firmware variant, which means this may fail if
  66                  * the capabilities are not updated when the firmware variant
  67                  * changes. This is not an issue on newer firmware, as it was
  68                  * changed in bug 69842 (v6.4.2.1007) to permit this flag to be
  69                  * specified on all firmware variants.
  70                  */
  71                 want_outer_classes = B_TRUE;
  72         } else {
  73                 want_outer_classes = B_FALSE;
  74         }
  75
  76         req.emr_cmd = MC_CMD_INIT_RXQ;
  77         req.emr_in_buf = payload;
  78         req.emr_in_length = MC_CMD_INIT_RXQ_V3_IN_LEN;
  79         req.emr_out_buf = payload;
  80         req.emr_out_length = MC_CMD_INIT_RXQ_V3_OUT_LEN;
  81
  82         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_SIZE, ndescs);
  83         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_TARGET_EVQ, target_evq);
  84         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_LABEL, label);
  85         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_INSTANCE, instance);
  86         MCDI_IN_POPULATE_DWORD_9(req, INIT_RXQ_EXT_IN_FLAGS,
  87             INIT_RXQ_EXT_IN_FLAG_BUFF_MODE, 0,
  88             INIT_RXQ_EXT_IN_FLAG_HDR_SPLIT, 0,
  89             INIT_RXQ_EXT_IN_FLAG_TIMESTAMP, 0,
  90             INIT_RXQ_EXT_IN_CRC_MODE, 0,
  91             INIT_RXQ_EXT_IN_FLAG_PREFIX, 1,
  92             INIT_RXQ_EXT_IN_FLAG_DISABLE_SCATTER, disable_scatter,
  93             INIT_RXQ_EXT_IN_DMA_MODE,
  94             dma_mode,
  95             INIT_RXQ_EXT_IN_PACKED_STREAM_BUFF_SIZE, ps_bufsize,
  96             INIT_RXQ_EXT_IN_FLAG_WANT_OUTER_CLASSES, want_outer_classes);
  97         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_OWNER_ID, 0);
  98         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);
  99
 100         if (es_bufs_per_desc > 0) {
 101                 MCDI_IN_SET_DWORD(req,
 102                     INIT_RXQ_V3_IN_ES_PACKET_BUFFERS_PER_BUCKET,
 103                     es_bufs_per_desc);
 104                 MCDI_IN_SET_DWORD(req,
 105                     INIT_RXQ_V3_IN_ES_MAX_DMA_LEN, es_max_dma_len);
 106                 MCDI_IN_SET_DWORD(req,
 107                     INIT_RXQ_V3_IN_ES_PACKET_STRIDE, es_buf_stride);
 108                 MCDI_IN_SET_DWORD(req,
 109                     INIT_RXQ_V3_IN_ES_HEAD_OF_LINE_BLOCK_TIMEOUT,
 110                     hol_block_timeout);
 111         }
 112
 113         dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR);
 114         addr = EFSYS_MEM_ADDR(esmp);
 115
 116         for (i = 0; i < npages; i++) {
 117                 EFX_POPULATE_QWORD_2(*dma_addr,
 118                     EFX_DWORD_1, (uint32_t)(addr >> 32),
 119                     EFX_DWORD_0, (uint32_t)(addr & 0xffffffff));
 120
 121                 dma_addr++;
 122                 addr += EFX_BUF_SIZE;
 123         }
 124
 125         efx_mcdi_execute(enp, &req);
 126
 127         if (req.emr_rc != 0) {
 128                 rc = req.emr_rc;
 129                 goto fail2;
 130         }
 131
 132         return (0);
 133
 134 fail2:
 135         EFSYS_PROBE(fail2);
 136 fail1:
 137         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 138
 139         return (rc);
 140 }
 141
 142 static  __checkReturn   efx_rc_t
 143 efx_mcdi_fini_rxq(
 144         __in            efx_nic_t *enp,
 145         __in            uint32_t instance)
 146 {
 147         efx_mcdi_req_t req;
 148         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_FINI_RXQ_IN_LEN,
 149                 MC_CMD_FINI_RXQ_OUT_LEN);
 150         efx_rc_t rc;
 151
 152         req.emr_cmd = MC_CMD_FINI_RXQ;
 153         req.emr_in_buf = payload;
 154         req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN;
 155         req.emr_out_buf = payload;
 156         req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN;
 157
 158         MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance);
 159
 160         efx_mcdi_execute_quiet(enp, &req);
 161
 162         if (req.emr_rc != 0) {
 163                 rc = req.emr_rc;
 164                 goto fail1;
 165         }
 166
 167         return (0);
 168
 169 fail1:
 170         /*
 171          * EALREADY is not an error, but indicates that the MC has rebooted and
 172          * that the RXQ has already been destroyed.
 173          */
 174         if (rc != EALREADY)
 175                 EFSYS_PROBE1(fail1, efx_rc_t, rc);
 176
 177         return (rc);
 178 }
 179
 180 #if EFSYS_OPT_RX_SCALE
 181 static  __checkReturn   efx_rc_t
 182 efx_mcdi_rss_context_alloc(
 183         __in            efx_nic_t *enp,
 184         __in            efx_rx_scale_context_type_t type,
 185         __in            uint32_t num_queues,
 186         __out           uint32_t *rss_contextp)
 187 {
 188         efx_mcdi_req_t req;
 189         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN,
 190                 MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN);
 191         uint32_t rss_context;
 192         uint32_t context_type;
 193         efx_rc_t rc;
 194
 195         if (num_queues > EFX_MAXRSS) {
 196                 rc = EINVAL;
 197                 goto fail1;
 198         }
 199
 200         switch (type) {
 201         case EFX_RX_SCALE_EXCLUSIVE:
 202                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE;
 203                 break;
 204         case EFX_RX_SCALE_SHARED:
 205                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED;
 206                 break;
 207         default:
 208                 rc = EINVAL;
 209                 goto fail2;
 210         }
 211
 212         req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC;
 213         req.emr_in_buf = payload;
 214         req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN;
 215         req.emr_out_buf = payload;
 216         req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN;
 217
 218         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
 219             EVB_PORT_ID_ASSIGNED);
 220         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type);
 221
 222         /*
 223          * For exclusive contexts, NUM_QUEUES is only used to validate
 224          * indirection table offsets.
 225          * For shared contexts, the provided context will spread traffic over
 226          * NUM_QUEUES many queues.
 227          */
 228         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues);
 229
 230         efx_mcdi_execute(enp, &req);
 231
 232         if (req.emr_rc != 0) {
 233                 rc = req.emr_rc;
 234                 goto fail3;
 235         }
 236
 237         if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) {
 238                 rc = EMSGSIZE;
 239                 goto fail4;
 240         }
 241
 242         rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);
 243         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 244                 rc = ENOENT;
 245                 goto fail5;
 246         }
 247
 248         *rss_contextp = rss_context;
 249
 250         return (0);
 251
 252 fail5:
 253         EFSYS_PROBE(fail5);
 254 fail4:
 255         EFSYS_PROBE(fail4);
 256 fail3:
 257         EFSYS_PROBE(fail3);
 258 fail2:
 259         EFSYS_PROBE(fail2);
 260 fail1:
 261         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 262
 263         return (rc);
 264 }
 265 #endif /* EFSYS_OPT_RX_SCALE */
 266
 267 #if EFSYS_OPT_RX_SCALE
 268 static                  efx_rc_t
 269 efx_mcdi_rss_context_free(
 270         __in            efx_nic_t *enp,
 271         __in            uint32_t rss_context)
 272 {
 273         efx_mcdi_req_t req;
 274         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_FREE_IN_LEN,
 275                 MC_CMD_RSS_CONTEXT_FREE_OUT_LEN);
 276         efx_rc_t rc;
 277
 278         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 279                 rc = EINVAL;
 280                 goto fail1;
 281         }
 282
 283         req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE;
 284         req.emr_in_buf = payload;
 285         req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN;
 286         req.emr_out_buf = payload;
 287         req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN;
 288
 289         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context);
 290
 291         efx_mcdi_execute_quiet(enp, &req);
 292
 293         if (req.emr_rc != 0) {
 294                 rc = req.emr_rc;
 295                 goto fail2;
 296         }
 297
 298         return (0);
 299
 300 fail2:
 301         EFSYS_PROBE(fail2);
 302 fail1:
 303         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 304
 305         return (rc);
 306 }
 307 #endif /* EFSYS_OPT_RX_SCALE */
 308
 309 #if EFSYS_OPT_RX_SCALE
 310 static                  efx_rc_t
 311 efx_mcdi_rss_context_set_flags(
 312         __in            efx_nic_t *enp,
 313         __in            uint32_t rss_context,
 314         __in            efx_rx_hash_type_t type)
 315 {
 316         efx_nic_cfg_t *encp = &enp->en_nic_cfg;
 317         efx_mcdi_req_t req;
 318         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN,
 319                 MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN);
 320         efx_rc_t rc;
 321
 322         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_LBN ==
 323                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_LBN);
 324         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_WIDTH ==
 325                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_WIDTH);
 326         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_LBN ==
 327                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_LBN);
 328         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_WIDTH ==
 329                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_WIDTH);
 330         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_TCP_LBN ==
 331                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_LBN);
 332         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_TCP_WIDTH ==
 333                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_WIDTH);
 334         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_LBN ==
 335                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_LBN);
 336         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_WIDTH ==
 337                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_WIDTH);
 338
 339         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 340                 rc = EINVAL;
 341                 goto fail1;
 342         }
 343
 344         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS;
 345         req.emr_in_buf = payload;
 346         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN;
 347         req.emr_out_buf = payload;
 348         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN;
 349
 350         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
 351             rss_context);
 352
 353         /*
 354          * If the firmware lacks support for additional modes, RSS_MODE
 355          * fields must contain zeros, otherwise the operation will fail.
 356          */
 357         if (encp->enc_rx_scale_additional_modes_supported == B_FALSE)
 358                 type &= EFX_RX_HASH_LEGACY_MASK;
 359
 360         MCDI_IN_POPULATE_DWORD_10(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS,
 361             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN,
 362             (type & EFX_RX_HASH_IPV4) ? 1 : 0,
 363             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN,
 364             (type & EFX_RX_HASH_TCPIPV4) ? 1 : 0,
 365             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN,
 366             (type & EFX_RX_HASH_IPV6) ? 1 : 0,
 367             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN,
 368             (type & EFX_RX_HASH_TCPIPV6) ? 1 : 0,
 369             RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE,
 370             (type >> EFX_RX_CLASS_IPV4_TCP_LBN) &
 371             EFX_MASK32(EFX_RX_CLASS_IPV4_TCP),
 372             RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV4_RSS_MODE,
 373             (type >> EFX_RX_CLASS_IPV4_UDP_LBN) &
 374             EFX_MASK32(EFX_RX_CLASS_IPV4_UDP),
 375             RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE,
 376             (type >> EFX_RX_CLASS_IPV4_LBN) & EFX_MASK32(EFX_RX_CLASS_IPV4),
 377             RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE,
 378             (type >> EFX_RX_CLASS_IPV6_TCP_LBN) &
 379             EFX_MASK32(EFX_RX_CLASS_IPV6_TCP),
 380             RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV6_RSS_MODE,
 381             (type >> EFX_RX_CLASS_IPV6_UDP_LBN) &
 382             EFX_MASK32(EFX_RX_CLASS_IPV6_UDP),
 383             RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE,
 384             (type >> EFX_RX_CLASS_IPV6_LBN) & EFX_MASK32(EFX_RX_CLASS_IPV6));
 385
 386         efx_mcdi_execute(enp, &req);
 387
 388         if (req.emr_rc != 0) {
 389                 rc = req.emr_rc;
 390                 goto fail2;
 391         }
 392
 393         return (0);
 394
 395 fail2:
 396         EFSYS_PROBE(fail2);
 397 fail1:
 398         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 399
 400         return (rc);
 401 }
 402 #endif /* EFSYS_OPT_RX_SCALE */
 403
 404 #if EFSYS_OPT_RX_SCALE
 405 static                  efx_rc_t
 406 efx_mcdi_rss_context_set_key(
 407         __in            efx_nic_t *enp,
 408         __in            uint32_t rss_context,
 409         __in_ecount(n)  uint8_t *key,
 410         __in            size_t n)
 411 {
 412         efx_mcdi_req_t req;
 413         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN,
 414                 MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN);
 415         efx_rc_t rc;
 416
 417         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 418                 rc = EINVAL;
 419                 goto fail1;
 420         }
 421
 422         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY;
 423         req.emr_in_buf = payload;
 424         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN;
 425         req.emr_out_buf = payload;
 426         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN;
 427
 428         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
 429             rss_context);
 430
 431         EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
 432         if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) {
 433                 rc = EINVAL;
 434                 goto fail2;
 435         }
 436
 437         memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY),
 438             key, n);
 439
 440         efx_mcdi_execute(enp, &req);
 441
 442         if (req.emr_rc != 0) {
 443                 rc = req.emr_rc;
 444                 goto fail3;
 445         }
 446
 447         return (0);
 448
 449 fail3:
 450         EFSYS_PROBE(fail3);
 451 fail2:
 452         EFSYS_PROBE(fail2);
 453 fail1:
 454         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 455
 456         return (rc);
 457 }
 458 #endif /* EFSYS_OPT_RX_SCALE */
 459
 460 #if EFSYS_OPT_RX_SCALE
 461 static                  efx_rc_t
 462 efx_mcdi_rss_context_set_table(
 463         __in            efx_nic_t *enp,
 464         __in            uint32_t rss_context,
 465         __in_ecount(n)  unsigned int *table,
 466         __in            size_t n)
 467 {
 468         efx_mcdi_req_t req;
 469         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN,
 470                 MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN);
 471         uint8_t *req_table;
 472         int i, rc;
 473
 474         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 475                 rc = EINVAL;
 476                 goto fail1;
 477         }
 478
 479         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE;
 480         req.emr_in_buf = payload;
 481         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN;
 482         req.emr_out_buf = payload;
 483         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN;
 484
 485         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
 486             rss_context);
 487
 488         req_table =
 489             MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE);
 490
 491         for (i = 0;
 492             i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN;
 493             i++) {
 494                 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0;
 495         }
 496
 497         efx_mcdi_execute(enp, &req);
 498
 499         if (req.emr_rc != 0) {
 500                 rc = req.emr_rc;
 501                 goto fail2;
 502         }
 503
 504         return (0);
 505
 506 fail2:
 507         EFSYS_PROBE(fail2);
 508 fail1:
 509         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 510
 511         return (rc);
 512 }
 513 #endif /* EFSYS_OPT_RX_SCALE */
 514
 515
 516         __checkReturn   efx_rc_t
 517 ef10_rx_init(
 518         __in            efx_nic_t *enp)
 519 {
 520 #if EFSYS_OPT_RX_SCALE
 521
 522         if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS,
 523                 &enp->en_rss_context) == 0) {
 524                 /*
 525                  * Allocated an exclusive RSS context, which allows both the
 526                  * indirection table and key to be modified.
 527                  */
 528                 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE;
 529                 enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
 530         } else {
 531                 /*
 532                  * Failed to allocate an exclusive RSS context. Continue
 533                  * operation without support for RSS. The pseudo-header in
 534                  * received packets will not contain a Toeplitz hash value.
 535                  */
 536                 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
 537                 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE;
 538         }
 539
 540 #endif /* EFSYS_OPT_RX_SCALE */
 541
 542         return (0);
 543 }
 544
 545 #if EFSYS_OPT_RX_SCATTER
 546         __checkReturn   efx_rc_t
 547 ef10_rx_scatter_enable(
 548         __in            efx_nic_t *enp,
 549         __in            unsigned int buf_size)
 550 {
 551         _NOTE(ARGUNUSED(enp, buf_size))
 552         return (0);
 553 }
 554 #endif  /* EFSYS_OPT_RX_SCATTER */
 555
 556 #if EFSYS_OPT_RX_SCALE
 557         __checkReturn   efx_rc_t
 558 ef10_rx_scale_context_alloc(
 559         __in            efx_nic_t *enp,
 560         __in            efx_rx_scale_context_type_t type,
 561         __in            uint32_t num_queues,
 562         __out           uint32_t *rss_contextp)
 563 {
 564         efx_rc_t rc;
 565
 566         rc = efx_mcdi_rss_context_alloc(enp, type, num_queues, rss_contextp);
 567         if (rc != 0)
 568                 goto fail1;
 569
 570         return (0);
 571
 572 fail1:
 573         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 574         return (rc);
 575 }
 576 #endif /* EFSYS_OPT_RX_SCALE */
 577
 578 #if EFSYS_OPT_RX_SCALE
 579         __checkReturn   efx_rc_t
 580 ef10_rx_scale_context_free(
 581         __in            efx_nic_t *enp,
 582         __in            uint32_t rss_context)
 583 {
 584         efx_rc_t rc;
 585
 586         rc = efx_mcdi_rss_context_free(enp, rss_context);
 587         if (rc != 0)
 588                 goto fail1;
 589
 590         return (0);
 591
 592 fail1:
 593         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 594         return (rc);
 595 }
 596 #endif /* EFSYS_OPT_RX_SCALE */
 597
 598 #if EFSYS_OPT_RX_SCALE
 599         __checkReturn   efx_rc_t
 600 ef10_rx_scale_mode_set(
 601         __in            efx_nic_t *enp,
 602         __in            uint32_t rss_context,
 603         __in            efx_rx_hash_alg_t alg,
 604         __in            efx_rx_hash_type_t type,
 605         __in            boolean_t insert)
 606 {
 607         efx_nic_cfg_t *encp = &enp->en_nic_cfg;
 608         efx_rc_t rc;
 609
 610         EFSYS_ASSERT3U(insert, ==, B_TRUE);
 611
 612         if ((encp->enc_rx_scale_hash_alg_mask & (1U << alg)) == 0 ||
 613             insert == B_FALSE) {
 614                 rc = EINVAL;
 615                 goto fail1;
 616         }
 617
 618         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 619                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 620                         rc = ENOTSUP;
 621                         goto fail2;
 622                 }
 623                 rss_context = enp->en_rss_context;
 624         }
 625
 626         if ((rc = efx_mcdi_rss_context_set_flags(enp,
 627                     rss_context, type)) != 0)
 628                 goto fail3;
 629
 630         return (0);
 631
 632 fail3:
 633         EFSYS_PROBE(fail3);
 634 fail2:
 635         EFSYS_PROBE(fail2);
 636 fail1:
 637         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 638
 639         return (rc);
 640 }
 641 #endif /* EFSYS_OPT_RX_SCALE */
 642
 643 #if EFSYS_OPT_RX_SCALE
 644         __checkReturn   efx_rc_t
 645 ef10_rx_scale_key_set(
 646         __in            efx_nic_t *enp,
 647         __in            uint32_t rss_context,
 648         __in_ecount(n)  uint8_t *key,
 649         __in            size_t n)
 650 {
 651         efx_rc_t rc;
 652
 653         EFX_STATIC_ASSERT(EFX_RSS_KEY_SIZE ==
 654             MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
 655
 656         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 657                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 658                         rc = ENOTSUP;
 659                         goto fail1;
 660                 }
 661                 rss_context = enp->en_rss_context;
 662         }
 663
 664         if ((rc = efx_mcdi_rss_context_set_key(enp, rss_context, key, n)) != 0)
 665                 goto fail2;
 666
 667         return (0);
 668
 669 fail2:
 670         EFSYS_PROBE(fail2);
 671 fail1:
 672         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 673
 674         return (rc);
 675 }
 676 #endif /* EFSYS_OPT_RX_SCALE */
 677
 678 #if EFSYS_OPT_RX_SCALE
 679         __checkReturn   efx_rc_t
 680 ef10_rx_scale_tbl_set(
 681         __in            efx_nic_t *enp,
 682         __in            uint32_t rss_context,
 683         __in_ecount(n)  unsigned int *table,
 684         __in            size_t n)
 685 {
 686         efx_rc_t rc;
 687
 688
 689         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 690                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 691                         rc = ENOTSUP;
 692                         goto fail1;
 693                 }
 694                 rss_context = enp->en_rss_context;
 695         }
 696
 697         if ((rc = efx_mcdi_rss_context_set_table(enp,
 698                     rss_context, table, n)) != 0)
 699                 goto fail2;
 700
 701         return (0);
 702
 703 fail2:
 704         EFSYS_PROBE(fail2);
 705 fail1:
 706         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 707
 708         return (rc);
 709 }
 710 #endif /* EFSYS_OPT_RX_SCALE */
 711
 712
 713 /*
 714  * EF10 RX pseudo-header
 715  * ---------------------
 716  *
 717  * Receive packets are prefixed by an (optional) 14 byte pseudo-header:
 718  *
 719  *  +00: Toeplitz hash value.
 720  *       (32bit little-endian)
 721  *  +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag.
 722  *       (16bit big-endian)
 723  *  +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag.
 724  *       (16bit big-endian)
 725  *  +08: Packet Length. Zero if the RX datapath was in cut-through mode.
 726  *       (16bit little-endian)
 727  *  +10: MAC timestamp. Zero if timestamping is not enabled.
 728  *       (32bit little-endian)
 729  *
 730  * See "The RX Pseudo-header" in SF-109306-TC.
 731  */
 732
 733         __checkReturn   efx_rc_t
 734 ef10_rx_prefix_pktlen(
 735         __in            efx_nic_t *enp,
 736         __in            uint8_t *buffer,
 737         __out           uint16_t *lengthp)
 738 {
 739         _NOTE(ARGUNUSED(enp))
 740
 741         /*
 742          * The RX pseudo-header contains the packet length, excluding the
 743          * pseudo-header. If the hardware receive datapath was operating in
 744          * cut-through mode then the length in the RX pseudo-header will be
 745          * zero, and the packet length must be obtained from the DMA length
 746          * reported in the RX event.
 747          */
 748         *lengthp = buffer[8] | (buffer[9] << 8);
 749         return (0);
 750 }
 751
 752 #if EFSYS_OPT_RX_SCALE
 753         __checkReturn   uint32_t
 754 ef10_rx_prefix_hash(
 755         __in            efx_nic_t *enp,
 756         __in            efx_rx_hash_alg_t func,
 757         __in            uint8_t *buffer)
 758 {
 759         _NOTE(ARGUNUSED(enp))
 760
 761         switch (func) {
 762         case EFX_RX_HASHALG_PACKED_STREAM:
 763         case EFX_RX_HASHALG_TOEPLITZ:
 764                 return (buffer[0] |
 765                     (buffer[1] << 8) |
 766                     (buffer[2] << 16) |
 767                     (buffer[3] << 24));
 768
 769         default:
 770                 EFSYS_ASSERT(0);
 771                 return (0);
 772         }
 773 }
 774 #endif /* EFSYS_OPT_RX_SCALE */
 775
 776 #if EFSYS_OPT_RX_PACKED_STREAM
 777 /*
 778  * Fake length for RXQ descriptors in packed stream mode
 779  * to make hardware happy
 780  */
 781 #define EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE 32
 782 #endif
 783
 784                                 void
 785 ef10_rx_qpost(
 786         __in                    efx_rxq_t *erp,
 787         __in_ecount(ndescs)     efsys_dma_addr_t *addrp,
 788         __in                    size_t size,
 789         __in                    unsigned int ndescs,
 790         __in                    unsigned int completed,
 791         __in                    unsigned int added)
 792 {
 793         efx_qword_t qword;
 794         unsigned int i;
 795         unsigned int offset;
 796         unsigned int id;
 797
 798         _NOTE(ARGUNUSED(completed))
 799
 800 #if EFSYS_OPT_RX_PACKED_STREAM
 801         /*
 802          * Real size of the buffer does not fit into ESF_DZ_RX_KER_BYTE_CNT
 803          * and equal to 0 after applying mask. Hardware does not like it.
 804          */
 805         if (erp->er_ev_qstate->eers_rx_packed_stream)
 806                 size = EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE;
 807 #endif
 808
 809         /* The client driver must not overfill the queue */
 810         EFSYS_ASSERT3U(added - completed + ndescs, <=,
 811             EFX_RXQ_LIMIT(erp->er_mask + 1));
 812
 813         id = added & (erp->er_mask);
 814         for (i = 0; i < ndescs; i++) {
 815                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
 816                     unsigned int, id, efsys_dma_addr_t, addrp[i],
 817                     size_t, size);
 818
 819                 EFX_POPULATE_QWORD_3(qword,
 820                     ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size),
 821                     ESF_DZ_RX_KER_BUF_ADDR_DW0,
 822                     (uint32_t)(addrp[i] & 0xffffffff),
 823                     ESF_DZ_RX_KER_BUF_ADDR_DW1,
 824                     (uint32_t)(addrp[i] >> 32));
 825
 826                 offset = id * sizeof (efx_qword_t);
 827                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
 828
 829                 id = (id + 1) & (erp->er_mask);
 830         }
 831 }
 832
 833                         void
 834 ef10_rx_qpush(
 835         __in    efx_rxq_t *erp,
 836         __in    unsigned int added,
 837         __inout unsigned int *pushedp)
 838 {
 839         efx_nic_t *enp = erp->er_enp;
 840         unsigned int pushed = *pushedp;
 841         uint32_t wptr;
 842         efx_dword_t dword;
 843
 844         /* Hardware has alignment restriction for WPTR */
 845         wptr = P2ALIGN(added, EF10_RX_WPTR_ALIGN);
 846         if (pushed == wptr)
 847                 return;
 848
 849         *pushedp = wptr;
 850
 851         /* Push the populated descriptors out */
 852         wptr &= erp->er_mask;
 853
 854         EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr);
 855
 856         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
 857         EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
 858             wptr, pushed & erp->er_mask);
 859         EFSYS_PIO_WRITE_BARRIER();
 860         EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
 861             erp->er_index, &dword, B_FALSE);
 862 }
 863
 864 #if EFSYS_OPT_RX_PACKED_STREAM
 865
 866                         void
 867 ef10_rx_qpush_ps_credits(
 868         __in            efx_rxq_t *erp)
 869 {
 870         efx_nic_t *enp = erp->er_enp;
 871         efx_dword_t dword;
 872         efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate;
 873         uint32_t credits;
 874
 875         EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
 876
 877         if (rxq_state->eers_rx_packed_stream_credits == 0)
 878                 return;
 879
 880         /*
 881          * It is a bug if we think that FW has utilized more
 882          * credits than it is allowed to have (maximum). However,
 883          * make sure that we do not credit more than maximum anyway.
 884          */
 885         credits = MIN(rxq_state->eers_rx_packed_stream_credits,
 886             EFX_RX_PACKED_STREAM_MAX_CREDITS);
 887         EFX_POPULATE_DWORD_3(dword,
 888             ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1,
 889             ERF_DZ_RX_DESC_MAGIC_CMD,
 890             ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS,
 891             ERF_DZ_RX_DESC_MAGIC_DATA, credits);
 892         EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
 893             erp->er_index, &dword, B_FALSE);
 894
 895         rxq_state->eers_rx_packed_stream_credits = 0;
 896 }
 897
 898 /*
 899  * In accordance with SF-112241-TC the received data has the following layout:
 900  *  - 8 byte pseudo-header which consist of:
 901  *    - 4 byte little-endian timestamp
 902  *    - 2 byte little-endian captured length in bytes
 903  *    - 2 byte little-endian original packet length in bytes
 904  *  - captured packet bytes
 905  *  - optional padding to align to 64 bytes boundary
 906  *  - 64 bytes scratch space for the host software
 907  */
 908         __checkReturn   uint8_t *
 909 ef10_rx_qps_packet_info(
 910         __in            efx_rxq_t *erp,
 911         __in            uint8_t *buffer,
 912         __in            uint32_t buffer_length,
 913         __in            uint32_t current_offset,
 914         __out           uint16_t *lengthp,
 915         __out           uint32_t *next_offsetp,
 916         __out           uint32_t *timestamp)
 917 {
 918         uint16_t buf_len;
 919         uint8_t *pkt_start;
 920         efx_qword_t *qwordp;
 921         efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate;
 922
 923         EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
 924
 925         buffer += current_offset;
 926         pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE;
 927
 928         qwordp = (efx_qword_t *)buffer;
 929         *timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP);
 930         *lengthp   = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN);
 931         buf_len    = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN);
 932
 933         buf_len = P2ROUNDUP(buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE,
 934                             EFX_RX_PACKED_STREAM_ALIGNMENT);
 935         *next_offsetp =
 936             current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT;
 937
 938         EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length);
 939         EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp);
 940
 941         if ((*next_offsetp ^ current_offset) &
 942             EFX_RX_PACKED_STREAM_MEM_PER_CREDIT)
 943                 rxq_state->eers_rx_packed_stream_credits++;
 944
 945         return (pkt_start);
 946 }
 947
 948
 949 #endif
 950
 951         __checkReturn   efx_rc_t
 952 ef10_rx_qflush(
 953         __in    efx_rxq_t *erp)
 954 {
 955         efx_nic_t *enp = erp->er_enp;
 956         efx_rc_t rc;
 957
 958         if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0)
 959                 goto fail1;
 960
 961         return (0);
 962
 963 fail1:
 964         /*
 965          * EALREADY is not an error, but indicates that the MC has rebooted and
 966          * that the RXQ has already been destroyed. Callers need to know that
 967          * the RXQ flush has completed to avoid waiting until timeout for a
 968          * flush done event that will not be delivered.
 969          */
 970         if (rc != EALREADY)
 971                 EFSYS_PROBE1(fail1, efx_rc_t, rc);
 972
 973         return (rc);
 974 }
 975
 976                 void
 977 ef10_rx_qenable(
 978         __in    efx_rxq_t *erp)
 979 {
 980         /* FIXME */
 981         _NOTE(ARGUNUSED(erp))
 982         /* FIXME */
 983 }
 984
 985         __checkReturn   efx_rc_t
 986 ef10_rx_qcreate(
 987         __in            efx_nic_t *enp,
 988         __in            unsigned int index,
 989         __in            unsigned int label,
 990         __in            efx_rxq_type_t type,
 991         __in            const efx_rxq_type_data_t *type_data,
 992         __in            efsys_mem_t *esmp,
 993         __in            size_t ndescs,
 994         __in            uint32_t id,
 995         __in            unsigned int flags,
 996         __in            efx_evq_t *eep,
 997         __in            efx_rxq_t *erp)
 998 {
 999         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1000         efx_rc_t rc;
1001         boolean_t disable_scatter;
1002         boolean_t want_inner_classes;
1003         unsigned int ps_buf_size;
1004         uint32_t es_bufs_per_desc = 0;
1005         uint32_t es_max_dma_len = 0;
1006         uint32_t es_buf_stride = 0;
1007         uint32_t hol_block_timeout = 0;
1008
1009         _NOTE(ARGUNUSED(id, erp, type_data))
1010
1011         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH));
1012         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
1013         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
1014
1015         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
1016         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
1017
1018         if (!ISP2(ndescs) ||
1019             (ndescs < EFX_RXQ_MINNDESCS) || (ndescs > EFX_RXQ_MAXNDESCS)) {
1020                 rc = EINVAL;
1021                 goto fail1;
1022         }
1023         if (index >= encp->enc_rxq_limit) {
1024                 rc = EINVAL;
1025                 goto fail2;
1026         }
1027
1028         switch (type) {
1029         case EFX_RXQ_TYPE_DEFAULT:
1030                 ps_buf_size = 0;
1031                 break;
1032 #if EFSYS_OPT_RX_PACKED_STREAM
1033         case EFX_RXQ_TYPE_PACKED_STREAM:
1034                 switch (type_data->ertd_packed_stream.eps_buf_size) {
1035                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_1M:
1036                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M;
1037                         break;
1038                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_512K:
1039                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K;
1040                         break;
1041                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_256K:
1042                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K;
1043                         break;
1044                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_128K:
1045                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K;
1046                         break;
1047                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_64K:
1048                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K;
1049                         break;
1050                 default:
1051                         rc = ENOTSUP;
1052                         goto fail3;
1053                 }
1054                 break;
1055 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1056 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
1057         case EFX_RXQ_TYPE_ES_SUPER_BUFFER:
1058                 ps_buf_size = 0;
1059                 es_bufs_per_desc =
1060                     type_data->ertd_es_super_buffer.eessb_bufs_per_desc;
1061                 es_max_dma_len =
1062                     type_data->ertd_es_super_buffer.eessb_max_dma_len;
1063                 es_buf_stride =
1064                     type_data->ertd_es_super_buffer.eessb_buf_stride;
1065                 hol_block_timeout =
1066                     type_data->ertd_es_super_buffer.eessb_hol_block_timeout;
1067                 break;
1068 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1069         default:
1070                 rc = ENOTSUP;
1071                 goto fail4;
1072         }
1073
1074 #if EFSYS_OPT_RX_PACKED_STREAM
1075         if (ps_buf_size != 0) {
1076                 /* Check if datapath firmware supports packed stream mode */
1077                 if (encp->enc_rx_packed_stream_supported == B_FALSE) {
1078                         rc = ENOTSUP;
1079                         goto fail5;
1080                 }
1081                 /* Check if packed stream allows configurable buffer sizes */
1082                 if ((ps_buf_size != MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M) &&
1083                     (encp->enc_rx_var_packed_stream_supported == B_FALSE)) {
1084                         rc = ENOTSUP;
1085                         goto fail6;
1086                 }
1087         }
1088 #else /* EFSYS_OPT_RX_PACKED_STREAM */
1089         EFSYS_ASSERT(ps_buf_size == 0);
1090 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1091
1092 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
1093         if (es_bufs_per_desc > 0) {
1094                 if (encp->enc_rx_es_super_buffer_supported == B_FALSE) {
1095                         rc = ENOTSUP;
1096                         goto fail7;
1097                 }
1098                 if (!IS_P2ALIGNED(es_max_dma_len,
1099                             EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) {
1100                         rc = EINVAL;
1101                         goto fail8;
1102                 }
1103                 if (!IS_P2ALIGNED(es_buf_stride,
1104                             EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) {
1105                         rc = EINVAL;
1106                         goto fail9;
1107                 }
1108         }
1109 #else /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1110         EFSYS_ASSERT(es_bufs_per_desc == 0);
1111 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1112
1113         /* Scatter can only be disabled if the firmware supports doing so */
1114         if (flags & EFX_RXQ_FLAG_SCATTER)
1115                 disable_scatter = B_FALSE;
1116         else
1117                 disable_scatter = encp->enc_rx_disable_scatter_supported;
1118
1119         if (flags & EFX_RXQ_FLAG_INNER_CLASSES)
1120                 want_inner_classes = B_TRUE;
1121         else
1122                 want_inner_classes = B_FALSE;
1123
1124         if ((rc = efx_mcdi_init_rxq(enp, ndescs, eep->ee_index, label, index,
1125                     esmp, disable_scatter, want_inner_classes,
1126                     ps_buf_size, es_bufs_per_desc, es_max_dma_len,
1127                     es_buf_stride, hol_block_timeout)) != 0)
1128                 goto fail10;
1129
1130         erp->er_eep = eep;
1131         erp->er_label = label;
1132
1133         ef10_ev_rxlabel_init(eep, erp, label, type);
1134
1135         erp->er_ev_qstate = &erp->er_eep->ee_rxq_state[label];
1136
1137         return (0);
1138
1139 fail10:
1140         EFSYS_PROBE(fail10);
1141 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
1142 fail9:
1143         EFSYS_PROBE(fail9);
1144 fail8:
1145         EFSYS_PROBE(fail8);
1146 fail7:
1147         EFSYS_PROBE(fail7);
1148 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1149 #if EFSYS_OPT_RX_PACKED_STREAM
1150 fail6:
1151         EFSYS_PROBE(fail6);
1152 fail5:
1153         EFSYS_PROBE(fail5);
1154 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1155 fail4:
1156         EFSYS_PROBE(fail4);
1157 #if EFSYS_OPT_RX_PACKED_STREAM
1158 fail3:
1159         EFSYS_PROBE(fail3);
1160 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1161 fail2:
1162         EFSYS_PROBE(fail2);
1163 fail1:
1164         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1165
1166         return (rc);
1167 }
1168
1169                 void
1170 ef10_rx_qdestroy(
1171         __in    efx_rxq_t *erp)
1172 {
1173         efx_nic_t *enp = erp->er_enp;
1174         efx_evq_t *eep = erp->er_eep;
1175         unsigned int label = erp->er_label;
1176
1177         ef10_ev_rxlabel_fini(eep, label);
1178
1179         EFSYS_ASSERT(enp->en_rx_qcount != 0);
1180         --enp->en_rx_qcount;
1181
1182         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1183 }
1184
1185                 void
1186 ef10_rx_fini(
1187         __in    efx_nic_t *enp)
1188 {
1189 #if EFSYS_OPT_RX_SCALE
1190         if (enp->en_rss_context_type != EFX_RX_SCALE_UNAVAILABLE)
1191                 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context);
1192         enp->en_rss_context = 0;
1193         enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
1194 #else
1195         _NOTE(ARGUNUSED(enp))
1196 #endif /* EFSYS_OPT_RX_SCALE */
1197 }
1198
1199 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 */