drivers/net/sfc/base/ef10_rx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright (c) 2012-2018 Solarflare Communications Inc.
   4  * All rights reserved.
   5  */
   6
   7 #include "efx.h"
   8 #include "efx_impl.h"
   9
  10
  11 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2
  12
  13
  14 static  __checkReturn   efx_rc_t
  15 efx_mcdi_init_rxq(
  16         __in            efx_nic_t *enp,
  17         __in            uint32_t ndescs,
  18         __in            uint32_t target_evq,
  19         __in            uint32_t label,
  20         __in            uint32_t instance,
  21         __in            efsys_mem_t *esmp,
  22         __in            boolean_t disable_scatter,
  23         __in            boolean_t want_inner_classes,
  24         __in            uint32_t ps_bufsize,
  25         __in            uint32_t es_bufs_per_desc,
  26         __in            uint32_t es_max_dma_len,
  27         __in            uint32_t es_buf_stride,
  28         __in            uint32_t hol_block_timeout)
  29 {
  30         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
  31         efx_mcdi_req_t req;
  32         uint8_t payload[MAX(MC_CMD_INIT_RXQ_V3_IN_LEN,
  33                             MC_CMD_INIT_RXQ_V3_OUT_LEN)];
  34         int npages = EFX_RXQ_NBUFS(ndescs);
  35         int i;
  36         efx_qword_t *dma_addr;
  37         uint64_t addr;
  38         efx_rc_t rc;
  39         uint32_t dma_mode;
  40         boolean_t want_outer_classes;
  41
  42         EFSYS_ASSERT3U(ndescs, <=, EFX_RXQ_MAXNDESCS);
  43
  44         if ((esmp == NULL) || (EFSYS_MEM_SIZE(esmp) < EFX_RXQ_SIZE(ndescs))) {
  45                 rc = EINVAL;
  46                 goto fail1;
  47         }
  48
  49         if (ps_bufsize > 0)
  50                 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_PACKED_STREAM;
  51         else if (es_bufs_per_desc > 0)
  52                 dma_mode = MC_CMD_INIT_RXQ_V3_IN_EQUAL_STRIDE_SUPER_BUFFER;
  53         else
  54                 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_SINGLE_PACKET;
  55
  56         if (encp->enc_tunnel_encapsulations_supported != 0 &&
  57             !want_inner_classes) {
  58                 /*
  59                  * WANT_OUTER_CLASSES can only be specified on hardware which
  60                  * supports tunnel encapsulation offloads, even though it is
  61                  * effectively the behaviour the hardware gives.
  62                  *
  63                  * Also, on hardware which does support such offloads, older
  64                  * firmware rejects the flag if the offloads are not supported
  65                  * by the current firmware variant, which means this may fail if
  66                  * the capabilities are not updated when the firmware variant
  67                  * changes. This is not an issue on newer firmware, as it was
  68                  * changed in bug 69842 (v6.4.2.1007) to permit this flag to be
  69                  * specified on all firmware variants.
  70                  */
  71                 want_outer_classes = B_TRUE;
  72         } else {
  73                 want_outer_classes = B_FALSE;
  74         }
  75
  76         (void) memset(payload, 0, sizeof (payload));
  77         req.emr_cmd = MC_CMD_INIT_RXQ;
  78         req.emr_in_buf = payload;
  79         req.emr_in_length = MC_CMD_INIT_RXQ_V3_IN_LEN;
  80         req.emr_out_buf = payload;
  81         req.emr_out_length = MC_CMD_INIT_RXQ_V3_OUT_LEN;
  82
  83         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_SIZE, ndescs);
  84         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_TARGET_EVQ, target_evq);
  85         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_LABEL, label);
  86         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_INSTANCE, instance);
  87         MCDI_IN_POPULATE_DWORD_9(req, INIT_RXQ_EXT_IN_FLAGS,
  88             INIT_RXQ_EXT_IN_FLAG_BUFF_MODE, 0,
  89             INIT_RXQ_EXT_IN_FLAG_HDR_SPLIT, 0,
  90             INIT_RXQ_EXT_IN_FLAG_TIMESTAMP, 0,
  91             INIT_RXQ_EXT_IN_CRC_MODE, 0,
  92             INIT_RXQ_EXT_IN_FLAG_PREFIX, 1,
  93             INIT_RXQ_EXT_IN_FLAG_DISABLE_SCATTER, disable_scatter,
  94             INIT_RXQ_EXT_IN_DMA_MODE,
  95             dma_mode,
  96             INIT_RXQ_EXT_IN_PACKED_STREAM_BUFF_SIZE, ps_bufsize,
  97             INIT_RXQ_EXT_IN_FLAG_WANT_OUTER_CLASSES, want_outer_classes);
  98         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_OWNER_ID, 0);
  99         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);
 100
 101         if (es_bufs_per_desc > 0) {
 102                 MCDI_IN_SET_DWORD(req,
 103                     INIT_RXQ_V3_IN_ES_PACKET_BUFFERS_PER_BUCKET,
 104                     es_bufs_per_desc);
 105                 MCDI_IN_SET_DWORD(req,
 106                     INIT_RXQ_V3_IN_ES_MAX_DMA_LEN, es_max_dma_len);
 107                 MCDI_IN_SET_DWORD(req,
 108                     INIT_RXQ_V3_IN_ES_PACKET_STRIDE, es_buf_stride);
 109                 MCDI_IN_SET_DWORD(req,
 110                     INIT_RXQ_V3_IN_ES_HEAD_OF_LINE_BLOCK_TIMEOUT,
 111                     hol_block_timeout);
 112         }
 113
 114         dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR);
 115         addr = EFSYS_MEM_ADDR(esmp);
 116
 117         for (i = 0; i < npages; i++) {
 118                 EFX_POPULATE_QWORD_2(*dma_addr,
 119                     EFX_DWORD_1, (uint32_t)(addr >> 32),
 120                     EFX_DWORD_0, (uint32_t)(addr & 0xffffffff));
 121
 122                 dma_addr++;
 123                 addr += EFX_BUF_SIZE;
 124         }
 125
 126         efx_mcdi_execute(enp, &req);
 127
 128         if (req.emr_rc != 0) {
 129                 rc = req.emr_rc;
 130                 goto fail2;
 131         }
 132
 133         return (0);
 134
 135 fail2:
 136         EFSYS_PROBE(fail2);
 137 fail1:
 138         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 139
 140         return (rc);
 141 }
 142
 143 static  __checkReturn   efx_rc_t
 144 efx_mcdi_fini_rxq(
 145         __in            efx_nic_t *enp,
 146         __in            uint32_t instance)
 147 {
 148         efx_mcdi_req_t req;
 149         uint8_t payload[MAX(MC_CMD_FINI_RXQ_IN_LEN,
 150                             MC_CMD_FINI_RXQ_OUT_LEN)];
 151         efx_rc_t rc;
 152
 153         (void) memset(payload, 0, sizeof (payload));
 154         req.emr_cmd = MC_CMD_FINI_RXQ;
 155         req.emr_in_buf = payload;
 156         req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN;
 157         req.emr_out_buf = payload;
 158         req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN;
 159
 160         MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance);
 161
 162         efx_mcdi_execute_quiet(enp, &req);
 163
 164         if (req.emr_rc != 0) {
 165                 rc = req.emr_rc;
 166                 goto fail1;
 167         }
 168
 169         return (0);
 170
 171 fail1:
 172         /*
 173          * EALREADY is not an error, but indicates that the MC has rebooted and
 174          * that the RXQ has already been destroyed.
 175          */
 176         if (rc != EALREADY)
 177                 EFSYS_PROBE1(fail1, efx_rc_t, rc);
 178
 179         return (rc);
 180 }
 181
 182 #if EFSYS_OPT_RX_SCALE
 183 static  __checkReturn   efx_rc_t
 184 efx_mcdi_rss_context_alloc(
 185         __in            efx_nic_t *enp,
 186         __in            efx_rx_scale_context_type_t type,
 187         __in            uint32_t num_queues,
 188         __out           uint32_t *rss_contextp)
 189 {
 190         efx_mcdi_req_t req;
 191         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN,
 192                             MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)];
 193         uint32_t rss_context;
 194         uint32_t context_type;
 195         efx_rc_t rc;
 196
 197         if (num_queues > EFX_MAXRSS) {
 198                 rc = EINVAL;
 199                 goto fail1;
 200         }
 201
 202         switch (type) {
 203         case EFX_RX_SCALE_EXCLUSIVE:
 204                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE;
 205                 break;
 206         case EFX_RX_SCALE_SHARED:
 207                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED;
 208                 break;
 209         default:
 210                 rc = EINVAL;
 211                 goto fail2;
 212         }
 213
 214         (void) memset(payload, 0, sizeof (payload));
 215         req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC;
 216         req.emr_in_buf = payload;
 217         req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN;
 218         req.emr_out_buf = payload;
 219         req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN;
 220
 221         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
 222             EVB_PORT_ID_ASSIGNED);
 223         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type);
 224
 225         /*
 226          * For exclusive contexts, NUM_QUEUES is only used to validate
 227          * indirection table offsets.
 228          * For shared contexts, the provided context will spread traffic over
 229          * NUM_QUEUES many queues.
 230          */
 231         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues);
 232
 233         efx_mcdi_execute(enp, &req);
 234
 235         if (req.emr_rc != 0) {
 236                 rc = req.emr_rc;
 237                 goto fail3;
 238         }
 239
 240         if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) {
 241                 rc = EMSGSIZE;
 242                 goto fail4;
 243         }
 244
 245         rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);
 246         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 247                 rc = ENOENT;
 248                 goto fail5;
 249         }
 250
 251         *rss_contextp = rss_context;
 252
 253         return (0);
 254
 255 fail5:
 256         EFSYS_PROBE(fail5);
 257 fail4:
 258         EFSYS_PROBE(fail4);
 259 fail3:
 260         EFSYS_PROBE(fail3);
 261 fail2:
 262         EFSYS_PROBE(fail2);
 263 fail1:
 264         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 265
 266         return (rc);
 267 }
 268 #endif /* EFSYS_OPT_RX_SCALE */
 269
 270 #if EFSYS_OPT_RX_SCALE
 271 static                  efx_rc_t
 272 efx_mcdi_rss_context_free(
 273         __in            efx_nic_t *enp,
 274         __in            uint32_t rss_context)
 275 {
 276         efx_mcdi_req_t req;
 277         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_FREE_IN_LEN,
 278                             MC_CMD_RSS_CONTEXT_FREE_OUT_LEN)];
 279         efx_rc_t rc;
 280
 281         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 282                 rc = EINVAL;
 283                 goto fail1;
 284         }
 285
 286         (void) memset(payload, 0, sizeof (payload));
 287         req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE;
 288         req.emr_in_buf = payload;
 289         req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN;
 290         req.emr_out_buf = payload;
 291         req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN;
 292
 293         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context);
 294
 295         efx_mcdi_execute_quiet(enp, &req);
 296
 297         if (req.emr_rc != 0) {
 298                 rc = req.emr_rc;
 299                 goto fail2;
 300         }
 301
 302         return (0);
 303
 304 fail2:
 305         EFSYS_PROBE(fail2);
 306 fail1:
 307         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 308
 309         return (rc);
 310 }
 311 #endif /* EFSYS_OPT_RX_SCALE */
 312
 313 #if EFSYS_OPT_RX_SCALE
 314 static                  efx_rc_t
 315 efx_mcdi_rss_context_set_flags(
 316         __in            efx_nic_t *enp,
 317         __in            uint32_t rss_context,
 318         __in            efx_rx_hash_type_t type)
 319 {
 320         efx_nic_cfg_t *encp = &enp->en_nic_cfg;
 321         efx_rx_hash_type_t type_ipv4;
 322         efx_rx_hash_type_t type_ipv4_tcp;
 323         efx_rx_hash_type_t type_ipv6;
 324         efx_rx_hash_type_t type_ipv6_tcp;
 325         efx_rx_hash_type_t modes;
 326         efx_mcdi_req_t req;
 327         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN,
 328                             MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN)];
 329         efx_rc_t rc;
 330
 331         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_LBN ==
 332                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_LBN);
 333         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_WIDTH ==
 334                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_WIDTH);
 335         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_LBN ==
 336                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_LBN);
 337         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_WIDTH ==
 338                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_WIDTH);
 339         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_TCP_LBN ==
 340                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_LBN);
 341         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_TCP_WIDTH ==
 342                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_WIDTH);
 343         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_LBN ==
 344                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_LBN);
 345         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_WIDTH ==
 346                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_WIDTH);
 347
 348         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 349                 rc = EINVAL;
 350                 goto fail1;
 351         }
 352
 353         (void) memset(payload, 0, sizeof (payload));
 354         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS;
 355         req.emr_in_buf = payload;
 356         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN;
 357         req.emr_out_buf = payload;
 358         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN;
 359
 360         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
 361             rss_context);
 362
 363         type_ipv4 = EFX_RX_HASH(IPV4, 2TUPLE) | EFX_RX_HASH(IPV4_TCP, 2TUPLE) |
 364                     EFX_RX_HASH(IPV4_UDP, 2TUPLE);
 365         type_ipv4_tcp = EFX_RX_HASH(IPV4_TCP, 4TUPLE);
 366         type_ipv6 = EFX_RX_HASH(IPV6, 2TUPLE) | EFX_RX_HASH(IPV6_TCP, 2TUPLE) |
 367                     EFX_RX_HASH(IPV6_UDP, 2TUPLE);
 368         type_ipv6_tcp = EFX_RX_HASH(IPV6_TCP, 4TUPLE);
 369
 370         /*
 371          * Create a copy of the original hash type.
 372          * The copy will be used to fill in RSS_MODE bits and
 373          * may be cleared beforehand. The original variable
 374          * and, thus, EN bits will remain unaffected.
 375          */
 376         modes = type;
 377
 378         /*
 379          * If the firmware lacks support for additional modes, RSS_MODE
 380          * fields must contain zeros, otherwise the operation will fail.
 381          */
 382         if (encp->enc_rx_scale_additional_modes_supported == B_FALSE)
 383                 modes = 0;
 384
 385 #define EXTRACT_RSS_MODE(_type, _class)         \
 386         (EFX_EXTRACT_NATIVE(_type, 0, 31,       \
 387         EFX_LOW_BIT(EFX_RX_CLASS_##_class),     \
 388         EFX_HIGH_BIT(EFX_RX_CLASS_##_class)) &  \
 389         EFX_MASK32(EFX_RX_CLASS_##_class))
 390
 391         MCDI_IN_POPULATE_DWORD_10(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS,
 392             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN,
 393             ((type & type_ipv4) == type_ipv4) ? 1 : 0,
 394             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN,
 395             ((type & type_ipv4_tcp) == type_ipv4_tcp) ? 1 : 0,
 396             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN,
 397             ((type & type_ipv6) == type_ipv6) ? 1 : 0,
 398             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN,
 399             ((type & type_ipv6_tcp) == type_ipv6_tcp) ? 1 : 0,
 400             RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE,
 401             EXTRACT_RSS_MODE(modes, IPV4_TCP),
 402             RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV4_RSS_MODE,
 403             EXTRACT_RSS_MODE(modes, IPV4_UDP),
 404             RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE,
 405             EXTRACT_RSS_MODE(modes, IPV4),
 406             RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE,
 407             EXTRACT_RSS_MODE(modes, IPV6_TCP),
 408             RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV6_RSS_MODE,
 409             EXTRACT_RSS_MODE(modes, IPV6_UDP),
 410             RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE,
 411             EXTRACT_RSS_MODE(modes, IPV6));
 412
 413 #undef EXTRACT_RSS_MODE
 414
 415         efx_mcdi_execute(enp, &req);
 416
 417         if (req.emr_rc != 0) {
 418                 rc = req.emr_rc;
 419                 goto fail2;
 420         }
 421
 422         return (0);
 423
 424 fail2:
 425         EFSYS_PROBE(fail2);
 426 fail1:
 427         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 428
 429         return (rc);
 430 }
 431 #endif /* EFSYS_OPT_RX_SCALE */
 432
 433 #if EFSYS_OPT_RX_SCALE
 434 static                  efx_rc_t
 435 efx_mcdi_rss_context_set_key(
 436         __in            efx_nic_t *enp,
 437         __in            uint32_t rss_context,
 438         __in_ecount(n)  uint8_t *key,
 439         __in            size_t n)
 440 {
 441         efx_mcdi_req_t req;
 442         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN,
 443                             MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN)];
 444         efx_rc_t rc;
 445
 446         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 447                 rc = EINVAL;
 448                 goto fail1;
 449         }
 450
 451         (void) memset(payload, 0, sizeof (payload));
 452         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY;
 453         req.emr_in_buf = payload;
 454         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN;
 455         req.emr_out_buf = payload;
 456         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN;
 457
 458         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
 459             rss_context);
 460
 461         EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
 462         if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) {
 463                 rc = EINVAL;
 464                 goto fail2;
 465         }
 466
 467         memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY),
 468             key, n);
 469
 470         efx_mcdi_execute(enp, &req);
 471
 472         if (req.emr_rc != 0) {
 473                 rc = req.emr_rc;
 474                 goto fail3;
 475         }
 476
 477         return (0);
 478
 479 fail3:
 480         EFSYS_PROBE(fail3);
 481 fail2:
 482         EFSYS_PROBE(fail2);
 483 fail1:
 484         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 485
 486         return (rc);
 487 }
 488 #endif /* EFSYS_OPT_RX_SCALE */
 489
 490 #if EFSYS_OPT_RX_SCALE
 491 static                  efx_rc_t
 492 efx_mcdi_rss_context_set_table(
 493         __in            efx_nic_t *enp,
 494         __in            uint32_t rss_context,
 495         __in_ecount(n)  unsigned int *table,
 496         __in            size_t n)
 497 {
 498         efx_mcdi_req_t req;
 499         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN,
 500                             MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN)];
 501         uint8_t *req_table;
 502         int i, rc;
 503
 504         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 505                 rc = EINVAL;
 506                 goto fail1;
 507         }
 508
 509         (void) memset(payload, 0, sizeof (payload));
 510         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE;
 511         req.emr_in_buf = payload;
 512         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN;
 513         req.emr_out_buf = payload;
 514         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN;
 515
 516         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
 517             rss_context);
 518
 519         req_table =
 520             MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE);
 521
 522         for (i = 0;
 523             i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN;
 524             i++) {
 525                 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0;
 526         }
 527
 528         efx_mcdi_execute(enp, &req);
 529
 530         if (req.emr_rc != 0) {
 531                 rc = req.emr_rc;
 532                 goto fail2;
 533         }
 534
 535         return (0);
 536
 537 fail2:
 538         EFSYS_PROBE(fail2);
 539 fail1:
 540         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 541
 542         return (rc);
 543 }
 544 #endif /* EFSYS_OPT_RX_SCALE */
 545
 546
 547         __checkReturn   efx_rc_t
 548 ef10_rx_init(
 549         __in            efx_nic_t *enp)
 550 {
 551 #if EFSYS_OPT_RX_SCALE
 552
 553         if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS,
 554                 &enp->en_rss_context) == 0) {
 555                 /*
 556                  * Allocated an exclusive RSS context, which allows both the
 557                  * indirection table and key to be modified.
 558                  */
 559                 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE;
 560                 enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
 561         } else {
 562                 /*
 563                  * Failed to allocate an exclusive RSS context. Continue
 564                  * operation without support for RSS. The pseudo-header in
 565                  * received packets will not contain a Toeplitz hash value.
 566                  */
 567                 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
 568                 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE;
 569         }
 570
 571 #endif /* EFSYS_OPT_RX_SCALE */
 572
 573         return (0);
 574 }
 575
 576 #if EFSYS_OPT_RX_SCATTER
 577         __checkReturn   efx_rc_t
 578 ef10_rx_scatter_enable(
 579         __in            efx_nic_t *enp,
 580         __in            unsigned int buf_size)
 581 {
 582         _NOTE(ARGUNUSED(enp, buf_size))
 583         return (0);
 584 }
 585 #endif  /* EFSYS_OPT_RX_SCATTER */
 586
 587 #if EFSYS_OPT_RX_SCALE
 588         __checkReturn   efx_rc_t
 589 ef10_rx_scale_context_alloc(
 590         __in            efx_nic_t *enp,
 591         __in            efx_rx_scale_context_type_t type,
 592         __in            uint32_t num_queues,
 593         __out           uint32_t *rss_contextp)
 594 {
 595         efx_rc_t rc;
 596
 597         rc = efx_mcdi_rss_context_alloc(enp, type, num_queues, rss_contextp);
 598         if (rc != 0)
 599                 goto fail1;
 600
 601         return (0);
 602
 603 fail1:
 604         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 605         return (rc);
 606 }
 607 #endif /* EFSYS_OPT_RX_SCALE */
 608
 609 #if EFSYS_OPT_RX_SCALE
 610         __checkReturn   efx_rc_t
 611 ef10_rx_scale_context_free(
 612         __in            efx_nic_t *enp,
 613         __in            uint32_t rss_context)
 614 {
 615         efx_rc_t rc;
 616
 617         rc = efx_mcdi_rss_context_free(enp, rss_context);
 618         if (rc != 0)
 619                 goto fail1;
 620
 621         return (0);
 622
 623 fail1:
 624         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 625         return (rc);
 626 }
 627 #endif /* EFSYS_OPT_RX_SCALE */
 628
 629 #if EFSYS_OPT_RX_SCALE
 630         __checkReturn   efx_rc_t
 631 ef10_rx_scale_mode_set(
 632         __in            efx_nic_t *enp,
 633         __in            uint32_t rss_context,
 634         __in            efx_rx_hash_alg_t alg,
 635         __in            efx_rx_hash_type_t type,
 636         __in            boolean_t insert)
 637 {
 638         efx_nic_cfg_t *encp = &enp->en_nic_cfg;
 639         efx_rc_t rc;
 640
 641         EFSYS_ASSERT3U(insert, ==, B_TRUE);
 642
 643         if ((encp->enc_rx_scale_hash_alg_mask & (1U << alg)) == 0 ||
 644             insert == B_FALSE) {
 645                 rc = EINVAL;
 646                 goto fail1;
 647         }
 648
 649         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 650                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 651                         rc = ENOTSUP;
 652                         goto fail2;
 653                 }
 654                 rss_context = enp->en_rss_context;
 655         }
 656
 657         if ((rc = efx_mcdi_rss_context_set_flags(enp,
 658                     rss_context, type)) != 0)
 659                 goto fail3;
 660
 661         return (0);
 662
 663 fail3:
 664         EFSYS_PROBE(fail3);
 665 fail2:
 666         EFSYS_PROBE(fail2);
 667 fail1:
 668         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 669
 670         return (rc);
 671 }
 672 #endif /* EFSYS_OPT_RX_SCALE */
 673
 674 #if EFSYS_OPT_RX_SCALE
 675         __checkReturn   efx_rc_t
 676 ef10_rx_scale_key_set(
 677         __in            efx_nic_t *enp,
 678         __in            uint32_t rss_context,
 679         __in_ecount(n)  uint8_t *key,
 680         __in            size_t n)
 681 {
 682         efx_rc_t rc;
 683
 684         EFX_STATIC_ASSERT(EFX_RSS_KEY_SIZE ==
 685             MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
 686
 687         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 688                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 689                         rc = ENOTSUP;
 690                         goto fail1;
 691                 }
 692                 rss_context = enp->en_rss_context;
 693         }
 694
 695         if ((rc = efx_mcdi_rss_context_set_key(enp, rss_context, key, n)) != 0)
 696                 goto fail2;
 697
 698         return (0);
 699
 700 fail2:
 701         EFSYS_PROBE(fail2);
 702 fail1:
 703         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 704
 705         return (rc);
 706 }
 707 #endif /* EFSYS_OPT_RX_SCALE */
 708
 709 #if EFSYS_OPT_RX_SCALE
 710         __checkReturn   efx_rc_t
 711 ef10_rx_scale_tbl_set(
 712         __in            efx_nic_t *enp,
 713         __in            uint32_t rss_context,
 714         __in_ecount(n)  unsigned int *table,
 715         __in            size_t n)
 716 {
 717         efx_rc_t rc;
 718
 719
 720         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 721                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 722                         rc = ENOTSUP;
 723                         goto fail1;
 724                 }
 725                 rss_context = enp->en_rss_context;
 726         }
 727
 728         if ((rc = efx_mcdi_rss_context_set_table(enp,
 729                     rss_context, table, n)) != 0)
 730                 goto fail2;
 731
 732         return (0);
 733
 734 fail2:
 735         EFSYS_PROBE(fail2);
 736 fail1:
 737         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 738
 739         return (rc);
 740 }
 741 #endif /* EFSYS_OPT_RX_SCALE */
 742
 743
 744 /*
 745  * EF10 RX pseudo-header
 746  * ---------------------
 747  *
 748  * Receive packets are prefixed by an (optional) 14 byte pseudo-header:
 749  *
 750  *  +00: Toeplitz hash value.
 751  *       (32bit little-endian)
 752  *  +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag.
 753  *       (16bit big-endian)
 754  *  +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag.
 755  *       (16bit big-endian)
 756  *  +08: Packet Length. Zero if the RX datapath was in cut-through mode.
 757  *       (16bit little-endian)
 758  *  +10: MAC timestamp. Zero if timestamping is not enabled.
 759  *       (32bit little-endian)
 760  *
 761  * See "The RX Pseudo-header" in SF-109306-TC.
 762  */
 763
 764         __checkReturn   efx_rc_t
 765 ef10_rx_prefix_pktlen(
 766         __in            efx_nic_t *enp,
 767         __in            uint8_t *buffer,
 768         __out           uint16_t *lengthp)
 769 {
 770         _NOTE(ARGUNUSED(enp))
 771
 772         /*
 773          * The RX pseudo-header contains the packet length, excluding the
 774          * pseudo-header. If the hardware receive datapath was operating in
 775          * cut-through mode then the length in the RX pseudo-header will be
 776          * zero, and the packet length must be obtained from the DMA length
 777          * reported in the RX event.
 778          */
 779         *lengthp = buffer[8] | (buffer[9] << 8);
 780         return (0);
 781 }
 782
 783 #if EFSYS_OPT_RX_SCALE
 784         __checkReturn   uint32_t
 785 ef10_rx_prefix_hash(
 786         __in            efx_nic_t *enp,
 787         __in            efx_rx_hash_alg_t func,
 788         __in            uint8_t *buffer)
 789 {
 790         _NOTE(ARGUNUSED(enp))
 791
 792         switch (func) {
 793         case EFX_RX_HASHALG_PACKED_STREAM:
 794         case EFX_RX_HASHALG_TOEPLITZ:
 795                 return (buffer[0] |
 796                     (buffer[1] << 8) |
 797                     (buffer[2] << 16) |
 798                     (buffer[3] << 24));
 799
 800         default:
 801                 EFSYS_ASSERT(0);
 802                 return (0);
 803         }
 804 }
 805 #endif /* EFSYS_OPT_RX_SCALE */
 806
 807 #if EFSYS_OPT_RX_PACKED_STREAM
 808 /*
 809  * Fake length for RXQ descriptors in packed stream mode
 810  * to make hardware happy
 811  */
 812 #define EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE 32
 813 #endif
 814
 815                                 void
 816 ef10_rx_qpost(
 817         __in                    efx_rxq_t *erp,
 818         __in_ecount(ndescs)     efsys_dma_addr_t *addrp,
 819         __in                    size_t size,
 820         __in                    unsigned int ndescs,
 821         __in                    unsigned int completed,
 822         __in                    unsigned int added)
 823 {
 824         efx_qword_t qword;
 825         unsigned int i;
 826         unsigned int offset;
 827         unsigned int id;
 828
 829         _NOTE(ARGUNUSED(completed))
 830
 831 #if EFSYS_OPT_RX_PACKED_STREAM
 832         /*
 833          * Real size of the buffer does not fit into ESF_DZ_RX_KER_BYTE_CNT
 834          * and equal to 0 after applying mask. Hardware does not like it.
 835          */
 836         if (erp->er_ev_qstate->eers_rx_packed_stream)
 837                 size = EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE;
 838 #endif
 839
 840         /* The client driver must not overfill the queue */
 841         EFSYS_ASSERT3U(added - completed + ndescs, <=,
 842             EFX_RXQ_LIMIT(erp->er_mask + 1));
 843
 844         id = added & (erp->er_mask);
 845         for (i = 0; i < ndescs; i++) {
 846                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
 847                     unsigned int, id, efsys_dma_addr_t, addrp[i],
 848                     size_t, size);
 849
 850                 EFX_POPULATE_QWORD_3(qword,
 851                     ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size),
 852                     ESF_DZ_RX_KER_BUF_ADDR_DW0,
 853                     (uint32_t)(addrp[i] & 0xffffffff),
 854                     ESF_DZ_RX_KER_BUF_ADDR_DW1,
 855                     (uint32_t)(addrp[i] >> 32));
 856
 857                 offset = id * sizeof (efx_qword_t);
 858                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
 859
 860                 id = (id + 1) & (erp->er_mask);
 861         }
 862 }
 863
 864                         void
 865 ef10_rx_qpush(
 866         __in    efx_rxq_t *erp,
 867         __in    unsigned int added,
 868         __inout unsigned int *pushedp)
 869 {
 870         efx_nic_t *enp = erp->er_enp;
 871         unsigned int pushed = *pushedp;
 872         uint32_t wptr;
 873         efx_dword_t dword;
 874
 875         /* Hardware has alignment restriction for WPTR */
 876         wptr = P2ALIGN(added, EF10_RX_WPTR_ALIGN);
 877         if (pushed == wptr)
 878                 return;
 879
 880         *pushedp = wptr;
 881
 882         /* Push the populated descriptors out */
 883         wptr &= erp->er_mask;
 884
 885         EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr);
 886
 887         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
 888         EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
 889             wptr, pushed & erp->er_mask);
 890         EFSYS_PIO_WRITE_BARRIER();
 891         EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
 892             erp->er_index, &dword, B_FALSE);
 893 }
 894
 895 #if EFSYS_OPT_RX_PACKED_STREAM
 896
 897                         void
 898 ef10_rx_qpush_ps_credits(
 899         __in            efx_rxq_t *erp)
 900 {
 901         efx_nic_t *enp = erp->er_enp;
 902         efx_dword_t dword;
 903         efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate;
 904         uint32_t credits;
 905
 906         EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
 907
 908         if (rxq_state->eers_rx_packed_stream_credits == 0)
 909                 return;
 910
 911         /*
 912          * It is a bug if we think that FW has utilized more
 913          * credits than it is allowed to have (maximum). However,
 914          * make sure that we do not credit more than maximum anyway.
 915          */
 916         credits = MIN(rxq_state->eers_rx_packed_stream_credits,
 917             EFX_RX_PACKED_STREAM_MAX_CREDITS);
 918         EFX_POPULATE_DWORD_3(dword,
 919             ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1,
 920             ERF_DZ_RX_DESC_MAGIC_CMD,
 921             ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS,
 922             ERF_DZ_RX_DESC_MAGIC_DATA, credits);
 923         EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
 924             erp->er_index, &dword, B_FALSE);
 925
 926         rxq_state->eers_rx_packed_stream_credits = 0;
 927 }
 928
 929 /*
 930  * In accordance with SF-112241-TC the received data has the following layout:
 931  *  - 8 byte pseudo-header which consist of:
 932  *    - 4 byte little-endian timestamp
 933  *    - 2 byte little-endian captured length in bytes
 934  *    - 2 byte little-endian original packet length in bytes
 935  *  - captured packet bytes
 936  *  - optional padding to align to 64 bytes boundary
 937  *  - 64 bytes scratch space for the host software
 938  */
 939         __checkReturn   uint8_t *
 940 ef10_rx_qps_packet_info(
 941         __in            efx_rxq_t *erp,
 942         __in            uint8_t *buffer,
 943         __in            uint32_t buffer_length,
 944         __in            uint32_t current_offset,
 945         __out           uint16_t *lengthp,
 946         __out           uint32_t *next_offsetp,
 947         __out           uint32_t *timestamp)
 948 {
 949         uint16_t buf_len;
 950         uint8_t *pkt_start;
 951         efx_qword_t *qwordp;
 952         efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate;
 953
 954         EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
 955
 956         buffer += current_offset;
 957         pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE;
 958
 959         qwordp = (efx_qword_t *)buffer;
 960         *timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP);
 961         *lengthp   = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN);
 962         buf_len    = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN);
 963
 964         buf_len = P2ROUNDUP(buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE,
 965                             EFX_RX_PACKED_STREAM_ALIGNMENT);
 966         *next_offsetp =
 967             current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT;
 968
 969         EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length);
 970         EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp);
 971
 972         if ((*next_offsetp ^ current_offset) &
 973             EFX_RX_PACKED_STREAM_MEM_PER_CREDIT)
 974                 rxq_state->eers_rx_packed_stream_credits++;
 975
 976         return (pkt_start);
 977 }
 978
 979
 980 #endif
 981
 982         __checkReturn   efx_rc_t
 983 ef10_rx_qflush(
 984         __in    efx_rxq_t *erp)
 985 {
 986         efx_nic_t *enp = erp->er_enp;
 987         efx_rc_t rc;
 988
 989         if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0)
 990                 goto fail1;
 991
 992         return (0);
 993
 994 fail1:
 995         /*
 996          * EALREADY is not an error, but indicates that the MC has rebooted and
 997          * that the RXQ has already been destroyed. Callers need to know that
 998          * the RXQ flush has completed to avoid waiting until timeout for a
 999          * flush done event that will not be delivered.
1000          */
1001         if (rc != EALREADY)
1002                 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1003
1004         return (rc);
1005 }
1006
1007                 void
1008 ef10_rx_qenable(
1009         __in    efx_rxq_t *erp)
1010 {
1011         /* FIXME */
1012         _NOTE(ARGUNUSED(erp))
1013         /* FIXME */
1014 }
1015
1016         __checkReturn   efx_rc_t
1017 ef10_rx_qcreate(
1018         __in            efx_nic_t *enp,
1019         __in            unsigned int index,
1020         __in            unsigned int label,
1021         __in            efx_rxq_type_t type,
1022         __in            const efx_rxq_type_data_t *type_data,
1023         __in            efsys_mem_t *esmp,
1024         __in            size_t ndescs,
1025         __in            uint32_t id,
1026         __in            unsigned int flags,
1027         __in            efx_evq_t *eep,
1028         __in            efx_rxq_t *erp)
1029 {
1030         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1031         efx_rc_t rc;
1032         boolean_t disable_scatter;
1033         boolean_t want_inner_classes;
1034         unsigned int ps_buf_size;
1035         uint32_t es_bufs_per_desc = 0;
1036         uint32_t es_max_dma_len = 0;
1037         uint32_t es_buf_stride = 0;
1038         uint32_t hol_block_timeout = 0;
1039
1040         _NOTE(ARGUNUSED(id, erp, type_data))
1041
1042         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH));
1043         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
1044         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
1045
1046         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
1047         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
1048
1049         if (!ISP2(ndescs) ||
1050             (ndescs < EFX_RXQ_MINNDESCS) || (ndescs > EFX_RXQ_MAXNDESCS)) {
1051                 rc = EINVAL;
1052                 goto fail1;
1053         }
1054         if (index >= encp->enc_rxq_limit) {
1055                 rc = EINVAL;
1056                 goto fail2;
1057         }
1058
1059         switch (type) {
1060         case EFX_RXQ_TYPE_DEFAULT:
1061                 ps_buf_size = 0;
1062                 break;
1063 #if EFSYS_OPT_RX_PACKED_STREAM
1064         case EFX_RXQ_TYPE_PACKED_STREAM:
1065                 switch (type_data->ertd_packed_stream.eps_buf_size) {
1066                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_1M:
1067                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M;
1068                         break;
1069                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_512K:
1070                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K;
1071                         break;
1072                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_256K:
1073                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K;
1074                         break;
1075                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_128K:
1076                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K;
1077                         break;
1078                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_64K:
1079                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K;
1080                         break;
1081                 default:
1082                         rc = ENOTSUP;
1083                         goto fail3;
1084                 }
1085                 break;
1086 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1087 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
1088         case EFX_RXQ_TYPE_ES_SUPER_BUFFER:
1089                 ps_buf_size = 0;
1090                 es_bufs_per_desc =
1091                     type_data->ertd_es_super_buffer.eessb_bufs_per_desc;
1092                 es_max_dma_len =
1093                     type_data->ertd_es_super_buffer.eessb_max_dma_len;
1094                 es_buf_stride =
1095                     type_data->ertd_es_super_buffer.eessb_buf_stride;
1096                 hol_block_timeout =
1097                     type_data->ertd_es_super_buffer.eessb_hol_block_timeout;
1098                 break;
1099 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1100         default:
1101                 rc = ENOTSUP;
1102                 goto fail4;
1103         }
1104
1105 #if EFSYS_OPT_RX_PACKED_STREAM
1106         if (ps_buf_size != 0) {
1107                 /* Check if datapath firmware supports packed stream mode */
1108                 if (encp->enc_rx_packed_stream_supported == B_FALSE) {
1109                         rc = ENOTSUP;
1110                         goto fail5;
1111                 }
1112                 /* Check if packed stream allows configurable buffer sizes */
1113                 if ((ps_buf_size != MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M) &&
1114                     (encp->enc_rx_var_packed_stream_supported == B_FALSE)) {
1115                         rc = ENOTSUP;
1116                         goto fail6;
1117                 }
1118         }
1119 #else /* EFSYS_OPT_RX_PACKED_STREAM */
1120         EFSYS_ASSERT(ps_buf_size == 0);
1121 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1122
1123 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
1124         if (es_bufs_per_desc > 0) {
1125                 if (encp->enc_rx_es_super_buffer_supported == B_FALSE) {
1126                         rc = ENOTSUP;
1127                         goto fail7;
1128                 }
1129                 if (!IS_P2ALIGNED(es_max_dma_len,
1130                             EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) {
1131                         rc = EINVAL;
1132                         goto fail8;
1133                 }
1134                 if (!IS_P2ALIGNED(es_buf_stride,
1135                             EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) {
1136                         rc = EINVAL;
1137                         goto fail9;
1138                 }
1139         }
1140 #else /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1141         EFSYS_ASSERT(es_bufs_per_desc == 0);
1142 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1143
1144         /* Scatter can only be disabled if the firmware supports doing so */
1145         if (flags & EFX_RXQ_FLAG_SCATTER)
1146                 disable_scatter = B_FALSE;
1147         else
1148                 disable_scatter = encp->enc_rx_disable_scatter_supported;
1149
1150         if (flags & EFX_RXQ_FLAG_INNER_CLASSES)
1151                 want_inner_classes = B_TRUE;
1152         else
1153                 want_inner_classes = B_FALSE;
1154
1155         if ((rc = efx_mcdi_init_rxq(enp, ndescs, eep->ee_index, label, index,
1156                     esmp, disable_scatter, want_inner_classes,
1157                     ps_buf_size, es_bufs_per_desc, es_max_dma_len,
1158                     es_buf_stride, hol_block_timeout)) != 0)
1159                 goto fail10;
1160
1161         erp->er_eep = eep;
1162         erp->er_label = label;
1163
1164         ef10_ev_rxlabel_init(eep, erp, label, type);
1165
1166         erp->er_ev_qstate = &erp->er_eep->ee_rxq_state[label];
1167
1168         return (0);
1169
1170 fail10:
1171         EFSYS_PROBE(fail10);
1172 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
1173 fail9:
1174         EFSYS_PROBE(fail9);
1175 fail8:
1176         EFSYS_PROBE(fail8);
1177 fail7:
1178         EFSYS_PROBE(fail7);
1179 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1180 #if EFSYS_OPT_RX_PACKED_STREAM
1181 fail6:
1182         EFSYS_PROBE(fail6);
1183 fail5:
1184         EFSYS_PROBE(fail5);
1185 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1186 fail4:
1187         EFSYS_PROBE(fail4);
1188 #if EFSYS_OPT_RX_PACKED_STREAM
1189 fail3:
1190         EFSYS_PROBE(fail3);
1191 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1192 fail2:
1193         EFSYS_PROBE(fail2);
1194 fail1:
1195         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1196
1197         return (rc);
1198 }
1199
1200                 void
1201 ef10_rx_qdestroy(
1202         __in    efx_rxq_t *erp)
1203 {
1204         efx_nic_t *enp = erp->er_enp;
1205         efx_evq_t *eep = erp->er_eep;
1206         unsigned int label = erp->er_label;
1207
1208         ef10_ev_rxlabel_fini(eep, label);
1209
1210         EFSYS_ASSERT(enp->en_rx_qcount != 0);
1211         --enp->en_rx_qcount;
1212
1213         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1214 }
1215
1216                 void
1217 ef10_rx_fini(
1218         __in    efx_nic_t *enp)
1219 {
1220 #if EFSYS_OPT_RX_SCALE
1221         if (enp->en_rss_context_type != EFX_RX_SCALE_UNAVAILABLE)
1222                 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context);
1223         enp->en_rss_context = 0;
1224         enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
1225 #else
1226         _NOTE(ARGUNUSED(enp))
1227 #endif /* EFSYS_OPT_RX_SCALE */
1228 }
1229
1230 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 */