drivers/net/sfc/base/ef10_rx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright (c) 2012-2018 Solarflare Communications Inc.
   4  * All rights reserved.
   5  */
   6
   7 #include "efx.h"
   8 #include "efx_impl.h"
   9
  10
  11 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
  12
  13
  14 static  __checkReturn   efx_rc_t
  15 efx_mcdi_init_rxq(
  16         __in            efx_nic_t *enp,
  17         __in            uint32_t ndescs,
  18         __in            uint32_t target_evq,
  19         __in            uint32_t label,
  20         __in            uint32_t instance,
  21         __in            efsys_mem_t *esmp,
  22         __in            boolean_t disable_scatter,
  23         __in            boolean_t want_inner_classes,
  24         __in            uint32_t ps_bufsize)
  25 {
  26         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
  27         efx_mcdi_req_t req;
  28         uint8_t payload[MAX(MC_CMD_INIT_RXQ_EXT_IN_LEN,
  29                             MC_CMD_INIT_RXQ_EXT_OUT_LEN)];
  30         int npages = EFX_RXQ_NBUFS(ndescs);
  31         int i;
  32         efx_qword_t *dma_addr;
  33         uint64_t addr;
  34         efx_rc_t rc;
  35         uint32_t dma_mode;
  36         boolean_t want_outer_classes;
  37
  38         EFSYS_ASSERT3U(ndescs, <=, EFX_RXQ_MAXNDESCS);
  39
  40         if (ps_bufsize > 0)
  41                 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_PACKED_STREAM;
  42         else
  43                 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_SINGLE_PACKET;
  44
  45         if (encp->enc_tunnel_encapsulations_supported != 0 &&
  46             !want_inner_classes) {
  47                 /*
  48                  * WANT_OUTER_CLASSES can only be specified on hardware which
  49                  * supports tunnel encapsulation offloads, even though it is
  50                  * effectively the behaviour the hardware gives.
  51                  *
  52                  * Also, on hardware which does support such offloads, older
  53                  * firmware rejects the flag if the offloads are not supported
  54                  * by the current firmware variant, which means this may fail if
  55                  * the capabilities are not updated when the firmware variant
  56                  * changes. This is not an issue on newer firmware, as it was
  57                  * changed in bug 69842 (v6.4.2.1007) to permit this flag to be
  58                  * specified on all firmware variants.
  59                  */
  60                 want_outer_classes = B_TRUE;
  61         } else {
  62                 want_outer_classes = B_FALSE;
  63         }
  64
  65         (void) memset(payload, 0, sizeof (payload));
  66         req.emr_cmd = MC_CMD_INIT_RXQ;
  67         req.emr_in_buf = payload;
  68         req.emr_in_length = MC_CMD_INIT_RXQ_EXT_IN_LEN;
  69         req.emr_out_buf = payload;
  70         req.emr_out_length = MC_CMD_INIT_RXQ_EXT_OUT_LEN;
  71
  72         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_SIZE, ndescs);
  73         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_TARGET_EVQ, target_evq);
  74         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_LABEL, label);
  75         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_INSTANCE, instance);
  76         MCDI_IN_POPULATE_DWORD_9(req, INIT_RXQ_EXT_IN_FLAGS,
  77             INIT_RXQ_EXT_IN_FLAG_BUFF_MODE, 0,
  78             INIT_RXQ_EXT_IN_FLAG_HDR_SPLIT, 0,
  79             INIT_RXQ_EXT_IN_FLAG_TIMESTAMP, 0,
  80             INIT_RXQ_EXT_IN_CRC_MODE, 0,
  81             INIT_RXQ_EXT_IN_FLAG_PREFIX, 1,
  82             INIT_RXQ_EXT_IN_FLAG_DISABLE_SCATTER, disable_scatter,
  83             INIT_RXQ_EXT_IN_DMA_MODE,
  84             dma_mode,
  85             INIT_RXQ_EXT_IN_PACKED_STREAM_BUFF_SIZE, ps_bufsize,
  86             INIT_RXQ_EXT_IN_FLAG_WANT_OUTER_CLASSES, want_outer_classes);
  87         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_OWNER_ID, 0);
  88         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);
  89
  90         dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR);
  91         addr = EFSYS_MEM_ADDR(esmp);
  92
  93         for (i = 0; i < npages; i++) {
  94                 EFX_POPULATE_QWORD_2(*dma_addr,
  95                     EFX_DWORD_1, (uint32_t)(addr >> 32),
  96                     EFX_DWORD_0, (uint32_t)(addr & 0xffffffff));
  97
  98                 dma_addr++;
  99                 addr += EFX_BUF_SIZE;
 100         }
 101
 102         efx_mcdi_execute(enp, &req);
 103
 104         if (req.emr_rc != 0) {
 105                 rc = req.emr_rc;
 106                 goto fail1;
 107         }
 108
 109         return (0);
 110
 111 fail1:
 112         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 113
 114         return (rc);
 115 }
 116
 117 static  __checkReturn   efx_rc_t
 118 efx_mcdi_fini_rxq(
 119         __in            efx_nic_t *enp,
 120         __in            uint32_t instance)
 121 {
 122         efx_mcdi_req_t req;
 123         uint8_t payload[MAX(MC_CMD_FINI_RXQ_IN_LEN,
 124                             MC_CMD_FINI_RXQ_OUT_LEN)];
 125         efx_rc_t rc;
 126
 127         (void) memset(payload, 0, sizeof (payload));
 128         req.emr_cmd = MC_CMD_FINI_RXQ;
 129         req.emr_in_buf = payload;
 130         req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN;
 131         req.emr_out_buf = payload;
 132         req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN;
 133
 134         MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance);
 135
 136         efx_mcdi_execute_quiet(enp, &req);
 137
 138         if (req.emr_rc != 0) {
 139                 rc = req.emr_rc;
 140                 goto fail1;
 141         }
 142
 143         return (0);
 144
 145 fail1:
 146         /*
 147          * EALREADY is not an error, but indicates that the MC has rebooted and
 148          * that the RXQ has already been destroyed.
 149          */
 150         if (rc != EALREADY)
 151                 EFSYS_PROBE1(fail1, efx_rc_t, rc);
 152
 153         return (rc);
 154 }
 155
 156 #if EFSYS_OPT_RX_SCALE
 157 static  __checkReturn   efx_rc_t
 158 efx_mcdi_rss_context_alloc(
 159         __in            efx_nic_t *enp,
 160         __in            efx_rx_scale_context_type_t type,
 161         __in            uint32_t num_queues,
 162         __out           uint32_t *rss_contextp)
 163 {
 164         efx_mcdi_req_t req;
 165         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN,
 166                             MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)];
 167         uint32_t rss_context;
 168         uint32_t context_type;
 169         efx_rc_t rc;
 170
 171         if (num_queues > EFX_MAXRSS) {
 172                 rc = EINVAL;
 173                 goto fail1;
 174         }
 175
 176         switch (type) {
 177         case EFX_RX_SCALE_EXCLUSIVE:
 178                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE;
 179                 break;
 180         case EFX_RX_SCALE_SHARED:
 181                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED;
 182                 break;
 183         default:
 184                 rc = EINVAL;
 185                 goto fail2;
 186         }
 187
 188         (void) memset(payload, 0, sizeof (payload));
 189         req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC;
 190         req.emr_in_buf = payload;
 191         req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN;
 192         req.emr_out_buf = payload;
 193         req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN;
 194
 195         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
 196             EVB_PORT_ID_ASSIGNED);
 197         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type);
 198
 199         /*
 200          * For exclusive contexts, NUM_QUEUES is only used to validate
 201          * indirection table offsets.
 202          * For shared contexts, the provided context will spread traffic over
 203          * NUM_QUEUES many queues.
 204          */
 205         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues);
 206
 207         efx_mcdi_execute(enp, &req);
 208
 209         if (req.emr_rc != 0) {
 210                 rc = req.emr_rc;
 211                 goto fail3;
 212         }
 213
 214         if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) {
 215                 rc = EMSGSIZE;
 216                 goto fail4;
 217         }
 218
 219         rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);
 220         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 221                 rc = ENOENT;
 222                 goto fail5;
 223         }
 224
 225         *rss_contextp = rss_context;
 226
 227         return (0);
 228
 229 fail5:
 230         EFSYS_PROBE(fail5);
 231 fail4:
 232         EFSYS_PROBE(fail4);
 233 fail3:
 234         EFSYS_PROBE(fail3);
 235 fail2:
 236         EFSYS_PROBE(fail2);
 237 fail1:
 238         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 239
 240         return (rc);
 241 }
 242 #endif /* EFSYS_OPT_RX_SCALE */
 243
 244 #if EFSYS_OPT_RX_SCALE
 245 static                  efx_rc_t
 246 efx_mcdi_rss_context_free(
 247         __in            efx_nic_t *enp,
 248         __in            uint32_t rss_context)
 249 {
 250         efx_mcdi_req_t req;
 251         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_FREE_IN_LEN,
 252                             MC_CMD_RSS_CONTEXT_FREE_OUT_LEN)];
 253         efx_rc_t rc;
 254
 255         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 256                 rc = EINVAL;
 257                 goto fail1;
 258         }
 259
 260         (void) memset(payload, 0, sizeof (payload));
 261         req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE;
 262         req.emr_in_buf = payload;
 263         req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN;
 264         req.emr_out_buf = payload;
 265         req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN;
 266
 267         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context);
 268
 269         efx_mcdi_execute_quiet(enp, &req);
 270
 271         if (req.emr_rc != 0) {
 272                 rc = req.emr_rc;
 273                 goto fail2;
 274         }
 275
 276         return (0);
 277
 278 fail2:
 279         EFSYS_PROBE(fail2);
 280 fail1:
 281         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 282
 283         return (rc);
 284 }
 285 #endif /* EFSYS_OPT_RX_SCALE */
 286
 287 #if EFSYS_OPT_RX_SCALE
 288 static                  efx_rc_t
 289 efx_mcdi_rss_context_set_flags(
 290         __in            efx_nic_t *enp,
 291         __in            uint32_t rss_context,
 292         __in            efx_rx_hash_type_t type)
 293 {
 294         efx_mcdi_req_t req;
 295         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN,
 296                             MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN)];
 297         efx_rc_t rc;
 298
 299         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 300                 rc = EINVAL;
 301                 goto fail1;
 302         }
 303
 304         (void) memset(payload, 0, sizeof (payload));
 305         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS;
 306         req.emr_in_buf = payload;
 307         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN;
 308         req.emr_out_buf = payload;
 309         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN;
 310
 311         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
 312             rss_context);
 313
 314         MCDI_IN_POPULATE_DWORD_4(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS,
 315             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN,
 316             (type & EFX_RX_HASH_IPV4) ? 1 : 0,
 317             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN,
 318             (type & EFX_RX_HASH_TCPIPV4) ? 1 : 0,
 319             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN,
 320             (type & EFX_RX_HASH_IPV6) ? 1 : 0,
 321             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN,
 322             (type & EFX_RX_HASH_TCPIPV6) ? 1 : 0);
 323
 324         efx_mcdi_execute(enp, &req);
 325
 326         if (req.emr_rc != 0) {
 327                 rc = req.emr_rc;
 328                 goto fail2;
 329         }
 330
 331         return (0);
 332
 333 fail2:
 334         EFSYS_PROBE(fail2);
 335 fail1:
 336         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 337
 338         return (rc);
 339 }
 340 #endif /* EFSYS_OPT_RX_SCALE */
 341
 342 #if EFSYS_OPT_RX_SCALE
 343 static                  efx_rc_t
 344 efx_mcdi_rss_context_set_key(
 345         __in            efx_nic_t *enp,
 346         __in            uint32_t rss_context,
 347         __in_ecount(n)  uint8_t *key,
 348         __in            size_t n)
 349 {
 350         efx_mcdi_req_t req;
 351         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN,
 352                             MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN)];
 353         efx_rc_t rc;
 354
 355         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 356                 rc = EINVAL;
 357                 goto fail1;
 358         }
 359
 360         (void) memset(payload, 0, sizeof (payload));
 361         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY;
 362         req.emr_in_buf = payload;
 363         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN;
 364         req.emr_out_buf = payload;
 365         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN;
 366
 367         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
 368             rss_context);
 369
 370         EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
 371         if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) {
 372                 rc = EINVAL;
 373                 goto fail2;
 374         }
 375
 376         memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY),
 377             key, n);
 378
 379         efx_mcdi_execute(enp, &req);
 380
 381         if (req.emr_rc != 0) {
 382                 rc = req.emr_rc;
 383                 goto fail3;
 384         }
 385
 386         return (0);
 387
 388 fail3:
 389         EFSYS_PROBE(fail3);
 390 fail2:
 391         EFSYS_PROBE(fail2);
 392 fail1:
 393         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 394
 395         return (rc);
 396 }
 397 #endif /* EFSYS_OPT_RX_SCALE */
 398
 399 #if EFSYS_OPT_RX_SCALE
 400 static                  efx_rc_t
 401 efx_mcdi_rss_context_set_table(
 402         __in            efx_nic_t *enp,
 403         __in            uint32_t rss_context,
 404         __in_ecount(n)  unsigned int *table,
 405         __in            size_t n)
 406 {
 407         efx_mcdi_req_t req;
 408         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN,
 409                             MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN)];
 410         uint8_t *req_table;
 411         int i, rc;
 412
 413         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 414                 rc = EINVAL;
 415                 goto fail1;
 416         }
 417
 418         (void) memset(payload, 0, sizeof (payload));
 419         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE;
 420         req.emr_in_buf = payload;
 421         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN;
 422         req.emr_out_buf = payload;
 423         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN;
 424
 425         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
 426             rss_context);
 427
 428         req_table =
 429             MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE);
 430
 431         for (i = 0;
 432             i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN;
 433             i++) {
 434                 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0;
 435         }
 436
 437         efx_mcdi_execute(enp, &req);
 438
 439         if (req.emr_rc != 0) {
 440                 rc = req.emr_rc;
 441                 goto fail2;
 442         }
 443
 444         return (0);
 445
 446 fail2:
 447         EFSYS_PROBE(fail2);
 448 fail1:
 449         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 450
 451         return (rc);
 452 }
 453 #endif /* EFSYS_OPT_RX_SCALE */
 454
 455
 456         __checkReturn   efx_rc_t
 457 ef10_rx_init(
 458         __in            efx_nic_t *enp)
 459 {
 460 #if EFSYS_OPT_RX_SCALE
 461
 462         if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS,
 463                 &enp->en_rss_context) == 0) {
 464                 /*
 465                  * Allocated an exclusive RSS context, which allows both the
 466                  * indirection table and key to be modified.
 467                  */
 468                 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE;
 469                 enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
 470         } else {
 471                 /*
 472                  * Failed to allocate an exclusive RSS context. Continue
 473                  * operation without support for RSS. The pseudo-header in
 474                  * received packets will not contain a Toeplitz hash value.
 475                  */
 476                 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
 477                 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE;
 478         }
 479
 480 #endif /* EFSYS_OPT_RX_SCALE */
 481
 482         return (0);
 483 }
 484
 485 #if EFSYS_OPT_RX_SCATTER
 486         __checkReturn   efx_rc_t
 487 ef10_rx_scatter_enable(
 488         __in            efx_nic_t *enp,
 489         __in            unsigned int buf_size)
 490 {
 491         _NOTE(ARGUNUSED(enp, buf_size))
 492         return (0);
 493 }
 494 #endif  /* EFSYS_OPT_RX_SCATTER */
 495
 496 #if EFSYS_OPT_RX_SCALE
 497         __checkReturn   efx_rc_t
 498 ef10_rx_scale_context_alloc(
 499         __in            efx_nic_t *enp,
 500         __in            efx_rx_scale_context_type_t type,
 501         __in            uint32_t num_queues,
 502         __out           uint32_t *rss_contextp)
 503 {
 504         efx_rc_t rc;
 505
 506         rc = efx_mcdi_rss_context_alloc(enp, type, num_queues, rss_contextp);
 507         if (rc != 0)
 508                 goto fail1;
 509
 510         return (0);
 511
 512 fail1:
 513         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 514         return (rc);
 515 }
 516 #endif /* EFSYS_OPT_RX_SCALE */
 517
 518 #if EFSYS_OPT_RX_SCALE
 519         __checkReturn   efx_rc_t
 520 ef10_rx_scale_context_free(
 521         __in            efx_nic_t *enp,
 522         __in            uint32_t rss_context)
 523 {
 524         efx_rc_t rc;
 525
 526         rc = efx_mcdi_rss_context_free(enp, rss_context);
 527         if (rc != 0)
 528                 goto fail1;
 529
 530         return (0);
 531
 532 fail1:
 533         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 534         return (rc);
 535 }
 536 #endif /* EFSYS_OPT_RX_SCALE */
 537
 538 #if EFSYS_OPT_RX_SCALE
 539         __checkReturn   efx_rc_t
 540 ef10_rx_scale_mode_set(
 541         __in            efx_nic_t *enp,
 542         __in            uint32_t rss_context,
 543         __in            efx_rx_hash_alg_t alg,
 544         __in            efx_rx_hash_type_t type,
 545         __in            boolean_t insert)
 546 {
 547         efx_rc_t rc;
 548
 549         EFSYS_ASSERT3U(alg, ==, EFX_RX_HASHALG_TOEPLITZ);
 550         EFSYS_ASSERT3U(insert, ==, B_TRUE);
 551
 552         if ((alg != EFX_RX_HASHALG_TOEPLITZ) || (insert == B_FALSE)) {
 553                 rc = EINVAL;
 554                 goto fail1;
 555         }
 556
 557         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 558                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 559                         rc = ENOTSUP;
 560                         goto fail2;
 561                 }
 562                 rss_context = enp->en_rss_context;
 563         }
 564
 565         if ((rc = efx_mcdi_rss_context_set_flags(enp,
 566                     rss_context, type)) != 0)
 567                 goto fail3;
 568
 569         return (0);
 570
 571 fail3:
 572         EFSYS_PROBE(fail3);
 573 fail2:
 574         EFSYS_PROBE(fail2);
 575 fail1:
 576         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 577
 578         return (rc);
 579 }
 580 #endif /* EFSYS_OPT_RX_SCALE */
 581
 582 #if EFSYS_OPT_RX_SCALE
 583         __checkReturn   efx_rc_t
 584 ef10_rx_scale_key_set(
 585         __in            efx_nic_t *enp,
 586         __in            uint32_t rss_context,
 587         __in_ecount(n)  uint8_t *key,
 588         __in            size_t n)
 589 {
 590         efx_rc_t rc;
 591
 592         EFX_STATIC_ASSERT(EFX_RSS_KEY_SIZE ==
 593             MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
 594
 595         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 596                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 597                         rc = ENOTSUP;
 598                         goto fail1;
 599                 }
 600                 rss_context = enp->en_rss_context;
 601         }
 602
 603         if ((rc = efx_mcdi_rss_context_set_key(enp, rss_context, key, n)) != 0)
 604                 goto fail2;
 605
 606         return (0);
 607
 608 fail2:
 609         EFSYS_PROBE(fail2);
 610 fail1:
 611         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 612
 613         return (rc);
 614 }
 615 #endif /* EFSYS_OPT_RX_SCALE */
 616
 617 #if EFSYS_OPT_RX_SCALE
 618         __checkReturn   efx_rc_t
 619 ef10_rx_scale_tbl_set(
 620         __in            efx_nic_t *enp,
 621         __in            uint32_t rss_context,
 622         __in_ecount(n)  unsigned int *table,
 623         __in            size_t n)
 624 {
 625         efx_rc_t rc;
 626
 627
 628         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 629                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 630                         rc = ENOTSUP;
 631                         goto fail1;
 632                 }
 633                 rss_context = enp->en_rss_context;
 634         }
 635
 636         if ((rc = efx_mcdi_rss_context_set_table(enp,
 637                     rss_context, table, n)) != 0)
 638                 goto fail2;
 639
 640         return (0);
 641
 642 fail2:
 643         EFSYS_PROBE(fail2);
 644 fail1:
 645         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 646
 647         return (rc);
 648 }
 649 #endif /* EFSYS_OPT_RX_SCALE */
 650
 651
 652 /*
 653  * EF10 RX pseudo-header
 654  * ---------------------
 655  *
 656  * Receive packets are prefixed by an (optional) 14 byte pseudo-header:
 657  *
 658  *  +00: Toeplitz hash value.
 659  *       (32bit little-endian)
 660  *  +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag.
 661  *       (16bit big-endian)
 662  *  +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag.
 663  *       (16bit big-endian)
 664  *  +08: Packet Length. Zero if the RX datapath was in cut-through mode.
 665  *       (16bit little-endian)
 666  *  +10: MAC timestamp. Zero if timestamping is not enabled.
 667  *       (32bit little-endian)
 668  *
 669  * See "The RX Pseudo-header" in SF-109306-TC.
 670  */
 671
 672         __checkReturn   efx_rc_t
 673 ef10_rx_prefix_pktlen(
 674         __in            efx_nic_t *enp,
 675         __in            uint8_t *buffer,
 676         __out           uint16_t *lengthp)
 677 {
 678         _NOTE(ARGUNUSED(enp))
 679
 680         /*
 681          * The RX pseudo-header contains the packet length, excluding the
 682          * pseudo-header. If the hardware receive datapath was operating in
 683          * cut-through mode then the length in the RX pseudo-header will be
 684          * zero, and the packet length must be obtained from the DMA length
 685          * reported in the RX event.
 686          */
 687         *lengthp = buffer[8] | (buffer[9] << 8);
 688         return (0);
 689 }
 690
 691 #if EFSYS_OPT_RX_SCALE
 692         __checkReturn   uint32_t
 693 ef10_rx_prefix_hash(
 694         __in            efx_nic_t *enp,
 695         __in            efx_rx_hash_alg_t func,
 696         __in            uint8_t *buffer)
 697 {
 698         _NOTE(ARGUNUSED(enp))
 699
 700         switch (func) {
 701         case EFX_RX_HASHALG_TOEPLITZ:
 702                 return (buffer[0] |
 703                     (buffer[1] << 8) |
 704                     (buffer[2] << 16) |
 705                     (buffer[3] << 24));
 706
 707         default:
 708                 EFSYS_ASSERT(0);
 709                 return (0);
 710         }
 711 }
 712 #endif /* EFSYS_OPT_RX_SCALE */
 713
 714 #if EFSYS_OPT_RX_PACKED_STREAM
 715 /*
 716  * Fake length for RXQ descriptors in packed stream mode
 717  * to make hardware happy
 718  */
 719 #define EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE 32
 720 #endif
 721
 722                                 void
 723 ef10_rx_qpost(
 724         __in                    efx_rxq_t *erp,
 725         __in_ecount(ndescs)     efsys_dma_addr_t *addrp,
 726         __in                    size_t size,
 727         __in                    unsigned int ndescs,
 728         __in                    unsigned int completed,
 729         __in                    unsigned int added)
 730 {
 731         efx_qword_t qword;
 732         unsigned int i;
 733         unsigned int offset;
 734         unsigned int id;
 735
 736         _NOTE(ARGUNUSED(completed))
 737
 738 #if EFSYS_OPT_RX_PACKED_STREAM
 739         /*
 740          * Real size of the buffer does not fit into ESF_DZ_RX_KER_BYTE_CNT
 741          * and equal to 0 after applying mask. Hardware does not like it.
 742          */
 743         if (erp->er_ev_qstate->eers_rx_packed_stream)
 744                 size = EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE;
 745 #endif
 746
 747         /* The client driver must not overfill the queue */
 748         EFSYS_ASSERT3U(added - completed + ndescs, <=,
 749             EFX_RXQ_LIMIT(erp->er_mask + 1));
 750
 751         id = added & (erp->er_mask);
 752         for (i = 0; i < ndescs; i++) {
 753                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
 754                     unsigned int, id, efsys_dma_addr_t, addrp[i],
 755                     size_t, size);
 756
 757                 EFX_POPULATE_QWORD_3(qword,
 758                     ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size),
 759                     ESF_DZ_RX_KER_BUF_ADDR_DW0,
 760                     (uint32_t)(addrp[i] & 0xffffffff),
 761                     ESF_DZ_RX_KER_BUF_ADDR_DW1,
 762                     (uint32_t)(addrp[i] >> 32));
 763
 764                 offset = id * sizeof (efx_qword_t);
 765                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
 766
 767                 id = (id + 1) & (erp->er_mask);
 768         }
 769 }
 770
 771                         void
 772 ef10_rx_qpush(
 773         __in    efx_rxq_t *erp,
 774         __in    unsigned int added,
 775         __inout unsigned int *pushedp)
 776 {
 777         efx_nic_t *enp = erp->er_enp;
 778         unsigned int pushed = *pushedp;
 779         uint32_t wptr;
 780         efx_dword_t dword;
 781
 782         /* Hardware has alignment restriction for WPTR */
 783         wptr = P2ALIGN(added, EF10_RX_WPTR_ALIGN);
 784         if (pushed == wptr)
 785                 return;
 786
 787         *pushedp = wptr;
 788
 789         /* Push the populated descriptors out */
 790         wptr &= erp->er_mask;
 791
 792         EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr);
 793
 794         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
 795         EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
 796             wptr, pushed & erp->er_mask);
 797         EFSYS_PIO_WRITE_BARRIER();
 798         EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
 799                             erp->er_index, &dword, B_FALSE);
 800 }
 801
 802 #if EFSYS_OPT_RX_PACKED_STREAM
 803
 804                         void
 805 ef10_rx_qpush_ps_credits(
 806         __in            efx_rxq_t *erp)
 807 {
 808         efx_nic_t *enp = erp->er_enp;
 809         efx_dword_t dword;
 810         efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate;
 811         uint32_t credits;
 812
 813         EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
 814
 815         if (rxq_state->eers_rx_packed_stream_credits == 0)
 816                 return;
 817
 818         /*
 819          * It is a bug if we think that FW has utilized more
 820          * credits than it is allowed to have (maximum). However,
 821          * make sure that we do not credit more than maximum anyway.
 822          */
 823         credits = MIN(rxq_state->eers_rx_packed_stream_credits,
 824             EFX_RX_PACKED_STREAM_MAX_CREDITS);
 825         EFX_POPULATE_DWORD_3(dword,
 826             ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1,
 827             ERF_DZ_RX_DESC_MAGIC_CMD,
 828             ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS,
 829             ERF_DZ_RX_DESC_MAGIC_DATA, credits);
 830         EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
 831             erp->er_index, &dword, B_FALSE);
 832
 833         rxq_state->eers_rx_packed_stream_credits = 0;
 834 }
 835
 836 /*
 837  * In accordance with SF-112241-TC the received data has the following layout:
 838  *  - 8 byte pseudo-header which consist of:
 839  *    - 4 byte little-endian timestamp
 840  *    - 2 byte little-endian captured length in bytes
 841  *    - 2 byte little-endian original packet length in bytes
 842  *  - captured packet bytes
 843  *  - optional padding to align to 64 bytes boundary
 844  *  - 64 bytes scratch space for the host software
 845  */
 846         __checkReturn   uint8_t *
 847 ef10_rx_qps_packet_info(
 848         __in            efx_rxq_t *erp,
 849         __in            uint8_t *buffer,
 850         __in            uint32_t buffer_length,
 851         __in            uint32_t current_offset,
 852         __out           uint16_t *lengthp,
 853         __out           uint32_t *next_offsetp,
 854         __out           uint32_t *timestamp)
 855 {
 856         uint16_t buf_len;
 857         uint8_t *pkt_start;
 858         efx_qword_t *qwordp;
 859         efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate;
 860
 861         EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
 862
 863         buffer += current_offset;
 864         pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE;
 865
 866         qwordp = (efx_qword_t *)buffer;
 867         *timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP);
 868         *lengthp   = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN);
 869         buf_len    = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN);
 870
 871         buf_len = P2ROUNDUP(buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE,
 872                             EFX_RX_PACKED_STREAM_ALIGNMENT);
 873         *next_offsetp =
 874             current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT;
 875
 876         EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length);
 877         EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp);
 878
 879         if ((*next_offsetp ^ current_offset) &
 880             EFX_RX_PACKED_STREAM_MEM_PER_CREDIT)
 881                 rxq_state->eers_rx_packed_stream_credits++;
 882
 883         return (pkt_start);
 884 }
 885
 886
 887 #endif
 888
 889         __checkReturn   efx_rc_t
 890 ef10_rx_qflush(
 891         __in    efx_rxq_t *erp)
 892 {
 893         efx_nic_t *enp = erp->er_enp;
 894         efx_rc_t rc;
 895
 896         if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0)
 897                 goto fail1;
 898
 899         return (0);
 900
 901 fail1:
 902         /*
 903          * EALREADY is not an error, but indicates that the MC has rebooted and
 904          * that the RXQ has already been destroyed. Callers need to know that
 905          * the RXQ flush has completed to avoid waiting until timeout for a
 906          * flush done event that will not be delivered.
 907          */
 908         if (rc != EALREADY)
 909                 EFSYS_PROBE1(fail1, efx_rc_t, rc);
 910
 911         return (rc);
 912 }
 913
 914                 void
 915 ef10_rx_qenable(
 916         __in    efx_rxq_t *erp)
 917 {
 918         /* FIXME */
 919         _NOTE(ARGUNUSED(erp))
 920         /* FIXME */
 921 }
 922
 923         __checkReturn   efx_rc_t
 924 ef10_rx_qcreate(
 925         __in            efx_nic_t *enp,
 926         __in            unsigned int index,
 927         __in            unsigned int label,
 928         __in            efx_rxq_type_t type,
 929         __in            uint32_t type_data,
 930         __in            efsys_mem_t *esmp,
 931         __in            size_t ndescs,
 932         __in            uint32_t id,
 933         __in            unsigned int flags,
 934         __in            efx_evq_t *eep,
 935         __in            efx_rxq_t *erp)
 936 {
 937         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 938         efx_rc_t rc;
 939         boolean_t disable_scatter;
 940         boolean_t want_inner_classes;
 941         unsigned int ps_buf_size;
 942
 943         _NOTE(ARGUNUSED(id, erp, type_data))
 944
 945         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH));
 946         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
 947         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
 948
 949         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
 950         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
 951
 952         if (!ISP2(ndescs) ||
 953             (ndescs < EFX_RXQ_MINNDESCS) || (ndescs > EFX_RXQ_MAXNDESCS)) {
 954                 rc = EINVAL;
 955                 goto fail1;
 956         }
 957         if (index >= encp->enc_rxq_limit) {
 958                 rc = EINVAL;
 959                 goto fail2;
 960         }
 961
 962         switch (type) {
 963         case EFX_RXQ_TYPE_DEFAULT:
 964                 ps_buf_size = 0;
 965                 break;
 966 #if EFSYS_OPT_RX_PACKED_STREAM
 967         case EFX_RXQ_TYPE_PACKED_STREAM:
 968                 switch (type_data) {
 969                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_1M:
 970                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M;
 971                         break;
 972                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_512K:
 973                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K;
 974                         break;
 975                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_256K:
 976                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K;
 977                         break;
 978                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_128K:
 979                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K;
 980                         break;
 981                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_64K:
 982                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K;
 983                         break;
 984                 default:
 985                         rc = ENOTSUP;
 986                         goto fail3;
 987                 }
 988                 break;
 989 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
 990         default:
 991                 rc = ENOTSUP;
 992                 goto fail4;
 993         }
 994
 995 #if EFSYS_OPT_RX_PACKED_STREAM
 996         if (ps_buf_size != 0) {
 997                 /* Check if datapath firmware supports packed stream mode */
 998                 if (encp->enc_rx_packed_stream_supported == B_FALSE) {
 999                         rc = ENOTSUP;
1000                         goto fail5;
1001                 }
1002                 /* Check if packed stream allows configurable buffer sizes */
1003                 if ((ps_buf_size != MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M) &&
1004                     (encp->enc_rx_var_packed_stream_supported == B_FALSE)) {
1005                         rc = ENOTSUP;
1006                         goto fail6;
1007                 }
1008         }
1009 #else /* EFSYS_OPT_RX_PACKED_STREAM */
1010         EFSYS_ASSERT(ps_buf_size == 0);
1011 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1012
1013         /* Scatter can only be disabled if the firmware supports doing so */
1014         if (flags & EFX_RXQ_FLAG_SCATTER)
1015                 disable_scatter = B_FALSE;
1016         else
1017                 disable_scatter = encp->enc_rx_disable_scatter_supported;
1018
1019         if (flags & EFX_RXQ_FLAG_INNER_CLASSES)
1020                 want_inner_classes = B_TRUE;
1021         else
1022                 want_inner_classes = B_FALSE;
1023
1024         if ((rc = efx_mcdi_init_rxq(enp, ndescs, eep->ee_index, label, index,
1025                     esmp, disable_scatter, want_inner_classes,
1026                     ps_buf_size)) != 0)
1027                 goto fail7;
1028
1029         erp->er_eep = eep;
1030         erp->er_label = label;
1031
1032         ef10_ev_rxlabel_init(eep, erp, label, type);
1033
1034         erp->er_ev_qstate = &erp->er_eep->ee_rxq_state[label];
1035
1036         return (0);
1037
1038 fail7:
1039         EFSYS_PROBE(fail7);
1040 #if EFSYS_OPT_RX_PACKED_STREAM
1041 fail6:
1042         EFSYS_PROBE(fail6);
1043 fail5:
1044         EFSYS_PROBE(fail5);
1045 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1046 fail4:
1047         EFSYS_PROBE(fail4);
1048 #if EFSYS_OPT_RX_PACKED_STREAM
1049 fail3:
1050         EFSYS_PROBE(fail3);
1051 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1052 fail2:
1053         EFSYS_PROBE(fail2);
1054 fail1:
1055         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1056
1057         return (rc);
1058 }
1059
1060                 void
1061 ef10_rx_qdestroy(
1062         __in    efx_rxq_t *erp)
1063 {
1064         efx_nic_t *enp = erp->er_enp;
1065         efx_evq_t *eep = erp->er_eep;
1066         unsigned int label = erp->er_label;
1067
1068         ef10_ev_rxlabel_fini(eep, label);
1069
1070         EFSYS_ASSERT(enp->en_rx_qcount != 0);
1071         --enp->en_rx_qcount;
1072
1073         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1074 }
1075
1076                 void
1077 ef10_rx_fini(
1078         __in    efx_nic_t *enp)
1079 {
1080 #if EFSYS_OPT_RX_SCALE
1081         if (enp->en_rss_context_type != EFX_RX_SCALE_UNAVAILABLE)
1082                 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context);
1083         enp->en_rss_context = 0;
1084         enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
1085 #else
1086         _NOTE(ARGUNUSED(enp))
1087 #endif /* EFSYS_OPT_RX_SCALE */
1088 }
1089
1090 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */