drivers/net/sfc/base/ef10_rx.c

   1 /*
   2  * Copyright (c) 2012-2016 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright notice,
  11  *    this list of conditions and the following disclaimer in the documentation
  12  *    and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * The views and conclusions contained in the software and documentation are
  27  * those of the authors and should not be interpreted as representing official
  28  * policies, either expressed or implied, of the FreeBSD Project.
  29  */
  30
  31 #include "efx.h"
  32 #include "efx_impl.h"
  33
  34
  35 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
  36
  37
  38 static  __checkReturn   efx_rc_t
  39 efx_mcdi_init_rxq(
  40         __in            efx_nic_t *enp,
  41         __in            uint32_t size,
  42         __in            uint32_t target_evq,
  43         __in            uint32_t label,
  44         __in            uint32_t instance,
  45         __in            efsys_mem_t *esmp,
  46         __in            boolean_t disable_scatter,
  47         __in            uint32_t ps_bufsize)
  48 {
  49         efx_mcdi_req_t req;
  50         uint8_t payload[MAX(MC_CMD_INIT_RXQ_EXT_IN_LEN,
  51                             MC_CMD_INIT_RXQ_EXT_OUT_LEN)];
  52         int npages = EFX_RXQ_NBUFS(size);
  53         int i;
  54         efx_qword_t *dma_addr;
  55         uint64_t addr;
  56         efx_rc_t rc;
  57         uint32_t dma_mode;
  58
  59         /* If this changes, then the payload size might need to change. */
  60         EFSYS_ASSERT3U(MC_CMD_INIT_RXQ_OUT_LEN, ==, 0);
  61         EFSYS_ASSERT3U(size, <=, EFX_RXQ_MAXNDESCS);
  62
  63         if (ps_bufsize > 0)
  64                 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_PACKED_STREAM;
  65         else
  66                 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_SINGLE_PACKET;
  67
  68         (void) memset(payload, 0, sizeof (payload));
  69         req.emr_cmd = MC_CMD_INIT_RXQ;
  70         req.emr_in_buf = payload;
  71         req.emr_in_length = MC_CMD_INIT_RXQ_EXT_IN_LEN;
  72         req.emr_out_buf = payload;
  73         req.emr_out_length = MC_CMD_INIT_RXQ_EXT_OUT_LEN;
  74
  75         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_SIZE, size);
  76         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_TARGET_EVQ, target_evq);
  77         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_LABEL, label);
  78         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_INSTANCE, instance);
  79         MCDI_IN_POPULATE_DWORD_8(req, INIT_RXQ_EXT_IN_FLAGS,
  80             INIT_RXQ_EXT_IN_FLAG_BUFF_MODE, 0,
  81             INIT_RXQ_EXT_IN_FLAG_HDR_SPLIT, 0,
  82             INIT_RXQ_EXT_IN_FLAG_TIMESTAMP, 0,
  83             INIT_RXQ_EXT_IN_CRC_MODE, 0,
  84             INIT_RXQ_EXT_IN_FLAG_PREFIX, 1,
  85             INIT_RXQ_EXT_IN_FLAG_DISABLE_SCATTER, disable_scatter,
  86             INIT_RXQ_EXT_IN_DMA_MODE,
  87             dma_mode,
  88             INIT_RXQ_EXT_IN_PACKED_STREAM_BUFF_SIZE, ps_bufsize);
  89         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_OWNER_ID, 0);
  90         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);
  91
  92         dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR);
  93         addr = EFSYS_MEM_ADDR(esmp);
  94
  95         for (i = 0; i < npages; i++) {
  96                 EFX_POPULATE_QWORD_2(*dma_addr,
  97                     EFX_DWORD_1, (uint32_t)(addr >> 32),
  98                     EFX_DWORD_0, (uint32_t)(addr & 0xffffffff));
  99
 100                 dma_addr++;
 101                 addr += EFX_BUF_SIZE;
 102         }
 103
 104         efx_mcdi_execute(enp, &req);
 105
 106         if (req.emr_rc != 0) {
 107                 rc = req.emr_rc;
 108                 goto fail1;
 109         }
 110
 111         return (0);
 112
 113 fail1:
 114         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 115
 116         return (rc);
 117 }
 118
 119 static  __checkReturn   efx_rc_t
 120 efx_mcdi_fini_rxq(
 121         __in            efx_nic_t *enp,
 122         __in            uint32_t instance)
 123 {
 124         efx_mcdi_req_t req;
 125         uint8_t payload[MAX(MC_CMD_FINI_RXQ_IN_LEN,
 126                             MC_CMD_FINI_RXQ_OUT_LEN)];
 127         efx_rc_t rc;
 128
 129         (void) memset(payload, 0, sizeof (payload));
 130         req.emr_cmd = MC_CMD_FINI_RXQ;
 131         req.emr_in_buf = payload;
 132         req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN;
 133         req.emr_out_buf = payload;
 134         req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN;
 135
 136         MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance);
 137
 138         efx_mcdi_execute_quiet(enp, &req);
 139
 140         if ((req.emr_rc != 0) && (req.emr_rc != MC_CMD_ERR_EALREADY)) {
 141                 rc = req.emr_rc;
 142                 goto fail1;
 143         }
 144
 145         return (0);
 146
 147 fail1:
 148         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 149
 150         return (rc);
 151 }
 152
 153 #if EFSYS_OPT_RX_SCALE
 154 static  __checkReturn   efx_rc_t
 155 efx_mcdi_rss_context_alloc(
 156         __in            efx_nic_t *enp,
 157         __in            efx_rx_scale_support_t scale_support,
 158         __in            uint32_t num_queues,
 159         __out           uint32_t *rss_contextp)
 160 {
 161         efx_mcdi_req_t req;
 162         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN,
 163                             MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)];
 164         uint32_t rss_context;
 165         uint32_t context_type;
 166         efx_rc_t rc;
 167
 168         if (num_queues > EFX_MAXRSS) {
 169                 rc = EINVAL;
 170                 goto fail1;
 171         }
 172
 173         switch (scale_support) {
 174         case EFX_RX_SCALE_EXCLUSIVE:
 175                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE;
 176                 break;
 177         case EFX_RX_SCALE_SHARED:
 178                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED;
 179                 break;
 180         default:
 181                 rc = EINVAL;
 182                 goto fail2;
 183         }
 184
 185         (void) memset(payload, 0, sizeof (payload));
 186         req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC;
 187         req.emr_in_buf = payload;
 188         req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN;
 189         req.emr_out_buf = payload;
 190         req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN;
 191
 192         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
 193             EVB_PORT_ID_ASSIGNED);
 194         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type);
 195         /* NUM_QUEUES is only used to validate indirection table offsets */
 196         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues);
 197
 198         efx_mcdi_execute(enp, &req);
 199
 200         if (req.emr_rc != 0) {
 201                 rc = req.emr_rc;
 202                 goto fail3;
 203         }
 204
 205         if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) {
 206                 rc = EMSGSIZE;
 207                 goto fail4;
 208         }
 209
 210         rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);
 211         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 212                 rc = ENOENT;
 213                 goto fail5;
 214         }
 215
 216         *rss_contextp = rss_context;
 217
 218         return (0);
 219
 220 fail5:
 221         EFSYS_PROBE(fail5);
 222 fail4:
 223         EFSYS_PROBE(fail4);
 224 fail3:
 225         EFSYS_PROBE(fail3);
 226 fail2:
 227         EFSYS_PROBE(fail2);
 228 fail1:
 229         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 230
 231         return (rc);
 232 }
 233 #endif /* EFSYS_OPT_RX_SCALE */
 234
 235 #if EFSYS_OPT_RX_SCALE
 236 static                  efx_rc_t
 237 efx_mcdi_rss_context_free(
 238         __in            efx_nic_t *enp,
 239         __in            uint32_t rss_context)
 240 {
 241         efx_mcdi_req_t req;
 242         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_FREE_IN_LEN,
 243                             MC_CMD_RSS_CONTEXT_FREE_OUT_LEN)];
 244         efx_rc_t rc;
 245
 246         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 247                 rc = EINVAL;
 248                 goto fail1;
 249         }
 250
 251         (void) memset(payload, 0, sizeof (payload));
 252         req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE;
 253         req.emr_in_buf = payload;
 254         req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN;
 255         req.emr_out_buf = payload;
 256         req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN;
 257
 258         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context);
 259
 260         efx_mcdi_execute_quiet(enp, &req);
 261
 262         if (req.emr_rc != 0) {
 263                 rc = req.emr_rc;
 264                 goto fail2;
 265         }
 266
 267         return (0);
 268
 269 fail2:
 270         EFSYS_PROBE(fail2);
 271 fail1:
 272         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 273
 274         return (rc);
 275 }
 276 #endif /* EFSYS_OPT_RX_SCALE */
 277
 278 #if EFSYS_OPT_RX_SCALE
 279 static                  efx_rc_t
 280 efx_mcdi_rss_context_set_flags(
 281         __in            efx_nic_t *enp,
 282         __in            uint32_t rss_context,
 283         __in            efx_rx_hash_type_t type)
 284 {
 285         efx_mcdi_req_t req;
 286         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN,
 287                             MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN)];
 288         efx_rc_t rc;
 289
 290         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 291                 rc = EINVAL;
 292                 goto fail1;
 293         }
 294
 295         (void) memset(payload, 0, sizeof (payload));
 296         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS;
 297         req.emr_in_buf = payload;
 298         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN;
 299         req.emr_out_buf = payload;
 300         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN;
 301
 302         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
 303             rss_context);
 304
 305         MCDI_IN_POPULATE_DWORD_4(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS,
 306             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN,
 307             (type & EFX_RX_HASH_IPV4) ? 1 : 0,
 308             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN,
 309             (type & EFX_RX_HASH_TCPIPV4) ? 1 : 0,
 310             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN,
 311             (type & EFX_RX_HASH_IPV6) ? 1 : 0,
 312             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN,
 313             (type & EFX_RX_HASH_TCPIPV6) ? 1 : 0);
 314
 315         efx_mcdi_execute(enp, &req);
 316
 317         if (req.emr_rc != 0) {
 318                 rc = req.emr_rc;
 319                 goto fail2;
 320         }
 321
 322         return (0);
 323
 324 fail2:
 325         EFSYS_PROBE(fail2);
 326 fail1:
 327         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 328
 329         return (rc);
 330 }
 331 #endif /* EFSYS_OPT_RX_SCALE */
 332
 333 #if EFSYS_OPT_RX_SCALE
 334 static                  efx_rc_t
 335 efx_mcdi_rss_context_set_key(
 336         __in            efx_nic_t *enp,
 337         __in            uint32_t rss_context,
 338         __in_ecount(n)  uint8_t *key,
 339         __in            size_t n)
 340 {
 341         efx_mcdi_req_t req;
 342         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN,
 343                             MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN)];
 344         efx_rc_t rc;
 345
 346         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 347                 rc = EINVAL;
 348                 goto fail1;
 349         }
 350
 351         (void) memset(payload, 0, sizeof (payload));
 352         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY;
 353         req.emr_in_buf = payload;
 354         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN;
 355         req.emr_out_buf = payload;
 356         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN;
 357
 358         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
 359             rss_context);
 360
 361         EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
 362         if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) {
 363                 rc = EINVAL;
 364                 goto fail2;
 365         }
 366
 367         memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY),
 368             key, n);
 369
 370         efx_mcdi_execute(enp, &req);
 371
 372         if (req.emr_rc != 0) {
 373                 rc = req.emr_rc;
 374                 goto fail3;
 375         }
 376
 377         return (0);
 378
 379 fail3:
 380         EFSYS_PROBE(fail3);
 381 fail2:
 382         EFSYS_PROBE(fail2);
 383 fail1:
 384         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 385
 386         return (rc);
 387 }
 388 #endif /* EFSYS_OPT_RX_SCALE */
 389
 390 #if EFSYS_OPT_RX_SCALE
 391 static                  efx_rc_t
 392 efx_mcdi_rss_context_set_table(
 393         __in            efx_nic_t *enp,
 394         __in            uint32_t rss_context,
 395         __in_ecount(n)  unsigned int *table,
 396         __in            size_t n)
 397 {
 398         efx_mcdi_req_t req;
 399         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN,
 400                             MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN)];
 401         uint8_t *req_table;
 402         int i, rc;
 403
 404         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 405                 rc = EINVAL;
 406                 goto fail1;
 407         }
 408
 409         (void) memset(payload, 0, sizeof (payload));
 410         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE;
 411         req.emr_in_buf = payload;
 412         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN;
 413         req.emr_out_buf = payload;
 414         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN;
 415
 416         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
 417             rss_context);
 418
 419         req_table =
 420             MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE);
 421
 422         for (i = 0;
 423             i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN;
 424             i++) {
 425                 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0;
 426         }
 427
 428         efx_mcdi_execute(enp, &req);
 429
 430         if (req.emr_rc != 0) {
 431                 rc = req.emr_rc;
 432                 goto fail2;
 433         }
 434
 435         return (0);
 436
 437 fail2:
 438         EFSYS_PROBE(fail2);
 439 fail1:
 440         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 441
 442         return (rc);
 443 }
 444 #endif /* EFSYS_OPT_RX_SCALE */
 445
 446
 447         __checkReturn   efx_rc_t
 448 ef10_rx_init(
 449         __in            efx_nic_t *enp)
 450 {
 451 #if EFSYS_OPT_RX_SCALE
 452
 453         if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS,
 454                 &enp->en_rss_context) == 0) {
 455                 /*
 456                  * Allocated an exclusive RSS context, which allows both the
 457                  * indirection table and key to be modified.
 458                  */
 459                 enp->en_rss_support = EFX_RX_SCALE_EXCLUSIVE;
 460                 enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
 461         } else {
 462                 /*
 463                  * Failed to allocate an exclusive RSS context. Continue
 464                  * operation without support for RSS. The pseudo-header in
 465                  * received packets will not contain a Toeplitz hash value.
 466                  */
 467                 enp->en_rss_support = EFX_RX_SCALE_UNAVAILABLE;
 468                 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE;
 469         }
 470
 471 #endif /* EFSYS_OPT_RX_SCALE */
 472
 473         return (0);
 474 }
 475
 476 #if EFSYS_OPT_RX_SCATTER
 477         __checkReturn   efx_rc_t
 478 ef10_rx_scatter_enable(
 479         __in            efx_nic_t *enp,
 480         __in            unsigned int buf_size)
 481 {
 482         _NOTE(ARGUNUSED(enp, buf_size))
 483         return (0);
 484 }
 485 #endif  /* EFSYS_OPT_RX_SCATTER */
 486
 487 #if EFSYS_OPT_RX_SCALE
 488         __checkReturn   efx_rc_t
 489 ef10_rx_scale_mode_set(
 490         __in            efx_nic_t *enp,
 491         __in            efx_rx_hash_alg_t alg,
 492         __in            efx_rx_hash_type_t type,
 493         __in            boolean_t insert)
 494 {
 495         efx_rc_t rc;
 496
 497         EFSYS_ASSERT3U(alg, ==, EFX_RX_HASHALG_TOEPLITZ);
 498         EFSYS_ASSERT3U(insert, ==, B_TRUE);
 499
 500         if ((alg != EFX_RX_HASHALG_TOEPLITZ) || (insert == B_FALSE)) {
 501                 rc = EINVAL;
 502                 goto fail1;
 503         }
 504
 505         if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) {
 506                 rc = ENOTSUP;
 507                 goto fail2;
 508         }
 509
 510         if ((rc = efx_mcdi_rss_context_set_flags(enp,
 511                     enp->en_rss_context, type)) != 0)
 512                 goto fail3;
 513
 514         return (0);
 515
 516 fail3:
 517         EFSYS_PROBE(fail3);
 518 fail2:
 519         EFSYS_PROBE(fail2);
 520 fail1:
 521         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 522
 523         return (rc);
 524 }
 525 #endif /* EFSYS_OPT_RX_SCALE */
 526
 527 #if EFSYS_OPT_RX_SCALE
 528         __checkReturn   efx_rc_t
 529 ef10_rx_scale_key_set(
 530         __in            efx_nic_t *enp,
 531         __in_ecount(n)  uint8_t *key,
 532         __in            size_t n)
 533 {
 534         efx_rc_t rc;
 535
 536         if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) {
 537                 rc = ENOTSUP;
 538                 goto fail1;
 539         }
 540
 541         if ((rc = efx_mcdi_rss_context_set_key(enp,
 542             enp->en_rss_context, key, n)) != 0)
 543                 goto fail2;
 544
 545         return (0);
 546
 547 fail2:
 548         EFSYS_PROBE(fail2);
 549 fail1:
 550         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 551
 552         return (rc);
 553 }
 554 #endif /* EFSYS_OPT_RX_SCALE */
 555
 556 #if EFSYS_OPT_RX_SCALE
 557         __checkReturn   efx_rc_t
 558 ef10_rx_scale_tbl_set(
 559         __in            efx_nic_t *enp,
 560         __in_ecount(n)  unsigned int *table,
 561         __in            size_t n)
 562 {
 563         efx_rc_t rc;
 564
 565         if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) {
 566                 rc = ENOTSUP;
 567                 goto fail1;
 568         }
 569
 570         if ((rc = efx_mcdi_rss_context_set_table(enp,
 571             enp->en_rss_context, table, n)) != 0)
 572                 goto fail2;
 573
 574         return (0);
 575
 576 fail2:
 577         EFSYS_PROBE(fail2);
 578 fail1:
 579         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 580
 581         return (rc);
 582 }
 583 #endif /* EFSYS_OPT_RX_SCALE */
 584
 585
 586 /*
 587  * EF10 RX pseudo-header
 588  * ---------------------
 589  *
 590  * Receive packets are prefixed by an (optional) 14 byte pseudo-header:
 591  *
 592  *  +00: Toeplitz hash value.
 593  *       (32bit little-endian)
 594  *  +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag.
 595  *       (16bit big-endian)
 596  *  +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag.
 597  *       (16bit big-endian)
 598  *  +08: Packet Length. Zero if the RX datapath was in cut-through mode.
 599  *       (16bit little-endian)
 600  *  +10: MAC timestamp. Zero if timestamping is not enabled.
 601  *       (32bit little-endian)
 602  *
 603  * See "The RX Pseudo-header" in SF-109306-TC.
 604  */
 605
 606         __checkReturn   efx_rc_t
 607 ef10_rx_prefix_pktlen(
 608         __in            efx_nic_t *enp,
 609         __in            uint8_t *buffer,
 610         __out           uint16_t *lengthp)
 611 {
 612         _NOTE(ARGUNUSED(enp))
 613
 614         /*
 615          * The RX pseudo-header contains the packet length, excluding the
 616          * pseudo-header. If the hardware receive datapath was operating in
 617          * cut-through mode then the length in the RX pseudo-header will be
 618          * zero, and the packet length must be obtained from the DMA length
 619          * reported in the RX event.
 620          */
 621         *lengthp = buffer[8] | (buffer[9] << 8);
 622         return (0);
 623 }
 624
 625 #if EFSYS_OPT_RX_SCALE
 626         __checkReturn   uint32_t
 627 ef10_rx_prefix_hash(
 628         __in            efx_nic_t *enp,
 629         __in            efx_rx_hash_alg_t func,
 630         __in            uint8_t *buffer)
 631 {
 632         _NOTE(ARGUNUSED(enp))
 633
 634         switch (func) {
 635         case EFX_RX_HASHALG_TOEPLITZ:
 636                 return (buffer[0] |
 637                     (buffer[1] << 8) |
 638                     (buffer[2] << 16) |
 639                     (buffer[3] << 24));
 640
 641         default:
 642                 EFSYS_ASSERT(0);
 643                 return (0);
 644         }
 645 }
 646 #endif /* EFSYS_OPT_RX_SCALE */
 647
 648                         void
 649 ef10_rx_qpost(
 650         __in            efx_rxq_t *erp,
 651         __in_ecount(n)  efsys_dma_addr_t *addrp,
 652         __in            size_t size,
 653         __in            unsigned int n,
 654         __in            unsigned int completed,
 655         __in            unsigned int added)
 656 {
 657         efx_qword_t qword;
 658         unsigned int i;
 659         unsigned int offset;
 660         unsigned int id;
 661
 662         /* The client driver must not overfill the queue */
 663         EFSYS_ASSERT3U(added - completed + n, <=,
 664             EFX_RXQ_LIMIT(erp->er_mask + 1));
 665
 666         id = added & (erp->er_mask);
 667         for (i = 0; i < n; i++) {
 668                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
 669                     unsigned int, id, efsys_dma_addr_t, addrp[i],
 670                     size_t, size);
 671
 672                 EFX_POPULATE_QWORD_3(qword,
 673                     ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size),
 674                     ESF_DZ_RX_KER_BUF_ADDR_DW0,
 675                     (uint32_t)(addrp[i] & 0xffffffff),
 676                     ESF_DZ_RX_KER_BUF_ADDR_DW1,
 677                     (uint32_t)(addrp[i] >> 32));
 678
 679                 offset = id * sizeof (efx_qword_t);
 680                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
 681
 682                 id = (id + 1) & (erp->er_mask);
 683         }
 684 }
 685
 686                         void
 687 ef10_rx_qpush(
 688         __in    efx_rxq_t *erp,
 689         __in    unsigned int added,
 690         __inout unsigned int *pushedp)
 691 {
 692         efx_nic_t *enp = erp->er_enp;
 693         unsigned int pushed = *pushedp;
 694         uint32_t wptr;
 695         efx_dword_t dword;
 696
 697         /* Hardware has alignment restriction for WPTR */
 698         wptr = P2ALIGN(added, EF10_RX_WPTR_ALIGN);
 699         if (pushed == wptr)
 700                 return;
 701
 702         *pushedp = wptr;
 703
 704         /* Push the populated descriptors out */
 705         wptr &= erp->er_mask;
 706
 707         EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr);
 708
 709         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
 710         EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
 711             wptr, pushed & erp->er_mask);
 712         EFSYS_PIO_WRITE_BARRIER();
 713         EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
 714                             erp->er_index, &dword, B_FALSE);
 715 }
 716
 717 #if EFSYS_OPT_RX_PACKED_STREAM
 718
 719                         void
 720 ef10_rx_qps_update_credits(
 721         __in    efx_rxq_t *erp)
 722 {
 723         efx_nic_t *enp = erp->er_enp;
 724         efx_dword_t dword;
 725         efx_evq_rxq_state_t *rxq_state =
 726                 &erp->er_eep->ee_rxq_state[erp->er_label];
 727
 728         EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
 729
 730         if (rxq_state->eers_rx_packed_stream_credits == 0)
 731                 return;
 732
 733         EFX_POPULATE_DWORD_3(dword,
 734             ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1,
 735             ERF_DZ_RX_DESC_MAGIC_CMD,
 736             ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS,
 737             ERF_DZ_RX_DESC_MAGIC_DATA,
 738             rxq_state->eers_rx_packed_stream_credits);
 739         EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
 740             erp->er_index, &dword, B_FALSE);
 741
 742         rxq_state->eers_rx_packed_stream_credits = 0;
 743 }
 744
 745         __checkReturn   uint8_t *
 746 ef10_rx_qps_packet_info(
 747         __in            efx_rxq_t *erp,
 748         __in            uint8_t *buffer,
 749         __in            uint32_t buffer_length,
 750         __in            uint32_t current_offset,
 751         __out           uint16_t *lengthp,
 752         __out           uint32_t *next_offsetp,
 753         __out           uint32_t *timestamp)
 754 {
 755         uint16_t buf_len;
 756         uint8_t *pkt_start;
 757         efx_qword_t *qwordp;
 758         efx_evq_rxq_state_t *rxq_state =
 759                 &erp->er_eep->ee_rxq_state[erp->er_label];
 760
 761         EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
 762
 763         buffer += current_offset;
 764         pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE;
 765
 766         qwordp = (efx_qword_t *)buffer;
 767         *timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP);
 768         *lengthp   = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN);
 769         buf_len    = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN);
 770
 771         buf_len = P2ROUNDUP(buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE,
 772                             EFX_RX_PACKED_STREAM_ALIGNMENT);
 773         *next_offsetp =
 774             current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT;
 775
 776         EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length);
 777         EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp);
 778
 779         if ((*next_offsetp ^ current_offset) &
 780             EFX_RX_PACKED_STREAM_MEM_PER_CREDIT) {
 781                 if (rxq_state->eers_rx_packed_stream_credits <
 782                     EFX_RX_PACKED_STREAM_MAX_CREDITS)
 783                         rxq_state->eers_rx_packed_stream_credits++;
 784         }
 785
 786         return (pkt_start);
 787 }
 788
 789
 790 #endif
 791
 792         __checkReturn   efx_rc_t
 793 ef10_rx_qflush(
 794         __in    efx_rxq_t *erp)
 795 {
 796         efx_nic_t *enp = erp->er_enp;
 797         efx_rc_t rc;
 798
 799         if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0)
 800                 goto fail1;
 801
 802         return (0);
 803
 804 fail1:
 805         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 806
 807         return (rc);
 808 }
 809
 810                 void
 811 ef10_rx_qenable(
 812         __in    efx_rxq_t *erp)
 813 {
 814         /* FIXME */
 815         _NOTE(ARGUNUSED(erp))
 816         /* FIXME */
 817 }
 818
 819         __checkReturn   efx_rc_t
 820 ef10_rx_qcreate(
 821         __in            efx_nic_t *enp,
 822         __in            unsigned int index,
 823         __in            unsigned int label,
 824         __in            efx_rxq_type_t type,
 825         __in            efsys_mem_t *esmp,
 826         __in            size_t n,
 827         __in            uint32_t id,
 828         __in            efx_evq_t *eep,
 829         __in            efx_rxq_t *erp)
 830 {
 831         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 832         efx_rc_t rc;
 833         boolean_t disable_scatter;
 834         unsigned int ps_buf_size;
 835
 836         _NOTE(ARGUNUSED(id, erp))
 837
 838         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH));
 839         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
 840         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
 841
 842         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
 843         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
 844
 845         if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) {
 846                 rc = EINVAL;
 847                 goto fail1;
 848         }
 849         if (index >= encp->enc_rxq_limit) {
 850                 rc = EINVAL;
 851                 goto fail2;
 852         }
 853
 854         switch (type) {
 855         case EFX_RXQ_TYPE_DEFAULT:
 856         case EFX_RXQ_TYPE_SCATTER:
 857                 ps_buf_size = 0;
 858                 break;
 859 #if EFSYS_OPT_RX_PACKED_STREAM
 860         case EFX_RXQ_TYPE_PACKED_STREAM_1M:
 861                 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M;
 862                 break;
 863         case EFX_RXQ_TYPE_PACKED_STREAM_512K:
 864                 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K;
 865                 break;
 866         case EFX_RXQ_TYPE_PACKED_STREAM_256K:
 867                 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K;
 868                 break;
 869         case EFX_RXQ_TYPE_PACKED_STREAM_128K:
 870                 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K;
 871                 break;
 872         case EFX_RXQ_TYPE_PACKED_STREAM_64K:
 873                 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K;
 874                 break;
 875 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
 876         default:
 877                 rc = ENOTSUP;
 878                 goto fail3;
 879         }
 880
 881 #if EFSYS_OPT_RX_PACKED_STREAM
 882         if (ps_buf_size != 0) {
 883                 /* Check if datapath firmware supports packed stream mode */
 884                 if (encp->enc_rx_packed_stream_supported == B_FALSE) {
 885                         rc = ENOTSUP;
 886                         goto fail4;
 887                 }
 888                 /* Check if packed stream allows configurable buffer sizes */
 889                 if ((type != EFX_RXQ_TYPE_PACKED_STREAM_1M) &&
 890                     (encp->enc_rx_var_packed_stream_supported == B_FALSE)) {
 891                         rc = ENOTSUP;
 892                         goto fail5;
 893                 }
 894         }
 895 #else /* EFSYS_OPT_RX_PACKED_STREAM */
 896         EFSYS_ASSERT(ps_buf_size == 0);
 897 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
 898
 899         /* Scatter can only be disabled if the firmware supports doing so */
 900         if (type == EFX_RXQ_TYPE_SCATTER)
 901                 disable_scatter = B_FALSE;
 902         else
 903                 disable_scatter = encp->enc_rx_disable_scatter_supported;
 904
 905         if ((rc = efx_mcdi_init_rxq(enp, n, eep->ee_index, label, index,
 906                     esmp, disable_scatter, ps_buf_size)) != 0)
 907                 goto fail6;
 908
 909         erp->er_eep = eep;
 910         erp->er_label = label;
 911
 912         ef10_ev_rxlabel_init(eep, erp, label, ps_buf_size != 0);
 913
 914         return (0);
 915
 916 fail6:
 917         EFSYS_PROBE(fail6);
 918 #if EFSYS_OPT_RX_PACKED_STREAM
 919 fail5:
 920         EFSYS_PROBE(fail5);
 921 fail4:
 922         EFSYS_PROBE(fail4);
 923 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
 924 fail3:
 925         EFSYS_PROBE(fail3);
 926 fail2:
 927         EFSYS_PROBE(fail2);
 928 fail1:
 929         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 930
 931         return (rc);
 932 }
 933
 934                 void
 935 ef10_rx_qdestroy(
 936         __in    efx_rxq_t *erp)
 937 {
 938         efx_nic_t *enp = erp->er_enp;
 939         efx_evq_t *eep = erp->er_eep;
 940         unsigned int label = erp->er_label;
 941
 942         ef10_ev_rxlabel_fini(eep, label);
 943
 944         EFSYS_ASSERT(enp->en_rx_qcount != 0);
 945         --enp->en_rx_qcount;
 946
 947         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
 948 }
 949
 950                 void
 951 ef10_rx_fini(
 952         __in    efx_nic_t *enp)
 953 {
 954 #if EFSYS_OPT_RX_SCALE
 955         if (enp->en_rss_support != EFX_RX_SCALE_UNAVAILABLE) {
 956                 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context);
 957         }
 958         enp->en_rss_context = 0;
 959         enp->en_rss_support = EFX_RX_SCALE_UNAVAILABLE;
 960 #else
 961         _NOTE(ARGUNUSED(enp))
 962 #endif /* EFSYS_OPT_RX_SCALE */
 963 }
 964
 965 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */