drivers/net/sfc/base/efx_types.h

   1 /*
   2  * Copyright (c) 2007-2016 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright notice,
  11  *    this list of conditions and the following disclaimer in the documentation
  12  *    and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * The views and conclusions contained in the software and documentation are
  27  * those of the authors and should not be interpreted as representing official
  28  * policies, either expressed or implied, of the FreeBSD Project.
  29  *
  30  * Ackowledgement to Fen Systems Ltd.
  31  */
  32
  33 #ifndef _SYS_EFX_TYPES_H
  34 #define _SYS_EFX_TYPES_H
  35
  36 #include "efsys.h"
  37
  38 #ifdef  __cplusplus
  39 extern "C" {
  40 #endif
  41
  42 /*
  43  * Bitfield access
  44  *
  45  * Solarflare NICs make extensive use of bitfields up to 128 bits
  46  * wide.  Since there is no native 128-bit datatype on most systems,
  47  * and since 64-bit datatypes are inefficient on 32-bit systems and
  48  * vice versa, we wrap accesses in a way that uses the most efficient
  49  * datatype.
  50  *
  51  * The NICs are PCI devices and therefore little-endian.  Since most
  52  * of the quantities that we deal with are DMAed to/from host memory,
  53  * we define    our datatypes (efx_oword_t, efx_qword_t and efx_dword_t)
  54  * to be little-endian.
  55  *
  56  * In the less common case of using PIO for individual register
  57  * writes, we construct the little-endian datatype in host memory and
  58  * then use non-swapping register access primitives, rather than
  59  * constructing a native-endian datatype and relying on implicit
  60  * byte-swapping.  (We use a similar strategy for register reads.)
  61  */
  62
  63 /*
  64  * NOTE: Field definitions here and elsewhere are done in terms of a lowest
  65  *       bit number (LBN) and a width.
  66  */
  67
  68 #define EFX_DUMMY_FIELD_LBN 0
  69 #define EFX_DUMMY_FIELD_WIDTH 0
  70
  71 #define EFX_BYTE_0_LBN 0
  72 #define EFX_BYTE_0_WIDTH 8
  73
  74 #define EFX_BYTE_1_LBN 8
  75 #define EFX_BYTE_1_WIDTH 8
  76
  77 #define EFX_BYTE_2_LBN 16
  78 #define EFX_BYTE_2_WIDTH 8
  79
  80 #define EFX_BYTE_3_LBN 24
  81 #define EFX_BYTE_3_WIDTH 8
  82
  83 #define EFX_BYTE_4_LBN 32
  84 #define EFX_BYTE_4_WIDTH 8
  85
  86 #define EFX_BYTE_5_LBN 40
  87 #define EFX_BYTE_5_WIDTH 8
  88
  89 #define EFX_BYTE_6_LBN 48
  90 #define EFX_BYTE_6_WIDTH 8
  91
  92 #define EFX_BYTE_7_LBN 56
  93 #define EFX_BYTE_7_WIDTH 8
  94
  95 #define EFX_WORD_0_LBN 0
  96 #define EFX_WORD_0_WIDTH 16
  97
  98 #define EFX_WORD_1_LBN 16
  99 #define EFX_WORD_1_WIDTH 16
 100
 101 #define EFX_WORD_2_LBN 32
 102 #define EFX_WORD_2_WIDTH 16
 103
 104 #define EFX_WORD_3_LBN 48
 105 #define EFX_WORD_3_WIDTH 16
 106
 107 #define EFX_DWORD_0_LBN 0
 108 #define EFX_DWORD_0_WIDTH 32
 109
 110 #define EFX_DWORD_1_LBN 32
 111 #define EFX_DWORD_1_WIDTH 32
 112
 113 #define EFX_DWORD_2_LBN 64
 114 #define EFX_DWORD_2_WIDTH 32
 115
 116 #define EFX_DWORD_3_LBN 96
 117 #define EFX_DWORD_3_WIDTH 32
 118
 119 /* There are intentionally no EFX_QWORD_0 or EFX_QWORD_1 field definitions
 120  * here as the implementaion of EFX_QWORD_FIELD and EFX_OWORD_FIELD do not
 121  * support field widths larger than 32 bits.
 122  */
 123
 124 /* Specified attribute (i.e. LBN ow WIDTH) of the specified field */
 125 #define EFX_VAL(_field, _attribute)                                     \
 126         _field ## _ ## _attribute
 127
 128 /* Lowest bit number of the specified field */
 129 #define EFX_LOW_BIT(_field)                                             \
 130         EFX_VAL(_field, LBN)
 131
 132 /* Width of the specified field */
 133 #define EFX_WIDTH(_field)                                               \
 134         EFX_VAL(_field, WIDTH)
 135
 136 /* Highest bit number of the specified field */
 137 #define EFX_HIGH_BIT(_field)                                            \
 138         (EFX_LOW_BIT(_field) + EFX_WIDTH(_field) - 1)
 139
 140 /*
 141  * 64-bit mask equal in width to the specified field.
 142  *
 143  * For example, a field with width 5 would have a mask of 0x000000000000001f.
 144  */
 145 #define EFX_MASK64(_field)                                              \
 146         ((EFX_WIDTH(_field) == 64) ? ~((uint64_t)0) :                   \
 147             (((((uint64_t)1) << EFX_WIDTH(_field))) - 1))
 148 /*
 149  * 32-bit mask equal in width to the specified field.
 150  *
 151  * For example, a field with width 5 would have a mask of 0x0000001f.
 152  */
 153 #define EFX_MASK32(_field)                                              \
 154         ((EFX_WIDTH(_field) == 32) ? ~((uint32_t)0) :                   \
 155             (((((uint32_t)1) << EFX_WIDTH(_field))) - 1))
 156
 157 /*
 158  * 16-bit mask equal in width to the specified field.
 159  *
 160  * For example, a field with width 5 would have a mask of 0x001f.
 161  */
 162 #define EFX_MASK16(_field)                                              \
 163         ((EFX_WIDTH(_field) == 16) ? 0xffffu :                          \
 164             (uint16_t)((1 << EFX_WIDTH(_field)) - 1))
 165
 166 /*
 167  * 8-bit mask equal in width to the specified field.
 168  *
 169  * For example, a field with width 5 would have a mask of 0x1f.
 170  */
 171 #define EFX_MASK8(_field)                                               \
 172         ((uint8_t)((1 << EFX_WIDTH(_field)) - 1))
 173
 174 #pragma pack(1)
 175
 176 /*
 177  * A byte (i.e. 8-bit) datatype
 178  */
 179 typedef union efx_byte_u {
 180         uint8_t eb_u8[1];
 181 } efx_byte_t;
 182
 183 /*
 184  * A word (i.e. 16-bit) datatype
 185  *
 186  * This datatype is defined to be little-endian.
 187  */
 188 typedef union efx_word_u {
 189         efx_byte_t ew_byte[2];
 190         uint16_t ew_u16[1];
 191         uint8_t ew_u8[2];
 192 } efx_word_t;
 193
 194 /*
 195  * A doubleword (i.e. 32-bit) datatype
 196  *
 197  * This datatype is defined to be little-endian.
 198  */
 199 typedef union efx_dword_u {
 200         efx_byte_t ed_byte[4];
 201         efx_word_t ed_word[2];
 202         uint32_t ed_u32[1];
 203         uint16_t ed_u16[2];
 204         uint8_t ed_u8[4];
 205 } efx_dword_t;
 206
 207 /*
 208  * A quadword (i.e. 64-bit) datatype
 209  *
 210  * This datatype is defined to be little-endian.
 211  */
 212 typedef union efx_qword_u {
 213         efx_byte_t eq_byte[8];
 214         efx_word_t eq_word[4];
 215         efx_dword_t eq_dword[2];
 216 #if EFSYS_HAS_UINT64
 217         uint64_t eq_u64[1];
 218 #endif
 219         uint32_t eq_u32[2];
 220         uint16_t eq_u16[4];
 221         uint8_t eq_u8[8];
 222 } efx_qword_t;
 223
 224 /*
 225  * An octword (i.e. 128-bit) datatype
 226  *
 227  * This datatype is defined to be little-endian.
 228  */
 229 typedef union efx_oword_u {
 230         efx_byte_t eo_byte[16];
 231         efx_word_t eo_word[8];
 232         efx_dword_t eo_dword[4];
 233         efx_qword_t eo_qword[2];
 234 #if EFSYS_HAS_SSE2_M128
 235         __m128i eo_u128[1];
 236 #endif
 237 #if EFSYS_HAS_UINT64
 238         uint64_t eo_u64[2];
 239 #endif
 240         uint32_t eo_u32[4];
 241         uint16_t eo_u16[8];
 242         uint8_t eo_u8[16];
 243 } efx_oword_t;
 244
 245 #pragma pack()
 246
 247 #define __SWAP16(_x)                            \
 248         ((((_x) & 0xff) << 8) |                 \
 249         (((_x) >> 8) & 0xff))
 250
 251 #define __SWAP32(_x)                            \
 252         ((__SWAP16((_x) & 0xffff) << 16) |      \
 253         __SWAP16(((_x) >> 16) & 0xffff))
 254
 255 #define __SWAP64(_x)                            \
 256         ((__SWAP32((_x) & 0xffffffff) << 32) |  \
 257         __SWAP32(((_x) >> 32) & 0xffffffff))
 258
 259 #define __NOSWAP16(_x)          (_x)
 260 #define __NOSWAP32(_x)          (_x)
 261 #define __NOSWAP64(_x)          (_x)
 262
 263 #if EFSYS_IS_BIG_ENDIAN
 264
 265 #define __CPU_TO_LE_16(_x)      ((uint16_t)__SWAP16(_x))
 266 #define __LE_TO_CPU_16(_x)      ((uint16_t)__SWAP16(_x))
 267 #define __CPU_TO_BE_16(_x)      ((uint16_t)__NOSWAP16(_x))
 268 #define __BE_TO_CPU_16(_x)      ((uint16_t)__NOSWAP16(_x))
 269
 270 #define __CPU_TO_LE_32(_x)      ((uint32_t)__SWAP32(_x))
 271 #define __LE_TO_CPU_32(_x)      ((uint32_t)__SWAP32(_x))
 272 #define __CPU_TO_BE_32(_x)      ((uint32_t)__NOSWAP32(_x))
 273 #define __BE_TO_CPU_32(_x)      ((uint32_t)__NOSWAP32(_x))
 274
 275 #define __CPU_TO_LE_64(_x)      ((uint64_t)__SWAP64(_x))
 276 #define __LE_TO_CPU_64(_x)      ((uint64_t)__SWAP64(_x))
 277 #define __CPU_TO_BE_64(_x)      ((uint64_t)__NOSWAP64(_x))
 278 #define __BE_TO_CPU_64(_x)      ((uint64_t)__NOSWAP64(_x))
 279
 280 #elif EFSYS_IS_LITTLE_ENDIAN
 281
 282 #define __CPU_TO_LE_16(_x)      ((uint16_t)__NOSWAP16(_x))
 283 #define __LE_TO_CPU_16(_x)      ((uint16_t)__NOSWAP16(_x))
 284 #define __CPU_TO_BE_16(_x)      ((uint16_t)__SWAP16(_x))
 285 #define __BE_TO_CPU_16(_x)      ((uint16_t)__SWAP16(_x))
 286
 287 #define __CPU_TO_LE_32(_x)      ((uint32_t)__NOSWAP32(_x))
 288 #define __LE_TO_CPU_32(_x)      ((uint32_t)__NOSWAP32(_x))
 289 #define __CPU_TO_BE_32(_x)      ((uint32_t)__SWAP32(_x))
 290 #define __BE_TO_CPU_32(_x)      ((uint32_t)__SWAP32(_x))
 291
 292 #define __CPU_TO_LE_64(_x)      ((uint64_t)__NOSWAP64(_x))
 293 #define __LE_TO_CPU_64(_x)      ((uint64_t)__NOSWAP64(_x))
 294 #define __CPU_TO_BE_64(_x)      ((uint64_t)__SWAP64(_x))
 295 #define __BE_TO_CPU_64(_x)      ((uint64_t)__SWAP64(_x))
 296
 297 #else
 298
 299 #error "Neither of EFSYS_IS_{BIG,LITTLE}_ENDIAN is set"
 300
 301 #endif
 302
 303 #define __NATIVE_8(_x)  (uint8_t)(_x)
 304
 305 /* Format string for printing an efx_byte_t */
 306 #define EFX_BYTE_FMT "0x%02x"
 307
 308 /* Format string for printing an efx_word_t */
 309 #define EFX_WORD_FMT "0x%04x"
 310
 311 /* Format string for printing an efx_dword_t */
 312 #define EFX_DWORD_FMT "0x%08x"
 313
 314 /* Format string for printing an efx_qword_t */
 315 #define EFX_QWORD_FMT "0x%08x:%08x"
 316
 317 /* Format string for printing an efx_oword_t */
 318 #define EFX_OWORD_FMT "0x%08x:%08x:%08x:%08x"
 319
 320 /* Parameters for printing an efx_byte_t */
 321 #define EFX_BYTE_VAL(_byte)                                     \
 322         ((unsigned int)__NATIVE_8((_byte).eb_u8[0]))
 323
 324 /* Parameters for printing an efx_word_t */
 325 #define EFX_WORD_VAL(_word)                                     \
 326         ((unsigned int)__LE_TO_CPU_16((_word).ew_u16[0]))
 327
 328 /* Parameters for printing an efx_dword_t */
 329 #define EFX_DWORD_VAL(_dword)                                   \
 330         ((unsigned int)__LE_TO_CPU_32((_dword).ed_u32[0]))
 331
 332 /* Parameters for printing an efx_qword_t */
 333 #define EFX_QWORD_VAL(_qword)                                   \
 334         ((unsigned int)__LE_TO_CPU_32((_qword).eq_u32[1])),     \
 335         ((unsigned int)__LE_TO_CPU_32((_qword).eq_u32[0]))
 336
 337 /* Parameters for printing an efx_oword_t */
 338 #define EFX_OWORD_VAL(_oword)                                   \
 339         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[3])),     \
 340         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[2])),     \
 341         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[1])),     \
 342         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[0]))
 343
 344 /*
 345  * Stop lint complaining about some shifts.
 346  */
 347 #ifdef  __lint
 348 extern int fix_lint;
 349 #define FIX_LINT(_x)    (_x + fix_lint)
 350 #else
 351 #define FIX_LINT(_x)    (_x)
 352 #endif
 353
 354 /*
 355  * Extract bit field portion [low,high) from the native-endian element
 356  * which contains bits [min,max).
 357  *
 358  * For example, suppose "element" represents the high 32 bits of a
 359  * 64-bit value, and we wish to extract the bits belonging to the bit
 360  * field occupying bits 28-45 of this 64-bit value.
 361  *
 362  * Then EFX_EXTRACT(_element, 32, 63, 28, 45) would give
 363  *
 364  *   (_element) << 4
 365  *
 366  * The result will contain the relevant bits filled in in the range
 367  * [0,high-low), with garbage in bits [high-low+1,...).
 368  */
 369 #define EFX_EXTRACT_NATIVE(_element, _min, _max, _low, _high)           \
 370         ((FIX_LINT(_low > _max) || FIX_LINT(_high < _min)) ?            \
 371                 0U :                                                    \
 372                 ((_low > _min) ?                                        \
 373                         ((_element) >> (_low - _min)) :                 \
 374                         ((_element) << (_min - _low))))
 375
 376 /*
 377  * Extract bit field portion [low,high) from the 64-bit little-endian
 378  * element which contains bits [min,max)
 379  */
 380 #define EFX_EXTRACT64(_element, _min, _max, _low, _high)                \
 381         EFX_EXTRACT_NATIVE(__LE_TO_CPU_64(_element), _min, _max, _low, _high)
 382
 383 /*
 384  * Extract bit field portion [low,high) from the 32-bit little-endian
 385  * element which contains bits [min,max)
 386  */
 387 #define EFX_EXTRACT32(_element, _min, _max, _low, _high)                \
 388         EFX_EXTRACT_NATIVE(__LE_TO_CPU_32(_element), _min, _max, _low, _high)
 389
 390 /*
 391  * Extract bit field portion [low,high) from the 16-bit little-endian
 392  * element which contains bits [min,max)
 393  */
 394 #define EFX_EXTRACT16(_element, _min, _max, _low, _high)                \
 395         EFX_EXTRACT_NATIVE(__LE_TO_CPU_16(_element), _min, _max, _low, _high)
 396
 397 /*
 398  * Extract bit field portion [low,high) from the 8-bit
 399  * element which contains bits [min,max)
 400  */
 401 #define EFX_EXTRACT8(_element, _min, _max, _low, _high)                 \
 402         EFX_EXTRACT_NATIVE(__NATIVE_8(_element), _min, _max, _low, _high)
 403
 404 #define EFX_EXTRACT_OWORD64(_oword, _low, _high)                        \
 405         (EFX_EXTRACT64((_oword).eo_u64[0], FIX_LINT(0), FIX_LINT(63),   \
 406             _low, _high) |                                              \
 407         EFX_EXTRACT64((_oword).eo_u64[1], FIX_LINT(64), FIX_LINT(127),  \
 408             _low, _high))
 409
 410 #define EFX_EXTRACT_OWORD32(_oword, _low, _high)                        \
 411         (EFX_EXTRACT32((_oword).eo_u32[0], FIX_LINT(0), FIX_LINT(31),   \
 412             _low, _high) |                                              \
 413         EFX_EXTRACT32((_oword).eo_u32[1], FIX_LINT(32), FIX_LINT(63),   \
 414             _low, _high) |                                              \
 415         EFX_EXTRACT32((_oword).eo_u32[2], FIX_LINT(64), FIX_LINT(95),   \
 416             _low, _high) |                                              \
 417         EFX_EXTRACT32((_oword).eo_u32[3], FIX_LINT(96), FIX_LINT(127),  \
 418             _low, _high))
 419
 420 #define EFX_EXTRACT_QWORD64(_qword, _low, _high)                        \
 421         (EFX_EXTRACT64((_qword).eq_u64[0], FIX_LINT(0), FIX_LINT(63),   \
 422             _low, _high))
 423
 424 #define EFX_EXTRACT_QWORD32(_qword, _low, _high)                        \
 425         (EFX_EXTRACT32((_qword).eq_u32[0], FIX_LINT(0), FIX_LINT(31),   \
 426             _low, _high) |                                              \
 427         EFX_EXTRACT32((_qword).eq_u32[1], FIX_LINT(32), FIX_LINT(63),   \
 428             _low, _high))
 429
 430 #define EFX_EXTRACT_DWORD(_dword, _low, _high)                          \
 431         (EFX_EXTRACT32((_dword).ed_u32[0], FIX_LINT(0), FIX_LINT(31),   \
 432             _low, _high))
 433
 434 #define EFX_EXTRACT_WORD(_word, _low, _high)                            \
 435         (EFX_EXTRACT16((_word).ew_u16[0], FIX_LINT(0), FIX_LINT(15),    \
 436             _low, _high))
 437
 438 #define EFX_EXTRACT_BYTE(_byte, _low, _high)                            \
 439         (EFX_EXTRACT8((_byte).eb_u8[0], FIX_LINT(0), FIX_LINT(7),       \
 440             _low, _high))
 441
 442
 443 #define EFX_OWORD_FIELD64(_oword, _field)                               \
 444         ((uint32_t)EFX_EXTRACT_OWORD64(_oword, EFX_LOW_BIT(_field),     \
 445             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 446
 447 #define EFX_OWORD_FIELD32(_oword, _field)                               \
 448         (EFX_EXTRACT_OWORD32(_oword, EFX_LOW_BIT(_field),               \
 449             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 450
 451 #define EFX_QWORD_FIELD64(_qword, _field)                               \
 452         ((uint32_t)EFX_EXTRACT_QWORD64(_qword, EFX_LOW_BIT(_field),     \
 453             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 454
 455 #define EFX_QWORD_FIELD32(_qword, _field)                               \
 456         (EFX_EXTRACT_QWORD32(_qword, EFX_LOW_BIT(_field),               \
 457             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 458
 459 #define EFX_DWORD_FIELD(_dword, _field)                                 \
 460         (EFX_EXTRACT_DWORD(_dword, EFX_LOW_BIT(_field),                 \
 461             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 462
 463 #define EFX_WORD_FIELD(_word, _field)                                   \
 464         (EFX_EXTRACT_WORD(_word, EFX_LOW_BIT(_field),                   \
 465             EFX_HIGH_BIT(_field)) & EFX_MASK16(_field))
 466
 467 #define EFX_BYTE_FIELD(_byte, _field)                                   \
 468         (EFX_EXTRACT_BYTE(_byte, EFX_LOW_BIT(_field),                   \
 469             EFX_HIGH_BIT(_field)) & EFX_MASK8(_field))
 470
 471
 472 #define EFX_OWORD_IS_EQUAL64(_oword_a, _oword_b)                        \
 473         ((_oword_a).eo_u64[0] == (_oword_b).eo_u64[0] &&                \
 474             (_oword_a).eo_u64[1] == (_oword_b).eo_u64[1])
 475
 476 #define EFX_OWORD_IS_EQUAL32(_oword_a, _oword_b)                        \
 477         ((_oword_a).eo_u32[0] == (_oword_b).eo_u32[0] &&                \
 478             (_oword_a).eo_u32[1] == (_oword_b).eo_u32[1] &&             \
 479             (_oword_a).eo_u32[2] == (_oword_b).eo_u32[2] &&             \
 480             (_oword_a).eo_u32[3] == (_oword_b).eo_u32[3])
 481
 482 #define EFX_QWORD_IS_EQUAL64(_qword_a, _qword_b)                        \
 483         ((_qword_a).eq_u64[0] == (_qword_b).eq_u64[0])
 484
 485 #define EFX_QWORD_IS_EQUAL32(_qword_a, _qword_b)                        \
 486         ((_qword_a).eq_u32[0] == (_qword_b).eq_u32[0] &&                \
 487             (_qword_a).eq_u32[1] == (_qword_b).eq_u32[1])
 488
 489 #define EFX_DWORD_IS_EQUAL(_dword_a, _dword_b)                          \
 490         ((_dword_a).ed_u32[0] == (_dword_b).ed_u32[0])
 491
 492 #define EFX_WORD_IS_EQUAL(_word_a, _word_b)                             \
 493         ((_word_a).ew_u16[0] == (_word_b).ew_u16[0])
 494
 495 #define EFX_BYTE_IS_EQUAL(_byte_a, _byte_b)                             \
 496         ((_byte_a).eb_u8[0] == (_byte_b).eb_u8[0])
 497
 498
 499 #define EFX_OWORD_IS_ZERO64(_oword)                                     \
 500         (((_oword).eo_u64[0] |                                          \
 501             (_oword).eo_u64[1]) == 0)
 502
 503 #define EFX_OWORD_IS_ZERO32(_oword)                                     \
 504         (((_oword).eo_u32[0] |                                          \
 505             (_oword).eo_u32[1] |                                        \
 506             (_oword).eo_u32[2] |                                        \
 507             (_oword).eo_u32[3]) == 0)
 508
 509 #define EFX_QWORD_IS_ZERO64(_qword)                                     \
 510         (((_qword).eq_u64[0]) == 0)
 511
 512 #define EFX_QWORD_IS_ZERO32(_qword)                                     \
 513         (((_qword).eq_u32[0] |                                          \
 514             (_qword).eq_u32[1]) == 0)
 515
 516 #define EFX_DWORD_IS_ZERO(_dword)                                       \
 517         (((_dword).ed_u32[0]) == 0)
 518
 519 #define EFX_WORD_IS_ZERO(_word)                                         \
 520         (((_word).ew_u16[0]) == 0)
 521
 522 #define EFX_BYTE_IS_ZERO(_byte)                                         \
 523         (((_byte).eb_u8[0]) == 0)
 524
 525
 526 #define EFX_OWORD_IS_SET64(_oword)                                      \
 527         (((_oword).eo_u64[0] &                                          \
 528             (_oword).eo_u64[1]) == ~((uint64_t)0))
 529
 530 #define EFX_OWORD_IS_SET32(_oword)                                      \
 531         (((_oword).eo_u32[0] &                                          \
 532             (_oword).eo_u32[1] &                                        \
 533             (_oword).eo_u32[2] &                                        \
 534             (_oword).eo_u32[3]) == ~((uint32_t)0))
 535
 536 #define EFX_QWORD_IS_SET64(_qword)                                      \
 537         (((_qword).eq_u64[0]) == ~((uint64_t)0))
 538
 539 #define EFX_QWORD_IS_SET32(_qword)                                      \
 540         (((_qword).eq_u32[0] &                                          \
 541             (_qword).eq_u32[1]) == ~((uint32_t)0))
 542
 543 #define EFX_DWORD_IS_SET(_dword)                                        \
 544         ((_dword).ed_u32[0] == ~((uint32_t)0))
 545
 546 #define EFX_WORD_IS_SET(_word)                                          \
 547         ((_word).ew_u16[0] == ~((uint16_t)0))
 548
 549 #define EFX_BYTE_IS_SET(_byte)                                          \
 550         ((_byte).eb_u8[0] == ~((uint8_t)0))
 551
 552 /*
 553  * Construct bit field portion
 554  *
 555  * Creates the portion of the bit field [low,high) that lies within
 556  * the range [min,max).
 557  */
 558
 559 #define EFX_INSERT_NATIVE64(_min, _max, _low, _high, _value)            \
 560         (((_low > _max) || (_high < _min)) ?                            \
 561                 0U :                                                    \
 562                 ((_low > _min) ?                                        \
 563                         (((uint64_t)(_value)) << (_low - _min)) :       \
 564                         (((uint64_t)(_value)) >> (_min - _low))))
 565
 566 #define EFX_INSERT_NATIVE32(_min, _max, _low, _high, _value)            \
 567         (((_low > _max) || (_high < _min)) ?                            \
 568                 0U :                                                    \
 569                 ((_low > _min) ?                                        \
 570                         (((uint32_t)(_value)) << (_low - _min)) :       \
 571                         (((uint32_t)(_value)) >> (_min - _low))))
 572
 573 #define EFX_INSERT_NATIVE16(_min, _max, _low, _high, _value)            \
 574         (((_low > _max) || (_high < _min)) ?                            \
 575                 0U :                                                    \
 576                 (uint16_t)((_low > _min) ?                              \
 577                                 ((_value) << (_low - _min)) :           \
 578                                 ((_value) >> (_min - _low))))
 579
 580 #define EFX_INSERT_NATIVE8(_min, _max, _low, _high, _value)             \
 581         (((_low > _max) || (_high < _min)) ?                            \
 582                 0U :                                                    \
 583                 (uint8_t)((_low > _min) ?                               \
 584                                 ((_value) << (_low - _min)) :   \
 585                                 ((_value) >> (_min - _low))))
 586
 587 /*
 588  * Construct bit field portion
 589  *
 590  * Creates the portion of the named bit field that lies within the
 591  * range [min,max).
 592  */
 593 #define EFX_INSERT_FIELD_NATIVE64(_min, _max, _field, _value)           \
 594         EFX_INSERT_NATIVE64(_min, _max, EFX_LOW_BIT(_field),            \
 595             EFX_HIGH_BIT(_field), _value)
 596
 597 #define EFX_INSERT_FIELD_NATIVE32(_min, _max, _field, _value)           \
 598         EFX_INSERT_NATIVE32(_min, _max, EFX_LOW_BIT(_field),            \
 599             EFX_HIGH_BIT(_field), _value)
 600
 601 #define EFX_INSERT_FIELD_NATIVE16(_min, _max, _field, _value)           \
 602         EFX_INSERT_NATIVE16(_min, _max, EFX_LOW_BIT(_field),            \
 603             EFX_HIGH_BIT(_field), _value)
 604
 605 #define EFX_INSERT_FIELD_NATIVE8(_min, _max, _field, _value)            \
 606         EFX_INSERT_NATIVE8(_min, _max, EFX_LOW_BIT(_field),             \
 607             EFX_HIGH_BIT(_field), _value)
 608
 609 /*
 610  * Construct bit field
 611  *
 612  * Creates the portion of the named bit fields that lie within the
 613  * range [min,max).
 614  */
 615 #define EFX_INSERT_FIELDS64(_min, _max,                                 \
 616             _field1, _value1, _field2, _value2, _field3, _value3,       \
 617             _field4, _value4, _field5, _value5, _field6, _value6,       \
 618             _field7, _value7, _field8, _value8, _field9, _value9,       \
 619             _field10, _value10)                                         \
 620         __CPU_TO_LE_64(                                                 \
 621             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field1, _value1) |   \
 622             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field2, _value2) |   \
 623             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field3, _value3) |   \
 624             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field4, _value4) |   \
 625             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field5, _value5) |   \
 626             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field6, _value6) |   \
 627             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field7, _value7) |   \
 628             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field8, _value8) |   \
 629             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field9, _value9) |   \
 630             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field10, _value10))
 631
 632 #define EFX_INSERT_FIELDS32(_min, _max,                                 \
 633             _field1, _value1, _field2, _value2, _field3, _value3,       \
 634             _field4, _value4, _field5, _value5, _field6, _value6,       \
 635             _field7, _value7, _field8, _value8, _field9, _value9,       \
 636             _field10, _value10)                                         \
 637         __CPU_TO_LE_32(                                                 \
 638             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field1, _value1) |   \
 639             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field2, _value2) |   \
 640             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field3, _value3) |   \
 641             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field4, _value4) |   \
 642             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field5, _value5) |   \
 643             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field6, _value6) |   \
 644             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field7, _value7) |   \
 645             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field8, _value8) |   \
 646             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field9, _value9) |   \
 647             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field10, _value10))
 648
 649 #define EFX_INSERT_FIELDS16(_min, _max,                                 \
 650             _field1, _value1, _field2, _value2, _field3, _value3,       \
 651             _field4, _value4, _field5, _value5, _field6, _value6,       \
 652             _field7, _value7, _field8, _value8, _field9, _value9,       \
 653             _field10, _value10)                                         \
 654         __CPU_TO_LE_16(                                                 \
 655             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field1, _value1) |   \
 656             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field2, _value2) |   \
 657             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field3, _value3) |   \
 658             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field4, _value4) |   \
 659             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field5, _value5) |   \
 660             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field6, _value6) |   \
 661             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field7, _value7) |   \
 662             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field8, _value8) |   \
 663             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field9, _value9) |   \
 664             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field10, _value10))
 665
 666 #define EFX_INSERT_FIELDS8(_min, _max,                                  \
 667             _field1, _value1, _field2, _value2, _field3, _value3,       \
 668             _field4, _value4, _field5, _value5, _field6, _value6,       \
 669             _field7, _value7, _field8, _value8, _field9, _value9,       \
 670             _field10, _value10)                                         \
 671         __NATIVE_8(                                                     \
 672             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field1, _value1) |    \
 673             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field2, _value2) |    \
 674             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field3, _value3) |    \
 675             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field4, _value4) |    \
 676             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field5, _value5) |    \
 677             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field6, _value6) |    \
 678             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field7, _value7) |    \
 679             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field8, _value8) |    \
 680             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field9, _value9) |    \
 681             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field10, _value10))
 682
 683 #define EFX_POPULATE_OWORD64(_oword,                                    \
 684             _field1, _value1, _field2, _value2, _field3, _value3,       \
 685             _field4, _value4, _field5, _value5, _field6, _value6,       \
 686             _field7, _value7, _field8, _value8, _field9, _value9,       \
 687             _field10, _value10)                                         \
 688         do {                                                            \
 689                 _NOTE(CONSTANTCONDITION)                                \
 690                 (_oword).eo_u64[0] = EFX_INSERT_FIELDS64(0, 63,         \
 691                     _field1, _value1, _field2, _value2,                 \
 692                     _field3, _value3, _field4, _value4,                 \
 693                     _field5, _value5, _field6, _value6,                 \
 694                     _field7, _value7, _field8, _value8,                 \
 695                     _field9, _value9, _field10, _value10);              \
 696                 _NOTE(CONSTANTCONDITION)                                \
 697                 (_oword).eo_u64[1] = EFX_INSERT_FIELDS64(64, 127,       \
 698                     _field1, _value1, _field2, _value2,                 \
 699                     _field3, _value3, _field4, _value4,                 \
 700                     _field5, _value5, _field6, _value6,                 \
 701                     _field7, _value7, _field8, _value8,                 \
 702                     _field9, _value9, _field10, _value10);              \
 703         _NOTE(CONSTANTCONDITION)                                        \
 704         } while (B_FALSE)
 705
 706 #define EFX_POPULATE_OWORD32(_oword,                                    \
 707             _field1, _value1, _field2, _value2, _field3, _value3,       \
 708             _field4, _value4, _field5, _value5, _field6, _value6,       \
 709             _field7, _value7, _field8, _value8, _field9, _value9,       \
 710             _field10, _value10)                                         \
 711         do {                                                            \
 712                 _NOTE(CONSTANTCONDITION)                                \
 713                 (_oword).eo_u32[0] = EFX_INSERT_FIELDS32(0, 31,         \
 714                     _field1, _value1, _field2, _value2,                 \
 715                     _field3, _value3, _field4, _value4,                 \
 716                     _field5, _value5, _field6, _value6,                 \
 717                     _field7, _value7, _field8, _value8,                 \
 718                     _field9, _value9, _field10, _value10);              \
 719                 _NOTE(CONSTANTCONDITION)                                \
 720                 (_oword).eo_u32[1] = EFX_INSERT_FIELDS32(32, 63,        \
 721                     _field1, _value1, _field2, _value2,                 \
 722                     _field3, _value3, _field4, _value4,                 \
 723                     _field5, _value5, _field6, _value6,                 \
 724                     _field7, _value7, _field8, _value8,                 \
 725                     _field9, _value9, _field10, _value10);              \
 726                 _NOTE(CONSTANTCONDITION)                                \
 727                 (_oword).eo_u32[2] = EFX_INSERT_FIELDS32(64, 95,        \
 728                     _field1, _value1, _field2, _value2,                 \
 729                     _field3, _value3, _field4, _value4,                 \
 730                     _field5, _value5, _field6, _value6,                 \
 731                     _field7, _value7, _field8, _value8,                 \
 732                     _field9, _value9, _field10, _value10);              \
 733                 _NOTE(CONSTANTCONDITION)                                \
 734                 (_oword).eo_u32[3] = EFX_INSERT_FIELDS32(96, 127,       \
 735                     _field1, _value1, _field2, _value2,                 \
 736                     _field3, _value3, _field4, _value4,                 \
 737                     _field5, _value5, _field6, _value6,                 \
 738                     _field7, _value7, _field8, _value8,                 \
 739                     _field9, _value9, _field10, _value10);              \
 740         _NOTE(CONSTANTCONDITION)                                        \
 741         } while (B_FALSE)
 742
 743 #define EFX_POPULATE_QWORD64(_qword,                                    \
 744             _field1, _value1, _field2, _value2, _field3, _value3,       \
 745             _field4, _value4, _field5, _value5, _field6, _value6,       \
 746             _field7, _value7, _field8, _value8, _field9, _value9,       \
 747             _field10, _value10)                                         \
 748         do {                                                            \
 749                 _NOTE(CONSTANTCONDITION)                                \
 750                 (_qword).eq_u64[0] = EFX_INSERT_FIELDS64(0, 63,         \
 751                     _field1, _value1, _field2, _value2,                 \
 752                     _field3, _value3, _field4, _value4,                 \
 753                     _field5, _value5, _field6, _value6,                 \
 754                     _field7, _value7, _field8, _value8,                 \
 755                     _field9, _value9, _field10, _value10);              \
 756         _NOTE(CONSTANTCONDITION)                                        \
 757         } while (B_FALSE)
 758
 759 #define EFX_POPULATE_QWORD32(_qword,                                    \
 760             _field1, _value1, _field2, _value2, _field3, _value3,       \
 761             _field4, _value4, _field5, _value5, _field6, _value6,       \
 762             _field7, _value7, _field8, _value8, _field9, _value9,       \
 763             _field10, _value10)                                         \
 764         do {                                                            \
 765                 _NOTE(CONSTANTCONDITION)                                \
 766                 (_qword).eq_u32[0] = EFX_INSERT_FIELDS32(0, 31,         \
 767                     _field1, _value1, _field2, _value2,                 \
 768                     _field3, _value3, _field4, _value4,                 \
 769                     _field5, _value5, _field6, _value6,                 \
 770                     _field7, _value7, _field8, _value8,                 \
 771                     _field9, _value9, _field10, _value10);              \
 772                 _NOTE(CONSTANTCONDITION)                                \
 773                 (_qword).eq_u32[1] = EFX_INSERT_FIELDS32(32, 63,        \
 774                     _field1, _value1, _field2, _value2,                 \
 775                     _field3, _value3, _field4, _value4,                 \
 776                     _field5, _value5, _field6, _value6,                 \
 777                     _field7, _value7, _field8, _value8,                 \
 778                     _field9, _value9, _field10, _value10);              \
 779         _NOTE(CONSTANTCONDITION)                                        \
 780         } while (B_FALSE)
 781
 782 #define EFX_POPULATE_DWORD(_dword,                                      \
 783             _field1, _value1, _field2, _value2, _field3, _value3,       \
 784             _field4, _value4, _field5, _value5, _field6, _value6,       \
 785             _field7, _value7, _field8, _value8, _field9, _value9,       \
 786             _field10, _value10)                                         \
 787         do {                                                            \
 788                 _NOTE(CONSTANTCONDITION)                                \
 789                 (_dword).ed_u32[0] = EFX_INSERT_FIELDS32(0, 31,         \
 790                     _field1, _value1, _field2, _value2,                 \
 791                     _field3, _value3, _field4, _value4,                 \
 792                     _field5, _value5, _field6, _value6,                 \
 793                     _field7, _value7, _field8, _value8,                 \
 794                     _field9, _value9, _field10, _value10);              \
 795         _NOTE(CONSTANTCONDITION)                                        \
 796         } while (B_FALSE)
 797
 798 #define EFX_POPULATE_WORD(_word,                                        \
 799             _field1, _value1, _field2, _value2, _field3, _value3,       \
 800             _field4, _value4, _field5, _value5, _field6, _value6,       \
 801             _field7, _value7, _field8, _value8, _field9, _value9,       \
 802             _field10, _value10)                                         \
 803         do {                                                            \
 804                 _NOTE(CONSTANTCONDITION)                                \
 805                 (_word).ew_u16[0] = EFX_INSERT_FIELDS16(0, 15,          \
 806                     _field1, _value1, _field2, _value2,                 \
 807                     _field3, _value3, _field4, _value4,                 \
 808                     _field5, _value5, _field6, _value6,                 \
 809                     _field7, _value7, _field8, _value8,                 \
 810                     _field9, _value9, _field10, _value10);              \
 811         _NOTE(CONSTANTCONDITION)                                        \
 812         } while (B_FALSE)
 813
 814 #define EFX_POPULATE_BYTE(_byte,                                        \
 815             _field1, _value1, _field2, _value2, _field3, _value3,       \
 816             _field4, _value4, _field5, _value5, _field6, _value6,       \
 817             _field7, _value7, _field8, _value8, _field9, _value9,       \
 818             _field10, _value10)                                         \
 819         do {                                                            \
 820                 _NOTE(CONSTANTCONDITION)                                \
 821                 (_byte).eb_u8[0] = EFX_INSERT_FIELDS8(0, 7,             \
 822                     _field1, _value1, _field2, _value2,                 \
 823                     _field3, _value3, _field4, _value4,                 \
 824                     _field5, _value5, _field6, _value6,                 \
 825                     _field7, _value7, _field8, _value8,                 \
 826                     _field9, _value9, _field10, _value10);              \
 827         _NOTE(CONSTANTCONDITION)                                        \
 828         } while (B_FALSE)
 829
 830 /* Populate an octword field with various numbers of arguments */
 831 #define EFX_POPULATE_OWORD_10 EFX_POPULATE_OWORD
 832
 833 #define EFX_POPULATE_OWORD_9(_oword,                                    \
 834             _field1, _value1, _field2, _value2, _field3, _value3,       \
 835             _field4, _value4, _field5, _value5, _field6, _value6,       \
 836             _field7, _value7, _field8, _value8, _field9, _value9)       \
 837         EFX_POPULATE_OWORD_10(_oword, EFX_DUMMY_FIELD, 0,               \
 838             _field1, _value1, _field2, _value2, _field3, _value3,       \
 839             _field4, _value4, _field5, _value5, _field6, _value6,       \
 840             _field7, _value7, _field8, _value8, _field9, _value9)
 841
 842 #define EFX_POPULATE_OWORD_8(_oword,                                    \
 843             _field1, _value1, _field2, _value2, _field3, _value3,       \
 844             _field4, _value4, _field5, _value5, _field6, _value6,       \
 845             _field7, _value7, _field8, _value8)                         \
 846         EFX_POPULATE_OWORD_9(_oword, EFX_DUMMY_FIELD, 0,                \
 847             _field1, _value1, _field2, _value2, _field3, _value3,       \
 848             _field4, _value4, _field5, _value5, _field6, _value6,       \
 849             _field7, _value7, _field8, _value8)
 850
 851 #define EFX_POPULATE_OWORD_7(_oword,                                    \
 852             _field1, _value1, _field2, _value2, _field3, _value3,       \
 853             _field4, _value4, _field5, _value5, _field6, _value6,       \
 854             _field7, _value7)                                           \
 855         EFX_POPULATE_OWORD_8(_oword, EFX_DUMMY_FIELD, 0,                \
 856             _field1, _value1, _field2, _value2, _field3, _value3,       \
 857             _field4, _value4, _field5, _value5, _field6, _value6,       \
 858             _field7, _value7)
 859
 860 #define EFX_POPULATE_OWORD_6(_oword,                                    \
 861             _field1, _value1, _field2, _value2, _field3, _value3,       \
 862             _field4, _value4, _field5, _value5, _field6, _value6)       \
 863         EFX_POPULATE_OWORD_7(_oword, EFX_DUMMY_FIELD, 0,                \
 864             _field1, _value1, _field2, _value2, _field3, _value3,       \
 865             _field4, _value4, _field5, _value5, _field6, _value6)
 866
 867 #define EFX_POPULATE_OWORD_5(_oword,                                    \
 868             _field1, _value1, _field2, _value2, _field3, _value3,       \
 869             _field4, _value4, _field5, _value5)                         \
 870         EFX_POPULATE_OWORD_6(_oword, EFX_DUMMY_FIELD, 0,                \
 871             _field1, _value1, _field2, _value2, _field3, _value3,       \
 872             _field4, _value4, _field5, _value5)
 873
 874 #define EFX_POPULATE_OWORD_4(_oword,                                    \
 875             _field1, _value1, _field2, _value2, _field3, _value3,       \
 876             _field4, _value4)                                           \
 877         EFX_POPULATE_OWORD_5(_oword, EFX_DUMMY_FIELD, 0,                \
 878             _field1, _value1, _field2, _value2, _field3, _value3,       \
 879             _field4, _value4)
 880
 881 #define EFX_POPULATE_OWORD_3(_oword,                                    \
 882             _field1, _value1, _field2, _value2, _field3, _value3)       \
 883         EFX_POPULATE_OWORD_4(_oword, EFX_DUMMY_FIELD, 0,                \
 884             _field1, _value1, _field2, _value2, _field3, _value3)
 885
 886 #define EFX_POPULATE_OWORD_2(_oword,                                    \
 887             _field1, _value1, _field2, _value2)                         \
 888         EFX_POPULATE_OWORD_3(_oword, EFX_DUMMY_FIELD, 0,                \
 889             _field1, _value1, _field2, _value2)
 890
 891 #define EFX_POPULATE_OWORD_1(_oword,                                    \
 892             _field1, _value1)                                           \
 893         EFX_POPULATE_OWORD_2(_oword, EFX_DUMMY_FIELD, 0,                \
 894             _field1, _value1)
 895
 896 #define EFX_ZERO_OWORD(_oword)                                          \
 897         EFX_POPULATE_OWORD_1(_oword, EFX_DUMMY_FIELD, 0)
 898
 899 #define EFX_SET_OWORD(_oword)                                           \
 900         EFX_POPULATE_OWORD_4(_oword,                                    \
 901             EFX_DWORD_0, 0xffffffff, EFX_DWORD_1, 0xffffffff,           \
 902             EFX_DWORD_2, 0xffffffff, EFX_DWORD_3, 0xffffffff)
 903
 904 /* Populate a quadword field with various numbers of arguments */
 905 #define EFX_POPULATE_QWORD_10 EFX_POPULATE_QWORD
 906
 907 #define EFX_POPULATE_QWORD_9(_qword,                                    \
 908             _field1, _value1, _field2, _value2, _field3, _value3,       \
 909             _field4, _value4, _field5, _value5, _field6, _value6,       \
 910             _field7, _value7, _field8, _value8, _field9, _value9)       \
 911         EFX_POPULATE_QWORD_10(_qword, EFX_DUMMY_FIELD, 0,               \
 912             _field1, _value1, _field2, _value2, _field3, _value3,       \
 913             _field4, _value4, _field5, _value5, _field6, _value6,       \
 914             _field7, _value7, _field8, _value8, _field9, _value9)
 915
 916 #define EFX_POPULATE_QWORD_8(_qword,                                    \
 917             _field1, _value1, _field2, _value2, _field3, _value3,       \
 918             _field4, _value4, _field5, _value5, _field6, _value6,       \
 919             _field7, _value7, _field8, _value8)                         \
 920         EFX_POPULATE_QWORD_9(_qword, EFX_DUMMY_FIELD, 0,                \
 921             _field1, _value1, _field2, _value2, _field3, _value3,       \
 922             _field4, _value4, _field5, _value5, _field6, _value6,       \
 923             _field7, _value7, _field8, _value8)
 924
 925 #define EFX_POPULATE_QWORD_7(_qword,                                    \
 926             _field1, _value1, _field2, _value2, _field3, _value3,       \
 927             _field4, _value4, _field5, _value5, _field6, _value6,       \
 928             _field7, _value7)                                           \
 929         EFX_POPULATE_QWORD_8(_qword, EFX_DUMMY_FIELD, 0,                \
 930             _field1, _value1, _field2, _value2, _field3, _value3,       \
 931             _field4, _value4, _field5, _value5, _field6, _value6,       \
 932             _field7, _value7)
 933
 934 #define EFX_POPULATE_QWORD_6(_qword,                                    \
 935             _field1, _value1, _field2, _value2, _field3, _value3,       \
 936             _field4, _value4, _field5, _value5, _field6, _value6)       \
 937         EFX_POPULATE_QWORD_7(_qword, EFX_DUMMY_FIELD, 0,                \
 938             _field1, _value1, _field2, _value2, _field3, _value3,       \
 939             _field4, _value4, _field5, _value5, _field6, _value6)
 940
 941 #define EFX_POPULATE_QWORD_5(_qword,                                    \
 942             _field1, _value1, _field2, _value2, _field3, _value3,       \
 943             _field4, _value4, _field5, _value5)                         \
 944         EFX_POPULATE_QWORD_6(_qword, EFX_DUMMY_FIELD, 0,                \
 945             _field1, _value1, _field2, _value2, _field3, _value3,       \
 946             _field4, _value4, _field5, _value5)
 947
 948 #define EFX_POPULATE_QWORD_4(_qword,                                    \
 949             _field1, _value1, _field2, _value2, _field3, _value3,       \
 950             _field4, _value4)                                           \
 951         EFX_POPULATE_QWORD_5(_qword, EFX_DUMMY_FIELD, 0,                \
 952             _field1, _value1, _field2, _value2, _field3, _value3,       \
 953             _field4, _value4)
 954
 955 #define EFX_POPULATE_QWORD_3(_qword,                                    \
 956             _field1, _value1, _field2, _value2, _field3, _value3)       \
 957         EFX_POPULATE_QWORD_4(_qword, EFX_DUMMY_FIELD, 0,                \
 958             _field1, _value1, _field2, _value2, _field3, _value3)
 959
 960 #define EFX_POPULATE_QWORD_2(_qword,                                    \
 961             _field1, _value1, _field2, _value2)                         \
 962         EFX_POPULATE_QWORD_3(_qword, EFX_DUMMY_FIELD, 0,                \
 963             _field1, _value1, _field2, _value2)
 964
 965 #define EFX_POPULATE_QWORD_1(_qword,                                    \
 966             _field1, _value1)                                           \
 967         EFX_POPULATE_QWORD_2(_qword, EFX_DUMMY_FIELD, 0,                \
 968             _field1, _value1)
 969
 970 #define EFX_ZERO_QWORD(_qword)                                          \
 971         EFX_POPULATE_QWORD_1(_qword, EFX_DUMMY_FIELD, 0)
 972
 973 #define EFX_SET_QWORD(_qword)                                           \
 974         EFX_POPULATE_QWORD_2(_qword,                                    \
 975             EFX_DWORD_0, 0xffffffff, EFX_DWORD_1, 0xffffffff)
 976
 977 /* Populate a dword field with various numbers of arguments */
 978 #define EFX_POPULATE_DWORD_10 EFX_POPULATE_DWORD
 979
 980 #define EFX_POPULATE_DWORD_9(_dword,                                    \
 981             _field1, _value1, _field2, _value2, _field3, _value3,       \
 982             _field4, _value4, _field5, _value5, _field6, _value6,       \
 983             _field7, _value7, _field8, _value8, _field9, _value9)       \
 984         EFX_POPULATE_DWORD_10(_dword, EFX_DUMMY_FIELD, 0,               \
 985             _field1, _value1, _field2, _value2, _field3, _value3,       \
 986             _field4, _value4, _field5, _value5, _field6, _value6,       \
 987             _field7, _value7, _field8, _value8, _field9, _value9)
 988
 989 #define EFX_POPULATE_DWORD_8(_dword,                                    \
 990             _field1, _value1, _field2, _value2, _field3, _value3,       \
 991             _field4, _value4, _field5, _value5, _field6, _value6,       \
 992             _field7, _value7, _field8, _value8)                         \
 993         EFX_POPULATE_DWORD_9(_dword, EFX_DUMMY_FIELD, 0,                \
 994             _field1, _value1, _field2, _value2, _field3, _value3,       \
 995             _field4, _value4, _field5, _value5, _field6, _value6,       \
 996             _field7, _value7, _field8, _value8)
 997
 998 #define EFX_POPULATE_DWORD_7(_dword,                                    \
 999             _field1, _value1, _field2, _value2, _field3, _value3,       \
1000             _field4, _value4, _field5, _value5, _field6, _value6,       \
1001             _field7, _value7)                                           \
1002         EFX_POPULATE_DWORD_8(_dword, EFX_DUMMY_FIELD, 0,                \
1003             _field1, _value1, _field2, _value2, _field3, _value3,       \
1004             _field4, _value4, _field5, _value5, _field6, _value6,       \
1005             _field7, _value7)
1006
1007 #define EFX_POPULATE_DWORD_6(_dword,                                    \
1008             _field1, _value1, _field2, _value2, _field3, _value3,       \
1009             _field4, _value4, _field5, _value5, _field6, _value6)       \
1010         EFX_POPULATE_DWORD_7(_dword, EFX_DUMMY_FIELD, 0,                \
1011             _field1, _value1, _field2, _value2, _field3, _value3,       \
1012             _field4, _value4, _field5, _value5, _field6, _value6)
1013
1014 #define EFX_POPULATE_DWORD_5(_dword,                                    \
1015             _field1, _value1, _field2, _value2, _field3, _value3,       \
1016             _field4, _value4, _field5, _value5)                         \
1017         EFX_POPULATE_DWORD_6(_dword, EFX_DUMMY_FIELD, 0,                \
1018             _field1, _value1, _field2, _value2, _field3, _value3,       \
1019             _field4, _value4, _field5, _value5)
1020
1021 #define EFX_POPULATE_DWORD_4(_dword,                                    \
1022             _field1, _value1, _field2, _value2, _field3, _value3,       \
1023             _field4, _value4)                                           \
1024         EFX_POPULATE_DWORD_5(_dword, EFX_DUMMY_FIELD, 0,                \
1025             _field1, _value1, _field2, _value2, _field3, _value3,       \
1026             _field4, _value4)
1027
1028 #define EFX_POPULATE_DWORD_3(_dword,                                    \
1029             _field1, _value1, _field2, _value2, _field3, _value3)       \
1030         EFX_POPULATE_DWORD_4(_dword, EFX_DUMMY_FIELD, 0,                \
1031             _field1, _value1, _field2, _value2, _field3, _value3)
1032
1033 #define EFX_POPULATE_DWORD_2(_dword,                                    \
1034             _field1, _value1, _field2, _value2)                         \
1035         EFX_POPULATE_DWORD_3(_dword, EFX_DUMMY_FIELD, 0,                \
1036             _field1, _value1, _field2, _value2)
1037
1038 #define EFX_POPULATE_DWORD_1(_dword,                                    \
1039             _field1, _value1)                                           \
1040         EFX_POPULATE_DWORD_2(_dword, EFX_DUMMY_FIELD, 0,                \
1041             _field1, _value1)
1042
1043 #define EFX_ZERO_DWORD(_dword)                                          \
1044         EFX_POPULATE_DWORD_1(_dword, EFX_DUMMY_FIELD, 0)
1045
1046 #define EFX_SET_DWORD(_dword)                                           \
1047         EFX_POPULATE_DWORD_1(_dword,                                    \
1048             EFX_DWORD_0, 0xffffffff)
1049
1050 /* Populate a word field with various numbers of arguments */
1051 #define EFX_POPULATE_WORD_10 EFX_POPULATE_WORD
1052
1053 #define EFX_POPULATE_WORD_9(_word,                                      \
1054             _field1, _value1, _field2, _value2, _field3, _value3,       \
1055             _field4, _value4, _field5, _value5, _field6, _value6,       \
1056             _field7, _value7, _field8, _value8, _field9, _value9)       \
1057         EFX_POPULATE_WORD_10(_word, EFX_DUMMY_FIELD, 0,                 \
1058             _field1, _value1, _field2, _value2, _field3, _value3,       \
1059             _field4, _value4, _field5, _value5, _field6, _value6,       \
1060             _field7, _value7, _field8, _value8, _field9, _value9)
1061
1062 #define EFX_POPULATE_WORD_8(_word,                                      \
1063             _field1, _value1, _field2, _value2, _field3, _value3,       \
1064             _field4, _value4, _field5, _value5, _field6, _value6,       \
1065             _field7, _value7, _field8, _value8)                         \
1066         EFX_POPULATE_WORD_9(_word, EFX_DUMMY_FIELD, 0,                  \
1067             _field1, _value1, _field2, _value2, _field3, _value3,       \
1068             _field4, _value4, _field5, _value5, _field6, _value6,       \
1069             _field7, _value7, _field8, _value8)
1070
1071 #define EFX_POPULATE_WORD_7(_word,                                      \
1072             _field1, _value1, _field2, _value2, _field3, _value3,       \
1073             _field4, _value4, _field5, _value5, _field6, _value6,       \
1074             _field7, _value7)                                           \
1075         EFX_POPULATE_WORD_8(_word, EFX_DUMMY_FIELD, 0,                  \
1076             _field1, _value1, _field2, _value2, _field3, _value3,       \
1077             _field4, _value4, _field5, _value5, _field6, _value6,       \
1078             _field7, _value7)
1079
1080 #define EFX_POPULATE_WORD_6(_word,                                      \
1081             _field1, _value1, _field2, _value2, _field3, _value3,       \
1082             _field4, _value4, _field5, _value5, _field6, _value6)       \
1083         EFX_POPULATE_WORD_7(_word, EFX_DUMMY_FIELD, 0,                  \
1084             _field1, _value1, _field2, _value2, _field3, _value3,       \
1085             _field4, _value4, _field5, _value5, _field6, _value6)
1086
1087 #define EFX_POPULATE_WORD_5(_word,                                      \
1088             _field1, _value1, _field2, _value2, _field3, _value3,       \
1089             _field4, _value4, _field5, _value5)                         \
1090         EFX_POPULATE_WORD_6(_word, EFX_DUMMY_FIELD, 0,                  \
1091             _field1, _value1, _field2, _value2, _field3, _value3,       \
1092             _field4, _value4, _field5, _value5)
1093
1094 #define EFX_POPULATE_WORD_4(_word,                                      \
1095             _field1, _value1, _field2, _value2, _field3, _value3,       \
1096             _field4, _value4)                                           \
1097         EFX_POPULATE_WORD_5(_word, EFX_DUMMY_FIELD, 0,                  \
1098             _field1, _value1, _field2, _value2, _field3, _value3,       \
1099             _field4, _value4)
1100
1101 #define EFX_POPULATE_WORD_3(_word,                                      \
1102             _field1, _value1, _field2, _value2, _field3, _value3)       \
1103         EFX_POPULATE_WORD_4(_word, EFX_DUMMY_FIELD, 0,                  \
1104             _field1, _value1, _field2, _value2, _field3, _value3)
1105
1106 #define EFX_POPULATE_WORD_2(_word,                                      \
1107             _field1, _value1, _field2, _value2)                         \
1108         EFX_POPULATE_WORD_3(_word, EFX_DUMMY_FIELD, 0,                  \
1109             _field1, _value1, _field2, _value2)
1110
1111 #define EFX_POPULATE_WORD_1(_word,                                      \
1112             _field1, _value1)                                           \
1113         EFX_POPULATE_WORD_2(_word, EFX_DUMMY_FIELD, 0,                  \
1114             _field1, _value1)
1115
1116 #define EFX_ZERO_WORD(_word)                                            \
1117         EFX_POPULATE_WORD_1(_word, EFX_DUMMY_FIELD, 0)
1118
1119 #define EFX_SET_WORD(_word)                                             \
1120         EFX_POPULATE_WORD_1(_word,                                      \
1121             EFX_WORD_0, 0xffff)
1122
1123 /* Populate a byte field with various numbers of arguments */
1124 #define EFX_POPULATE_BYTE_10 EFX_POPULATE_BYTE
1125
1126 #define EFX_POPULATE_BYTE_9(_byte,                                      \
1127             _field1, _value1, _field2, _value2, _field3, _value3,       \
1128             _field4, _value4, _field5, _value5, _field6, _value6,       \
1129             _field7, _value7, _field8, _value8, _field9, _value9)       \
1130         EFX_POPULATE_BYTE_10(_byte, EFX_DUMMY_FIELD, 0,                 \
1131             _field1, _value1, _field2, _value2, _field3, _value3,       \
1132             _field4, _value4, _field5, _value5, _field6, _value6,       \
1133             _field7, _value7, _field8, _value8, _field9, _value9)
1134
1135 #define EFX_POPULATE_BYTE_8(_byte,                                      \
1136             _field1, _value1, _field2, _value2, _field3, _value3,       \
1137             _field4, _value4, _field5, _value5, _field6, _value6,       \
1138             _field7, _value7, _field8, _value8)                         \
1139         EFX_POPULATE_BYTE_9(_byte, EFX_DUMMY_FIELD, 0,                  \
1140             _field1, _value1, _field2, _value2, _field3, _value3,       \
1141             _field4, _value4, _field5, _value5, _field6, _value6,       \
1142             _field7, _value7, _field8, _value8)
1143
1144 #define EFX_POPULATE_BYTE_7(_byte,                                      \
1145             _field1, _value1, _field2, _value2, _field3, _value3,       \
1146             _field4, _value4, _field5, _value5, _field6, _value6,       \
1147             _field7, _value7)                                           \
1148         EFX_POPULATE_BYTE_8(_byte, EFX_DUMMY_FIELD, 0,                  \
1149             _field1, _value1, _field2, _value2, _field3, _value3,       \
1150             _field4, _value4, _field5, _value5, _field6, _value6,       \
1151             _field7, _value7)
1152
1153 #define EFX_POPULATE_BYTE_6(_byte,                                      \
1154             _field1, _value1, _field2, _value2, _field3, _value3,       \
1155             _field4, _value4, _field5, _value5, _field6, _value6)       \
1156         EFX_POPULATE_BYTE_7(_byte, EFX_DUMMY_FIELD, 0,                  \
1157             _field1, _value1, _field2, _value2, _field3, _value3,       \
1158             _field4, _value4, _field5, _value5, _field6, _value6)
1159
1160 #define EFX_POPULATE_BYTE_5(_byte,                                      \
1161             _field1, _value1, _field2, _value2, _field3, _value3,       \
1162             _field4, _value4, _field5, _value5)                         \
1163         EFX_POPULATE_BYTE_6(_byte, EFX_DUMMY_FIELD, 0,                  \
1164             _field1, _value1, _field2, _value2, _field3, _value3,       \
1165             _field4, _value4, _field5, _value5)
1166
1167 #define EFX_POPULATE_BYTE_4(_byte,                                      \
1168             _field1, _value1, _field2, _value2, _field3, _value3,       \
1169             _field4, _value4)                                           \
1170         EFX_POPULATE_BYTE_5(_byte, EFX_DUMMY_FIELD, 0,                  \
1171             _field1, _value1, _field2, _value2, _field3, _value3,       \
1172             _field4, _value4)
1173
1174 #define EFX_POPULATE_BYTE_3(_byte,                                      \
1175             _field1, _value1, _field2, _value2, _field3, _value3)       \
1176         EFX_POPULATE_BYTE_4(_byte, EFX_DUMMY_FIELD, 0,                  \
1177             _field1, _value1, _field2, _value2, _field3, _value3)
1178
1179 #define EFX_POPULATE_BYTE_2(_byte,                                      \
1180             _field1, _value1, _field2, _value2)                         \
1181         EFX_POPULATE_BYTE_3(_byte, EFX_DUMMY_FIELD, 0,                  \
1182             _field1, _value1, _field2, _value2)
1183
1184 #define EFX_POPULATE_BYTE_1(_byte,                                      \
1185             _field1, _value1)                                           \
1186         EFX_POPULATE_BYTE_2(_byte, EFX_DUMMY_FIELD, 0,                  \
1187             _field1, _value1)
1188
1189 #define EFX_ZERO_BYTE(_byte)                                            \
1190         EFX_POPULATE_BYTE_1(_byte, EFX_DUMMY_FIELD, 0)
1191
1192 #define EFX_SET_BYTE(_byte)                                             \
1193         EFX_POPULATE_BYTE_1(_byte,                                      \
1194             EFX_BYTE_0, 0xff)
1195
1196 /*
1197  * Modify a named field within an already-populated structure.  Used
1198  * for read-modify-write operations.
1199  */
1200
1201 #define EFX_INSERT_FIELD64(_min, _max, _field, _value)                  \
1202         __CPU_TO_LE_64(EFX_INSERT_FIELD_NATIVE64(_min, _max, _field, _value))
1203
1204 #define EFX_INSERT_FIELD32(_min, _max, _field, _value)                  \
1205         __CPU_TO_LE_32(EFX_INSERT_FIELD_NATIVE32(_min, _max, _field, _value))
1206
1207 #define EFX_INSERT_FIELD16(_min, _max, _field, _value)                  \
1208         __CPU_TO_LE_16(EFX_INSERT_FIELD_NATIVE16(_min, _max, _field, _value))
1209
1210 #define EFX_INSERT_FIELD8(_min, _max, _field, _value)                   \
1211         __NATIVE_8(EFX_INSERT_FIELD_NATIVE8(_min, _max, _field, _value))
1212
1213 #define EFX_INPLACE_MASK64(_min, _max, _field)                          \
1214         EFX_INSERT_FIELD64(_min, _max, _field, EFX_MASK64(_field))
1215
1216 #define EFX_INPLACE_MASK32(_min, _max, _field)                          \
1217         EFX_INSERT_FIELD32(_min, _max, _field, EFX_MASK32(_field))
1218
1219 #define EFX_INPLACE_MASK16(_min, _max, _field)                          \
1220         EFX_INSERT_FIELD16(_min, _max, _field, EFX_MASK16(_field))
1221
1222 #define EFX_INPLACE_MASK8(_min, _max, _field)                           \
1223         EFX_INSERT_FIELD8(_min, _max, _field, EFX_MASK8(_field))
1224
1225 #define EFX_SET_OWORD_FIELD64(_oword, _field, _value)                   \
1226         do {                                                            \
1227                 _NOTE(CONSTANTCONDITION)                                \
1228                 (_oword).eo_u64[0] = (((_oword).eo_u64[0] &             \
1229                     ~EFX_INPLACE_MASK64(0, 63, _field)) |               \
1230                     EFX_INSERT_FIELD64(0, 63, _field, _value));         \
1231                 _NOTE(CONSTANTCONDITION)                                \
1232                 (_oword).eo_u64[1] = (((_oword).eo_u64[1] &             \
1233                     ~EFX_INPLACE_MASK64(64, 127, _field)) |             \
1234                     EFX_INSERT_FIELD64(64, 127, _field, _value));       \
1235         _NOTE(CONSTANTCONDITION)                                        \
1236         } while (B_FALSE)
1237
1238 #define EFX_SET_OWORD_FIELD32(_oword, _field, _value)                   \
1239         do {                                                            \
1240                 _NOTE(CONSTANTCONDITION)                                \
1241                 (_oword).eo_u32[0] = (((_oword).eo_u32[0] &             \
1242                     ~EFX_INPLACE_MASK32(0, 31, _field)) |               \
1243                     EFX_INSERT_FIELD32(0, 31, _field, _value));         \
1244                 _NOTE(CONSTANTCONDITION)                                \
1245                 (_oword).eo_u32[1] = (((_oword).eo_u32[1] &             \
1246                     ~EFX_INPLACE_MASK32(32, 63, _field)) |              \
1247                     EFX_INSERT_FIELD32(32, 63, _field, _value));        \
1248                 _NOTE(CONSTANTCONDITION)                                \
1249                 (_oword).eo_u32[2] = (((_oword).eo_u32[2] &             \
1250                     ~EFX_INPLACE_MASK32(64, 95, _field)) |              \
1251                     EFX_INSERT_FIELD32(64, 95, _field, _value));        \
1252                 _NOTE(CONSTANTCONDITION)                                \
1253                 (_oword).eo_u32[3] = (((_oword).eo_u32[3] &             \
1254                     ~EFX_INPLACE_MASK32(96, 127, _field)) |             \
1255                     EFX_INSERT_FIELD32(96, 127, _field, _value));       \
1256         _NOTE(CONSTANTCONDITION)                                        \
1257         } while (B_FALSE)
1258
1259 #define EFX_SET_QWORD_FIELD64(_qword, _field, _value)                   \
1260         do {                                                            \
1261                 _NOTE(CONSTANTCONDITION)                                \
1262                 (_qword).eq_u64[0] = (((_qword).eq_u64[0] &             \
1263                     ~EFX_INPLACE_MASK64(0, 63, _field)) |               \
1264                     EFX_INSERT_FIELD64(0, 63, _field, _value));         \
1265         _NOTE(CONSTANTCONDITION)                                        \
1266         } while (B_FALSE)
1267
1268 #define EFX_SET_QWORD_FIELD32(_qword, _field, _value)                   \
1269         do {                                                            \
1270                 _NOTE(CONSTANTCONDITION)                                \
1271                 (_qword).eq_u32[0] = (((_qword).eq_u32[0] &             \
1272                     ~EFX_INPLACE_MASK32(0, 31, _field)) |               \
1273                     EFX_INSERT_FIELD32(0, 31, _field, _value));         \
1274                 _NOTE(CONSTANTCONDITION)                                \
1275                 (_qword).eq_u32[1] = (((_qword).eq_u32[1] &             \
1276                     ~EFX_INPLACE_MASK32(32, 63, _field)) |              \
1277                     EFX_INSERT_FIELD32(32, 63, _field, _value));        \
1278         _NOTE(CONSTANTCONDITION)                                        \
1279         } while (B_FALSE)
1280
1281 #define EFX_SET_DWORD_FIELD(_dword, _field, _value)                     \
1282         do {                                                            \
1283                 _NOTE(CONSTANTCONDITION)                                \
1284                 (_dword).ed_u32[0] = (((_dword).ed_u32[0] &             \
1285                     ~EFX_INPLACE_MASK32(0, 31, _field)) |               \
1286                     EFX_INSERT_FIELD32(0, 31, _field, _value));         \
1287         _NOTE(CONSTANTCONDITION)                                        \
1288         } while (B_FALSE)
1289
1290 #define EFX_SET_WORD_FIELD(_word, _field, _value)                       \
1291         do {                                                            \
1292                 _NOTE(CONSTANTCONDITION)                                \
1293                 (_word).ew_u16[0] = (((_word).ew_u16[0] &               \
1294                     ~EFX_INPLACE_MASK16(0, 15, _field)) |               \
1295                     EFX_INSERT_FIELD16(0, 15, _field, _value));         \
1296         _NOTE(CONSTANTCONDITION)                                        \
1297         } while (B_FALSE)
1298
1299 #define EFX_SET_BYTE_FIELD(_byte, _field, _value)                       \
1300         do {                                                            \
1301                 _NOTE(CONSTANTCONDITION)                                \
1302                 (_byte).eb_u8[0] = (((_byte).eb_u8[0] &                 \
1303                     ~EFX_INPLACE_MASK8(0, 7, _field)) |                 \
1304                     EFX_INSERT_FIELD8(0, 7, _field, _value));           \
1305         _NOTE(CONSTANTCONDITION)                                        \
1306         } while (B_FALSE)
1307
1308 /*
1309  * Set or clear a numbered bit within an octword.
1310  */
1311
1312 #define EFX_SHIFT64(_bit, _base)                                        \
1313         (((_bit) >= (_base) && (_bit) < (_base) + 64) ?                 \
1314                 ((uint64_t)1 << ((_bit) - (_base))) :                   \
1315                 0U)
1316
1317 #define EFX_SHIFT32(_bit, _base)                                        \
1318         (((_bit) >= (_base) && (_bit) < (_base) + 32) ?                 \
1319                 ((uint32_t)1 << ((_bit) - (_base))) :                   \
1320                 0U)
1321
1322 #define EFX_SHIFT16(_bit, _base)                                        \
1323         (((_bit) >= (_base) && (_bit) < (_base) + 16) ?                 \
1324                 (uint16_t)(1 << ((_bit) - (_base))) :                   \
1325                 0U)
1326
1327 #define EFX_SHIFT8(_bit, _base)                                         \
1328         (((_bit) >= (_base) && (_bit) < (_base) + 8) ?                  \
1329                 (uint8_t)(1 << ((_bit) - (_base))) :                    \
1330                 0U)
1331
1332 #define EFX_SET_OWORD_BIT64(_oword, _bit)                               \
1333         do {                                                            \
1334                 _NOTE(CONSTANTCONDITION)                                \
1335                 (_oword).eo_u64[0] |=                                   \
1336                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)));     \
1337                 (_oword).eo_u64[1] |=                                   \
1338                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(64)));    \
1339         _NOTE(CONSTANTCONDITION)                                        \
1340         } while (B_FALSE)
1341
1342 #define EFX_SET_OWORD_BIT32(_oword, _bit)                               \
1343         do {                                                            \
1344                 _NOTE(CONSTANTCONDITION)                                \
1345                 (_oword).eo_u32[0] |=                                   \
1346                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));     \
1347                 (_oword).eo_u32[1] |=                                   \
1348                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)));    \
1349                 (_oword).eo_u32[2] |=                                   \
1350                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(64)));    \
1351                 (_oword).eo_u32[3] |=                                   \
1352                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(96)));    \
1353         _NOTE(CONSTANTCONDITION)                                        \
1354         } while (B_FALSE)
1355
1356 #define EFX_CLEAR_OWORD_BIT64(_oword, _bit)                             \
1357         do {                                                            \
1358                 _NOTE(CONSTANTCONDITION)                                \
1359                 (_oword).eo_u64[0] &=                                   \
1360                     __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(0)));    \
1361                 (_oword).eo_u64[1] &=                                   \
1362                     __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(64)));   \
1363         _NOTE(CONSTANTCONDITION)                                        \
1364         } while (B_FALSE)
1365
1366 #define EFX_CLEAR_OWORD_BIT32(_oword, _bit)                             \
1367         do {                                                            \
1368                 _NOTE(CONSTANTCONDITION)                                \
1369                 (_oword).eo_u32[0] &=                                   \
1370                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));    \
1371                 (_oword).eo_u32[1] &=                                   \
1372                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(32)));   \
1373                 (_oword).eo_u32[2] &=                                   \
1374                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(64)));   \
1375                 (_oword).eo_u32[3] &=                                   \
1376                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(96)));   \
1377         _NOTE(CONSTANTCONDITION)                                        \
1378         } while (B_FALSE)
1379
1380 #define EFX_TEST_OWORD_BIT64(_oword, _bit)                              \
1381         (((_oword).eo_u64[0] &                                          \
1382                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)))) ||  \
1383         ((_oword).eo_u64[1] &                                           \
1384                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(64)))))
1385
1386 #define EFX_TEST_OWORD_BIT32(_oword, _bit)                              \
1387         (((_oword).eo_u32[0] &                                          \
1388                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)))) ||  \
1389         ((_oword).eo_u32[1] &                                           \
1390                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)))) || \
1391         ((_oword).eo_u32[2] &                                           \
1392                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(64)))) || \
1393         ((_oword).eo_u32[3] &                                           \
1394                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(96)))))
1395
1396
1397 #define EFX_SET_QWORD_BIT64(_qword, _bit)                               \
1398         do {                                                            \
1399                 _NOTE(CONSTANTCONDITION)                                \
1400                 (_qword).eq_u64[0] |=                                   \
1401                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)));     \
1402         _NOTE(CONSTANTCONDITION)                                        \
1403         } while (B_FALSE)
1404
1405 #define EFX_SET_QWORD_BIT32(_qword, _bit)                               \
1406         do {                                                            \
1407                 _NOTE(CONSTANTCONDITION)                                \
1408                 (_qword).eq_u32[0] |=                                   \
1409                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));     \
1410                 (_qword).eq_u32[1] |=                                   \
1411                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)));    \
1412         _NOTE(CONSTANTCONDITION)                                        \
1413         } while (B_FALSE)
1414
1415 #define EFX_CLEAR_QWORD_BIT64(_qword, _bit)                             \
1416         do {                                                            \
1417                 _NOTE(CONSTANTCONDITION)                                \
1418                 (_qword).eq_u64[0] &=                                   \
1419                     __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(0)));    \
1420         _NOTE(CONSTANTCONDITION)                                        \
1421         } while (B_FALSE)
1422
1423 #define EFX_CLEAR_QWORD_BIT32(_qword, _bit)                             \
1424         do {                                                            \
1425                 _NOTE(CONSTANTCONDITION)                                \
1426                 (_qword).eq_u32[0] &=                                   \
1427                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));    \
1428                 (_qword).eq_u32[1] &=                                   \
1429                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(32)));   \
1430         _NOTE(CONSTANTCONDITION)                                        \
1431         } while (B_FALSE)
1432
1433 #define EFX_TEST_QWORD_BIT64(_qword, _bit)                              \
1434         (((_qword).eq_u64[0] &                                          \
1435                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)))) != 0)
1436
1437 #define EFX_TEST_QWORD_BIT32(_qword, _bit)                              \
1438         (((_qword).eq_u32[0] &                                          \
1439                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)))) ||  \
1440         ((_qword).eq_u32[1] &                                           \
1441                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)))))
1442
1443
1444 #define EFX_SET_DWORD_BIT(_dword, _bit)                                 \
1445         do {                                                            \
1446                 (_dword).ed_u32[0] |=                                   \
1447                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));     \
1448         _NOTE(CONSTANTCONDITION)                                        \
1449         } while (B_FALSE)
1450
1451 #define EFX_CLEAR_DWORD_BIT(_dword, _bit)                               \
1452         do {                                                            \
1453                 (_dword).ed_u32[0] &=                                   \
1454                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));    \
1455         _NOTE(CONSTANTCONDITION)                                        \
1456         } while (B_FALSE)
1457
1458 #define EFX_TEST_DWORD_BIT(_dword, _bit)                                \
1459         (((_dword).ed_u32[0] &                                          \
1460                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)))) != 0)
1461
1462
1463 #define EFX_SET_WORD_BIT(_word, _bit)                                   \
1464         do {                                                            \
1465                 (_word).ew_u16[0] |=                                    \
1466                     __CPU_TO_LE_16(EFX_SHIFT16(_bit, FIX_LINT(0)));     \
1467         _NOTE(CONSTANTCONDITION)                                        \
1468         } while (B_FALSE)
1469
1470 #define EFX_CLEAR_WORD_BIT(_word, _bit)                                 \
1471         do {                                                            \
1472                 (_word).ew_u32[0] &=                                    \
1473                     __CPU_TO_LE_16(~EFX_SHIFT16(_bit, FIX_LINT(0)));    \
1474         _NOTE(CONSTANTCONDITION)                                        \
1475         } while (B_FALSE)
1476
1477 #define EFX_TEST_WORD_BIT(_word, _bit)                                  \
1478         (((_word).ew_u16[0] &                                           \
1479                     __CPU_TO_LE_16(EFX_SHIFT16(_bit, FIX_LINT(0)))) != 0)
1480
1481
1482 #define EFX_SET_BYTE_BIT(_byte, _bit)                                   \
1483         do {                                                            \
1484                 (_byte).eb_u8[0] |=                                     \
1485                     __NATIVE_8(EFX_SHIFT8(_bit, FIX_LINT(0)));          \
1486         _NOTE(CONSTANTCONDITION)                                        \
1487         } while (B_FALSE)
1488
1489 #define EFX_CLEAR_BYTE_BIT(_byte, _bit)                                 \
1490         do {                                                            \
1491                 (_byte).eb_u8[0] &=                                     \
1492                     __NATIVE_8(~EFX_SHIFT8(_bit, FIX_LINT(0)));         \
1493         _NOTE(CONSTANTCONDITION)                                        \
1494         } while (B_FALSE)
1495
1496 #define EFX_TEST_BYTE_BIT(_byte, _bit)                                  \
1497         (((_byte).eb_u8[0] &                                            \
1498                     __NATIVE_8(EFX_SHIFT8(_bit, FIX_LINT(0)))) != 0)
1499
1500
1501 #define EFX_OR_OWORD64(_oword1, _oword2)                                \
1502         do {                                                            \
1503                 (_oword1).eo_u64[0] |= (_oword2).eo_u64[0];             \
1504                 (_oword1).eo_u64[1] |= (_oword2).eo_u64[1];             \
1505         _NOTE(CONSTANTCONDITION)                                        \
1506         } while (B_FALSE)
1507
1508 #define EFX_OR_OWORD32(_oword1, _oword2)                                \
1509         do {                                                            \
1510                 (_oword1).eo_u32[0] |= (_oword2).eo_u32[0];             \
1511                 (_oword1).eo_u32[1] |= (_oword2).eo_u32[1];             \
1512                 (_oword1).eo_u32[2] |= (_oword2).eo_u32[2];             \
1513                 (_oword1).eo_u32[3] |= (_oword2).eo_u32[3];             \
1514         _NOTE(CONSTANTCONDITION)                                        \
1515         } while (B_FALSE)
1516
1517 #define EFX_AND_OWORD64(_oword1, _oword2)                               \
1518         do {                                                            \
1519                 (_oword1).eo_u64[0] &= (_oword2).eo_u64[0];             \
1520                 (_oword1).eo_u64[1] &= (_oword2).eo_u64[1];             \
1521         _NOTE(CONSTANTCONDITION)                                        \
1522         } while (B_FALSE)
1523
1524 #define EFX_AND_OWORD32(_oword1, _oword2)                               \
1525         do {                                                            \
1526                 (_oword1).eo_u32[0] &= (_oword2).eo_u32[0];             \
1527                 (_oword1).eo_u32[1] &= (_oword2).eo_u32[1];             \
1528                 (_oword1).eo_u32[2] &= (_oword2).eo_u32[2];             \
1529                 (_oword1).eo_u32[3] &= (_oword2).eo_u32[3];             \
1530         _NOTE(CONSTANTCONDITION)                                        \
1531         } while (B_FALSE)
1532
1533 #define EFX_OR_QWORD64(_qword1, _qword2)                                \
1534         do {                                                            \
1535                 (_qword1).eq_u64[0] |= (_qword2).eq_u64[0];             \
1536         _NOTE(CONSTANTCONDITION)                                        \
1537         } while (B_FALSE)
1538
1539 #define EFX_OR_QWORD32(_qword1, _qword2)                                \
1540         do {                                                            \
1541                 (_qword1).eq_u32[0] |= (_qword2).eq_u32[0];             \
1542                 (_qword1).eq_u32[1] |= (_qword2).eq_u32[1];             \
1543         _NOTE(CONSTANTCONDITION)                                        \
1544         } while (B_FALSE)
1545
1546 #define EFX_AND_QWORD64(_qword1, _qword2)                               \
1547         do {                                                            \
1548                 (_qword1).eq_u64[0] &= (_qword2).eq_u64[0];             \
1549         _NOTE(CONSTANTCONDITION)                                        \
1550         } while (B_FALSE)
1551
1552 #define EFX_AND_QWORD32(_qword1, _qword2)                               \
1553         do {                                                            \
1554                 (_qword1).eq_u32[0] &= (_qword2).eq_u32[0];             \
1555                 (_qword1).eq_u32[1] &= (_qword2).eq_u32[1];             \
1556         _NOTE(CONSTANTCONDITION)                                        \
1557         } while (B_FALSE)
1558
1559 #define EFX_OR_DWORD(_dword1, _dword2)                                  \
1560         do {                                                            \
1561                 (_dword1).ed_u32[0] |= (_dword2).ed_u32[0];             \
1562         _NOTE(CONSTANTCONDITION)                                        \
1563         } while (B_FALSE)
1564
1565 #define EFX_AND_DWORD(_dword1, _dword2)                                 \
1566         do {                                                            \
1567                 (_dword1).ed_u32[0] &= (_dword2).ed_u32[0];             \
1568         _NOTE(CONSTANTCONDITION)                                        \
1569         } while (B_FALSE)
1570
1571 #define EFX_OR_WORD(_word1, _word2)                                     \
1572         do {                                                            \
1573                 (_word1).ew_u16[0] |= (_word2).ew_u16[0];               \
1574         _NOTE(CONSTANTCONDITION)                                        \
1575         } while (B_FALSE)
1576
1577 #define EFX_AND_WORD(_word1, _word2)                                    \
1578         do {                                                            \
1579                 (_word1).ew_u16[0] &= (_word2).ew_u16[0];               \
1580         _NOTE(CONSTANTCONDITION)                                        \
1581         } while (B_FALSE)
1582
1583 #define EFX_OR_BYTE(_byte1, _byte2)                                     \
1584         do {                                                            \
1585                 (_byte1).eb_u8[0] |= (_byte2).eb_u8[0];                 \
1586         _NOTE(CONSTANTCONDITION)                                        \
1587         } while (B_FALSE)
1588
1589 #define EFX_AND_BYTE(_byte1, _byte2)                                    \
1590         do {                                                            \
1591                 (_byte1).eb_u8[0] &= (_byte2).eb_u8[0];                 \
1592         _NOTE(CONSTANTCONDITION)                                        \
1593         } while (B_FALSE)
1594
1595 #if EFSYS_USE_UINT64
1596 #define EFX_OWORD_FIELD         EFX_OWORD_FIELD64
1597 #define EFX_QWORD_FIELD         EFX_QWORD_FIELD64
1598 #define EFX_OWORD_IS_EQUAL      EFX_OWORD_IS_EQUAL64
1599 #define EFX_QWORD_IS_EQUAL      EFX_QWORD_IS_EQUAL64
1600 #define EFX_OWORD_IS_ZERO       EFX_OWORD_IS_ZERO64
1601 #define EFX_QWORD_IS_ZERO       EFX_QWORD_IS_ZERO64
1602 #define EFX_OWORD_IS_SET        EFX_OWORD_IS_SET64
1603 #define EFX_QWORD_IS_SET        EFX_QWORD_IS_SET64
1604 #define EFX_POPULATE_OWORD      EFX_POPULATE_OWORD64
1605 #define EFX_POPULATE_QWORD      EFX_POPULATE_QWORD64
1606 #define EFX_SET_OWORD_FIELD     EFX_SET_OWORD_FIELD64
1607 #define EFX_SET_QWORD_FIELD     EFX_SET_QWORD_FIELD64
1608 #define EFX_SET_OWORD_BIT       EFX_SET_OWORD_BIT64
1609 #define EFX_CLEAR_OWORD_BIT     EFX_CLEAR_OWORD_BIT64
1610 #define EFX_TEST_OWORD_BIT      EFX_TEST_OWORD_BIT64
1611 #define EFX_SET_QWORD_BIT       EFX_SET_QWORD_BIT64
1612 #define EFX_CLEAR_QWORD_BIT     EFX_CLEAR_QWORD_BIT64
1613 #define EFX_TEST_QWORD_BIT      EFX_TEST_QWORD_BIT64
1614 #define EFX_OR_OWORD            EFX_OR_OWORD64
1615 #define EFX_AND_OWORD           EFX_AND_OWORD64
1616 #define EFX_OR_QWORD            EFX_OR_QWORD64
1617 #define EFX_AND_QWORD           EFX_AND_QWORD64
1618 #else
1619 #define EFX_OWORD_FIELD         EFX_OWORD_FIELD32
1620 #define EFX_QWORD_FIELD         EFX_QWORD_FIELD32
1621 #define EFX_OWORD_IS_EQUAL      EFX_OWORD_IS_EQUAL32
1622 #define EFX_QWORD_IS_EQUAL      EFX_QWORD_IS_EQUAL32
1623 #define EFX_OWORD_IS_ZERO       EFX_OWORD_IS_ZERO32
1624 #define EFX_QWORD_IS_ZERO       EFX_QWORD_IS_ZERO32
1625 #define EFX_OWORD_IS_SET        EFX_OWORD_IS_SET32
1626 #define EFX_QWORD_IS_SET        EFX_QWORD_IS_SET32
1627 #define EFX_POPULATE_OWORD      EFX_POPULATE_OWORD32
1628 #define EFX_POPULATE_QWORD      EFX_POPULATE_QWORD32
1629 #define EFX_SET_OWORD_FIELD     EFX_SET_OWORD_FIELD32
1630 #define EFX_SET_QWORD_FIELD     EFX_SET_QWORD_FIELD32
1631 #define EFX_SET_OWORD_BIT       EFX_SET_OWORD_BIT32
1632 #define EFX_CLEAR_OWORD_BIT     EFX_CLEAR_OWORD_BIT32
1633 #define EFX_TEST_OWORD_BIT      EFX_TEST_OWORD_BIT32
1634 #define EFX_SET_QWORD_BIT       EFX_SET_QWORD_BIT32
1635 #define EFX_CLEAR_QWORD_BIT     EFX_CLEAR_QWORD_BIT32
1636 #define EFX_TEST_QWORD_BIT      EFX_TEST_QWORD_BIT32
1637 #define EFX_OR_OWORD            EFX_OR_OWORD32
1638 #define EFX_AND_OWORD           EFX_AND_OWORD32
1639 #define EFX_OR_QWORD            EFX_OR_QWORD32
1640 #define EFX_AND_QWORD           EFX_AND_QWORD32
1641 #endif
1642
1643 #ifdef  __cplusplus
1644 }
1645 #endif
1646
1647 #endif  /* _SYS_EFX_TYPES_H */