drivers/net/sfc/base/efx_types.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright (c) 2007-2018 Solarflare Communications Inc.
   4  * All rights reserved.
   5  *
   6  * Ackowledgement to Fen Systems Ltd.
   7  */
   8
   9 #ifndef _SYS_EFX_TYPES_H
  10 #define _SYS_EFX_TYPES_H
  11
  12 #include "efsys.h"
  13
  14 #ifdef  __cplusplus
  15 extern "C" {
  16 #endif
  17
  18 /*
  19  * Bitfield access
  20  *
  21  * Solarflare NICs make extensive use of bitfields up to 128 bits
  22  * wide.  Since there is no native 128-bit datatype on most systems,
  23  * and since 64-bit datatypes are inefficient on 32-bit systems and
  24  * vice versa, we wrap accesses in a way that uses the most efficient
  25  * datatype.
  26  *
  27  * The NICs are PCI devices and therefore little-endian.  Since most
  28  * of the quantities that we deal with are DMAed to/from host memory,
  29  * we define    our datatypes (efx_oword_t, efx_qword_t and efx_dword_t)
  30  * to be little-endian.
  31  *
  32  * In the less common case of using PIO for individual register
  33  * writes, we construct the little-endian datatype in host memory and
  34  * then use non-swapping register access primitives, rather than
  35  * constructing a native-endian datatype and relying on implicit
  36  * byte-swapping.  (We use a similar strategy for register reads.)
  37  */
  38
  39 /*
  40  * NOTE: Field definitions here and elsewhere are done in terms of a lowest
  41  *       bit number (LBN) and a width.
  42  */
  43
  44 #define EFX_DUMMY_FIELD_LBN 0
  45 #define EFX_DUMMY_FIELD_WIDTH 0
  46
  47 #define EFX_BYTE_0_LBN 0
  48 #define EFX_BYTE_0_WIDTH 8
  49
  50 #define EFX_BYTE_1_LBN 8
  51 #define EFX_BYTE_1_WIDTH 8
  52
  53 #define EFX_BYTE_2_LBN 16
  54 #define EFX_BYTE_2_WIDTH 8
  55
  56 #define EFX_BYTE_3_LBN 24
  57 #define EFX_BYTE_3_WIDTH 8
  58
  59 #define EFX_BYTE_4_LBN 32
  60 #define EFX_BYTE_4_WIDTH 8
  61
  62 #define EFX_BYTE_5_LBN 40
  63 #define EFX_BYTE_5_WIDTH 8
  64
  65 #define EFX_BYTE_6_LBN 48
  66 #define EFX_BYTE_6_WIDTH 8
  67
  68 #define EFX_BYTE_7_LBN 56
  69 #define EFX_BYTE_7_WIDTH 8
  70
  71 #define EFX_WORD_0_LBN 0
  72 #define EFX_WORD_0_WIDTH 16
  73
  74 #define EFX_WORD_1_LBN 16
  75 #define EFX_WORD_1_WIDTH 16
  76
  77 #define EFX_WORD_2_LBN 32
  78 #define EFX_WORD_2_WIDTH 16
  79
  80 #define EFX_WORD_3_LBN 48
  81 #define EFX_WORD_3_WIDTH 16
  82
  83 #define EFX_DWORD_0_LBN 0
  84 #define EFX_DWORD_0_WIDTH 32
  85
  86 #define EFX_DWORD_1_LBN 32
  87 #define EFX_DWORD_1_WIDTH 32
  88
  89 #define EFX_DWORD_2_LBN 64
  90 #define EFX_DWORD_2_WIDTH 32
  91
  92 #define EFX_DWORD_3_LBN 96
  93 #define EFX_DWORD_3_WIDTH 32
  94
  95 /*
  96  * There are intentionally no EFX_QWORD_0 or EFX_QWORD_1 field definitions
  97  * here as the implementaion of EFX_QWORD_FIELD and EFX_OWORD_FIELD do not
  98  * support field widths larger than 32 bits.
  99  */
 100
 101 /* Specified attribute (i.e. LBN ow WIDTH) of the specified field */
 102 #define EFX_VAL(_field, _attribute)                                     \
 103         _field ## _ ## _attribute
 104
 105 /* Lowest bit number of the specified field */
 106 #define EFX_LOW_BIT(_field)                                             \
 107         EFX_VAL(_field, LBN)
 108
 109 /* Width of the specified field */
 110 #define EFX_WIDTH(_field)                                               \
 111         EFX_VAL(_field, WIDTH)
 112
 113 /* Highest bit number of the specified field */
 114 #define EFX_HIGH_BIT(_field)                                            \
 115         (EFX_LOW_BIT(_field) + EFX_WIDTH(_field) - 1)
 116
 117 /*
 118  * 64-bit mask equal in width to the specified field.
 119  *
 120  * For example, a field with width 5 would have a mask of 0x000000000000001f.
 121  */
 122 #define EFX_MASK64(_field)                                              \
 123         ((EFX_WIDTH(_field) == 64) ? ~((uint64_t)0) :                   \
 124             (((((uint64_t)1) << EFX_WIDTH(_field))) - 1))
 125 /*
 126  * 32-bit mask equal in width to the specified field.
 127  *
 128  * For example, a field with width 5 would have a mask of 0x0000001f.
 129  */
 130 #define EFX_MASK32(_field)                                              \
 131         ((EFX_WIDTH(_field) == 32) ? ~((uint32_t)0) :                   \
 132             (((((uint32_t)1) << EFX_WIDTH(_field))) - 1))
 133
 134 /*
 135  * 16-bit mask equal in width to the specified field.
 136  *
 137  * For example, a field with width 5 would have a mask of 0x001f.
 138  */
 139 #define EFX_MASK16(_field)                                              \
 140         ((EFX_WIDTH(_field) == 16) ? 0xffffu :                          \
 141             (uint16_t)((1 << EFX_WIDTH(_field)) - 1))
 142
 143 /*
 144  * 8-bit mask equal in width to the specified field.
 145  *
 146  * For example, a field with width 5 would have a mask of 0x1f.
 147  */
 148 #define EFX_MASK8(_field)                                               \
 149         ((uint8_t)((1 << EFX_WIDTH(_field)) - 1))
 150
 151 #pragma pack(1)
 152
 153 /*
 154  * A byte (i.e. 8-bit) datatype
 155  */
 156 typedef union efx_byte_u {
 157         uint8_t eb_u8[1];
 158 } efx_byte_t;
 159
 160 /*
 161  * A word (i.e. 16-bit) datatype
 162  *
 163  * This datatype is defined to be little-endian.
 164  */
 165 typedef union efx_word_u {
 166         efx_byte_t ew_byte[2];
 167         uint16_t ew_u16[1];
 168         uint8_t ew_u8[2];
 169 } efx_word_t;
 170
 171 /*
 172  * A doubleword (i.e. 32-bit) datatype
 173  *
 174  * This datatype is defined to be little-endian.
 175  */
 176 typedef union efx_dword_u {
 177         efx_byte_t ed_byte[4];
 178         efx_word_t ed_word[2];
 179         uint32_t ed_u32[1];
 180         uint16_t ed_u16[2];
 181         uint8_t ed_u8[4];
 182 } efx_dword_t;
 183
 184 /*
 185  * A quadword (i.e. 64-bit) datatype
 186  *
 187  * This datatype is defined to be little-endian.
 188  */
 189 typedef union efx_qword_u {
 190         efx_byte_t eq_byte[8];
 191         efx_word_t eq_word[4];
 192         efx_dword_t eq_dword[2];
 193 #if EFSYS_HAS_UINT64
 194         uint64_t eq_u64[1];
 195 #endif
 196         uint32_t eq_u32[2];
 197         uint16_t eq_u16[4];
 198         uint8_t eq_u8[8];
 199 } efx_qword_t;
 200
 201 /*
 202  * An octword (i.e. 128-bit) datatype
 203  *
 204  * This datatype is defined to be little-endian.
 205  */
 206 typedef union efx_oword_u {
 207         efx_byte_t eo_byte[16];
 208         efx_word_t eo_word[8];
 209         efx_dword_t eo_dword[4];
 210         efx_qword_t eo_qword[2];
 211 #if EFSYS_HAS_SSE2_M128
 212         __m128i eo_u128[1];
 213 #endif
 214 #if EFSYS_HAS_UINT64
 215         uint64_t eo_u64[2];
 216 #endif
 217         uint32_t eo_u32[4];
 218         uint16_t eo_u16[8];
 219         uint8_t eo_u8[16];
 220 } efx_oword_t;
 221
 222 #pragma pack()
 223
 224 #define __SWAP16(_x)                            \
 225         ((((_x) & 0xff) << 8) |                 \
 226         (((_x) >> 8) & 0xff))
 227
 228 #define __SWAP32(_x)                            \
 229         ((__SWAP16((_x) & 0xffff) << 16) |      \
 230         __SWAP16(((_x) >> 16) & 0xffff))
 231
 232 #define __SWAP64(_x)                            \
 233         ((__SWAP32((_x) & 0xffffffff) << 32) |  \
 234         __SWAP32(((_x) >> 32) & 0xffffffff))
 235
 236 #define __NOSWAP16(_x)          (_x)
 237 #define __NOSWAP32(_x)          (_x)
 238 #define __NOSWAP64(_x)          (_x)
 239
 240 #if EFSYS_IS_BIG_ENDIAN
 241
 242 #define __CPU_TO_LE_16(_x)      ((uint16_t)__SWAP16(_x))
 243 #define __LE_TO_CPU_16(_x)      ((uint16_t)__SWAP16(_x))
 244 #define __CPU_TO_BE_16(_x)      ((uint16_t)__NOSWAP16(_x))
 245 #define __BE_TO_CPU_16(_x)      ((uint16_t)__NOSWAP16(_x))
 246
 247 #define __CPU_TO_LE_32(_x)      ((uint32_t)__SWAP32(_x))
 248 #define __LE_TO_CPU_32(_x)      ((uint32_t)__SWAP32(_x))
 249 #define __CPU_TO_BE_32(_x)      ((uint32_t)__NOSWAP32(_x))
 250 #define __BE_TO_CPU_32(_x)      ((uint32_t)__NOSWAP32(_x))
 251
 252 #define __CPU_TO_LE_64(_x)      ((uint64_t)__SWAP64(_x))
 253 #define __LE_TO_CPU_64(_x)      ((uint64_t)__SWAP64(_x))
 254 #define __CPU_TO_BE_64(_x)      ((uint64_t)__NOSWAP64(_x))
 255 #define __BE_TO_CPU_64(_x)      ((uint64_t)__NOSWAP64(_x))
 256
 257 #elif EFSYS_IS_LITTLE_ENDIAN
 258
 259 #define __CPU_TO_LE_16(_x)      ((uint16_t)__NOSWAP16(_x))
 260 #define __LE_TO_CPU_16(_x)      ((uint16_t)__NOSWAP16(_x))
 261 #define __CPU_TO_BE_16(_x)      ((uint16_t)__SWAP16(_x))
 262 #define __BE_TO_CPU_16(_x)      ((uint16_t)__SWAP16(_x))
 263
 264 #define __CPU_TO_LE_32(_x)      ((uint32_t)__NOSWAP32(_x))
 265 #define __LE_TO_CPU_32(_x)      ((uint32_t)__NOSWAP32(_x))
 266 #define __CPU_TO_BE_32(_x)      ((uint32_t)__SWAP32(_x))
 267 #define __BE_TO_CPU_32(_x)      ((uint32_t)__SWAP32(_x))
 268
 269 #define __CPU_TO_LE_64(_x)      ((uint64_t)__NOSWAP64(_x))
 270 #define __LE_TO_CPU_64(_x)      ((uint64_t)__NOSWAP64(_x))
 271 #define __CPU_TO_BE_64(_x)      ((uint64_t)__SWAP64(_x))
 272 #define __BE_TO_CPU_64(_x)      ((uint64_t)__SWAP64(_x))
 273
 274 #else
 275
 276 #error "Neither of EFSYS_IS_{BIG,LITTLE}_ENDIAN is set"
 277
 278 #endif
 279
 280 #define __NATIVE_8(_x)  (uint8_t)(_x)
 281
 282 /* Format string for printing an efx_byte_t */
 283 #define EFX_BYTE_FMT "0x%02x"
 284
 285 /* Format string for printing an efx_word_t */
 286 #define EFX_WORD_FMT "0x%04x"
 287
 288 /* Format string for printing an efx_dword_t */
 289 #define EFX_DWORD_FMT "0x%08x"
 290
 291 /* Format string for printing an efx_qword_t */
 292 #define EFX_QWORD_FMT "0x%08x:%08x"
 293
 294 /* Format string for printing an efx_oword_t */
 295 #define EFX_OWORD_FMT "0x%08x:%08x:%08x:%08x"
 296
 297 /* Parameters for printing an efx_byte_t */
 298 #define EFX_BYTE_VAL(_byte)                                     \
 299         ((unsigned int)__NATIVE_8((_byte).eb_u8[0]))
 300
 301 /* Parameters for printing an efx_word_t */
 302 #define EFX_WORD_VAL(_word)                                     \
 303         ((unsigned int)__LE_TO_CPU_16((_word).ew_u16[0]))
 304
 305 /* Parameters for printing an efx_dword_t */
 306 #define EFX_DWORD_VAL(_dword)                                   \
 307         ((unsigned int)__LE_TO_CPU_32((_dword).ed_u32[0]))
 308
 309 /* Parameters for printing an efx_qword_t */
 310 #define EFX_QWORD_VAL(_qword)                                   \
 311         ((unsigned int)__LE_TO_CPU_32((_qword).eq_u32[1])),     \
 312         ((unsigned int)__LE_TO_CPU_32((_qword).eq_u32[0]))
 313
 314 /* Parameters for printing an efx_oword_t */
 315 #define EFX_OWORD_VAL(_oword)                                   \
 316         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[3])),     \
 317         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[2])),     \
 318         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[1])),     \
 319         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[0]))
 320
 321 /*
 322  * Stop lint complaining about some shifts.
 323  */
 324 #ifdef  __lint
 325 extern int fix_lint;
 326 #define FIX_LINT(_x)    (_x + fix_lint)
 327 #else
 328 #define FIX_LINT(_x)    (_x)
 329 #endif
 330
 331 /*
 332  * Extract bit field portion [low,high) from the native-endian element
 333  * which contains bits [min,max).
 334  *
 335  * For example, suppose "element" represents the high 32 bits of a
 336  * 64-bit value, and we wish to extract the bits belonging to the bit
 337  * field occupying bits 28-45 of this 64-bit value.
 338  *
 339  * Then EFX_EXTRACT(_element, 32, 63, 28, 45) would give
 340  *
 341  *   (_element) << 4
 342  *
 343  * The result will contain the relevant bits filled in in the range
 344  * [0,high-low), with garbage in bits [high-low+1,...).
 345  */
 346 #define EFX_EXTRACT_NATIVE(_element, _min, _max, _low, _high)           \
 347         ((FIX_LINT(_low > _max) || FIX_LINT(_high < _min)) ?            \
 348                 0U :                                                    \
 349                 ((_low > _min) ?                                        \
 350                         ((_element) >> (_low - _min)) :                 \
 351                         ((_element) << (_min - _low))))
 352
 353 /*
 354  * Extract bit field portion [low,high) from the 64-bit little-endian
 355  * element which contains bits [min,max)
 356  */
 357 #define EFX_EXTRACT64(_element, _min, _max, _low, _high)                \
 358         EFX_EXTRACT_NATIVE(__LE_TO_CPU_64(_element), _min, _max, _low, _high)
 359
 360 /*
 361  * Extract bit field portion [low,high) from the 32-bit little-endian
 362  * element which contains bits [min,max)
 363  */
 364 #define EFX_EXTRACT32(_element, _min, _max, _low, _high)                \
 365         EFX_EXTRACT_NATIVE(__LE_TO_CPU_32(_element), _min, _max, _low, _high)
 366
 367 /*
 368  * Extract bit field portion [low,high) from the 16-bit little-endian
 369  * element which contains bits [min,max)
 370  */
 371 #define EFX_EXTRACT16(_element, _min, _max, _low, _high)                \
 372         EFX_EXTRACT_NATIVE(__LE_TO_CPU_16(_element), _min, _max, _low, _high)
 373
 374 /*
 375  * Extract bit field portion [low,high) from the 8-bit
 376  * element which contains bits [min,max)
 377  */
 378 #define EFX_EXTRACT8(_element, _min, _max, _low, _high)                 \
 379         EFX_EXTRACT_NATIVE(__NATIVE_8(_element), _min, _max, _low, _high)
 380
 381 #define EFX_EXTRACT_OWORD64(_oword, _low, _high)                        \
 382         (EFX_EXTRACT64((_oword).eo_u64[0], FIX_LINT(0), FIX_LINT(63),   \
 383             _low, _high) |                                              \
 384         EFX_EXTRACT64((_oword).eo_u64[1], FIX_LINT(64), FIX_LINT(127),  \
 385             _low, _high))
 386
 387 #define EFX_EXTRACT_OWORD32(_oword, _low, _high)                        \
 388         (EFX_EXTRACT32((_oword).eo_u32[0], FIX_LINT(0), FIX_LINT(31),   \
 389             _low, _high) |                                              \
 390         EFX_EXTRACT32((_oword).eo_u32[1], FIX_LINT(32), FIX_LINT(63),   \
 391             _low, _high) |                                              \
 392         EFX_EXTRACT32((_oword).eo_u32[2], FIX_LINT(64), FIX_LINT(95),   \
 393             _low, _high) |                                              \
 394         EFX_EXTRACT32((_oword).eo_u32[3], FIX_LINT(96), FIX_LINT(127),  \
 395             _low, _high))
 396
 397 #define EFX_EXTRACT_QWORD64(_qword, _low, _high)                        \
 398         (EFX_EXTRACT64((_qword).eq_u64[0], FIX_LINT(0), FIX_LINT(63),   \
 399             _low, _high))
 400
 401 #define EFX_EXTRACT_QWORD32(_qword, _low, _high)                        \
 402         (EFX_EXTRACT32((_qword).eq_u32[0], FIX_LINT(0), FIX_LINT(31),   \
 403             _low, _high) |                                              \
 404         EFX_EXTRACT32((_qword).eq_u32[1], FIX_LINT(32), FIX_LINT(63),   \
 405             _low, _high))
 406
 407 #define EFX_EXTRACT_DWORD(_dword, _low, _high)                          \
 408         (EFX_EXTRACT32((_dword).ed_u32[0], FIX_LINT(0), FIX_LINT(31),   \
 409             _low, _high))
 410
 411 #define EFX_EXTRACT_WORD(_word, _low, _high)                            \
 412         (EFX_EXTRACT16((_word).ew_u16[0], FIX_LINT(0), FIX_LINT(15),    \
 413             _low, _high))
 414
 415 #define EFX_EXTRACT_BYTE(_byte, _low, _high)                            \
 416         (EFX_EXTRACT8((_byte).eb_u8[0], FIX_LINT(0), FIX_LINT(7),       \
 417             _low, _high))
 418
 419
 420 #define EFX_OWORD_FIELD64(_oword, _field)                               \
 421         ((uint32_t)EFX_EXTRACT_OWORD64(_oword, EFX_LOW_BIT(_field),     \
 422             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 423
 424 #define EFX_OWORD_FIELD32(_oword, _field)                               \
 425         (EFX_EXTRACT_OWORD32(_oword, EFX_LOW_BIT(_field),               \
 426             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 427
 428 #define EFX_QWORD_FIELD64(_qword, _field)                               \
 429         ((uint32_t)EFX_EXTRACT_QWORD64(_qword, EFX_LOW_BIT(_field),     \
 430             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 431
 432 #define EFX_QWORD_FIELD32(_qword, _field)                               \
 433         (EFX_EXTRACT_QWORD32(_qword, EFX_LOW_BIT(_field),               \
 434             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 435
 436 #define EFX_DWORD_FIELD(_dword, _field)                                 \
 437         (EFX_EXTRACT_DWORD(_dword, EFX_LOW_BIT(_field),                 \
 438             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 439
 440 #define EFX_WORD_FIELD(_word, _field)                                   \
 441         (EFX_EXTRACT_WORD(_word, EFX_LOW_BIT(_field),                   \
 442             EFX_HIGH_BIT(_field)) & EFX_MASK16(_field))
 443
 444 #define EFX_BYTE_FIELD(_byte, _field)                                   \
 445         (EFX_EXTRACT_BYTE(_byte, EFX_LOW_BIT(_field),                   \
 446             EFX_HIGH_BIT(_field)) & EFX_MASK8(_field))
 447
 448
 449 #define EFX_OWORD_IS_EQUAL64(_oword_a, _oword_b)                        \
 450         ((_oword_a).eo_u64[0] == (_oword_b).eo_u64[0] &&                \
 451             (_oword_a).eo_u64[1] == (_oword_b).eo_u64[1])
 452
 453 #define EFX_OWORD_IS_EQUAL32(_oword_a, _oword_b)                        \
 454         ((_oword_a).eo_u32[0] == (_oword_b).eo_u32[0] &&                \
 455             (_oword_a).eo_u32[1] == (_oword_b).eo_u32[1] &&             \
 456             (_oword_a).eo_u32[2] == (_oword_b).eo_u32[2] &&             \
 457             (_oword_a).eo_u32[3] == (_oword_b).eo_u32[3])
 458
 459 #define EFX_QWORD_IS_EQUAL64(_qword_a, _qword_b)                        \
 460         ((_qword_a).eq_u64[0] == (_qword_b).eq_u64[0])
 461
 462 #define EFX_QWORD_IS_EQUAL32(_qword_a, _qword_b)                        \
 463         ((_qword_a).eq_u32[0] == (_qword_b).eq_u32[0] &&                \
 464             (_qword_a).eq_u32[1] == (_qword_b).eq_u32[1])
 465
 466 #define EFX_DWORD_IS_EQUAL(_dword_a, _dword_b)                          \
 467         ((_dword_a).ed_u32[0] == (_dword_b).ed_u32[0])
 468
 469 #define EFX_WORD_IS_EQUAL(_word_a, _word_b)                             \
 470         ((_word_a).ew_u16[0] == (_word_b).ew_u16[0])
 471
 472 #define EFX_BYTE_IS_EQUAL(_byte_a, _byte_b)                             \
 473         ((_byte_a).eb_u8[0] == (_byte_b).eb_u8[0])
 474
 475
 476 #define EFX_OWORD_IS_ZERO64(_oword)                                     \
 477         (((_oword).eo_u64[0] |                                          \
 478             (_oword).eo_u64[1]) == 0)
 479
 480 #define EFX_OWORD_IS_ZERO32(_oword)                                     \
 481         (((_oword).eo_u32[0] |                                          \
 482             (_oword).eo_u32[1] |                                        \
 483             (_oword).eo_u32[2] |                                        \
 484             (_oword).eo_u32[3]) == 0)
 485
 486 #define EFX_QWORD_IS_ZERO64(_qword)                                     \
 487         (((_qword).eq_u64[0]) == 0)
 488
 489 #define EFX_QWORD_IS_ZERO32(_qword)                                     \
 490         (((_qword).eq_u32[0] |                                          \
 491             (_qword).eq_u32[1]) == 0)
 492
 493 #define EFX_DWORD_IS_ZERO(_dword)                                       \
 494         (((_dword).ed_u32[0]) == 0)
 495
 496 #define EFX_WORD_IS_ZERO(_word)                                         \
 497         (((_word).ew_u16[0]) == 0)
 498
 499 #define EFX_BYTE_IS_ZERO(_byte)                                         \
 500         (((_byte).eb_u8[0]) == 0)
 501
 502
 503 #define EFX_OWORD_IS_SET64(_oword)                                      \
 504         (((_oword).eo_u64[0] &                                          \
 505             (_oword).eo_u64[1]) == ~((uint64_t)0))
 506
 507 #define EFX_OWORD_IS_SET32(_oword)                                      \
 508         (((_oword).eo_u32[0] &                                          \
 509             (_oword).eo_u32[1] &                                        \
 510             (_oword).eo_u32[2] &                                        \
 511             (_oword).eo_u32[3]) == ~((uint32_t)0))
 512
 513 #define EFX_QWORD_IS_SET64(_qword)                                      \
 514         (((_qword).eq_u64[0]) == ~((uint64_t)0))
 515
 516 #define EFX_QWORD_IS_SET32(_qword)                                      \
 517         (((_qword).eq_u32[0] &                                          \
 518             (_qword).eq_u32[1]) == ~((uint32_t)0))
 519
 520 #define EFX_DWORD_IS_SET(_dword)                                        \
 521         ((_dword).ed_u32[0] == ~((uint32_t)0))
 522
 523 #define EFX_WORD_IS_SET(_word)                                          \
 524         ((_word).ew_u16[0] == ~((uint16_t)0))
 525
 526 #define EFX_BYTE_IS_SET(_byte)                                          \
 527         ((_byte).eb_u8[0] == ~((uint8_t)0))
 528
 529 /*
 530  * Construct bit field portion
 531  *
 532  * Creates the portion of the bit field [low,high) that lies within
 533  * the range [min,max).
 534  */
 535
 536 #define EFX_INSERT_NATIVE64(_min, _max, _low, _high, _value)            \
 537         (((_low > _max) || (_high < _min)) ?                            \
 538                 0U :                                                    \
 539                 ((_low > _min) ?                                        \
 540                         (((uint64_t)(_value)) << (_low - _min)) :       \
 541                         (((uint64_t)(_value)) >> (_min - _low))))
 542
 543 #define EFX_INSERT_NATIVE32(_min, _max, _low, _high, _value)            \
 544         (((_low > _max) || (_high < _min)) ?                            \
 545                 0U :                                                    \
 546                 ((_low > _min) ?                                        \
 547                         (((uint32_t)(_value)) << (_low - _min)) :       \
 548                         (((uint32_t)(_value)) >> (_min - _low))))
 549
 550 #define EFX_INSERT_NATIVE16(_min, _max, _low, _high, _value)            \
 551         (((_low > _max) || (_high < _min)) ?                            \
 552                 0U :                                                    \
 553                 (uint16_t)((_low > _min) ?                              \
 554                                 ((_value) << (_low - _min)) :           \
 555                                 ((_value) >> (_min - _low))))
 556
 557 #define EFX_INSERT_NATIVE8(_min, _max, _low, _high, _value)             \
 558         (((_low > _max) || (_high < _min)) ?                            \
 559                 0U :                                                    \
 560                 (uint8_t)((_low > _min) ?                               \
 561                                 ((_value) << (_low - _min)) :   \
 562                                 ((_value) >> (_min - _low))))
 563
 564 /*
 565  * Construct bit field portion
 566  *
 567  * Creates the portion of the named bit field that lies within the
 568  * range [min,max).
 569  */
 570 #define EFX_INSERT_FIELD_NATIVE64(_min, _max, _field, _value)           \
 571         EFX_INSERT_NATIVE64(_min, _max, EFX_LOW_BIT(_field),            \
 572             EFX_HIGH_BIT(_field), _value)
 573
 574 #define EFX_INSERT_FIELD_NATIVE32(_min, _max, _field, _value)           \
 575         EFX_INSERT_NATIVE32(_min, _max, EFX_LOW_BIT(_field),            \
 576             EFX_HIGH_BIT(_field), _value)
 577
 578 #define EFX_INSERT_FIELD_NATIVE16(_min, _max, _field, _value)           \
 579         EFX_INSERT_NATIVE16(_min, _max, EFX_LOW_BIT(_field),            \
 580             EFX_HIGH_BIT(_field), _value)
 581
 582 #define EFX_INSERT_FIELD_NATIVE8(_min, _max, _field, _value)            \
 583         EFX_INSERT_NATIVE8(_min, _max, EFX_LOW_BIT(_field),             \
 584             EFX_HIGH_BIT(_field), _value)
 585
 586 /*
 587  * Construct bit field
 588  *
 589  * Creates the portion of the named bit fields that lie within the
 590  * range [min,max).
 591  */
 592 #define EFX_INSERT_FIELDS64(_min, _max,                                 \
 593             _field1, _value1, _field2, _value2, _field3, _value3,       \
 594             _field4, _value4, _field5, _value5, _field6, _value6,       \
 595             _field7, _value7, _field8, _value8, _field9, _value9,       \
 596             _field10, _value10)                                         \
 597         __CPU_TO_LE_64(                                                 \
 598             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field1, _value1) |   \
 599             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field2, _value2) |   \
 600             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field3, _value3) |   \
 601             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field4, _value4) |   \
 602             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field5, _value5) |   \
 603             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field6, _value6) |   \
 604             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field7, _value7) |   \
 605             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field8, _value8) |   \
 606             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field9, _value9) |   \
 607             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field10, _value10))
 608
 609 #define EFX_INSERT_FIELDS32(_min, _max,                                 \
 610             _field1, _value1, _field2, _value2, _field3, _value3,       \
 611             _field4, _value4, _field5, _value5, _field6, _value6,       \
 612             _field7, _value7, _field8, _value8, _field9, _value9,       \
 613             _field10, _value10)                                         \
 614         __CPU_TO_LE_32(                                                 \
 615             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field1, _value1) |   \
 616             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field2, _value2) |   \
 617             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field3, _value3) |   \
 618             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field4, _value4) |   \
 619             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field5, _value5) |   \
 620             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field6, _value6) |   \
 621             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field7, _value7) |   \
 622             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field8, _value8) |   \
 623             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field9, _value9) |   \
 624             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field10, _value10))
 625
 626 #define EFX_INSERT_FIELDS16(_min, _max,                                 \
 627             _field1, _value1, _field2, _value2, _field3, _value3,       \
 628             _field4, _value4, _field5, _value5, _field6, _value6,       \
 629             _field7, _value7, _field8, _value8, _field9, _value9,       \
 630             _field10, _value10)                                         \
 631         __CPU_TO_LE_16(                                                 \
 632             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field1, _value1) |   \
 633             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field2, _value2) |   \
 634             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field3, _value3) |   \
 635             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field4, _value4) |   \
 636             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field5, _value5) |   \
 637             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field6, _value6) |   \
 638             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field7, _value7) |   \
 639             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field8, _value8) |   \
 640             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field9, _value9) |   \
 641             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field10, _value10))
 642
 643 #define EFX_INSERT_FIELDS8(_min, _max,                                  \
 644             _field1, _value1, _field2, _value2, _field3, _value3,       \
 645             _field4, _value4, _field5, _value5, _field6, _value6,       \
 646             _field7, _value7, _field8, _value8, _field9, _value9,       \
 647             _field10, _value10)                                         \
 648         __NATIVE_8(                                                     \
 649             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field1, _value1) |    \
 650             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field2, _value2) |    \
 651             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field3, _value3) |    \
 652             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field4, _value4) |    \
 653             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field5, _value5) |    \
 654             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field6, _value6) |    \
 655             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field7, _value7) |    \
 656             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field8, _value8) |    \
 657             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field9, _value9) |    \
 658             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field10, _value10))
 659
 660 #define EFX_POPULATE_OWORD64(_oword,                                    \
 661             _field1, _value1, _field2, _value2, _field3, _value3,       \
 662             _field4, _value4, _field5, _value5, _field6, _value6,       \
 663             _field7, _value7, _field8, _value8, _field9, _value9,       \
 664             _field10, _value10)                                         \
 665         do {                                                            \
 666                 _NOTE(CONSTANTCONDITION)                                \
 667                 (_oword).eo_u64[0] = EFX_INSERT_FIELDS64(0, 63,         \
 668                     _field1, _value1, _field2, _value2,                 \
 669                     _field3, _value3, _field4, _value4,                 \
 670                     _field5, _value5, _field6, _value6,                 \
 671                     _field7, _value7, _field8, _value8,                 \
 672                     _field9, _value9, _field10, _value10);              \
 673                 _NOTE(CONSTANTCONDITION)                                \
 674                 (_oword).eo_u64[1] = EFX_INSERT_FIELDS64(64, 127,       \
 675                     _field1, _value1, _field2, _value2,                 \
 676                     _field3, _value3, _field4, _value4,                 \
 677                     _field5, _value5, _field6, _value6,                 \
 678                     _field7, _value7, _field8, _value8,                 \
 679                     _field9, _value9, _field10, _value10);              \
 680         _NOTE(CONSTANTCONDITION)                                        \
 681         } while (B_FALSE)
 682
 683 #define EFX_POPULATE_OWORD32(_oword,                                    \
 684             _field1, _value1, _field2, _value2, _field3, _value3,       \
 685             _field4, _value4, _field5, _value5, _field6, _value6,       \
 686             _field7, _value7, _field8, _value8, _field9, _value9,       \
 687             _field10, _value10)                                         \
 688         do {                                                            \
 689                 _NOTE(CONSTANTCONDITION)                                \
 690                 (_oword).eo_u32[0] = EFX_INSERT_FIELDS32(0, 31,         \
 691                     _field1, _value1, _field2, _value2,                 \
 692                     _field3, _value3, _field4, _value4,                 \
 693                     _field5, _value5, _field6, _value6,                 \
 694                     _field7, _value7, _field8, _value8,                 \
 695                     _field9, _value9, _field10, _value10);              \
 696                 _NOTE(CONSTANTCONDITION)                                \
 697                 (_oword).eo_u32[1] = EFX_INSERT_FIELDS32(32, 63,        \
 698                     _field1, _value1, _field2, _value2,                 \
 699                     _field3, _value3, _field4, _value4,                 \
 700                     _field5, _value5, _field6, _value6,                 \
 701                     _field7, _value7, _field8, _value8,                 \
 702                     _field9, _value9, _field10, _value10);              \
 703                 _NOTE(CONSTANTCONDITION)                                \
 704                 (_oword).eo_u32[2] = EFX_INSERT_FIELDS32(64, 95,        \
 705                     _field1, _value1, _field2, _value2,                 \
 706                     _field3, _value3, _field4, _value4,                 \
 707                     _field5, _value5, _field6, _value6,                 \
 708                     _field7, _value7, _field8, _value8,                 \
 709                     _field9, _value9, _field10, _value10);              \
 710                 _NOTE(CONSTANTCONDITION)                                \
 711                 (_oword).eo_u32[3] = EFX_INSERT_FIELDS32(96, 127,       \
 712                     _field1, _value1, _field2, _value2,                 \
 713                     _field3, _value3, _field4, _value4,                 \
 714                     _field5, _value5, _field6, _value6,                 \
 715                     _field7, _value7, _field8, _value8,                 \
 716                     _field9, _value9, _field10, _value10);              \
 717         _NOTE(CONSTANTCONDITION)                                        \
 718         } while (B_FALSE)
 719
 720 #define EFX_POPULATE_QWORD64(_qword,                                    \
 721             _field1, _value1, _field2, _value2, _field3, _value3,       \
 722             _field4, _value4, _field5, _value5, _field6, _value6,       \
 723             _field7, _value7, _field8, _value8, _field9, _value9,       \
 724             _field10, _value10)                                         \
 725         do {                                                            \
 726                 _NOTE(CONSTANTCONDITION)                                \
 727                 (_qword).eq_u64[0] = EFX_INSERT_FIELDS64(0, 63,         \
 728                     _field1, _value1, _field2, _value2,                 \
 729                     _field3, _value3, _field4, _value4,                 \
 730                     _field5, _value5, _field6, _value6,                 \
 731                     _field7, _value7, _field8, _value8,                 \
 732                     _field9, _value9, _field10, _value10);              \
 733         _NOTE(CONSTANTCONDITION)                                        \
 734         } while (B_FALSE)
 735
 736 #define EFX_POPULATE_QWORD32(_qword,                                    \
 737             _field1, _value1, _field2, _value2, _field3, _value3,       \
 738             _field4, _value4, _field5, _value5, _field6, _value6,       \
 739             _field7, _value7, _field8, _value8, _field9, _value9,       \
 740             _field10, _value10)                                         \
 741         do {                                                            \
 742                 _NOTE(CONSTANTCONDITION)                                \
 743                 (_qword).eq_u32[0] = EFX_INSERT_FIELDS32(0, 31,         \
 744                     _field1, _value1, _field2, _value2,                 \
 745                     _field3, _value3, _field4, _value4,                 \
 746                     _field5, _value5, _field6, _value6,                 \
 747                     _field7, _value7, _field8, _value8,                 \
 748                     _field9, _value9, _field10, _value10);              \
 749                 _NOTE(CONSTANTCONDITION)                                \
 750                 (_qword).eq_u32[1] = EFX_INSERT_FIELDS32(32, 63,        \
 751                     _field1, _value1, _field2, _value2,                 \
 752                     _field3, _value3, _field4, _value4,                 \
 753                     _field5, _value5, _field6, _value6,                 \
 754                     _field7, _value7, _field8, _value8,                 \
 755                     _field9, _value9, _field10, _value10);              \
 756         _NOTE(CONSTANTCONDITION)                                        \
 757         } while (B_FALSE)
 758
 759 #define EFX_POPULATE_DWORD(_dword,                                      \
 760             _field1, _value1, _field2, _value2, _field3, _value3,       \
 761             _field4, _value4, _field5, _value5, _field6, _value6,       \
 762             _field7, _value7, _field8, _value8, _field9, _value9,       \
 763             _field10, _value10)                                         \
 764         do {                                                            \
 765                 _NOTE(CONSTANTCONDITION)                                \
 766                 (_dword).ed_u32[0] = EFX_INSERT_FIELDS32(0, 31,         \
 767                     _field1, _value1, _field2, _value2,                 \
 768                     _field3, _value3, _field4, _value4,                 \
 769                     _field5, _value5, _field6, _value6,                 \
 770                     _field7, _value7, _field8, _value8,                 \
 771                     _field9, _value9, _field10, _value10);              \
 772         _NOTE(CONSTANTCONDITION)                                        \
 773         } while (B_FALSE)
 774
 775 #define EFX_POPULATE_WORD(_word,                                        \
 776             _field1, _value1, _field2, _value2, _field3, _value3,       \
 777             _field4, _value4, _field5, _value5, _field6, _value6,       \
 778             _field7, _value7, _field8, _value8, _field9, _value9,       \
 779             _field10, _value10)                                         \
 780         do {                                                            \
 781                 _NOTE(CONSTANTCONDITION)                                \
 782                 (_word).ew_u16[0] = EFX_INSERT_FIELDS16(0, 15,          \
 783                     _field1, _value1, _field2, _value2,                 \
 784                     _field3, _value3, _field4, _value4,                 \
 785                     _field5, _value5, _field6, _value6,                 \
 786                     _field7, _value7, _field8, _value8,                 \
 787                     _field9, _value9, _field10, _value10);              \
 788         _NOTE(CONSTANTCONDITION)                                        \
 789         } while (B_FALSE)
 790
 791 #define EFX_POPULATE_BYTE(_byte,                                        \
 792             _field1, _value1, _field2, _value2, _field3, _value3,       \
 793             _field4, _value4, _field5, _value5, _field6, _value6,       \
 794             _field7, _value7, _field8, _value8, _field9, _value9,       \
 795             _field10, _value10)                                         \
 796         do {                                                            \
 797                 _NOTE(CONSTANTCONDITION)                                \
 798                 (_byte).eb_u8[0] = EFX_INSERT_FIELDS8(0, 7,             \
 799                     _field1, _value1, _field2, _value2,                 \
 800                     _field3, _value3, _field4, _value4,                 \
 801                     _field5, _value5, _field6, _value6,                 \
 802                     _field7, _value7, _field8, _value8,                 \
 803                     _field9, _value9, _field10, _value10);              \
 804         _NOTE(CONSTANTCONDITION)                                        \
 805         } while (B_FALSE)
 806
 807 /* Populate an octword field with various numbers of arguments */
 808 #define EFX_POPULATE_OWORD_10 EFX_POPULATE_OWORD
 809
 810 #define EFX_POPULATE_OWORD_9(_oword,                                    \
 811             _field1, _value1, _field2, _value2, _field3, _value3,       \
 812             _field4, _value4, _field5, _value5, _field6, _value6,       \
 813             _field7, _value7, _field8, _value8, _field9, _value9)       \
 814         EFX_POPULATE_OWORD_10(_oword, EFX_DUMMY_FIELD, 0,               \
 815             _field1, _value1, _field2, _value2, _field3, _value3,       \
 816             _field4, _value4, _field5, _value5, _field6, _value6,       \
 817             _field7, _value7, _field8, _value8, _field9, _value9)
 818
 819 #define EFX_POPULATE_OWORD_8(_oword,                                    \
 820             _field1, _value1, _field2, _value2, _field3, _value3,       \
 821             _field4, _value4, _field5, _value5, _field6, _value6,       \
 822             _field7, _value7, _field8, _value8)                         \
 823         EFX_POPULATE_OWORD_9(_oword, EFX_DUMMY_FIELD, 0,                \
 824             _field1, _value1, _field2, _value2, _field3, _value3,       \
 825             _field4, _value4, _field5, _value5, _field6, _value6,       \
 826             _field7, _value7, _field8, _value8)
 827
 828 #define EFX_POPULATE_OWORD_7(_oword,                                    \
 829             _field1, _value1, _field2, _value2, _field3, _value3,       \
 830             _field4, _value4, _field5, _value5, _field6, _value6,       \
 831             _field7, _value7)                                           \
 832         EFX_POPULATE_OWORD_8(_oword, EFX_DUMMY_FIELD, 0,                \
 833             _field1, _value1, _field2, _value2, _field3, _value3,       \
 834             _field4, _value4, _field5, _value5, _field6, _value6,       \
 835             _field7, _value7)
 836
 837 #define EFX_POPULATE_OWORD_6(_oword,                                    \
 838             _field1, _value1, _field2, _value2, _field3, _value3,       \
 839             _field4, _value4, _field5, _value5, _field6, _value6)       \
 840         EFX_POPULATE_OWORD_7(_oword, EFX_DUMMY_FIELD, 0,                \
 841             _field1, _value1, _field2, _value2, _field3, _value3,       \
 842             _field4, _value4, _field5, _value5, _field6, _value6)
 843
 844 #define EFX_POPULATE_OWORD_5(_oword,                                    \
 845             _field1, _value1, _field2, _value2, _field3, _value3,       \
 846             _field4, _value4, _field5, _value5)                         \
 847         EFX_POPULATE_OWORD_6(_oword, EFX_DUMMY_FIELD, 0,                \
 848             _field1, _value1, _field2, _value2, _field3, _value3,       \
 849             _field4, _value4, _field5, _value5)
 850
 851 #define EFX_POPULATE_OWORD_4(_oword,                                    \
 852             _field1, _value1, _field2, _value2, _field3, _value3,       \
 853             _field4, _value4)                                           \
 854         EFX_POPULATE_OWORD_5(_oword, EFX_DUMMY_FIELD, 0,                \
 855             _field1, _value1, _field2, _value2, _field3, _value3,       \
 856             _field4, _value4)
 857
 858 #define EFX_POPULATE_OWORD_3(_oword,                                    \
 859             _field1, _value1, _field2, _value2, _field3, _value3)       \
 860         EFX_POPULATE_OWORD_4(_oword, EFX_DUMMY_FIELD, 0,                \
 861             _field1, _value1, _field2, _value2, _field3, _value3)
 862
 863 #define EFX_POPULATE_OWORD_2(_oword,                                    \
 864             _field1, _value1, _field2, _value2)                         \
 865         EFX_POPULATE_OWORD_3(_oword, EFX_DUMMY_FIELD, 0,                \
 866             _field1, _value1, _field2, _value2)
 867
 868 #define EFX_POPULATE_OWORD_1(_oword,                                    \
 869             _field1, _value1)                                           \
 870         EFX_POPULATE_OWORD_2(_oword, EFX_DUMMY_FIELD, 0,                \
 871             _field1, _value1)
 872
 873 #define EFX_ZERO_OWORD(_oword)                                          \
 874         EFX_POPULATE_OWORD_1(_oword, EFX_DUMMY_FIELD, 0)
 875
 876 #define EFX_SET_OWORD(_oword)                                           \
 877         EFX_POPULATE_OWORD_4(_oword,                                    \
 878             EFX_DWORD_0, 0xffffffff, EFX_DWORD_1, 0xffffffff,           \
 879             EFX_DWORD_2, 0xffffffff, EFX_DWORD_3, 0xffffffff)
 880
 881 /* Populate a quadword field with various numbers of arguments */
 882 #define EFX_POPULATE_QWORD_10 EFX_POPULATE_QWORD
 883
 884 #define EFX_POPULATE_QWORD_9(_qword,                                    \
 885             _field1, _value1, _field2, _value2, _field3, _value3,       \
 886             _field4, _value4, _field5, _value5, _field6, _value6,       \
 887             _field7, _value7, _field8, _value8, _field9, _value9)       \
 888         EFX_POPULATE_QWORD_10(_qword, EFX_DUMMY_FIELD, 0,               \
 889             _field1, _value1, _field2, _value2, _field3, _value3,       \
 890             _field4, _value4, _field5, _value5, _field6, _value6,       \
 891             _field7, _value7, _field8, _value8, _field9, _value9)
 892
 893 #define EFX_POPULATE_QWORD_8(_qword,                                    \
 894             _field1, _value1, _field2, _value2, _field3, _value3,       \
 895             _field4, _value4, _field5, _value5, _field6, _value6,       \
 896             _field7, _value7, _field8, _value8)                         \
 897         EFX_POPULATE_QWORD_9(_qword, EFX_DUMMY_FIELD, 0,                \
 898             _field1, _value1, _field2, _value2, _field3, _value3,       \
 899             _field4, _value4, _field5, _value5, _field6, _value6,       \
 900             _field7, _value7, _field8, _value8)
 901
 902 #define EFX_POPULATE_QWORD_7(_qword,                                    \
 903             _field1, _value1, _field2, _value2, _field3, _value3,       \
 904             _field4, _value4, _field5, _value5, _field6, _value6,       \
 905             _field7, _value7)                                           \
 906         EFX_POPULATE_QWORD_8(_qword, EFX_DUMMY_FIELD, 0,                \
 907             _field1, _value1, _field2, _value2, _field3, _value3,       \
 908             _field4, _value4, _field5, _value5, _field6, _value6,       \
 909             _field7, _value7)
 910
 911 #define EFX_POPULATE_QWORD_6(_qword,                                    \
 912             _field1, _value1, _field2, _value2, _field3, _value3,       \
 913             _field4, _value4, _field5, _value5, _field6, _value6)       \
 914         EFX_POPULATE_QWORD_7(_qword, EFX_DUMMY_FIELD, 0,                \
 915             _field1, _value1, _field2, _value2, _field3, _value3,       \
 916             _field4, _value4, _field5, _value5, _field6, _value6)
 917
 918 #define EFX_POPULATE_QWORD_5(_qword,                                    \
 919             _field1, _value1, _field2, _value2, _field3, _value3,       \
 920             _field4, _value4, _field5, _value5)                         \
 921         EFX_POPULATE_QWORD_6(_qword, EFX_DUMMY_FIELD, 0,                \
 922             _field1, _value1, _field2, _value2, _field3, _value3,       \
 923             _field4, _value4, _field5, _value5)
 924
 925 #define EFX_POPULATE_QWORD_4(_qword,                                    \
 926             _field1, _value1, _field2, _value2, _field3, _value3,       \
 927             _field4, _value4)                                           \
 928         EFX_POPULATE_QWORD_5(_qword, EFX_DUMMY_FIELD, 0,                \
 929             _field1, _value1, _field2, _value2, _field3, _value3,       \
 930             _field4, _value4)
 931
 932 #define EFX_POPULATE_QWORD_3(_qword,                                    \
 933             _field1, _value1, _field2, _value2, _field3, _value3)       \
 934         EFX_POPULATE_QWORD_4(_qword, EFX_DUMMY_FIELD, 0,                \
 935             _field1, _value1, _field2, _value2, _field3, _value3)
 936
 937 #define EFX_POPULATE_QWORD_2(_qword,                                    \
 938             _field1, _value1, _field2, _value2)                         \
 939         EFX_POPULATE_QWORD_3(_qword, EFX_DUMMY_FIELD, 0,                \
 940             _field1, _value1, _field2, _value2)
 941
 942 #define EFX_POPULATE_QWORD_1(_qword,                                    \
 943             _field1, _value1)                                           \
 944         EFX_POPULATE_QWORD_2(_qword, EFX_DUMMY_FIELD, 0,                \
 945             _field1, _value1)
 946
 947 #define EFX_ZERO_QWORD(_qword)                                          \
 948         EFX_POPULATE_QWORD_1(_qword, EFX_DUMMY_FIELD, 0)
 949
 950 #define EFX_SET_QWORD(_qword)                                           \
 951         EFX_POPULATE_QWORD_2(_qword,                                    \
 952             EFX_DWORD_0, 0xffffffff, EFX_DWORD_1, 0xffffffff)
 953
 954 /* Populate a dword field with various numbers of arguments */
 955 #define EFX_POPULATE_DWORD_10 EFX_POPULATE_DWORD
 956
 957 #define EFX_POPULATE_DWORD_9(_dword,                                    \
 958             _field1, _value1, _field2, _value2, _field3, _value3,       \
 959             _field4, _value4, _field5, _value5, _field6, _value6,       \
 960             _field7, _value7, _field8, _value8, _field9, _value9)       \
 961         EFX_POPULATE_DWORD_10(_dword, EFX_DUMMY_FIELD, 0,               \
 962             _field1, _value1, _field2, _value2, _field3, _value3,       \
 963             _field4, _value4, _field5, _value5, _field6, _value6,       \
 964             _field7, _value7, _field8, _value8, _field9, _value9)
 965
 966 #define EFX_POPULATE_DWORD_8(_dword,                                    \
 967             _field1, _value1, _field2, _value2, _field3, _value3,       \
 968             _field4, _value4, _field5, _value5, _field6, _value6,       \
 969             _field7, _value7, _field8, _value8)                         \
 970         EFX_POPULATE_DWORD_9(_dword, EFX_DUMMY_FIELD, 0,                \
 971             _field1, _value1, _field2, _value2, _field3, _value3,       \
 972             _field4, _value4, _field5, _value5, _field6, _value6,       \
 973             _field7, _value7, _field8, _value8)
 974
 975 #define EFX_POPULATE_DWORD_7(_dword,                                    \
 976             _field1, _value1, _field2, _value2, _field3, _value3,       \
 977             _field4, _value4, _field5, _value5, _field6, _value6,       \
 978             _field7, _value7)                                           \
 979         EFX_POPULATE_DWORD_8(_dword, EFX_DUMMY_FIELD, 0,                \
 980             _field1, _value1, _field2, _value2, _field3, _value3,       \
 981             _field4, _value4, _field5, _value5, _field6, _value6,       \
 982             _field7, _value7)
 983
 984 #define EFX_POPULATE_DWORD_6(_dword,                                    \
 985             _field1, _value1, _field2, _value2, _field3, _value3,       \
 986             _field4, _value4, _field5, _value5, _field6, _value6)       \
 987         EFX_POPULATE_DWORD_7(_dword, EFX_DUMMY_FIELD, 0,                \
 988             _field1, _value1, _field2, _value2, _field3, _value3,       \
 989             _field4, _value4, _field5, _value5, _field6, _value6)
 990
 991 #define EFX_POPULATE_DWORD_5(_dword,                                    \
 992             _field1, _value1, _field2, _value2, _field3, _value3,       \
 993             _field4, _value4, _field5, _value5)                         \
 994         EFX_POPULATE_DWORD_6(_dword, EFX_DUMMY_FIELD, 0,                \
 995             _field1, _value1, _field2, _value2, _field3, _value3,       \
 996             _field4, _value4, _field5, _value5)
 997
 998 #define EFX_POPULATE_DWORD_4(_dword,                                    \
 999             _field1, _value1, _field2, _value2, _field3, _value3,       \
1000             _field4, _value4)                                           \
1001         EFX_POPULATE_DWORD_5(_dword, EFX_DUMMY_FIELD, 0,                \
1002             _field1, _value1, _field2, _value2, _field3, _value3,       \
1003             _field4, _value4)
1004
1005 #define EFX_POPULATE_DWORD_3(_dword,                                    \
1006             _field1, _value1, _field2, _value2, _field3, _value3)       \
1007         EFX_POPULATE_DWORD_4(_dword, EFX_DUMMY_FIELD, 0,                \
1008             _field1, _value1, _field2, _value2, _field3, _value3)
1009
1010 #define EFX_POPULATE_DWORD_2(_dword,                                    \
1011             _field1, _value1, _field2, _value2)                         \
1012         EFX_POPULATE_DWORD_3(_dword, EFX_DUMMY_FIELD, 0,                \
1013             _field1, _value1, _field2, _value2)
1014
1015 #define EFX_POPULATE_DWORD_1(_dword,                                    \
1016             _field1, _value1)                                           \
1017         EFX_POPULATE_DWORD_2(_dword, EFX_DUMMY_FIELD, 0,                \
1018             _field1, _value1)
1019
1020 #define EFX_ZERO_DWORD(_dword)                                          \
1021         EFX_POPULATE_DWORD_1(_dword, EFX_DUMMY_FIELD, 0)
1022
1023 #define EFX_SET_DWORD(_dword)                                           \
1024         EFX_POPULATE_DWORD_1(_dword,                                    \
1025             EFX_DWORD_0, 0xffffffff)
1026
1027 /* Populate a word field with various numbers of arguments */
1028 #define EFX_POPULATE_WORD_10 EFX_POPULATE_WORD
1029
1030 #define EFX_POPULATE_WORD_9(_word,                                      \
1031             _field1, _value1, _field2, _value2, _field3, _value3,       \
1032             _field4, _value4, _field5, _value5, _field6, _value6,       \
1033             _field7, _value7, _field8, _value8, _field9, _value9)       \
1034         EFX_POPULATE_WORD_10(_word, EFX_DUMMY_FIELD, 0,                 \
1035             _field1, _value1, _field2, _value2, _field3, _value3,       \
1036             _field4, _value4, _field5, _value5, _field6, _value6,       \
1037             _field7, _value7, _field8, _value8, _field9, _value9)
1038
1039 #define EFX_POPULATE_WORD_8(_word,                                      \
1040             _field1, _value1, _field2, _value2, _field3, _value3,       \
1041             _field4, _value4, _field5, _value5, _field6, _value6,       \
1042             _field7, _value7, _field8, _value8)                         \
1043         EFX_POPULATE_WORD_9(_word, EFX_DUMMY_FIELD, 0,                  \
1044             _field1, _value1, _field2, _value2, _field3, _value3,       \
1045             _field4, _value4, _field5, _value5, _field6, _value6,       \
1046             _field7, _value7, _field8, _value8)
1047
1048 #define EFX_POPULATE_WORD_7(_word,                                      \
1049             _field1, _value1, _field2, _value2, _field3, _value3,       \
1050             _field4, _value4, _field5, _value5, _field6, _value6,       \
1051             _field7, _value7)                                           \
1052         EFX_POPULATE_WORD_8(_word, EFX_DUMMY_FIELD, 0,                  \
1053             _field1, _value1, _field2, _value2, _field3, _value3,       \
1054             _field4, _value4, _field5, _value5, _field6, _value6,       \
1055             _field7, _value7)
1056
1057 #define EFX_POPULATE_WORD_6(_word,                                      \
1058             _field1, _value1, _field2, _value2, _field3, _value3,       \
1059             _field4, _value4, _field5, _value5, _field6, _value6)       \
1060         EFX_POPULATE_WORD_7(_word, EFX_DUMMY_FIELD, 0,                  \
1061             _field1, _value1, _field2, _value2, _field3, _value3,       \
1062             _field4, _value4, _field5, _value5, _field6, _value6)
1063
1064 #define EFX_POPULATE_WORD_5(_word,                                      \
1065             _field1, _value1, _field2, _value2, _field3, _value3,       \
1066             _field4, _value4, _field5, _value5)                         \
1067         EFX_POPULATE_WORD_6(_word, EFX_DUMMY_FIELD, 0,                  \
1068             _field1, _value1, _field2, _value2, _field3, _value3,       \
1069             _field4, _value4, _field5, _value5)
1070
1071 #define EFX_POPULATE_WORD_4(_word,                                      \
1072             _field1, _value1, _field2, _value2, _field3, _value3,       \
1073             _field4, _value4)                                           \
1074         EFX_POPULATE_WORD_5(_word, EFX_DUMMY_FIELD, 0,                  \
1075             _field1, _value1, _field2, _value2, _field3, _value3,       \
1076             _field4, _value4)
1077
1078 #define EFX_POPULATE_WORD_3(_word,                                      \
1079             _field1, _value1, _field2, _value2, _field3, _value3)       \
1080         EFX_POPULATE_WORD_4(_word, EFX_DUMMY_FIELD, 0,                  \
1081             _field1, _value1, _field2, _value2, _field3, _value3)
1082
1083 #define EFX_POPULATE_WORD_2(_word,                                      \
1084             _field1, _value1, _field2, _value2)                         \
1085         EFX_POPULATE_WORD_3(_word, EFX_DUMMY_FIELD, 0,                  \
1086             _field1, _value1, _field2, _value2)
1087
1088 #define EFX_POPULATE_WORD_1(_word,                                      \
1089             _field1, _value1)                                           \
1090         EFX_POPULATE_WORD_2(_word, EFX_DUMMY_FIELD, 0,                  \
1091             _field1, _value1)
1092
1093 #define EFX_ZERO_WORD(_word)                                            \
1094         EFX_POPULATE_WORD_1(_word, EFX_DUMMY_FIELD, 0)
1095
1096 #define EFX_SET_WORD(_word)                                             \
1097         EFX_POPULATE_WORD_1(_word,                                      \
1098             EFX_WORD_0, 0xffff)
1099
1100 /* Populate a byte field with various numbers of arguments */
1101 #define EFX_POPULATE_BYTE_10 EFX_POPULATE_BYTE
1102
1103 #define EFX_POPULATE_BYTE_9(_byte,                                      \
1104             _field1, _value1, _field2, _value2, _field3, _value3,       \
1105             _field4, _value4, _field5, _value5, _field6, _value6,       \
1106             _field7, _value7, _field8, _value8, _field9, _value9)       \
1107         EFX_POPULATE_BYTE_10(_byte, EFX_DUMMY_FIELD, 0,                 \
1108             _field1, _value1, _field2, _value2, _field3, _value3,       \
1109             _field4, _value4, _field5, _value5, _field6, _value6,       \
1110             _field7, _value7, _field8, _value8, _field9, _value9)
1111
1112 #define EFX_POPULATE_BYTE_8(_byte,                                      \
1113             _field1, _value1, _field2, _value2, _field3, _value3,       \
1114             _field4, _value4, _field5, _value5, _field6, _value6,       \
1115             _field7, _value7, _field8, _value8)                         \
1116         EFX_POPULATE_BYTE_9(_byte, EFX_DUMMY_FIELD, 0,                  \
1117             _field1, _value1, _field2, _value2, _field3, _value3,       \
1118             _field4, _value4, _field5, _value5, _field6, _value6,       \
1119             _field7, _value7, _field8, _value8)
1120
1121 #define EFX_POPULATE_BYTE_7(_byte,                                      \
1122             _field1, _value1, _field2, _value2, _field3, _value3,       \
1123             _field4, _value4, _field5, _value5, _field6, _value6,       \
1124             _field7, _value7)                                           \
1125         EFX_POPULATE_BYTE_8(_byte, EFX_DUMMY_FIELD, 0,                  \
1126             _field1, _value1, _field2, _value2, _field3, _value3,       \
1127             _field4, _value4, _field5, _value5, _field6, _value6,       \
1128             _field7, _value7)
1129
1130 #define EFX_POPULATE_BYTE_6(_byte,                                      \
1131             _field1, _value1, _field2, _value2, _field3, _value3,       \
1132             _field4, _value4, _field5, _value5, _field6, _value6)       \
1133         EFX_POPULATE_BYTE_7(_byte, EFX_DUMMY_FIELD, 0,                  \
1134             _field1, _value1, _field2, _value2, _field3, _value3,       \
1135             _field4, _value4, _field5, _value5, _field6, _value6)
1136
1137 #define EFX_POPULATE_BYTE_5(_byte,                                      \
1138             _field1, _value1, _field2, _value2, _field3, _value3,       \
1139             _field4, _value4, _field5, _value5)                         \
1140         EFX_POPULATE_BYTE_6(_byte, EFX_DUMMY_FIELD, 0,                  \
1141             _field1, _value1, _field2, _value2, _field3, _value3,       \
1142             _field4, _value4, _field5, _value5)
1143
1144 #define EFX_POPULATE_BYTE_4(_byte,                                      \
1145             _field1, _value1, _field2, _value2, _field3, _value3,       \
1146             _field4, _value4)                                           \
1147         EFX_POPULATE_BYTE_5(_byte, EFX_DUMMY_FIELD, 0,                  \
1148             _field1, _value1, _field2, _value2, _field3, _value3,       \
1149             _field4, _value4)
1150
1151 #define EFX_POPULATE_BYTE_3(_byte,                                      \
1152             _field1, _value1, _field2, _value2, _field3, _value3)       \
1153         EFX_POPULATE_BYTE_4(_byte, EFX_DUMMY_FIELD, 0,                  \
1154             _field1, _value1, _field2, _value2, _field3, _value3)
1155
1156 #define EFX_POPULATE_BYTE_2(_byte,                                      \
1157             _field1, _value1, _field2, _value2)                         \
1158         EFX_POPULATE_BYTE_3(_byte, EFX_DUMMY_FIELD, 0,                  \
1159             _field1, _value1, _field2, _value2)
1160
1161 #define EFX_POPULATE_BYTE_1(_byte,                                      \
1162             _field1, _value1)                                           \
1163         EFX_POPULATE_BYTE_2(_byte, EFX_DUMMY_FIELD, 0,                  \
1164             _field1, _value1)
1165
1166 #define EFX_ZERO_BYTE(_byte)                                            \
1167         EFX_POPULATE_BYTE_1(_byte, EFX_DUMMY_FIELD, 0)
1168
1169 #define EFX_SET_BYTE(_byte)                                             \
1170         EFX_POPULATE_BYTE_1(_byte,                                      \
1171             EFX_BYTE_0, 0xff)
1172
1173 /*
1174  * Modify a named field within an already-populated structure.  Used
1175  * for read-modify-write operations.
1176  */
1177
1178 #define EFX_INSERT_FIELD64(_min, _max, _field, _value)                  \
1179         __CPU_TO_LE_64(EFX_INSERT_FIELD_NATIVE64(_min, _max, _field, _value))
1180
1181 #define EFX_INSERT_FIELD32(_min, _max, _field, _value)                  \
1182         __CPU_TO_LE_32(EFX_INSERT_FIELD_NATIVE32(_min, _max, _field, _value))
1183
1184 #define EFX_INSERT_FIELD16(_min, _max, _field, _value)                  \
1185         __CPU_TO_LE_16(EFX_INSERT_FIELD_NATIVE16(_min, _max, _field, _value))
1186
1187 #define EFX_INSERT_FIELD8(_min, _max, _field, _value)                   \
1188         __NATIVE_8(EFX_INSERT_FIELD_NATIVE8(_min, _max, _field, _value))
1189
1190 #define EFX_INPLACE_MASK64(_min, _max, _field)                          \
1191         EFX_INSERT_FIELD64(_min, _max, _field, EFX_MASK64(_field))
1192
1193 #define EFX_INPLACE_MASK32(_min, _max, _field)                          \
1194         EFX_INSERT_FIELD32(_min, _max, _field, EFX_MASK32(_field))
1195
1196 #define EFX_INPLACE_MASK16(_min, _max, _field)                          \
1197         EFX_INSERT_FIELD16(_min, _max, _field, EFX_MASK16(_field))
1198
1199 #define EFX_INPLACE_MASK8(_min, _max, _field)                           \
1200         EFX_INSERT_FIELD8(_min, _max, _field, EFX_MASK8(_field))
1201
1202 #define EFX_SET_OWORD_FIELD64(_oword, _field, _value)                   \
1203         do {                                                            \
1204                 _NOTE(CONSTANTCONDITION)                                \
1205                 (_oword).eo_u64[0] = (((_oword).eo_u64[0] &             \
1206                     ~EFX_INPLACE_MASK64(0, 63, _field)) |               \
1207                     EFX_INSERT_FIELD64(0, 63, _field, _value));         \
1208                 _NOTE(CONSTANTCONDITION)                                \
1209                 (_oword).eo_u64[1] = (((_oword).eo_u64[1] &             \
1210                     ~EFX_INPLACE_MASK64(64, 127, _field)) |             \
1211                     EFX_INSERT_FIELD64(64, 127, _field, _value));       \
1212         _NOTE(CONSTANTCONDITION)                                        \
1213         } while (B_FALSE)
1214
1215 #define EFX_SET_OWORD_FIELD32(_oword, _field, _value)                   \
1216         do {                                                            \
1217                 _NOTE(CONSTANTCONDITION)                                \
1218                 (_oword).eo_u32[0] = (((_oword).eo_u32[0] &             \
1219                     ~EFX_INPLACE_MASK32(0, 31, _field)) |               \
1220                     EFX_INSERT_FIELD32(0, 31, _field, _value));         \
1221                 _NOTE(CONSTANTCONDITION)                                \
1222                 (_oword).eo_u32[1] = (((_oword).eo_u32[1] &             \
1223                     ~EFX_INPLACE_MASK32(32, 63, _field)) |              \
1224                     EFX_INSERT_FIELD32(32, 63, _field, _value));        \
1225                 _NOTE(CONSTANTCONDITION)                                \
1226                 (_oword).eo_u32[2] = (((_oword).eo_u32[2] &             \
1227                     ~EFX_INPLACE_MASK32(64, 95, _field)) |              \
1228                     EFX_INSERT_FIELD32(64, 95, _field, _value));        \
1229                 _NOTE(CONSTANTCONDITION)                                \
1230                 (_oword).eo_u32[3] = (((_oword).eo_u32[3] &             \
1231                     ~EFX_INPLACE_MASK32(96, 127, _field)) |             \
1232                     EFX_INSERT_FIELD32(96, 127, _field, _value));       \
1233         _NOTE(CONSTANTCONDITION)                                        \
1234         } while (B_FALSE)
1235
1236 #define EFX_SET_QWORD_FIELD64(_qword, _field, _value)                   \
1237         do {                                                            \
1238                 _NOTE(CONSTANTCONDITION)                                \
1239                 (_qword).eq_u64[0] = (((_qword).eq_u64[0] &             \
1240                     ~EFX_INPLACE_MASK64(0, 63, _field)) |               \
1241                     EFX_INSERT_FIELD64(0, 63, _field, _value));         \
1242         _NOTE(CONSTANTCONDITION)                                        \
1243         } while (B_FALSE)
1244
1245 #define EFX_SET_QWORD_FIELD32(_qword, _field, _value)                   \
1246         do {                                                            \
1247                 _NOTE(CONSTANTCONDITION)                                \
1248                 (_qword).eq_u32[0] = (((_qword).eq_u32[0] &             \
1249                     ~EFX_INPLACE_MASK32(0, 31, _field)) |               \
1250                     EFX_INSERT_FIELD32(0, 31, _field, _value));         \
1251                 _NOTE(CONSTANTCONDITION)                                \
1252                 (_qword).eq_u32[1] = (((_qword).eq_u32[1] &             \
1253                     ~EFX_INPLACE_MASK32(32, 63, _field)) |              \
1254                     EFX_INSERT_FIELD32(32, 63, _field, _value));        \
1255         _NOTE(CONSTANTCONDITION)                                        \
1256         } while (B_FALSE)
1257
1258 #define EFX_SET_DWORD_FIELD(_dword, _field, _value)                     \
1259         do {                                                            \
1260                 _NOTE(CONSTANTCONDITION)                                \
1261                 (_dword).ed_u32[0] = (((_dword).ed_u32[0] &             \
1262                     ~EFX_INPLACE_MASK32(0, 31, _field)) |               \
1263                     EFX_INSERT_FIELD32(0, 31, _field, _value));         \
1264         _NOTE(CONSTANTCONDITION)                                        \
1265         } while (B_FALSE)
1266
1267 #define EFX_SET_WORD_FIELD(_word, _field, _value)                       \
1268         do {                                                            \
1269                 _NOTE(CONSTANTCONDITION)                                \
1270                 (_word).ew_u16[0] = (((_word).ew_u16[0] &               \
1271                     ~EFX_INPLACE_MASK16(0, 15, _field)) |               \
1272                     EFX_INSERT_FIELD16(0, 15, _field, _value));         \
1273         _NOTE(CONSTANTCONDITION)                                        \
1274         } while (B_FALSE)
1275
1276 #define EFX_SET_BYTE_FIELD(_byte, _field, _value)                       \
1277         do {                                                            \
1278                 _NOTE(CONSTANTCONDITION)                                \
1279                 (_byte).eb_u8[0] = (((_byte).eb_u8[0] &                 \
1280                     ~EFX_INPLACE_MASK8(0, 7, _field)) |                 \
1281                     EFX_INSERT_FIELD8(0, 7, _field, _value));           \
1282         _NOTE(CONSTANTCONDITION)                                        \
1283         } while (B_FALSE)
1284
1285 /*
1286  * Set or clear a numbered bit within an octword.
1287  */
1288
1289 #define EFX_SHIFT64(_bit, _base)                                        \
1290         (((_bit) >= (_base) && (_bit) < (_base) + 64) ?                 \
1291                 ((uint64_t)1 << ((_bit) - (_base))) :                   \
1292                 0U)
1293
1294 #define EFX_SHIFT32(_bit, _base)                                        \
1295         (((_bit) >= (_base) && (_bit) < (_base) + 32) ?                 \
1296                 ((uint32_t)1 << ((_bit) - (_base))) :                   \
1297                 0U)
1298
1299 #define EFX_SHIFT16(_bit, _base)                                        \
1300         (((_bit) >= (_base) && (_bit) < (_base) + 16) ?                 \
1301                 (uint16_t)(1 << ((_bit) - (_base))) :                   \
1302                 0U)
1303
1304 #define EFX_SHIFT8(_bit, _base)                                         \
1305         (((_bit) >= (_base) && (_bit) < (_base) + 8) ?                  \
1306                 (uint8_t)(1 << ((_bit) - (_base))) :                    \
1307                 0U)
1308
1309 #define EFX_SET_OWORD_BIT64(_oword, _bit)                               \
1310         do {                                                            \
1311                 _NOTE(CONSTANTCONDITION)                                \
1312                 (_oword).eo_u64[0] |=                                   \
1313                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)));     \
1314                 (_oword).eo_u64[1] |=                                   \
1315                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(64)));    \
1316         _NOTE(CONSTANTCONDITION)                                        \
1317         } while (B_FALSE)
1318
1319 #define EFX_SET_OWORD_BIT32(_oword, _bit)                               \
1320         do {                                                            \
1321                 _NOTE(CONSTANTCONDITION)                                \
1322                 (_oword).eo_u32[0] |=                                   \
1323                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));     \
1324                 (_oword).eo_u32[1] |=                                   \
1325                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)));    \
1326                 (_oword).eo_u32[2] |=                                   \
1327                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(64)));    \
1328                 (_oword).eo_u32[3] |=                                   \
1329                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(96)));    \
1330         _NOTE(CONSTANTCONDITION)                                        \
1331         } while (B_FALSE)
1332
1333 #define EFX_CLEAR_OWORD_BIT64(_oword, _bit)                             \
1334         do {                                                            \
1335                 _NOTE(CONSTANTCONDITION)                                \
1336                 (_oword).eo_u64[0] &=                                   \
1337                     __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(0)));    \
1338                 (_oword).eo_u64[1] &=                                   \
1339                     __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(64)));   \
1340         _NOTE(CONSTANTCONDITION)                                        \
1341         } while (B_FALSE)
1342
1343 #define EFX_CLEAR_OWORD_BIT32(_oword, _bit)                             \
1344         do {                                                            \
1345                 _NOTE(CONSTANTCONDITION)                                \
1346                 (_oword).eo_u32[0] &=                                   \
1347                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));    \
1348                 (_oword).eo_u32[1] &=                                   \
1349                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(32)));   \
1350                 (_oword).eo_u32[2] &=                                   \
1351                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(64)));   \
1352                 (_oword).eo_u32[3] &=                                   \
1353                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(96)));   \
1354         _NOTE(CONSTANTCONDITION)                                        \
1355         } while (B_FALSE)
1356
1357 #define EFX_TEST_OWORD_BIT64(_oword, _bit)                              \
1358         (((_oword).eo_u64[0] &                                          \
1359                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)))) ||  \
1360         ((_oword).eo_u64[1] &                                           \
1361                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(64)))))
1362
1363 #define EFX_TEST_OWORD_BIT32(_oword, _bit)                              \
1364         (((_oword).eo_u32[0] &                                          \
1365                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)))) ||  \
1366         ((_oword).eo_u32[1] &                                           \
1367                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)))) || \
1368         ((_oword).eo_u32[2] &                                           \
1369                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(64)))) || \
1370         ((_oword).eo_u32[3] &                                           \
1371                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(96)))))
1372
1373
1374 #define EFX_SET_QWORD_BIT64(_qword, _bit)                               \
1375         do {                                                            \
1376                 _NOTE(CONSTANTCONDITION)                                \
1377                 (_qword).eq_u64[0] |=                                   \
1378                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)));     \
1379         _NOTE(CONSTANTCONDITION)                                        \
1380         } while (B_FALSE)
1381
1382 #define EFX_SET_QWORD_BIT32(_qword, _bit)                               \
1383         do {                                                            \
1384                 _NOTE(CONSTANTCONDITION)                                \
1385                 (_qword).eq_u32[0] |=                                   \
1386                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));     \
1387                 (_qword).eq_u32[1] |=                                   \
1388                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)));    \
1389         _NOTE(CONSTANTCONDITION)                                        \
1390         } while (B_FALSE)
1391
1392 #define EFX_CLEAR_QWORD_BIT64(_qword, _bit)                             \
1393         do {                                                            \
1394                 _NOTE(CONSTANTCONDITION)                                \
1395                 (_qword).eq_u64[0] &=                                   \
1396                     __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(0)));    \
1397         _NOTE(CONSTANTCONDITION)                                        \
1398         } while (B_FALSE)
1399
1400 #define EFX_CLEAR_QWORD_BIT32(_qword, _bit)                             \
1401         do {                                                            \
1402                 _NOTE(CONSTANTCONDITION)                                \
1403                 (_qword).eq_u32[0] &=                                   \
1404                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));    \
1405                 (_qword).eq_u32[1] &=                                   \
1406                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(32)));   \
1407         _NOTE(CONSTANTCONDITION)                                        \
1408         } while (B_FALSE)
1409
1410 #define EFX_TEST_QWORD_BIT64(_qword, _bit)                              \
1411         (((_qword).eq_u64[0] &                                          \
1412                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)))) != 0)
1413
1414 #define EFX_TEST_QWORD_BIT32(_qword, _bit)                              \
1415         (((_qword).eq_u32[0] &                                          \
1416                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)))) ||  \
1417         ((_qword).eq_u32[1] &                                           \
1418                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)))))
1419
1420
1421 #define EFX_SET_DWORD_BIT(_dword, _bit)                                 \
1422         do {                                                            \
1423                 (_dword).ed_u32[0] |=                                   \
1424                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));     \
1425         _NOTE(CONSTANTCONDITION)                                        \
1426         } while (B_FALSE)
1427
1428 #define EFX_CLEAR_DWORD_BIT(_dword, _bit)                               \
1429         do {                                                            \
1430                 (_dword).ed_u32[0] &=                                   \
1431                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));    \
1432         _NOTE(CONSTANTCONDITION)                                        \
1433         } while (B_FALSE)
1434
1435 #define EFX_TEST_DWORD_BIT(_dword, _bit)                                \
1436         (((_dword).ed_u32[0] &                                          \
1437                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)))) != 0)
1438
1439
1440 #define EFX_SET_WORD_BIT(_word, _bit)                                   \
1441         do {                                                            \
1442                 (_word).ew_u16[0] |=                                    \
1443                     __CPU_TO_LE_16(EFX_SHIFT16(_bit, FIX_LINT(0)));     \
1444         _NOTE(CONSTANTCONDITION)                                        \
1445         } while (B_FALSE)
1446
1447 #define EFX_CLEAR_WORD_BIT(_word, _bit)                                 \
1448         do {                                                            \
1449                 (_word).ew_u32[0] &=                                    \
1450                     __CPU_TO_LE_16(~EFX_SHIFT16(_bit, FIX_LINT(0)));    \
1451         _NOTE(CONSTANTCONDITION)                                        \
1452         } while (B_FALSE)
1453
1454 #define EFX_TEST_WORD_BIT(_word, _bit)                                  \
1455         (((_word).ew_u16[0] &                                           \
1456                     __CPU_TO_LE_16(EFX_SHIFT16(_bit, FIX_LINT(0)))) != 0)
1457
1458
1459 #define EFX_SET_BYTE_BIT(_byte, _bit)                                   \
1460         do {                                                            \
1461                 (_byte).eb_u8[0] |=                                     \
1462                     __NATIVE_8(EFX_SHIFT8(_bit, FIX_LINT(0)));          \
1463         _NOTE(CONSTANTCONDITION)                                        \
1464         } while (B_FALSE)
1465
1466 #define EFX_CLEAR_BYTE_BIT(_byte, _bit)                                 \
1467         do {                                                            \
1468                 (_byte).eb_u8[0] &=                                     \
1469                     __NATIVE_8(~EFX_SHIFT8(_bit, FIX_LINT(0)));         \
1470         _NOTE(CONSTANTCONDITION)                                        \
1471         } while (B_FALSE)
1472
1473 #define EFX_TEST_BYTE_BIT(_byte, _bit)                                  \
1474         (((_byte).eb_u8[0] &                                            \
1475                     __NATIVE_8(EFX_SHIFT8(_bit, FIX_LINT(0)))) != 0)
1476
1477
1478 #define EFX_OR_OWORD64(_oword1, _oword2)                                \
1479         do {                                                            \
1480                 (_oword1).eo_u64[0] |= (_oword2).eo_u64[0];             \
1481                 (_oword1).eo_u64[1] |= (_oword2).eo_u64[1];             \
1482         _NOTE(CONSTANTCONDITION)                                        \
1483         } while (B_FALSE)
1484
1485 #define EFX_OR_OWORD32(_oword1, _oword2)                                \
1486         do {                                                            \
1487                 (_oword1).eo_u32[0] |= (_oword2).eo_u32[0];             \
1488                 (_oword1).eo_u32[1] |= (_oword2).eo_u32[1];             \
1489                 (_oword1).eo_u32[2] |= (_oword2).eo_u32[2];             \
1490                 (_oword1).eo_u32[3] |= (_oword2).eo_u32[3];             \
1491         _NOTE(CONSTANTCONDITION)                                        \
1492         } while (B_FALSE)
1493
1494 #define EFX_AND_OWORD64(_oword1, _oword2)                               \
1495         do {                                                            \
1496                 (_oword1).eo_u64[0] &= (_oword2).eo_u64[0];             \
1497                 (_oword1).eo_u64[1] &= (_oword2).eo_u64[1];             \
1498         _NOTE(CONSTANTCONDITION)                                        \
1499         } while (B_FALSE)
1500
1501 #define EFX_AND_OWORD32(_oword1, _oword2)                               \
1502         do {                                                            \
1503                 (_oword1).eo_u32[0] &= (_oword2).eo_u32[0];             \
1504                 (_oword1).eo_u32[1] &= (_oword2).eo_u32[1];             \
1505                 (_oword1).eo_u32[2] &= (_oword2).eo_u32[2];             \
1506                 (_oword1).eo_u32[3] &= (_oword2).eo_u32[3];             \
1507         _NOTE(CONSTANTCONDITION)                                        \
1508         } while (B_FALSE)
1509
1510 #define EFX_OR_QWORD64(_qword1, _qword2)                                \
1511         do {                                                            \
1512                 (_qword1).eq_u64[0] |= (_qword2).eq_u64[0];             \
1513         _NOTE(CONSTANTCONDITION)                                        \
1514         } while (B_FALSE)
1515
1516 #define EFX_OR_QWORD32(_qword1, _qword2)                                \
1517         do {                                                            \
1518                 (_qword1).eq_u32[0] |= (_qword2).eq_u32[0];             \
1519                 (_qword1).eq_u32[1] |= (_qword2).eq_u32[1];             \
1520         _NOTE(CONSTANTCONDITION)                                        \
1521         } while (B_FALSE)
1522
1523 #define EFX_AND_QWORD64(_qword1, _qword2)                               \
1524         do {                                                            \
1525                 (_qword1).eq_u64[0] &= (_qword2).eq_u64[0];             \
1526         _NOTE(CONSTANTCONDITION)                                        \
1527         } while (B_FALSE)
1528
1529 #define EFX_AND_QWORD32(_qword1, _qword2)                               \
1530         do {                                                            \
1531                 (_qword1).eq_u32[0] &= (_qword2).eq_u32[0];             \
1532                 (_qword1).eq_u32[1] &= (_qword2).eq_u32[1];             \
1533         _NOTE(CONSTANTCONDITION)                                        \
1534         } while (B_FALSE)
1535
1536 #define EFX_OR_DWORD(_dword1, _dword2)                                  \
1537         do {                                                            \
1538                 (_dword1).ed_u32[0] |= (_dword2).ed_u32[0];             \
1539         _NOTE(CONSTANTCONDITION)                                        \
1540         } while (B_FALSE)
1541
1542 #define EFX_AND_DWORD(_dword1, _dword2)                                 \
1543         do {                                                            \
1544                 (_dword1).ed_u32[0] &= (_dword2).ed_u32[0];             \
1545         _NOTE(CONSTANTCONDITION)                                        \
1546         } while (B_FALSE)
1547
1548 #define EFX_OR_WORD(_word1, _word2)                                     \
1549         do {                                                            \
1550                 (_word1).ew_u16[0] |= (_word2).ew_u16[0];               \
1551         _NOTE(CONSTANTCONDITION)                                        \
1552         } while (B_FALSE)
1553
1554 #define EFX_AND_WORD(_word1, _word2)                                    \
1555         do {                                                            \
1556                 (_word1).ew_u16[0] &= (_word2).ew_u16[0];               \
1557         _NOTE(CONSTANTCONDITION)                                        \
1558         } while (B_FALSE)
1559
1560 #define EFX_OR_BYTE(_byte1, _byte2)                                     \
1561         do {                                                            \
1562                 (_byte1).eb_u8[0] |= (_byte2).eb_u8[0];                 \
1563         _NOTE(CONSTANTCONDITION)                                        \
1564         } while (B_FALSE)
1565
1566 #define EFX_AND_BYTE(_byte1, _byte2)                                    \
1567         do {                                                            \
1568                 (_byte1).eb_u8[0] &= (_byte2).eb_u8[0];                 \
1569         _NOTE(CONSTANTCONDITION)                                        \
1570         } while (B_FALSE)
1571
1572 #if EFSYS_USE_UINT64
1573 #define EFX_OWORD_FIELD         EFX_OWORD_FIELD64
1574 #define EFX_QWORD_FIELD         EFX_QWORD_FIELD64
1575 #define EFX_OWORD_IS_EQUAL      EFX_OWORD_IS_EQUAL64
1576 #define EFX_QWORD_IS_EQUAL      EFX_QWORD_IS_EQUAL64
1577 #define EFX_OWORD_IS_ZERO       EFX_OWORD_IS_ZERO64
1578 #define EFX_QWORD_IS_ZERO       EFX_QWORD_IS_ZERO64
1579 #define EFX_OWORD_IS_SET        EFX_OWORD_IS_SET64
1580 #define EFX_QWORD_IS_SET        EFX_QWORD_IS_SET64
1581 #define EFX_POPULATE_OWORD      EFX_POPULATE_OWORD64
1582 #define EFX_POPULATE_QWORD      EFX_POPULATE_QWORD64
1583 #define EFX_SET_OWORD_FIELD     EFX_SET_OWORD_FIELD64
1584 #define EFX_SET_QWORD_FIELD     EFX_SET_QWORD_FIELD64
1585 #define EFX_SET_OWORD_BIT       EFX_SET_OWORD_BIT64
1586 #define EFX_CLEAR_OWORD_BIT     EFX_CLEAR_OWORD_BIT64
1587 #define EFX_TEST_OWORD_BIT      EFX_TEST_OWORD_BIT64
1588 #define EFX_SET_QWORD_BIT       EFX_SET_QWORD_BIT64
1589 #define EFX_CLEAR_QWORD_BIT     EFX_CLEAR_QWORD_BIT64
1590 #define EFX_TEST_QWORD_BIT      EFX_TEST_QWORD_BIT64
1591 #define EFX_OR_OWORD            EFX_OR_OWORD64
1592 #define EFX_AND_OWORD           EFX_AND_OWORD64
1593 #define EFX_OR_QWORD            EFX_OR_QWORD64
1594 #define EFX_AND_QWORD           EFX_AND_QWORD64
1595 #else
1596 #define EFX_OWORD_FIELD         EFX_OWORD_FIELD32
1597 #define EFX_QWORD_FIELD         EFX_QWORD_FIELD32
1598 #define EFX_OWORD_IS_EQUAL      EFX_OWORD_IS_EQUAL32
1599 #define EFX_QWORD_IS_EQUAL      EFX_QWORD_IS_EQUAL32
1600 #define EFX_OWORD_IS_ZERO       EFX_OWORD_IS_ZERO32
1601 #define EFX_QWORD_IS_ZERO       EFX_QWORD_IS_ZERO32
1602 #define EFX_OWORD_IS_SET        EFX_OWORD_IS_SET32
1603 #define EFX_QWORD_IS_SET        EFX_QWORD_IS_SET32
1604 #define EFX_POPULATE_OWORD      EFX_POPULATE_OWORD32
1605 #define EFX_POPULATE_QWORD      EFX_POPULATE_QWORD32
1606 #define EFX_SET_OWORD_FIELD     EFX_SET_OWORD_FIELD32
1607 #define EFX_SET_QWORD_FIELD     EFX_SET_QWORD_FIELD32
1608 #define EFX_SET_OWORD_BIT       EFX_SET_OWORD_BIT32
1609 #define EFX_CLEAR_OWORD_BIT     EFX_CLEAR_OWORD_BIT32
1610 #define EFX_TEST_OWORD_BIT      EFX_TEST_OWORD_BIT32
1611 #define EFX_SET_QWORD_BIT       EFX_SET_QWORD_BIT32
1612 #define EFX_CLEAR_QWORD_BIT     EFX_CLEAR_QWORD_BIT32
1613 #define EFX_TEST_QWORD_BIT      EFX_TEST_QWORD_BIT32
1614 #define EFX_OR_OWORD            EFX_OR_OWORD32
1615 #define EFX_AND_OWORD           EFX_AND_OWORD32
1616 #define EFX_OR_QWORD            EFX_OR_QWORD32
1617 #define EFX_AND_QWORD           EFX_AND_QWORD32
1618 #endif
1619
1620 #ifdef  __cplusplus
1621 }
1622 #endif
1623
1624 #endif  /* _SYS_EFX_TYPES_H */