drivers/net/sfc/base/efx_types.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright (c) 2007-2018 Solarflare Communications Inc.
   4  * All rights reserved.
   5  *
   6  * Ackowledgement to Fen Systems Ltd.
   7  */
   8
   9 #ifndef _SYS_EFX_TYPES_H
  10 #define _SYS_EFX_TYPES_H
  11
  12 #include "efsys.h"
  13
  14 #ifdef  __cplusplus
  15 extern "C" {
  16 #endif
  17
  18 /*
  19  * Bitfield access
  20  *
  21  * Solarflare NICs make extensive use of bitfields up to 128 bits
  22  * wide.  Since there is no native 128-bit datatype on most systems,
  23  * and since 64-bit datatypes are inefficient on 32-bit systems and
  24  * vice versa, we wrap accesses in a way that uses the most efficient
  25  * datatype.
  26  *
  27  * The NICs are PCI devices and therefore little-endian.  Since most
  28  * of the quantities that we deal with are DMAed to/from host memory,
  29  * we define    our datatypes (efx_oword_t, efx_qword_t and efx_dword_t)
  30  * to be little-endian.
  31  *
  32  * In the less common case of using PIO for individual register
  33  * writes, we construct the little-endian datatype in host memory and
  34  * then use non-swapping register access primitives, rather than
  35  * constructing a native-endian datatype and relying on implicit
  36  * byte-swapping.  (We use a similar strategy for register reads.)
  37  */
  38
  39 /*
  40  * NOTE: Field definitions here and elsewhere are done in terms of a lowest
  41  *       bit number (LBN) and a width.
  42  */
  43
  44 #define EFX_DUMMY_FIELD_LBN 0
  45 #define EFX_DUMMY_FIELD_WIDTH 0
  46
  47 #define EFX_BYTE_0_LBN 0
  48 #define EFX_BYTE_0_WIDTH 8
  49
  50 #define EFX_BYTE_1_LBN 8
  51 #define EFX_BYTE_1_WIDTH 8
  52
  53 #define EFX_BYTE_2_LBN 16
  54 #define EFX_BYTE_2_WIDTH 8
  55
  56 #define EFX_BYTE_3_LBN 24
  57 #define EFX_BYTE_3_WIDTH 8
  58
  59 #define EFX_BYTE_4_LBN 32
  60 #define EFX_BYTE_4_WIDTH 8
  61
  62 #define EFX_BYTE_5_LBN 40
  63 #define EFX_BYTE_5_WIDTH 8
  64
  65 #define EFX_BYTE_6_LBN 48
  66 #define EFX_BYTE_6_WIDTH 8
  67
  68 #define EFX_BYTE_7_LBN 56
  69 #define EFX_BYTE_7_WIDTH 8
  70
  71 #define EFX_WORD_0_LBN 0
  72 #define EFX_WORD_0_WIDTH 16
  73
  74 #define EFX_WORD_1_LBN 16
  75 #define EFX_WORD_1_WIDTH 16
  76
  77 #define EFX_WORD_2_LBN 32
  78 #define EFX_WORD_2_WIDTH 16
  79
  80 #define EFX_WORD_3_LBN 48
  81 #define EFX_WORD_3_WIDTH 16
  82
  83 #define EFX_DWORD_0_LBN 0
  84 #define EFX_DWORD_0_WIDTH 32
  85
  86 #define EFX_DWORD_1_LBN 32
  87 #define EFX_DWORD_1_WIDTH 32
  88
  89 #define EFX_DWORD_2_LBN 64
  90 #define EFX_DWORD_2_WIDTH 32
  91
  92 #define EFX_DWORD_3_LBN 96
  93 #define EFX_DWORD_3_WIDTH 32
  94
  95 /*
  96  * There are intentionally no EFX_QWORD_0 or EFX_QWORD_1 field definitions
  97  * here as the implementaion of EFX_QWORD_FIELD and EFX_OWORD_FIELD do not
  98  * support field widths larger than 32 bits.
  99  */
 100
 101 /* Specified attribute (i.e. LBN ow WIDTH) of the specified field */
 102 #define EFX_VAL(_field, _attribute)                                     \
 103         _field ## _ ## _attribute
 104
 105 /* Lowest bit number of the specified field */
 106 #define EFX_LOW_BIT(_field)                                             \
 107         EFX_VAL(_field, LBN)
 108
 109 /* Width of the specified field */
 110 #define EFX_WIDTH(_field)                                               \
 111         EFX_VAL(_field, WIDTH)
 112
 113 /* Highest bit number of the specified field */
 114 #define EFX_HIGH_BIT(_field)                                            \
 115         (EFX_LOW_BIT(_field) + EFX_WIDTH(_field) - 1)
 116
 117 /*
 118  * 64-bit mask equal in width to the specified field.
 119  *
 120  * For example, a field with width 5 would have a mask of 0x000000000000001f.
 121  */
 122 #define EFX_MASK64(_field)                                              \
 123         ((EFX_WIDTH(_field) == 64) ? ~((uint64_t)0) :                   \
 124             (((((uint64_t)1) << EFX_WIDTH(_field))) - 1))
 125 /*
 126  * 32-bit mask equal in width to the specified field.
 127  *
 128  * For example, a field with width 5 would have a mask of 0x0000001f.
 129  */
 130 #define EFX_MASK32(_field)                                              \
 131         ((EFX_WIDTH(_field) == 32) ? ~((uint32_t)0) :                   \
 132             (((((uint32_t)1) << EFX_WIDTH(_field))) - 1))
 133
 134 /*
 135  * 16-bit mask equal in width to the specified field.
 136  *
 137  * For example, a field with width 5 would have a mask of 0x001f.
 138  */
 139 #define EFX_MASK16(_field)                                              \
 140         ((EFX_WIDTH(_field) == 16) ? 0xffffu :                          \
 141             (uint16_t)((1 << EFX_WIDTH(_field)) - 1))
 142
 143 /*
 144  * 8-bit mask equal in width to the specified field.
 145  *
 146  * For example, a field with width 5 would have a mask of 0x1f.
 147  */
 148 #define EFX_MASK8(_field)                                               \
 149         ((uint8_t)((1 << EFX_WIDTH(_field)) - 1))
 150
 151 #pragma pack(1)
 152
 153 /*
 154  * A byte (i.e. 8-bit) datatype
 155  */
 156 typedef union efx_byte_u {
 157         uint8_t eb_u8[1];
 158 } efx_byte_t;
 159
 160 /*
 161  * A word (i.e. 16-bit) datatype
 162  *
 163  * This datatype is defined to be little-endian.
 164  */
 165 typedef union efx_word_u {
 166         efx_byte_t ew_byte[2];
 167         uint16_t ew_u16[1];
 168         uint8_t ew_u8[2];
 169 } efx_word_t;
 170
 171 /*
 172  * A doubleword (i.e. 32-bit) datatype
 173  *
 174  * This datatype is defined to be little-endian.
 175  */
 176 typedef union efx_dword_u {
 177         efx_byte_t ed_byte[4];
 178         efx_word_t ed_word[2];
 179         uint32_t ed_u32[1];
 180         uint16_t ed_u16[2];
 181         uint8_t ed_u8[4];
 182 } efx_dword_t;
 183
 184 /*
 185  * A quadword (i.e. 64-bit) datatype
 186  *
 187  * This datatype is defined to be little-endian.
 188  */
 189 typedef union efx_qword_u {
 190         efx_byte_t eq_byte[8];
 191         efx_word_t eq_word[4];
 192         efx_dword_t eq_dword[2];
 193 #if EFSYS_HAS_UINT64
 194         uint64_t eq_u64[1];
 195 #endif
 196         uint32_t eq_u32[2];
 197         uint16_t eq_u16[4];
 198         uint8_t eq_u8[8];
 199 } efx_qword_t;
 200
 201 /*
 202  * An octword (i.e. 128-bit) datatype
 203  *
 204  * This datatype is defined to be little-endian.
 205  */
 206 typedef union efx_oword_u {
 207         efx_byte_t eo_byte[16];
 208         efx_word_t eo_word[8];
 209         efx_dword_t eo_dword[4];
 210         efx_qword_t eo_qword[2];
 211 #if EFSYS_HAS_SSE2_M128
 212         __m128i eo_u128[1];
 213 #endif
 214 #if EFSYS_HAS_UINT64
 215         uint64_t eo_u64[2];
 216 #endif
 217         uint32_t eo_u32[4];
 218         uint16_t eo_u16[8];
 219         uint8_t eo_u8[16];
 220 } efx_oword_t;
 221
 222 #pragma pack()
 223
 224 #define __SWAP16(_x)                            \
 225         ((((_x) & 0xff) << 8) |                 \
 226         (((_x) >> 8) & 0xff))
 227
 228 #define __SWAP32(_x)                            \
 229         ((__SWAP16((_x) & 0xffff) << 16) |      \
 230         __SWAP16(((_x) >> 16) & 0xffff))
 231
 232 #define __SWAP64(_x)                            \
 233         ((__SWAP32((_x) & 0xffffffff) << 32) |  \
 234         __SWAP32(((_x) >> 32) & 0xffffffff))
 235
 236 #define __NOSWAP16(_x)          (_x)
 237 #define __NOSWAP32(_x)          (_x)
 238 #define __NOSWAP64(_x)          (_x)
 239
 240 #if EFSYS_IS_BIG_ENDIAN
 241
 242 #define __CPU_TO_LE_16(_x)      ((uint16_t)__SWAP16(_x))
 243 #define __LE_TO_CPU_16(_x)      ((uint16_t)__SWAP16(_x))
 244 #define __CPU_TO_BE_16(_x)      ((uint16_t)__NOSWAP16(_x))
 245 #define __BE_TO_CPU_16(_x)      ((uint16_t)__NOSWAP16(_x))
 246
 247 #define __CPU_TO_LE_32(_x)      ((uint32_t)__SWAP32(_x))
 248 #define __LE_TO_CPU_32(_x)      ((uint32_t)__SWAP32(_x))
 249 #define __CPU_TO_BE_32(_x)      ((uint32_t)__NOSWAP32(_x))
 250 #define __BE_TO_CPU_32(_x)      ((uint32_t)__NOSWAP32(_x))
 251
 252 #define __CPU_TO_LE_64(_x)      ((uint64_t)__SWAP64(_x))
 253 #define __LE_TO_CPU_64(_x)      ((uint64_t)__SWAP64(_x))
 254 #define __CPU_TO_BE_64(_x)      ((uint64_t)__NOSWAP64(_x))
 255 #define __BE_TO_CPU_64(_x)      ((uint64_t)__NOSWAP64(_x))
 256
 257 #elif EFSYS_IS_LITTLE_ENDIAN
 258
 259 #define __CPU_TO_LE_16(_x)      ((uint16_t)__NOSWAP16(_x))
 260 #define __LE_TO_CPU_16(_x)      ((uint16_t)__NOSWAP16(_x))
 261 #define __CPU_TO_BE_16(_x)      ((uint16_t)__SWAP16(_x))
 262 #define __BE_TO_CPU_16(_x)      ((uint16_t)__SWAP16(_x))
 263
 264 #define __CPU_TO_LE_32(_x)      ((uint32_t)__NOSWAP32(_x))
 265 #define __LE_TO_CPU_32(_x)      ((uint32_t)__NOSWAP32(_x))
 266 #define __CPU_TO_BE_32(_x)      ((uint32_t)__SWAP32(_x))
 267 #define __BE_TO_CPU_32(_x)      ((uint32_t)__SWAP32(_x))
 268
 269 #define __CPU_TO_LE_64(_x)      ((uint64_t)__NOSWAP64(_x))
 270 #define __LE_TO_CPU_64(_x)      ((uint64_t)__NOSWAP64(_x))
 271 #define __CPU_TO_BE_64(_x)      ((uint64_t)__SWAP64(_x))
 272 #define __BE_TO_CPU_64(_x)      ((uint64_t)__SWAP64(_x))
 273
 274 #else
 275
 276 #error "Neither of EFSYS_IS_{BIG,LITTLE}_ENDIAN is set"
 277
 278 #endif
 279
 280 #define __NATIVE_8(_x)  (uint8_t)(_x)
 281
 282 /* Format string for printing an efx_byte_t */
 283 #define EFX_BYTE_FMT "0x%02x"
 284
 285 /* Format string for printing an efx_word_t */
 286 #define EFX_WORD_FMT "0x%04x"
 287
 288 /* Format string for printing an efx_dword_t */
 289 #define EFX_DWORD_FMT "0x%08x"
 290
 291 /* Format string for printing an efx_qword_t */
 292 #define EFX_QWORD_FMT "0x%08x:%08x"
 293
 294 /* Format string for printing an efx_oword_t */
 295 #define EFX_OWORD_FMT "0x%08x:%08x:%08x:%08x"
 296
 297 /* Parameters for printing an efx_byte_t */
 298 #define EFX_BYTE_VAL(_byte)                                     \
 299         ((unsigned int)__NATIVE_8((_byte).eb_u8[0]))
 300
 301 /* Parameters for printing an efx_word_t */
 302 #define EFX_WORD_VAL(_word)                                     \
 303         ((unsigned int)__LE_TO_CPU_16((_word).ew_u16[0]))
 304
 305 /* Parameters for printing an efx_dword_t */
 306 #define EFX_DWORD_VAL(_dword)                                   \
 307         ((unsigned int)__LE_TO_CPU_32((_dword).ed_u32[0]))
 308
 309 /* Parameters for printing an efx_qword_t */
 310 #define EFX_QWORD_VAL(_qword)                                   \
 311         ((unsigned int)__LE_TO_CPU_32((_qword).eq_u32[1])),     \
 312         ((unsigned int)__LE_TO_CPU_32((_qword).eq_u32[0]))
 313
 314 /* Parameters for printing an efx_oword_t */
 315 #define EFX_OWORD_VAL(_oword)                                   \
 316         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[3])),     \
 317         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[2])),     \
 318         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[1])),     \
 319         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[0]))
 320
 321 /*
 322  * Stop lint complaining about some shifts.
 323  */
 324 #ifdef  __lint
 325 extern int fix_lint;
 326 #define FIX_LINT(_x)    (_x + fix_lint)
 327 #else
 328 #define FIX_LINT(_x)    (_x)
 329 #endif
 330
 331 /*
 332  * Saturation arithmetic subtract with minimum equal to zero.
 333  *
 334  * Use saturating arithmetic to ensure a non-negative result. This
 335  * avoids undefined behaviour (and compiler warnings) when used as a
 336  * shift count.
 337  */
 338 #define EFX_SSUB(_val, _sub) \
 339         ((_val) > (_sub) ? ((_val) - (_sub)) : 0)
 340
 341 /*
 342  * Extract bit field portion [low,high) from the native-endian element
 343  * which contains bits [min,max).
 344  *
 345  * For example, suppose "element" represents the high 32 bits of a
 346  * 64-bit value, and we wish to extract the bits belonging to the bit
 347  * field occupying bits 28-45 of this 64-bit value.
 348  *
 349  * Then EFX_EXTRACT(_element, 32, 63, 28, 45) would give
 350  *
 351  *   (_element) << 4
 352  *
 353  * The result will contain the relevant bits filled in in the range
 354  * [0,high-low), with garbage in bits [high-low+1,...).
 355  */
 356 #define EFX_EXTRACT_NATIVE(_element, _min, _max, _low, _high)           \
 357         ((FIX_LINT(_low > _max) || FIX_LINT(_high < _min)) ?            \
 358                 0U :                                                    \
 359                 ((_low > _min) ?                                        \
 360                         ((_element) >> EFX_SSUB(_low, _min)) :          \
 361                         ((_element) << EFX_SSUB(_min, _low))))
 362
 363 /*
 364  * Extract bit field portion [low,high) from the 64-bit little-endian
 365  * element which contains bits [min,max)
 366  */
 367 #define EFX_EXTRACT64(_element, _min, _max, _low, _high)                \
 368         EFX_EXTRACT_NATIVE(__LE_TO_CPU_64(_element), _min, _max, _low, _high)
 369
 370 /*
 371  * Extract bit field portion [low,high) from the 32-bit little-endian
 372  * element which contains bits [min,max)
 373  */
 374 #define EFX_EXTRACT32(_element, _min, _max, _low, _high)                \
 375         EFX_EXTRACT_NATIVE(__LE_TO_CPU_32(_element), _min, _max, _low, _high)
 376
 377 /*
 378  * Extract bit field portion [low,high) from the 16-bit little-endian
 379  * element which contains bits [min,max)
 380  */
 381 #define EFX_EXTRACT16(_element, _min, _max, _low, _high)                \
 382         EFX_EXTRACT_NATIVE(__LE_TO_CPU_16(_element), _min, _max, _low, _high)
 383
 384 /*
 385  * Extract bit field portion [low,high) from the 8-bit
 386  * element which contains bits [min,max)
 387  */
 388 #define EFX_EXTRACT8(_element, _min, _max, _low, _high)                 \
 389         EFX_EXTRACT_NATIVE(__NATIVE_8(_element), _min, _max, _low, _high)
 390
 391 #define EFX_EXTRACT_OWORD64(_oword, _low, _high)                        \
 392         (EFX_EXTRACT64((_oword).eo_u64[0], FIX_LINT(0), FIX_LINT(63),   \
 393             _low, _high) |                                              \
 394         EFX_EXTRACT64((_oword).eo_u64[1], FIX_LINT(64), FIX_LINT(127),  \
 395             _low, _high))
 396
 397 #define EFX_EXTRACT_OWORD32(_oword, _low, _high)                        \
 398         (EFX_EXTRACT32((_oword).eo_u32[0], FIX_LINT(0), FIX_LINT(31),   \
 399             _low, _high) |                                              \
 400         EFX_EXTRACT32((_oword).eo_u32[1], FIX_LINT(32), FIX_LINT(63),   \
 401             _low, _high) |                                              \
 402         EFX_EXTRACT32((_oword).eo_u32[2], FIX_LINT(64), FIX_LINT(95),   \
 403             _low, _high) |                                              \
 404         EFX_EXTRACT32((_oword).eo_u32[3], FIX_LINT(96), FIX_LINT(127),  \
 405             _low, _high))
 406
 407 #define EFX_EXTRACT_QWORD64(_qword, _low, _high)                        \
 408         (EFX_EXTRACT64((_qword).eq_u64[0], FIX_LINT(0), FIX_LINT(63),   \
 409             _low, _high))
 410
 411 #define EFX_EXTRACT_QWORD32(_qword, _low, _high)                        \
 412         (EFX_EXTRACT32((_qword).eq_u32[0], FIX_LINT(0), FIX_LINT(31),   \
 413             _low, _high) |                                              \
 414         EFX_EXTRACT32((_qword).eq_u32[1], FIX_LINT(32), FIX_LINT(63),   \
 415             _low, _high))
 416
 417 #define EFX_EXTRACT_DWORD(_dword, _low, _high)                          \
 418         (EFX_EXTRACT32((_dword).ed_u32[0], FIX_LINT(0), FIX_LINT(31),   \
 419             _low, _high))
 420
 421 #define EFX_EXTRACT_WORD(_word, _low, _high)                            \
 422         (EFX_EXTRACT16((_word).ew_u16[0], FIX_LINT(0), FIX_LINT(15),    \
 423             _low, _high))
 424
 425 #define EFX_EXTRACT_BYTE(_byte, _low, _high)                            \
 426         (EFX_EXTRACT8((_byte).eb_u8[0], FIX_LINT(0), FIX_LINT(7),       \
 427             _low, _high))
 428
 429
 430 #define EFX_OWORD_FIELD64(_oword, _field)                               \
 431         ((uint32_t)EFX_EXTRACT_OWORD64(_oword, EFX_LOW_BIT(_field),     \
 432             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 433
 434 #define EFX_OWORD_FIELD32(_oword, _field)                               \
 435         (EFX_EXTRACT_OWORD32(_oword, EFX_LOW_BIT(_field),               \
 436             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 437
 438 #define EFX_QWORD_FIELD64(_qword, _field)                               \
 439         ((uint32_t)EFX_EXTRACT_QWORD64(_qword, EFX_LOW_BIT(_field),     \
 440             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 441
 442 #define EFX_QWORD_FIELD32(_qword, _field)                               \
 443         (EFX_EXTRACT_QWORD32(_qword, EFX_LOW_BIT(_field),               \
 444             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 445
 446 #define EFX_DWORD_FIELD(_dword, _field)                                 \
 447         (EFX_EXTRACT_DWORD(_dword, EFX_LOW_BIT(_field),                 \
 448             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 449
 450 #define EFX_WORD_FIELD(_word, _field)                                   \
 451         (EFX_EXTRACT_WORD(_word, EFX_LOW_BIT(_field),                   \
 452             EFX_HIGH_BIT(_field)) & EFX_MASK16(_field))
 453
 454 #define EFX_BYTE_FIELD(_byte, _field)                                   \
 455         (EFX_EXTRACT_BYTE(_byte, EFX_LOW_BIT(_field),                   \
 456             EFX_HIGH_BIT(_field)) & EFX_MASK8(_field))
 457
 458
 459 #define EFX_OWORD_IS_EQUAL64(_oword_a, _oword_b)                        \
 460         ((_oword_a).eo_u64[0] == (_oword_b).eo_u64[0] &&                \
 461             (_oword_a).eo_u64[1] == (_oword_b).eo_u64[1])
 462
 463 #define EFX_OWORD_IS_EQUAL32(_oword_a, _oword_b)                        \
 464         ((_oword_a).eo_u32[0] == (_oword_b).eo_u32[0] &&                \
 465             (_oword_a).eo_u32[1] == (_oword_b).eo_u32[1] &&             \
 466             (_oword_a).eo_u32[2] == (_oword_b).eo_u32[2] &&             \
 467             (_oword_a).eo_u32[3] == (_oword_b).eo_u32[3])
 468
 469 #define EFX_QWORD_IS_EQUAL64(_qword_a, _qword_b)                        \
 470         ((_qword_a).eq_u64[0] == (_qword_b).eq_u64[0])
 471
 472 #define EFX_QWORD_IS_EQUAL32(_qword_a, _qword_b)                        \
 473         ((_qword_a).eq_u32[0] == (_qword_b).eq_u32[0] &&                \
 474             (_qword_a).eq_u32[1] == (_qword_b).eq_u32[1])
 475
 476 #define EFX_DWORD_IS_EQUAL(_dword_a, _dword_b)                          \
 477         ((_dword_a).ed_u32[0] == (_dword_b).ed_u32[0])
 478
 479 #define EFX_WORD_IS_EQUAL(_word_a, _word_b)                             \
 480         ((_word_a).ew_u16[0] == (_word_b).ew_u16[0])
 481
 482 #define EFX_BYTE_IS_EQUAL(_byte_a, _byte_b)                             \
 483         ((_byte_a).eb_u8[0] == (_byte_b).eb_u8[0])
 484
 485
 486 #define EFX_OWORD_IS_ZERO64(_oword)                                     \
 487         (((_oword).eo_u64[0] |                                          \
 488             (_oword).eo_u64[1]) == 0)
 489
 490 #define EFX_OWORD_IS_ZERO32(_oword)                                     \
 491         (((_oword).eo_u32[0] |                                          \
 492             (_oword).eo_u32[1] |                                        \
 493             (_oword).eo_u32[2] |                                        \
 494             (_oword).eo_u32[3]) == 0)
 495
 496 #define EFX_QWORD_IS_ZERO64(_qword)                                     \
 497         (((_qword).eq_u64[0]) == 0)
 498
 499 #define EFX_QWORD_IS_ZERO32(_qword)                                     \
 500         (((_qword).eq_u32[0] |                                          \
 501             (_qword).eq_u32[1]) == 0)
 502
 503 #define EFX_DWORD_IS_ZERO(_dword)                                       \
 504         (((_dword).ed_u32[0]) == 0)
 505
 506 #define EFX_WORD_IS_ZERO(_word)                                         \
 507         (((_word).ew_u16[0]) == 0)
 508
 509 #define EFX_BYTE_IS_ZERO(_byte)                                         \
 510         (((_byte).eb_u8[0]) == 0)
 511
 512
 513 #define EFX_OWORD_IS_SET64(_oword)                                      \
 514         (((_oword).eo_u64[0] &                                          \
 515             (_oword).eo_u64[1]) == ~((uint64_t)0))
 516
 517 #define EFX_OWORD_IS_SET32(_oword)                                      \
 518         (((_oword).eo_u32[0] &                                          \
 519             (_oword).eo_u32[1] &                                        \
 520             (_oword).eo_u32[2] &                                        \
 521             (_oword).eo_u32[3]) == ~((uint32_t)0))
 522
 523 #define EFX_QWORD_IS_SET64(_qword)                                      \
 524         (((_qword).eq_u64[0]) == ~((uint64_t)0))
 525
 526 #define EFX_QWORD_IS_SET32(_qword)                                      \
 527         (((_qword).eq_u32[0] &                                          \
 528             (_qword).eq_u32[1]) == ~((uint32_t)0))
 529
 530 #define EFX_DWORD_IS_SET(_dword)                                        \
 531         ((_dword).ed_u32[0] == ~((uint32_t)0))
 532
 533 #define EFX_WORD_IS_SET(_word)                                          \
 534         ((_word).ew_u16[0] == ~((uint16_t)0))
 535
 536 #define EFX_BYTE_IS_SET(_byte)                                          \
 537         ((_byte).eb_u8[0] == ~((uint8_t)0))
 538
 539 /*
 540  * Construct bit field portion
 541  *
 542  * Creates the portion of the bit field [low,high) that lies within
 543  * the range [min,max).
 544  */
 545
 546 #define EFX_INSERT_NATIVE64(_min, _max, _low, _high, _value)            \
 547         (((_low > _max) || (_high < _min)) ?                            \
 548                 0U :                                                    \
 549                 ((_low > _min) ?                                        \
 550                         (((uint64_t)(_value)) << EFX_SSUB(_low, _min)) :\
 551                         (((uint64_t)(_value)) >> EFX_SSUB(_min, _low))))
 552
 553 #define EFX_INSERT_NATIVE32(_min, _max, _low, _high, _value)            \
 554         (((_low > _max) || (_high < _min)) ?                            \
 555                 0U :                                                    \
 556                 ((_low > _min) ?                                        \
 557                         (((uint32_t)(_value)) << EFX_SSUB(_low, _min)) :\
 558                         (((uint32_t)(_value)) >> EFX_SSUB(_min, _low))))
 559
 560 #define EFX_INSERT_NATIVE16(_min, _max, _low, _high, _value)            \
 561         (((_low > _max) || (_high < _min)) ?                            \
 562                 0U :                                                    \
 563                 (uint16_t)((_low > _min) ?                              \
 564                                 ((_value) << EFX_SSUB(_low, _min)) :    \
 565                                 ((_value) >> EFX_SSUB(_min, _low))))
 566
 567 #define EFX_INSERT_NATIVE8(_min, _max, _low, _high, _value)             \
 568         (((_low > _max) || (_high < _min)) ?                            \
 569                 0U :                                                    \
 570                 (uint8_t)((_low > _min) ?                               \
 571                                 ((_value) << EFX_SSUB(_low, _min)) :    \
 572                                 ((_value) >> EFX_SSUB(_min, _low))))
 573
 574 /*
 575  * Construct bit field portion
 576  *
 577  * Creates the portion of the named bit field that lies within the
 578  * range [min,max).
 579  */
 580 #define EFX_INSERT_FIELD_NATIVE64(_min, _max, _field, _value)           \
 581         EFX_INSERT_NATIVE64(_min, _max, EFX_LOW_BIT(_field),            \
 582             EFX_HIGH_BIT(_field), _value)
 583
 584 #define EFX_INSERT_FIELD_NATIVE32(_min, _max, _field, _value)           \
 585         EFX_INSERT_NATIVE32(_min, _max, EFX_LOW_BIT(_field),            \
 586             EFX_HIGH_BIT(_field), _value)
 587
 588 #define EFX_INSERT_FIELD_NATIVE16(_min, _max, _field, _value)           \
 589         EFX_INSERT_NATIVE16(_min, _max, EFX_LOW_BIT(_field),            \
 590             EFX_HIGH_BIT(_field), _value)
 591
 592 #define EFX_INSERT_FIELD_NATIVE8(_min, _max, _field, _value)            \
 593         EFX_INSERT_NATIVE8(_min, _max, EFX_LOW_BIT(_field),             \
 594             EFX_HIGH_BIT(_field), _value)
 595
 596 /*
 597  * Construct bit field
 598  *
 599  * Creates the portion of the named bit fields that lie within the
 600  * range [min,max).
 601  */
 602 #define EFX_INSERT_FIELDS64(_min, _max,                                 \
 603             _field1, _value1, _field2, _value2, _field3, _value3,       \
 604             _field4, _value4, _field5, _value5, _field6, _value6,       \
 605             _field7, _value7, _field8, _value8, _field9, _value9,       \
 606             _field10, _value10)                                         \
 607         __CPU_TO_LE_64(                                                 \
 608             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field1, _value1) |   \
 609             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field2, _value2) |   \
 610             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field3, _value3) |   \
 611             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field4, _value4) |   \
 612             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field5, _value5) |   \
 613             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field6, _value6) |   \
 614             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field7, _value7) |   \
 615             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field8, _value8) |   \
 616             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field9, _value9) |   \
 617             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field10, _value10))
 618
 619 #define EFX_INSERT_FIELDS32(_min, _max,                                 \
 620             _field1, _value1, _field2, _value2, _field3, _value3,       \
 621             _field4, _value4, _field5, _value5, _field6, _value6,       \
 622             _field7, _value7, _field8, _value8, _field9, _value9,       \
 623             _field10, _value10)                                         \
 624         __CPU_TO_LE_32(                                                 \
 625             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field1, _value1) |   \
 626             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field2, _value2) |   \
 627             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field3, _value3) |   \
 628             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field4, _value4) |   \
 629             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field5, _value5) |   \
 630             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field6, _value6) |   \
 631             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field7, _value7) |   \
 632             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field8, _value8) |   \
 633             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field9, _value9) |   \
 634             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field10, _value10))
 635
 636 #define EFX_INSERT_FIELDS16(_min, _max,                                 \
 637             _field1, _value1, _field2, _value2, _field3, _value3,       \
 638             _field4, _value4, _field5, _value5, _field6, _value6,       \
 639             _field7, _value7, _field8, _value8, _field9, _value9,       \
 640             _field10, _value10)                                         \
 641         __CPU_TO_LE_16(                                                 \
 642             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field1, _value1) |   \
 643             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field2, _value2) |   \
 644             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field3, _value3) |   \
 645             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field4, _value4) |   \
 646             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field5, _value5) |   \
 647             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field6, _value6) |   \
 648             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field7, _value7) |   \
 649             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field8, _value8) |   \
 650             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field9, _value9) |   \
 651             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field10, _value10))
 652
 653 #define EFX_INSERT_FIELDS8(_min, _max,                                  \
 654             _field1, _value1, _field2, _value2, _field3, _value3,       \
 655             _field4, _value4, _field5, _value5, _field6, _value6,       \
 656             _field7, _value7, _field8, _value8, _field9, _value9,       \
 657             _field10, _value10)                                         \
 658         __NATIVE_8(                                                     \
 659             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field1, _value1) |    \
 660             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field2, _value2) |    \
 661             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field3, _value3) |    \
 662             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field4, _value4) |    \
 663             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field5, _value5) |    \
 664             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field6, _value6) |    \
 665             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field7, _value7) |    \
 666             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field8, _value8) |    \
 667             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field9, _value9) |    \
 668             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field10, _value10))
 669
 670 #define EFX_POPULATE_OWORD64(_oword,                                    \
 671             _field1, _value1, _field2, _value2, _field3, _value3,       \
 672             _field4, _value4, _field5, _value5, _field6, _value6,       \
 673             _field7, _value7, _field8, _value8, _field9, _value9,       \
 674             _field10, _value10)                                         \
 675         do {                                                            \
 676                 _NOTE(CONSTANTCONDITION)                                \
 677                 (_oword).eo_u64[0] = EFX_INSERT_FIELDS64(0, 63,         \
 678                     _field1, _value1, _field2, _value2,                 \
 679                     _field3, _value3, _field4, _value4,                 \
 680                     _field5, _value5, _field6, _value6,                 \
 681                     _field7, _value7, _field8, _value8,                 \
 682                     _field9, _value9, _field10, _value10);              \
 683                 _NOTE(CONSTANTCONDITION)                                \
 684                 (_oword).eo_u64[1] = EFX_INSERT_FIELDS64(64, 127,       \
 685                     _field1, _value1, _field2, _value2,                 \
 686                     _field3, _value3, _field4, _value4,                 \
 687                     _field5, _value5, _field6, _value6,                 \
 688                     _field7, _value7, _field8, _value8,                 \
 689                     _field9, _value9, _field10, _value10);              \
 690         _NOTE(CONSTANTCONDITION)                                        \
 691         } while (B_FALSE)
 692
 693 #define EFX_POPULATE_OWORD32(_oword,                                    \
 694             _field1, _value1, _field2, _value2, _field3, _value3,       \
 695             _field4, _value4, _field5, _value5, _field6, _value6,       \
 696             _field7, _value7, _field8, _value8, _field9, _value9,       \
 697             _field10, _value10)                                         \
 698         do {                                                            \
 699                 _NOTE(CONSTANTCONDITION)                                \
 700                 (_oword).eo_u32[0] = EFX_INSERT_FIELDS32(0, 31,         \
 701                     _field1, _value1, _field2, _value2,                 \
 702                     _field3, _value3, _field4, _value4,                 \
 703                     _field5, _value5, _field6, _value6,                 \
 704                     _field7, _value7, _field8, _value8,                 \
 705                     _field9, _value9, _field10, _value10);              \
 706                 _NOTE(CONSTANTCONDITION)                                \
 707                 (_oword).eo_u32[1] = EFX_INSERT_FIELDS32(32, 63,        \
 708                     _field1, _value1, _field2, _value2,                 \
 709                     _field3, _value3, _field4, _value4,                 \
 710                     _field5, _value5, _field6, _value6,                 \
 711                     _field7, _value7, _field8, _value8,                 \
 712                     _field9, _value9, _field10, _value10);              \
 713                 _NOTE(CONSTANTCONDITION)                                \
 714                 (_oword).eo_u32[2] = EFX_INSERT_FIELDS32(64, 95,        \
 715                     _field1, _value1, _field2, _value2,                 \
 716                     _field3, _value3, _field4, _value4,                 \
 717                     _field5, _value5, _field6, _value6,                 \
 718                     _field7, _value7, _field8, _value8,                 \
 719                     _field9, _value9, _field10, _value10);              \
 720                 _NOTE(CONSTANTCONDITION)                                \
 721                 (_oword).eo_u32[3] = EFX_INSERT_FIELDS32(96, 127,       \
 722                     _field1, _value1, _field2, _value2,                 \
 723                     _field3, _value3, _field4, _value4,                 \
 724                     _field5, _value5, _field6, _value6,                 \
 725                     _field7, _value7, _field8, _value8,                 \
 726                     _field9, _value9, _field10, _value10);              \
 727         _NOTE(CONSTANTCONDITION)                                        \
 728         } while (B_FALSE)
 729
 730 #define EFX_POPULATE_QWORD64(_qword,                                    \
 731             _field1, _value1, _field2, _value2, _field3, _value3,       \
 732             _field4, _value4, _field5, _value5, _field6, _value6,       \
 733             _field7, _value7, _field8, _value8, _field9, _value9,       \
 734             _field10, _value10)                                         \
 735         do {                                                            \
 736                 _NOTE(CONSTANTCONDITION)                                \
 737                 (_qword).eq_u64[0] = EFX_INSERT_FIELDS64(0, 63,         \
 738                     _field1, _value1, _field2, _value2,                 \
 739                     _field3, _value3, _field4, _value4,                 \
 740                     _field5, _value5, _field6, _value6,                 \
 741                     _field7, _value7, _field8, _value8,                 \
 742                     _field9, _value9, _field10, _value10);              \
 743         _NOTE(CONSTANTCONDITION)                                        \
 744         } while (B_FALSE)
 745
 746 #define EFX_POPULATE_QWORD32(_qword,                                    \
 747             _field1, _value1, _field2, _value2, _field3, _value3,       \
 748             _field4, _value4, _field5, _value5, _field6, _value6,       \
 749             _field7, _value7, _field8, _value8, _field9, _value9,       \
 750             _field10, _value10)                                         \
 751         do {                                                            \
 752                 _NOTE(CONSTANTCONDITION)                                \
 753                 (_qword).eq_u32[0] = EFX_INSERT_FIELDS32(0, 31,         \
 754                     _field1, _value1, _field2, _value2,                 \
 755                     _field3, _value3, _field4, _value4,                 \
 756                     _field5, _value5, _field6, _value6,                 \
 757                     _field7, _value7, _field8, _value8,                 \
 758                     _field9, _value9, _field10, _value10);              \
 759                 _NOTE(CONSTANTCONDITION)                                \
 760                 (_qword).eq_u32[1] = EFX_INSERT_FIELDS32(32, 63,        \
 761                     _field1, _value1, _field2, _value2,                 \
 762                     _field3, _value3, _field4, _value4,                 \
 763                     _field5, _value5, _field6, _value6,                 \
 764                     _field7, _value7, _field8, _value8,                 \
 765                     _field9, _value9, _field10, _value10);              \
 766         _NOTE(CONSTANTCONDITION)                                        \
 767         } while (B_FALSE)
 768
 769 #define EFX_POPULATE_DWORD(_dword,                                      \
 770             _field1, _value1, _field2, _value2, _field3, _value3,       \
 771             _field4, _value4, _field5, _value5, _field6, _value6,       \
 772             _field7, _value7, _field8, _value8, _field9, _value9,       \
 773             _field10, _value10)                                         \
 774         do {                                                            \
 775                 _NOTE(CONSTANTCONDITION)                                \
 776                 (_dword).ed_u32[0] = EFX_INSERT_FIELDS32(0, 31,         \
 777                     _field1, _value1, _field2, _value2,                 \
 778                     _field3, _value3, _field4, _value4,                 \
 779                     _field5, _value5, _field6, _value6,                 \
 780                     _field7, _value7, _field8, _value8,                 \
 781                     _field9, _value9, _field10, _value10);              \
 782         _NOTE(CONSTANTCONDITION)                                        \
 783         } while (B_FALSE)
 784
 785 #define EFX_POPULATE_WORD(_word,                                        \
 786             _field1, _value1, _field2, _value2, _field3, _value3,       \
 787             _field4, _value4, _field5, _value5, _field6, _value6,       \
 788             _field7, _value7, _field8, _value8, _field9, _value9,       \
 789             _field10, _value10)                                         \
 790         do {                                                            \
 791                 _NOTE(CONSTANTCONDITION)                                \
 792                 (_word).ew_u16[0] = EFX_INSERT_FIELDS16(0, 15,          \
 793                     _field1, _value1, _field2, _value2,                 \
 794                     _field3, _value3, _field4, _value4,                 \
 795                     _field5, _value5, _field6, _value6,                 \
 796                     _field7, _value7, _field8, _value8,                 \
 797                     _field9, _value9, _field10, _value10);              \
 798         _NOTE(CONSTANTCONDITION)                                        \
 799         } while (B_FALSE)
 800
 801 #define EFX_POPULATE_BYTE(_byte,                                        \
 802             _field1, _value1, _field2, _value2, _field3, _value3,       \
 803             _field4, _value4, _field5, _value5, _field6, _value6,       \
 804             _field7, _value7, _field8, _value8, _field9, _value9,       \
 805             _field10, _value10)                                         \
 806         do {                                                            \
 807                 _NOTE(CONSTANTCONDITION)                                \
 808                 (_byte).eb_u8[0] = EFX_INSERT_FIELDS8(0, 7,             \
 809                     _field1, _value1, _field2, _value2,                 \
 810                     _field3, _value3, _field4, _value4,                 \
 811                     _field5, _value5, _field6, _value6,                 \
 812                     _field7, _value7, _field8, _value8,                 \
 813                     _field9, _value9, _field10, _value10);              \
 814         _NOTE(CONSTANTCONDITION)                                        \
 815         } while (B_FALSE)
 816
 817 /* Populate an octword field with various numbers of arguments */
 818 #define EFX_POPULATE_OWORD_10 EFX_POPULATE_OWORD
 819
 820 #define EFX_POPULATE_OWORD_9(_oword,                                    \
 821             _field1, _value1, _field2, _value2, _field3, _value3,       \
 822             _field4, _value4, _field5, _value5, _field6, _value6,       \
 823             _field7, _value7, _field8, _value8, _field9, _value9)       \
 824         EFX_POPULATE_OWORD_10(_oword, EFX_DUMMY_FIELD, 0,               \
 825             _field1, _value1, _field2, _value2, _field3, _value3,       \
 826             _field4, _value4, _field5, _value5, _field6, _value6,       \
 827             _field7, _value7, _field8, _value8, _field9, _value9)
 828
 829 #define EFX_POPULATE_OWORD_8(_oword,                                    \
 830             _field1, _value1, _field2, _value2, _field3, _value3,       \
 831             _field4, _value4, _field5, _value5, _field6, _value6,       \
 832             _field7, _value7, _field8, _value8)                         \
 833         EFX_POPULATE_OWORD_9(_oword, EFX_DUMMY_FIELD, 0,                \
 834             _field1, _value1, _field2, _value2, _field3, _value3,       \
 835             _field4, _value4, _field5, _value5, _field6, _value6,       \
 836             _field7, _value7, _field8, _value8)
 837
 838 #define EFX_POPULATE_OWORD_7(_oword,                                    \
 839             _field1, _value1, _field2, _value2, _field3, _value3,       \
 840             _field4, _value4, _field5, _value5, _field6, _value6,       \
 841             _field7, _value7)                                           \
 842         EFX_POPULATE_OWORD_8(_oword, EFX_DUMMY_FIELD, 0,                \
 843             _field1, _value1, _field2, _value2, _field3, _value3,       \
 844             _field4, _value4, _field5, _value5, _field6, _value6,       \
 845             _field7, _value7)
 846
 847 #define EFX_POPULATE_OWORD_6(_oword,                                    \
 848             _field1, _value1, _field2, _value2, _field3, _value3,       \
 849             _field4, _value4, _field5, _value5, _field6, _value6)       \
 850         EFX_POPULATE_OWORD_7(_oword, EFX_DUMMY_FIELD, 0,                \
 851             _field1, _value1, _field2, _value2, _field3, _value3,       \
 852             _field4, _value4, _field5, _value5, _field6, _value6)
 853
 854 #define EFX_POPULATE_OWORD_5(_oword,                                    \
 855             _field1, _value1, _field2, _value2, _field3, _value3,       \
 856             _field4, _value4, _field5, _value5)                         \
 857         EFX_POPULATE_OWORD_6(_oword, EFX_DUMMY_FIELD, 0,                \
 858             _field1, _value1, _field2, _value2, _field3, _value3,       \
 859             _field4, _value4, _field5, _value5)
 860
 861 #define EFX_POPULATE_OWORD_4(_oword,                                    \
 862             _field1, _value1, _field2, _value2, _field3, _value3,       \
 863             _field4, _value4)                                           \
 864         EFX_POPULATE_OWORD_5(_oword, EFX_DUMMY_FIELD, 0,                \
 865             _field1, _value1, _field2, _value2, _field3, _value3,       \
 866             _field4, _value4)
 867
 868 #define EFX_POPULATE_OWORD_3(_oword,                                    \
 869             _field1, _value1, _field2, _value2, _field3, _value3)       \
 870         EFX_POPULATE_OWORD_4(_oword, EFX_DUMMY_FIELD, 0,                \
 871             _field1, _value1, _field2, _value2, _field3, _value3)
 872
 873 #define EFX_POPULATE_OWORD_2(_oword,                                    \
 874             _field1, _value1, _field2, _value2)                         \
 875         EFX_POPULATE_OWORD_3(_oword, EFX_DUMMY_FIELD, 0,                \
 876             _field1, _value1, _field2, _value2)
 877
 878 #define EFX_POPULATE_OWORD_1(_oword,                                    \
 879             _field1, _value1)                                           \
 880         EFX_POPULATE_OWORD_2(_oword, EFX_DUMMY_FIELD, 0,                \
 881             _field1, _value1)
 882
 883 #define EFX_ZERO_OWORD(_oword)                                          \
 884         EFX_POPULATE_OWORD_1(_oword, EFX_DUMMY_FIELD, 0)
 885
 886 #define EFX_SET_OWORD(_oword)                                           \
 887         EFX_POPULATE_OWORD_4(_oword,                                    \
 888             EFX_DWORD_0, 0xffffffff, EFX_DWORD_1, 0xffffffff,           \
 889             EFX_DWORD_2, 0xffffffff, EFX_DWORD_3, 0xffffffff)
 890
 891 /* Populate a quadword field with various numbers of arguments */
 892 #define EFX_POPULATE_QWORD_10 EFX_POPULATE_QWORD
 893
 894 #define EFX_POPULATE_QWORD_9(_qword,                                    \
 895             _field1, _value1, _field2, _value2, _field3, _value3,       \
 896             _field4, _value4, _field5, _value5, _field6, _value6,       \
 897             _field7, _value7, _field8, _value8, _field9, _value9)       \
 898         EFX_POPULATE_QWORD_10(_qword, EFX_DUMMY_FIELD, 0,               \
 899             _field1, _value1, _field2, _value2, _field3, _value3,       \
 900             _field4, _value4, _field5, _value5, _field6, _value6,       \
 901             _field7, _value7, _field8, _value8, _field9, _value9)
 902
 903 #define EFX_POPULATE_QWORD_8(_qword,                                    \
 904             _field1, _value1, _field2, _value2, _field3, _value3,       \
 905             _field4, _value4, _field5, _value5, _field6, _value6,       \
 906             _field7, _value7, _field8, _value8)                         \
 907         EFX_POPULATE_QWORD_9(_qword, EFX_DUMMY_FIELD, 0,                \
 908             _field1, _value1, _field2, _value2, _field3, _value3,       \
 909             _field4, _value4, _field5, _value5, _field6, _value6,       \
 910             _field7, _value7, _field8, _value8)
 911
 912 #define EFX_POPULATE_QWORD_7(_qword,                                    \
 913             _field1, _value1, _field2, _value2, _field3, _value3,       \
 914             _field4, _value4, _field5, _value5, _field6, _value6,       \
 915             _field7, _value7)                                           \
 916         EFX_POPULATE_QWORD_8(_qword, EFX_DUMMY_FIELD, 0,                \
 917             _field1, _value1, _field2, _value2, _field3, _value3,       \
 918             _field4, _value4, _field5, _value5, _field6, _value6,       \
 919             _field7, _value7)
 920
 921 #define EFX_POPULATE_QWORD_6(_qword,                                    \
 922             _field1, _value1, _field2, _value2, _field3, _value3,       \
 923             _field4, _value4, _field5, _value5, _field6, _value6)       \
 924         EFX_POPULATE_QWORD_7(_qword, EFX_DUMMY_FIELD, 0,                \
 925             _field1, _value1, _field2, _value2, _field3, _value3,       \
 926             _field4, _value4, _field5, _value5, _field6, _value6)
 927
 928 #define EFX_POPULATE_QWORD_5(_qword,                                    \
 929             _field1, _value1, _field2, _value2, _field3, _value3,       \
 930             _field4, _value4, _field5, _value5)                         \
 931         EFX_POPULATE_QWORD_6(_qword, EFX_DUMMY_FIELD, 0,                \
 932             _field1, _value1, _field2, _value2, _field3, _value3,       \
 933             _field4, _value4, _field5, _value5)
 934
 935 #define EFX_POPULATE_QWORD_4(_qword,                                    \
 936             _field1, _value1, _field2, _value2, _field3, _value3,       \
 937             _field4, _value4)                                           \
 938         EFX_POPULATE_QWORD_5(_qword, EFX_DUMMY_FIELD, 0,                \
 939             _field1, _value1, _field2, _value2, _field3, _value3,       \
 940             _field4, _value4)
 941
 942 #define EFX_POPULATE_QWORD_3(_qword,                                    \
 943             _field1, _value1, _field2, _value2, _field3, _value3)       \
 944         EFX_POPULATE_QWORD_4(_qword, EFX_DUMMY_FIELD, 0,                \
 945             _field1, _value1, _field2, _value2, _field3, _value3)
 946
 947 #define EFX_POPULATE_QWORD_2(_qword,                                    \
 948             _field1, _value1, _field2, _value2)                         \
 949         EFX_POPULATE_QWORD_3(_qword, EFX_DUMMY_FIELD, 0,                \
 950             _field1, _value1, _field2, _value2)
 951
 952 #define EFX_POPULATE_QWORD_1(_qword,                                    \
 953             _field1, _value1)                                           \
 954         EFX_POPULATE_QWORD_2(_qword, EFX_DUMMY_FIELD, 0,                \
 955             _field1, _value1)
 956
 957 #define EFX_ZERO_QWORD(_qword)                                          \
 958         EFX_POPULATE_QWORD_1(_qword, EFX_DUMMY_FIELD, 0)
 959
 960 #define EFX_SET_QWORD(_qword)                                           \
 961         EFX_POPULATE_QWORD_2(_qword,                                    \
 962             EFX_DWORD_0, 0xffffffff, EFX_DWORD_1, 0xffffffff)
 963
 964 /* Populate a dword field with various numbers of arguments */
 965 #define EFX_POPULATE_DWORD_10 EFX_POPULATE_DWORD
 966
 967 #define EFX_POPULATE_DWORD_9(_dword,                                    \
 968             _field1, _value1, _field2, _value2, _field3, _value3,       \
 969             _field4, _value4, _field5, _value5, _field6, _value6,       \
 970             _field7, _value7, _field8, _value8, _field9, _value9)       \
 971         EFX_POPULATE_DWORD_10(_dword, EFX_DUMMY_FIELD, 0,               \
 972             _field1, _value1, _field2, _value2, _field3, _value3,       \
 973             _field4, _value4, _field5, _value5, _field6, _value6,       \
 974             _field7, _value7, _field8, _value8, _field9, _value9)
 975
 976 #define EFX_POPULATE_DWORD_8(_dword,                                    \
 977             _field1, _value1, _field2, _value2, _field3, _value3,       \
 978             _field4, _value4, _field5, _value5, _field6, _value6,       \
 979             _field7, _value7, _field8, _value8)                         \
 980         EFX_POPULATE_DWORD_9(_dword, EFX_DUMMY_FIELD, 0,                \
 981             _field1, _value1, _field2, _value2, _field3, _value3,       \
 982             _field4, _value4, _field5, _value5, _field6, _value6,       \
 983             _field7, _value7, _field8, _value8)
 984
 985 #define EFX_POPULATE_DWORD_7(_dword,                                    \
 986             _field1, _value1, _field2, _value2, _field3, _value3,       \
 987             _field4, _value4, _field5, _value5, _field6, _value6,       \
 988             _field7, _value7)                                           \
 989         EFX_POPULATE_DWORD_8(_dword, EFX_DUMMY_FIELD, 0,                \
 990             _field1, _value1, _field2, _value2, _field3, _value3,       \
 991             _field4, _value4, _field5, _value5, _field6, _value6,       \
 992             _field7, _value7)
 993
 994 #define EFX_POPULATE_DWORD_6(_dword,                                    \
 995             _field1, _value1, _field2, _value2, _field3, _value3,       \
 996             _field4, _value4, _field5, _value5, _field6, _value6)       \
 997         EFX_POPULATE_DWORD_7(_dword, EFX_DUMMY_FIELD, 0,                \
 998             _field1, _value1, _field2, _value2, _field3, _value3,       \
 999             _field4, _value4, _field5, _value5, _field6, _value6)
1000
1001 #define EFX_POPULATE_DWORD_5(_dword,                                    \
1002             _field1, _value1, _field2, _value2, _field3, _value3,       \
1003             _field4, _value4, _field5, _value5)                         \
1004         EFX_POPULATE_DWORD_6(_dword, EFX_DUMMY_FIELD, 0,                \
1005             _field1, _value1, _field2, _value2, _field3, _value3,       \
1006             _field4, _value4, _field5, _value5)
1007
1008 #define EFX_POPULATE_DWORD_4(_dword,                                    \
1009             _field1, _value1, _field2, _value2, _field3, _value3,       \
1010             _field4, _value4)                                           \
1011         EFX_POPULATE_DWORD_5(_dword, EFX_DUMMY_FIELD, 0,                \
1012             _field1, _value1, _field2, _value2, _field3, _value3,       \
1013             _field4, _value4)
1014
1015 #define EFX_POPULATE_DWORD_3(_dword,                                    \
1016             _field1, _value1, _field2, _value2, _field3, _value3)       \
1017         EFX_POPULATE_DWORD_4(_dword, EFX_DUMMY_FIELD, 0,                \
1018             _field1, _value1, _field2, _value2, _field3, _value3)
1019
1020 #define EFX_POPULATE_DWORD_2(_dword,                                    \
1021             _field1, _value1, _field2, _value2)                         \
1022         EFX_POPULATE_DWORD_3(_dword, EFX_DUMMY_FIELD, 0,                \
1023             _field1, _value1, _field2, _value2)
1024
1025 #define EFX_POPULATE_DWORD_1(_dword,                                    \
1026             _field1, _value1)                                           \
1027         EFX_POPULATE_DWORD_2(_dword, EFX_DUMMY_FIELD, 0,                \
1028             _field1, _value1)
1029
1030 #define EFX_ZERO_DWORD(_dword)                                          \
1031         EFX_POPULATE_DWORD_1(_dword, EFX_DUMMY_FIELD, 0)
1032
1033 #define EFX_SET_DWORD(_dword)                                           \
1034         EFX_POPULATE_DWORD_1(_dword,                                    \
1035             EFX_DWORD_0, 0xffffffff)
1036
1037 /* Populate a word field with various numbers of arguments */
1038 #define EFX_POPULATE_WORD_10 EFX_POPULATE_WORD
1039
1040 #define EFX_POPULATE_WORD_9(_word,                                      \
1041             _field1, _value1, _field2, _value2, _field3, _value3,       \
1042             _field4, _value4, _field5, _value5, _field6, _value6,       \
1043             _field7, _value7, _field8, _value8, _field9, _value9)       \
1044         EFX_POPULATE_WORD_10(_word, EFX_DUMMY_FIELD, 0,                 \
1045             _field1, _value1, _field2, _value2, _field3, _value3,       \
1046             _field4, _value4, _field5, _value5, _field6, _value6,       \
1047             _field7, _value7, _field8, _value8, _field9, _value9)
1048
1049 #define EFX_POPULATE_WORD_8(_word,                                      \
1050             _field1, _value1, _field2, _value2, _field3, _value3,       \
1051             _field4, _value4, _field5, _value5, _field6, _value6,       \
1052             _field7, _value7, _field8, _value8)                         \
1053         EFX_POPULATE_WORD_9(_word, EFX_DUMMY_FIELD, 0,                  \
1054             _field1, _value1, _field2, _value2, _field3, _value3,       \
1055             _field4, _value4, _field5, _value5, _field6, _value6,       \
1056             _field7, _value7, _field8, _value8)
1057
1058 #define EFX_POPULATE_WORD_7(_word,                                      \
1059             _field1, _value1, _field2, _value2, _field3, _value3,       \
1060             _field4, _value4, _field5, _value5, _field6, _value6,       \
1061             _field7, _value7)                                           \
1062         EFX_POPULATE_WORD_8(_word, EFX_DUMMY_FIELD, 0,                  \
1063             _field1, _value1, _field2, _value2, _field3, _value3,       \
1064             _field4, _value4, _field5, _value5, _field6, _value6,       \
1065             _field7, _value7)
1066
1067 #define EFX_POPULATE_WORD_6(_word,                                      \
1068             _field1, _value1, _field2, _value2, _field3, _value3,       \
1069             _field4, _value4, _field5, _value5, _field6, _value6)       \
1070         EFX_POPULATE_WORD_7(_word, EFX_DUMMY_FIELD, 0,                  \
1071             _field1, _value1, _field2, _value2, _field3, _value3,       \
1072             _field4, _value4, _field5, _value5, _field6, _value6)
1073
1074 #define EFX_POPULATE_WORD_5(_word,                                      \
1075             _field1, _value1, _field2, _value2, _field3, _value3,       \
1076             _field4, _value4, _field5, _value5)                         \
1077         EFX_POPULATE_WORD_6(_word, EFX_DUMMY_FIELD, 0,                  \
1078             _field1, _value1, _field2, _value2, _field3, _value3,       \
1079             _field4, _value4, _field5, _value5)
1080
1081 #define EFX_POPULATE_WORD_4(_word,                                      \
1082             _field1, _value1, _field2, _value2, _field3, _value3,       \
1083             _field4, _value4)                                           \
1084         EFX_POPULATE_WORD_5(_word, EFX_DUMMY_FIELD, 0,                  \
1085             _field1, _value1, _field2, _value2, _field3, _value3,       \
1086             _field4, _value4)
1087
1088 #define EFX_POPULATE_WORD_3(_word,                                      \
1089             _field1, _value1, _field2, _value2, _field3, _value3)       \
1090         EFX_POPULATE_WORD_4(_word, EFX_DUMMY_FIELD, 0,                  \
1091             _field1, _value1, _field2, _value2, _field3, _value3)
1092
1093 #define EFX_POPULATE_WORD_2(_word,                                      \
1094             _field1, _value1, _field2, _value2)                         \
1095         EFX_POPULATE_WORD_3(_word, EFX_DUMMY_FIELD, 0,                  \
1096             _field1, _value1, _field2, _value2)
1097
1098 #define EFX_POPULATE_WORD_1(_word,                                      \
1099             _field1, _value1)                                           \
1100         EFX_POPULATE_WORD_2(_word, EFX_DUMMY_FIELD, 0,                  \
1101             _field1, _value1)
1102
1103 #define EFX_ZERO_WORD(_word)                                            \
1104         EFX_POPULATE_WORD_1(_word, EFX_DUMMY_FIELD, 0)
1105
1106 #define EFX_SET_WORD(_word)                                             \
1107         EFX_POPULATE_WORD_1(_word,                                      \
1108             EFX_WORD_0, 0xffff)
1109
1110 /* Populate a byte field with various numbers of arguments */
1111 #define EFX_POPULATE_BYTE_10 EFX_POPULATE_BYTE
1112
1113 #define EFX_POPULATE_BYTE_9(_byte,                                      \
1114             _field1, _value1, _field2, _value2, _field3, _value3,       \
1115             _field4, _value4, _field5, _value5, _field6, _value6,       \
1116             _field7, _value7, _field8, _value8, _field9, _value9)       \
1117         EFX_POPULATE_BYTE_10(_byte, EFX_DUMMY_FIELD, 0,                 \
1118             _field1, _value1, _field2, _value2, _field3, _value3,       \
1119             _field4, _value4, _field5, _value5, _field6, _value6,       \
1120             _field7, _value7, _field8, _value8, _field9, _value9)
1121
1122 #define EFX_POPULATE_BYTE_8(_byte,                                      \
1123             _field1, _value1, _field2, _value2, _field3, _value3,       \
1124             _field4, _value4, _field5, _value5, _field6, _value6,       \
1125             _field7, _value7, _field8, _value8)                         \
1126         EFX_POPULATE_BYTE_9(_byte, EFX_DUMMY_FIELD, 0,                  \
1127             _field1, _value1, _field2, _value2, _field3, _value3,       \
1128             _field4, _value4, _field5, _value5, _field6, _value6,       \
1129             _field7, _value7, _field8, _value8)
1130
1131 #define EFX_POPULATE_BYTE_7(_byte,                                      \
1132             _field1, _value1, _field2, _value2, _field3, _value3,       \
1133             _field4, _value4, _field5, _value5, _field6, _value6,       \
1134             _field7, _value7)                                           \
1135         EFX_POPULATE_BYTE_8(_byte, EFX_DUMMY_FIELD, 0,                  \
1136             _field1, _value1, _field2, _value2, _field3, _value3,       \
1137             _field4, _value4, _field5, _value5, _field6, _value6,       \
1138             _field7, _value7)
1139
1140 #define EFX_POPULATE_BYTE_6(_byte,                                      \
1141             _field1, _value1, _field2, _value2, _field3, _value3,       \
1142             _field4, _value4, _field5, _value5, _field6, _value6)       \
1143         EFX_POPULATE_BYTE_7(_byte, EFX_DUMMY_FIELD, 0,                  \
1144             _field1, _value1, _field2, _value2, _field3, _value3,       \
1145             _field4, _value4, _field5, _value5, _field6, _value6)
1146
1147 #define EFX_POPULATE_BYTE_5(_byte,                                      \
1148             _field1, _value1, _field2, _value2, _field3, _value3,       \
1149             _field4, _value4, _field5, _value5)                         \
1150         EFX_POPULATE_BYTE_6(_byte, EFX_DUMMY_FIELD, 0,                  \
1151             _field1, _value1, _field2, _value2, _field3, _value3,       \
1152             _field4, _value4, _field5, _value5)
1153
1154 #define EFX_POPULATE_BYTE_4(_byte,                                      \
1155             _field1, _value1, _field2, _value2, _field3, _value3,       \
1156             _field4, _value4)                                           \
1157         EFX_POPULATE_BYTE_5(_byte, EFX_DUMMY_FIELD, 0,                  \
1158             _field1, _value1, _field2, _value2, _field3, _value3,       \
1159             _field4, _value4)
1160
1161 #define EFX_POPULATE_BYTE_3(_byte,                                      \
1162             _field1, _value1, _field2, _value2, _field3, _value3)       \
1163         EFX_POPULATE_BYTE_4(_byte, EFX_DUMMY_FIELD, 0,                  \
1164             _field1, _value1, _field2, _value2, _field3, _value3)
1165
1166 #define EFX_POPULATE_BYTE_2(_byte,                                      \
1167             _field1, _value1, _field2, _value2)                         \
1168         EFX_POPULATE_BYTE_3(_byte, EFX_DUMMY_FIELD, 0,                  \
1169             _field1, _value1, _field2, _value2)
1170
1171 #define EFX_POPULATE_BYTE_1(_byte,                                      \
1172             _field1, _value1)                                           \
1173         EFX_POPULATE_BYTE_2(_byte, EFX_DUMMY_FIELD, 0,                  \
1174             _field1, _value1)
1175
1176 #define EFX_ZERO_BYTE(_byte)                                            \
1177         EFX_POPULATE_BYTE_1(_byte, EFX_DUMMY_FIELD, 0)
1178
1179 #define EFX_SET_BYTE(_byte)                                             \
1180         EFX_POPULATE_BYTE_1(_byte,                                      \
1181             EFX_BYTE_0, 0xff)
1182
1183 /*
1184  * Modify a named field within an already-populated structure.  Used
1185  * for read-modify-write operations.
1186  */
1187
1188 #define EFX_INSERT_FIELD64(_min, _max, _field, _value)                  \
1189         __CPU_TO_LE_64(EFX_INSERT_FIELD_NATIVE64(_min, _max, _field, _value))
1190
1191 #define EFX_INSERT_FIELD32(_min, _max, _field, _value)                  \
1192         __CPU_TO_LE_32(EFX_INSERT_FIELD_NATIVE32(_min, _max, _field, _value))
1193
1194 #define EFX_INSERT_FIELD16(_min, _max, _field, _value)                  \
1195         __CPU_TO_LE_16(EFX_INSERT_FIELD_NATIVE16(_min, _max, _field, _value))
1196
1197 #define EFX_INSERT_FIELD8(_min, _max, _field, _value)                   \
1198         __NATIVE_8(EFX_INSERT_FIELD_NATIVE8(_min, _max, _field, _value))
1199
1200 #define EFX_INPLACE_MASK64(_min, _max, _field)                          \
1201         EFX_INSERT_FIELD64(_min, _max, _field, EFX_MASK64(_field))
1202
1203 #define EFX_INPLACE_MASK32(_min, _max, _field)                          \
1204         EFX_INSERT_FIELD32(_min, _max, _field, EFX_MASK32(_field))
1205
1206 #define EFX_INPLACE_MASK16(_min, _max, _field)                          \
1207         EFX_INSERT_FIELD16(_min, _max, _field, EFX_MASK16(_field))
1208
1209 #define EFX_INPLACE_MASK8(_min, _max, _field)                           \
1210         EFX_INSERT_FIELD8(_min, _max, _field, EFX_MASK8(_field))
1211
1212 #define EFX_SET_OWORD_FIELD64(_oword, _field, _value)                   \
1213         do {                                                            \
1214                 _NOTE(CONSTANTCONDITION)                                \
1215                 (_oword).eo_u64[0] = (((_oword).eo_u64[0] &             \
1216                     ~EFX_INPLACE_MASK64(0, 63, _field)) |               \
1217                     EFX_INSERT_FIELD64(0, 63, _field, _value));         \
1218                 _NOTE(CONSTANTCONDITION)                                \
1219                 (_oword).eo_u64[1] = (((_oword).eo_u64[1] &             \
1220                     ~EFX_INPLACE_MASK64(64, 127, _field)) |             \
1221                     EFX_INSERT_FIELD64(64, 127, _field, _value));       \
1222         _NOTE(CONSTANTCONDITION)                                        \
1223         } while (B_FALSE)
1224
1225 #define EFX_SET_OWORD_FIELD32(_oword, _field, _value)                   \
1226         do {                                                            \
1227                 _NOTE(CONSTANTCONDITION)                                \
1228                 (_oword).eo_u32[0] = (((_oword).eo_u32[0] &             \
1229                     ~EFX_INPLACE_MASK32(0, 31, _field)) |               \
1230                     EFX_INSERT_FIELD32(0, 31, _field, _value));         \
1231                 _NOTE(CONSTANTCONDITION)                                \
1232                 (_oword).eo_u32[1] = (((_oword).eo_u32[1] &             \
1233                     ~EFX_INPLACE_MASK32(32, 63, _field)) |              \
1234                     EFX_INSERT_FIELD32(32, 63, _field, _value));        \
1235                 _NOTE(CONSTANTCONDITION)                                \
1236                 (_oword).eo_u32[2] = (((_oword).eo_u32[2] &             \
1237                     ~EFX_INPLACE_MASK32(64, 95, _field)) |              \
1238                     EFX_INSERT_FIELD32(64, 95, _field, _value));        \
1239                 _NOTE(CONSTANTCONDITION)                                \
1240                 (_oword).eo_u32[3] = (((_oword).eo_u32[3] &             \
1241                     ~EFX_INPLACE_MASK32(96, 127, _field)) |             \
1242                     EFX_INSERT_FIELD32(96, 127, _field, _value));       \
1243         _NOTE(CONSTANTCONDITION)                                        \
1244         } while (B_FALSE)
1245
1246 #define EFX_SET_QWORD_FIELD64(_qword, _field, _value)                   \
1247         do {                                                            \
1248                 _NOTE(CONSTANTCONDITION)                                \
1249                 (_qword).eq_u64[0] = (((_qword).eq_u64[0] &             \
1250                     ~EFX_INPLACE_MASK64(0, 63, _field)) |               \
1251                     EFX_INSERT_FIELD64(0, 63, _field, _value));         \
1252         _NOTE(CONSTANTCONDITION)                                        \
1253         } while (B_FALSE)
1254
1255 #define EFX_SET_QWORD_FIELD32(_qword, _field, _value)                   \
1256         do {                                                            \
1257                 _NOTE(CONSTANTCONDITION)                                \
1258                 (_qword).eq_u32[0] = (((_qword).eq_u32[0] &             \
1259                     ~EFX_INPLACE_MASK32(0, 31, _field)) |               \
1260                     EFX_INSERT_FIELD32(0, 31, _field, _value));         \
1261                 _NOTE(CONSTANTCONDITION)                                \
1262                 (_qword).eq_u32[1] = (((_qword).eq_u32[1] &             \
1263                     ~EFX_INPLACE_MASK32(32, 63, _field)) |              \
1264                     EFX_INSERT_FIELD32(32, 63, _field, _value));        \
1265         _NOTE(CONSTANTCONDITION)                                        \
1266         } while (B_FALSE)
1267
1268 #define EFX_SET_DWORD_FIELD(_dword, _field, _value)                     \
1269         do {                                                            \
1270                 _NOTE(CONSTANTCONDITION)                                \
1271                 (_dword).ed_u32[0] = (((_dword).ed_u32[0] &             \
1272                     ~EFX_INPLACE_MASK32(0, 31, _field)) |               \
1273                     EFX_INSERT_FIELD32(0, 31, _field, _value));         \
1274         _NOTE(CONSTANTCONDITION)                                        \
1275         } while (B_FALSE)
1276
1277 #define EFX_SET_WORD_FIELD(_word, _field, _value)                       \
1278         do {                                                            \
1279                 _NOTE(CONSTANTCONDITION)                                \
1280                 (_word).ew_u16[0] = (((_word).ew_u16[0] &               \
1281                     ~EFX_INPLACE_MASK16(0, 15, _field)) |               \
1282                     EFX_INSERT_FIELD16(0, 15, _field, _value));         \
1283         _NOTE(CONSTANTCONDITION)                                        \
1284         } while (B_FALSE)
1285
1286 #define EFX_SET_BYTE_FIELD(_byte, _field, _value)                       \
1287         do {                                                            \
1288                 _NOTE(CONSTANTCONDITION)                                \
1289                 (_byte).eb_u8[0] = (((_byte).eb_u8[0] &                 \
1290                     ~EFX_INPLACE_MASK8(0, 7, _field)) |                 \
1291                     EFX_INSERT_FIELD8(0, 7, _field, _value));           \
1292         _NOTE(CONSTANTCONDITION)                                        \
1293         } while (B_FALSE)
1294
1295 /*
1296  * Set or clear a numbered bit within an octword.
1297  */
1298
1299 #define EFX_SHIFT64(_bit, _base)                                        \
1300         (((_bit) >= (_base) && (_bit) < (_base) + 64) ?                 \
1301                 ((uint64_t)1 << EFX_SSUB((_bit), (_base))) :            \
1302                 0U)
1303
1304 #define EFX_SHIFT32(_bit, _base)                                        \
1305         (((_bit) >= (_base) && (_bit) < (_base) + 32) ?                 \
1306                 ((uint32_t)1 << EFX_SSUB((_bit),(_base))) :             \
1307                 0U)
1308
1309 #define EFX_SHIFT16(_bit, _base)                                        \
1310         (((_bit) >= (_base) && (_bit) < (_base) + 16) ?                 \
1311                 (uint16_t)(1 << EFX_SSUB((_bit), (_base))) :            \
1312                 0U)
1313
1314 #define EFX_SHIFT8(_bit, _base)                                         \
1315         (((_bit) >= (_base) && (_bit) < (_base) + 8) ?                  \
1316                 (uint8_t)(1 << EFX_SSUB((_bit), (_base))) :             \
1317                 0U)
1318
1319 #define EFX_SET_OWORD_BIT64(_oword, _bit)                               \
1320         do {                                                            \
1321                 _NOTE(CONSTANTCONDITION)                                \
1322                 (_oword).eo_u64[0] |=                                   \
1323                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)));     \
1324                 (_oword).eo_u64[1] |=                                   \
1325                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(64)));    \
1326         _NOTE(CONSTANTCONDITION)                                        \
1327         } while (B_FALSE)
1328
1329 #define EFX_SET_OWORD_BIT32(_oword, _bit)                               \
1330         do {                                                            \
1331                 _NOTE(CONSTANTCONDITION)                                \
1332                 (_oword).eo_u32[0] |=                                   \
1333                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));     \
1334                 (_oword).eo_u32[1] |=                                   \
1335                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)));    \
1336                 (_oword).eo_u32[2] |=                                   \
1337                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(64)));    \
1338                 (_oword).eo_u32[3] |=                                   \
1339                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(96)));    \
1340         _NOTE(CONSTANTCONDITION)                                        \
1341         } while (B_FALSE)
1342
1343 #define EFX_CLEAR_OWORD_BIT64(_oword, _bit)                             \
1344         do {                                                            \
1345                 _NOTE(CONSTANTCONDITION)                                \
1346                 (_oword).eo_u64[0] &=                                   \
1347                     __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(0)));    \
1348                 (_oword).eo_u64[1] &=                                   \
1349                     __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(64)));   \
1350         _NOTE(CONSTANTCONDITION)                                        \
1351         } while (B_FALSE)
1352
1353 #define EFX_CLEAR_OWORD_BIT32(_oword, _bit)                             \
1354         do {                                                            \
1355                 _NOTE(CONSTANTCONDITION)                                \
1356                 (_oword).eo_u32[0] &=                                   \
1357                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));    \
1358                 (_oword).eo_u32[1] &=                                   \
1359                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(32)));   \
1360                 (_oword).eo_u32[2] &=                                   \
1361                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(64)));   \
1362                 (_oword).eo_u32[3] &=                                   \
1363                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(96)));   \
1364         _NOTE(CONSTANTCONDITION)                                        \
1365         } while (B_FALSE)
1366
1367 #define EFX_TEST_OWORD_BIT64(_oword, _bit)                              \
1368         (((_oword).eo_u64[0] &                                          \
1369                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)))) ||  \
1370         ((_oword).eo_u64[1] &                                           \
1371                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(64)))))
1372
1373 #define EFX_TEST_OWORD_BIT32(_oword, _bit)                              \
1374         (((_oword).eo_u32[0] &                                          \
1375                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)))) ||  \
1376         ((_oword).eo_u32[1] &                                           \
1377                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)))) || \
1378         ((_oword).eo_u32[2] &                                           \
1379                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(64)))) || \
1380         ((_oword).eo_u32[3] &                                           \
1381                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(96)))))
1382
1383
1384 #define EFX_SET_QWORD_BIT64(_qword, _bit)                               \
1385         do {                                                            \
1386                 _NOTE(CONSTANTCONDITION)                                \
1387                 (_qword).eq_u64[0] |=                                   \
1388                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)));     \
1389         _NOTE(CONSTANTCONDITION)                                        \
1390         } while (B_FALSE)
1391
1392 #define EFX_SET_QWORD_BIT32(_qword, _bit)                               \
1393         do {                                                            \
1394                 _NOTE(CONSTANTCONDITION)                                \
1395                 (_qword).eq_u32[0] |=                                   \
1396                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));     \
1397                 (_qword).eq_u32[1] |=                                   \
1398                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)));    \
1399         _NOTE(CONSTANTCONDITION)                                        \
1400         } while (B_FALSE)
1401
1402 #define EFX_CLEAR_QWORD_BIT64(_qword, _bit)                             \
1403         do {                                                            \
1404                 _NOTE(CONSTANTCONDITION)                                \
1405                 (_qword).eq_u64[0] &=                                   \
1406                     __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(0)));    \
1407         _NOTE(CONSTANTCONDITION)                                        \
1408         } while (B_FALSE)
1409
1410 #define EFX_CLEAR_QWORD_BIT32(_qword, _bit)                             \
1411         do {                                                            \
1412                 _NOTE(CONSTANTCONDITION)                                \
1413                 (_qword).eq_u32[0] &=                                   \
1414                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));    \
1415                 (_qword).eq_u32[1] &=                                   \
1416                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(32)));   \
1417         _NOTE(CONSTANTCONDITION)                                        \
1418         } while (B_FALSE)
1419
1420 #define EFX_TEST_QWORD_BIT64(_qword, _bit)                              \
1421         (((_qword).eq_u64[0] &                                          \
1422                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)))) != 0)
1423
1424 #define EFX_TEST_QWORD_BIT32(_qword, _bit)                              \
1425         (((_qword).eq_u32[0] &                                          \
1426                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)))) ||  \
1427         ((_qword).eq_u32[1] &                                           \
1428                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)))))
1429
1430
1431 #define EFX_SET_DWORD_BIT(_dword, _bit)                                 \
1432         do {                                                            \
1433                 (_dword).ed_u32[0] |=                                   \
1434                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));     \
1435         _NOTE(CONSTANTCONDITION)                                        \
1436         } while (B_FALSE)
1437
1438 #define EFX_CLEAR_DWORD_BIT(_dword, _bit)                               \
1439         do {                                                            \
1440                 (_dword).ed_u32[0] &=                                   \
1441                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));    \
1442         _NOTE(CONSTANTCONDITION)                                        \
1443         } while (B_FALSE)
1444
1445 #define EFX_TEST_DWORD_BIT(_dword, _bit)                                \
1446         (((_dword).ed_u32[0] &                                          \
1447                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)))) != 0)
1448
1449
1450 #define EFX_SET_WORD_BIT(_word, _bit)                                   \
1451         do {                                                            \
1452                 (_word).ew_u16[0] |=                                    \
1453                     __CPU_TO_LE_16(EFX_SHIFT16(_bit, FIX_LINT(0)));     \
1454         _NOTE(CONSTANTCONDITION)                                        \
1455         } while (B_FALSE)
1456
1457 #define EFX_CLEAR_WORD_BIT(_word, _bit)                                 \
1458         do {                                                            \
1459                 (_word).ew_u32[0] &=                                    \
1460                     __CPU_TO_LE_16(~EFX_SHIFT16(_bit, FIX_LINT(0)));    \
1461         _NOTE(CONSTANTCONDITION)                                        \
1462         } while (B_FALSE)
1463
1464 #define EFX_TEST_WORD_BIT(_word, _bit)                                  \
1465         (((_word).ew_u16[0] &                                           \
1466                     __CPU_TO_LE_16(EFX_SHIFT16(_bit, FIX_LINT(0)))) != 0)
1467
1468
1469 #define EFX_SET_BYTE_BIT(_byte, _bit)                                   \
1470         do {                                                            \
1471                 (_byte).eb_u8[0] |=                                     \
1472                     __NATIVE_8(EFX_SHIFT8(_bit, FIX_LINT(0)));          \
1473         _NOTE(CONSTANTCONDITION)                                        \
1474         } while (B_FALSE)
1475
1476 #define EFX_CLEAR_BYTE_BIT(_byte, _bit)                                 \
1477         do {                                                            \
1478                 (_byte).eb_u8[0] &=                                     \
1479                     __NATIVE_8(~EFX_SHIFT8(_bit, FIX_LINT(0)));         \
1480         _NOTE(CONSTANTCONDITION)                                        \
1481         } while (B_FALSE)
1482
1483 #define EFX_TEST_BYTE_BIT(_byte, _bit)                                  \
1484         (((_byte).eb_u8[0] &                                            \
1485                     __NATIVE_8(EFX_SHIFT8(_bit, FIX_LINT(0)))) != 0)
1486
1487
1488 #define EFX_OR_OWORD64(_oword1, _oword2)                                \
1489         do {                                                            \
1490                 (_oword1).eo_u64[0] |= (_oword2).eo_u64[0];             \
1491                 (_oword1).eo_u64[1] |= (_oword2).eo_u64[1];             \
1492         _NOTE(CONSTANTCONDITION)                                        \
1493         } while (B_FALSE)
1494
1495 #define EFX_OR_OWORD32(_oword1, _oword2)                                \
1496         do {                                                            \
1497                 (_oword1).eo_u32[0] |= (_oword2).eo_u32[0];             \
1498                 (_oword1).eo_u32[1] |= (_oword2).eo_u32[1];             \
1499                 (_oword1).eo_u32[2] |= (_oword2).eo_u32[2];             \
1500                 (_oword1).eo_u32[3] |= (_oword2).eo_u32[3];             \
1501         _NOTE(CONSTANTCONDITION)                                        \
1502         } while (B_FALSE)
1503
1504 #define EFX_AND_OWORD64(_oword1, _oword2)                               \
1505         do {                                                            \
1506                 (_oword1).eo_u64[0] &= (_oword2).eo_u64[0];             \
1507                 (_oword1).eo_u64[1] &= (_oword2).eo_u64[1];             \
1508         _NOTE(CONSTANTCONDITION)                                        \
1509         } while (B_FALSE)
1510
1511 #define EFX_AND_OWORD32(_oword1, _oword2)                               \
1512         do {                                                            \
1513                 (_oword1).eo_u32[0] &= (_oword2).eo_u32[0];             \
1514                 (_oword1).eo_u32[1] &= (_oword2).eo_u32[1];             \
1515                 (_oword1).eo_u32[2] &= (_oword2).eo_u32[2];             \
1516                 (_oword1).eo_u32[3] &= (_oword2).eo_u32[3];             \
1517         _NOTE(CONSTANTCONDITION)                                        \
1518         } while (B_FALSE)
1519
1520 #define EFX_OR_QWORD64(_qword1, _qword2)                                \
1521         do {                                                            \
1522                 (_qword1).eq_u64[0] |= (_qword2).eq_u64[0];             \
1523         _NOTE(CONSTANTCONDITION)                                        \
1524         } while (B_FALSE)
1525
1526 #define EFX_OR_QWORD32(_qword1, _qword2)                                \
1527         do {                                                            \
1528                 (_qword1).eq_u32[0] |= (_qword2).eq_u32[0];             \
1529                 (_qword1).eq_u32[1] |= (_qword2).eq_u32[1];             \
1530         _NOTE(CONSTANTCONDITION)                                        \
1531         } while (B_FALSE)
1532
1533 #define EFX_AND_QWORD64(_qword1, _qword2)                               \
1534         do {                                                            \
1535                 (_qword1).eq_u64[0] &= (_qword2).eq_u64[0];             \
1536         _NOTE(CONSTANTCONDITION)                                        \
1537         } while (B_FALSE)
1538
1539 #define EFX_AND_QWORD32(_qword1, _qword2)                               \
1540         do {                                                            \
1541                 (_qword1).eq_u32[0] &= (_qword2).eq_u32[0];             \
1542                 (_qword1).eq_u32[1] &= (_qword2).eq_u32[1];             \
1543         _NOTE(CONSTANTCONDITION)                                        \
1544         } while (B_FALSE)
1545
1546 #define EFX_OR_DWORD(_dword1, _dword2)                                  \
1547         do {                                                            \
1548                 (_dword1).ed_u32[0] |= (_dword2).ed_u32[0];             \
1549         _NOTE(CONSTANTCONDITION)                                        \
1550         } while (B_FALSE)
1551
1552 #define EFX_AND_DWORD(_dword1, _dword2)                                 \
1553         do {                                                            \
1554                 (_dword1).ed_u32[0] &= (_dword2).ed_u32[0];             \
1555         _NOTE(CONSTANTCONDITION)                                        \
1556         } while (B_FALSE)
1557
1558 #define EFX_OR_WORD(_word1, _word2)                                     \
1559         do {                                                            \
1560                 (_word1).ew_u16[0] |= (_word2).ew_u16[0];               \
1561         _NOTE(CONSTANTCONDITION)                                        \
1562         } while (B_FALSE)
1563
1564 #define EFX_AND_WORD(_word1, _word2)                                    \
1565         do {                                                            \
1566                 (_word1).ew_u16[0] &= (_word2).ew_u16[0];               \
1567         _NOTE(CONSTANTCONDITION)                                        \
1568         } while (B_FALSE)
1569
1570 #define EFX_OR_BYTE(_byte1, _byte2)                                     \
1571         do {                                                            \
1572                 (_byte1).eb_u8[0] |= (_byte2).eb_u8[0];                 \
1573         _NOTE(CONSTANTCONDITION)                                        \
1574         } while (B_FALSE)
1575
1576 #define EFX_AND_BYTE(_byte1, _byte2)                                    \
1577         do {                                                            \
1578                 (_byte1).eb_u8[0] &= (_byte2).eb_u8[0];                 \
1579         _NOTE(CONSTANTCONDITION)                                        \
1580         } while (B_FALSE)
1581
1582 #if EFSYS_USE_UINT64
1583 #define EFX_OWORD_FIELD         EFX_OWORD_FIELD64
1584 #define EFX_QWORD_FIELD         EFX_QWORD_FIELD64
1585 #define EFX_OWORD_IS_EQUAL      EFX_OWORD_IS_EQUAL64
1586 #define EFX_QWORD_IS_EQUAL      EFX_QWORD_IS_EQUAL64
1587 #define EFX_OWORD_IS_ZERO       EFX_OWORD_IS_ZERO64
1588 #define EFX_QWORD_IS_ZERO       EFX_QWORD_IS_ZERO64
1589 #define EFX_OWORD_IS_SET        EFX_OWORD_IS_SET64
1590 #define EFX_QWORD_IS_SET        EFX_QWORD_IS_SET64
1591 #define EFX_POPULATE_OWORD      EFX_POPULATE_OWORD64
1592 #define EFX_POPULATE_QWORD      EFX_POPULATE_QWORD64
1593 #define EFX_SET_OWORD_FIELD     EFX_SET_OWORD_FIELD64
1594 #define EFX_SET_QWORD_FIELD     EFX_SET_QWORD_FIELD64
1595 #define EFX_SET_OWORD_BIT       EFX_SET_OWORD_BIT64
1596 #define EFX_CLEAR_OWORD_BIT     EFX_CLEAR_OWORD_BIT64
1597 #define EFX_TEST_OWORD_BIT      EFX_TEST_OWORD_BIT64
1598 #define EFX_SET_QWORD_BIT       EFX_SET_QWORD_BIT64
1599 #define EFX_CLEAR_QWORD_BIT     EFX_CLEAR_QWORD_BIT64
1600 #define EFX_TEST_QWORD_BIT      EFX_TEST_QWORD_BIT64
1601 #define EFX_OR_OWORD            EFX_OR_OWORD64
1602 #define EFX_AND_OWORD           EFX_AND_OWORD64
1603 #define EFX_OR_QWORD            EFX_OR_QWORD64
1604 #define EFX_AND_QWORD           EFX_AND_QWORD64
1605 #else
1606 #define EFX_OWORD_FIELD         EFX_OWORD_FIELD32
1607 #define EFX_QWORD_FIELD         EFX_QWORD_FIELD32
1608 #define EFX_OWORD_IS_EQUAL      EFX_OWORD_IS_EQUAL32
1609 #define EFX_QWORD_IS_EQUAL      EFX_QWORD_IS_EQUAL32
1610 #define EFX_OWORD_IS_ZERO       EFX_OWORD_IS_ZERO32
1611 #define EFX_QWORD_IS_ZERO       EFX_QWORD_IS_ZERO32
1612 #define EFX_OWORD_IS_SET        EFX_OWORD_IS_SET32
1613 #define EFX_QWORD_IS_SET        EFX_QWORD_IS_SET32
1614 #define EFX_POPULATE_OWORD      EFX_POPULATE_OWORD32
1615 #define EFX_POPULATE_QWORD      EFX_POPULATE_QWORD32
1616 #define EFX_SET_OWORD_FIELD     EFX_SET_OWORD_FIELD32
1617 #define EFX_SET_QWORD_FIELD     EFX_SET_QWORD_FIELD32
1618 #define EFX_SET_OWORD_BIT       EFX_SET_OWORD_BIT32
1619 #define EFX_CLEAR_OWORD_BIT     EFX_CLEAR_OWORD_BIT32
1620 #define EFX_TEST_OWORD_BIT      EFX_TEST_OWORD_BIT32
1621 #define EFX_SET_QWORD_BIT       EFX_SET_QWORD_BIT32
1622 #define EFX_CLEAR_QWORD_BIT     EFX_CLEAR_QWORD_BIT32
1623 #define EFX_TEST_QWORD_BIT      EFX_TEST_QWORD_BIT32
1624 #define EFX_OR_OWORD            EFX_OR_OWORD32
1625 #define EFX_AND_OWORD           EFX_AND_OWORD32
1626 #define EFX_OR_QWORD            EFX_OR_QWORD32
1627 #define EFX_AND_QWORD           EFX_AND_QWORD32
1628 #endif
1629
1630 #ifdef  __cplusplus
1631 }
1632 #endif
1633
1634 #endif  /* _SYS_EFX_TYPES_H */