drivers/crypto/caam_jr/caam_jr_hw_specific.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2017 NXP
   3  */
   4
   5 #ifndef CAAM_JR_HW_SPECIFIC_H
   6 #define CAAM_JR_HW_SPECIFIC_H
   7
   8 #include <caam_jr_config.h>
   9
  10 /*
  11  * Offset to the registers of a job ring.
  12  * Is different for each job ring.
  13  */
  14 #define CHAN_BASE(jr)   ((size_t)(jr)->register_base_addr)
  15
  16 #define SEC_JOB_RING_IS_FULL(pi, ci, ring_max_size, ring_threshold) \
  17                 ((((pi) + 1 + ((ring_max_size) - (ring_threshold))) & \
  18                   (ring_max_size - 1))  == ((ci)))
  19
  20 #define SEC_CIRCULAR_COUNTER(x, max)   (((x) + 1) & (max - 1))
  21
  22 /*
  23  * Assert that cond is true. If !cond is true, display str and the vararg list
  24  * in a printf-like syntax. also, if !cond is true, return altRet.
  25  *
  26  * \param cond          A boolean expression to be asserted true
  27  * \param altRet        The value to be returned if cond doesn't hold true
  28  * \param str           A quoted char string
  29  *
  30  * E.g.:
  31  *      SEC_ASSERT(ret > 0, 0, "ERROR initializing app: code = %d\n", ret);
  32  */
  33 #define SEC_ASSERT(cond, altRet, ...) do {\
  34         if (unlikely(!(cond))) {\
  35                 CAAM_JR_ERR(__VA_ARGS__); \
  36                 return altRet; \
  37         } \
  38 } while (0)
  39
  40 #define SEC_DP_ASSERT(cond, altRet, ...) do {\
  41         if (unlikely(!(cond))) {\
  42                 CAAM_JR_DP_ERR(__VA_ARGS__); \
  43                 return altRet; \
  44         } \
  45 } while (0)
  46
  47 #define ASSERT(x)
  48
  49 /*
  50  * Constants representing various job ring registers
  51  */
  52 #if CAAM_BYTE_ORDER == __BIG_ENDIAN
  53 #define JR_REG_IRBA_OFFSET              0x0000
  54 #define JR_REG_IRBA_OFFSET_LO           0x0004
  55 #else
  56 #define JR_REG_IRBA_OFFSET              0x0004
  57 #define JR_REG_IRBA_OFFSET_LO           0x0000
  58 #endif
  59
  60 #define JR_REG_IRSR_OFFSET              0x000C
  61 #define JR_REG_IRSA_OFFSET              0x0014
  62 #define JR_REG_IRJA_OFFSET              0x001C
  63
  64 #if CAAM_BYTE_ORDER == __BIG_ENDIAN
  65 #define JR_REG_ORBA_OFFSET              0x0020
  66 #define JR_REG_ORBA_OFFSET_LO           0x0024
  67 #else
  68 #define JR_REG_ORBA_OFFSET              0x0024
  69 #define JR_REG_ORBA_OFFSET_LO           0x0020
  70 #endif
  71
  72 #define JR_REG_ORSR_OFFSET              0x002C
  73 #define JR_REG_ORJR_OFFSET              0x0034
  74 #define JR_REG_ORSFR_OFFSET             0x003C
  75 #define JR_REG_JROSR_OFFSET             0x0044
  76 #define JR_REG_JRINT_OFFSET             0x004C
  77
  78 #define JR_REG_JRCFG_OFFSET             0x0050
  79 #define JR_REG_JRCFG_OFFSET_LO          0x0054
  80
  81 #define JR_REG_IRRI_OFFSET              0x005C
  82 #define JR_REG_ORWI_OFFSET              0x0064
  83 #define JR_REG_JRCR_OFFSET              0x006C
  84
  85 /*
  86  * Constants for error handling on job ring
  87  */
  88 #define JR_REG_JRINT_ERR_TYPE_SHIFT     8
  89 #define JR_REG_JRINT_ERR_ORWI_SHIFT     16
  90 #define JR_REG_JRINIT_JRE_SHIFT         1
  91
  92 #define JRINT_JRE                       (1 << JR_REG_JRINIT_JRE_SHIFT)
  93 #define JRINT_ERR_WRITE_STATUS          (1 << JR_REG_JRINT_ERR_TYPE_SHIFT)
  94 #define JRINT_ERR_BAD_INPUT_BASE        (3 << JR_REG_JRINT_ERR_TYPE_SHIFT)
  95 #define JRINT_ERR_BAD_OUTPUT_BASE       (4 << JR_REG_JRINT_ERR_TYPE_SHIFT)
  96 #define JRINT_ERR_WRITE_2_IRBA          (5 << JR_REG_JRINT_ERR_TYPE_SHIFT)
  97 #define JRINT_ERR_WRITE_2_ORBA          (6 << JR_REG_JRINT_ERR_TYPE_SHIFT)
  98 #define JRINT_ERR_RES_B4_HALT           (7 << JR_REG_JRINT_ERR_TYPE_SHIFT)
  99 #define JRINT_ERR_REM_TOO_MANY          (8 << JR_REG_JRINT_ERR_TYPE_SHIFT)
 100 #define JRINT_ERR_ADD_TOO_MANY          (9 << JR_REG_JRINT_ERR_TYPE_SHIFT)
 101 #define JRINT_ERR_HALT_MASK             0x0C
 102 #define JRINT_ERR_HALT_INPROGRESS       0x04
 103 #define JRINT_ERR_HALT_COMPLETE         0x08
 104
 105 #define JR_REG_JRCR_VAL_RESET           0x00000001
 106
 107 #define JR_REG_JRCFG_LO_ICTT_SHIFT      0x10
 108 #define JR_REG_JRCFG_LO_ICDCT_SHIFT     0x08
 109 #define JR_REG_JRCFG_LO_ICEN_EN         0x02
 110
 111 /*
 112  * Constants for Descriptor Processing errors
 113  */
 114 #define SEC_HW_ERR_SSRC_NO_SRC          0x00
 115 #define SEC_HW_ERR_SSRC_CCB_ERR         0x02
 116 #define SEC_HW_ERR_SSRC_JMP_HALT_U      0x03
 117 #define SEC_HW_ERR_SSRC_DECO            0x04
 118 #define SEC_HW_ERR_SSRC_JR              0x06
 119 #define SEC_HW_ERR_SSRC_JMP_HALT_COND   0x07
 120
 121 #define SEC_HW_ERR_DECO_HFN_THRESHOLD   0xF1
 122 #define SEC_HW_ERR_CCB_ICV_CHECK_FAIL   0x0A
 123
 124 /*
 125  * Constants for descriptors
 126  */
 127 /* Return higher 32 bits of physical address */
 128 #define PHYS_ADDR_HI(phys_addr) \
 129             (uint32_t)(((uint64_t)phys_addr) >> 32)
 130
 131 /* Return lower 32 bits of physical address */
 132 #define PHYS_ADDR_LO(phys_addr) \
 133             (uint32_t)(((uint64_t)phys_addr) & 0xFFFFFFFF)
 134
 135 /*
 136  * Macros for extracting error codes for the job ring
 137  */
 138 #define JR_REG_JRINT_ERR_TYPE_EXTRACT(value)      ((value) & 0x00000F00)
 139 #define JR_REG_JRINT_ERR_ORWI_EXTRACT(value)     \
 140         (((value) & 0x3FFF0000) >> JR_REG_JRINT_ERR_ORWI_SHIFT)
 141 #define JR_REG_JRINT_JRE_EXTRACT(value)    ((value) & JRINT_JRE)
 142
 143 /*
 144  * Macros for managing the job ring
 145  */
 146 /* Read pointer to job ring input ring start address */
 147 #if defined(RTE_ARCH_ARM64)
 148 #define hw_get_inp_queue_base(jr) ((((dma_addr_t)GET_JR_REG(IRBA, \
 149                                       (jr))) << 32) | \
 150                                       (GET_JR_REG_LO(IRBA, (jr))))
 151
 152 /* Read pointer to job ring output ring start address */
 153 #define hw_get_out_queue_base(jr) (((dma_addr_t)(GET_JR_REG(ORBA, \
 154                                      (jr))) << 32) | \
 155                                      (GET_JR_REG_LO(ORBA, (jr))))
 156 #else
 157 #define hw_get_inp_queue_base(jr)   ((dma_addr_t)(GET_JR_REG_LO(IRBA, (jr))))
 158
 159 #define hw_get_out_queue_base(jr)   ((dma_addr_t)(GET_JR_REG_LO(ORBA, (jr))))
 160 #endif
 161
 162 /*
 163  * IRJA - Input Ring Jobs Added Register shows
 164  * how many new jobs were added to the Input Ring.
 165  */
 166 #define hw_enqueue_desc_on_job_ring(job_ring) SET_JR_REG(IRJA, (job_ring), 1)
 167
 168 #define hw_set_input_ring_size(job_ring, size) SET_JR_REG(IRSR, job_ring, \
 169                                                          (size))
 170
 171 #define hw_set_output_ring_size(job_ring, size) SET_JR_REG(ORSR, job_ring, \
 172                                                           (size))
 173
 174 #if defined(RTE_ARCH_ARM64)
 175 #define hw_set_input_ring_start_addr(job_ring, start_addr)      \
 176 {                                                               \
 177         SET_JR_REG(IRBA, job_ring, PHYS_ADDR_HI(start_addr));   \
 178         SET_JR_REG_LO(IRBA, job_ring, PHYS_ADDR_LO(start_addr));\
 179 }
 180
 181 #define hw_set_output_ring_start_addr(job_ring, start_addr) \
 182 {                                                               \
 183         SET_JR_REG(ORBA, job_ring, PHYS_ADDR_HI(start_addr));   \
 184         SET_JR_REG_LO(ORBA, job_ring, PHYS_ADDR_LO(start_addr));\
 185 }
 186
 187 #else
 188 #define hw_set_input_ring_start_addr(job_ring, start_addr)      \
 189 {                                                               \
 190         SET_JR_REG(IRBA, job_ring, 0);  \
 191         SET_JR_REG_LO(IRBA, job_ring, PHYS_ADDR_LO(start_addr));\
 192 }
 193
 194 #define hw_set_output_ring_start_addr(job_ring, start_addr) \
 195 {                                                               \
 196         SET_JR_REG(ORBA, job_ring, 0);  \
 197         SET_JR_REG_LO(ORBA, job_ring, PHYS_ADDR_LO(start_addr));\
 198 }
 199 #endif
 200
 201 /* ORJR - Output Ring Jobs Removed Register shows how many jobs were
 202  * removed from the Output Ring for processing by software. This is done after
 203  * the software has processed the entries.
 204  */
 205 #define hw_remove_entries(jr, no_entries) SET_JR_REG(ORJR, (jr), (no_entries))
 206
 207 /* IRSA - Input Ring Slots Available register holds the number of entries in
 208  * the Job Ring's input ring. Once a job is enqueued, the value returned is
 209  * decremented by the hardware by the number of jobs enqueued.
 210  */
 211 #define hw_get_available_slots(jr)              GET_JR_REG(IRSA, jr)
 212
 213 /* ORSFR - Output Ring Slots Full register holds the number of jobs which were
 214  * processed by the SEC and can be retrieved by the software. Once a job has
 215  * been processed by software, the user will call hw_remove_one_entry in order
 216  * to notify the SEC that the entry was processed.
 217  */
 218 #define hw_get_no_finished_jobs(jr)             GET_JR_REG(ORSFR, jr)
 219
 220 /*
 221  * Macros for manipulating JR registers
 222  */
 223 #if CORE_BYTE_ORDER == CAAM_BYTE_ORDER
 224 #define sec_read_32(addr)       (*(volatile unsigned int *)(addr))
 225 #define sec_write_32(addr, val) (*(volatile unsigned int *)(addr) = (val))
 226
 227 #else
 228 #define sec_read_32(addr)       rte_bswap32((*(volatile unsigned int *)(addr)))
 229 #define sec_write_32(addr, val) \
 230                         (*(volatile unsigned int *)(addr) = rte_bswap32(val))
 231 #endif
 232
 233 #if CAAM_BYTE_ORDER == __LITTLE_ENDIAN
 234 #define sec_read_64(addr)       (((u64)sec_read_32((u32 *)(addr) + 1) << 32) | \
 235                                 (sec_read_32((u32 *)(addr))))
 236
 237 #define sec_write_64(addr, val) {                               \
 238         sec_write_32((u32 *)(addr) + 1, (u32)((val) >> 32));    \
 239         sec_write_32((u32 *)(addr), (u32)(val));                \
 240 }
 241 #else /* CAAM_BYTE_ORDER == __BIG_ENDIAN */
 242 #define sec_read_64(addr)       (((u64)sec_read_32((u32 *)(addr)) << 32) | \
 243                                 (sec_read_32((u32 *)(addr) + 1)))
 244
 245 #define sec_write_64(addr, val) {                               \
 246         sec_write_32((u32 *)(addr), (u32)((val) >> 32));        \
 247         sec_write_32((u32 *)(addr) + 1, (u32)(val));            \
 248 }
 249 #endif
 250
 251 #if defined(RTE_ARCH_ARM64)
 252 #define sec_read_addr(a)        sec_read_64((a))
 253 #define sec_write_addr(a, v)    sec_write_64((a), (v))
 254 #else
 255 #define sec_read_addr(a)        sec_read_32((a))
 256 #define sec_write_addr(a, v)    sec_write_32((a), (v))
 257 #endif
 258
 259 #define JR_REG(name, jr)        (CHAN_BASE(jr) + JR_REG_##name##_OFFSET)
 260 #define JR_REG_LO(name, jr)     (CHAN_BASE(jr) + JR_REG_##name##_OFFSET_LO)
 261
 262 #define GET_JR_REG(name, jr)    (sec_read_32(JR_REG(name, (jr))))
 263 #define GET_JR_REG_LO(name, jr) (sec_read_32(JR_REG_LO(name, (jr))))
 264
 265 #define SET_JR_REG(name, jr, value) \
 266                                 (sec_write_32(JR_REG(name, (jr)), value))
 267 #define SET_JR_REG_LO(name, jr, value) \
 268                                 (sec_write_32(JR_REG_LO(name, (jr)), value))
 269
 270 /* Lists the possible states for a job ring. */
 271 typedef enum sec_job_ring_state_e {
 272         SEC_JOB_RING_STATE_STARTED,     /* Job ring is initialized */
 273         SEC_JOB_RING_STATE_RESET,       /* Job ring reset is in progress */
 274 } sec_job_ring_state_t;
 275
 276 /* code or cmd block to caam */
 277 struct sec_cdb {
 278         struct {
 279                 union {
 280                         uint32_t word;
 281                         struct {
 282 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
 283                                 uint16_t rsvd63_48;
 284                                 unsigned int rsvd47_39:9;
 285                                 unsigned int idlen:7;
 286 #else
 287                                 unsigned int idlen:7;
 288                                 unsigned int rsvd47_39:9;
 289                                 uint16_t rsvd63_48;
 290 #endif
 291                         } field;
 292                 } __rte_packed hi;
 293
 294                 union {
 295                         uint32_t word;
 296                         struct {
 297 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
 298                                 unsigned int rsvd31_30:2;
 299                                 unsigned int fsgt:1;
 300                                 unsigned int lng:1;
 301                                 unsigned int offset:2;
 302                                 unsigned int abs:1;
 303                                 unsigned int add_buf:1;
 304                                 uint8_t pool_id;
 305                                 uint16_t pool_buffer_size;
 306 #else
 307                                 uint16_t pool_buffer_size;
 308                                 uint8_t pool_id;
 309                                 unsigned int add_buf:1;
 310                                 unsigned int abs:1;
 311                                 unsigned int offset:2;
 312                                 unsigned int lng:1;
 313                                 unsigned int fsgt:1;
 314                                 unsigned int rsvd31_30:2;
 315 #endif
 316                         } field;
 317                 } __rte_packed lo;
 318         } __rte_packed sh_hdr;
 319
 320         uint32_t sh_desc[SEC_JOB_DESCRIPTOR_SIZE];
 321 };
 322
 323 struct caam_jr_qp {
 324         struct sec_job_ring_t *ring;
 325         uint64_t rx_pkts;
 326         uint64_t rx_errs;
 327         uint64_t rx_poll_err;
 328         uint64_t tx_pkts;
 329         uint64_t tx_errs;
 330         uint64_t tx_ring_full;
 331 };
 332
 333 struct sec_job_ring_t {
 334         /* TODO: Add wrapper macro to make it obvious this is the consumer index
 335          * on the output ring
 336          */
 337         uint32_t cidx;          /* Consumer index for job ring (jobs array).
 338                                  * @note: cidx and pidx are accessed from
 339                                  * different threads. Place the cidx and pidx
 340                                  * inside the structure so that they lay on
 341                                  * different cachelines, to avoid false sharing
 342                                  * between threads when the threads run on
 343                                  * different cores!
 344                                  */
 345         /* TODO: Add wrapper macro to make it obvious this is the producer index
 346          * on the input ring
 347          */
 348         uint32_t pidx;          /* Producer index for job ring (jobs array) */
 349
 350         phys_addr_t *input_ring;/* Ring of output descriptors received from SEC.
 351                                  * Size of array is power of 2 to allow fast
 352                                  * update of producer/consumer indexes with
 353                                  * bitwise operations.
 354                                  */
 355
 356         struct sec_outring_entry *output_ring;
 357                                 /* Ring of output descriptors received from SEC.
 358                                  * Size of array is power of 2 to allow fast
 359                                  * update of producer/consumer indexes with
 360                                  * bitwise operations.
 361                                  */
 362
 363         uint32_t irq_fd;        /* The file descriptor used for polling from
 364                                  * user space for interrupts notifications
 365                                  */
 366         uint32_t jr_mode;       /* Model used by SEC Driver to receive
 367                                  * notifications from SEC.  Can be either
 368                                  * of the three: #SEC_NOTIFICATION_TYPE_NAPI
 369                                  * #SEC_NOTIFICATION_TYPE_IRQ or
 370                                  * #SEC_NOTIFICATION_TYPE_POLL
 371                                  */
 372         uint32_t napi_mode;     /* Job ring mode if NAPI mode is chosen
 373                                  * Used only when jr_mode is set to
 374                                  * #SEC_NOTIFICATION_TYPE_NAPI
 375                                  */
 376         void *register_base_addr;       /* Base address for SEC's
 377                                          * register memory for this job ring.
 378                                          */
 379         uint8_t coalescing_en;          /* notifies if coelescing is
 380                                          * enabled for the job ring
 381                                          */
 382         sec_job_ring_state_t jr_state;  /* The state of this job ring */
 383
 384         struct rte_mempool *ctx_pool;   /* per dev mempool for caam_jr_op_ctx */
 385         unsigned int max_nb_queue_pairs;
 386         unsigned int max_nb_sessions;
 387         struct caam_jr_qp qps[RTE_CAAM_MAX_NB_SEC_QPS]; /* i/o queue for sec */
 388 };
 389
 390 /* Union describing the possible error codes that
 391  * can be set in the descriptor status word
 392  */
 393 union hw_error_code {
 394         uint32_t error;
 395         union {
 396                 struct {
 397                         uint32_t ssrc:4;
 398                         uint32_t ssed_val:28;
 399                 } __rte_packed value;
 400                 struct {
 401                         uint32_t ssrc:4;
 402                         uint32_t res:28;
 403                 } __rte_packed no_status_src;
 404                 struct {
 405                         uint32_t ssrc:4;
 406                         uint32_t jmp:1;
 407                         uint32_t res:11;
 408                         uint32_t desc_idx:8;
 409                         uint32_t cha_id:4;
 410                         uint32_t err_id:4;
 411                 } __rte_packed ccb_status_src;
 412                 struct {
 413                         uint32_t ssrc:4;
 414                         uint32_t jmp:1;
 415                         uint32_t res:11;
 416                         uint32_t desc_idx:8;
 417                         uint32_t offset:8;
 418                 } __rte_packed jmp_halt_user_src;
 419                 struct {
 420                         uint32_t ssrc:4;
 421                         uint32_t jmp:1;
 422                         uint32_t res:11;
 423                         uint32_t desc_idx:8;
 424                         uint32_t desc_err:8;
 425                 } __rte_packed deco_src;
 426                 struct {
 427                         uint32_t ssrc:4;
 428                         uint32_t res:17;
 429                         uint32_t naddr:3;
 430                         uint32_t desc_err:8;
 431                 } __rte_packed jr_src;
 432                 struct {
 433                         uint32_t ssrc:4;
 434                         uint32_t jmp:1;
 435                         uint32_t res:11;
 436                         uint32_t desc_idx:8;
 437                         uint32_t cond:8;
 438                 } __rte_packed jmp_halt_cond_src;
 439         } __rte_packed error_desc;
 440 } __rte_packed;
 441
 442 /* @brief Initialize a job ring/channel in SEC device.
 443  * Write configuration register/s to properly initialize a job ring.
 444  *
 445  * @param [in] job_ring     The job ring
 446  *
 447  * @retval 0 for success
 448  * @retval other for error
 449  */
 450 int hw_reset_job_ring(struct sec_job_ring_t *job_ring);
 451
 452 /* @brief Reset a job ring/channel in SEC device.
 453  * Write configuration register/s to reset a job ring.
 454  *
 455  * @param [in] job_ring     The job ring
 456  *
 457  * @retval 0 for success
 458  * @retval -1 in case job ring reset failed
 459  */
 460 int hw_shutdown_job_ring(struct sec_job_ring_t *job_ring);
 461
 462 /* @brief Handle a job ring/channel error in SEC device.
 463  * Identify the error type and clear error bits if required.
 464  *
 465  * @param [in]  job_ring        The job ring
 466  * @param [in]  sec_error_code  The job ring's error code
 467  */
 468 void hw_handle_job_ring_error(struct sec_job_ring_t *job_ring,
 469                               uint32_t sec_error_code);
 470
 471 /* @brief Handle a job ring error in the device.
 472  * Identify the error type and printout a explanatory
 473  * messages.
 474  *
 475  * @param [in]  job_ring        The job ring
 476  *
 477  */
 478 void hw_job_ring_error_print(struct sec_job_ring_t *job_ring, int code);
 479
 480 /* @brief Set interrupt coalescing parameters on the Job Ring.
 481  * @param [in]  job_ring                The job ring
 482  * @param [in]  irq_coalesing_timer     Interrupt coalescing timer threshold.
 483  *                                      This value determines the maximum
 484  *                                      amount of time after processing a
 485  *                                      descriptor before raising an interrupt.
 486  * @param [in]  irq_coalescing_count    Interrupt coalescing descriptor count
 487  *                                      threshold.
 488  */
 489 int hw_job_ring_set_coalescing_param(struct sec_job_ring_t *job_ring,
 490                                      uint16_t irq_coalescing_timer,
 491                                      uint8_t irq_coalescing_count);
 492
 493 /* @brief Enable interrupt coalescing on a job ring
 494  * @param [in]  job_ring                The job ring
 495  */
 496 int hw_job_ring_enable_coalescing(struct sec_job_ring_t *job_ring);
 497
 498 /* @brief Disable interrupt coalescing on a job ring
 499  * @param [in]  job_ring                The job ring
 500  */
 501 int hw_job_ring_disable_coalescing(struct sec_job_ring_t *job_ring);
 502
 503 #endif /* CAAM_JR_HW_SPECIFIC_H */