drivers/net/qede/base/ecore_hw.c

   1 /*
   2  * Copyright (c) 2016 QLogic Corporation.
   3  * All rights reserved.
   4  * www.qlogic.com
   5  *
   6  * See LICENSE.qede_pmd for copyright and licensing details.
   7  */
   8
   9 #include "bcm_osal.h"
  10 #include "ecore_hsi_common.h"
  11 #include "ecore_status.h"
  12 #include "ecore.h"
  13 #include "ecore_hw.h"
  14 #include "reg_addr.h"
  15 #include "ecore_utils.h"
  16 #include "ecore_iov_api.h"
  17
  18 #ifndef ASIC_ONLY
  19 #define ECORE_EMUL_FACTOR 2000
  20 #define ECORE_FPGA_FACTOR 200
  21 #endif
  22
  23 #define ECORE_BAR_ACQUIRE_TIMEOUT 1000
  24
  25 /* Invalid values */
  26 #define ECORE_BAR_INVALID_OFFSET        (OSAL_CPU_TO_LE32(-1))
  27
  28 struct ecore_ptt {
  29         osal_list_entry_t list_entry;
  30         unsigned int idx;
  31         struct pxp_ptt_entry pxp;
  32         u8 hwfn_id;
  33 };
  34
  35 struct ecore_ptt_pool {
  36         osal_list_t free_list;
  37         osal_spinlock_t lock; /* ptt synchronized access */
  38         struct ecore_ptt ptts[PXP_EXTERNAL_BAR_PF_WINDOW_NUM];
  39 };
  40
  41 enum _ecore_status_t ecore_ptt_pool_alloc(struct ecore_hwfn *p_hwfn)
  42 {
  43         struct ecore_ptt_pool *p_pool = OSAL_ALLOC(p_hwfn->p_dev,
  44                                                    GFP_KERNEL,
  45                                                    sizeof(*p_pool));
  46         int i;
  47
  48         if (!p_pool)
  49                 return ECORE_NOMEM;
  50
  51         OSAL_LIST_INIT(&p_pool->free_list);
  52         for (i = 0; i < PXP_EXTERNAL_BAR_PF_WINDOW_NUM; i++) {
  53                 p_pool->ptts[i].idx = i;
  54                 p_pool->ptts[i].pxp.offset = ECORE_BAR_INVALID_OFFSET;
  55                 p_pool->ptts[i].pxp.pretend.control = 0;
  56                 p_pool->ptts[i].hwfn_id = p_hwfn->my_id;
  57
  58                 /* There are special PTT entries that are taken only by design.
  59                  * The rest are added ot the list for general usage.
  60                  */
  61                 if (i >= RESERVED_PTT_MAX)
  62                         OSAL_LIST_PUSH_HEAD(&p_pool->ptts[i].list_entry,
  63                                             &p_pool->free_list);
  64         }
  65
  66         p_hwfn->p_ptt_pool = p_pool;
  67         OSAL_SPIN_LOCK_ALLOC(p_hwfn, &p_pool->lock);
  68         OSAL_SPIN_LOCK_INIT(&p_pool->lock);
  69
  70         return ECORE_SUCCESS;
  71 }
  72
  73 void ecore_ptt_invalidate(struct ecore_hwfn *p_hwfn)
  74 {
  75         struct ecore_ptt *p_ptt;
  76         int i;
  77
  78         for (i = 0; i < PXP_EXTERNAL_BAR_PF_WINDOW_NUM; i++) {
  79                 p_ptt = &p_hwfn->p_ptt_pool->ptts[i];
  80                 p_ptt->pxp.offset = ECORE_BAR_INVALID_OFFSET;
  81         }
  82 }
  83
  84 void ecore_ptt_pool_free(struct ecore_hwfn *p_hwfn)
  85 {
  86         if (p_hwfn->p_ptt_pool)
  87                 OSAL_SPIN_LOCK_DEALLOC(&p_hwfn->p_ptt_pool->lock);
  88         OSAL_FREE(p_hwfn->p_dev, p_hwfn->p_ptt_pool);
  89 }
  90
  91 struct ecore_ptt *ecore_ptt_acquire(struct ecore_hwfn *p_hwfn)
  92 {
  93         struct ecore_ptt *p_ptt;
  94         unsigned int i;
  95
  96         /* Take the free PTT from the list */
  97         for (i = 0; i < ECORE_BAR_ACQUIRE_TIMEOUT; i++) {
  98                 OSAL_SPIN_LOCK(&p_hwfn->p_ptt_pool->lock);
  99                 if (!OSAL_LIST_IS_EMPTY(&p_hwfn->p_ptt_pool->free_list)) {
 100                         p_ptt = OSAL_LIST_FIRST_ENTRY(
 101                                                 &p_hwfn->p_ptt_pool->free_list,
 102                                                 struct ecore_ptt, list_entry);
 103                         OSAL_LIST_REMOVE_ENTRY(&p_ptt->list_entry,
 104                                                &p_hwfn->p_ptt_pool->free_list);
 105
 106                         OSAL_SPIN_UNLOCK(&p_hwfn->p_ptt_pool->lock);
 107
 108                         DP_VERBOSE(p_hwfn, ECORE_MSG_HW,
 109                                    "allocated ptt %d\n", p_ptt->idx);
 110
 111                         return p_ptt;
 112                 }
 113
 114                 OSAL_SPIN_UNLOCK(&p_hwfn->p_ptt_pool->lock);
 115                 OSAL_MSLEEP(1);
 116         }
 117
 118         DP_NOTICE(p_hwfn, true,
 119                   "PTT acquire timeout - failed to allocate PTT\n");
 120         return OSAL_NULL;
 121 }
 122
 123 void ecore_ptt_release(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt)
 124 {
 125         /* This PTT should not be set to pretend if it is being released */
 126         /* TODO - add some pretend sanity checks, to make sure pretend
 127          * isn't set on this ptt
 128          */
 129
 130         OSAL_SPIN_LOCK(&p_hwfn->p_ptt_pool->lock);
 131         OSAL_LIST_PUSH_HEAD(&p_ptt->list_entry, &p_hwfn->p_ptt_pool->free_list);
 132         OSAL_SPIN_UNLOCK(&p_hwfn->p_ptt_pool->lock);
 133 }
 134
 135 u32 ecore_ptt_get_hw_addr(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt)
 136 {
 137         /* The HW is using DWORDS and we need to translate it to Bytes */
 138         return OSAL_LE32_TO_CPU(p_ptt->pxp.offset) << 2;
 139 }
 140
 141 static u32 ecore_ptt_config_addr(struct ecore_ptt *p_ptt)
 142 {
 143         return PXP_PF_WINDOW_ADMIN_PER_PF_START +
 144             p_ptt->idx * sizeof(struct pxp_ptt_entry);
 145 }
 146
 147 u32 ecore_ptt_get_bar_addr(struct ecore_ptt *p_ptt)
 148 {
 149         return PXP_EXTERNAL_BAR_PF_WINDOW_START +
 150             p_ptt->idx * PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE;
 151 }
 152
 153 void ecore_ptt_set_win(struct ecore_hwfn *p_hwfn,
 154                        struct ecore_ptt *p_ptt, u32 new_hw_addr)
 155 {
 156         u32 prev_hw_addr;
 157
 158         prev_hw_addr = ecore_ptt_get_hw_addr(p_hwfn, p_ptt);
 159
 160         if (new_hw_addr == prev_hw_addr)
 161                 return;
 162
 163         /* Update PTT entery in admin window */
 164         DP_VERBOSE(p_hwfn, ECORE_MSG_HW,
 165                    "Updating PTT entry %d to offset 0x%x\n",
 166                    p_ptt->idx, new_hw_addr);
 167
 168         /* The HW is using DWORDS and the address is in Bytes */
 169         p_ptt->pxp.offset = OSAL_CPU_TO_LE32(new_hw_addr >> 2);
 170
 171         REG_WR(p_hwfn,
 172                ecore_ptt_config_addr(p_ptt) +
 173                OFFSETOF(struct pxp_ptt_entry, offset),
 174                OSAL_LE32_TO_CPU(p_ptt->pxp.offset));
 175 }
 176
 177 static u32 ecore_set_ptt(struct ecore_hwfn *p_hwfn,
 178                          struct ecore_ptt *p_ptt, u32 hw_addr)
 179 {
 180         u32 win_hw_addr = ecore_ptt_get_hw_addr(p_hwfn, p_ptt);
 181         u32 offset;
 182
 183         offset = hw_addr - win_hw_addr;
 184
 185         if (p_ptt->hwfn_id != p_hwfn->my_id)
 186                 DP_NOTICE(p_hwfn, true,
 187                           "ptt[%d] of hwfn[%02x] is used by hwfn[%02x]!\n",
 188                           p_ptt->idx, p_ptt->hwfn_id, p_hwfn->my_id);
 189
 190         /* Verify the address is within the window */
 191         if (hw_addr < win_hw_addr ||
 192             offset >= PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE) {
 193                 ecore_ptt_set_win(p_hwfn, p_ptt, hw_addr);
 194                 offset = 0;
 195         }
 196
 197         return ecore_ptt_get_bar_addr(p_ptt) + offset;
 198 }
 199
 200 struct ecore_ptt *ecore_get_reserved_ptt(struct ecore_hwfn *p_hwfn,
 201                                          enum reserved_ptts ptt_idx)
 202 {
 203         if (ptt_idx >= RESERVED_PTT_MAX) {
 204                 DP_NOTICE(p_hwfn, true,
 205                           "Requested PTT %d is out of range\n", ptt_idx);
 206                 return OSAL_NULL;
 207         }
 208
 209         return &p_hwfn->p_ptt_pool->ptts[ptt_idx];
 210 }
 211
 212 static bool ecore_is_reg_fifo_empty(struct ecore_hwfn *p_hwfn,
 213                                     struct ecore_ptt *p_ptt)
 214 {
 215         bool is_empty = true;
 216         u32 bar_addr;
 217
 218         if (!p_hwfn->p_dev->chk_reg_fifo)
 219                 goto out;
 220
 221         /* ecore_rd() cannot be used here since it calls this function */
 222         bar_addr = ecore_set_ptt(p_hwfn, p_ptt, GRC_REG_TRACE_FIFO_VALID_DATA);
 223         is_empty = REG_RD(p_hwfn, bar_addr) == 0;
 224
 225 #ifndef ASIC_ONLY
 226         if (CHIP_REV_IS_SLOW(p_hwfn->p_dev))
 227                 OSAL_UDELAY(100);
 228 #endif
 229
 230 out:
 231         return is_empty;
 232 }
 233
 234 void ecore_wr(struct ecore_hwfn *p_hwfn,
 235               struct ecore_ptt *p_ptt, u32 hw_addr, u32 val)
 236 {
 237         bool prev_fifo_err;
 238         u32 bar_addr;
 239
 240         prev_fifo_err = !ecore_is_reg_fifo_empty(p_hwfn, p_ptt);
 241
 242         bar_addr = ecore_set_ptt(p_hwfn, p_ptt, hw_addr);
 243         REG_WR(p_hwfn, bar_addr, val);
 244         DP_VERBOSE(p_hwfn, ECORE_MSG_HW,
 245                    "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n",
 246                    bar_addr, hw_addr, val);
 247
 248 #ifndef ASIC_ONLY
 249         if (CHIP_REV_IS_SLOW(p_hwfn->p_dev))
 250                 OSAL_UDELAY(100);
 251 #endif
 252
 253         OSAL_WARN(!prev_fifo_err && !ecore_is_reg_fifo_empty(p_hwfn, p_ptt),
 254                   "reg_fifo err was caused by a call to ecore_wr(0x%x, 0x%x)\n",
 255                   hw_addr, val);
 256 }
 257
 258 u32 ecore_rd(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u32 hw_addr)
 259 {
 260         bool prev_fifo_err;
 261         u32 bar_addr, val;
 262
 263         prev_fifo_err = !ecore_is_reg_fifo_empty(p_hwfn, p_ptt);
 264
 265         bar_addr = ecore_set_ptt(p_hwfn, p_ptt, hw_addr);
 266         val = REG_RD(p_hwfn, bar_addr);
 267
 268         DP_VERBOSE(p_hwfn, ECORE_MSG_HW,
 269                    "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n",
 270                    bar_addr, hw_addr, val);
 271
 272 #ifndef ASIC_ONLY
 273         if (CHIP_REV_IS_SLOW(p_hwfn->p_dev))
 274                 OSAL_UDELAY(100);
 275 #endif
 276
 277         OSAL_WARN(!prev_fifo_err && !ecore_is_reg_fifo_empty(p_hwfn, p_ptt),
 278                   "reg_fifo error was caused by a call to ecore_rd(0x%x)\n",
 279                   hw_addr);
 280
 281         return val;
 282 }
 283
 284 static void ecore_memcpy_hw(struct ecore_hwfn *p_hwfn,
 285                             struct ecore_ptt *p_ptt,
 286                             void *addr,
 287                             u32 hw_addr, osal_size_t n, bool to_device)
 288 {
 289         u32 dw_count, *host_addr, hw_offset;
 290         osal_size_t quota, done = 0;
 291         u32 OSAL_IOMEM *reg_addr;
 292
 293         while (done < n) {
 294                 quota = OSAL_MIN_T(osal_size_t, n - done,
 295                                    PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE);
 296
 297                 if (IS_PF(p_hwfn->p_dev)) {
 298                         ecore_ptt_set_win(p_hwfn, p_ptt, hw_addr + done);
 299                         hw_offset = ecore_ptt_get_bar_addr(p_ptt);
 300                 } else {
 301                         hw_offset = hw_addr + done;
 302                 }
 303
 304                 dw_count = quota / 4;
 305                 host_addr = (u32 *)((u8 *)addr + done);
 306                 reg_addr = (u32 OSAL_IOMEM *)OSAL_REG_ADDR(p_hwfn, hw_offset);
 307
 308                 if (to_device)
 309                         while (dw_count--)
 310                                 DIRECT_REG_WR(p_hwfn, reg_addr++, *host_addr++);
 311                 else
 312                         while (dw_count--)
 313                                 *host_addr++ = DIRECT_REG_RD(p_hwfn,
 314                                                              reg_addr++);
 315
 316                 done += quota;
 317         }
 318 }
 319
 320 void ecore_memcpy_from(struct ecore_hwfn *p_hwfn,
 321                        struct ecore_ptt *p_ptt,
 322                        void *dest, u32 hw_addr, osal_size_t n)
 323 {
 324         DP_VERBOSE(p_hwfn, ECORE_MSG_HW,
 325                    "hw_addr 0x%x, dest %p hw_addr 0x%x, size %lu\n",
 326                    hw_addr, dest, hw_addr, (unsigned long)n);
 327
 328         ecore_memcpy_hw(p_hwfn, p_ptt, dest, hw_addr, n, false);
 329 }
 330
 331 void ecore_memcpy_to(struct ecore_hwfn *p_hwfn,
 332                      struct ecore_ptt *p_ptt,
 333                      u32 hw_addr, void *src, osal_size_t n)
 334 {
 335         DP_VERBOSE(p_hwfn, ECORE_MSG_HW,
 336                    "hw_addr 0x%x, hw_addr 0x%x, src %p size %lu\n",
 337                    hw_addr, hw_addr, src, (unsigned long)n);
 338
 339         ecore_memcpy_hw(p_hwfn, p_ptt, src, hw_addr, n, true);
 340 }
 341
 342 void ecore_fid_pretend(struct ecore_hwfn *p_hwfn,
 343                        struct ecore_ptt *p_ptt, u16 fid)
 344 {
 345         u16 control = 0;
 346
 347         SET_FIELD(control, PXP_PRETEND_CMD_IS_CONCRETE, 1);
 348         SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_FUNCTION, 1);
 349
 350 /* Every pretend undos prev pretends, including previous port pretend */
 351
 352         SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0);
 353         SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0);
 354         SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1);
 355
 356         if (!GET_FIELD(fid, PXP_CONCRETE_FID_VFVALID))
 357                 fid = GET_FIELD(fid, PXP_CONCRETE_FID_PFID);
 358
 359         p_ptt->pxp.pretend.control = OSAL_CPU_TO_LE16(control);
 360         p_ptt->pxp.pretend.fid.concrete_fid.fid = OSAL_CPU_TO_LE16(fid);
 361
 362         REG_WR(p_hwfn,
 363                ecore_ptt_config_addr(p_ptt) +
 364                OFFSETOF(struct pxp_ptt_entry, pretend),
 365                         *(u32 *)&p_ptt->pxp.pretend);
 366 }
 367
 368 void ecore_port_pretend(struct ecore_hwfn *p_hwfn,
 369                         struct ecore_ptt *p_ptt, u8 port_id)
 370 {
 371         u16 control = 0;
 372
 373         SET_FIELD(control, PXP_PRETEND_CMD_PORT, port_id);
 374         SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 1);
 375         SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1);
 376         p_ptt->pxp.pretend.control = OSAL_CPU_TO_LE16(control);
 377
 378         REG_WR(p_hwfn,
 379                ecore_ptt_config_addr(p_ptt) +
 380                OFFSETOF(struct pxp_ptt_entry, pretend),
 381                         *(u32 *)&p_ptt->pxp.pretend);
 382 }
 383
 384 void ecore_port_unpretend(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt)
 385 {
 386         u16 control = 0;
 387
 388         SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0);
 389         SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0);
 390         SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1);
 391
 392         p_ptt->pxp.pretend.control = OSAL_CPU_TO_LE16(control);
 393
 394         REG_WR(p_hwfn,
 395                ecore_ptt_config_addr(p_ptt) +
 396                OFFSETOF(struct pxp_ptt_entry, pretend),
 397                         *(u32 *)&p_ptt->pxp.pretend);
 398 }
 399
 400 u32 ecore_vfid_to_concrete(struct ecore_hwfn *p_hwfn, u8 vfid)
 401 {
 402         u32 concrete_fid = 0;
 403
 404         SET_FIELD(concrete_fid, PXP_CONCRETE_FID_PFID, p_hwfn->rel_pf_id);
 405         SET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFID, vfid);
 406         SET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFVALID, 1);
 407
 408         return concrete_fid;
 409 }
 410
 411 /* Not in use @DPDK
 412  * Ecore HW lock
 413  * =============
 414  * Although the implementation is ready, today we don't have any flow that
 415  * utliizes said locks - and we want to keep it this way.
 416  * If this changes, this needs to be revisted.
 417  */
 418
 419 /* Ecore DMAE
 420  * =============
 421  */
 422 static void ecore_dmae_opcode(struct ecore_hwfn *p_hwfn,
 423                               const u8 is_src_type_grc,
 424                               const u8 is_dst_type_grc,
 425                               struct ecore_dmae_params *p_params)
 426 {
 427         u16 opcode_b = 0;
 428         u32 opcode = 0;
 429
 430         /* Whether the source is the PCIe or the GRC.
 431          * 0- The source is the PCIe
 432          * 1- The source is the GRC.
 433          */
 434         opcode |= (is_src_type_grc ? DMAE_CMD_SRC_MASK_GRC
 435                    : DMAE_CMD_SRC_MASK_PCIE) << DMAE_CMD_SRC_SHIFT;
 436         opcode |= (p_hwfn->rel_pf_id & DMAE_CMD_SRC_PF_ID_MASK) <<
 437             DMAE_CMD_SRC_PF_ID_SHIFT;
 438
 439         /* The destination of the DMA can be: 0-None 1-PCIe 2-GRC 3-None */
 440         opcode |= (is_dst_type_grc ? DMAE_CMD_DST_MASK_GRC
 441                    : DMAE_CMD_DST_MASK_PCIE) << DMAE_CMD_DST_SHIFT;
 442         opcode |= (p_hwfn->rel_pf_id & DMAE_CMD_DST_PF_ID_MASK) <<
 443             DMAE_CMD_DST_PF_ID_SHIFT;
 444
 445         /* DMAE_E4_TODO need to check which value to specifiy here. */
 446         /* opcode |= (!b_complete_to_host)<< DMAE_CMD_C_DST_SHIFT; */
 447
 448         /* Whether to write a completion word to the completion destination:
 449          * 0-Do not write a completion word
 450          * 1-Write the completion word
 451          */
 452         opcode |= DMAE_CMD_COMP_WORD_EN_MASK << DMAE_CMD_COMP_WORD_EN_SHIFT;
 453         opcode |= DMAE_CMD_SRC_ADDR_RESET_MASK << DMAE_CMD_SRC_ADDR_RESET_SHIFT;
 454
 455         if (p_params->flags & ECORE_DMAE_FLAG_COMPLETION_DST)
 456                 opcode |= 1 << DMAE_CMD_COMP_FUNC_SHIFT;
 457
 458         /* swapping mode 3 - big endian there should be a define ifdefed in
 459          * the HSI somewhere. Since it is currently
 460          */
 461         opcode |= DMAE_CMD_ENDIANITY << DMAE_CMD_ENDIANITY_MODE_SHIFT;
 462
 463         opcode |= p_hwfn->port_id << DMAE_CMD_PORT_ID_SHIFT;
 464
 465         /* reset source address in next go */
 466         opcode |= DMAE_CMD_SRC_ADDR_RESET_MASK << DMAE_CMD_SRC_ADDR_RESET_SHIFT;
 467
 468         /* reset dest address in next go */
 469         opcode |= DMAE_CMD_DST_ADDR_RESET_MASK << DMAE_CMD_DST_ADDR_RESET_SHIFT;
 470
 471         /* SRC/DST VFID: all 1's - pf, otherwise VF id */
 472         if (p_params->flags & ECORE_DMAE_FLAG_VF_SRC) {
 473                 opcode |= (1 << DMAE_CMD_SRC_VF_ID_VALID_SHIFT);
 474                 opcode_b |= (p_params->src_vfid << DMAE_CMD_SRC_VF_ID_SHIFT);
 475         } else {
 476                 opcode_b |= (DMAE_CMD_SRC_VF_ID_MASK <<
 477                              DMAE_CMD_SRC_VF_ID_SHIFT);
 478         }
 479         if (p_params->flags & ECORE_DMAE_FLAG_VF_DST) {
 480                 opcode |= 1 << DMAE_CMD_DST_VF_ID_VALID_SHIFT;
 481                 opcode_b |= p_params->dst_vfid << DMAE_CMD_DST_VF_ID_SHIFT;
 482         } else {
 483                 opcode_b |= DMAE_CMD_DST_VF_ID_MASK << DMAE_CMD_DST_VF_ID_SHIFT;
 484         }
 485
 486         p_hwfn->dmae_info.p_dmae_cmd->opcode = OSAL_CPU_TO_LE32(opcode);
 487         p_hwfn->dmae_info.p_dmae_cmd->opcode_b = OSAL_CPU_TO_LE16(opcode_b);
 488 }
 489
 490 static u32 ecore_dmae_idx_to_go_cmd(u8 idx)
 491 {
 492         OSAL_BUILD_BUG_ON((DMAE_REG_GO_C31 - DMAE_REG_GO_C0) != 31 * 4);
 493
 494         /* All the DMAE 'go' registers form an array in internal memory */
 495         return DMAE_REG_GO_C0 + (idx << 2);
 496 }
 497
 498 static enum _ecore_status_t ecore_dmae_post_command(struct ecore_hwfn *p_hwfn,
 499                                                     struct ecore_ptt *p_ptt)
 500 {
 501         struct dmae_cmd *p_command = p_hwfn->dmae_info.p_dmae_cmd;
 502         u8 idx_cmd = p_hwfn->dmae_info.channel, i;
 503         enum _ecore_status_t ecore_status = ECORE_SUCCESS;
 504
 505         /* verify address is not OSAL_NULL */
 506         if ((((!p_command->dst_addr_lo) && (!p_command->dst_addr_hi)) ||
 507              ((!p_command->src_addr_lo) && (!p_command->src_addr_hi)))) {
 508                 DP_NOTICE(p_hwfn, true,
 509                           "source or destination address 0 idx_cmd=%d\n"
 510                           "opcode = [0x%08x,0x%04x] len=0x%x"
 511                           " src=0x%x:%x dst=0x%x:%x\n",
 512                           idx_cmd,
 513                           OSAL_LE32_TO_CPU(p_command->opcode),
 514                           OSAL_LE16_TO_CPU(p_command->opcode_b),
 515                           OSAL_LE16_TO_CPU(p_command->length_dw),
 516                           OSAL_LE32_TO_CPU(p_command->src_addr_hi),
 517                           OSAL_LE32_TO_CPU(p_command->src_addr_lo),
 518                           OSAL_LE32_TO_CPU(p_command->dst_addr_hi),
 519                           OSAL_LE32_TO_CPU(p_command->dst_addr_lo));
 520
 521                 return ECORE_INVAL;
 522         }
 523
 524         DP_VERBOSE(p_hwfn, ECORE_MSG_HW,
 525                    "Posting DMAE command [idx %d]: opcode = [0x%08x,0x%04x]"
 526                    "len=0x%x src=0x%x:%x dst=0x%x:%x\n",
 527                    idx_cmd,
 528                    OSAL_LE32_TO_CPU(p_command->opcode),
 529                    OSAL_LE16_TO_CPU(p_command->opcode_b),
 530                    OSAL_LE16_TO_CPU(p_command->length_dw),
 531                    OSAL_LE32_TO_CPU(p_command->src_addr_hi),
 532                    OSAL_LE32_TO_CPU(p_command->src_addr_lo),
 533                    OSAL_LE32_TO_CPU(p_command->dst_addr_hi),
 534                    OSAL_LE32_TO_CPU(p_command->dst_addr_lo));
 535
 536         /* Copy the command to DMAE - need to do it before every call
 537          * for source/dest address no reset.
 538          * The number of commands have been increased to 16 (previous was 14)
 539          * The first 9 DWs are the command registers, the 10 DW is the
 540          * GO register, and
 541          * the rest are result registers (which are read only by the client).
 542          */
 543         for (i = 0; i < DMAE_CMD_SIZE; i++) {
 544                 u32 data = (i < DMAE_CMD_SIZE_TO_FILL) ?
 545                     *(((u32 *)p_command) + i) : 0;
 546
 547                 ecore_wr(p_hwfn, p_ptt,
 548                          DMAE_REG_CMD_MEM +
 549                          (idx_cmd * DMAE_CMD_SIZE * sizeof(u32)) +
 550                          (i * sizeof(u32)), data);
 551         }
 552
 553         ecore_wr(p_hwfn, p_ptt,
 554                  ecore_dmae_idx_to_go_cmd(idx_cmd), DMAE_GO_VALUE);
 555
 556         return ecore_status;
 557 }
 558
 559 enum _ecore_status_t ecore_dmae_info_alloc(struct ecore_hwfn *p_hwfn)
 560 {
 561         dma_addr_t *p_addr = &p_hwfn->dmae_info.completion_word_phys_addr;
 562         struct dmae_cmd **p_cmd = &p_hwfn->dmae_info.p_dmae_cmd;
 563         u32 **p_buff = &p_hwfn->dmae_info.p_intermediate_buffer;
 564         u32 **p_comp = &p_hwfn->dmae_info.p_completion_word;
 565
 566         *p_comp = OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, p_addr, sizeof(u32));
 567         if (*p_comp == OSAL_NULL) {
 568                 DP_NOTICE(p_hwfn, true,
 569                           "Failed to allocate `p_completion_word'\n");
 570                 goto err;
 571         }
 572
 573         p_addr = &p_hwfn->dmae_info.dmae_cmd_phys_addr;
 574         *p_cmd = OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, p_addr,
 575                                          sizeof(struct dmae_cmd));
 576         if (*p_cmd == OSAL_NULL) {
 577                 DP_NOTICE(p_hwfn, true,
 578                           "Failed to allocate `struct dmae_cmd'\n");
 579                 goto err;
 580         }
 581
 582         p_addr = &p_hwfn->dmae_info.intermediate_buffer_phys_addr;
 583         *p_buff = OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, p_addr,
 584                                           sizeof(u32) * DMAE_MAX_RW_SIZE);
 585         if (*p_buff == OSAL_NULL) {
 586                 DP_NOTICE(p_hwfn, true,
 587                           "Failed to allocate `intermediate_buffer'\n");
 588                 goto err;
 589         }
 590
 591         p_hwfn->dmae_info.channel = p_hwfn->rel_pf_id;
 592
 593         return ECORE_SUCCESS;
 594 err:
 595         ecore_dmae_info_free(p_hwfn);
 596         return ECORE_NOMEM;
 597 }
 598
 599 void ecore_dmae_info_free(struct ecore_hwfn *p_hwfn)
 600 {
 601         dma_addr_t p_phys;
 602
 603         /* Just make sure no one is in the middle */
 604         OSAL_MUTEX_ACQUIRE(&p_hwfn->dmae_info.mutex);
 605
 606         if (p_hwfn->dmae_info.p_completion_word != OSAL_NULL) {
 607                 p_phys = p_hwfn->dmae_info.completion_word_phys_addr;
 608                 OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev,
 609                                        p_hwfn->dmae_info.p_completion_word,
 610                                        p_phys, sizeof(u32));
 611                 p_hwfn->dmae_info.p_completion_word = OSAL_NULL;
 612         }
 613
 614         if (p_hwfn->dmae_info.p_dmae_cmd != OSAL_NULL) {
 615                 p_phys = p_hwfn->dmae_info.dmae_cmd_phys_addr;
 616                 OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev,
 617                                        p_hwfn->dmae_info.p_dmae_cmd,
 618                                        p_phys, sizeof(struct dmae_cmd));
 619                 p_hwfn->dmae_info.p_dmae_cmd = OSAL_NULL;
 620         }
 621
 622         if (p_hwfn->dmae_info.p_intermediate_buffer != OSAL_NULL) {
 623                 p_phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr;
 624                 OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev,
 625                                        p_hwfn->dmae_info.p_intermediate_buffer,
 626                                        p_phys, sizeof(u32) * DMAE_MAX_RW_SIZE);
 627                 p_hwfn->dmae_info.p_intermediate_buffer = OSAL_NULL;
 628         }
 629
 630         OSAL_MUTEX_RELEASE(&p_hwfn->dmae_info.mutex);
 631 }
 632
 633 static enum _ecore_status_t ecore_dmae_operation_wait(struct ecore_hwfn *p_hwfn)
 634 {
 635         u32 wait_cnt_limit = 10000, wait_cnt = 0;
 636         enum _ecore_status_t ecore_status = ECORE_SUCCESS;
 637
 638 #ifndef ASIC_ONLY
 639         u32 factor = (CHIP_REV_IS_EMUL(p_hwfn->p_dev) ?
 640                       ECORE_EMUL_FACTOR :
 641                       (CHIP_REV_IS_FPGA(p_hwfn->p_dev) ?
 642                        ECORE_FPGA_FACTOR : 1));
 643
 644         wait_cnt_limit *= factor;
 645 #endif
 646
 647         /* DMAE_E4_TODO : TODO check if we have to call any other function
 648          * other than BARRIER to sync the completion_word since we are not
 649          * using the volatile keyword for this
 650          */
 651         OSAL_BARRIER(p_hwfn->p_dev);
 652         while (*p_hwfn->dmae_info.p_completion_word != DMAE_COMPLETION_VAL) {
 653                 OSAL_UDELAY(DMAE_MIN_WAIT_TIME);
 654                 if (++wait_cnt > wait_cnt_limit) {
 655                         DP_NOTICE(p_hwfn->p_dev, ECORE_MSG_HW,
 656                                   "Timed-out waiting for operation to"
 657                                   " complete. Completion word is 0x%08x"
 658                                   " expected 0x%08x.\n",
 659                                   *p_hwfn->dmae_info.p_completion_word,
 660                                   DMAE_COMPLETION_VAL);
 661                         ecore_status = ECORE_TIMEOUT;
 662                         break;
 663                 }
 664                 /* to sync the completion_word since we are not
 665                  * using the volatile keyword for p_completion_word
 666                  */
 667                 OSAL_BARRIER(p_hwfn->p_dev);
 668         }
 669
 670         if (ecore_status == ECORE_SUCCESS)
 671                 *p_hwfn->dmae_info.p_completion_word = 0;
 672
 673         return ecore_status;
 674 }
 675
 676 static enum _ecore_status_t
 677 ecore_dmae_execute_sub_operation(struct ecore_hwfn *p_hwfn,
 678                                  struct ecore_ptt *p_ptt,
 679                                  u64 src_addr,
 680                                  u64 dst_addr,
 681                                  u8 src_type, u8 dst_type, u32 length_dw)
 682 {
 683         dma_addr_t phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr;
 684         struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd;
 685         enum _ecore_status_t ecore_status = ECORE_SUCCESS;
 686
 687         switch (src_type) {
 688         case ECORE_DMAE_ADDRESS_GRC:
 689         case ECORE_DMAE_ADDRESS_HOST_PHYS:
 690                 cmd->src_addr_hi = OSAL_CPU_TO_LE32(DMA_HI(src_addr));
 691                 cmd->src_addr_lo = OSAL_CPU_TO_LE32(DMA_LO(src_addr));
 692                 break;
 693                 /* for virt source addresses we use the intermediate buffer. */
 694         case ECORE_DMAE_ADDRESS_HOST_VIRT:
 695                 cmd->src_addr_hi = OSAL_CPU_TO_LE32(DMA_HI(phys));
 696                 cmd->src_addr_lo = OSAL_CPU_TO_LE32(DMA_LO(phys));
 697                 OSAL_MEMCPY(&p_hwfn->dmae_info.p_intermediate_buffer[0],
 698                             (void *)(osal_uintptr_t)src_addr,
 699                             length_dw * sizeof(u32));
 700                 break;
 701         default:
 702                 return ECORE_INVAL;
 703         }
 704
 705         switch (dst_type) {
 706         case ECORE_DMAE_ADDRESS_GRC:
 707         case ECORE_DMAE_ADDRESS_HOST_PHYS:
 708                 cmd->dst_addr_hi = OSAL_CPU_TO_LE32(DMA_HI(dst_addr));
 709                 cmd->dst_addr_lo = OSAL_CPU_TO_LE32(DMA_LO(dst_addr));
 710                 break;
 711                 /* for virt destination address we use the intermediate buff. */
 712         case ECORE_DMAE_ADDRESS_HOST_VIRT:
 713                 cmd->dst_addr_hi = OSAL_CPU_TO_LE32(DMA_HI(phys));
 714                 cmd->dst_addr_lo = OSAL_CPU_TO_LE32(DMA_LO(phys));
 715                 break;
 716         default:
 717                 return ECORE_INVAL;
 718         }
 719
 720         cmd->length_dw = OSAL_CPU_TO_LE16((u16)length_dw);
 721
 722         if (src_type == ECORE_DMAE_ADDRESS_HOST_VIRT ||
 723             src_type == ECORE_DMAE_ADDRESS_HOST_PHYS)
 724                 OSAL_DMA_SYNC(p_hwfn->p_dev,
 725                               (void *)HILO_U64(cmd->src_addr_hi,
 726                                                cmd->src_addr_lo),
 727                               length_dw * sizeof(u32), false);
 728
 729         ecore_dmae_post_command(p_hwfn, p_ptt);
 730
 731         ecore_status = ecore_dmae_operation_wait(p_hwfn);
 732
 733         /* TODO - is it true ? */
 734         if (src_type == ECORE_DMAE_ADDRESS_HOST_VIRT ||
 735             src_type == ECORE_DMAE_ADDRESS_HOST_PHYS)
 736                 OSAL_DMA_SYNC(p_hwfn->p_dev,
 737                               (void *)HILO_U64(cmd->src_addr_hi,
 738                                                cmd->src_addr_lo),
 739                               length_dw * sizeof(u32), true);
 740
 741         if (ecore_status != ECORE_SUCCESS) {
 742                 DP_NOTICE(p_hwfn, ECORE_MSG_HW,
 743                           "ecore_dmae_host2grc: Wait Failed. source_addr"
 744                           " 0x%lx, grc_addr 0x%lx, size_in_dwords 0x%x\n",
 745                           (unsigned long)src_addr, (unsigned long)dst_addr,
 746                           length_dw);
 747                 return ecore_status;
 748         }
 749
 750         if (dst_type == ECORE_DMAE_ADDRESS_HOST_VIRT)
 751                 OSAL_MEMCPY((void *)(osal_uintptr_t)(dst_addr),
 752                             &p_hwfn->dmae_info.p_intermediate_buffer[0],
 753                             length_dw * sizeof(u32));
 754
 755         return ECORE_SUCCESS;
 756 }
 757
 758 static enum _ecore_status_t
 759 ecore_dmae_execute_command(struct ecore_hwfn *p_hwfn,
 760                            struct ecore_ptt *p_ptt,
 761                            u64 src_addr,
 762                            u64 dst_addr,
 763                            u8 src_type,
 764                            u8 dst_type,
 765                            u32 size_in_dwords,
 766                            struct ecore_dmae_params *p_params)
 767 {
 768         dma_addr_t phys = p_hwfn->dmae_info.completion_word_phys_addr;
 769         u16 length_cur = 0, i = 0, cnt_split = 0, length_mod = 0;
 770         struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd;
 771         u64 src_addr_split = 0, dst_addr_split = 0;
 772         u16 length_limit = DMAE_MAX_RW_SIZE;
 773         enum _ecore_status_t ecore_status = ECORE_SUCCESS;
 774         u32 offset = 0;
 775
 776         if (p_hwfn->p_dev->recov_in_prog) {
 777                 DP_VERBOSE(p_hwfn, ECORE_MSG_HW,
 778                            "Recovery is in progress. Avoid DMAE transaction [{src: addr 0x%lx, type %d}, {dst: addr 0x%lx, type %d}, size %d].\n",
 779                            (unsigned long)src_addr, src_type,
 780                            (unsigned long)dst_addr, dst_type,
 781                            size_in_dwords);
 782                 /* Return success to let the flow to be completed successfully
 783                  * w/o any error handling.
 784                  */
 785                 return ECORE_SUCCESS;
 786         }
 787
 788         ecore_dmae_opcode(p_hwfn,
 789                           (src_type == ECORE_DMAE_ADDRESS_GRC),
 790                           (dst_type == ECORE_DMAE_ADDRESS_GRC), p_params);
 791
 792         cmd->comp_addr_lo = OSAL_CPU_TO_LE32(DMA_LO(phys));
 793         cmd->comp_addr_hi = OSAL_CPU_TO_LE32(DMA_HI(phys));
 794         cmd->comp_val = OSAL_CPU_TO_LE32(DMAE_COMPLETION_VAL);
 795
 796         /* Check if the grc_addr is valid like < MAX_GRC_OFFSET */
 797         cnt_split = size_in_dwords / length_limit;
 798         length_mod = size_in_dwords % length_limit;
 799
 800         src_addr_split = src_addr;
 801         dst_addr_split = dst_addr;
 802
 803         for (i = 0; i <= cnt_split; i++) {
 804                 offset = length_limit * i;
 805
 806                 if (!(p_params->flags & ECORE_DMAE_FLAG_RW_REPL_SRC)) {
 807                         if (src_type == ECORE_DMAE_ADDRESS_GRC)
 808                                 src_addr_split = src_addr + offset;
 809                         else
 810                                 src_addr_split = src_addr + (offset * 4);
 811                 }
 812
 813                 if (dst_type == ECORE_DMAE_ADDRESS_GRC)
 814                         dst_addr_split = dst_addr + offset;
 815                 else
 816                         dst_addr_split = dst_addr + (offset * 4);
 817
 818                 length_cur = (cnt_split == i) ? length_mod : length_limit;
 819
 820                 /* might be zero on last iteration */
 821                 if (!length_cur)
 822                         continue;
 823
 824                 ecore_status = ecore_dmae_execute_sub_operation(p_hwfn,
 825                                                                 p_ptt,
 826                                                                 src_addr_split,
 827                                                                 dst_addr_split,
 828                                                                 src_type,
 829                                                                 dst_type,
 830                                                                 length_cur);
 831                 if (ecore_status != ECORE_SUCCESS) {
 832                         DP_NOTICE(p_hwfn, false,
 833                                   "ecore_dmae_execute_sub_operation Failed"
 834                                   " with error 0x%x. source_addr 0x%lx,"
 835                                   " dest addr 0x%lx, size_in_dwords 0x%x\n",
 836                                   ecore_status, (unsigned long)src_addr,
 837                                   (unsigned long)dst_addr, length_cur);
 838
 839                         ecore_hw_err_notify(p_hwfn, ECORE_HW_ERR_DMAE_FAIL);
 840                         break;
 841                 }
 842         }
 843
 844         return ecore_status;
 845 }
 846
 847 enum _ecore_status_t
 848 ecore_dmae_host2grc(struct ecore_hwfn *p_hwfn,
 849                     struct ecore_ptt *p_ptt,
 850                     u64 source_addr,
 851                     u32 grc_addr, u32 size_in_dwords, u32 flags)
 852 {
 853         u32 grc_addr_in_dw = grc_addr / sizeof(u32);
 854         struct ecore_dmae_params params;
 855         enum _ecore_status_t rc;
 856
 857         OSAL_MEMSET(&params, 0, sizeof(struct ecore_dmae_params));
 858         params.flags = flags;
 859
 860         OSAL_MUTEX_ACQUIRE(&p_hwfn->dmae_info.mutex);
 861
 862         rc = ecore_dmae_execute_command(p_hwfn, p_ptt, source_addr,
 863                                         grc_addr_in_dw,
 864                                         ECORE_DMAE_ADDRESS_HOST_VIRT,
 865                                         ECORE_DMAE_ADDRESS_GRC,
 866                                         size_in_dwords, &params);
 867
 868         OSAL_MUTEX_RELEASE(&p_hwfn->dmae_info.mutex);
 869
 870         return rc;
 871 }
 872
 873 enum _ecore_status_t
 874 ecore_dmae_grc2host(struct ecore_hwfn *p_hwfn,
 875                     struct ecore_ptt *p_ptt,
 876                     u32 grc_addr,
 877                     dma_addr_t dest_addr, u32 size_in_dwords, u32 flags)
 878 {
 879         u32 grc_addr_in_dw = grc_addr / sizeof(u32);
 880         struct ecore_dmae_params params;
 881         enum _ecore_status_t rc;
 882
 883         OSAL_MEMSET(&params, 0, sizeof(struct ecore_dmae_params));
 884         params.flags = flags;
 885
 886         OSAL_MUTEX_ACQUIRE(&p_hwfn->dmae_info.mutex);
 887
 888         rc = ecore_dmae_execute_command(p_hwfn, p_ptt, grc_addr_in_dw,
 889                                         dest_addr, ECORE_DMAE_ADDRESS_GRC,
 890                                         ECORE_DMAE_ADDRESS_HOST_VIRT,
 891                                         size_in_dwords, &params);
 892
 893         OSAL_MUTEX_RELEASE(&p_hwfn->dmae_info.mutex);
 894
 895         return rc;
 896 }
 897
 898 enum _ecore_status_t
 899 ecore_dmae_host2host(struct ecore_hwfn *p_hwfn,
 900                      struct ecore_ptt *p_ptt,
 901                      dma_addr_t source_addr,
 902                      dma_addr_t dest_addr,
 903                      u32 size_in_dwords, struct ecore_dmae_params *p_params)
 904 {
 905         enum _ecore_status_t rc;
 906
 907         OSAL_MUTEX_ACQUIRE(&p_hwfn->dmae_info.mutex);
 908
 909         rc = ecore_dmae_execute_command(p_hwfn, p_ptt, source_addr,
 910                                         dest_addr,
 911                                         ECORE_DMAE_ADDRESS_HOST_PHYS,
 912                                         ECORE_DMAE_ADDRESS_HOST_PHYS,
 913                                         size_in_dwords, p_params);
 914
 915         OSAL_MUTEX_RELEASE(&p_hwfn->dmae_info.mutex);
 916
 917         return rc;
 918 }
 919
 920 void ecore_hw_err_notify(struct ecore_hwfn *p_hwfn,
 921                          enum ecore_hw_err_type err_type)
 922 {
 923         /* Fan failure cannot be masked by handling of another HW error */
 924         if (p_hwfn->p_dev->recov_in_prog && err_type != ECORE_HW_ERR_FAN_FAIL) {
 925                 DP_VERBOSE(p_hwfn, ECORE_MSG_DRV,
 926                            "Recovery is in progress."
 927                            "Avoid notifying about HW error %d.\n",
 928                            err_type);
 929                 return;
 930         }
 931
 932         OSAL_HW_ERROR_OCCURRED(p_hwfn, err_type);
 933 }