drivers/net/qede/base/ecore_hw.c

   1 /*
   2  * Copyright (c) 2016 QLogic Corporation.
   3  * All rights reserved.
   4  * www.qlogic.com
   5  *
   6  * See LICENSE.qede_pmd for copyright and licensing details.
   7  */
   8
   9 #include "bcm_osal.h"
  10 #include "ecore_hsi_common.h"
  11 #include "ecore_status.h"
  12 #include "ecore.h"
  13 #include "ecore_hw.h"
  14 #include "reg_addr.h"
  15 #include "ecore_utils.h"
  16 #include "ecore_iov_api.h"
  17
  18 #ifndef ASIC_ONLY
  19 #define ECORE_EMUL_FACTOR 2000
  20 #define ECORE_FPGA_FACTOR 200
  21 #endif
  22
  23 #define ECORE_BAR_ACQUIRE_TIMEOUT 1000
  24
  25 /* Invalid values */
  26 #define ECORE_BAR_INVALID_OFFSET        (OSAL_CPU_TO_LE32(-1))
  27
  28 struct ecore_ptt {
  29         osal_list_entry_t list_entry;
  30         unsigned int idx;
  31         struct pxp_ptt_entry pxp;
  32         u8 hwfn_id;
  33 };
  34
  35 struct ecore_ptt_pool {
  36         osal_list_t free_list;
  37         osal_spinlock_t lock; /* ptt synchronized access */
  38         struct ecore_ptt ptts[PXP_EXTERNAL_BAR_PF_WINDOW_NUM];
  39 };
  40
  41 enum _ecore_status_t ecore_ptt_pool_alloc(struct ecore_hwfn *p_hwfn)
  42 {
  43         struct ecore_ptt_pool *p_pool = OSAL_ALLOC(p_hwfn->p_dev,
  44                                                    GFP_KERNEL,
  45                                                    sizeof(*p_pool));
  46         int i;
  47
  48         if (!p_pool)
  49                 return ECORE_NOMEM;
  50
  51         OSAL_LIST_INIT(&p_pool->free_list);
  52         for (i = 0; i < PXP_EXTERNAL_BAR_PF_WINDOW_NUM; i++) {
  53                 p_pool->ptts[i].idx = i;
  54                 p_pool->ptts[i].pxp.offset = ECORE_BAR_INVALID_OFFSET;
  55                 p_pool->ptts[i].pxp.pretend.control = 0;
  56                 p_pool->ptts[i].hwfn_id = p_hwfn->my_id;
  57
  58                 /* There are special PTT entries that are taken only by design.
  59                  * The rest are added ot the list for general usage.
  60                  */
  61                 if (i >= RESERVED_PTT_MAX)
  62                         OSAL_LIST_PUSH_HEAD(&p_pool->ptts[i].list_entry,
  63                                             &p_pool->free_list);
  64         }
  65
  66         p_hwfn->p_ptt_pool = p_pool;
  67 #ifdef CONFIG_ECORE_LOCK_ALLOC
  68         OSAL_SPIN_LOCK_ALLOC(p_hwfn, &p_pool->lock);
  69 #endif
  70         OSAL_SPIN_LOCK_INIT(&p_pool->lock);
  71
  72         return ECORE_SUCCESS;
  73 }
  74
  75 void ecore_ptt_invalidate(struct ecore_hwfn *p_hwfn)
  76 {
  77         struct ecore_ptt *p_ptt;
  78         int i;
  79
  80         for (i = 0; i < PXP_EXTERNAL_BAR_PF_WINDOW_NUM; i++) {
  81                 p_ptt = &p_hwfn->p_ptt_pool->ptts[i];
  82                 p_ptt->pxp.offset = ECORE_BAR_INVALID_OFFSET;
  83         }
  84 }
  85
  86 void ecore_ptt_pool_free(struct ecore_hwfn *p_hwfn)
  87 {
  88 #ifdef CONFIG_ECORE_LOCK_ALLOC
  89         if (p_hwfn->p_ptt_pool)
  90                 OSAL_SPIN_LOCK_DEALLOC(&p_hwfn->p_ptt_pool->lock);
  91 #endif
  92         OSAL_FREE(p_hwfn->p_dev, p_hwfn->p_ptt_pool);
  93 }
  94
  95 struct ecore_ptt *ecore_ptt_acquire(struct ecore_hwfn *p_hwfn)
  96 {
  97         struct ecore_ptt *p_ptt;
  98         unsigned int i;
  99
 100         /* Take the free PTT from the list */
 101         for (i = 0; i < ECORE_BAR_ACQUIRE_TIMEOUT; i++) {
 102                 OSAL_SPIN_LOCK(&p_hwfn->p_ptt_pool->lock);
 103                 if (!OSAL_LIST_IS_EMPTY(&p_hwfn->p_ptt_pool->free_list)) {
 104                         p_ptt = OSAL_LIST_FIRST_ENTRY(
 105                                                 &p_hwfn->p_ptt_pool->free_list,
 106                                                 struct ecore_ptt, list_entry);
 107                         OSAL_LIST_REMOVE_ENTRY(&p_ptt->list_entry,
 108                                                &p_hwfn->p_ptt_pool->free_list);
 109
 110                         OSAL_SPIN_UNLOCK(&p_hwfn->p_ptt_pool->lock);
 111
 112                         DP_VERBOSE(p_hwfn, ECORE_MSG_HW,
 113                                    "allocated ptt %d\n", p_ptt->idx);
 114
 115                         return p_ptt;
 116                 }
 117
 118                 OSAL_SPIN_UNLOCK(&p_hwfn->p_ptt_pool->lock);
 119                 OSAL_MSLEEP(1);
 120         }
 121
 122         DP_NOTICE(p_hwfn, true,
 123                   "PTT acquire timeout - failed to allocate PTT\n");
 124         return OSAL_NULL;
 125 }
 126
 127 void ecore_ptt_release(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt)
 128 {
 129         /* This PTT should not be set to pretend if it is being released */
 130         /* TODO - add some pretend sanity checks, to make sure pretend
 131          * isn't set on this ptt
 132          */
 133
 134         OSAL_SPIN_LOCK(&p_hwfn->p_ptt_pool->lock);
 135         OSAL_LIST_PUSH_HEAD(&p_ptt->list_entry, &p_hwfn->p_ptt_pool->free_list);
 136         OSAL_SPIN_UNLOCK(&p_hwfn->p_ptt_pool->lock);
 137 }
 138
 139 static u32 ecore_ptt_get_hw_addr(struct ecore_ptt *p_ptt)
 140 {
 141         /* The HW is using DWORDS and we need to translate it to Bytes */
 142         return OSAL_LE32_TO_CPU(p_ptt->pxp.offset) << 2;
 143 }
 144
 145 static u32 ecore_ptt_config_addr(struct ecore_ptt *p_ptt)
 146 {
 147         return PXP_PF_WINDOW_ADMIN_PER_PF_START +
 148             p_ptt->idx * sizeof(struct pxp_ptt_entry);
 149 }
 150
 151 u32 ecore_ptt_get_bar_addr(struct ecore_ptt *p_ptt)
 152 {
 153         return PXP_EXTERNAL_BAR_PF_WINDOW_START +
 154             p_ptt->idx * PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE;
 155 }
 156
 157 void ecore_ptt_set_win(struct ecore_hwfn *p_hwfn,
 158                        struct ecore_ptt *p_ptt, u32 new_hw_addr)
 159 {
 160         u32 prev_hw_addr;
 161
 162         prev_hw_addr = ecore_ptt_get_hw_addr(p_ptt);
 163
 164         if (new_hw_addr == prev_hw_addr)
 165                 return;
 166
 167         /* Update PTT entery in admin window */
 168         DP_VERBOSE(p_hwfn, ECORE_MSG_HW,
 169                    "Updating PTT entry %d to offset 0x%x\n",
 170                    p_ptt->idx, new_hw_addr);
 171
 172         /* The HW is using DWORDS and the address is in Bytes */
 173         p_ptt->pxp.offset = OSAL_CPU_TO_LE32(new_hw_addr >> 2);
 174
 175         REG_WR(p_hwfn,
 176                ecore_ptt_config_addr(p_ptt) +
 177                OFFSETOF(struct pxp_ptt_entry, offset),
 178                OSAL_LE32_TO_CPU(p_ptt->pxp.offset));
 179 }
 180
 181 static u32 ecore_set_ptt(struct ecore_hwfn *p_hwfn,
 182                          struct ecore_ptt *p_ptt, u32 hw_addr)
 183 {
 184         u32 win_hw_addr = ecore_ptt_get_hw_addr(p_ptt);
 185         u32 offset;
 186
 187         offset = hw_addr - win_hw_addr;
 188
 189         if (p_ptt->hwfn_id != p_hwfn->my_id)
 190                 DP_NOTICE(p_hwfn, true,
 191                           "ptt[%d] of hwfn[%02x] is used by hwfn[%02x]!\n",
 192                           p_ptt->idx, p_ptt->hwfn_id, p_hwfn->my_id);
 193
 194         /* Verify the address is within the window */
 195         if (hw_addr < win_hw_addr ||
 196             offset >= PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE) {
 197                 ecore_ptt_set_win(p_hwfn, p_ptt, hw_addr);
 198                 offset = 0;
 199         }
 200
 201         return ecore_ptt_get_bar_addr(p_ptt) + offset;
 202 }
 203
 204 struct ecore_ptt *ecore_get_reserved_ptt(struct ecore_hwfn *p_hwfn,
 205                                          enum reserved_ptts ptt_idx)
 206 {
 207         if (ptt_idx >= RESERVED_PTT_MAX) {
 208                 DP_NOTICE(p_hwfn, true,
 209                           "Requested PTT %d is out of range\n", ptt_idx);
 210                 return OSAL_NULL;
 211         }
 212
 213         return &p_hwfn->p_ptt_pool->ptts[ptt_idx];
 214 }
 215
 216 static bool ecore_is_reg_fifo_empty(struct ecore_hwfn *p_hwfn,
 217                                     struct ecore_ptt *p_ptt)
 218 {
 219         bool is_empty = true;
 220         u32 bar_addr;
 221
 222         if (!p_hwfn->p_dev->chk_reg_fifo)
 223                 goto out;
 224
 225         /* ecore_rd() cannot be used here since it calls this function */
 226         bar_addr = ecore_set_ptt(p_hwfn, p_ptt, GRC_REG_TRACE_FIFO_VALID_DATA);
 227         is_empty = REG_RD(p_hwfn, bar_addr) == 0;
 228
 229 #ifndef ASIC_ONLY
 230         if (CHIP_REV_IS_SLOW(p_hwfn->p_dev))
 231                 OSAL_UDELAY(100);
 232 #endif
 233
 234 out:
 235         return is_empty;
 236 }
 237
 238 void ecore_wr(struct ecore_hwfn *p_hwfn,
 239               struct ecore_ptt *p_ptt, u32 hw_addr, u32 val)
 240 {
 241         bool prev_fifo_err;
 242         u32 bar_addr;
 243
 244         prev_fifo_err = !ecore_is_reg_fifo_empty(p_hwfn, p_ptt);
 245
 246         bar_addr = ecore_set_ptt(p_hwfn, p_ptt, hw_addr);
 247         REG_WR(p_hwfn, bar_addr, val);
 248         DP_VERBOSE(p_hwfn, ECORE_MSG_HW,
 249                    "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n",
 250                    bar_addr, hw_addr, val);
 251
 252 #ifndef ASIC_ONLY
 253         if (CHIP_REV_IS_SLOW(p_hwfn->p_dev))
 254                 OSAL_UDELAY(100);
 255 #endif
 256
 257         OSAL_WARN(!prev_fifo_err && !ecore_is_reg_fifo_empty(p_hwfn, p_ptt),
 258                   "reg_fifo err was caused by a call to ecore_wr(0x%x, 0x%x)\n",
 259                   hw_addr, val);
 260 }
 261
 262 u32 ecore_rd(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u32 hw_addr)
 263 {
 264         bool prev_fifo_err;
 265         u32 bar_addr, val;
 266
 267         prev_fifo_err = !ecore_is_reg_fifo_empty(p_hwfn, p_ptt);
 268
 269         bar_addr = ecore_set_ptt(p_hwfn, p_ptt, hw_addr);
 270         val = REG_RD(p_hwfn, bar_addr);
 271
 272         DP_VERBOSE(p_hwfn, ECORE_MSG_HW,
 273                    "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n",
 274                    bar_addr, hw_addr, val);
 275
 276 #ifndef ASIC_ONLY
 277         if (CHIP_REV_IS_SLOW(p_hwfn->p_dev))
 278                 OSAL_UDELAY(100);
 279 #endif
 280
 281         OSAL_WARN(!prev_fifo_err && !ecore_is_reg_fifo_empty(p_hwfn, p_ptt),
 282                   "reg_fifo error was caused by a call to ecore_rd(0x%x)\n",
 283                   hw_addr);
 284
 285         return val;
 286 }
 287
 288 static void ecore_memcpy_hw(struct ecore_hwfn *p_hwfn,
 289                             struct ecore_ptt *p_ptt,
 290                             void *addr,
 291                             u32 hw_addr, osal_size_t n, bool to_device)
 292 {
 293         u32 dw_count, *host_addr, hw_offset;
 294         osal_size_t quota, done = 0;
 295         u32 OSAL_IOMEM *reg_addr;
 296
 297         while (done < n) {
 298                 quota = OSAL_MIN_T(osal_size_t, n - done,
 299                                    PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE);
 300
 301                 if (IS_PF(p_hwfn->p_dev)) {
 302                         ecore_ptt_set_win(p_hwfn, p_ptt, hw_addr + done);
 303                         hw_offset = ecore_ptt_get_bar_addr(p_ptt);
 304                 } else {
 305                         hw_offset = hw_addr + done;
 306                 }
 307
 308                 dw_count = quota / 4;
 309                 host_addr = (u32 *)((u8 *)addr + done);
 310                 reg_addr = (u32 OSAL_IOMEM *)OSAL_REG_ADDR(p_hwfn, hw_offset);
 311
 312                 if (to_device)
 313                         while (dw_count--)
 314                                 DIRECT_REG_WR(p_hwfn, reg_addr++, *host_addr++);
 315                 else
 316                         while (dw_count--)
 317                                 *host_addr++ = DIRECT_REG_RD(p_hwfn,
 318                                                              reg_addr++);
 319
 320                 done += quota;
 321         }
 322 }
 323
 324 void ecore_memcpy_from(struct ecore_hwfn *p_hwfn,
 325                        struct ecore_ptt *p_ptt,
 326                        void *dest, u32 hw_addr, osal_size_t n)
 327 {
 328         DP_VERBOSE(p_hwfn, ECORE_MSG_HW,
 329                    "hw_addr 0x%x, dest %p hw_addr 0x%x, size %lu\n",
 330                    hw_addr, dest, hw_addr, (unsigned long)n);
 331
 332         ecore_memcpy_hw(p_hwfn, p_ptt, dest, hw_addr, n, false);
 333 }
 334
 335 void ecore_memcpy_to(struct ecore_hwfn *p_hwfn,
 336                      struct ecore_ptt *p_ptt,
 337                      u32 hw_addr, void *src, osal_size_t n)
 338 {
 339         DP_VERBOSE(p_hwfn, ECORE_MSG_HW,
 340                    "hw_addr 0x%x, hw_addr 0x%x, src %p size %lu\n",
 341                    hw_addr, hw_addr, src, (unsigned long)n);
 342
 343         ecore_memcpy_hw(p_hwfn, p_ptt, src, hw_addr, n, true);
 344 }
 345
 346 void ecore_fid_pretend(struct ecore_hwfn *p_hwfn,
 347                        struct ecore_ptt *p_ptt, u16 fid)
 348 {
 349         u16 control = 0;
 350
 351         SET_FIELD(control, PXP_PRETEND_CMD_IS_CONCRETE, 1);
 352         SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_FUNCTION, 1);
 353
 354 /* Every pretend undos prev pretends, including previous port pretend */
 355
 356         SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0);
 357         SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0);
 358         SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1);
 359
 360         if (!GET_FIELD(fid, PXP_CONCRETE_FID_VFVALID))
 361                 fid = GET_FIELD(fid, PXP_CONCRETE_FID_PFID);
 362
 363         p_ptt->pxp.pretend.control = OSAL_CPU_TO_LE16(control);
 364         p_ptt->pxp.pretend.fid.concrete_fid.fid = OSAL_CPU_TO_LE16(fid);
 365
 366         REG_WR(p_hwfn,
 367                ecore_ptt_config_addr(p_ptt) +
 368                OFFSETOF(struct pxp_ptt_entry, pretend),
 369                         *(u32 *)&p_ptt->pxp.pretend);
 370 }
 371
 372 void ecore_port_pretend(struct ecore_hwfn *p_hwfn,
 373                         struct ecore_ptt *p_ptt, u8 port_id)
 374 {
 375         u16 control = 0;
 376
 377         SET_FIELD(control, PXP_PRETEND_CMD_PORT, port_id);
 378         SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 1);
 379         SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1);
 380         p_ptt->pxp.pretend.control = OSAL_CPU_TO_LE16(control);
 381
 382         REG_WR(p_hwfn,
 383                ecore_ptt_config_addr(p_ptt) +
 384                OFFSETOF(struct pxp_ptt_entry, pretend),
 385                         *(u32 *)&p_ptt->pxp.pretend);
 386 }
 387
 388 void ecore_port_unpretend(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt)
 389 {
 390         u16 control = 0;
 391
 392         SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0);
 393         SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0);
 394         SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1);
 395
 396         p_ptt->pxp.pretend.control = OSAL_CPU_TO_LE16(control);
 397
 398         REG_WR(p_hwfn,
 399                ecore_ptt_config_addr(p_ptt) +
 400                OFFSETOF(struct pxp_ptt_entry, pretend),
 401                         *(u32 *)&p_ptt->pxp.pretend);
 402 }
 403
 404 u32 ecore_vfid_to_concrete(struct ecore_hwfn *p_hwfn, u8 vfid)
 405 {
 406         u32 concrete_fid = 0;
 407
 408         SET_FIELD(concrete_fid, PXP_CONCRETE_FID_PFID, p_hwfn->rel_pf_id);
 409         SET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFID, vfid);
 410         SET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFVALID, 1);
 411
 412         return concrete_fid;
 413 }
 414
 415 /* Not in use @DPDK
 416  * Ecore HW lock
 417  * =============
 418  * Although the implementation is ready, today we don't have any flow that
 419  * utliizes said locks - and we want to keep it this way.
 420  * If this changes, this needs to be revisted.
 421  */
 422
 423 /* Ecore DMAE
 424  * =============
 425  */
 426 static void ecore_dmae_opcode(struct ecore_hwfn *p_hwfn,
 427                               const u8 is_src_type_grc,
 428                               const u8 is_dst_type_grc,
 429                               struct ecore_dmae_params *p_params)
 430 {
 431         u16 opcode_b = 0;
 432         u32 opcode = 0;
 433
 434         /* Whether the source is the PCIe or the GRC.
 435          * 0- The source is the PCIe
 436          * 1- The source is the GRC.
 437          */
 438         opcode |= (is_src_type_grc ? DMAE_CMD_SRC_MASK_GRC
 439                    : DMAE_CMD_SRC_MASK_PCIE) << DMAE_CMD_SRC_SHIFT;
 440         opcode |= (p_hwfn->rel_pf_id & DMAE_CMD_SRC_PF_ID_MASK) <<
 441             DMAE_CMD_SRC_PF_ID_SHIFT;
 442
 443         /* The destination of the DMA can be: 0-None 1-PCIe 2-GRC 3-None */
 444         opcode |= (is_dst_type_grc ? DMAE_CMD_DST_MASK_GRC
 445                    : DMAE_CMD_DST_MASK_PCIE) << DMAE_CMD_DST_SHIFT;
 446         opcode |= (p_hwfn->rel_pf_id & DMAE_CMD_DST_PF_ID_MASK) <<
 447             DMAE_CMD_DST_PF_ID_SHIFT;
 448
 449         /* DMAE_E4_TODO need to check which value to specifiy here. */
 450         /* opcode |= (!b_complete_to_host)<< DMAE_CMD_C_DST_SHIFT; */
 451
 452         /* Whether to write a completion word to the completion destination:
 453          * 0-Do not write a completion word
 454          * 1-Write the completion word
 455          */
 456         opcode |= DMAE_CMD_COMP_WORD_EN_MASK << DMAE_CMD_COMP_WORD_EN_SHIFT;
 457         opcode |= DMAE_CMD_SRC_ADDR_RESET_MASK << DMAE_CMD_SRC_ADDR_RESET_SHIFT;
 458
 459         if (p_params->flags & ECORE_DMAE_FLAG_COMPLETION_DST)
 460                 opcode |= 1 << DMAE_CMD_COMP_FUNC_SHIFT;
 461
 462         /* swapping mode 3 - big endian there should be a define ifdefed in
 463          * the HSI somewhere. Since it is currently
 464          */
 465         opcode |= DMAE_CMD_ENDIANITY << DMAE_CMD_ENDIANITY_MODE_SHIFT;
 466
 467         opcode |= p_hwfn->port_id << DMAE_CMD_PORT_ID_SHIFT;
 468
 469         /* reset source address in next go */
 470         opcode |= DMAE_CMD_SRC_ADDR_RESET_MASK << DMAE_CMD_SRC_ADDR_RESET_SHIFT;
 471
 472         /* reset dest address in next go */
 473         opcode |= DMAE_CMD_DST_ADDR_RESET_MASK << DMAE_CMD_DST_ADDR_RESET_SHIFT;
 474
 475         /* SRC/DST VFID: all 1's - pf, otherwise VF id */
 476         if (p_params->flags & ECORE_DMAE_FLAG_VF_SRC) {
 477                 opcode |= (1 << DMAE_CMD_SRC_VF_ID_VALID_SHIFT);
 478                 opcode_b |= (p_params->src_vfid << DMAE_CMD_SRC_VF_ID_SHIFT);
 479         } else {
 480                 opcode_b |= (DMAE_CMD_SRC_VF_ID_MASK <<
 481                              DMAE_CMD_SRC_VF_ID_SHIFT);
 482         }
 483         if (p_params->flags & ECORE_DMAE_FLAG_VF_DST) {
 484                 opcode |= 1 << DMAE_CMD_DST_VF_ID_VALID_SHIFT;
 485                 opcode_b |= p_params->dst_vfid << DMAE_CMD_DST_VF_ID_SHIFT;
 486         } else {
 487                 opcode_b |= DMAE_CMD_DST_VF_ID_MASK << DMAE_CMD_DST_VF_ID_SHIFT;
 488         }
 489
 490         p_hwfn->dmae_info.p_dmae_cmd->opcode = OSAL_CPU_TO_LE32(opcode);
 491         p_hwfn->dmae_info.p_dmae_cmd->opcode_b = OSAL_CPU_TO_LE16(opcode_b);
 492 }
 493
 494 static u32 ecore_dmae_idx_to_go_cmd(u8 idx)
 495 {
 496         OSAL_BUILD_BUG_ON((DMAE_REG_GO_C31 - DMAE_REG_GO_C0) != 31 * 4);
 497
 498         /* All the DMAE 'go' registers form an array in internal memory */
 499         return DMAE_REG_GO_C0 + (idx << 2);
 500 }
 501
 502 static enum _ecore_status_t ecore_dmae_post_command(struct ecore_hwfn *p_hwfn,
 503                                                     struct ecore_ptt *p_ptt)
 504 {
 505         struct dmae_cmd *p_command = p_hwfn->dmae_info.p_dmae_cmd;
 506         u8 idx_cmd = p_hwfn->dmae_info.channel, i;
 507         enum _ecore_status_t ecore_status = ECORE_SUCCESS;
 508
 509         /* verify address is not OSAL_NULL */
 510         if ((((!p_command->dst_addr_lo) && (!p_command->dst_addr_hi)) ||
 511              ((!p_command->src_addr_lo) && (!p_command->src_addr_hi)))) {
 512                 DP_NOTICE(p_hwfn, true,
 513                           "source or destination address 0 idx_cmd=%d\n"
 514                           "opcode = [0x%08x,0x%04x] len=0x%x"
 515                           " src=0x%x:%x dst=0x%x:%x\n",
 516                           idx_cmd,
 517                           OSAL_LE32_TO_CPU(p_command->opcode),
 518                           OSAL_LE16_TO_CPU(p_command->opcode_b),
 519                           OSAL_LE16_TO_CPU(p_command->length_dw),
 520                           OSAL_LE32_TO_CPU(p_command->src_addr_hi),
 521                           OSAL_LE32_TO_CPU(p_command->src_addr_lo),
 522                           OSAL_LE32_TO_CPU(p_command->dst_addr_hi),
 523                           OSAL_LE32_TO_CPU(p_command->dst_addr_lo));
 524
 525                 return ECORE_INVAL;
 526         }
 527
 528         DP_VERBOSE(p_hwfn, ECORE_MSG_HW,
 529                    "Posting DMAE command [idx %d]: opcode = [0x%08x,0x%04x]"
 530                    "len=0x%x src=0x%x:%x dst=0x%x:%x\n",
 531                    idx_cmd,
 532                    OSAL_LE32_TO_CPU(p_command->opcode),
 533                    OSAL_LE16_TO_CPU(p_command->opcode_b),
 534                    OSAL_LE16_TO_CPU(p_command->length_dw),
 535                    OSAL_LE32_TO_CPU(p_command->src_addr_hi),
 536                    OSAL_LE32_TO_CPU(p_command->src_addr_lo),
 537                    OSAL_LE32_TO_CPU(p_command->dst_addr_hi),
 538                    OSAL_LE32_TO_CPU(p_command->dst_addr_lo));
 539
 540         /* Copy the command to DMAE - need to do it before every call
 541          * for source/dest address no reset.
 542          * The number of commands have been increased to 16 (previous was 14)
 543          * The first 9 DWs are the command registers, the 10 DW is the
 544          * GO register, and
 545          * the rest are result registers (which are read only by the client).
 546          */
 547         for (i = 0; i < DMAE_CMD_SIZE; i++) {
 548                 u32 data = (i < DMAE_CMD_SIZE_TO_FILL) ?
 549                     *(((u32 *)p_command) + i) : 0;
 550
 551                 ecore_wr(p_hwfn, p_ptt,
 552                          DMAE_REG_CMD_MEM +
 553                          (idx_cmd * DMAE_CMD_SIZE * sizeof(u32)) +
 554                          (i * sizeof(u32)), data);
 555         }
 556
 557         ecore_wr(p_hwfn, p_ptt,
 558                  ecore_dmae_idx_to_go_cmd(idx_cmd), DMAE_GO_VALUE);
 559
 560         return ecore_status;
 561 }
 562
 563 enum _ecore_status_t ecore_dmae_info_alloc(struct ecore_hwfn *p_hwfn)
 564 {
 565         dma_addr_t *p_addr = &p_hwfn->dmae_info.completion_word_phys_addr;
 566         struct dmae_cmd **p_cmd = &p_hwfn->dmae_info.p_dmae_cmd;
 567         u32 **p_buff = &p_hwfn->dmae_info.p_intermediate_buffer;
 568         u32 **p_comp = &p_hwfn->dmae_info.p_completion_word;
 569
 570         *p_comp = OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, p_addr, sizeof(u32));
 571         if (*p_comp == OSAL_NULL) {
 572                 DP_NOTICE(p_hwfn, true,
 573                           "Failed to allocate `p_completion_word'\n");
 574                 goto err;
 575         }
 576
 577         p_addr = &p_hwfn->dmae_info.dmae_cmd_phys_addr;
 578         *p_cmd = OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, p_addr,
 579                                          sizeof(struct dmae_cmd));
 580         if (*p_cmd == OSAL_NULL) {
 581                 DP_NOTICE(p_hwfn, true,
 582                           "Failed to allocate `struct dmae_cmd'\n");
 583                 goto err;
 584         }
 585
 586         p_addr = &p_hwfn->dmae_info.intermediate_buffer_phys_addr;
 587         *p_buff = OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, p_addr,
 588                                           sizeof(u32) * DMAE_MAX_RW_SIZE);
 589         if (*p_buff == OSAL_NULL) {
 590                 DP_NOTICE(p_hwfn, true,
 591                           "Failed to allocate `intermediate_buffer'\n");
 592                 goto err;
 593         }
 594
 595         p_hwfn->dmae_info.channel = p_hwfn->rel_pf_id;
 596
 597         return ECORE_SUCCESS;
 598 err:
 599         ecore_dmae_info_free(p_hwfn);
 600         return ECORE_NOMEM;
 601 }
 602
 603 void ecore_dmae_info_free(struct ecore_hwfn *p_hwfn)
 604 {
 605         dma_addr_t p_phys;
 606
 607         /* Just make sure no one is in the middle */
 608         OSAL_MUTEX_ACQUIRE(&p_hwfn->dmae_info.mutex);
 609
 610         if (p_hwfn->dmae_info.p_completion_word != OSAL_NULL) {
 611                 p_phys = p_hwfn->dmae_info.completion_word_phys_addr;
 612                 OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev,
 613                                        p_hwfn->dmae_info.p_completion_word,
 614                                        p_phys, sizeof(u32));
 615                 p_hwfn->dmae_info.p_completion_word = OSAL_NULL;
 616         }
 617
 618         if (p_hwfn->dmae_info.p_dmae_cmd != OSAL_NULL) {
 619                 p_phys = p_hwfn->dmae_info.dmae_cmd_phys_addr;
 620                 OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev,
 621                                        p_hwfn->dmae_info.p_dmae_cmd,
 622                                        p_phys, sizeof(struct dmae_cmd));
 623                 p_hwfn->dmae_info.p_dmae_cmd = OSAL_NULL;
 624         }
 625
 626         if (p_hwfn->dmae_info.p_intermediate_buffer != OSAL_NULL) {
 627                 p_phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr;
 628                 OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev,
 629                                        p_hwfn->dmae_info.p_intermediate_buffer,
 630                                        p_phys, sizeof(u32) * DMAE_MAX_RW_SIZE);
 631                 p_hwfn->dmae_info.p_intermediate_buffer = OSAL_NULL;
 632         }
 633
 634         OSAL_MUTEX_RELEASE(&p_hwfn->dmae_info.mutex);
 635 }
 636
 637 static enum _ecore_status_t ecore_dmae_operation_wait(struct ecore_hwfn *p_hwfn)
 638 {
 639         u32 wait_cnt_limit = 10000, wait_cnt = 0;
 640         enum _ecore_status_t ecore_status = ECORE_SUCCESS;
 641
 642 #ifndef ASIC_ONLY
 643         u32 factor = (CHIP_REV_IS_EMUL(p_hwfn->p_dev) ?
 644                       ECORE_EMUL_FACTOR :
 645                       (CHIP_REV_IS_FPGA(p_hwfn->p_dev) ?
 646                        ECORE_FPGA_FACTOR : 1));
 647
 648         wait_cnt_limit *= factor;
 649 #endif
 650
 651         /* DMAE_E4_TODO : TODO check if we have to call any other function
 652          * other than BARRIER to sync the completion_word since we are not
 653          * using the volatile keyword for this
 654          */
 655         OSAL_BARRIER(p_hwfn->p_dev);
 656         while (*p_hwfn->dmae_info.p_completion_word != DMAE_COMPLETION_VAL) {
 657                 OSAL_UDELAY(DMAE_MIN_WAIT_TIME);
 658                 if (++wait_cnt > wait_cnt_limit) {
 659                         DP_NOTICE(p_hwfn->p_dev, ECORE_MSG_HW,
 660                                   "Timed-out waiting for operation to"
 661                                   " complete. Completion word is 0x%08x"
 662                                   " expected 0x%08x.\n",
 663                                   *p_hwfn->dmae_info.p_completion_word,
 664                                   DMAE_COMPLETION_VAL);
 665                         ecore_status = ECORE_TIMEOUT;
 666                         break;
 667                 }
 668                 /* to sync the completion_word since we are not
 669                  * using the volatile keyword for p_completion_word
 670                  */
 671                 OSAL_BARRIER(p_hwfn->p_dev);
 672         }
 673
 674         if (ecore_status == ECORE_SUCCESS)
 675                 *p_hwfn->dmae_info.p_completion_word = 0;
 676
 677         return ecore_status;
 678 }
 679
 680 static enum _ecore_status_t
 681 ecore_dmae_execute_sub_operation(struct ecore_hwfn *p_hwfn,
 682                                  struct ecore_ptt *p_ptt,
 683                                  u64 src_addr,
 684                                  u64 dst_addr,
 685                                  u8 src_type, u8 dst_type, u32 length_dw)
 686 {
 687         dma_addr_t phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr;
 688         struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd;
 689         enum _ecore_status_t ecore_status = ECORE_SUCCESS;
 690
 691         switch (src_type) {
 692         case ECORE_DMAE_ADDRESS_GRC:
 693         case ECORE_DMAE_ADDRESS_HOST_PHYS:
 694                 cmd->src_addr_hi = OSAL_CPU_TO_LE32(DMA_HI(src_addr));
 695                 cmd->src_addr_lo = OSAL_CPU_TO_LE32(DMA_LO(src_addr));
 696                 break;
 697                 /* for virt source addresses we use the intermediate buffer. */
 698         case ECORE_DMAE_ADDRESS_HOST_VIRT:
 699                 cmd->src_addr_hi = OSAL_CPU_TO_LE32(DMA_HI(phys));
 700                 cmd->src_addr_lo = OSAL_CPU_TO_LE32(DMA_LO(phys));
 701                 OSAL_MEMCPY(&p_hwfn->dmae_info.p_intermediate_buffer[0],
 702                             (void *)(osal_uintptr_t)src_addr,
 703                             length_dw * sizeof(u32));
 704                 break;
 705         default:
 706                 return ECORE_INVAL;
 707         }
 708
 709         switch (dst_type) {
 710         case ECORE_DMAE_ADDRESS_GRC:
 711         case ECORE_DMAE_ADDRESS_HOST_PHYS:
 712                 cmd->dst_addr_hi = OSAL_CPU_TO_LE32(DMA_HI(dst_addr));
 713                 cmd->dst_addr_lo = OSAL_CPU_TO_LE32(DMA_LO(dst_addr));
 714                 break;
 715                 /* for virt destination address we use the intermediate buff. */
 716         case ECORE_DMAE_ADDRESS_HOST_VIRT:
 717                 cmd->dst_addr_hi = OSAL_CPU_TO_LE32(DMA_HI(phys));
 718                 cmd->dst_addr_lo = OSAL_CPU_TO_LE32(DMA_LO(phys));
 719                 break;
 720         default:
 721                 return ECORE_INVAL;
 722         }
 723
 724         cmd->length_dw = OSAL_CPU_TO_LE16((u16)length_dw);
 725
 726         if (src_type == ECORE_DMAE_ADDRESS_HOST_VIRT ||
 727             src_type == ECORE_DMAE_ADDRESS_HOST_PHYS)
 728                 OSAL_DMA_SYNC(p_hwfn->p_dev,
 729                               (void *)HILO_U64(cmd->src_addr_hi,
 730                                                cmd->src_addr_lo),
 731                               length_dw * sizeof(u32), false);
 732
 733         ecore_dmae_post_command(p_hwfn, p_ptt);
 734
 735         ecore_status = ecore_dmae_operation_wait(p_hwfn);
 736
 737         /* TODO - is it true ? */
 738         if (src_type == ECORE_DMAE_ADDRESS_HOST_VIRT ||
 739             src_type == ECORE_DMAE_ADDRESS_HOST_PHYS)
 740                 OSAL_DMA_SYNC(p_hwfn->p_dev,
 741                               (void *)HILO_U64(cmd->src_addr_hi,
 742                                                cmd->src_addr_lo),
 743                               length_dw * sizeof(u32), true);
 744
 745         if (ecore_status != ECORE_SUCCESS) {
 746                 DP_NOTICE(p_hwfn, ECORE_MSG_HW,
 747                           "Wait Failed. source_addr 0x%lx, grc_addr 0x%lx, size_in_dwords 0x%x, intermediate buffer 0x%lx.\n",
 748                           (unsigned long)src_addr, (unsigned long)dst_addr,
 749                           length_dw,
 750                           (unsigned long)p_hwfn->dmae_info.intermediate_buffer_phys_addr);
 751                 return ecore_status;
 752         }
 753
 754         if (dst_type == ECORE_DMAE_ADDRESS_HOST_VIRT)
 755                 OSAL_MEMCPY((void *)(osal_uintptr_t)(dst_addr),
 756                             &p_hwfn->dmae_info.p_intermediate_buffer[0],
 757                             length_dw * sizeof(u32));
 758
 759         return ECORE_SUCCESS;
 760 }
 761
 762 static enum _ecore_status_t
 763 ecore_dmae_execute_command(struct ecore_hwfn *p_hwfn,
 764                            struct ecore_ptt *p_ptt,
 765                            u64 src_addr,
 766                            u64 dst_addr,
 767                            u8 src_type,
 768                            u8 dst_type,
 769                            u32 size_in_dwords,
 770                            struct ecore_dmae_params *p_params)
 771 {
 772         dma_addr_t phys = p_hwfn->dmae_info.completion_word_phys_addr;
 773         u16 length_cur = 0, i = 0, cnt_split = 0, length_mod = 0;
 774         struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd;
 775         u64 src_addr_split = 0, dst_addr_split = 0;
 776         u16 length_limit = DMAE_MAX_RW_SIZE;
 777         enum _ecore_status_t ecore_status = ECORE_SUCCESS;
 778         u32 offset = 0;
 779
 780         if (p_hwfn->p_dev->recov_in_prog) {
 781                 DP_VERBOSE(p_hwfn, ECORE_MSG_HW,
 782                            "Recovery is in progress. Avoid DMAE transaction [{src: addr 0x%lx, type %d}, {dst: addr 0x%lx, type %d}, size %d].\n",
 783                            (unsigned long)src_addr, src_type,
 784                            (unsigned long)dst_addr, dst_type,
 785                            size_in_dwords);
 786                 /* Return success to let the flow to be completed successfully
 787                  * w/o any error handling.
 788                  */
 789                 return ECORE_SUCCESS;
 790         }
 791
 792         if (!cmd) {
 793                 DP_NOTICE(p_hwfn, true,
 794                           "ecore_dmae_execute_sub_operation failed. Invalid state. source_addr 0x%lx, destination addr 0x%lx, size_in_dwords 0x%x\n",
 795                           (unsigned long)src_addr,
 796                           (unsigned long)dst_addr,
 797                           length_cur);
 798                 return ECORE_INVAL;
 799         }
 800
 801         ecore_dmae_opcode(p_hwfn,
 802                           (src_type == ECORE_DMAE_ADDRESS_GRC),
 803                           (dst_type == ECORE_DMAE_ADDRESS_GRC), p_params);
 804
 805         cmd->comp_addr_lo = OSAL_CPU_TO_LE32(DMA_LO(phys));
 806         cmd->comp_addr_hi = OSAL_CPU_TO_LE32(DMA_HI(phys));
 807         cmd->comp_val = OSAL_CPU_TO_LE32(DMAE_COMPLETION_VAL);
 808
 809         /* Check if the grc_addr is valid like < MAX_GRC_OFFSET */
 810         cnt_split = size_in_dwords / length_limit;
 811         length_mod = size_in_dwords % length_limit;
 812
 813         src_addr_split = src_addr;
 814         dst_addr_split = dst_addr;
 815
 816         for (i = 0; i <= cnt_split; i++) {
 817                 offset = length_limit * i;
 818
 819                 if (!(p_params->flags & ECORE_DMAE_FLAG_RW_REPL_SRC)) {
 820                         if (src_type == ECORE_DMAE_ADDRESS_GRC)
 821                                 src_addr_split = src_addr + offset;
 822                         else
 823                                 src_addr_split = src_addr + (offset * 4);
 824                 }
 825
 826                 if (dst_type == ECORE_DMAE_ADDRESS_GRC)
 827                         dst_addr_split = dst_addr + offset;
 828                 else
 829                         dst_addr_split = dst_addr + (offset * 4);
 830
 831                 length_cur = (cnt_split == i) ? length_mod : length_limit;
 832
 833                 /* might be zero on last iteration */
 834                 if (!length_cur)
 835                         continue;
 836
 837                 ecore_status = ecore_dmae_execute_sub_operation(p_hwfn,
 838                                                                 p_ptt,
 839                                                                 src_addr_split,
 840                                                                 dst_addr_split,
 841                                                                 src_type,
 842                                                                 dst_type,
 843                                                                 length_cur);
 844                 if (ecore_status != ECORE_SUCCESS) {
 845                         DP_NOTICE(p_hwfn, false,
 846                                   "ecore_dmae_execute_sub_operation Failed"
 847                                   " with error 0x%x. source_addr 0x%lx,"
 848                                   " dest addr 0x%lx, size_in_dwords 0x%x\n",
 849                                   ecore_status, (unsigned long)src_addr,
 850                                   (unsigned long)dst_addr, length_cur);
 851
 852                         ecore_hw_err_notify(p_hwfn, ECORE_HW_ERR_DMAE_FAIL);
 853                         break;
 854                 }
 855         }
 856
 857         return ecore_status;
 858 }
 859
 860 enum _ecore_status_t
 861 ecore_dmae_host2grc(struct ecore_hwfn *p_hwfn,
 862                     struct ecore_ptt *p_ptt,
 863                     u64 source_addr,
 864                     u32 grc_addr, u32 size_in_dwords, u32 flags)
 865 {
 866         u32 grc_addr_in_dw = grc_addr / sizeof(u32);
 867         struct ecore_dmae_params params;
 868         enum _ecore_status_t rc;
 869
 870         OSAL_MEMSET(&params, 0, sizeof(struct ecore_dmae_params));
 871         params.flags = flags;
 872
 873         OSAL_MUTEX_ACQUIRE(&p_hwfn->dmae_info.mutex);
 874
 875         rc = ecore_dmae_execute_command(p_hwfn, p_ptt, source_addr,
 876                                         grc_addr_in_dw,
 877                                         ECORE_DMAE_ADDRESS_HOST_VIRT,
 878                                         ECORE_DMAE_ADDRESS_GRC,
 879                                         size_in_dwords, &params);
 880
 881         OSAL_MUTEX_RELEASE(&p_hwfn->dmae_info.mutex);
 882
 883         return rc;
 884 }
 885
 886 enum _ecore_status_t
 887 ecore_dmae_grc2host(struct ecore_hwfn *p_hwfn,
 888                     struct ecore_ptt *p_ptt,
 889                     u32 grc_addr,
 890                     dma_addr_t dest_addr, u32 size_in_dwords, u32 flags)
 891 {
 892         u32 grc_addr_in_dw = grc_addr / sizeof(u32);
 893         struct ecore_dmae_params params;
 894         enum _ecore_status_t rc;
 895
 896         OSAL_MEMSET(&params, 0, sizeof(struct ecore_dmae_params));
 897         params.flags = flags;
 898
 899         OSAL_MUTEX_ACQUIRE(&p_hwfn->dmae_info.mutex);
 900
 901         rc = ecore_dmae_execute_command(p_hwfn, p_ptt, grc_addr_in_dw,
 902                                         dest_addr, ECORE_DMAE_ADDRESS_GRC,
 903                                         ECORE_DMAE_ADDRESS_HOST_VIRT,
 904                                         size_in_dwords, &params);
 905
 906         OSAL_MUTEX_RELEASE(&p_hwfn->dmae_info.mutex);
 907
 908         return rc;
 909 }
 910
 911 enum _ecore_status_t
 912 ecore_dmae_host2host(struct ecore_hwfn *p_hwfn,
 913                      struct ecore_ptt *p_ptt,
 914                      dma_addr_t source_addr,
 915                      dma_addr_t dest_addr,
 916                      u32 size_in_dwords, struct ecore_dmae_params *p_params)
 917 {
 918         enum _ecore_status_t rc;
 919
 920         OSAL_MUTEX_ACQUIRE(&p_hwfn->dmae_info.mutex);
 921
 922         rc = ecore_dmae_execute_command(p_hwfn, p_ptt, source_addr,
 923                                         dest_addr,
 924                                         ECORE_DMAE_ADDRESS_HOST_PHYS,
 925                                         ECORE_DMAE_ADDRESS_HOST_PHYS,
 926                                         size_in_dwords, p_params);
 927
 928         OSAL_MUTEX_RELEASE(&p_hwfn->dmae_info.mutex);
 929
 930         return rc;
 931 }
 932
 933 void ecore_hw_err_notify(struct ecore_hwfn *p_hwfn,
 934                          enum ecore_hw_err_type err_type)
 935 {
 936         /* Fan failure cannot be masked by handling of another HW error */
 937         if (p_hwfn->p_dev->recov_in_prog && err_type != ECORE_HW_ERR_FAN_FAIL) {
 938                 DP_VERBOSE(p_hwfn, ECORE_MSG_DRV,
 939                            "Recovery is in progress."
 940                            "Avoid notifying about HW error %d.\n",
 941                            err_type);
 942                 return;
 943         }
 944
 945         OSAL_HW_ERROR_OCCURRED(p_hwfn, err_type);
 946 }