imx8m: Add DDR4 DVFS support
Add DDR4 DVFS support for i.MX8M. Currently, only tested on i.MX8MM.
Signed-off-by: Bai Ping <ping.bai@nxp.com>
(cherry picked from commit 058a759ffed26797314bc0a52db3c440240fa6a5)
diff --git a/plat/imx/common/imx8m/ddr4_dvfs.c b/plat/imx/common/imx8m/ddr4_dvfs.c
new file mode 100644
index 0000000..71237f0
--- /dev/null
+++ b/plat/imx/common/imx8m/ddr4_dvfs.c
@@ -0,0 +1,355 @@
+/*
+ * Copyright 2018 NXP
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <debug.h>
+#include <ddrc.h>
+#include <dram.h>
+#include <mmio.h>
+
+unsigned fsp_init_reg[3][4] = {
+ { DDRC_INIT3(0), DDRC_INIT4(0), DDRC_INIT6(0), DDRC_INIT7(0) },
+ { DDRC_FREQ1_INIT3(0), DDRC_FREQ1_INIT4(0), DDRC_FREQ1_INIT6(0), DDRC_FREQ1_INIT7(0) },
+ { DDRC_FREQ2_INIT3(0), DDRC_FREQ2_INIT4(0), DDRC_FREQ2_INIT6(0), DDRC_FREQ2_INIT7(0) },
+};
+
+static unsigned int mr_value[7];
+
+void ddr4_mr_write(unsigned int mr, unsigned int data, unsigned int mr_type, unsigned int rank)
+{
+ unsigned int tmp, mr_mirror, data_mirror;
+ /*
+ * 1. Poll MRSTAT.mr_wr_busy until it is 0 to make sure that there is
+ * no outstanding MR transAction. No
+ */
+ do {
+ tmp = mmio_read_32(DDRC_MRSTAT(0));
+ } while (tmp & 0x1);
+
+ /*
+ * 2. Write the MRCTRL0.mr_type, MRCTRL0.mr_addr, MRCTRL0.mr_rank
+ * and (for MRWs) MRCTRL1.mr_data to define the MR transaction.
+ */
+ tmp = mmio_read_32(DDRC_DIMMCTL(0));
+ if ((tmp & 0x2) && (rank == 0x2)) {
+ mr_mirror = (mr & 0x4) | ((mr & 0x1) << 1) | ((mr & 0x2) >> 1); /* BA0, BA1 swap */
+ data_mirror = (data & 0x1607) | ((data & 0x8) << 1) | ((data & 0x10) >> 1) |
+ ((data & 0x20) << 1) | ((data & 0x40) >> 1) | ((data & 0x80) << 1) |
+ ((data & 0x100) >> 1) | ((data & 0x800) << 2) | ((data & 0x2000) >> 2) ;
+ } else {
+ mr_mirror = mr;
+ data_mirror = data;
+ }
+
+ mmio_write_32(DDRC_MRCTRL0(0), mr_type | (mr_mirror << 12) | (rank << 4) );
+ mmio_write_32(DDRC_MRCTRL1(0), data_mirror );
+
+ /*
+ * 3. In a separate APB transaction, write the MRCTRL0.mr_wr to 1. This bit is
+ * self-clearing, and triggers the MR transaction. The uMCTL2 then asserts the
+ * MRSTAT.mr_wr_busy while it performs the MR transaction to SDRAM, and no further
+ * accesses can be initiated until it is deasserted.
+ */
+ mmio_setbits_32(DDRC_MRCTRL0(0), (1 << 31));
+ do {
+ tmp = mmio_read_32(DDRC_MRSTAT(0));
+ } while (tmp);
+}
+
+void dram_cfg_all_mr(unsigned int num_rank, unsigned int pstate)
+{
+ /*
+ * 15. Perform MRS commands as required to re-program timing registers
+ * in the SDRAM for the new frequency (in particular, CL, CWL and WR
+ * may need to be changed).
+ */
+
+ for (int i = 1; i <= num_rank; i++) {
+ for (int j = 0; j < 7; j++)
+ ddr4_mr_write(j, mr_value[j], 0, i);
+ }
+}
+
+void sw_pstate(unsigned int pstate)
+{
+ volatile unsigned int tmp;
+ unsigned int i;
+
+ mmio_write_32(DDRC_SWCTL(0), 0x0000);
+ /* 12. Change the clock frequency to the desired value. */
+
+ /*
+ * 13. Update any registers which may be required to change for the new frequency. This includes
+ * quasidynamic and dynamic registers. This includes both uMCTL2 registers and PHY registers.
+ */
+ mmio_write_32(DDRC_MSTR2(0), pstate); /* UMCTL2_REGS_FREQ1 */
+ mmio_setbits_32(DDRC_MSTR(0), (0x1 << 29));
+
+ /*
+ * dvfs.18. Toggle RFSHCTL3.refresh_update_level to allow the new refresh-related
+ * register values to propagate to the refresh logic.
+ */
+ tmp = mmio_read_32(DDRC_RFSHCTL3(0));
+ if ((tmp & 0x2) == 0x2)
+ mmio_write_32(DDRC_RFSHCTL3(0), tmp & 0xFFFFFFFD);
+ else
+ mmio_write_32(DDRC_RFSHCTL3(0), tmp | 0x2);
+ /*
+ *dvfs.19. If required, trigger the initialization in the PHY. If using the gen2
+ * multiPHY, PLL initialization should be triggered at this point. See the PHY
+ * databook for details about the frequency change procedure.
+ */
+ /* before write Dynamic reg, sw_done should be 0 */
+ mmio_write_32(DDRC_DFIMISC(0), 0x00000000 | (pstate << 8));//pstate1
+ mmio_write_32(DDRC_DFIMISC(0), 0x00000020 | (pstate << 8));
+
+ /* wait DFISTAT.dfi_init_complete to 0 */
+ do {
+ tmp = 0x1 & mmio_read_32(DDRC_DFISTAT(0));
+ } while (tmp);
+
+ /* change the clock to the target frequency */
+ dram_clock_switch(pstate);
+
+ mmio_write_32(DDRC_DFIMISC(0), 0x00000000 | (pstate << 8));
+ /* wait DFISTAT.dfi_init_complete to 1 */
+ do {
+ tmp = 0x1 & mmio_read_32(DDRC_DFISTAT(0));
+ } while (!tmp);
+
+ /*
+ * When changing frequencies the controller may violate the JEDEC
+ * requirement that no more than 16 refreshes should be issued within
+ * 2*tREFI. These extra refreshes are not expected to cause a problem
+ * in the SDRAM. This issue can be avoided by waiting for at least 2*tREFI
+ * before exiting self-refresh in step 19.
+ */
+ for (i = 20; i > 0; i--)
+ printf("waiting for 2*tREF (2*7.8us)\n");
+
+ /* 14. Exit the self-refresh state by setting PWRCTL.selfref_sw = 0. */
+ mmio_clrbits_32(DDRC_PWRCTL(0), (1 << 5));
+ do {
+ tmp = 0x3f & (mmio_read_32((DDRC_STAT(0))));
+ } while (tmp == 0x23);
+}
+
+void ddr4_dll_change(unsigned int num_rank, unsigned int pstate, unsigned int cur_pstate)
+{
+ volatile unsigned int tmp;
+ enum DLL_STATE { NO_CHANGE=0, ON2OFF=1, OFF2ON=2} dll_sw; /* 0-no change, 1-on2off, 2-off2on.; */
+
+ if (pstate != 0 && cur_pstate == 0)
+ dll_sw = ON2OFF;
+ else if (pstate == 0 && cur_pstate != 0)
+ dll_sw = OFF2ON;
+ else
+ dll_sw = NO_CHANGE;
+
+ /* the the following software programming sequence to switch from DLL-on to DLL-off, or reverse: */
+ mmio_write_32(DDRC_SWCTL(0), 0x0000);
+
+ mmio_write_32(DDRC_PCTRL_0(0), 0x00000000);
+
+ /* 1. Set the DBG1.dis_hif = 1. This prevents further reads/writes being received on the HIF. */
+ mmio_setbits_32(DDRC_DBG1(0), (0x1 << 1));
+
+ /* 2. Set ZQCTL0.dis_auto_zq=1, to disable automatic generation of ZQCS/MPC(ZQ calibration) */
+ mmio_setbits_32(DDRC_FREQ1_ZQCTL0(0), (1 << 31));
+ mmio_setbits_32(DDRC_FREQ2_ZQCTL0(0), (1 << 31));
+ mmio_setbits_32(DDRC_ZQCTL0(0), (1 << 31));
+
+ /* 3. Set RFSHCTL3.dis_auto_refresh=1, to disable automatic refreshes */
+ /* mmio_setbits_32(DDRC_RFSHCTL3(0), 0x1); */
+ /* 4. Ensure all commands have been flushed from the uMCTL2 by polling */
+ do {
+ tmp = 0x36000000 & mmio_read_32(DDRC_DBGCAM(0));
+ } while (tmp != 0x36000000);
+
+ /* mmio_write_32(DDRC_PCTRL_0(0), 0x00000000); */
+
+ /* 5. Perform an MRS command (using MRCTRL0 and MRCTRL1 registers) to disable RTT_NOM: */
+ /* a. DDR3: Write 0 to MR1[9], MR1[6] and MR1[2] */
+ /* b. DDR4: Write 0 to MR1[10:8] */
+ for (int i = 1; i <= num_rank; i++) {
+ if (mr_value[1] & 0x700)
+ ddr4_mr_write(1, mr_value[1] & 0xF8FF, 0, i);
+ }
+
+ /* 6. For DDR4 only: Perform an MRS command (using MRCTRL0 and MRCTRL1 registers) to write 0 to */
+ /* MR5[8:6] to disable RTT_PARK */
+ for (int i = i; i < num_rank; i++) {
+ if (mr_value[5] & 0x1C0)
+ ddr4_mr_write(5, mr_value[5] & 0xFE3F, 0, i);
+ }
+
+ if(dll_sw == ON2OFF) {
+ /* 7. Perform an MRS command (using MRCTRL0 and MRCTRL1 registers) to write 0 to MR2[11:9], to */
+ /* disable RTT_WR (and therefore disable dynamic ODT). This applies for both DDR3 and DDR4. */
+ for (int i = 1; i <= num_rank; i++) {
+ if (mr_value[2] & 0xE00)
+ ddr4_mr_write(2, mr_value[2] & 0xF1FF, 0, i);
+ }
+
+ /* 8. Perform an MRS command (using MRCTRL0 and MRCTRL1 registers) to disable the DLL. The */
+ /* timing of this MRS is automatically handled by the uMCTL2. */
+ /* a. DDR3: Write 1 to MR1[0] */
+ /* b. DDR4: Write 0 to MR1[0] */
+ for (int i = 1; i <= num_rank; i++) {
+ ddr4_mr_write(1, mr_value[1] & 0xFFFE, 0, i);
+ }
+ }
+
+ /* 9. Put the SDRAM into self-refresh mode by setting PWRCTL.selfref_sw = 1, and polling */
+ /* STAT.operating_mode to ensure the DDRC has entered self-refresh. */
+ mmio_setbits_32(DDRC_PWRCTL(0), (1 << 5));
+ /*
+ * 10. Wait until STAT.operating_mode[1:0]==11 indicating that the DWC_ddr_umctl2
+ * core is in selfrefresh mode. Ensure transition to self-refresh was due to software
+ * by checking that
+ */
+ /* STAT.selfref_type[1:0]=2`b10. */
+ do {
+ tmp = 0x3f & (mmio_read_32((DDRC_STAT(0))));
+ } while (tmp != 0x23);
+
+ /* FIXME Set RFSHCTL3.dis_auto_refresh=1, to disable automatic refreshes ??? */
+ mmio_setbits_32(DDRC_RFSHCTL3(0), 0x1);
+
+ /* 11. Set the MSTR.dll_off_mode = 1 or 0. */
+ if (dll_sw == ON2OFF)
+ mmio_setbits_32(DDRC_MSTR(0), (1 << 15));
+
+ if(dll_sw == OFF2ON)
+ mmio_clrbits_32(DDRC_MSTR(0), (1 << 15));
+
+ sw_pstate(pstate);
+
+ /* DRAM dll enable */
+ if (dll_sw == OFF2ON) {
+ for (int i = 1; i <= num_rank; i++) {
+ ddr4_mr_write(1, mr_value[1] | 0x1, 0, i);
+ }
+
+ for (int i = 1; i <= num_rank; i++) {
+ ddr4_mr_write(0, mr_value[0] | 0x100, 0, i);
+ }
+ }
+
+ dram_cfg_all_mr(num_rank, pstate);
+
+ /* 16. Re-enable automatic generation of ZQCS/MPC(ZQ calibration) commands */
+ mmio_clrbits_32(DDRC_FREQ1_ZQCTL0(0), (1 << 31));
+ mmio_clrbits_32(DDRC_FREQ2_ZQCTL0(0), (1 << 31));
+ mmio_clrbits_32(DDRC_ZQCTL0(0), (1 << 31));
+
+ /* 17. Re-enable automatic refreshes (RFSHCTL3.dis_auto_refresh = 0) if they have been previously disabled. */
+ mmio_clrbits_32(DDRC_RFSHCTL3(0), 0x1);
+ /* 18. Restore ZQCTL0.dis_srx_zqcl */
+ /* 19. Write DBG1.dis_hif = 0 to re-enable reads and writes. */
+ mmio_clrbits_32(DDRC_DBG1(0), (0x1 << 1));
+
+ mmio_write_32(DDRC_PCTRL_0(0), 0x00000001);
+ /* 27. Write 1 to SBRCTL.scrub_en. Enable SBR if desired, only required if SBR instantiated. */
+
+ /* set SWCTL.sw_done to enable quasi-dynamic register programming outside reset. */
+ mmio_write_32(DDRC_SWCTL(0), 0x0001);
+
+ /* wait SWSTAT.sw_done_ack to 1 */
+ do {
+ tmp = 0x1 & mmio_read_32(DDRC_SWSTAT(0));
+ } while (!tmp);
+}
+
+void ddr4_dll_no_change(unsigned int num_rank, unsigned int pstate)
+{
+ volatile unsigned int tmp;
+ /* 1. set SWCTL.sw_done to disable quasi-dynamic register programming outside reset. */
+ mmio_write_32(DDRC_SWCTL(0), 0x0000);
+
+ /* 2. Write 0 to PCTRL_n.port_en. This blocks AXI port(s) from taking any transaction (blocks traffic on
+ AXI ports). */
+ mmio_write_32(DDRC_PCTRL_0(0), 0x00000000);
+
+ /* 3. Poll PSTAT.rd_port_busy_n=0 and PSTAT.wr_port_busy_n=0. Wait until all AXI ports are idle (the
+ uMCTL2 core has to be idle). */
+ do {
+ tmp = mmio_read_32(DDRC_PSTAT(0));
+ } while (tmp & 0x10001);
+
+ /* 4. Write 0 to SBRCTL.scrub_en. Disable SBR, required only if SBR instantiated. */
+ /* 5. Poll SBRSTAT.scrub_busy=0. Indicates that there are no outstanding SBR read commands */
+ /* 6. Set DERATEEN.derate_enable = 0, if DERATEEN.derate_eanble = 1 and the read latency (RL) value */
+ //needs to change after the frequency change (LPDDR2/3/4 only).
+ /* 7. Set DBG1.dis_hif=1 so that no new commands will be accepted by the uMCTL2. */
+ mmio_setbits_32(DDRC_DBG1(0), (0x1 << 1));
+ /* 8. Poll DBGCAM.dbg_wr_q_empty and DBGCAM.dbg_rd_q_empty to ensure that write and read data buffers are empty. */
+ do {
+ tmp = 0x06000000 & mmio_read_32(DDRC_DBGCAM(0));
+ } while (tmp != 0x06000000);
+ /* 9. For DDR4, update MR6 with the new tDLLK value via the Mode Register Write signals */
+
+ /* 10. Set DFILPCFG0.dfi_lp_en_sr = 0, if DFILPCFG0.dfi_lp_en_sr = 1, and wait until DFISTAT.dfi_lp_ack */
+ /* 11. If DFI PHY Master interface is active in uMCTL2 (DFIPHYMSTR.phymstr_en == 1'b1) then disable it */
+ /* 12. Wait until STAT.operating_mode[1:0]!=11 indicating that the DWC_ddr_umctl2 controller is not in self-refresh mode. */
+ tmp = 0x3 & (mmio_read_32((DDRC_STAT(0))));
+ if (tmp == 0x3) {
+ printf("C: Error DRAM should not in Self Refresh\n");
+ }
+ /* 13. Assert PWRCTL.selfref_sw for the DWC_ddr_umctl2 core to enter the self-refresh mode. */
+ mmio_setbits_32(DDRC_PWRCTL(0), (1 << 5));
+ /* 14. Wait until STAT.operating_mode[1:0]==11 indicating that the DWC_ddr_umctl2 core is in selfrefresh mode. */
+ do {
+ tmp = 0x3f & (mmio_read_32((DDRC_STAT(0))));
+ } while (tmp != 0x23);
+
+ sw_pstate(pstate);
+ dram_cfg_all_mr(num_rank, pstate);
+
+ /* 23. Enable HIF commands by setting DBG1.dis_hif=0. */
+ mmio_clrbits_32(DDRC_DBG1(0), (0x1 << 1));
+ /* 24. Reset DERATEEN.derate_enable = 1 if DERATEEN.derate_enable has been set to 0 in step 6. */
+ /* 25. If DFI PHY Master interface was active in uMCTL2 (DFIPHYMSTR.phymstr_en == 1'b1) before the */
+ /* step 11 then enable it back by programming DFIPHYMSTR.phymstr_en = 1'b1. */
+ /* 26. Write 1 to PCTRL_n.port_en. AXI port(s) are no longer blocked from taking transactions (Re-enable traffic on AXI ports). */
+ /* Re-enable traffic on the AXI ports */
+ mmio_write_32(DDRC_PCTRL_0(0), 0x00000001);
+ /* 27. Write 1 to SBRCTL.scrub_en. Enable SBR if desired, only required if SBR instantiated. */
+
+ /* set SWCTL.sw_done to enable quasi-dynamic register programming outside reset. */
+ mmio_write_32(DDRC_SWCTL(0), 0x0001);
+
+ /* wait SWSTAT.sw_done_ack to 1 */
+ do {
+ tmp = 0x1 & mmio_read_32(DDRC_SWSTAT(0));
+ } while (!tmp);
+}
+
+void get_mr_values(unsigned int pstate)
+{
+ uint32_t init_val;
+
+ for (int i = 0; i < 3; i++) {
+ init_val = mmio_read_32(fsp_init_reg[pstate][i]);
+ mr_value[2*i] = init_val >> 16;
+ mr_value[2*i + 1] = init_val & 0xFFFF;
+ }
+
+ mr_value[6] = mmio_read_32(fsp_init_reg[pstate][3]) & 0xFFFF;
+}
+
+void ddr4_swffc(struct dram_info *dram_info, unsigned int pstate)
+{
+ unsigned int cur_pstate = dram_info->current_fsp;
+
+ get_mr_values(pstate);
+
+ if ((pstate == 0 && cur_pstate !=0) || (pstate != 0 && cur_pstate == 0))
+ ddr4_dll_change(dram_info->num_rank, pstate, cur_pstate);
+ else
+ ddr4_dll_no_change(dram_info->num_rank, pstate);
+
+ dram_info->current_fsp = pstate;
+}
diff --git a/plat/imx/common/imx8m/dram.c b/plat/imx/common/imx8m/dram.c
index e00ca96..df34c4b 100644
--- a/plat/imx/common/imx8m/dram.c
+++ b/plat/imx/common/imx8m/dram.c
@@ -67,13 +67,17 @@
}
}
+
void dram_info_init(unsigned long dram_timing_base)
{
- uint32_t current_fsp, ddr_type;
+ uint32_t current_fsp, ddr_type, ddrc_mstr;
/* get the dram type */
- ddr_type = mmio_read_32(DDRC_MSTR(0)) & DDR_TYPE_MASK;
+ ddrc_mstr = mmio_read_32(DDRC_MSTR(0));
+ ddr_type = ddrc_mstr & DDR_TYPE_MASK;
+
dram_info.dram_type = ddr_type;
+ dram_info.num_rank = (ddrc_mstr >> 24) & DDRC_ACTIVE_RANK_MASK;
/* init the boot_fsp & current_fsp */
current_fsp = mmio_read_32(DDRC_DFIMISC(0));
@@ -93,6 +97,10 @@
dcsw_op_all(DCCSW);
lpddr4_swffc(dev_fsp, 0x0);
dev_fsp = (~dev_fsp) & 0x1;
+ } else if (ddr_type == DDRC_DDR4 && current_fsp != 0x0) {
+ /* flush the L1/L2 cache */
+ dcsw_op_all(DCCSW);
+ ddr4_swffc(&dram_info, 0x0);
}
}
@@ -122,10 +130,6 @@
unsigned int target_freq = x1;
uint32_t online_cores = x2;
- /* TODO add ddr4 dvfs support later */
- if (dram_info.dram_type != DDRC_LPDDR4)
- return 0;
-
if (target_freq == 0xf) {
/* set the WFE done status */
spin_lock(&dfs_lock);
@@ -159,8 +163,12 @@
/* flush the L1/L2 cache */
dcsw_op_all(DCCSW);
- lpddr4_swffc(dev_fsp, target_freq);
- dev_fsp = (~dev_fsp) & 0x1;
+ if (dram_info.dram_type == DDRC_LPDDR4) {
+ lpddr4_swffc(dev_fsp, target_freq);
+ dev_fsp = (~dev_fsp) & 0x1;
+ } else if (dram_info.dram_type == DDRC_DDR4) {
+ ddr4_swffc(&dram_info, target_freq);
+ }
wait_ddrc_hwffc_done = false;
wfe_done = 0;
diff --git a/plat/imx/common/include/dram.h b/plat/imx/common/include/dram.h
index fa203da..f3d72dd 100644
--- a/plat/imx/common/include/dram.h
+++ b/plat/imx/common/include/dram.h
@@ -30,6 +30,7 @@
#define DDRC_LPDDR4 BIT(5)
#define DDRC_DDR4 BIT(4)
#define DDR_TYPE_MASK 0x3f
+#define DDRC_ACTIVE_RANK_MASK 0x3
#define DDRPHY_REG(x) (0x3c000000 + 4*x)
@@ -59,6 +60,7 @@
struct dram_info {
int dram_type;
+ unsigned int num_rank;
int current_fsp;
int boot_fsp;
struct dram_timing_info *timing_info;
@@ -78,6 +80,8 @@
void ddr4_exit_retention(void);
/* lpddr4 swffc for dvfs */
void lpddr4_swffc(unsigned int dev_fsp, unsigned int tgt_freq);
+/* ddr4 swffw for dvfs */
+void ddr4_swffc(struct dram_info *dram_info, unsigned int target_fsp);
/* dram retention */
void dram_enter_retention(void);
diff --git a/plat/imx/imx8mm/platform.mk b/plat/imx/imx8mm/platform.mk
index 4c922f0..7707332 100644
--- a/plat/imx/imx8mm/platform.mk
+++ b/plat/imx/imx8mm/platform.mk
@@ -14,7 +14,8 @@
plat/imx/common/imx8m/lpddr4_retention.c \
plat/imx/common/imx8m/ddr4_retention.c \
plat/imx/common/imx8m/lpddr4_helper.c \
- plat/imx/common/imx8m/lpddr4_dvfs.c
+ plat/imx/common/imx8m/lpddr4_dvfs.c \
+ plat/imx/common/imx8m/ddr4_dvfs.c
BL31_SOURCES += plat/imx/common/imx8_helpers.S \
plat/imx/common/mxcuart_console.S \
