plat/intel/soc/agilex/soc/agilex_memory_controller.c - tf-a-mtk - Git at Google

 /*
  * Copyright (c) 2019, Intel Corporation. All rights reserved.
  *
  * SPDX-License-Identifier: BSD-3-Clause
  */

 #include <errno.h>
 #include <lib/mmio.h>
 #include <lib/utils.h>
 #include <common/debug.h>
 #include <drivers/delay_timer.h>
 #include <platform_def.h>

 #include "agilex_memory_controller.h"

 #define ALT_CCU_NOC_DI_SET_MSK		0x10

 #define DDR_READ_LATENCY_DELAY		40
 #define MAX_MEM_CAL_RETRY		3
 #define PRE_CALIBRATION_DELAY		1
 #define POST_CALIBRATION_DELAY		1
 #define TIMEOUT_EMIF_CALIBRATION	1000
 #define CLEAR_EMIF_DELAY		50000
 #define CLEAR_EMIF_TIMEOUT		0x100000
 #define TIMEOUT_INT_RESP		10000

 #define DDR_CONFIG(A, B, C, R)	(((A) << 24) | ((B) << 16) | ((C) << 8) | (R))
 #define DDR_CONFIG_ELEMENTS	(sizeof(ddr_config)/sizeof(uint32_t))

 /* tWR = Min. 15ns constant, see JEDEC standard eg. DDR4 is JESD79-4.pdf */
 #define tWR_IN_NS 15

 void configure_hmc_adaptor_regs(void);
 void configure_ddr_sched_ctrl_regs(void);

 /* The followring are the supported configurations */
 uint32_t ddr_config[] = {
 	/* DDR_CONFIG(Address order,Bank,Column,Row) */
 	/* List for DDR3 or LPDDR3 (pinout order > chip, row, bank, column) */
 	DDR_CONFIG(0, 3, 10, 12),
 	DDR_CONFIG(0, 3,  9, 13),
 	DDR_CONFIG(0, 3, 10, 13),
 	DDR_CONFIG(0, 3,  9, 14),
 	DDR_CONFIG(0, 3, 10, 14),
 	DDR_CONFIG(0, 3, 10, 15),
 	DDR_CONFIG(0, 3, 11, 14),
 	DDR_CONFIG(0, 3, 11, 15),
 	DDR_CONFIG(0, 3, 10, 16),
 	DDR_CONFIG(0, 3, 11, 16),
 	DDR_CONFIG(0, 3, 12, 15),	/* 0xa */
 	/* List for DDR4 only (pinout order > chip, bank, row, column) */
 	DDR_CONFIG(1, 3, 10, 14),
 	DDR_CONFIG(1, 4, 10, 14),
 	DDR_CONFIG(1, 3, 10, 15),
 	DDR_CONFIG(1, 4, 10, 15),
 	DDR_CONFIG(1, 3, 10, 16),
 	DDR_CONFIG(1, 4, 10, 16),
 	DDR_CONFIG(1, 3, 10, 17),
 	DDR_CONFIG(1, 4, 10, 17),
 };

 static int match_ddr_conf(uint32_t ddr_conf)
 {
 	int i;

 	for (i = 0; i < DDR_CONFIG_ELEMENTS; i++) {
 		if (ddr_conf == ddr_config[i])
 			return i;
 	}
 	return 0;
 }

 static int check_hmc_clk(void)
 {
 	unsigned long timeout = 0;
 	uint32_t hmc_clk;

 	do {
 		hmc_clk = mmio_read_32(AGX_SYSMGR_CORE_HMC_CLK);
 		if (hmc_clk & AGX_SYSMGR_CORE_HMC_CLK_STATUS)
 			break;
 		udelay(1);
 	} while (++timeout < 1000);
 	if (timeout >= 1000)
 		return -ETIMEDOUT;

 	return 0;
 }

 static int clear_emif(void)
 {
 	uint32_t data;
 	unsigned long timeout;

 	mmio_write_32(AGX_MPFE_HMC_ADP_RSTHANDSHAKECTRL, 0);

 	timeout = 0;
 	do {
 		data = mmio_read_32(AGX_MPFE_HMC_ADP_RSTHANDSHAKESTAT);
 		if ((data & AGX_MPFE_HMC_ADP_RSTHANDSHAKESTAT_SEQ2CORE) == 0)
 			break;
 		udelay(CLEAR_EMIF_DELAY);
 	} while (++timeout < CLEAR_EMIF_TIMEOUT);
 	if (timeout >= CLEAR_EMIF_TIMEOUT)
 		return -ETIMEDOUT;

 	return 0;
 }

 static int mem_calibration(void)
 {
 	int status;
 	uint32_t data;
 	unsigned long timeout;
 	unsigned long retry = 0;

 	udelay(PRE_CALIBRATION_DELAY);

 	do {
 		if (retry != 0)
 			INFO("DDR: Retrying DRAM calibration\n");

 		timeout = 0;
 		do {
 			data = mmio_read_32(AGX_MPFE_HMC_ADP_DDRCALSTAT);
 			if (AGX_MPFE_HMC_ADP_DDRCALSTAT_CAL(data) == 1)
 				break;
 			mdelay(1);
 		} while (++timeout < TIMEOUT_EMIF_CALIBRATION);

 		if (AGX_MPFE_HMC_ADP_DDRCALSTAT_CAL(data) == 0) {
 			status = clear_emif();
 			if (status)
 				ERROR("Failed to clear Emif\n");
 		} else {
 			break;
 		}
 	} while (++retry < MAX_MEM_CAL_RETRY);

 	if (AGX_MPFE_HMC_ADP_DDRCALSTAT_CAL(data) == 0) {
 		ERROR("DDR: DRAM calibration failed.\n");
 		status = -EIO;
 	} else {
 		INFO("DDR: DRAM calibration success.\n");
 		status = 0;
 	}

 	udelay(POST_CALIBRATION_DELAY);

 	return status;
 }

 int init_hard_memory_controller(void)
 {
 	int status;

 	status = check_hmc_clk();
 	if (status) {
 		ERROR("DDR: Error, HMC clock not running\n");
 		return status;
 	}

 	status = mem_calibration();
 	if (status) {
 		ERROR("DDR: Memory Calibration Failed\n");
 		return status;
 	}

 	configure_hmc_adaptor_regs();

 	return 0;
 }

 void configure_ddr_sched_ctrl_regs(void)
 {
 	uint32_t data, dram_addr_order, ddr_conf, bank, row, col,
 		rd_to_miss, wr_to_miss, burst_len, burst_len_ddr_clk,
 		burst_len_sched_clk, act_to_act, rd_to_wr, wr_to_rd, bw_ratio,
 		t_rtp, t_rp, t_rcd, rd_latency, tw_rin_clk_cycles,
 		bw_ratio_extended, auto_precharge = 0, act_to_act_bank, faw,
 		faw_bank, bus_rd_to_rd, bus_rd_to_wr, bus_wr_to_rd;

 	INFO("Init HPS NOC's DDR Scheduler.\n");

 	data = mmio_read_32(AGX_MPFE_IOHMC_CTRLCFG1);
 	dram_addr_order = AGX_MPFE_IOHMC_CTRLCFG1_CFG_ADDR_ORDER(data);

 	data = mmio_read_32(AGX_MPFE_IOHMC_DRAMADDRW);

 	col  = IOHMC_DRAMADDRW_COL_ADDR_WIDTH(data);
 	row  = IOHMC_DRAMADDRW_ROW_ADDR_WIDTH(data);
 	bank = IOHMC_DRAMADDRW_BANK_ADDR_WIDTH(data) +
 		IOHMC_DRAMADDRW_BANK_GRP_ADDR_WIDTH(data);

 	ddr_conf = match_ddr_conf(DDR_CONFIG(dram_addr_order, bank, col, row));

 	if (ddr_conf) {
 		mmio_clrsetbits_32(
 			AGX_MPFE_DDR_MAIN_SCHED_DDRCONF,
 			AGX_MPFE_DDR_MAIN_SCHED_DDRCONF_SET_MSK,
 			AGX_MPFE_DDR_MAIN_SCHED_DDRCONF_SET(ddr_conf));
 	} else {
 		ERROR("DDR: Cannot find predefined ddrConf configuration.\n");
 	}

 	mmio_write_32(AGX_MPFE_HMC_ADP(ADP_DRAMADDRWIDTH), data);

 	data = mmio_read_32(AGX_MPFE_IOHMC_DRAMTIMING0);
 	rd_latency = AGX_MPFE_IOHMC_REG_DRAMTIMING0_CFG_TCL(data);

 	data = mmio_read_32(AGX_MPFE_IOHMC_CALTIMING0);
 	act_to_act = ACT_TO_ACT(data);
 	t_rcd = ACT_TO_RDWR(data);
 	act_to_act_bank = ACT_TO_ACT_DIFF_BANK(data);

 	data = mmio_read_32(AGX_MPFE_IOHMC_CALTIMING1);
 	rd_to_wr = RD_TO_WR(data);
 	bus_rd_to_rd = RD_TO_RD_DIFF_CHIP(data);
 	bus_rd_to_wr = RD_TO_WR_DIFF_CHIP(data);

 	data = mmio_read_32(AGX_MPFE_IOHMC_CALTIMING2);
 	t_rtp = RD_TO_PCH(data);

 	data = mmio_read_32(AGX_MPFE_IOHMC_CALTIMING3);
 	wr_to_rd = CALTIMING3_WR_TO_RD(data);
 	bus_wr_to_rd = CALTIMING3_WR_TO_RD_DIFF_CHIP(data);

 	data = mmio_read_32(AGX_MPFE_IOHMC_CALTIMING4);
 	t_rp = PCH_TO_VALID(data);

 	data = mmio_read_32(AGX_MPFE_HMC_ADP(HMC_ADP_DDRIOCTRL));
 	bw_ratio = ((HMC_ADP_DDRIOCTRL_IO_SIZE(data) == 0) ? 0 : 1);

 	data = mmio_read_32(AGX_MPFE_IOHMC_CTRLCFG0);
 	burst_len = HMC_ADP_DDRIOCTRL_CTRL_BURST_LENGTH(data);
 	burst_len_ddr_clk = burst_len / 2;
 	burst_len_sched_clk = ((burst_len/2) / 2);

 	data = mmio_read_32(AGX_MPFE_IOHMC_CTRLCFG0);
 	switch (AGX_MPFE_IOHMC_REG_CTRLCFG0_CFG_MEM_TYPE(data)) {
 	case 1:
 		/* DDR4 - 1333MHz */
 		/* 20 (19.995) clock cycles = 15ns */
 		/* Calculate with rounding */
 		tw_rin_clk_cycles = (((tWR_IN_NS * 1333) % 1000) >= 500) ?
 			((tWR_IN_NS * 1333) / 1000) + 1 :
 			((tWR_IN_NS * 1333) / 1000);
 		break;
 	default:
 		/* Others - 1066MHz or slower */
 		/* 16 (15.990) clock cycles = 15ns */
 		/* Calculate with rounding */
 		tw_rin_clk_cycles = (((tWR_IN_NS * 1066) % 1000) >= 500) ?
 			((tWR_IN_NS * 1066) / 1000) + 1 :
 			((tWR_IN_NS * 1066) / 1000);
 		break;
 	}

 	rd_to_miss = t_rtp + t_rp + t_rcd - burst_len_sched_clk;
 	wr_to_miss = ((rd_latency + burst_len_ddr_clk + 2 + tw_rin_clk_cycles)
 			/ 2) - rd_to_wr + t_rp + t_rcd;

 	mmio_write_32(AGX_MPFE_DDR_MAIN_SCHED_DDRTIMING,
 		bw_ratio << DDRTIMING_BWRATIO_OFST |
 		wr_to_rd << DDRTIMING_WRTORD_OFST|
 		rd_to_wr << DDRTIMING_RDTOWR_OFST |
 		burst_len_sched_clk << DDRTIMING_BURSTLEN_OFST |
 		wr_to_miss << DDRTIMING_WRTOMISS_OFST |
 		rd_to_miss << DDRTIMING_RDTOMISS_OFST |
 		act_to_act << DDRTIMING_ACTTOACT_OFST);

 	data = mmio_read_32(AGX_MPFE_HMC_ADP(HMC_ADP_DDRIOCTRL));
 	bw_ratio_extended = ((ADP_DDRIOCTRL_IO_SIZE(data) == 0) ? 1 : 0);

 	mmio_write_32(AGX_MPFE_DDR_MAIN_SCHED_DDRMODE,
 		bw_ratio_extended << DDRMODE_BWRATIOEXTENDED_OFST |
 		auto_precharge << DDRMODE_AUTOPRECHARGE_OFST);

 	mmio_write_32(AGX_MPFE_DDR_MAIN_SCHED_READLATENCY,
 		(rd_latency / 2) + DDR_READ_LATENCY_DELAY);

 	data = mmio_read_32(AGX_MPFE_IOHMC_CALTIMING9);
 	faw = AGX_MPFE_IOHMC_CALTIMING9_ACT_TO_ACT(data);

 	faw_bank = 1; // always 1 because we always have 4 bank DDR.

 	mmio_write_32(AGX_MPFE_DDR_MAIN_SCHED_ACTIVATE,
 		faw_bank << AGX_MPFE_DDR_MAIN_SCHED_ACTIVATE_FAWBANK_OFST |
 		faw << AGX_MPFE_DDR_MAIN_SCHED_ACTIVATE_FAW_OFST |
 		act_to_act_bank << AGX_MPFE_DDR_MAIN_SCHED_ACTIVATE_RRD_OFST);

 	mmio_write_32(AGX_MPFE_DDR_MAIN_SCHED_DEVTODEV,
 		((bus_rd_to_rd
 			<< AGX_MPFE_DDR_MAIN_SCHED_DEVTODEV_BUSRDTORD_OFST)
 			& AGX_MPFE_DDR_MAIN_SCHED_DEVTODEV_BUSRDTORD_MSK) |
 		((bus_rd_to_wr
 			<< AGX_MPFE_DDR_MAIN_SCHED_DEVTODEV_BUSRDTOWR_OFST)
 			& AGX_MPFE_DDR_MAIN_SCHED_DEVTODEV_BUSRDTOWR_MSK) |
 		((bus_wr_to_rd
 			<< AGX_MPFE_DDR_MAIN_SCHED_DEVTODEV_BUSWRTORD_OFST)
 			& AGX_MPFE_DDR_MAIN_SCHED_DEVTODEV_BUSWRTORD_MSK));

 }

 unsigned long get_physical_dram_size(void)
 {
 	uint32_t data;
 	unsigned long ram_addr_width, ram_ext_if_io_width;

 	data = mmio_read_32(AGX_MPFE_HMC_ADP_DDRIOCTRL);
 	switch (AGX_MPFE_HMC_ADP_DDRIOCTRL_IO_SIZE(data)) {
 	case 0:
 		ram_ext_if_io_width = 16;
 		break;
 	case 1:
 		ram_ext_if_io_width = 32;
 		break;
 	case 2:
 		ram_ext_if_io_width = 64;
 		break;
 	default:
 		ram_ext_if_io_width = 0;
 		break;
 	}

 	data = mmio_read_32(AGX_MPFE_IOHMC_REG_DRAMADDRW);
 	ram_addr_width = IOHMC_DRAMADDRW_CFG_COL_ADDR_WIDTH(data) +
 		IOHMC_DRAMADDRW_CFG_ROW_ADDR_WIDTH(data) +
 		IOHMC_DRAMADDRW_CFG_BANK_ADDR_WIDTH(data) +
 		IOHMC_DRAMADDRW_CFG_BANK_GROUP_ADDR_WIDTH(data) +
 		IOHMC_DRAMADDRW_CFG_CS_ADDR_WIDTH(data);

 	return (1 << ram_addr_width) * (ram_ext_if_io_width / 8);
 }


 void configure_hmc_adaptor_regs(void)
 {
 	uint32_t data;
 	uint32_t dram_io_width;

 	/* Configure DDR data rate */
 	dram_io_width = AGX_MPFE_IOHMC_NIOSRESERVE0_NIOS_RESERVE0(
 		mmio_read_32(AGX_MPFE_IOHMC_REG_NIOSRESERVE0_OFST));
 	dram_io_width = (dram_io_width & 0xFF) >> 5;

 	data = mmio_read_32(AGX_MPFE_IOHMC_CTRLCFG3);

 	dram_io_width |= (data & 0x4);

 	mmio_write_32(AGX_MPFE_HMC_ADP_DDRIOCTRL, dram_io_width);

 	/* Copy dram addr width from IOHMC to HMC ADP */
 	data = mmio_read_32(AGX_MPFE_IOHMC_DRAMADDRW);
 	mmio_write_32(AGX_MPFE_HMC_ADP(ADP_DRAMADDRWIDTH), data);

 	/* Enable nonsecure access to DDR */
 	data = get_physical_dram_size();

 	if (data < AGX_DDR_SIZE)
 		data = AGX_DDR_SIZE;

 	mmio_write_32(AGX_NOC_FW_DDR_SCR_MPUREGION0ADDR_LIMIT, data - 1);
 	mmio_write_32(AGX_NOC_FW_DDR_SCR_MPUREGION0ADDR_LIMITEXT, 0x1f);

 	mmio_write_32(AGX_NOC_FW_DDR_SCR_NONMPUREGION0ADDR_LIMIT, data - 1);

 	mmio_write_32(AGX_SOC_NOC_FW_DDR_SCR_ENABLESET, BIT(0) | BIT(8));

 	/* ECC enablement */
 	data = mmio_read_32(AGX_MPFE_IOHMC_REG_CTRLCFG1);
 	if (data & (1 << AGX_IOHMC_CTRLCFG1_ENABLE_ECC_OFST)) {
 		mmio_clrsetbits_32(AGX_MPFE_HMC_ADP_ECCCTRL1,
 			AGX_MPFE_HMC_ADP_ECCCTRL1_AUTOWB_CNT_RST_SET_MSK |
 			AGX_MPFE_HMC_ADP_ECCCTRL1_CNT_RST_SET_MSK |
 			AGX_MPFE_HMC_ADP_ECCCTRL1_ECC_EN_SET_MSK,
 			AGX_MPFE_HMC_ADP_ECCCTRL1_AUTOWB_CNT_RST_SET_MSK |
 			AGX_MPFE_HMC_ADP_ECCCTRL1_CNT_RST_SET_MSK);

 		mmio_clrsetbits_32(AGX_MPFE_HMC_ADP_ECCCTRL2,
 			AGX_MPFE_HMC_ADP_ECCCTRL2_OVRW_RB_ECC_EN_SET_MSK |
 			AGX_MPFE_HMC_ADP_ECCCTRL2_RMW_EN_SET_MSK |
 			AGX_MPFE_HMC_ADP_ECCCTRL2_AUTOWB_EN_SET_MSK,
 			AGX_MPFE_HMC_ADP_ECCCTRL2_RMW_EN_SET_MSK |
 			AGX_MPFE_HMC_ADP_ECCCTRL2_AUTOWB_EN_SET_MSK);

 		mmio_clrsetbits_32(AGX_MPFE_HMC_ADP_ECCCTRL1,
 			AGX_MPFE_HMC_ADP_ECCCTRL1_AUTOWB_CNT_RST_SET_MSK |
 			AGX_MPFE_HMC_ADP_ECCCTRL1_CNT_RST_SET_MSK |
 			AGX_MPFE_HMC_ADP_ECCCTRL1_ECC_EN_SET_MSK,
 			AGX_MPFE_HMC_ADP_ECCCTRL1_ECC_EN_SET_MSK);
 		INFO("Scrubbing ECC\n");

 		/* ECC Scrubbing */
 		zeromem(DRAM_BASE, DRAM_SIZE);
 	} else {
 		INFO("ECC is disabled.\n");
 	}
 }
	/*
	* Copyright (c) 2019, Intel Corporation. All rights reserved.
	*
	* SPDX-License-Identifier: BSD-3-Clause
	*/

	#include <errno.h>
	#include <lib/mmio.h>
	#include <lib/utils.h>
	#include <common/debug.h>
	#include <drivers/delay_timer.h>
	#include <platform_def.h>

	#include "agilex_memory_controller.h"

	#define ALT_CCU_NOC_DI_SET_MSK 0x10

	#define DDR_READ_LATENCY_DELAY 40
	#define MAX_MEM_CAL_RETRY 3
	#define PRE_CALIBRATION_DELAY 1
	#define POST_CALIBRATION_DELAY 1
	#define TIMEOUT_EMIF_CALIBRATION 1000
	#define CLEAR_EMIF_DELAY 50000
	#define CLEAR_EMIF_TIMEOUT 0x100000
	#define TIMEOUT_INT_RESP 10000

	#define DDR_CONFIG(A, B, C, R) (((A) << 24) \| ((B) << 16) \| ((C) << 8) \| (R))
	#define DDR_CONFIG_ELEMENTS (sizeof(ddr_config)/sizeof(uint32_t))

	/* tWR = Min. 15ns constant, see JEDEC standard eg. DDR4 is JESD79-4.pdf */
	#define tWR_IN_NS 15

	void configure_hmc_adaptor_regs(void);
	void configure_ddr_sched_ctrl_regs(void);

	/* The followring are the supported configurations */
	uint32_t ddr_config[] = {
	/* DDR_CONFIG(Address order,Bank,Column,Row) */
	/* List for DDR3 or LPDDR3 (pinout order > chip, row, bank, column) */
	DDR_CONFIG(0, 3, 10, 12),
	DDR_CONFIG(0, 3, 9, 13),
	DDR_CONFIG(0, 3, 10, 13),
	DDR_CONFIG(0, 3, 9, 14),
	DDR_CONFIG(0, 3, 10, 14),
	DDR_CONFIG(0, 3, 10, 15),
	DDR_CONFIG(0, 3, 11, 14),
	DDR_CONFIG(0, 3, 11, 15),
	DDR_CONFIG(0, 3, 10, 16),
	DDR_CONFIG(0, 3, 11, 16),
	DDR_CONFIG(0, 3, 12, 15), /* 0xa */
	/* List for DDR4 only (pinout order > chip, bank, row, column) */
	DDR_CONFIG(1, 3, 10, 14),
	DDR_CONFIG(1, 4, 10, 14),
	DDR_CONFIG(1, 3, 10, 15),
	DDR_CONFIG(1, 4, 10, 15),
	DDR_CONFIG(1, 3, 10, 16),
	DDR_CONFIG(1, 4, 10, 16),
	DDR_CONFIG(1, 3, 10, 17),
	DDR_CONFIG(1, 4, 10, 17),
	};

	static int match_ddr_conf(uint32_t ddr_conf)
	{
	int i;

	for (i = 0; i < DDR_CONFIG_ELEMENTS; i++) {
	if (ddr_conf == ddr_config[i])
	return i;
	}
	return 0;
	}

	static int check_hmc_clk(void)
	{
	unsigned long timeout = 0;
	uint32_t hmc_clk;

	do {
	hmc_clk = mmio_read_32(AGX_SYSMGR_CORE_HMC_CLK);
	if (hmc_clk & AGX_SYSMGR_CORE_HMC_CLK_STATUS)
	break;
	udelay(1);
	} while (++timeout < 1000);
	if (timeout >= 1000)
	return -ETIMEDOUT;

	return 0;
	}

	static int clear_emif(void)
	{
	uint32_t data;
	unsigned long timeout;

	mmio_write_32(AGX_MPFE_HMC_ADP_RSTHANDSHAKECTRL, 0);

	timeout = 0;
	do {
	data = mmio_read_32(AGX_MPFE_HMC_ADP_RSTHANDSHAKESTAT);
	if ((data & AGX_MPFE_HMC_ADP_RSTHANDSHAKESTAT_SEQ2CORE) == 0)
	break;
	udelay(CLEAR_EMIF_DELAY);
	} while (++timeout < CLEAR_EMIF_TIMEOUT);
	if (timeout >= CLEAR_EMIF_TIMEOUT)
	return -ETIMEDOUT;

	return 0;
	}

	static int mem_calibration(void)
	{
	int status;
	uint32_t data;
	unsigned long timeout;
	unsigned long retry = 0;

	udelay(PRE_CALIBRATION_DELAY);

	do {
	if (retry != 0)
	INFO("DDR: Retrying DRAM calibration\n");

	timeout = 0;
	do {
	data = mmio_read_32(AGX_MPFE_HMC_ADP_DDRCALSTAT);
	if (AGX_MPFE_HMC_ADP_DDRCALSTAT_CAL(data) == 1)
	break;
	mdelay(1);
	} while (++timeout < TIMEOUT_EMIF_CALIBRATION);

	if (AGX_MPFE_HMC_ADP_DDRCALSTAT_CAL(data) == 0) {
	status = clear_emif();
	if (status)
	ERROR("Failed to clear Emif\n");
	} else {
	break;
	}
	} while (++retry < MAX_MEM_CAL_RETRY);

	if (AGX_MPFE_HMC_ADP_DDRCALSTAT_CAL(data) == 0) {
	ERROR("DDR: DRAM calibration failed.\n");
	status = -EIO;
	} else {
	INFO("DDR: DRAM calibration success.\n");
	status = 0;
	}

	udelay(POST_CALIBRATION_DELAY);

	return status;
	}

	int init_hard_memory_controller(void)
	{
	int status;

	status = check_hmc_clk();
	if (status) {
	ERROR("DDR: Error, HMC clock not running\n");
	return status;
	}

	status = mem_calibration();
	if (status) {
	ERROR("DDR: Memory Calibration Failed\n");
	return status;
	}

	configure_hmc_adaptor_regs();

	return 0;
	}

	void configure_ddr_sched_ctrl_regs(void)
	{
	uint32_t data, dram_addr_order, ddr_conf, bank, row, col,
	rd_to_miss, wr_to_miss, burst_len, burst_len_ddr_clk,
	burst_len_sched_clk, act_to_act, rd_to_wr, wr_to_rd, bw_ratio,
	t_rtp, t_rp, t_rcd, rd_latency, tw_rin_clk_cycles,
	bw_ratio_extended, auto_precharge = 0, act_to_act_bank, faw,
	faw_bank, bus_rd_to_rd, bus_rd_to_wr, bus_wr_to_rd;

	INFO("Init HPS NOC's DDR Scheduler.\n");

	data = mmio_read_32(AGX_MPFE_IOHMC_CTRLCFG1);
	dram_addr_order = AGX_MPFE_IOHMC_CTRLCFG1_CFG_ADDR_ORDER(data);

	data = mmio_read_32(AGX_MPFE_IOHMC_DRAMADDRW);

	col = IOHMC_DRAMADDRW_COL_ADDR_WIDTH(data);
	row = IOHMC_DRAMADDRW_ROW_ADDR_WIDTH(data);
	bank = IOHMC_DRAMADDRW_BANK_ADDR_WIDTH(data) +
	IOHMC_DRAMADDRW_BANK_GRP_ADDR_WIDTH(data);

	ddr_conf = match_ddr_conf(DDR_CONFIG(dram_addr_order, bank, col, row));

	if (ddr_conf) {
	mmio_clrsetbits_32(
	AGX_MPFE_DDR_MAIN_SCHED_DDRCONF,
	AGX_MPFE_DDR_MAIN_SCHED_DDRCONF_SET_MSK,
	AGX_MPFE_DDR_MAIN_SCHED_DDRCONF_SET(ddr_conf));
	} else {
	ERROR("DDR: Cannot find predefined ddrConf configuration.\n");
	}

	mmio_write_32(AGX_MPFE_HMC_ADP(ADP_DRAMADDRWIDTH), data);

	data = mmio_read_32(AGX_MPFE_IOHMC_DRAMTIMING0);
	rd_latency = AGX_MPFE_IOHMC_REG_DRAMTIMING0_CFG_TCL(data);

	data = mmio_read_32(AGX_MPFE_IOHMC_CALTIMING0);
	act_to_act = ACT_TO_ACT(data);
	t_rcd = ACT_TO_RDWR(data);
	act_to_act_bank = ACT_TO_ACT_DIFF_BANK(data);

	data = mmio_read_32(AGX_MPFE_IOHMC_CALTIMING1);
	rd_to_wr = RD_TO_WR(data);
	bus_rd_to_rd = RD_TO_RD_DIFF_CHIP(data);
	bus_rd_to_wr = RD_TO_WR_DIFF_CHIP(data);

	data = mmio_read_32(AGX_MPFE_IOHMC_CALTIMING2);
	t_rtp = RD_TO_PCH(data);

	data = mmio_read_32(AGX_MPFE_IOHMC_CALTIMING3);
	wr_to_rd = CALTIMING3_WR_TO_RD(data);
	bus_wr_to_rd = CALTIMING3_WR_TO_RD_DIFF_CHIP(data);

	data = mmio_read_32(AGX_MPFE_IOHMC_CALTIMING4);
	t_rp = PCH_TO_VALID(data);

	data = mmio_read_32(AGX_MPFE_HMC_ADP(HMC_ADP_DDRIOCTRL));
	bw_ratio = ((HMC_ADP_DDRIOCTRL_IO_SIZE(data) == 0) ? 0 : 1);

	data = mmio_read_32(AGX_MPFE_IOHMC_CTRLCFG0);
	burst_len = HMC_ADP_DDRIOCTRL_CTRL_BURST_LENGTH(data);
	burst_len_ddr_clk = burst_len / 2;
	burst_len_sched_clk = ((burst_len/2) / 2);

	data = mmio_read_32(AGX_MPFE_IOHMC_CTRLCFG0);
	switch (AGX_MPFE_IOHMC_REG_CTRLCFG0_CFG_MEM_TYPE(data)) {
	case 1:
	/* DDR4 - 1333MHz */
	/* 20 (19.995) clock cycles = 15ns */
	/* Calculate with rounding */
	tw_rin_clk_cycles = (((tWR_IN_NS * 1333) % 1000) >= 500) ?
	((tWR_IN_NS * 1333) / 1000) + 1 :
	((tWR_IN_NS * 1333) / 1000);
	break;
	default:
	/* Others - 1066MHz or slower */
	/* 16 (15.990) clock cycles = 15ns */
	/* Calculate with rounding */
	tw_rin_clk_cycles = (((tWR_IN_NS * 1066) % 1000) >= 500) ?
	((tWR_IN_NS * 1066) / 1000) + 1 :
	((tWR_IN_NS * 1066) / 1000);
	break;
	}

	rd_to_miss = t_rtp + t_rp + t_rcd - burst_len_sched_clk;
	wr_to_miss = ((rd_latency + burst_len_ddr_clk + 2 + tw_rin_clk_cycles)
	/ 2) - rd_to_wr + t_rp + t_rcd;

	mmio_write_32(AGX_MPFE_DDR_MAIN_SCHED_DDRTIMING,
	bw_ratio << DDRTIMING_BWRATIO_OFST \|
	wr_to_rd << DDRTIMING_WRTORD_OFST\|
	rd_to_wr << DDRTIMING_RDTOWR_OFST \|
	burst_len_sched_clk << DDRTIMING_BURSTLEN_OFST \|
	wr_to_miss << DDRTIMING_WRTOMISS_OFST \|
	rd_to_miss << DDRTIMING_RDTOMISS_OFST \|
	act_to_act << DDRTIMING_ACTTOACT_OFST);

	data = mmio_read_32(AGX_MPFE_HMC_ADP(HMC_ADP_DDRIOCTRL));
	bw_ratio_extended = ((ADP_DDRIOCTRL_IO_SIZE(data) == 0) ? 1 : 0);

	mmio_write_32(AGX_MPFE_DDR_MAIN_SCHED_DDRMODE,
	bw_ratio_extended << DDRMODE_BWRATIOEXTENDED_OFST \|
	auto_precharge << DDRMODE_AUTOPRECHARGE_OFST);

	mmio_write_32(AGX_MPFE_DDR_MAIN_SCHED_READLATENCY,
	(rd_latency / 2) + DDR_READ_LATENCY_DELAY);

	data = mmio_read_32(AGX_MPFE_IOHMC_CALTIMING9);
	faw = AGX_MPFE_IOHMC_CALTIMING9_ACT_TO_ACT(data);

	faw_bank = 1; // always 1 because we always have 4 bank DDR.

	mmio_write_32(AGX_MPFE_DDR_MAIN_SCHED_ACTIVATE,
	faw_bank << AGX_MPFE_DDR_MAIN_SCHED_ACTIVATE_FAWBANK_OFST \|
	faw << AGX_MPFE_DDR_MAIN_SCHED_ACTIVATE_FAW_OFST \|
	act_to_act_bank << AGX_MPFE_DDR_MAIN_SCHED_ACTIVATE_RRD_OFST);

	mmio_write_32(AGX_MPFE_DDR_MAIN_SCHED_DEVTODEV,
	((bus_rd_to_rd
	<< AGX_MPFE_DDR_MAIN_SCHED_DEVTODEV_BUSRDTORD_OFST)
	& AGX_MPFE_DDR_MAIN_SCHED_DEVTODEV_BUSRDTORD_MSK) \|
	((bus_rd_to_wr
	<< AGX_MPFE_DDR_MAIN_SCHED_DEVTODEV_BUSRDTOWR_OFST)
	& AGX_MPFE_DDR_MAIN_SCHED_DEVTODEV_BUSRDTOWR_MSK) \|
	((bus_wr_to_rd
	<< AGX_MPFE_DDR_MAIN_SCHED_DEVTODEV_BUSWRTORD_OFST)
	& AGX_MPFE_DDR_MAIN_SCHED_DEVTODEV_BUSWRTORD_MSK));

	}

	unsigned long get_physical_dram_size(void)
	{
	uint32_t data;
	unsigned long ram_addr_width, ram_ext_if_io_width;

	data = mmio_read_32(AGX_MPFE_HMC_ADP_DDRIOCTRL);
	switch (AGX_MPFE_HMC_ADP_DDRIOCTRL_IO_SIZE(data)) {
	case 0:
	ram_ext_if_io_width = 16;
	break;
	case 1:
	ram_ext_if_io_width = 32;
	break;
	case 2:
	ram_ext_if_io_width = 64;
	break;
	default:
	ram_ext_if_io_width = 0;
	break;
	}

	data = mmio_read_32(AGX_MPFE_IOHMC_REG_DRAMADDRW);
	ram_addr_width = IOHMC_DRAMADDRW_CFG_COL_ADDR_WIDTH(data) +
	IOHMC_DRAMADDRW_CFG_ROW_ADDR_WIDTH(data) +
	IOHMC_DRAMADDRW_CFG_BANK_ADDR_WIDTH(data) +
	IOHMC_DRAMADDRW_CFG_BANK_GROUP_ADDR_WIDTH(data) +
	IOHMC_DRAMADDRW_CFG_CS_ADDR_WIDTH(data);

	return (1 << ram_addr_width) * (ram_ext_if_io_width / 8);
	}



	void configure_hmc_adaptor_regs(void)
	{
	uint32_t data;
	uint32_t dram_io_width;

	/* Configure DDR data rate */
	dram_io_width = AGX_MPFE_IOHMC_NIOSRESERVE0_NIOS_RESERVE0(
	mmio_read_32(AGX_MPFE_IOHMC_REG_NIOSRESERVE0_OFST));
	dram_io_width = (dram_io_width & 0xFF) >> 5;

	data = mmio_read_32(AGX_MPFE_IOHMC_CTRLCFG3);

	dram_io_width \|= (data & 0x4);

	mmio_write_32(AGX_MPFE_HMC_ADP_DDRIOCTRL, dram_io_width);

	/* Copy dram addr width from IOHMC to HMC ADP */
	data = mmio_read_32(AGX_MPFE_IOHMC_DRAMADDRW);
	mmio_write_32(AGX_MPFE_HMC_ADP(ADP_DRAMADDRWIDTH), data);

	/* Enable nonsecure access to DDR */
	data = get_physical_dram_size();

	if (data < AGX_DDR_SIZE)
	data = AGX_DDR_SIZE;

	mmio_write_32(AGX_NOC_FW_DDR_SCR_MPUREGION0ADDR_LIMIT, data - 1);
	mmio_write_32(AGX_NOC_FW_DDR_SCR_MPUREGION0ADDR_LIMITEXT, 0x1f);

	mmio_write_32(AGX_NOC_FW_DDR_SCR_NONMPUREGION0ADDR_LIMIT, data - 1);

	mmio_write_32(AGX_SOC_NOC_FW_DDR_SCR_ENABLESET, BIT(0) \| BIT(8));

	/* ECC enablement */
	data = mmio_read_32(AGX_MPFE_IOHMC_REG_CTRLCFG1);
	if (data & (1 << AGX_IOHMC_CTRLCFG1_ENABLE_ECC_OFST)) {
	mmio_clrsetbits_32(AGX_MPFE_HMC_ADP_ECCCTRL1,
	AGX_MPFE_HMC_ADP_ECCCTRL1_AUTOWB_CNT_RST_SET_MSK \|
	AGX_MPFE_HMC_ADP_ECCCTRL1_CNT_RST_SET_MSK \|
	AGX_MPFE_HMC_ADP_ECCCTRL1_ECC_EN_SET_MSK,
	AGX_MPFE_HMC_ADP_ECCCTRL1_AUTOWB_CNT_RST_SET_MSK \|
	AGX_MPFE_HMC_ADP_ECCCTRL1_CNT_RST_SET_MSK);

	mmio_clrsetbits_32(AGX_MPFE_HMC_ADP_ECCCTRL2,
	AGX_MPFE_HMC_ADP_ECCCTRL2_OVRW_RB_ECC_EN_SET_MSK \|
	AGX_MPFE_HMC_ADP_ECCCTRL2_RMW_EN_SET_MSK \|
	AGX_MPFE_HMC_ADP_ECCCTRL2_AUTOWB_EN_SET_MSK,
	AGX_MPFE_HMC_ADP_ECCCTRL2_RMW_EN_SET_MSK \|
	AGX_MPFE_HMC_ADP_ECCCTRL2_AUTOWB_EN_SET_MSK);

	mmio_clrsetbits_32(AGX_MPFE_HMC_ADP_ECCCTRL1,
	AGX_MPFE_HMC_ADP_ECCCTRL1_AUTOWB_CNT_RST_SET_MSK \|
	AGX_MPFE_HMC_ADP_ECCCTRL1_CNT_RST_SET_MSK \|
	AGX_MPFE_HMC_ADP_ECCCTRL1_ECC_EN_SET_MSK,
	AGX_MPFE_HMC_ADP_ECCCTRL1_ECC_EN_SET_MSK);
	INFO("Scrubbing ECC\n");

	/* ECC Scrubbing */
	zeromem(DRAM_BASE, DRAM_SIZE);
	} else {
	INFO("ECC is disabled.\n");
	}
	}