x86: quark: Add System Memory Controller support

The codes are actually doing the memory initialization stuff.

Signed-off-by: Bin Meng <bmeng.cn@gmail.com>
Acked-by: Simon Glass <sjg@chromium.org>

arch/x86/cpu/quark/smc.c

@@ -0,0 +1,2764 @@
+/*
+ * Copyright (C) 2013, Intel Corporation
+ * Copyright (C) 2015, Bin Meng <bmeng.cn@gmail.com>
+ *
+ * Ported from Intel released Quark UEFI BIOS
+ * QuarkSocPkg/QuarkNorthCluster/MemoryInit/Pei
+ *
+ * SPDX-License-Identifier:	Intel
+ */
+
+#include <common.h>
+#include <pci.h>
+#include <asm/arch/device.h>
+#include <asm/arch/mrc.h>
+#include <asm/arch/msg_port.h>
+#include "mrc_util.h"
+#include "hte.h"
+#include "smc.h"
+
+/* t_rfc values (in picoseconds) per density */
+static const uint32_t t_rfc[5] = {
+	90000,	/* 512Mb */
+	110000,	/* 1Gb */
+	160000,	/* 2Gb */
+	300000,	/* 4Gb */
+	350000,	/* 8Gb */
+};
+
+/* t_ck clock period in picoseconds per speed index 800, 1066, 1333 */
+static const uint32_t t_ck[3] = {
+	2500,
+	1875,
+	1500
+};
+
+/* Global variables */
+static const uint16_t ddr_wclk[] = {193, 158};
+static const uint16_t ddr_wctl[] = {1, 217};
+static const uint16_t ddr_wcmd[] = {1, 220};
+
+#ifdef BACKUP_RCVN
+static const uint16_t ddr_rcvn[] = {129, 498};
+#endif
+
+#ifdef BACKUP_WDQS
+static const uint16_t ddr_wdqs[] = {65, 289};
+#endif
+
+#ifdef BACKUP_RDQS
+static const uint8_t ddr_rdqs[] = {32, 24};
+#endif
+
+#ifdef BACKUP_WDQ
+static const uint16_t ddr_wdq[] = {32, 257};
+#endif
+
+/* Stop self refresh driven by MCU */
+void clear_self_refresh(struct mrc_params *mrc_params)
+{
+	ENTERFN();
+
+	/* clear the PMSTS Channel Self Refresh bits */
+	mrc_write_mask(MEM_CTLR, PMSTS, BIT0, BIT0);
+
+	LEAVEFN();
+}
+
+/* It will initialize timing registers in the MCU (DTR0..DTR4) */
+void prog_ddr_timing_control(struct mrc_params *mrc_params)
+{
+	uint8_t tcl, wl;
+	uint8_t trp, trcd, tras, twr, twtr, trrd, trtp, tfaw;
+	uint32_t tck;
+	u32 dtr0, dtr1, dtr2, dtr3, dtr4;
+	u32 tmp1, tmp2;
+
+	ENTERFN();
+
+	/* mcu_init starts */
+	mrc_post_code(0x02, 0x00);
+
+	dtr0 = msg_port_read(MEM_CTLR, DTR0);
+	dtr1 = msg_port_read(MEM_CTLR, DTR1);
+	dtr2 = msg_port_read(MEM_CTLR, DTR2);
+	dtr3 = msg_port_read(MEM_CTLR, DTR3);
+	dtr4 = msg_port_read(MEM_CTLR, DTR4);
+
+	tck = t_ck[mrc_params->ddr_speed];	/* Clock in picoseconds */
+	tcl = mrc_params->params.cl;		/* CAS latency in clocks */
+	trp = tcl;	/* Per CAT MRC */
+	trcd = tcl;	/* Per CAT MRC */
+	tras = MCEIL(mrc_params->params.ras, tck);
+
+	/* Per JEDEC: tWR=15000ps DDR2/3 from 800-1600 */
+	twr = MCEIL(15000, tck);
+
+	twtr = MCEIL(mrc_params->params.wtr, tck);
+	trrd = MCEIL(mrc_params->params.rrd, tck);
+	trtp = 4;	/* Valid for 800 and 1066, use 5 for 1333 */
+	tfaw = MCEIL(mrc_params->params.faw, tck);
+
+	wl = 5 + mrc_params->ddr_speed;
+
+	dtr0 &= ~(BIT0 | BIT1);
+	dtr0 |= mrc_params->ddr_speed;
+	dtr0 &= ~(BIT12 | BIT13 | BIT14);
+	tmp1 = tcl - 5;
+	dtr0 |= ((tcl - 5) << 12);
+	dtr0 &= ~(BIT4 | BIT5 | BIT6 | BIT7);
+	dtr0 |= ((trp - 5) << 4);	/* 5 bit DRAM Clock */
+	dtr0 &= ~(BIT8 | BIT9 | BIT10 | BIT11);
+	dtr0 |= ((trcd - 5) << 8);	/* 5 bit DRAM Clock */
+
+	dtr1 &= ~(BIT0 | BIT1 | BIT2);
+	tmp2 = wl - 3;
+	dtr1 |= (wl - 3);
+	dtr1 &= ~(BIT8 | BIT9 | BIT10 | BIT11);
+	dtr1 |= ((wl + 4 + twr - 14) << 8);	/* Change to tWTP */
+	dtr1 &= ~(BIT28 | BIT29 | BIT30);
+	dtr1 |= ((MMAX(trtp, 4) - 3) << 28);	/* 4 bit DRAM Clock */
+	dtr1 &= ~(BIT24 | BIT25);
+	dtr1 |= ((trrd - 4) << 24);		/* 4 bit DRAM Clock */
+	dtr1 &= ~(BIT4 | BIT5);
+	dtr1 |= (1 << 4);
+	dtr1 &= ~(BIT20 | BIT21 | BIT22 | BIT23);
+	dtr1 |= ((tras - 14) << 20);		/* 6 bit DRAM Clock */
+	dtr1 &= ~(BIT16 | BIT17 | BIT18 | BIT19);
+	dtr1 |= ((((tfaw + 1) >> 1) - 5) << 16);/* 4 bit DRAM Clock */
+	/* Set 4 Clock CAS to CAS delay (multi-burst) */
+	dtr1 &= ~(BIT12 | BIT13);
+
+	dtr2 &= ~(BIT0 | BIT1 | BIT2);
+	dtr2 |= 1;
+	dtr2 &= ~(BIT8 | BIT9 | BIT10);
+	dtr2 |= (2 << 8);
+	dtr2 &= ~(BIT16 | BIT17 | BIT18 | BIT19);
+	dtr2 |= (2 << 16);
+
+	dtr3 &= ~(BIT0 | BIT1 | BIT2);
+	dtr3 |= 2;
+	dtr3 &= ~(BIT4 | BIT5 | BIT6);
+	dtr3 |= (2 << 4);
+
+	dtr3 &= ~(BIT8 | BIT9 | BIT10 | BIT11);
+	if (mrc_params->ddr_speed == DDRFREQ_800) {
+		/* Extended RW delay (+1) */
+		dtr3 |= ((tcl - 5 + 1) << 8);
+	} else if (mrc_params->ddr_speed == DDRFREQ_1066) {
+		/* Extended RW delay (+1) */
+		dtr3 |= ((tcl - 5 + 1) << 8);
+	}
+
+	dtr3 &= ~(BIT13 | BIT14 | BIT15 | BIT16);
+	dtr3 |= ((4 + wl + twtr - 11) << 13);
+
+	dtr3 &= ~(BIT22 | BIT23);
+	if (mrc_params->ddr_speed == DDRFREQ_800)
+		dtr3 |= ((MMAX(0, 1 - 1)) << 22);
+	else
+		dtr3 |= ((MMAX(0, 2 - 1)) << 22);
+
+	dtr4 &= ~(BIT0 | BIT1);
+	dtr4 |= 1;
+	dtr4 &= ~(BIT4 | BIT5 | BIT6);
+	dtr4 |= (1 << 4);
+	dtr4 &= ~(BIT8 | BIT9 | BIT10);
+	dtr4 |= ((1 + tmp1 - tmp2 + 2) << 8);
+	dtr4 &= ~(BIT12 | BIT13 | BIT14);
+	dtr4 |= ((1 + tmp1 - tmp2 + 2) << 12);
+	dtr4 &= ~(BIT15 | BIT16);
+
+	msg_port_write(MEM_CTLR, DTR0, dtr0);
+	msg_port_write(MEM_CTLR, DTR1, dtr1);
+	msg_port_write(MEM_CTLR, DTR2, dtr2);
+	msg_port_write(MEM_CTLR, DTR3, dtr3);
+	msg_port_write(MEM_CTLR, DTR4, dtr4);
+
+	LEAVEFN();
+}
+
+/* Configure MCU before jedec init sequence */
+void prog_decode_before_jedec(struct mrc_params *mrc_params)
+{
+	u32 drp;
+	u32 drfc;
+	u32 dcal;
+	u32 dsch;
+	u32 dpmc0;
+
+	ENTERFN();
+
+	/* Disable power saving features */
+	dpmc0 = msg_port_read(MEM_CTLR, DPMC0);
+	dpmc0 |= (BIT24 | BIT25);
+	dpmc0 &= ~(BIT16 | BIT17 | BIT18);
+	dpmc0 &= ~BIT23;
+	msg_port_write(MEM_CTLR, DPMC0, dpmc0);
+
+	/* Disable out of order transactions */
+	dsch = msg_port_read(MEM_CTLR, DSCH);
+	dsch |= (BIT8 | BIT12);
+	msg_port_write(MEM_CTLR, DSCH, dsch);
+
+	/* Disable issuing the REF command */
+	drfc = msg_port_read(MEM_CTLR, DRFC);
+	drfc &= ~(BIT12 | BIT13 | BIT14);
+	msg_port_write(MEM_CTLR, DRFC, drfc);
+
+	/* Disable ZQ calibration short */
+	dcal = msg_port_read(MEM_CTLR, DCAL);
+	dcal &= ~(BIT8 | BIT9 | BIT10);
+	dcal &= ~(BIT12 | BIT13);
+	msg_port_write(MEM_CTLR, DCAL, dcal);
+
+	/*
+	 * Training performed in address mode 0, rank population has limited
+	 * impact, however simulator complains if enabled non-existing rank.
+	 */
+	drp = 0;
+	if (mrc_params->rank_enables & 1)
+		drp |= BIT0;
+	if (mrc_params->rank_enables & 2)
+		drp |= BIT1;
+	msg_port_write(MEM_CTLR, DRP, drp);
+
+	LEAVEFN();
+}
+
+/*
+ * After Cold Reset, BIOS should set COLDWAKE bit to 1 before
+ * sending the WAKE message to the Dunit.
+ *
+ * For Standby Exit, or any other mode in which the DRAM is in
+ * SR, this bit must be set to 0.
+ */
+void perform_ddr_reset(struct mrc_params *mrc_params)
+{
+	ENTERFN();
+
+	/* Set COLDWAKE bit before sending the WAKE message */
+	mrc_write_mask(MEM_CTLR, DRMC, BIT16, BIT16);
+
+	/* Send wake command to DUNIT (MUST be done before JEDEC) */
+	dram_wake_command();
+
+	/* Set default value */
+	msg_port_write(MEM_CTLR, DRMC,
+		       (mrc_params->rd_odt_value == 0 ? BIT12 : 0));
+
+	LEAVEFN();
+}
+
+
+/*
+ * This function performs some initialization on the DDRIO unit.
+ * This function is dependent on BOARD_ID, DDR_SPEED, and CHANNEL_ENABLES.
+ */
+void ddrphy_init(struct mrc_params *mrc_params)
+{
+	uint32_t temp;
+	uint8_t ch;	/* channel counter */
+	uint8_t rk;	/* rank counter */
+	uint8_t bl_grp;	/*  byte lane group counter (2 BLs per module) */
+	uint8_t bl_divisor = 1;	/* byte lane divisor */
+	/* For DDR3 --> 0 == 800, 1 == 1066, 2 == 1333 */
+	uint8_t speed = mrc_params->ddr_speed & (BIT1 | BIT0);
+	uint8_t cas;
+	uint8_t cwl;
+
+	ENTERFN();
+
+	cas = mrc_params->params.cl;
+	cwl = 5 + mrc_params->ddr_speed;
+
+	/* ddrphy_init starts */
+	mrc_post_code(0x03, 0x00);
+
+	/*
+	 * HSD#231531
+	 * Make sure IOBUFACT is deasserted before initializing the DDR PHY
+	 *
+	 * HSD#234845
+	 * Make sure WRPTRENABLE is deasserted before initializing the DDR PHY
+	 */
+	for (ch = 0; ch < NUM_CHANNELS; ch++) {
+		if (mrc_params->channel_enables & (1 << ch)) {
+			/* Deassert DDRPHY Initialization Complete */
+			mrc_alt_write_mask(DDRPHY,
+				(CMDPMCONFIG0 + (ch * DDRIOCCC_CH_OFFSET)),
+				~BIT20, BIT20);	/* SPID_INIT_COMPLETE=0 */
+			/* Deassert IOBUFACT */
+			mrc_alt_write_mask(DDRPHY,
+				(CMDCFGREG0 + (ch * DDRIOCCC_CH_OFFSET)),
+				~BIT2, BIT2);	/* IOBUFACTRST_N=0 */
+			/* Disable WRPTR */
+			mrc_alt_write_mask(DDRPHY,
+				(CMDPTRREG + (ch * DDRIOCCC_CH_OFFSET)),
+				~BIT0, BIT0);	/* WRPTRENABLE=0 */
+		}
+	}
+
+	/* Put PHY in reset */
+	mrc_alt_write_mask(DDRPHY, MASTERRSTN, 0, BIT0);
+
+	/* Initialize DQ01, DQ23, CMD, CLK-CTL, COMP modules */
+
+	/* STEP0 */
+	mrc_post_code(0x03, 0x10);
+	for (ch = 0; ch < NUM_CHANNELS; ch++) {
+		if (mrc_params->channel_enables & (1 << ch)) {
+			/* DQ01-DQ23 */
+			for (bl_grp = 0;
+			     bl_grp < ((NUM_BYTE_LANES / bl_divisor) / 2);
+			     bl_grp++) {
+				/* Analog MUX select - IO2xCLKSEL */
+				mrc_alt_write_mask(DDRPHY,
+					(DQOBSCKEBBCTL +
+					(bl_grp * DDRIODQ_BL_OFFSET) +
+					(ch * DDRIODQ_CH_OFFSET)),
+					((bl_grp) ? (0x00) : (BIT22)), (BIT22));
+
+				/* ODT Strength */
+				switch (mrc_params->rd_odt_value) {
+				case 1:
+					temp = 0x3;
+					break;	/* 60 ohm */
+				case 2:
+					temp = 0x3;
+					break;	/* 120 ohm */
+				case 3:
+					temp = 0x3;
+					break;	/* 180 ohm */
+				default:
+					temp = 0x3;
+					break;	/* 120 ohm */
+				}
+
+				/* ODT strength */
+				mrc_alt_write_mask(DDRPHY,
+					(B0RXIOBUFCTL +
+					(bl_grp * DDRIODQ_BL_OFFSET) +
+					(ch * DDRIODQ_CH_OFFSET)),
+					(temp << 5), (BIT6 | BIT5));
+				/* ODT strength */
+				mrc_alt_write_mask(DDRPHY,
+					(B1RXIOBUFCTL +
+					(bl_grp * DDRIODQ_BL_OFFSET) +
+					(ch * DDRIODQ_CH_OFFSET)),
+					(temp << 5), (BIT6 | BIT5));
+
+				/* Dynamic ODT/DIFFAMP */
+				temp = (((cas) << 24) | ((cas) << 16) |
+					((cas) << 8) | ((cas) << 0));
+				switch (speed) {
+				case 0:
+					temp -= 0x01010101;
+					break;	/* 800 */
+				case 1:
+					temp -= 0x02020202;
+					break;	/* 1066 */
+				case 2:
+					temp -= 0x03030303;
+					break;	/* 1333 */
+				case 3:
+					temp -= 0x04040404;
+					break;	/* 1600 */
+				}
+
+				/* Launch Time: ODT, DIFFAMP, ODT, DIFFAMP */
+				mrc_alt_write_mask(DDRPHY,
+					(B01LATCTL1 +
+					(bl_grp * DDRIODQ_BL_OFFSET) +
+					(ch * DDRIODQ_CH_OFFSET)),
+					temp,
+					(BIT28 | BIT27 | BIT26 | BIT25 | BIT24 |
+					BIT20 | BIT19 | BIT18 | BIT17 | BIT16 |
+					BIT12 | BIT11 | BIT10 | BIT9 | BIT8 |
+					BIT4 | BIT3 | BIT2 | BIT1 | BIT0));
+				switch (speed) {
+				/* HSD#234715 */
+				case 0:
+					temp = ((0x06 << 16) | (0x07 << 8));
+					break;	/* 800 */
+				case 1:
+					temp = ((0x07 << 16) | (0x08 << 8));
+					break;	/* 1066 */
+				case 2:
+					temp = ((0x09 << 16) | (0x0A << 8));
+					break;	/* 1333 */
+				case 3:
+					temp = ((0x0A << 16) | (0x0B << 8));
+					break;	/* 1600 */
+				}
+
+				/* On Duration: ODT, DIFFAMP */
+				mrc_alt_write_mask(DDRPHY,
+					(B0ONDURCTL +
+					(bl_grp * DDRIODQ_BL_OFFSET) +
+					(ch * DDRIODQ_CH_OFFSET)),
+					temp,
+					(BIT21 | BIT20 | BIT19 | BIT18 | BIT17 |
+					BIT16 | BIT13 | BIT12 | BIT11 | BIT10 |
+					BIT9 | BIT8));
+				/* On Duration: ODT, DIFFAMP */
+				mrc_alt_write_mask(DDRPHY,
+					(B1ONDURCTL +
+					(bl_grp * DDRIODQ_BL_OFFSET) +
+					(ch * DDRIODQ_CH_OFFSET)),
+					temp,
+					(BIT21 | BIT20 | BIT19 | BIT18 | BIT17 |
+					BIT16 | BIT13 | BIT12 | BIT11 | BIT10 |
+					BIT9 | BIT8));
+
+				switch (mrc_params->rd_odt_value) {
+				case 0:
+					/* override DIFFAMP=on, ODT=off */
+					temp = ((0x3F << 16) | (0x3f << 10));
+					break;
+				default:
+					/* override DIFFAMP=on, ODT=on */
+					temp = ((0x3F << 16) | (0x2A << 10));
+					break;
+				}
+
+				/* Override: DIFFAMP, ODT */
+				mrc_alt_write_mask(DDRPHY,
+					(B0OVRCTL +
+					(bl_grp * DDRIODQ_BL_OFFSET) +
+					(ch * DDRIODQ_CH_OFFSET)),
+					temp,
+					(BIT21 | BIT20 | BIT19 | BIT18 | BIT17 |
+					BIT16 | BIT15 | BIT14 | BIT13 | BIT12 |
+					BIT11 | BIT10));
+				/* Override: DIFFAMP, ODT */
+				mrc_alt_write_mask(DDRPHY,
+					(B1OVRCTL +
+					(bl_grp * DDRIODQ_BL_OFFSET) +
+					(ch * DDRIODQ_CH_OFFSET)),
+					temp,
+					(BIT21 | BIT20 | BIT19 | BIT18 | BIT17 |
+					BIT16 | BIT15 | BIT14 | BIT13 | BIT12 |
+					BIT11 | BIT10));
+
+				/* DLL Setup */
+
+				/* 1xCLK Domain Timings: tEDP,RCVEN,WDQS (PO) */
+				mrc_alt_write_mask(DDRPHY,
+					(B0LATCTL0 +
+					(bl_grp * DDRIODQ_BL_OFFSET) +
+					(ch * DDRIODQ_CH_OFFSET)),
+					(((cas + 7) << 16) | ((cas - 4) << 8) |
+					((cwl - 2) << 0)),
+					(BIT21 | BIT20 | BIT19 | BIT18 | BIT17 |
+					BIT16 | BIT12 | BIT11 | BIT10 | BIT9 |
+					BIT8 | BIT4 | BIT3 | BIT2 | BIT1 |
+					BIT0));
+				mrc_alt_write_mask(DDRPHY,
+					(B1LATCTL0 +
+					(bl_grp * DDRIODQ_BL_OFFSET) +
+					(ch * DDRIODQ_CH_OFFSET)),
+					(((cas + 7) << 16) | ((cas - 4) << 8) |
+					((cwl - 2) << 0)),
+					(BIT21 | BIT20 | BIT19 | BIT18 | BIT17 |
+					BIT16 | BIT12 | BIT11 | BIT10 | BIT9 |
+					BIT8 | BIT4 | BIT3 | BIT2 | BIT1 |
+					BIT0));
+
+				/* RCVEN Bypass (PO) */
+				mrc_alt_write_mask(DDRPHY,
+					(B0RXIOBUFCTL +
+					(bl_grp * DDRIODQ_BL_OFFSET) +
+					(ch * DDRIODQ_CH_OFFSET)),
+					((0x0 << 7) | (0x0 << 0)),
+					(BIT7 | BIT0));
+				mrc_alt_write_mask(DDRPHY,
+					(B1RXIOBUFCTL +
+					(bl_grp * DDRIODQ_BL_OFFSET) +
+					(ch * DDRIODQ_CH_OFFSET)),
+					((0x0 << 7) | (0x0 << 0)),
+					(BIT7 | BIT0));
+
+				/* TX */
+				mrc_alt_write_mask(DDRPHY,
+					(DQCTL +
+					(bl_grp * DDRIODQ_BL_OFFSET) +
+					(ch * DDRIODQ_CH_OFFSET)),
+					(BIT16), (BIT16));
+				mrc_alt_write_mask(DDRPHY,
+					(B01PTRCTL1 +
+					(bl_grp * DDRIODQ_BL_OFFSET) +
+					(ch * DDRIODQ_CH_OFFSET)),
+					(BIT8), (BIT8));
+
+				/* RX (PO) */
+				/* Internal Vref Code, Enable#, Ext_or_Int (1=Ext) */
+				mrc_alt_write_mask(DDRPHY,
+					(B0VREFCTL +
+					(bl_grp * DDRIODQ_BL_OFFSET) +
+					(ch * DDRIODQ_CH_OFFSET)),
+					((0x03 << 2) | (0x0 << 1) | (0x0 << 0)),
+					(BIT7 | BIT6 | BIT5 | BIT4 | BIT3 |
+					BIT2 | BIT1 | BIT0));
+				/* Internal Vref Code, Enable#, Ext_or_Int (1=Ext) */
+				mrc_alt_write_mask(DDRPHY,
+					(B1VREFCTL +
+					(bl_grp * DDRIODQ_BL_OFFSET) +
+					(ch * DDRIODQ_CH_OFFSET)),
+					((0x03 << 2) | (0x0 << 1) | (0x0 << 0)),
+					(BIT7 | BIT6 | BIT5 | BIT4 | BIT3 |
+					BIT2 | BIT1 | BIT0));
+				/* Per-Bit De-Skew Enable */
+				mrc_alt_write_mask(DDRPHY,
+					(B0RXIOBUFCTL +
+					(bl_grp * DDRIODQ_BL_OFFSET) +
+					(ch * DDRIODQ_CH_OFFSET)),
+					(0), (BIT4));
+				/* Per-Bit De-Skew Enable */
+				mrc_alt_write_mask(DDRPHY,
+					(B1RXIOBUFCTL +
+					(bl_grp * DDRIODQ_BL_OFFSET) +
+					(ch * DDRIODQ_CH_OFFSET)),
+					(0), (BIT4));
+			}
+
+			/* CLKEBB */
+			mrc_alt_write_mask(DDRPHY,
+				(CMDOBSCKEBBCTL + (ch * DDRIOCCC_CH_OFFSET)),
+				0, (BIT23));
+
+			/* Enable tristate control of cmd/address bus */
+			mrc_alt_write_mask(DDRPHY,
+				(CMDCFGREG0 + (ch * DDRIOCCC_CH_OFFSET)),
+				0, (BIT1 | BIT0));
+
+			/* ODT RCOMP */
+			mrc_alt_write_mask(DDRPHY,
+				(CMDRCOMPODT + (ch * DDRIOCCC_CH_OFFSET)),
+				((0x03 << 5) | (0x03 << 0)),
+				(BIT9 | BIT8 | BIT7 | BIT6 | BIT5 | BIT4 |
+				BIT3 | BIT2 | BIT1 | BIT0));
+
+			/* CMDPM* registers must be programmed in this order */
+
+			/* Turn On Delays: SFR (regulator), MPLL */
+			mrc_alt_write_mask(DDRPHY,
+				(CMDPMDLYREG4 + (ch * DDRIOCCC_CH_OFFSET)),
+				((0xFFFFU << 16) | (0xFFFF << 0)),
+				0xFFFFFFFF);
+			/*
+			 * Delays: ASSERT_IOBUFACT_to_ALLON0_for_PM_MSG_3,
+			 * VREG (MDLL) Turn On, ALLON0_to_DEASSERT_IOBUFACT
+			 * for_PM_MSG_gt0, MDLL Turn On
+			 */
+			mrc_alt_write_mask(DDRPHY,
+				(CMDPMDLYREG3 + (ch * DDRIOCCC_CH_OFFSET)),
+				((0xFU << 28) | (0xFFF << 16) | (0xF << 12) |
+				(0x616 << 0)), 0xFFFFFFFF);
+			/* MPLL Divider Reset Delays */
+			mrc_alt_write_mask(DDRPHY,
+				(CMDPMDLYREG2 + (ch * DDRIOCCC_CH_OFFSET)),
+				((0xFFU << 24) | (0xFF << 16) | (0xFF << 8) |
+				(0xFF << 0)), 0xFFFFFFFF);
+			/* Turn Off Delays: VREG, Staggered MDLL, MDLL, PI */
+			mrc_alt_write_mask(DDRPHY,
+				(CMDPMDLYREG1 + (ch * DDRIOCCC_CH_OFFSET)),
+				((0xFFU << 24) | (0xFF << 16) | (0xFF << 8) |
+				(0xFF << 0)), 0xFFFFFFFF);
+			/* Turn On Delays: MPLL, Staggered MDLL, PI, IOBUFACT */
+			mrc_alt_write_mask(DDRPHY,
+				(CMDPMDLYREG0 + (ch * DDRIOCCC_CH_OFFSET)),
+				((0xFFU << 24) | (0xFF << 16) | (0xFF << 8) |
+				(0xFF << 0)), 0xFFFFFFFF);
+			/* Allow PUnit signals */
+			mrc_alt_write_mask(DDRPHY,
+				(CMDPMCONFIG0 + (ch * DDRIOCCC_CH_OFFSET)),
+				((0x6 << 8) | BIT6 | (0x4 << 0)),
+				(BIT31 | BIT30 | BIT29 | BIT28 | BIT27 | BIT26 |
+				BIT25 | BIT24 | BIT23 | BIT22 | BIT21 | BIT11 |
+				BIT10 | BIT9 | BIT8 | BIT6 | BIT3 | BIT2 |
+				BIT1 | BIT0));
+			/* DLL_VREG Bias Trim, VREF Tuning for DLL_VREG */
+			mrc_alt_write_mask(DDRPHY,
+				(CMDMDLLCTL + (ch * DDRIOCCC_CH_OFFSET)),
+				((0x3 << 4) | (0x7 << 0)),
+				(BIT6 | BIT5 | BIT4 | BIT3 | BIT2 | BIT1 |
+				BIT0));
+
+			/* CLK-CTL */
+			mrc_alt_write_mask(DDRPHY,
+				(CCOBSCKEBBCTL + (ch * DDRIOCCC_CH_OFFSET)),
+				0, BIT24);	/* CLKEBB */
+			/* Buffer Enable: CS,CKE,ODT,CLK */
+			mrc_alt_write_mask(DDRPHY,
+				(CCCFGREG0 + (ch * DDRIOCCC_CH_OFFSET)),
+				((0x0 << 16) | (0x0 << 12) | (0x0 << 8) |
+				(0xF << 4) | BIT0),
+				(BIT19 | BIT18 | BIT17 | BIT16 | BIT15 | BIT14 |
+				BIT13 | BIT12 | BIT11 | BIT10 | BIT9 | BIT8 |
+				BIT7 | BIT6 | BIT5 | BIT4 | BIT0));
+			/* ODT RCOMP */
+			mrc_alt_write_mask(DDRPHY,
+				(CCRCOMPODT + (ch * DDRIOCCC_CH_OFFSET)),
+				((0x03 << 8) | (0x03 << 0)),
+				(BIT12 | BIT11 | BIT10 | BIT9 | BIT8 | BIT4 |
+				BIT3 | BIT2 | BIT1 | BIT0));
+			/* DLL_VREG Bias Trim, VREF Tuning for DLL_VREG */
+			mrc_alt_write_mask(DDRPHY,
+				(CCMDLLCTL + (ch * DDRIOCCC_CH_OFFSET)),
+				((0x3 << 4) | (0x7 << 0)),
+				(BIT6 | BIT5 | BIT4 | BIT3 | BIT2 | BIT1 |
+				BIT0));
+
+			/*
+			 * COMP (RON channel specific)
+			 * - DQ/DQS/DM RON: 32 Ohm
+			 * - CTRL/CMD RON: 27 Ohm
+			 * - CLK RON: 26 Ohm
+			 */
+			/* RCOMP Vref PU/PD */
+			mrc_alt_write_mask(DDRPHY,
+				(DQVREFCH0 +  (ch * DDRCOMP_CH_OFFSET)),
+				((0x08 << 24) | (0x03 << 16)),
+				(BIT29 | BIT28 | BIT27 | BIT26 | BIT25 |
+				BIT24 | BIT21 | BIT20 | BIT19 | BIT18 |
+				BIT17 | BIT16));
+			/* RCOMP Vref PU/PD */
+			mrc_alt_write_mask(DDRPHY,
+				(CMDVREFCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				((0x0C << 24) | (0x03 << 16)),
+				(BIT29 | BIT28 | BIT27 | BIT26 | BIT25 |
+				BIT24 | BIT21 | BIT20 | BIT19 | BIT18 |
+				BIT17 | BIT16));
+			/* RCOMP Vref PU/PD */
+			mrc_alt_write_mask(DDRPHY,
+				(CLKVREFCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				((0x0F << 24) | (0x03 << 16)),
+				(BIT29 | BIT28 | BIT27 | BIT26 | BIT25 |
+				BIT24 | BIT21 | BIT20 | BIT19 | BIT18 |
+				BIT17 | BIT16));
+			/* RCOMP Vref PU/PD */
+			mrc_alt_write_mask(DDRPHY,
+				(DQSVREFCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				((0x08 << 24) | (0x03 << 16)),
+				(BIT29 | BIT28 | BIT27 | BIT26 | BIT25 |
+				BIT24 | BIT21 | BIT20 | BIT19 | BIT18 |
+				BIT17 | BIT16));
+			/* RCOMP Vref PU/PD */
+			mrc_alt_write_mask(DDRPHY,
+				(CTLVREFCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				((0x0C << 24) | (0x03 << 16)),
+				(BIT29 | BIT28 | BIT27 | BIT26 | BIT25 |
+				BIT24 | BIT21 | BIT20 | BIT19 | BIT18 |
+				BIT17 | BIT16));
+
+			/* DQS Swapped Input Enable */
+			mrc_alt_write_mask(DDRPHY,
+				(COMPEN1CH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT19 | BIT17),
+				(BIT31 | BIT30 | BIT19 | BIT17 |
+				BIT15 | BIT14));
+
+			/* ODT VREF = 1.5 x 274/360+274 = 0.65V (code of ~50) */
+			/* ODT Vref PU/PD */
+			mrc_alt_write_mask(DDRPHY,
+				(DQVREFCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				((0x32 << 8) | (0x03 << 0)),
+				(BIT13 | BIT12 | BIT11 | BIT10 | BIT9 | BIT8 |
+				BIT5 | BIT4 | BIT3 | BIT2 | BIT1 | BIT0));
+			/* ODT Vref PU/PD */
+			mrc_alt_write_mask(DDRPHY,
+				(DQSVREFCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				((0x32 << 8) | (0x03 << 0)),
+				(BIT13 | BIT12 | BIT11 | BIT10 | BIT9 | BIT8 |
+				BIT5 | BIT4 | BIT3 | BIT2 | BIT1 | BIT0));
+			/* ODT Vref PU/PD */
+			mrc_alt_write_mask(DDRPHY,
+				(CLKVREFCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				((0x0E << 8) | (0x05 << 0)),
+				(BIT13 | BIT12 | BIT11 | BIT10 | BIT9 | BIT8 |
+				BIT5 | BIT4 | BIT3 | BIT2 | BIT1 | BIT0));
+
+			/*
+			 * Slew rate settings are frequency specific,
+			 * numbers below are for 800Mhz (speed == 0)
+			 * - DQ/DQS/DM/CLK SR: 4V/ns,
+			 * - CTRL/CMD SR: 1.5V/ns
+			 */
+			temp = (0x0E << 16) | (0x0E << 12) | (0x08 << 8) |
+				(0x0B << 4) | (0x0B << 0);
+			/* DCOMP Delay Select: CTL,CMD,CLK,DQS,DQ */
+			mrc_alt_write_mask(DDRPHY,
+				(DLYSELCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				temp,
+				(BIT19 | BIT18 | BIT17 | BIT16 | BIT15 |
+				BIT14 | BIT13 | BIT12 | BIT11 | BIT10 |
+				BIT9 | BIT8 | BIT7 | BIT6 | BIT5 | BIT4 |
+				BIT3 | BIT2 | BIT1 | BIT0));
+			/* TCO Vref CLK,DQS,DQ */
+			mrc_alt_write_mask(DDRPHY,
+				(TCOVREFCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				((0x05 << 16) | (0x05 << 8) | (0x05 << 0)),
+				(BIT21 | BIT20 | BIT19 | BIT18 | BIT17 |
+				BIT16 | BIT13 | BIT12 | BIT11 | BIT10 |
+				BIT9 | BIT8 | BIT5 | BIT4 | BIT3 | BIT2 |
+				BIT1 | BIT0));
+			/* ODTCOMP CMD/CTL PU/PD */
+			mrc_alt_write_mask(DDRPHY,
+				(CCBUFODTCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				((0x03 << 8) | (0x03 << 0)),
+				(BIT12 | BIT11 | BIT10 | BIT9 | BIT8 |
+				BIT4 | BIT3 | BIT2 | BIT1 | BIT0));
+			/* COMP */
+			mrc_alt_write_mask(DDRPHY,
+				(COMPEN0CH0 + (ch * DDRCOMP_CH_OFFSET)),
+				0, (BIT31 | BIT30 | BIT8));
+
+#ifdef BACKUP_COMPS
+			/* DQ COMP Overrides */
+			/* RCOMP PU */
+			mrc_alt_write_mask(DDRPHY,
+				(DQDRVPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x0A << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* RCOMP PD */
+			mrc_alt_write_mask(DDRPHY,
+				(DQDRVPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x0A << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* DCOMP PU */
+			mrc_alt_write_mask(DDRPHY,
+				(DQDLYPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x10 << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* DCOMP PD */
+			mrc_alt_write_mask(DDRPHY,
+				(DQDLYPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x10 << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* ODTCOMP PU */
+			mrc_alt_write_mask(DDRPHY,
+				(DQODTPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x0B << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* ODTCOMP PD */
+			mrc_alt_write_mask(DDRPHY,
+				(DQODTPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x0B << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* TCOCOMP PU */
+			mrc_alt_write_mask(DDRPHY,
+				(DQTCOPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31), (BIT31));
+			/* TCOCOMP PD */
+			mrc_alt_write_mask(DDRPHY,
+				(DQTCOPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31), (BIT31));
+
+			/* DQS COMP Overrides */
+			/* RCOMP PU */
+			mrc_alt_write_mask(DDRPHY,
+				(DQSDRVPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x0A << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* RCOMP PD */
+			mrc_alt_write_mask(DDRPHY,
+				(DQSDRVPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x0A << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* DCOMP PU */
+			mrc_alt_write_mask(DDRPHY,
+				(DQSDLYPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x10 << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* DCOMP PD */
+			mrc_alt_write_mask(DDRPHY,
+				(DQSDLYPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x10 << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* ODTCOMP PU */
+			mrc_alt_write_mask(DDRPHY,
+				(DQSODTPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x0B << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* ODTCOMP PD */
+			mrc_alt_write_mask(DDRPHY,
+				(DQSODTPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x0B << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* TCOCOMP PU */
+			mrc_alt_write_mask(DDRPHY,
+				(DQSTCOPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31), (BIT31));
+			/* TCOCOMP PD */
+			mrc_alt_write_mask(DDRPHY,
+				(DQSTCOPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31), (BIT31));
+
+			/* CLK COMP Overrides */
+			/* RCOMP PU */
+			mrc_alt_write_mask(DDRPHY,
+				(CLKDRVPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x0C << 16)),
+				(BIT31 | (0x0B << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* RCOMP PD */
+			mrc_alt_write_mask(DDRPHY,
+				(CLKDRVPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x0C << 16)),
+				(BIT31 | (0x0B << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* DCOMP PU */
+			mrc_alt_write_mask(DDRPHY,
+				(CLKDLYPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x07 << 16)),
+				(BIT31 | (0x0B << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* DCOMP PD */
+			mrc_alt_write_mask(DDRPHY,
+				(CLKDLYPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x07 << 16)),
+				(BIT31 | (0x0B << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* ODTCOMP PU */
+			mrc_alt_write_mask(DDRPHY,
+				(CLKODTPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x0B << 16)),
+				(BIT31 | (0x0B << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* ODTCOMP PD */
+			mrc_alt_write_mask(DDRPHY,
+				(CLKODTPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x0B << 16)),
+				(BIT31 | (0x0B << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* TCOCOMP PU */
+			mrc_alt_write_mask(DDRPHY,
+				(CLKTCOPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31), (BIT31));
+			/* TCOCOMP PD */
+			mrc_alt_write_mask(DDRPHY,
+				(CLKTCOPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31), (BIT31));
+
+			/* CMD COMP Overrides */
+			/* RCOMP PU */
+			mrc_alt_write_mask(DDRPHY,
+				(CMDDRVPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x0D << 16)),
+				(BIT31 | BIT21 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* RCOMP PD */
+			mrc_alt_write_mask(DDRPHY,
+				(CMDDRVPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x0D << 16)),
+				(BIT31 | BIT21 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* DCOMP PU */
+			mrc_alt_write_mask(DDRPHY,
+				(CMDDLYPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x0A << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* DCOMP PD */
+			mrc_alt_write_mask(DDRPHY,
+				(CMDDLYPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x0A << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+
+			/* CTL COMP Overrides */
+			/* RCOMP PU */
+			mrc_alt_write_mask(DDRPHY,
+				(CTLDRVPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x0D << 16)),
+				(BIT31 | BIT21 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* RCOMP PD */
+			mrc_alt_write_mask(DDRPHY,
+				(CTLDRVPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x0D << 16)),
+				(BIT31 | BIT21 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* DCOMP PU */
+			mrc_alt_write_mask(DDRPHY,
+				(CTLDLYPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x0A << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* DCOMP PD */
+			mrc_alt_write_mask(DDRPHY,
+				(CTLDLYPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x0A << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+#else
+			/* DQ TCOCOMP Overrides */
+			/* TCOCOMP PU */
+			mrc_alt_write_mask(DDRPHY,
+				(DQTCOPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x1F << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* TCOCOMP PD */
+			mrc_alt_write_mask(DDRPHY,
+				(DQTCOPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x1F << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+
+			/* DQS TCOCOMP Overrides */
+			/* TCOCOMP PU */
+			mrc_alt_write_mask(DDRPHY,
+				(DQSTCOPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x1F << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* TCOCOMP PD */
+			mrc_alt_write_mask(DDRPHY,
+				(DQSTCOPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x1F << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+
+			/* CLK TCOCOMP Overrides */
+			/* TCOCOMP PU */
+			mrc_alt_write_mask(DDRPHY,
+				(CLKTCOPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x1F << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+			/* TCOCOMP PD */
+			mrc_alt_write_mask(DDRPHY,
+				(CLKTCOPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
+				(BIT31 | (0x1F << 16)),
+				(BIT31 | BIT20 | BIT19 |
+				BIT18 | BIT17 | BIT16));
+#endif
+
+			/* program STATIC delays */
+#ifdef BACKUP_WCMD
+			set_wcmd(ch, ddr_wcmd[PLATFORM_ID]);
+#else
+			set_wcmd(ch, ddr_wclk[PLATFORM_ID] + HALF_CLK);
+#endif
+
+			for (rk = 0; rk < NUM_RANKS; rk++) {
+				if (mrc_params->rank_enables & (1<<rk)) {
+					set_wclk(ch, rk, ddr_wclk[PLATFORM_ID]);
+#ifdef BACKUP_WCTL
+					set_wctl(ch, rk, ddr_wctl[PLATFORM_ID]);
+#else
+					set_wctl(ch, rk, ddr_wclk[PLATFORM_ID] + HALF_CLK);
+#endif
+				}
+			}
+		}
+	}
+
+	/* COMP (non channel specific) */
+	/* RCOMP: Dither PU Enable */
+	mrc_alt_write_mask(DDRPHY, (DQANADRVPUCTL), (BIT30), (BIT30));
+	/* RCOMP: Dither PD Enable */
+	mrc_alt_write_mask(DDRPHY, (DQANADRVPDCTL), (BIT30), (BIT30));
+	/* RCOMP: Dither PU Enable */
+	mrc_alt_write_mask(DDRPHY, (CMDANADRVPUCTL), (BIT30), (BIT30));
+	/* RCOMP: Dither PD Enable */
+	mrc_alt_write_mask(DDRPHY, (CMDANADRVPDCTL), (BIT30), (BIT30));
+	/* RCOMP: Dither PU Enable */
+	mrc_alt_write_mask(DDRPHY, (CLKANADRVPUCTL), (BIT30), (BIT30));
+	/* RCOMP: Dither PD Enable */
+	mrc_alt_write_mask(DDRPHY, (CLKANADRVPDCTL), (BIT30), (BIT30));
+	/* RCOMP: Dither PU Enable */
+	mrc_alt_write_mask(DDRPHY, (DQSANADRVPUCTL), (BIT30), (BIT30));
+	/* RCOMP: Dither PD Enable */
+	mrc_alt_write_mask(DDRPHY, (DQSANADRVPDCTL), (BIT30), (BIT30));
+	/* RCOMP: Dither PU Enable */
+	mrc_alt_write_mask(DDRPHY, (CTLANADRVPUCTL), (BIT30), (BIT30));
+	/* RCOMP: Dither PD Enable */
+	mrc_alt_write_mask(DDRPHY, (CTLANADRVPDCTL), (BIT30), (BIT30));
+	/* ODT: Dither PU Enable */
+	mrc_alt_write_mask(DDRPHY, (DQANAODTPUCTL), (BIT30), (BIT30));
+	/* ODT: Dither PD Enable */
+	mrc_alt_write_mask(DDRPHY, (DQANAODTPDCTL), (BIT30), (BIT30));
+	/* ODT: Dither PU Enable */
+	mrc_alt_write_mask(DDRPHY, (CLKANAODTPUCTL), (BIT30), (BIT30));
+	/* ODT: Dither PD Enable */
+	mrc_alt_write_mask(DDRPHY, (CLKANAODTPDCTL), (BIT30), (BIT30));
+	/* ODT: Dither PU Enable */
+	mrc_alt_write_mask(DDRPHY, (DQSANAODTPUCTL), (BIT30), (BIT30));
+	/* ODT: Dither PD Enable */
+	mrc_alt_write_mask(DDRPHY, (DQSANAODTPDCTL), (BIT30), (BIT30));
+	/* DCOMP: Dither PU Enable */
+	mrc_alt_write_mask(DDRPHY, (DQANADLYPUCTL), (BIT30), (BIT30));
+	/* DCOMP: Dither PD Enable */
+	mrc_alt_write_mask(DDRPHY, (DQANADLYPDCTL), (BIT30), (BIT30));
+	/* DCOMP: Dither PU Enable */
+	mrc_alt_write_mask(DDRPHY, (CMDANADLYPUCTL), (BIT30), (BIT30));
+	/* DCOMP: Dither PD Enable */
+	mrc_alt_write_mask(DDRPHY, (CMDANADLYPDCTL), (BIT30), (BIT30));
+	/* DCOMP: Dither PU Enable */
+	mrc_alt_write_mask(DDRPHY, (CLKANADLYPUCTL), (BIT30), (BIT30));
+	/* DCOMP: Dither PD Enable */
+	mrc_alt_write_mask(DDRPHY, (CLKANADLYPDCTL), (BIT30), (BIT30));
+	/* DCOMP: Dither PU Enable */
+	mrc_alt_write_mask(DDRPHY, (DQSANADLYPUCTL), (BIT30), (BIT30));
+	/* DCOMP: Dither PD Enable */
+	mrc_alt_write_mask(DDRPHY, (DQSANADLYPDCTL), (BIT30), (BIT30));
+	/* DCOMP: Dither PU Enable */
+	mrc_alt_write_mask(DDRPHY, (CTLANADLYPUCTL), (BIT30), (BIT30));
+	/* DCOMP: Dither PD Enable */
+	mrc_alt_write_mask(DDRPHY, (CTLANADLYPDCTL), (BIT30), (BIT30));
+	/* TCO: Dither PU Enable */
+	mrc_alt_write_mask(DDRPHY, (DQANATCOPUCTL), (BIT30), (BIT30));
+	/* TCO: Dither PD Enable */
+	mrc_alt_write_mask(DDRPHY, (DQANATCOPDCTL), (BIT30), (BIT30));
+	/* TCO: Dither PU Enable */
+	mrc_alt_write_mask(DDRPHY, (CLKANATCOPUCTL), (BIT30), (BIT30));
+	/* TCO: Dither PD Enable */
+	mrc_alt_write_mask(DDRPHY, (CLKANATCOPDCTL), (BIT30), (BIT30));
+	/* TCO: Dither PU Enable */
+	mrc_alt_write_mask(DDRPHY, (DQSANATCOPUCTL), (BIT30), (BIT30));
+	/* TCO: Dither PD Enable */
+	mrc_alt_write_mask(DDRPHY, (DQSANATCOPDCTL), (BIT30), (BIT30));
+	/* TCOCOMP: Pulse Count */
+	mrc_alt_write_mask(DDRPHY, (TCOCNTCTRL), (0x1 << 0), (BIT1 | BIT0));
+	/* ODT: CMD/CTL PD/PU */
+	mrc_alt_write_mask(DDRPHY,
+		(CHNLBUFSTATIC), ((0x03 << 24) | (0x03 << 16)),
+		(BIT28 | BIT27 | BIT26 | BIT25 | BIT24 |
+		BIT20 | BIT19 | BIT18 | BIT17 | BIT16));
+	/* Set 1us counter */
+	mrc_alt_write_mask(DDRPHY,
+		(MSCNTR), (0x64 << 0),
+		(BIT7 | BIT6 | BIT5 | BIT4 | BIT3 | BIT2 | BIT1 | BIT0));
+	mrc_alt_write_mask(DDRPHY,
+		(LATCH1CTL), (0x1 << 28),
+		(BIT30 | BIT29 | BIT28));
+
+	/* Release PHY from reset */
+	mrc_alt_write_mask(DDRPHY, MASTERRSTN, BIT0, BIT0);
+
+	/* STEP1 */
+	mrc_post_code(0x03, 0x11);
+
+	for (ch = 0; ch < NUM_CHANNELS; ch++) {
+		if (mrc_params->channel_enables & (1 << ch)) {
+			/* DQ01-DQ23 */
+			for (bl_grp = 0;
+			     bl_grp < ((NUM_BYTE_LANES / bl_divisor) / 2);
+			     bl_grp++) {
+				mrc_alt_write_mask(DDRPHY,
+					(DQMDLLCTL +
+					(bl_grp * DDRIODQ_BL_OFFSET) +
+					(ch * DDRIODQ_CH_OFFSET)),
+					(BIT13),
+					(BIT13));	/* Enable VREG */
+				delay_n(3);
+			}
+
+			/* ECC */
+			mrc_alt_write_mask(DDRPHY, (ECCMDLLCTL),
+				(BIT13), (BIT13));	/* Enable VREG */
+			delay_n(3);
+			/* CMD */
+			mrc_alt_write_mask(DDRPHY,
+				(CMDMDLLCTL + (ch * DDRIOCCC_CH_OFFSET)),
+				(BIT13), (BIT13));	/* Enable VREG */
+			delay_n(3);
+			/* CLK-CTL */
+			mrc_alt_write_mask(DDRPHY,
+				(CCMDLLCTL + (ch * DDRIOCCC_CH_OFFSET)),
+				(BIT13), (BIT13));	/* Enable VREG */
+			delay_n(3);
+		}
+	}
+
+	/* STEP2 */
+	mrc_post_code(0x03, 0x12);
+	delay_n(200);
+
+	for (ch = 0; ch < NUM_CHANNELS; ch++) {
+		if (mrc_params->channel_enables & (1 << ch)) {
+			/* DQ01-DQ23 */
+			for (bl_grp = 0;
+			     bl_grp < ((NUM_BYTE_LANES / bl_divisor) / 2);
+			     bl_grp++) {
+				mrc_alt_write_mask(DDRPHY,
+					(DQMDLLCTL +
+					(bl_grp * DDRIODQ_BL_OFFSET) +
+					(ch * DDRIODQ_CH_OFFSET)),
+					(BIT17),
+					(BIT17));	/* Enable MCDLL */
+				delay_n(50);
+			}
+
+		/* ECC */
+		mrc_alt_write_mask(DDRPHY, (ECCMDLLCTL),
+			(BIT17), (BIT17));	/* Enable MCDLL */
+		delay_n(50);
+		/* CMD */
+		mrc_alt_write_mask(DDRPHY,
+			(CMDMDLLCTL + (ch * DDRIOCCC_CH_OFFSET)),
+			(BIT18), (BIT18));	/* Enable MCDLL */
+		delay_n(50);
+		/* CLK-CTL */
+		mrc_alt_write_mask(DDRPHY,
+			(CCMDLLCTL + (ch * DDRIOCCC_CH_OFFSET)),
+			(BIT18), (BIT18));	/* Enable MCDLL */
+		delay_n(50);
+		}
+	}
+
+	/* STEP3: */
+	mrc_post_code(0x03, 0x13);
+	delay_n(100);
+
+	for (ch = 0; ch < NUM_CHANNELS; ch++) {
+		if (mrc_params->channel_enables & (1 << ch)) {
+			/* DQ01-DQ23 */
+			for (bl_grp = 0;
+			     bl_grp < ((NUM_BYTE_LANES / bl_divisor) / 2);
+			     bl_grp++) {
+#ifdef FORCE_16BIT_DDRIO
+				temp = ((bl_grp) &&
+					(mrc_params->channel_width == X16)) ?
+					((0x1 << 12) | (0x1 << 8) |
+					(0xF << 4) | (0xF << 0)) :
+					((0xF << 12) | (0xF << 8) |
+					(0xF << 4) | (0xF << 0));
+#else
+				temp = ((0xF << 12) | (0xF << 8) |
+					(0xF << 4) | (0xF << 0));
+#endif
+				/* Enable TXDLL */
+				mrc_alt_write_mask(DDRPHY,
+					(DQDLLTXCTL +
+					(bl_grp * DDRIODQ_BL_OFFSET) +
+					(ch * DDRIODQ_CH_OFFSET)),
+					temp, 0xFFFF);
+				delay_n(3);
+				/* Enable RXDLL */
+				mrc_alt_write_mask(DDRPHY,
+					(DQDLLRXCTL +
+					(bl_grp * DDRIODQ_BL_OFFSET) +
+					(ch * DDRIODQ_CH_OFFSET)),
+					(BIT3 | BIT2 | BIT1 | BIT0),
+					(BIT3 | BIT2 | BIT1 | BIT0));
+				delay_n(3);
+				/* Enable RXDLL Overrides BL0 */
+				mrc_alt_write_mask(DDRPHY,
+					(B0OVRCTL +
+					(bl_grp * DDRIODQ_BL_OFFSET) +
+					(ch * DDRIODQ_CH_OFFSET)),
+					(BIT3 | BIT2 | BIT1 | BIT0),
+					(BIT3 | BIT2 | BIT1 | BIT0));
+			}
+
+			/* ECC */
+			temp = ((0xF << 12) | (0xF << 8) |
+				(0xF << 4) | (0xF << 0));
+			mrc_alt_write_mask(DDRPHY, (ECCDLLTXCTL),
+				temp, 0xFFFF);
+			delay_n(3);
+
+			/* CMD (PO) */
+			mrc_alt_write_mask(DDRPHY,
+				(CMDDLLTXCTL + (ch * DDRIOCCC_CH_OFFSET)),
+				temp, 0xFFFF);
+			delay_n(3);
+		}
+	}
+
+	/* STEP4 */
+	mrc_post_code(0x03, 0x14);
+
+	for (ch = 0; ch < NUM_CHANNELS; ch++) {
+		if (mrc_params->channel_enables & (1 << ch)) {
+			/* Host To Memory Clock Alignment (HMC) for 800/1066 */
+			for (bl_grp = 0;
+			     bl_grp < ((NUM_BYTE_LANES / bl_divisor) / 2);
+			     bl_grp++) {
+				/* CLK_ALIGN_MOD_ID */
+				mrc_alt_write_mask(DDRPHY,
+					(DQCLKALIGNREG2 +
+					(bl_grp * DDRIODQ_BL_OFFSET) +
+					(ch * DDRIODQ_CH_OFFSET)),
+					(bl_grp) ? (0x3) : (0x1),
+					(BIT3 | BIT2 | BIT1 | BIT0));
+			}
+
+			mrc_alt_write_mask(DDRPHY,
+				(ECCCLKALIGNREG2 + (ch * DDRIODQ_CH_OFFSET)),
+				0x2,
+				(BIT3 | BIT2 | BIT1 | BIT0));
+			mrc_alt_write_mask(DDRPHY,
+				(CMDCLKALIGNREG2 + (ch * DDRIODQ_CH_OFFSET)),
+				0x0,
+				(BIT3 | BIT2 | BIT1 | BIT0));
+			mrc_alt_write_mask(DDRPHY,
+				(CCCLKALIGNREG2 + (ch * DDRIODQ_CH_OFFSET)),
+				0x2,
+				(BIT3 | BIT2 | BIT1 | BIT0));
+			mrc_alt_write_mask(DDRPHY,
+				(CMDCLKALIGNREG0 + (ch * DDRIOCCC_CH_OFFSET)),
+				(0x2 << 4), (BIT5 | BIT4));
+			/*
+			 * NUM_SAMPLES, MAX_SAMPLES,
+			 * MACRO_PI_STEP, MICRO_PI_STEP
+			 */
+			mrc_alt_write_mask(DDRPHY,
+				(CMDCLKALIGNREG1 + (ch * DDRIOCCC_CH_OFFSET)),
+				((0x18 << 16) | (0x10 << 8) |
+				(0x8 << 2) | (0x1 << 0)),
+				(BIT22 | BIT21 | BIT20 | BIT19 | BIT18 | BIT17 |
+				BIT16 | BIT14 | BIT13 | BIT12 | BIT11 | BIT10 |
+				BIT9 | BIT8 | BIT7 | BIT6 | BIT5 | BIT4 | BIT3 |
+				BIT2 | BIT1 | BIT0));
+			/* TOTAL_NUM_MODULES, FIRST_U_PARTITION */
+			mrc_alt_write_mask(DDRPHY,
+				(CMDCLKALIGNREG2 + (ch * DDRIOCCC_CH_OFFSET)),
+				((0x10 << 16) | (0x4 << 8) | (0x2 << 4)),
+				(BIT20 | BIT19 | BIT18 | BIT17 | BIT16 |
+				BIT11 | BIT10 | BIT9 | BIT8 | BIT7 | BIT6 |
+				BIT5 | BIT4));
+#ifdef HMC_TEST
+			/* START_CLK_ALIGN=1 */
+			mrc_alt_write_mask(DDRPHY,
+				(CMDCLKALIGNREG0 + (ch * DDRIOCCC_CH_OFFSET)),
+				BIT24, BIT24);
+			while (msg_port_alt_read(DDRPHY,
+				(CMDCLKALIGNREG0 + (ch * DDRIOCCC_CH_OFFSET))) &
+				BIT24)
+				;	/* wait for START_CLK_ALIGN=0 */
+#endif
+
+			/* Set RD/WR Pointer Seperation & COUNTEN & FIFOPTREN */
+			mrc_alt_write_mask(DDRPHY,
+				(CMDPTRREG + (ch * DDRIOCCC_CH_OFFSET)),
+				BIT0, BIT0);	/* WRPTRENABLE=1 */
+
+			/* COMP initial */
+			/* enable bypass for CLK buffer (PO) */
+			mrc_alt_write_mask(DDRPHY,
+				(COMPEN0CH0 + (ch * DDRCOMP_CH_OFFSET)),
+				BIT5, BIT5);
+			/* Initial COMP Enable */
+			mrc_alt_write_mask(DDRPHY, (CMPCTRL),
+				(BIT0), (BIT0));
+			/* wait for Initial COMP Enable = 0 */
+			while (msg_port_alt_read(DDRPHY, (CMPCTRL)) & BIT0)
+				;
+			/* disable bypass for CLK buffer (PO) */
+			mrc_alt_write_mask(DDRPHY,
+				(COMPEN0CH0 + (ch * DDRCOMP_CH_OFFSET)),
+				~BIT5, BIT5);
+
+			/* IOBUFACT */
+
+			/* STEP4a */
+			mrc_alt_write_mask(DDRPHY,
+				(CMDCFGREG0 + (ch * DDRIOCCC_CH_OFFSET)),
+				BIT2, BIT2);	/* IOBUFACTRST_N=1 */
+
+			/* DDRPHY initialization complete */
+			mrc_alt_write_mask(DDRPHY,
+				(CMDPMCONFIG0 + (ch * DDRIOCCC_CH_OFFSET)),
+				BIT20, BIT20);	/* SPID_INIT_COMPLETE=1 */
+		}
+	}
+
+	LEAVEFN();
+}
+
+/* This function performs JEDEC initialization on all enabled channels */
+void perform_jedec_init(struct mrc_params *mrc_params)
+{
+	uint8_t twr, wl, rank;
+	uint32_t tck;
+	u32 dtr0;
+	u32 drp;
+	u32 drmc;
+	u32 mrs0_cmd = 0;
+	u32 emrs1_cmd = 0;
+	u32 emrs2_cmd = 0;
+	u32 emrs3_cmd = 0;
+
+	ENTERFN();
+
+	/* jedec_init starts */
+	mrc_post_code(0x04, 0x00);
+
+	/* DDR3_RESET_SET=0, DDR3_RESET_RESET=1 */
+	mrc_alt_write_mask(DDRPHY, CCDDR3RESETCTL, BIT1, (BIT8 | BIT1));
+
+	/* Assert RESET# for 200us */
+	delay_u(200);
+
+	/* DDR3_RESET_SET=1, DDR3_RESET_RESET=0 */
+	mrc_alt_write_mask(DDRPHY, CCDDR3RESETCTL, BIT8, (BIT8 | BIT1));
+
+	dtr0 = msg_port_read(MEM_CTLR, DTR0);
+
+	/*
+	 * Set CKEVAL for populated ranks
+	 * then send NOP to each rank (#4550197)
+	 */
+
+	drp = msg_port_read(MEM_CTLR, DRP);
+	drp &= 0x3;
+
+	drmc = msg_port_read(MEM_CTLR, DRMC);
+	drmc &= 0xFFFFFFFC;
+	drmc |= (BIT4 | drp);
+
+	msg_port_write(MEM_CTLR, DRMC, drmc);
+
+	for (rank = 0; rank < NUM_RANKS; rank++) {
+		/* Skip to next populated rank */
+		if ((mrc_params->rank_enables & (1 << rank)) == 0)
+			continue;
+
+		dram_init_command(DCMD_NOP(rank));
+	}
+
+	msg_port_write(MEM_CTLR, DRMC,
+		(mrc_params->rd_odt_value == 0 ? BIT12 : 0));
+
+	/*
+	 * setup for emrs 2
+	 * BIT[15:11] --> Always "0"
+	 * BIT[10:09] --> Rtt_WR: want "Dynamic ODT Off" (0)
+	 * BIT[08]    --> Always "0"
+	 * BIT[07]    --> SRT: use sr_temp_range
+	 * BIT[06]    --> ASR: want "Manual SR Reference" (0)
+	 * BIT[05:03] --> CWL: use oem_tCWL
+	 * BIT[02:00] --> PASR: want "Full Array" (0)
+	 */
+	emrs2_cmd |= (2 << 3);
+	wl = 5 + mrc_params->ddr_speed;
+	emrs2_cmd |= ((wl - 5) << 9);
+	emrs2_cmd |= (mrc_params->sr_temp_range << 13);
+
+	/*
+	 * setup for emrs 3
+	 * BIT[15:03] --> Always "0"
+	 * BIT[02]    --> MPR: want "Normal Operation" (0)
+	 * BIT[01:00] --> MPR_Loc: want "Predefined Pattern" (0)
+	 */
+	emrs3_cmd |= (3 << 3);
+
+	/*
+	 * setup for emrs 1
+	 * BIT[15:13]     --> Always "0"
+	 * BIT[12:12]     --> Qoff: want "Output Buffer Enabled" (0)
+	 * BIT[11:11]     --> TDQS: want "Disabled" (0)
+	 * BIT[10:10]     --> Always "0"
+	 * BIT[09,06,02]  --> Rtt_nom: use rtt_nom_value
+	 * BIT[08]        --> Always "0"
+	 * BIT[07]        --> WR_LVL: want "Disabled" (0)
+	 * BIT[05,01]     --> DIC: use ron_value
+	 * BIT[04:03]     --> AL: additive latency want "0" (0)
+	 * BIT[00]        --> DLL: want "Enable" (0)
+	 *
+	 * (BIT5|BIT1) set Ron value
+	 * 00 --> RZQ/6 (40ohm)
+	 * 01 --> RZQ/7 (34ohm)
+	 * 1* --> RESERVED
+	 *
+	 * (BIT9|BIT6|BIT2) set Rtt_nom value
+	 * 000 --> Disabled
+	 * 001 --> RZQ/4 ( 60ohm)
+	 * 010 --> RZQ/2 (120ohm)
+	 * 011 --> RZQ/6 ( 40ohm)
+	 * 1** --> RESERVED
+	 */
+	emrs1_cmd |= (1 << 3);
+	emrs1_cmd &= ~BIT6;
+
+	if (mrc_params->ron_value == 0)
+		emrs1_cmd |= BIT7;
+	else
+		emrs1_cmd &= ~BIT7;
+
+	if (mrc_params->rtt_nom_value == 0)
+		emrs1_cmd |= (DDR3_EMRS1_RTTNOM_40 << 6);
+	else if (mrc_params->rtt_nom_value == 1)
+		emrs1_cmd |= (DDR3_EMRS1_RTTNOM_60 << 6);
+	else if (mrc_params->rtt_nom_value == 2)
+		emrs1_cmd |= (DDR3_EMRS1_RTTNOM_120 << 6);
+
+	/* save MRS1 value (excluding control fields) */
+	mrc_params->mrs1 = emrs1_cmd >> 6;
+
+	/*
+	 * setup for mrs 0
+	 * BIT[15:13]     --> Always "0"
+	 * BIT[12]        --> PPD: for Quark (1)
+	 * BIT[11:09]     --> WR: use oem_tWR
+	 * BIT[08]        --> DLL: want "Reset" (1, self clearing)
+	 * BIT[07]        --> MODE: want "Normal" (0)
+	 * BIT[06:04,02]  --> CL: use oem_tCAS
+	 * BIT[03]        --> RD_BURST_TYPE: want "Interleave" (1)
+	 * BIT[01:00]     --> BL: want "8 Fixed" (0)
+	 * WR:
+	 * 0 --> 16
+	 * 1 --> 5
+	 * 2 --> 6
+	 * 3 --> 7
+	 * 4 --> 8
+	 * 5 --> 10
+	 * 6 --> 12
+	 * 7 --> 14
+	 * CL:
+	 * BIT[02:02] "0" if oem_tCAS <= 11 (1866?)
+	 * BIT[06:04] use oem_tCAS-4
+	 */
+	mrs0_cmd |= BIT14;
+	mrs0_cmd |= BIT18;
+	mrs0_cmd |= ((((dtr0 >> 12) & 7) + 1) << 10);
+
+	tck = t_ck[mrc_params->ddr_speed];
+	/* Per JEDEC: tWR=15000ps DDR2/3 from 800-1600 */
+	twr = MCEIL(15000, tck);
+	mrs0_cmd |= ((twr - 4) << 15);
+
+	for (rank = 0; rank < NUM_RANKS; rank++) {
+		/* Skip to next populated rank */
+		if ((mrc_params->rank_enables & (1 << rank)) == 0)
+			continue;
+
+		emrs2_cmd |= (rank << 22);
+		dram_init_command(emrs2_cmd);
+
+		emrs3_cmd |= (rank << 22);
+		dram_init_command(emrs3_cmd);
+
+		emrs1_cmd |= (rank << 22);
+		dram_init_command(emrs1_cmd);
+
+		mrs0_cmd |= (rank << 22);
+		dram_init_command(mrs0_cmd);
+
+		dram_init_command(DCMD_ZQCL(rank));
+	}
+
+	LEAVEFN();
+}
+
+/*
+ * Dunit Initialization Complete
+ *
+ * Indicates that initialization of the Dunit has completed.
+ *
+ * Memory accesses are permitted and maintenance operation begins.
+ * Until this bit is set to a 1, the memory controller will not accept
+ * DRAM requests from the MEMORY_MANAGER or HTE.
+ */
+void set_ddr_init_complete(struct mrc_params *mrc_params)
+{
+	u32 dco;
+
+	ENTERFN();
+
+	dco = msg_port_read(MEM_CTLR, DCO);
+	dco &= ~BIT28;
+	dco |= BIT31;
+	msg_port_write(MEM_CTLR, DCO, dco);
+
+	LEAVEFN();
+}
+
+/*
+ * This function will retrieve relevant timing data
+ *
+ * This data will be used on subsequent boots to speed up boot times
+ * and is required for Suspend To RAM capabilities.
+ */
+void restore_timings(struct mrc_params *mrc_params)
+{
+	uint8_t ch, rk, bl;
+	const struct mrc_timings *mt = &mrc_params->timings;
+
+	for (ch = 0; ch < NUM_CHANNELS; ch++) {
+		for (rk = 0; rk < NUM_RANKS; rk++) {
+			for (bl = 0; bl < NUM_BYTE_LANES; bl++) {
+				set_rcvn(ch, rk, bl, mt->rcvn[ch][rk][bl]);
+				set_rdqs(ch, rk, bl, mt->rdqs[ch][rk][bl]);
+				set_wdqs(ch, rk, bl, mt->wdqs[ch][rk][bl]);
+				set_wdq(ch, rk, bl, mt->wdq[ch][rk][bl]);
+				if (rk == 0) {
+					/* VREF (RANK0 only) */
+					set_vref(ch, bl, mt->vref[ch][bl]);
+				}
+			}
+			set_wctl(ch, rk, mt->wctl[ch][rk]);
+		}
+		set_wcmd(ch, mt->wcmd[ch]);
+	}
+}
+
+/*
+ * Configure default settings normally set as part of read training
+ *
+ * Some defaults have to be set earlier as they may affect earlier
+ * training steps.
+ */
+void default_timings(struct mrc_params *mrc_params)
+{
+	uint8_t ch, rk, bl;
+
+	for (ch = 0; ch < NUM_CHANNELS; ch++) {
+		for (rk = 0; rk < NUM_RANKS; rk++) {
+			for (bl = 0; bl < NUM_BYTE_LANES; bl++) {
+				set_rdqs(ch, rk, bl, 24);
+				if (rk == 0) {
+					/* VREF (RANK0 only) */
+					set_vref(ch, bl, 32);
+				}
+			}
+		}
+	}
+}
+
+/*
+ * This function will perform our RCVEN Calibration Algorithm.
+ * We will only use the 2xCLK domain timings to perform RCVEN Calibration.
+ * All byte lanes will be calibrated "simultaneously" per channel per rank.
+ */
+void rcvn_cal(struct mrc_params *mrc_params)
+{
+	uint8_t ch;	/* channel counter */
+	uint8_t rk;	/* rank counter */
+	uint8_t bl;	/* byte lane counter */
+	uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1;
+
+#ifdef R2R_SHARING
+	/* used to find placement for rank2rank sharing configs */
+	uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES];
+#ifndef BACKUP_RCVN
+	/* used to find placement for rank2rank sharing configs */
+	uint32_t num_ranks_enabled = 0;
+#endif
+#endif
+
+#ifdef BACKUP_RCVN
+#else
+	uint32_t temp;
+	/* absolute PI value to be programmed on the byte lane */
+	uint32_t delay[NUM_BYTE_LANES];
+	u32 dtr1, dtr1_save;
+#endif
+
+	ENTERFN();
+
+	/* rcvn_cal starts */
+	mrc_post_code(0x05, 0x00);
+
+#ifndef BACKUP_RCVN
+	/* need separate burst to sample DQS preamble */
+	dtr1 = msg_port_read(MEM_CTLR, DTR1);
+	dtr1_save = dtr1;
+	dtr1 |= BIT12;
+	msg_port_write(MEM_CTLR, DTR1, dtr1);
+#endif
+
+#ifdef R2R_SHARING
+	/* need to set "final_delay[][]" elements to "0" */
+	memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay));
+#endif
+
+	/* loop through each enabled channel */
+	for (ch = 0; ch < NUM_CHANNELS; ch++) {
+		if (mrc_params->channel_enables & (1 << ch)) {
+			/* perform RCVEN Calibration on a per rank basis */
+			for (rk = 0; rk < NUM_RANKS; rk++) {
+				if (mrc_params->rank_enables & (1 << rk)) {
+					/*
+					 * POST_CODE here indicates the current
+					 * channel and rank being calibrated
+					 */
+					mrc_post_code(0x05, (0x10 + ((ch << 4) | rk)));
+
+#ifdef BACKUP_RCVN
+					/* et hard-coded timing values */
+					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++)
+						set_rcvn(ch, rk, bl, ddr_rcvn[PLATFORM_ID]);
+#else
+					/* enable FIFORST */
+					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl += 2) {
+						mrc_alt_write_mask(DDRPHY,
+							(B01PTRCTL1 +
+							((bl >> 1) * DDRIODQ_BL_OFFSET) +
+							(ch * DDRIODQ_CH_OFFSET)),
+							0, BIT8);
+					}
+					/* initialize the starting delay to 128 PI (cas +1 CLK) */
+					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
+						/* 1x CLK domain timing is cas-4 */
+						delay[bl] = (4 + 1) * FULL_CLK;
+
+						set_rcvn(ch, rk, bl, delay[bl]);
+					}
+
+					/* now find the rising edge */
+					find_rising_edge(mrc_params, delay, ch, rk, true);
+
+					/* Now increase delay by 32 PI (1/4 CLK) to place in center of high pulse */
+					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
+						delay[bl] += QRTR_CLK;
+						set_rcvn(ch, rk, bl, delay[bl]);
+					}
+					/* Now decrement delay by 128 PI (1 CLK) until we sample a "0" */
+					do {
+						temp = sample_dqs(mrc_params, ch, rk, true);
+						for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
+							if (temp & (1 << bl)) {
+								if (delay[bl] >= FULL_CLK) {
+									delay[bl] -= FULL_CLK;
+									set_rcvn(ch, rk, bl, delay[bl]);
+								} else {
+									/* not enough delay */
+									training_message(ch, rk, bl);
+									mrc_post_code(0xEE, 0x50);
+								}
+							}
+						}
+					} while (temp & 0xFF);
+
+#ifdef R2R_SHARING
+					/* increment "num_ranks_enabled" */
+					num_ranks_enabled++;
+					/* Finally increment delay by 32 PI (1/4 CLK) to place in center of preamble */
+					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
+						delay[bl] += QRTR_CLK;
+						/* add "delay[]" values to "final_delay[][]" for rolling average */
+						final_delay[ch][bl] += delay[bl];
+						/* set timing based on rolling average values */
+						set_rcvn(ch, rk, bl, ((final_delay[ch][bl]) / num_ranks_enabled));
+					}
+#else
+					/* Finally increment delay by 32 PI (1/4 CLK) to place in center of preamble */
+					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
+						delay[bl] += QRTR_CLK;
+						set_rcvn(ch, rk, bl, delay[bl]);
+					}
+#endif
+
+					/* disable FIFORST */
+					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl += 2) {
+						mrc_alt_write_mask(DDRPHY,
+							(B01PTRCTL1 +
+							((bl >> 1) * DDRIODQ_BL_OFFSET) +
+							(ch * DDRIODQ_CH_OFFSET)),
+							BIT8, BIT8);
+					}
+#endif
+				}
+			}
+		}
+	}
+
+#ifndef BACKUP_RCVN
+	/* restore original */
+	msg_port_write(MEM_CTLR, DTR1, dtr1_save);
+#endif
+
+	LEAVEFN();
+}
+
+/*
+ * This function will perform the Write Levelling algorithm
+ * (align WCLK and WDQS).
+ *
+ * This algorithm will act on each rank in each channel separately.
+ */
+void wr_level(struct mrc_params *mrc_params)
+{
+	uint8_t ch;	/* channel counter */
+	uint8_t rk;	/* rank counter */
+	uint8_t bl;	/* byte lane counter */
+	uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1;
+
+#ifdef R2R_SHARING
+	/* used to find placement for rank2rank sharing configs */
+	uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES];
+#ifndef BACKUP_WDQS
+	/* used to find placement for rank2rank sharing configs */
+	uint32_t num_ranks_enabled = 0;
+#endif
+#endif
+
+#ifdef BACKUP_WDQS
+#else
+	/* determines stop condition for CRS_WR_LVL */
+	bool all_edges_found;
+	/* absolute PI value to be programmed on the byte lane */
+	uint32_t delay[NUM_BYTE_LANES];
+	/*
+	 * static makes it so the data is loaded in the heap once by shadow(),
+	 * where non-static copies the data onto the stack every time this
+	 * function is called
+	 */
+	uint32_t address;	/* address to be checked during COARSE_WR_LVL */
+	u32 dtr4, dtr4_save;
+#endif
+
+	ENTERFN();
+
+	/* wr_level starts */
+	mrc_post_code(0x06, 0x00);
+
+#ifdef R2R_SHARING
+	/* need to set "final_delay[][]" elements to "0" */
+	memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay));
+#endif
+
+	/* loop through each enabled channel */
+	for (ch = 0; ch < NUM_CHANNELS; ch++) {
+		if (mrc_params->channel_enables & (1 << ch)) {
+			/* perform WRITE LEVELING algorithm on a per rank basis */
+			for (rk = 0; rk < NUM_RANKS; rk++) {
+				if (mrc_params->rank_enables & (1 << rk)) {
+					/*
+					 * POST_CODE here indicates the current
+					 * rank and channel being calibrated
+					 */
+					mrc_post_code(0x06, (0x10 + ((ch << 4) | rk)));
+
+#ifdef BACKUP_WDQS
+					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
+						set_wdqs(ch, rk, bl, ddr_wdqs[PLATFORM_ID]);
+						set_wdq(ch, rk, bl, (ddr_wdqs[PLATFORM_ID] - QRTR_CLK));
+					}
+#else
+					/*
+					 * perform a single PRECHARGE_ALL command to
+					 * make DRAM state machine go to IDLE state
+					 */
+					dram_init_command(DCMD_PREA(rk));
+
+					/*
+					 * enable Write Levelling Mode
+					 * (EMRS1 w/ Write Levelling Mode Enable)
+					 */
+					dram_init_command(DCMD_MRS1(rk, 0x0082));
+
+					/*
+					 * set ODT DRAM Full Time Termination
+					 * disable in MCU
+					 */
+
+					dtr4 = msg_port_read(MEM_CTLR, DTR4);
+					dtr4_save = dtr4;
+					dtr4 |= BIT15;
+					msg_port_write(MEM_CTLR, DTR4, dtr4);
+
+					for (bl = 0; bl < ((NUM_BYTE_LANES / bl_divisor) / 2); bl++) {
+						/*
+						 * Enable Sandy Bridge Mode (WDQ Tri-State) &
+						 * Ensure 5 WDQS pulses during Write Leveling
+						 */
+						mrc_alt_write_mask(DDRPHY,
+							DQCTL + (DDRIODQ_BL_OFFSET * bl) + (DDRIODQ_CH_OFFSET * ch),
+							(BIT28 | BIT8 | BIT6 | BIT4 | BIT2),
+							(BIT28 | BIT9 | BIT8 | BIT7 | BIT6 | BIT5 | BIT4 | BIT3 | BIT2));
+					}
+
+					/* Write Leveling Mode enabled in IO */
+					mrc_alt_write_mask(DDRPHY,
+						CCDDR3RESETCTL + (DDRIOCCC_CH_OFFSET * ch),
+						BIT16, BIT16);
+
+					/* Initialize the starting delay to WCLK */
+					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
+						/*
+						 * CLK0 --> RK0
+						 * CLK1 --> RK1
+						 */
+						delay[bl] = get_wclk(ch, rk);
+
+						set_wdqs(ch, rk, bl, delay[bl]);
+					}
+
+					/* now find the rising edge */
+					find_rising_edge(mrc_params, delay, ch, rk, false);
+
+					/* disable Write Levelling Mode */
+					mrc_alt_write_mask(DDRPHY,
+						CCDDR3RESETCTL + (DDRIOCCC_CH_OFFSET * ch),
+						0, BIT16);
+
+					for (bl = 0; bl < ((NUM_BYTE_LANES / bl_divisor) / 2); bl++) {
+						/* Disable Sandy Bridge Mode & Ensure 4 WDQS pulses during normal operation */
+						mrc_alt_write_mask(DDRPHY,
+							DQCTL + (DDRIODQ_BL_OFFSET * bl) + (DDRIODQ_CH_OFFSET * ch),
+							(BIT8 | BIT6 | BIT4 | BIT2),
+							(BIT28 | BIT9 | BIT8 | BIT7 | BIT6 | BIT5 | BIT4 | BIT3 | BIT2));
+					}
+
+					/* restore original DTR4 */
+					msg_port_write(MEM_CTLR, DTR4, dtr4_save);
+
+					/*
+					 * restore original value
+					 * (Write Levelling Mode Disable)
+					 */
+					dram_init_command(DCMD_MRS1(rk, mrc_params->mrs1));
+
+					/*
+					 * perform a single PRECHARGE_ALL command to
+					 * make DRAM state machine go to IDLE state
+					 */
+					dram_init_command(DCMD_PREA(rk));
+
+					mrc_post_code(0x06, (0x30 + ((ch << 4) | rk)));
+
+					/*
+					 * COARSE WRITE LEVEL:
+					 * check that we're on the correct clock edge
+					 */
+
+					/* hte reconfiguration request */
+					mrc_params->hte_setup = 1;
+
+					/* start CRS_WR_LVL with WDQS = WDQS + 128 PI */
+					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
+						delay[bl] = get_wdqs(ch, rk, bl) + FULL_CLK;
+						set_wdqs(ch, rk, bl, delay[bl]);
+						/*
+						 * program WDQ timings based on WDQS
+						 * (WDQ = WDQS - 32 PI)
+						 */
+						set_wdq(ch, rk, bl, (delay[bl] - QRTR_CLK));
+					}
+
+					/* get an address in the targeted channel/rank */
+					address = get_addr(ch, rk);
+					do {
+						uint32_t coarse_result = 0x00;
+						uint32_t coarse_result_mask = byte_lane_mask(mrc_params);
+						/* assume pass */
+						all_edges_found = true;
+
+						mrc_params->hte_setup = 1;
+						coarse_result = check_rw_coarse(mrc_params, address);
+
+						/* check for failures and margin the byte lane back 128 PI (1 CLK) */
+						for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
+							if (coarse_result & (coarse_result_mask << bl)) {
+								all_edges_found = false;
+								delay[bl] -= FULL_CLK;
+								set_wdqs(ch, rk, bl, delay[bl]);
+								/* program WDQ timings based on WDQS (WDQ = WDQS - 32 PI) */
+								set_wdq(ch, rk, bl, (delay[bl] - QRTR_CLK));
+							}
+						}
+					} while (!all_edges_found);
+
+#ifdef R2R_SHARING
+					/* increment "num_ranks_enabled" */
+					 num_ranks_enabled++;
+					/* accumulate "final_delay[][]" values from "delay[]" values for rolling average */
+					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
+						final_delay[ch][bl] += delay[bl];
+						set_wdqs(ch, rk, bl, ((final_delay[ch][bl]) / num_ranks_enabled));
+						/* program WDQ timings based on WDQS (WDQ = WDQS - 32 PI) */
+						set_wdq(ch, rk, bl, ((final_delay[ch][bl]) / num_ranks_enabled) - QRTR_CLK);
+					}
+#endif
+#endif
+				}
+			}
+		}
+	}
+
+	LEAVEFN();
+}
+
+void prog_page_ctrl(struct mrc_params *mrc_params)
+{
+	u32 dpmc0;
+
+	ENTERFN();
+
+	dpmc0 = msg_port_read(MEM_CTLR, DPMC0);
+	dpmc0 &= ~(BIT16 | BIT17 | BIT18);
+	dpmc0 |= (4 << 16);
+	dpmc0 |= BIT21;
+	msg_port_write(MEM_CTLR, DPMC0, dpmc0);
+}
+
+/*
+ * This function will perform the READ TRAINING Algorithm on all
+ * channels/ranks/byte_lanes simultaneously to minimize execution time.
+ *
+ * The idea here is to train the VREF and RDQS (and eventually RDQ) values
+ * to achieve maximum READ margins. The algorithm will first determine the
+ * X coordinate (RDQS setting). This is done by collapsing the VREF eye
+ * until we find a minimum required RDQS eye for VREF_MIN and VREF_MAX.
+ * Then we take the averages of the RDQS eye at VREF_MIN and VREF_MAX,
+ * then average those; this will be the final X coordinate. The algorithm
+ * will then determine the Y coordinate (VREF setting). This is done by
+ * collapsing the RDQS eye until we find a minimum required VREF eye for
+ * RDQS_MIN and RDQS_MAX. Then we take the averages of the VREF eye at
+ * RDQS_MIN and RDQS_MAX, then average those; this will be the final Y
+ * coordinate.
+ *
+ * NOTE: this algorithm assumes the eye curves have a one-to-one relationship,
+ * meaning for each X the curve has only one Y and vice-a-versa.
+ */
+void rd_train(struct mrc_params *mrc_params)
+{
+	uint8_t ch;	/* channel counter */
+	uint8_t rk;	/* rank counter */
+	uint8_t bl;	/* byte lane counter */
+	uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1;
+#ifdef BACKUP_RDQS
+#else
+	uint8_t side_x;	/* tracks LEFT/RIGHT approach vectors */
+	uint8_t side_y;	/* tracks BOTTOM/TOP approach vectors */
+	/* X coordinate data (passing RDQS values) for approach vectors */
+	uint8_t x_coordinate[2][2][NUM_CHANNELS][NUM_RANKS][NUM_BYTE_LANES];
+	/* Y coordinate data (passing VREF values) for approach vectors */
+	uint8_t y_coordinate[2][2][NUM_CHANNELS][NUM_BYTE_LANES];
+	/* centered X (RDQS) */
+	uint8_t x_center[NUM_CHANNELS][NUM_RANKS][NUM_BYTE_LANES];
+	/* centered Y (VREF) */
+	uint8_t y_center[NUM_CHANNELS][NUM_BYTE_LANES];
+	uint32_t address;	/* target address for check_bls_ex() */
+	uint32_t result;	/* result of check_bls_ex() */
+	uint32_t bl_mask;	/* byte lane mask for result checking */
+#ifdef R2R_SHARING
+	/* used to find placement for rank2rank sharing configs */
+	uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES];
+	/* used to find placement for rank2rank sharing configs */
+	uint32_t num_ranks_enabled = 0;
+#endif
+#endif
+
+	/* rd_train starts */
+	mrc_post_code(0x07, 0x00);
+
+	ENTERFN();
+
+#ifdef BACKUP_RDQS
+	for (ch = 0; ch < NUM_CHANNELS; ch++) {
+		if (mrc_params->channel_enables & (1 << ch)) {
+			for (rk = 0; rk < NUM_RANKS; rk++) {
+				if (mrc_params->rank_enables & (1 << rk)) {
+					for (bl = 0;
+					     bl < (NUM_BYTE_LANES / bl_divisor);
+					     bl++) {
+						set_rdqs(ch, rk, bl, ddr_rdqs[PLATFORM_ID]);
+					}
+				}
+			}
+		}
+	}
+#else
+	/* initialize x/y_coordinate arrays */
+	for (ch = 0; ch < NUM_CHANNELS; ch++) {
+		if (mrc_params->channel_enables & (1 << ch)) {
+			for (rk = 0; rk < NUM_RANKS; rk++) {
+				if (mrc_params->rank_enables & (1 << rk)) {
+					for (bl = 0;
+					     bl < (NUM_BYTE_LANES / bl_divisor);
+					     bl++) {
+						/* x_coordinate */
+						x_coordinate[L][B][ch][rk][bl] = RDQS_MIN;
+						x_coordinate[R][B][ch][rk][bl] = RDQS_MAX;
+						x_coordinate[L][T][ch][rk][bl] = RDQS_MIN;
+						x_coordinate[R][T][ch][rk][bl] = RDQS_MAX;
+						/* y_coordinate */
+						y_coordinate[L][B][ch][bl] = VREF_MIN;
+						y_coordinate[R][B][ch][bl] = VREF_MIN;
+						y_coordinate[L][T][ch][bl] = VREF_MAX;
+						y_coordinate[R][T][ch][bl] = VREF_MAX;
+					}
+				}
+			}
+		}
+	}
+
+	/* initialize other variables */
+	bl_mask = byte_lane_mask(mrc_params);
+	address = get_addr(0, 0);
+
+#ifdef R2R_SHARING
+	/* need to set "final_delay[][]" elements to "0" */
+	memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay));
+#endif
+
+	/* look for passing coordinates */
+	for (side_y = B; side_y <= T; side_y++) {
+		for (side_x = L; side_x <= R; side_x++) {
+			mrc_post_code(0x07, (0x10 + (side_y * 2) + (side_x)));
+
+			/* find passing values */
+			for (ch = 0; ch < NUM_CHANNELS; ch++) {
+				if (mrc_params->channel_enables & (0x1 << ch)) {
+					for (rk = 0; rk < NUM_RANKS; rk++) {
+						if (mrc_params->rank_enables &
+							(0x1 << rk)) {
+							/* set x/y_coordinate search starting settings */
+							for (bl = 0;
+							     bl < (NUM_BYTE_LANES / bl_divisor);
+							     bl++) {
+								set_rdqs(ch, rk, bl,
+									 x_coordinate[side_x][side_y][ch][rk][bl]);
+								set_vref(ch, bl,
+									 y_coordinate[side_x][side_y][ch][bl]);
+							}
+
+							/* get an address in the target channel/rank */
+							address = get_addr(ch, rk);
+
+							/* request HTE reconfiguration */
+							mrc_params->hte_setup = 1;
+
+							/* test the settings */
+							do {
+								/* result[07:00] == failing byte lane (MAX 8) */
+								result = check_bls_ex(mrc_params, address);
+
+								/* check for failures */
+								if (result & 0xFF) {
+									/* at least 1 byte lane failed */
+									for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
+										if (result &
+											(bl_mask << bl)) {
+											/* adjust the RDQS values accordingly */
+											if (side_x == L)
+												x_coordinate[L][side_y][ch][rk][bl] += RDQS_STEP;
+											else
+												x_coordinate[R][side_y][ch][rk][bl] -= RDQS_STEP;
+
+											/* check that we haven't closed the RDQS_EYE too much */
+											if ((x_coordinate[L][side_y][ch][rk][bl] > (RDQS_MAX - MIN_RDQS_EYE)) ||
+												(x_coordinate[R][side_y][ch][rk][bl] < (RDQS_MIN + MIN_RDQS_EYE)) ||
+												(x_coordinate[L][side_y][ch][rk][bl] ==
+												x_coordinate[R][side_y][ch][rk][bl])) {
+												/*
+												 * not enough RDQS margin available at this VREF
+												 * update VREF values accordingly
+												 */
+												if (side_y == B)
+													y_coordinate[side_x][B][ch][bl] += VREF_STEP;
+												else
+													y_coordinate[side_x][T][ch][bl] -= VREF_STEP;
+
+												/* check that we haven't closed the VREF_EYE too much */
+												if ((y_coordinate[side_x][B][ch][bl] > (VREF_MAX - MIN_VREF_EYE)) ||
+													(y_coordinate[side_x][T][ch][bl] < (VREF_MIN + MIN_VREF_EYE)) ||
+													(y_coordinate[side_x][B][ch][bl] == y_coordinate[side_x][T][ch][bl])) {
+													/* VREF_EYE collapsed below MIN_VREF_EYE */
+													training_message(ch, rk, bl);
+													mrc_post_code(0xEE, (0x70 + (side_y * 2) + (side_x)));
+												} else {
+													/* update the VREF setting */
+													set_vref(ch, bl, y_coordinate[side_x][side_y][ch][bl]);
+													/* reset the X coordinate to begin the search at the new VREF */
+													x_coordinate[side_x][side_y][ch][rk][bl] =
+														(side_x == L) ? (RDQS_MIN) : (RDQS_MAX);
+												}
+											}
+
+											/* update the RDQS setting */
+											set_rdqs(ch, rk, bl, x_coordinate[side_x][side_y][ch][rk][bl]);
+										}
+									}
+								}
+							} while (result & 0xFF);
+						}
+					}
+				}
+			}
+		}
+	}
+
+	mrc_post_code(0x07, 0x20);
+
+	/* find final RDQS (X coordinate) & final VREF (Y coordinate) */
+	for (ch = 0; ch < NUM_CHANNELS; ch++) {
+		if (mrc_params->channel_enables & (1 << ch)) {
+			for (rk = 0; rk < NUM_RANKS; rk++) {
+				if (mrc_params->rank_enables & (1 << rk)) {
+					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
+						uint32_t temp1;
+						uint32_t temp2;
+
+						/* x_coordinate */
+						DPF(D_INFO,
+						    "RDQS T/B eye rank%d lane%d : %d-%d %d-%d\n",
+						    rk, bl,
+						    x_coordinate[L][T][ch][rk][bl],
+						    x_coordinate[R][T][ch][rk][bl],
+						    x_coordinate[L][B][ch][rk][bl],
+						    x_coordinate[R][B][ch][rk][bl]);
+
+						/* average the TOP side LEFT & RIGHT values */
+						temp1 = (x_coordinate[R][T][ch][rk][bl] + x_coordinate[L][T][ch][rk][bl]) / 2;
+						/* average the BOTTOM side LEFT & RIGHT values */
+						temp2 = (x_coordinate[R][B][ch][rk][bl] + x_coordinate[L][B][ch][rk][bl]) / 2;
+						/* average the above averages */
+						x_center[ch][rk][bl] = (uint8_t) ((temp1 + temp2) / 2);
+
+						/* y_coordinate */
+						DPF(D_INFO,
+						    "VREF R/L eye lane%d : %d-%d %d-%d\n",
+						    bl,
+						    y_coordinate[R][B][ch][bl],
+						    y_coordinate[R][T][ch][bl],
+						    y_coordinate[L][B][ch][bl],
+						    y_coordinate[L][T][ch][bl]);
+
+						/* average the RIGHT side TOP & BOTTOM values */
+						temp1 = (y_coordinate[R][T][ch][bl] + y_coordinate[R][B][ch][bl]) / 2;
+						/* average the LEFT side TOP & BOTTOM values */
+						temp2 = (y_coordinate[L][T][ch][bl] + y_coordinate[L][B][ch][bl]) / 2;
+						/* average the above averages */
+						y_center[ch][bl] = (uint8_t) ((temp1 + temp2) / 2);
+					}
+				}
+			}
+		}
+	}
+
+#ifdef RX_EYE_CHECK
+	/* perform an eye check */
+	for (side_y = B; side_y <= T; side_y++) {
+		for (side_x = L; side_x <= R; side_x++) {
+			mrc_post_code(0x07, (0x30 + (side_y * 2) + (side_x)));
+
+			/* update the settings for the eye check */
+			for (ch = 0; ch < NUM_CHANNELS; ch++) {
+				if (mrc_params->channel_enables & (1 << ch)) {
+					for (rk = 0; rk < NUM_RANKS; rk++) {
+						if (mrc_params->rank_enables & (1 << rk)) {
+							for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
+								if (side_x == L)
+									set_rdqs(ch, rk, bl, (x_center[ch][rk][bl] - (MIN_RDQS_EYE / 2)));
+								else
+									set_rdqs(ch, rk, bl, (x_center[ch][rk][bl] + (MIN_RDQS_EYE / 2)));
+
+								if (side_y == B)
+									set_vref(ch, bl, (y_center[ch][bl] - (MIN_VREF_EYE / 2)));
+								else
+									set_vref(ch, bl, (y_center[ch][bl] + (MIN_VREF_EYE / 2)));
+							}
+						}
+					}
+				}
+			}
+
+			/* request HTE reconfiguration */
+			mrc_params->hte_setup = 1;
+
+			/* check the eye */
+			if (check_bls_ex(mrc_params, address) & 0xFF) {
+				/* one or more byte lanes failed */
+				mrc_post_code(0xEE, (0x74 + (side_x * 2) + (side_y)));
+			}
+		}
+	}
+#endif
+
+	mrc_post_code(0x07, 0x40);
+
+	/* set final placements */
+	for (ch = 0; ch < NUM_CHANNELS; ch++) {
+		if (mrc_params->channel_enables & (1 << ch)) {
+			for (rk = 0; rk < NUM_RANKS; rk++) {
+				if (mrc_params->rank_enables & (1 << rk)) {
+#ifdef R2R_SHARING
+					/* increment "num_ranks_enabled" */
+					num_ranks_enabled++;
+#endif
+					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
+						/* x_coordinate */
+#ifdef R2R_SHARING
+						final_delay[ch][bl] += x_center[ch][rk][bl];
+						set_rdqs(ch, rk, bl, ((final_delay[ch][bl]) / num_ranks_enabled));
+#else
+						set_rdqs(ch, rk, bl, x_center[ch][rk][bl]);
+#endif
+						/* y_coordinate */
+						set_vref(ch, bl, y_center[ch][bl]);
+					}
+				}
+			}
+		}
+	}
+#endif
+
+	LEAVEFN();
+}
+
+/*
+ * This function will perform the WRITE TRAINING Algorithm on all
+ * channels/ranks/byte_lanes simultaneously to minimize execution time.
+ *
+ * The idea here is to train the WDQ timings to achieve maximum WRITE margins.
+ * The algorithm will start with WDQ at the current WDQ setting (tracks WDQS
+ * in WR_LVL) +/- 32 PIs (+/- 1/4 CLK) and collapse the eye until all data
+ * patterns pass. This is because WDQS will be aligned to WCLK by the
+ * Write Leveling algorithm and WDQ will only ever have a 1/2 CLK window
+ * of validity.
+ */
+void wr_train(struct mrc_params *mrc_params)
+{
+	uint8_t ch;	/* channel counter */
+	uint8_t rk;	/* rank counter */
+	uint8_t bl;	/* byte lane counter */
+	uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1;
+#ifdef BACKUP_WDQ
+#else
+	uint8_t side;		/* LEFT/RIGHT side indicator (0=L, 1=R) */
+	uint32_t temp;		/* temporary DWORD */
+	/* 2 arrays, for L & R side passing delays */
+	uint32_t delay[2][NUM_CHANNELS][NUM_RANKS][NUM_BYTE_LANES];
+	uint32_t address;	/* target address for check_bls_ex() */
+	uint32_t result;	/* result of check_bls_ex() */
+	uint32_t bl_mask;	/* byte lane mask for result checking */
+#ifdef R2R_SHARING
+	/* used to find placement for rank2rank sharing configs */
+	uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES];
+	/* used to find placement for rank2rank sharing configs */
+	uint32_t num_ranks_enabled = 0;
+#endif
+#endif
+
+	/* wr_train starts */
+	mrc_post_code(0x08, 0x00);
+
+	ENTERFN();
+
+#ifdef BACKUP_WDQ
+	for (ch = 0; ch < NUM_CHANNELS; ch++) {
+		if (mrc_params->channel_enables & (1 << ch)) {
+			for (rk = 0; rk < NUM_RANKS; rk++) {
+				if (mrc_params->rank_enables & (1 << rk)) {
+					for (bl = 0;
+					     bl < (NUM_BYTE_LANES / bl_divisor);
+					     bl++) {
+						set_wdq(ch, rk, bl, ddr_wdq[PLATFORM_ID]);
+					}
+				}
+			}
+		}
+	}
+#else
+	/* initialize "delay" */
+	for (ch = 0; ch < NUM_CHANNELS; ch++) {
+		if (mrc_params->channel_enables & (1 << ch)) {
+			for (rk = 0; rk < NUM_RANKS; rk++) {
+				if (mrc_params->rank_enables & (1 << rk)) {
+					for (bl = 0;
+					     bl < (NUM_BYTE_LANES / bl_divisor);
+					     bl++) {
+						/*
+						 * want to start with
+						 * WDQ = (WDQS - QRTR_CLK)
+						 * +/- QRTR_CLK
+						 */
+						temp = get_wdqs(ch, rk, bl) - QRTR_CLK;
+						delay[L][ch][rk][bl] = temp - QRTR_CLK;
+						delay[R][ch][rk][bl] = temp + QRTR_CLK;
+					}
+				}
+			}
+		}
+	}
+
+	/* initialize other variables */
+	bl_mask = byte_lane_mask(mrc_params);
+	address = get_addr(0, 0);
+
+#ifdef R2R_SHARING
+	/* need to set "final_delay[][]" elements to "0" */
+	memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay));
+#endif
+
+	/*
+	 * start algorithm on the LEFT side and train each channel/bl
+	 * until no failures are observed, then repeat for the RIGHT side.
+	 */
+	for (side = L; side <= R; side++) {
+		mrc_post_code(0x08, (0x10 + (side)));
+
+		/* set starting values */
+		for (ch = 0; ch < NUM_CHANNELS; ch++) {
+			if (mrc_params->channel_enables & (1 << ch)) {
+				for (rk = 0; rk < NUM_RANKS; rk++) {
+					if (mrc_params->rank_enables &
+						(1 << rk)) {
+						for (bl = 0;
+						     bl < (NUM_BYTE_LANES / bl_divisor);
+						     bl++) {
+							set_wdq(ch, rk, bl, delay[side][ch][rk][bl]);
+						}
+					}
+				}
+			}
+		}
+
+		/* find passing values */
+		for (ch = 0; ch < NUM_CHANNELS; ch++) {
+			if (mrc_params->channel_enables & (1 << ch)) {
+				for (rk = 0; rk < NUM_RANKS; rk++) {
+					if (mrc_params->rank_enables &
+						(1 << rk)) {
+						/* get an address in the target channel/rank */
+						address = get_addr(ch, rk);
+
+						/* request HTE reconfiguration */
+						mrc_params->hte_setup = 1;
+
+						/* check the settings */
+						do {
+							/* result[07:00] == failing byte lane (MAX 8) */
+							result = check_bls_ex(mrc_params, address);
+							/* check for failures */
+							if (result & 0xFF) {
+								/* at least 1 byte lane failed */
+								for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
+									if (result &
+										(bl_mask << bl)) {
+										if (side == L)
+											delay[L][ch][rk][bl] += WDQ_STEP;
+										else
+											delay[R][ch][rk][bl] -= WDQ_STEP;
+
+										/* check for algorithm failure */
+										if (delay[L][ch][rk][bl] != delay[R][ch][rk][bl]) {
+											/*
+											 * margin available
+											 * update delay setting
+											 */
+											set_wdq(ch, rk, bl,
+												delay[side][ch][rk][bl]);
+										} else {
+											/*
+											 * no margin available
+											 * notify the user and halt
+											 */
+											training_message(ch, rk, bl);
+											mrc_post_code(0xEE, (0x80 + side));
+										}
+									}
+								}
+							}
+						/* stop when all byte lanes pass */
+						} while (result & 0xFF);
+					}
+				}
+			}
+		}
+	}
+
+	/* program WDQ to the middle of passing window */
+	for (ch = 0; ch < NUM_CHANNELS; ch++) {
+		if (mrc_params->channel_enables & (1 << ch)) {
+			for (rk = 0; rk < NUM_RANKS; rk++) {
+				if (mrc_params->rank_enables & (1 << rk)) {
+#ifdef R2R_SHARING
+					/* increment "num_ranks_enabled" */
+					num_ranks_enabled++;
+#endif
+					for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
+						DPF(D_INFO,
+						    "WDQ eye rank%d lane%d : %d-%d\n",
+						    rk, bl,
+						    delay[L][ch][rk][bl],
+						    delay[R][ch][rk][bl]);
+
+						temp = (delay[R][ch][rk][bl] + delay[L][ch][rk][bl]) / 2;
+
+#ifdef R2R_SHARING
+						final_delay[ch][bl] += temp;
+						set_wdq(ch, rk, bl,
+							((final_delay[ch][bl]) / num_ranks_enabled));
+#else
+						set_wdq(ch, rk, bl, temp);
+#endif
+					}
+				}
+			}
+		}
+	}
+#endif
+
+	LEAVEFN();
+}
+
+/*
+ * This function will store relevant timing data
+ *
+ * This data will be used on subsequent boots to speed up boot times
+ * and is required for Suspend To RAM capabilities.
+ */
+void store_timings(struct mrc_params *mrc_params)
+{
+	uint8_t ch, rk, bl;
+	struct mrc_timings *mt = &mrc_params->timings;
+
+	for (ch = 0; ch < NUM_CHANNELS; ch++) {
+		for (rk = 0; rk < NUM_RANKS; rk++) {
+			for (bl = 0; bl < NUM_BYTE_LANES; bl++) {
+				mt->rcvn[ch][rk][bl] = get_rcvn(ch, rk, bl);
+				mt->rdqs[ch][rk][bl] = get_rdqs(ch, rk, bl);
+				mt->wdqs[ch][rk][bl] = get_wdqs(ch, rk, bl);
+				mt->wdq[ch][rk][bl] = get_wdq(ch, rk, bl);
+
+				if (rk == 0)
+					mt->vref[ch][bl] = get_vref(ch, bl);
+			}
+
+			mt->wctl[ch][rk] = get_wctl(ch, rk);
+		}
+
+		mt->wcmd[ch] = get_wcmd(ch);
+	}
+
+	/* need to save for a case of changing frequency after warm reset */
+	mt->ddr_speed = mrc_params->ddr_speed;
+}
+
+/*
+ * The purpose of this function is to ensure the SEC comes out of reset
+ * and IA initiates the SEC enabling Memory Scrambling.
+ */
+void enable_scrambling(struct mrc_params *mrc_params)
+{
+	uint32_t lfsr = 0;
+	uint8_t i;
+
+	if (mrc_params->scrambling_enables == 0)
+		return;
+
+	ENTERFN();
+
+	/* 32 bit seed is always stored in BIOS NVM */
+	lfsr = mrc_params->timings.scrambler_seed;
+
+	if (mrc_params->boot_mode == BM_COLD) {
+		/*
+		 * factory value is 0 and in first boot,
+		 * a clock based seed is loaded.
+		 */
+		if (lfsr == 0) {
+			/*
+			 * get seed from system clock
+			 * and make sure it is not all 1's
+			 */
+			lfsr = rdtsc() & 0x0FFFFFFF;
+		} else {
+			/*
+			 * Need to replace scrambler
+			 *
+			 * get next 32bit LFSR 16 times which is the last
+			 * part of the previous scrambler vector
+			 */
+			for (i = 0; i < 16; i++)
+				lfsr32(&lfsr);
+		}
+
+		/* save new seed */
+		mrc_params->timings.scrambler_seed = lfsr;
+	}
+
+	/*
+	 * In warm boot or S3 exit, we have the previous seed.
+	 * In cold boot, we have the last 32bit LFSR which is the new seed.
+	 */
+	lfsr32(&lfsr);	/* shift to next value */
+	msg_port_write(MEM_CTLR, SCRMSEED, (lfsr & 0x0003FFFF));
+
+	for (i = 0; i < 2; i++)
+		msg_port_write(MEM_CTLR, SCRMLO + i, (lfsr & 0xAAAAAAAA));
+
+	LEAVEFN();
+}
+
+/*
+ * Configure MCU Power Management Control Register
+ * and Scheduler Control Register
+ */
+void prog_ddr_control(struct mrc_params *mrc_params)
+{
+	u32 dsch;
+	u32 dpmc0;
+
+	ENTERFN();
+
+	dsch = msg_port_read(MEM_CTLR, DSCH);
+	dsch &= ~(BIT8 | BIT9 | BIT12);
+	msg_port_write(MEM_CTLR, DSCH, dsch);
+
+	dpmc0 = msg_port_read(MEM_CTLR, DPMC0);
+	dpmc0 &= ~BIT25;
+	dpmc0 |= (mrc_params->power_down_disable << 25);
+	dpmc0 &= ~BIT24;
+	dpmc0 &= ~(BIT16 | BIT17 | BIT18);
+	dpmc0 |= (4 << 16);
+	dpmc0 |= BIT21;
+	msg_port_write(MEM_CTLR, DPMC0, dpmc0);
+
+	/* CMDTRIST = 2h - CMD/ADDR are tristated when no valid command */
+	mrc_write_mask(MEM_CTLR, DPMC1, 2 << 4, BIT4 | BIT5);
+
+	LEAVEFN();
+}
+
+/*
+ * After training complete configure MCU Rank Population Register
+ * specifying: ranks enabled, device width, density, address mode
+ */
+void prog_dra_drb(struct mrc_params *mrc_params)
+{
+	u32 drp;
+	u32 dco;
+	u8 density = mrc_params->params.density;
+
+	ENTERFN();
+
+	dco = msg_port_read(MEM_CTLR, DCO);
+	dco &= ~BIT31;
+	msg_port_write(MEM_CTLR, DCO, dco);
+
+	drp = 0;
+	if (mrc_params->rank_enables & 1)
+		drp |= BIT0;
+	if (mrc_params->rank_enables & 2)
+		drp |= BIT1;
+	if (mrc_params->dram_width == X16) {
+		drp |= (1 << 4);
+		drp |= (1 << 9);
+	}
+
+	/*
+	 * Density encoding in struct dram_params: 0=512Mb, 1=Gb, 2=2Gb, 3=4Gb
+	 * has to be mapped RANKDENSx encoding (0=1Gb)
+	 */
+	if (density == 0)
+		density = 4;
+
+	drp |= ((density - 1) << 6);
+	drp |= ((density - 1) << 11);
+
+	/* Address mode can be overwritten if ECC enabled */
+	drp |= (mrc_params->address_mode << 14);
+
+	msg_port_write(MEM_CTLR, DRP, drp);
+
+	dco &= ~BIT28;
+	dco |= BIT31;
+	msg_port_write(MEM_CTLR, DCO, dco);
+
+	LEAVEFN();
+}
+
+/* Send DRAM wake command */
+void perform_wake(struct mrc_params *mrc_params)
+{
+	ENTERFN();
+
+	dram_wake_command();
+
+	LEAVEFN();
+}
+
+/*
+ * Configure refresh rate and short ZQ calibration interval
+ * Activate dynamic self refresh
+ */
+void change_refresh_period(struct mrc_params *mrc_params)
+{
+	u32 drfc;
+	u32 dcal;
+	u32 dpmc0;
+
+	ENTERFN();
+
+	drfc = msg_port_read(MEM_CTLR, DRFC);
+	drfc &= ~(BIT12 | BIT13 | BIT14);
+	drfc |= (mrc_params->refresh_rate << 12);
+	drfc |= BIT21;
+	msg_port_write(MEM_CTLR, DRFC, drfc);
+
+	dcal = msg_port_read(MEM_CTLR, DCAL);
+	dcal &= ~(BIT8 | BIT9 | BIT10);
+	dcal |= (3 << 8);	/* 63ms */
+	msg_port_write(MEM_CTLR, DCAL, dcal);
+
+	dpmc0 = msg_port_read(MEM_CTLR, DPMC0);
+	dpmc0 |= (BIT23 | BIT29);
+	msg_port_write(MEM_CTLR, DPMC0, dpmc0);
+
+	LEAVEFN();
+}
+
+/*
+ * Configure DDRPHY for Auto-Refresh, Periodic Compensations,
+ * Dynamic Diff-Amp, ZQSPERIOD, Auto-Precharge, CKE Power-Down
+ */
+void set_auto_refresh(struct mrc_params *mrc_params)
+{
+	uint32_t channel;
+	uint32_t rank;
+	uint32_t bl;
+	uint32_t bl_divisor = 1;
+	uint32_t temp;
+
+	ENTERFN();
+
+	/*
+	 * Enable Auto-Refresh, Periodic Compensations, Dynamic Diff-Amp,
+	 * ZQSPERIOD, Auto-Precharge, CKE Power-Down
+	 */
+	for (channel = 0; channel < NUM_CHANNELS; channel++) {
+		if (mrc_params->channel_enables & (1 << channel)) {
+			/* Enable Periodic RCOMPS */
+			mrc_alt_write_mask(DDRPHY, CMPCTRL, BIT1, BIT1);
+
+			/* Enable Dynamic DiffAmp & Set Read ODT Value */
+			switch (mrc_params->rd_odt_value) {
+			case 0:
+				temp = 0x3F;	/* OFF */
+				break;
+			default:
+				temp = 0x00;	/* Auto */
+				break;
+			}
+
+			for (bl = 0; bl < ((NUM_BYTE_LANES / bl_divisor) / 2); bl++) {
+				/* Override: DIFFAMP, ODT */
+				mrc_alt_write_mask(DDRPHY,
+					(B0OVRCTL + (bl * DDRIODQ_BL_OFFSET) +
+					(channel * DDRIODQ_CH_OFFSET)),
+					(0x00 << 16) | (temp << 10),
+					(BIT21 | BIT20 | BIT19 | BIT18 |
+					 BIT17 | BIT16 | BIT15 | BIT14 |
+					 BIT13 | BIT12 | BIT11 | BIT10));
+
+				/* Override: DIFFAMP, ODT */
+				mrc_alt_write_mask(DDRPHY,
+					(B1OVRCTL + (bl * DDRIODQ_BL_OFFSET) +
+					(channel * DDRIODQ_CH_OFFSET)),
+					(0x00 << 16) | (temp << 10),
+					(BIT21 | BIT20 | BIT19 | BIT18 |
+					 BIT17 | BIT16 | BIT15 | BIT14 |
+					 BIT13 | BIT12 | BIT11 | BIT10));
+			}
+
+			/* Issue ZQCS command */
+			for (rank = 0; rank < NUM_RANKS; rank++) {
+				if (mrc_params->rank_enables & (1 << rank))
+					dram_init_command(DCMD_ZQCS(rank));
+			}
+		}
+	}
+
+	clear_pointers();
+
+	LEAVEFN();
+}
+
+/*
+ * Depending on configuration enables ECC support
+ *
+ * Available memory size is decreased, and updated with 0s
+ * in order to clear error status. Address mode 2 forced.
+ */
+void ecc_enable(struct mrc_params *mrc_params)
+{
+	u32 drp;
+	u32 dsch;
+	u32 ecc_ctrl;
+
+	if (mrc_params->ecc_enables == 0)
+		return;
+
+	ENTERFN();
+
+	/* Configuration required in ECC mode */
+	drp = msg_port_read(MEM_CTLR, DRP);
+	drp &= ~(BIT14 | BIT15);
+	drp |= BIT15;
+	drp |= BIT13;
+	msg_port_write(MEM_CTLR, DRP, drp);
+
+	/* Disable new request bypass */
+	dsch = msg_port_read(MEM_CTLR, DSCH);
+	dsch |= BIT12;
+	msg_port_write(MEM_CTLR, DSCH, dsch);
+
+	/* Enable ECC */
+	ecc_ctrl = (BIT0 | BIT1 | BIT17);
+	msg_port_write(MEM_CTLR, DECCCTRL, ecc_ctrl);
+
+	/* Assume 8 bank memory, one bank is gone for ECC */
+	mrc_params->mem_size -= mrc_params->mem_size / 8;
+
+	/* For S3 resume memory content has to be preserved */
+	if (mrc_params->boot_mode != BM_S3) {
+		select_hte();
+		hte_mem_init(mrc_params, MRC_MEM_INIT);
+		select_mem_mgr();
+	}
+
+	LEAVEFN();
+}
+
+/*
+ * Execute memory test
+ * if error detected it is indicated in mrc_params->status
+ */
+void memory_test(struct mrc_params *mrc_params)
+{
+	uint32_t result = 0;
+
+	ENTERFN();
+
+	select_hte();
+	result = hte_mem_init(mrc_params, MRC_MEM_TEST);
+	select_mem_mgr();
+
+	DPF(D_INFO, "Memory test result %x\n", result);
+	mrc_params->status = ((result == 0) ? MRC_SUCCESS : MRC_E_MEMTEST);
+	LEAVEFN();
+}
+
+/* Lock MCU registers at the end of initialization sequence */
+void lock_registers(struct mrc_params *mrc_params)
+{
+	u32 dco;
+
+	ENTERFN();
+
+	dco = msg_port_read(MEM_CTLR, DCO);
+	dco &= ~(BIT28 | BIT29);
+	dco |= (BIT0 | BIT8);
+	msg_port_write(MEM_CTLR, DCO, dco);
+
+	LEAVEFN();
+}

arch/x86/cpu/quark/smc.h

@@ -0,0 +1,446 @@
+/*
+ * Copyright (C) 2013, Intel Corporation
+ * Copyright (C) 2015, Bin Meng <bmeng.cn@gmail.com>
+ *
+ * Ported from Intel released Quark UEFI BIOS
+ * QuarkSocPkg/QuarkNorthCluster/MemoryInit/Pei
+ *
+ * SPDX-License-Identifier:	Intel
+ */
+
+#ifndef _SMC_H_
+#define _SMC_H_
+
+/* System Memory Controller Register Defines */
+
+/* Memory Controller Message Bus Registers Offsets */
+#define DRP			0x00
+#define DTR0			0x01
+#define DTR1			0x02
+#define DTR2			0x03
+#define DTR3			0x04
+#define DTR4			0x05
+#define DPMC0			0x06
+#define DPMC1			0x07
+#define DRFC			0x08
+#define DSCH			0x09
+#define DCAL			0x0A
+#define DRMC			0x0B
+#define PMSTS			0x0C
+#define DCO			0x0F
+#define DSTAT			0x20
+#define SSKPD0			0x4A
+#define SSKPD1			0x4B
+#define DECCCTRL		0x60
+#define DECCSTAT		0x61
+#define DECCSBECNT		0x62
+#define DECCSBECA		0x68
+#define DECCSBECS		0x69
+#define DECCDBECA		0x6A
+#define DECCDBECS		0x6B
+#define DFUSESTAT		0x70
+#define SCRMSEED		0x80
+#define SCRMLO			0x81
+#define SCRMHI			0x82
+
+/* DRAM init command */
+#define DCMD_MRS1(rnk, dat)	(0 | ((rnk) << 22) | (1 << 3) | ((dat) << 6))
+#define DCMD_REF(rnk)		(1 | ((rnk) << 22))
+#define DCMD_PRE(rnk)		(2 | ((rnk) << 22))
+#define DCMD_PREA(rnk)		(2 | ((rnk) << 22) | (BIT10 << 6))
+#define DCMD_ACT(rnk, row)	(3 | ((rnk) << 22) | ((row) << 6))
+#define DCMD_WR(rnk, col)	(4 | ((rnk) << 22) | ((col) << 6))
+#define DCMD_RD(rnk, col)	(5 | ((rnk) << 22) | ((col) << 6))
+#define DCMD_ZQCS(rnk)		(6 | ((rnk) << 22))
+#define DCMD_ZQCL(rnk)		(6 | ((rnk) << 22) | (BIT10 << 6))
+#define DCMD_NOP(rnk)		(7 | ((rnk) << 22))
+
+#define DDR3_EMRS1_DIC_40	(0)
+#define DDR3_EMRS1_DIC_34	(1)
+
+#define DDR3_EMRS1_RTTNOM_0	(0)
+#define DDR3_EMRS1_RTTNOM_60	(0x04)
+#define DDR3_EMRS1_RTTNOM_120	(0x40)
+#define DDR3_EMRS1_RTTNOM_40	(0x44)
+#define DDR3_EMRS1_RTTNOM_20	(0x200)
+#define DDR3_EMRS1_RTTNOM_30	(0x204)
+
+#define DDR3_EMRS2_RTTWR_60	(1 << 9)
+#define DDR3_EMRS2_RTTWR_120	(1 << 10)
+
+/* BEGIN DDRIO Registers */
+
+/* DDR IOs & COMPs */
+#define DDRIODQ_BL_OFFSET	0x0800
+#define DDRIODQ_CH_OFFSET	((NUM_BYTE_LANES / 2) * DDRIODQ_BL_OFFSET)
+#define DDRIOCCC_CH_OFFSET	0x0800
+#define DDRCOMP_CH_OFFSET	0x0100
+
+/* CH0-BL01-DQ */
+#define DQOBSCKEBBCTL		0x0000
+#define DQDLLTXCTL		0x0004
+#define DQDLLRXCTL		0x0008
+#define DQMDLLCTL		0x000C
+#define B0RXIOBUFCTL		0x0010
+#define B0VREFCTL		0x0014
+#define B0RXOFFSET1		0x0018
+#define B0RXOFFSET0		0x001C
+#define B1RXIOBUFCTL		0x0020
+#define B1VREFCTL		0x0024
+#define B1RXOFFSET1		0x0028
+#define B1RXOFFSET0		0x002C
+#define DQDFTCTL		0x0030
+#define DQTRAINSTS		0x0034
+#define B1DLLPICODER0		0x0038
+#define B0DLLPICODER0		0x003C
+#define B1DLLPICODER1		0x0040
+#define B0DLLPICODER1		0x0044
+#define B1DLLPICODER2		0x0048
+#define B0DLLPICODER2		0x004C
+#define B1DLLPICODER3		0x0050
+#define B0DLLPICODER3		0x0054
+#define B1RXDQSPICODE		0x0058
+#define B0RXDQSPICODE		0x005C
+#define B1RXDQPICODER32		0x0060
+#define B1RXDQPICODER10		0x0064
+#define B0RXDQPICODER32		0x0068
+#define B0RXDQPICODER10		0x006C
+#define B01PTRCTL0		0x0070
+#define B01PTRCTL1		0x0074
+#define B01DBCTL0		0x0078
+#define B01DBCTL1		0x007C
+#define B0LATCTL0		0x0080
+#define B1LATCTL0		0x0084
+#define B01LATCTL1		0x0088
+#define B0ONDURCTL		0x008C
+#define B1ONDURCTL		0x0090
+#define B0OVRCTL		0x0094
+#define B1OVRCTL		0x0098
+#define DQCTL			0x009C
+#define B0RK2RKCHGPTRCTRL	0x00A0
+#define B1RK2RKCHGPTRCTRL	0x00A4
+#define DQRK2RKCTL		0x00A8
+#define DQRK2RKPTRCTL		0x00AC
+#define B0RK2RKLAT		0x00B0
+#define B1RK2RKLAT		0x00B4
+#define DQCLKALIGNREG0		0x00B8
+#define DQCLKALIGNREG1		0x00BC
+#define DQCLKALIGNREG2		0x00C0
+#define DQCLKALIGNSTS0		0x00C4
+#define DQCLKALIGNSTS1		0x00C8
+#define DQCLKGATE		0x00CC
+#define B0COMPSLV1		0x00D0
+#define B1COMPSLV1		0x00D4
+#define B0COMPSLV2		0x00D8
+#define B1COMPSLV2		0x00DC
+#define B0COMPSLV3		0x00E0
+#define B1COMPSLV3		0x00E4
+#define DQVISALANECR0TOP	0x00E8
+#define DQVISALANECR1TOP	0x00EC
+#define DQVISACONTROLCRTOP	0x00F0
+#define DQVISALANECR0BL		0x00F4
+#define DQVISALANECR1BL		0x00F8
+#define DQVISACONTROLCRBL	0x00FC
+#define DQTIMINGCTRL		0x010C
+
+/* CH0-ECC */
+#define ECCDLLTXCTL		0x2004
+#define ECCDLLRXCTL		0x2008
+#define ECCMDLLCTL		0x200C
+#define ECCB1DLLPICODER0	0x2038
+#define ECCB1DLLPICODER1	0x2040
+#define ECCB1DLLPICODER2	0x2048
+#define ECCB1DLLPICODER3	0x2050
+#define ECCB01DBCTL0		0x2078
+#define ECCB01DBCTL1		0x207C
+#define ECCCLKALIGNREG0		0x20B8
+#define ECCCLKALIGNREG1		0x20BC
+#define ECCCLKALIGNREG2		0x20C0
+
+/* CH0-CMD */
+#define CMDOBSCKEBBCTL		0x4800
+#define CMDDLLTXCTL		0x4808
+#define CMDDLLRXCTL		0x480C
+#define CMDMDLLCTL		0x4810
+#define CMDRCOMPODT		0x4814
+#define CMDDLLPICODER0		0x4820
+#define CMDDLLPICODER1		0x4824
+#define CMDCFGREG0		0x4840
+#define CMDPTRREG		0x4844
+#define CMDCLKALIGNREG0		0x4850
+#define CMDCLKALIGNREG1		0x4854
+#define CMDCLKALIGNREG2		0x4858
+#define CMDPMCONFIG0		0x485C
+#define CMDPMDLYREG0		0x4860
+#define CMDPMDLYREG1		0x4864
+#define CMDPMDLYREG2		0x4868
+#define CMDPMDLYREG3		0x486C
+#define CMDPMDLYREG4		0x4870
+#define CMDCLKALIGNSTS0		0x4874
+#define CMDCLKALIGNSTS1		0x4878
+#define CMDPMSTS0		0x487C
+#define CMDPMSTS1		0x4880
+#define CMDCOMPSLV		0x4884
+#define CMDBONUS0		0x488C
+#define CMDBONUS1		0x4890
+#define CMDVISALANECR0		0x4894
+#define CMDVISALANECR1		0x4898
+#define CMDVISACONTROLCR	0x489C
+#define CMDCLKGATE		0x48A0
+#define CMDTIMINGCTRL		0x48A4
+
+/* CH0-CLK-CTL */
+#define CCOBSCKEBBCTL		0x5800
+#define CCRCOMPIO		0x5804
+#define CCDLLTXCTL		0x5808
+#define CCDLLRXCTL		0x580C
+#define CCMDLLCTL		0x5810
+#define CCRCOMPODT		0x5814
+#define CCDLLPICODER0		0x5820
+#define CCDLLPICODER1		0x5824
+#define CCDDR3RESETCTL		0x5830
+#define CCCFGREG0		0x5838
+#define CCCFGREG1		0x5840
+#define CCPTRREG		0x5844
+#define CCCLKALIGNREG0		0x5850
+#define CCCLKALIGNREG1		0x5854
+#define CCCLKALIGNREG2		0x5858
+#define CCPMCONFIG0		0x585C
+#define CCPMDLYREG0		0x5860
+#define CCPMDLYREG1		0x5864
+#define CCPMDLYREG2		0x5868
+#define CCPMDLYREG3		0x586C
+#define CCPMDLYREG4		0x5870
+#define CCCLKALIGNSTS0		0x5874
+#define CCCLKALIGNSTS1		0x5878
+#define CCPMSTS0		0x587C
+#define CCPMSTS1		0x5880
+#define CCCOMPSLV1		0x5884
+#define CCCOMPSLV2		0x5888
+#define CCCOMPSLV3		0x588C
+#define CCBONUS0		0x5894
+#define CCBONUS1		0x5898
+#define CCVISALANECR0		0x589C
+#define CCVISALANECR1		0x58A0
+#define CCVISACONTROLCR		0x58A4
+#define CCCLKGATE		0x58A8
+#define CCTIMINGCTL		0x58AC
+
+/* COMP */
+#define CMPCTRL			0x6800
+#define SOFTRSTCNTL		0x6804
+#define MSCNTR			0x6808
+#define NMSCNTRL		0x680C
+#define LATCH1CTL		0x6814
+#define COMPVISALANECR0		0x681C
+#define COMPVISALANECR1		0x6820
+#define COMPVISACONTROLCR	0x6824
+#define COMPBONUS0		0x6830
+#define TCOCNTCTRL		0x683C
+#define DQANAODTPUCTL		0x6840
+#define DQANAODTPDCTL		0x6844
+#define DQANADRVPUCTL		0x6848
+#define DQANADRVPDCTL		0x684C
+#define DQANADLYPUCTL		0x6850
+#define DQANADLYPDCTL		0x6854
+#define DQANATCOPUCTL		0x6858
+#define DQANATCOPDCTL		0x685C
+#define CMDANADRVPUCTL		0x6868
+#define CMDANADRVPDCTL		0x686C
+#define CMDANADLYPUCTL		0x6870
+#define CMDANADLYPDCTL		0x6874
+#define CLKANAODTPUCTL		0x6880
+#define CLKANAODTPDCTL		0x6884
+#define CLKANADRVPUCTL		0x6888
+#define CLKANADRVPDCTL		0x688C
+#define CLKANADLYPUCTL		0x6890
+#define CLKANADLYPDCTL		0x6894
+#define CLKANATCOPUCTL		0x6898
+#define CLKANATCOPDCTL		0x689C
+#define DQSANAODTPUCTL		0x68A0
+#define DQSANAODTPDCTL		0x68A4
+#define DQSANADRVPUCTL		0x68A8
+#define DQSANADRVPDCTL		0x68AC
+#define DQSANADLYPUCTL		0x68B0
+#define DQSANADLYPDCTL		0x68B4
+#define DQSANATCOPUCTL		0x68B8
+#define DQSANATCOPDCTL		0x68BC
+#define CTLANADRVPUCTL		0x68C8
+#define CTLANADRVPDCTL		0x68CC
+#define CTLANADLYPUCTL		0x68D0
+#define CTLANADLYPDCTL		0x68D4
+#define CHNLBUFSTATIC		0x68F0
+#define COMPOBSCNTRL		0x68F4
+#define COMPBUFFDBG0		0x68F8
+#define COMPBUFFDBG1		0x68FC
+#define CFGMISCCH0		0x6900
+#define COMPEN0CH0		0x6904
+#define COMPEN1CH0		0x6908
+#define COMPEN2CH0		0x690C
+#define STATLEGEN0CH0		0x6910
+#define STATLEGEN1CH0		0x6914
+#define DQVREFCH0		0x6918
+#define CMDVREFCH0		0x691C
+#define CLKVREFCH0		0x6920
+#define DQSVREFCH0		0x6924
+#define CTLVREFCH0		0x6928
+#define TCOVREFCH0		0x692C
+#define DLYSELCH0		0x6930
+#define TCODRAMBUFODTCH0	0x6934
+#define CCBUFODTCH0		0x6938
+#define RXOFFSETCH0		0x693C
+#define DQODTPUCTLCH0		0x6940
+#define DQODTPDCTLCH0		0x6944
+#define DQDRVPUCTLCH0		0x6948
+#define DQDRVPDCTLCH0		0x694C
+#define DQDLYPUCTLCH0		0x6950
+#define DQDLYPDCTLCH0		0x6954
+#define DQTCOPUCTLCH0		0x6958
+#define DQTCOPDCTLCH0		0x695C
+#define CMDDRVPUCTLCH0		0x6968
+#define CMDDRVPDCTLCH0		0x696C
+#define CMDDLYPUCTLCH0		0x6970
+#define CMDDLYPDCTLCH0		0x6974
+#define CLKODTPUCTLCH0		0x6980
+#define CLKODTPDCTLCH0		0x6984
+#define CLKDRVPUCTLCH0		0x6988
+#define CLKDRVPDCTLCH0		0x698C
+#define CLKDLYPUCTLCH0		0x6990
+#define CLKDLYPDCTLCH0		0x6994
+#define CLKTCOPUCTLCH0		0x6998
+#define CLKTCOPDCTLCH0		0x699C
+#define DQSODTPUCTLCH0		0x69A0
+#define DQSODTPDCTLCH0		0x69A4
+#define DQSDRVPUCTLCH0		0x69A8
+#define DQSDRVPDCTLCH0		0x69AC
+#define DQSDLYPUCTLCH0		0x69B0
+#define DQSDLYPDCTLCH0		0x69B4
+#define DQSTCOPUCTLCH0		0x69B8
+#define DQSTCOPDCTLCH0		0x69BC
+#define CTLDRVPUCTLCH0		0x69C8
+#define CTLDRVPDCTLCH0		0x69CC
+#define CTLDLYPUCTLCH0		0x69D0
+#define CTLDLYPDCTLCH0		0x69D4
+#define FNLUPDTCTLCH0		0x69F0
+
+/* PLL */
+#define MPLLCTRL0		0x7800
+#define MPLLCTRL1		0x7808
+#define MPLLCSR0		0x7810
+#define MPLLCSR1		0x7814
+#define MPLLCSR2		0x7820
+#define MPLLDFT			0x7828
+#define MPLLMON0CTL		0x7830
+#define MPLLMON1CTL		0x7838
+#define MPLLMON2CTL		0x783C
+#define SFRTRIM			0x7850
+#define MPLLDFTOUT0		0x7858
+#define MPLLDFTOUT1		0x785C
+#define MASTERRSTN		0x7880
+#define PLLLOCKDEL		0x7884
+#define SFRDEL			0x7888
+#define CRUVISALANECR0		0x78F0
+#define CRUVISALANECR1		0x78F4
+#define CRUVISACONTROLCR	0x78F8
+#define IOSFVISALANECR0		0x78FC
+#define IOSFVISALANECR1		0x7900
+#define IOSFVISACONTROLCR	0x7904
+
+/* END DDRIO Registers */
+
+/* DRAM Specific Message Bus OpCodes */
+#define MSG_OP_DRAM_INIT	0x68
+#define MSG_OP_DRAM_WAKE	0xCA
+
+#define SAMPLE_SIZE		6
+
+/* must be less than this number to enable early deadband */
+#define EARLY_DB		0x12
+/* must be greater than this number to enable late deadband */
+#define LATE_DB			0x34
+
+#define CHX_REGS		(11 * 4)
+#define FULL_CLK		128
+#define HALF_CLK		64
+#define QRTR_CLK		32
+
+#define MCEIL(num, den)		((uint8_t)((num + den - 1) / den))
+#define MMAX(a, b)		((a) > (b) ? (a) : (b))
+#define DEAD_LOOP()		for (;;);
+
+#define MIN_RDQS_EYE		10	/* in PI Codes */
+#define MIN_VREF_EYE		10	/* in VREF Codes */
+/* how many RDQS codes to jump while margining */
+#define RDQS_STEP		1
+/* how many VREF codes to jump while margining */
+#define VREF_STEP		1
+/* offset into "vref_codes[]" for minimum allowed VREF setting */
+#define VREF_MIN		0x00
+/* offset into "vref_codes[]" for maximum allowed VREF setting */
+#define VREF_MAX		0x3F
+#define RDQS_MIN		0x00	/* minimum RDQS delay value */
+#define RDQS_MAX		0x3F	/* maximum RDQS delay value */
+
+/* how many WDQ codes to jump while margining */
+#define WDQ_STEP		1
+
+enum {
+	B,	/* BOTTOM VREF */
+	T	/* TOP VREF */
+};
+
+enum {
+	L,	/* LEFT RDQS */
+	R	/* RIGHT RDQS */
+};
+
+/* Memory Options */
+
+/* enable STATIC timing settings for RCVN (BACKUP_MODE) */
+#undef BACKUP_RCVN
+/* enable STATIC timing settings for WDQS (BACKUP_MODE) */
+#undef BACKUP_WDQS
+/* enable STATIC timing settings for RDQS (BACKUP_MODE) */
+#undef BACKUP_RDQS
+/* enable STATIC timing settings for WDQ (BACKUP_MODE) */
+#undef BACKUP_WDQ
+/* enable *COMP overrides (BACKUP_MODE) */
+#undef BACKUP_COMPS
+/* enable the RD_TRAIN eye check */
+#undef RX_EYE_CHECK
+
+/* enable Host to Memory Clock Alignment */
+#define HMC_TEST
+/* enable multi-rank support via rank2rank sharing */
+#define R2R_SHARING
+/* disable signals not used in 16bit mode of DDRIO */
+#define FORCE_16BIT_DDRIO
+
+#define PLATFORM_ID		1
+
+void clear_self_refresh(struct mrc_params *mrc_params);
+void prog_ddr_timing_control(struct mrc_params *mrc_params);
+void prog_decode_before_jedec(struct mrc_params *mrc_params);
+void perform_ddr_reset(struct mrc_params *mrc_params);
+void ddrphy_init(struct mrc_params *mrc_params);
+void perform_jedec_init(struct mrc_params *mrc_params);
+void set_ddr_init_complete(struct mrc_params *mrc_params);
+void restore_timings(struct mrc_params *mrc_params);
+void default_timings(struct mrc_params *mrc_params);
+void rcvn_cal(struct mrc_params *mrc_params);
+void wr_level(struct mrc_params *mrc_params);
+void prog_page_ctrl(struct mrc_params *mrc_params);
+void rd_train(struct mrc_params *mrc_params);
+void wr_train(struct mrc_params *mrc_params);
+void store_timings(struct mrc_params *mrc_params);
+void enable_scrambling(struct mrc_params *mrc_params);
+void prog_ddr_control(struct mrc_params *mrc_params);
+void prog_dra_drb(struct mrc_params *mrc_params);
+void perform_wake(struct mrc_params *mrc_params);
+void change_refresh_period(struct mrc_params *mrc_params);
+void set_auto_refresh(struct mrc_params *mrc_params);
+void ecc_enable(struct mrc_params *mrc_params);
+void memory_test(struct mrc_params *mrc_params);
+void lock_registers(struct mrc_params *mrc_params);
+
+#endif /* _SMC_H_ */