drivers/ddr/marvell/axp/ddr3_read_leveling.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) Marvell International Ltd. and its affiliates
 */

#include <common.h>
#include <i2c.h>
#include <spl.h>
#include <asm/io.h>
#include <asm/arch/cpu.h>
#include <asm/arch/soc.h>

#include "ddr3_hw_training.h"

/*
 * Debug
 */
#define DEBUG_RL_C(s, d, l) \
	DEBUG_RL_S(s); DEBUG_RL_D(d, l); DEBUG_RL_S("\n")
#define DEBUG_RL_FULL_C(s, d, l) \
	DEBUG_RL_FULL_S(s); DEBUG_RL_FULL_D(d, l); DEBUG_RL_FULL_S("\n")

#ifdef MV_DEBUG_RL
#define DEBUG_RL_S(s) \
	debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%s", s)
#define DEBUG_RL_D(d, l) \
	debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%x", d)
#else
#define DEBUG_RL_S(s)
#define DEBUG_RL_D(d, l)
#endif

#ifdef MV_DEBUG_RL_FULL
#define DEBUG_RL_FULL_S(s)		puts(s)
#define DEBUG_RL_FULL_D(d, l)		printf("%x", d)
#else
#define DEBUG_RL_FULL_S(s)
#define DEBUG_RL_FULL_D(d, l)
#endif

extern u32 rl_pattern[LEN_STD_PATTERN];

#ifdef RL_MODE
static int ddr3_read_leveling_single_cs_rl_mode(u32 cs, u32 freq,
						int ratio_2to1, u32 ecc,
						MV_DRAM_INFO *dram_info);
#else
static int ddr3_read_leveling_single_cs_window_mode(u32 cs, u32 freq,
						    int ratio_2to1, u32 ecc,
						    MV_DRAM_INFO *dram_info);
#endif

/*
 * Name:     ddr3_read_leveling_hw
 * Desc:     Execute the Read leveling phase by HW
 * Args:     dram_info - main struct
 *           freq      - current sequence frequency
 * Notes:
 * Returns:  MV_OK if success, MV_FAIL if fail.
 */
int ddr3_read_leveling_hw(u32 freq, MV_DRAM_INFO *dram_info)
{
	u32 reg;

	/* Debug message - Start Read leveling procedure */
	DEBUG_RL_S("DDR3 - Read Leveling - Starting HW RL procedure\n");

	/* Start Auto Read Leveling procedure */
	reg = 1 << REG_DRAM_TRAINING_RL_OFFS;
	/* Config the retest number */
	reg |= (COUNT_HW_RL << REG_DRAM_TRAINING_RETEST_OFFS);

	/* Enable CS in the automatic process */
	reg |= (dram_info->cs_ena << REG_DRAM_TRAINING_CS_OFFS);

	reg_write(REG_DRAM_TRAINING_ADDR, reg);	/* 0x15B0 - Training Register */

	reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) |
		(1 << REG_DRAM_TRAINING_AUTO_OFFS);
	reg_write(REG_DRAM_TRAINING_SHADOW_ADDR, reg);

	/* Wait */
	do {
		reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
			(1 << REG_DRAM_TRAINING_AUTO_OFFS);
	} while (reg);		/* Wait for '0' */

	/* Check if Successful */
	if (reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
	    (1 << REG_DRAM_TRAINING_ERROR_OFFS)) {
		u32 delay, phase, pup, cs;

		dram_info->rl_max_phase = 0;
		dram_info->rl_min_phase = 10;

		/* Read results to arrays */
		for (cs = 0; cs < MAX_CS; cs++) {
			if (dram_info->cs_ena & (1 << cs)) {
				for (pup = 0;
				     pup < dram_info->num_of_total_pups;
				     pup++) {
					if (pup == dram_info->num_of_std_pups
					    && dram_info->ecc_ena)
						pup = ECC_PUP;
					reg =
					    ddr3_read_pup_reg(PUP_RL_MODE, cs,
							      pup);
					phase = (reg >> REG_PHY_PHASE_OFFS) &
						PUP_PHASE_MASK;
					delay = reg & PUP_DELAY_MASK;
					dram_info->rl_val[cs][pup][P] = phase;
					if (phase > dram_info->rl_max_phase)
						dram_info->rl_max_phase = phase;
					if (phase < dram_info->rl_min_phase)
						dram_info->rl_min_phase = phase;
					dram_info->rl_val[cs][pup][D] = delay;
					dram_info->rl_val[cs][pup][S] =
					    RL_FINAL_STATE;
					reg =
					    ddr3_read_pup_reg(PUP_RL_MODE + 0x1,
							      cs, pup);
					dram_info->rl_val[cs][pup][DQS] =
					    (reg & 0x3F);
				}
#ifdef MV_DEBUG_RL
				/* Print results */
				DEBUG_RL_C("DDR3 - Read Leveling - Results for CS - ",
					   (u32) cs, 1);

				for (pup = 0;
				     pup < (dram_info->num_of_total_pups);
				     pup++) {
					if (pup == dram_info->num_of_std_pups
					    && dram_info->ecc_ena)
						pup = ECC_PUP;
					DEBUG_RL_S("DDR3 - Read Leveling - PUP: ");
					DEBUG_RL_D((u32) pup, 1);
					DEBUG_RL_S(", Phase: ");
					DEBUG_RL_D((u32) dram_info->
						   rl_val[cs][pup][P], 1);
					DEBUG_RL_S(", Delay: ");
					DEBUG_RL_D((u32) dram_info->
						   rl_val[cs][pup][D], 2);
					DEBUG_RL_S("\n");
				}
#endif
			}
		}

		dram_info->rd_rdy_dly =
			reg_read(REG_READ_DATA_READY_DELAYS_ADDR) &
			REG_READ_DATA_SAMPLE_DELAYS_MASK;
		dram_info->rd_smpl_dly =
			reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR) &
			REG_READ_DATA_READY_DELAYS_MASK;
#ifdef MV_DEBUG_RL
		DEBUG_RL_C("DDR3 - Read Leveling - Read Sample Delay: ",
			   dram_info->rd_smpl_dly, 2);
		DEBUG_RL_C("DDR3 - Read Leveling - Read Ready Delay: ",
			   dram_info->rd_rdy_dly, 2);
		DEBUG_RL_S("DDR3 - Read Leveling - HW RL Ended Successfully\n");
#endif
		return MV_OK;

	} else {
		DEBUG_RL_S("DDR3 - Read Leveling - HW RL Error\n");
		return MV_FAIL;
	}
}

/*
 * Name:     ddr3_read_leveling_sw
 * Desc:     Execute the Read leveling phase by SW
 * Args:     dram_info - main struct
 *           freq      - current sequence frequency
 * Notes:
 * Returns:  MV_OK if success, MV_FAIL if fail.
 */
int ddr3_read_leveling_sw(u32 freq, int ratio_2to1, MV_DRAM_INFO *dram_info)
{
	u32 reg, cs, ecc, pup_num, phase, delay, pup;
	int status;

	/* Debug message - Start Read leveling procedure */
	DEBUG_RL_S("DDR3 - Read Leveling - Starting SW RL procedure\n");

	/* Enable SW Read Leveling */
	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
		(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
	reg &= ~(1 << REG_DRAM_TRAINING_2_RL_MODE_OFFS);
	/* [0]=1 - Enable SW override  */
	/* 0x15B8 - Training SW 2 Register */
	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);

#ifdef RL_MODE
	reg = (dram_info->cs_ena << REG_DRAM_TRAINING_CS_OFFS) |
		(1 << REG_DRAM_TRAINING_AUTO_OFFS);
	reg_write(REG_DRAM_TRAINING_ADDR, reg);	/* 0x15B0 - Training Register */
#endif

	/* Loop for each CS */
	for (cs = 0; cs < dram_info->num_cs; cs++) {
		DEBUG_RL_C("DDR3 - Read Leveling - CS - ", (u32) cs, 1);

		for (ecc = 0; ecc <= (dram_info->ecc_ena); ecc++) {
			/* ECC Support - Switch ECC Mux on ecc=1 */
			reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
				~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
			reg |= (dram_info->ecc_ena *
				ecc << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
			reg_write(REG_DRAM_TRAINING_2_ADDR, reg);

			if (ecc)
				DEBUG_RL_S("DDR3 - Read Leveling - ECC Mux Enabled\n");
			else
				DEBUG_RL_S("DDR3 - Read Leveling - ECC Mux Disabled\n");

			/* Set current sample delays */
			reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR);
			reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK <<
				 (REG_READ_DATA_SAMPLE_DELAYS_OFFS * cs));
			reg |= (dram_info->cl <<
				(REG_READ_DATA_SAMPLE_DELAYS_OFFS * cs));
			reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR, reg);

			/* Set current Ready delay */
			reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
			reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
				 (REG_READ_DATA_READY_DELAYS_OFFS * cs));
			if (!ratio_2to1) {
				/* 1:1 mode */
				reg |= ((dram_info->cl + 1) <<
					(REG_READ_DATA_READY_DELAYS_OFFS * cs));
			} else {
				/* 2:1 mode */
				reg |= ((dram_info->cl + 2) <<
					(REG_READ_DATA_READY_DELAYS_OFFS * cs));
			}
			reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);

			/* Read leveling Single CS[cs] */
#ifdef RL_MODE
			status =
			    ddr3_read_leveling_single_cs_rl_mode(cs, freq,
								 ratio_2to1,
								 ecc,
								 dram_info);
			if (MV_OK != status)
				return status;
#else
			status =
			    ddr3_read_leveling_single_cs_window_mode(cs, freq,
								     ratio_2to1,
								     ecc,
								     dram_info)
			    if (MV_OK != status)
				return status;
#endif
		}

		/* Print results */
		DEBUG_RL_C("DDR3 - Read Leveling - Results for CS - ", (u32) cs,
			   1);

		for (pup = 0;
		     pup < (dram_info->num_of_std_pups + dram_info->ecc_ena);
		     pup++) {
			DEBUG_RL_S("DDR3 - Read Leveling - PUP: ");
			DEBUG_RL_D((u32) pup, 1);
			DEBUG_RL_S(", Phase: ");
			DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][P], 1);
			DEBUG_RL_S(", Delay: ");
			DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][D], 2);
			DEBUG_RL_S("\n");
		}

		DEBUG_RL_C("DDR3 - Read Leveling - Read Sample Delay: ",
			   dram_info->rd_smpl_dly, 2);
		DEBUG_RL_C("DDR3 - Read Leveling - Read Ready Delay: ",
			   dram_info->rd_rdy_dly, 2);

		/* Configure PHY with average of 3 locked leveling settings */
		for (pup = 0;
		     pup < (dram_info->num_of_std_pups + dram_info->ecc_ena);
		     pup++) {
			/* ECC support - bit 8 */
			pup_num = (pup == dram_info->num_of_std_pups) ? ECC_BIT : pup;

			/* For now, set last cnt result */
			phase = dram_info->rl_val[cs][pup][P];
			delay = dram_info->rl_val[cs][pup][D];
			ddr3_write_pup_reg(PUP_RL_MODE, cs, pup_num, phase,
					   delay);
		}
	}

	/* Reset PHY read FIFO */
	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
		(1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
	/* 0x15B8 - Training SW 2 Register */
	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);

	do {
		reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) &
			(1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
	} while (reg);		/* Wait for '0' */

	/* ECC Support - Switch ECC Mux off ecc=0 */
	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
		~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);

#ifdef RL_MODE
	reg_write(REG_DRAM_TRAINING_ADDR, 0);	/* 0x15B0 - Training Register */
#endif

	/* Disable SW Read Leveling */
	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
		~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
	/* [0] = 0 - Disable SW override  */
	reg = (reg | (0x1 << REG_DRAM_TRAINING_2_RL_MODE_OFFS));
	/* [3] = 1 - Disable RL MODE */
	/* 0x15B8 - Training SW 2 Register */
	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);

	DEBUG_RL_S("DDR3 - Read Leveling - Finished RL procedure for all CS\n");
	return MV_OK;
}

#ifdef RL_MODE
/*
 * overrun() extracted from ddr3_read_leveling_single_cs_rl_mode().
 * This just got too much indented making it hard to read / edit.
 */
static void overrun(u32 cs, MV_DRAM_INFO *info, u32 pup, u32 locked_pups,
		    u32 *locked_sum, u32 ecc, int *first_octet_locked,
		    int *counter_in_progress, int final_delay, u32 delay,
		    u32 phase)
{
	/* If no OverRun */
	if (((~locked_pups >> pup) & 0x1) && (final_delay == 0)) {
		int idx;

		idx = pup + ecc * ECC_BIT;

		/* PUP passed, start examining */
		if (info->rl_val[cs][idx][S] == RL_UNLOCK_STATE) {
			/* Must be RL_UNLOCK_STATE */
			/* Match expected value ? - Update State Machine */
			if (info->rl_val[cs][idx][C] < RL_RETRY_COUNT) {
				DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have no overrun and a match on pup: ",
						(u32)pup, 1);
				info->rl_val[cs][idx][C]++;

				/* If pup got to last state - lock the delays */
				if (info->rl_val[cs][idx][C] == RL_RETRY_COUNT) {
					info->rl_val[cs][idx][C] = 0;
					info->rl_val[cs][idx][DS] = delay;
					info->rl_val[cs][idx][PS] = phase;

					/* Go to Final State */
					info->rl_val[cs][idx][S] = RL_FINAL_STATE;
					*locked_sum = *locked_sum + 1;
					DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have locked pup: ",
							(u32)pup, 1);

					/*
					 * If first lock - need to lock delays
					 */
					if (*first_octet_locked == 0) {
						DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got first lock on pup: ",
								(u32)pup, 1);
						*first_octet_locked = 1;
					}

					/*
					 * If pup is in not in final state but
					 * there was match - dont increment
					 * counter
					 */
				} else {
					*counter_in_progress = 1;
				}
			}
		}
	}
}

/*
 * Name:     ddr3_read_leveling_single_cs_rl_mode
 * Desc:     Execute Read leveling for single Chip select
 * Args:     cs        - current chip select
 *           freq      - current sequence frequency
 *           ecc       - ecc iteration indication
 *           dram_info - main struct
 * Notes:
 * Returns:  MV_OK if success, MV_FAIL if fail.
 */
static int ddr3_read_leveling_single_cs_rl_mode(u32 cs, u32 freq,
						int ratio_2to1, u32 ecc,
						MV_DRAM_INFO *dram_info)
{
	u32 reg, delay, phase, pup, rd_sample_delay, add, locked_pups,
		repeat_max_cnt, sdram_offset, locked_sum;
	u32 phase_min, ui_max_delay;
	int all_locked, first_octet_locked, counter_in_progress;
	int final_delay = 0;

	DEBUG_RL_FULL_C("DDR3 - Read Leveling - Single CS - ", (u32) cs, 1);

	/* Init values */
	phase = 0;
	delay = 0;
	rd_sample_delay = dram_info->cl;
	all_locked = 0;
	first_octet_locked = 0;
	repeat_max_cnt = 0;
	locked_sum = 0;

	for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc);
	     pup++)
		dram_info->rl_val[cs][pup + ecc * ECC_BIT][S] = 0;

	/* Main loop */
	while (!all_locked) {
		counter_in_progress = 0;

		DEBUG_RL_FULL_S("DDR3 - Read Leveling - RdSmplDly = ");
		DEBUG_RL_FULL_D(rd_sample_delay, 2);
		DEBUG_RL_FULL_S(", RdRdyDly = ");
		DEBUG_RL_FULL_D(dram_info->rd_rdy_dly, 2);
		DEBUG_RL_FULL_S(", Phase = ");
		DEBUG_RL_FULL_D(phase, 1);
		DEBUG_RL_FULL_S(", Delay = ");
		DEBUG_RL_FULL_D(delay, 2);
		DEBUG_RL_FULL_S("\n");

		/*
		 * Broadcast to all PUPs current RL delays: DQS phase,
		 * leveling delay
		 */
		ddr3_write_pup_reg(PUP_RL_MODE, cs, PUP_BC, phase, delay);

		/* Reset PHY read FIFO */
		reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
			(1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
		/* 0x15B8 - Training SW 2 Register */
		reg_write(REG_DRAM_TRAINING_2_ADDR, reg);

		do {
			reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) &
				(1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
		} while (reg);	/* Wait for '0' */

		/* Read pattern from SDRAM */
		sdram_offset = cs * (SDRAM_CS_SIZE + 1) + SDRAM_RL_OFFS;
		locked_pups = 0;
		if (MV_OK !=
		    ddr3_sdram_compare(dram_info, 0xFF, &locked_pups,
				       rl_pattern, LEN_STD_PATTERN,
				       sdram_offset, 0, 0, NULL, 0))
			return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PATTERN;

		/* Octet evaluation */
		/* pup_num = Q or 1 for ECC */
		for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) {
			/* Check Overrun */
			if (!((reg_read(REG_DRAM_TRAINING_2_ADDR) >>
			       (REG_DRAM_TRAINING_2_OVERRUN_OFFS + pup)) & 0x1)) {
				overrun(cs, dram_info, pup, locked_pups,
					&locked_sum, ecc, &first_octet_locked,
					&counter_in_progress, final_delay,
					delay, phase);
			} else {
				DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got overrun on pup: ",
						(u32)pup, 1);
			}
		}

		if (locked_sum == (dram_info->num_of_std_pups *
				   (1 - ecc) + ecc)) {
			all_locked = 1;
			DEBUG_RL_FULL_S("DDR3 - Read Leveling - Single Cs - All pups locked\n");
		}

		/*
		 * This is a fix for unstable condition where pups are
		 * toggling between match and no match
		 */
		/*
		 * If some of the pups is >1 <3, check if we did it too
		 * many times
		 */
		if (counter_in_progress == 1) {
			/* Notify at least one Counter is >=1 and < 3 */
			if (repeat_max_cnt < RL_RETRY_COUNT) {
				repeat_max_cnt++;
				counter_in_progress = 1;
				DEBUG_RL_FULL_S("DDR3 - Read Leveling - Counter is >=1 and <3\n");
				DEBUG_RL_FULL_S("DDR3 - Read Leveling - So we will not increment the delay to see if locked again\n");
			} else {
				DEBUG_RL_FULL_S("DDR3 - Read Leveling - repeat_max_cnt reached max so now we will increment the delay\n");
				counter_in_progress = 0;
			}
		}

		/*
		 * Check some of the pups are in the middle of state machine
		 * and don't increment the delays
		 */
		if (!counter_in_progress && !all_locked) {
			int idx;

			idx = pup + ecc * ECC_BIT;

			repeat_max_cnt = 0;
			/* if 1:1 mode */
			if ((!ratio_2to1) && ((phase == 0) || (phase == 4)))
				ui_max_delay = MAX_DELAY_INV;
			else
				ui_max_delay = MAX_DELAY;

			/* Increment Delay */
			if (delay < ui_max_delay) {
				delay++;
				/*
				 * Mark the last delay/pahse place for
				 * window final place
				 */
				if (delay == ui_max_delay) {
					if ((!ratio_2to1 && phase ==
					     MAX_PHASE_RL_L_1TO1)
					    || (ratio_2to1 && phase ==
						MAX_PHASE_RL_L_2TO1))
						final_delay = 1;
				}
			} else {
				/* Phase+CL Incrementation */
				delay = 0;

				if (!ratio_2to1) {
					/* 1:1 mode */
					if (first_octet_locked) {
						/* some Pup was Locked */
						if (phase < MAX_PHASE_RL_L_1TO1) {
							if (phase == 1) {
								phase = 4;
							} else {
								phase++;
								delay = MIN_DELAY_PHASE_1_LIMIT;
							}
						} else {
							DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
							DEBUG_RL_S("1)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked n");
							return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PUP_UNLOCK;
						}
					} else {
						/* NO Pup was Locked */
						if (phase < MAX_PHASE_RL_UL_1TO1) {
							phase++;
							delay =
							    MIN_DELAY_PHASE_1_LIMIT;
						} else {
							phase = 0;
						}
					}
				} else {
					/* 2:1 mode */
					if (first_octet_locked) {
						/* some Pup was Locked */
						if (phase < MAX_PHASE_RL_L_2TO1) {
							phase++;
						} else {
							DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
							DEBUG_RL_S("2)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
							for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) {
								/* pup_num = Q or 1 for ECC */
								if (dram_info->rl_val[cs][idx][S]
								    == 0) {
									DEBUG_RL_C("Failed byte is = ",
										   pup, 1);
								}
							}
							return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PUP_UNLOCK;
						}
					} else {
						/* No Pup was Locked */
						if (phase < MAX_PHASE_RL_UL_2TO1)
							phase++;
						else
							phase = 0;
					}
				}

				/*
				 * If we finished a full Phases cycle (so now
				 * phase = 0, need to increment rd_sample_dly
				 */
				if (phase == 0 && first_octet_locked == 0) {
					rd_sample_delay++;
					if (rd_sample_delay == 0x10) {
						DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
						DEBUG_RL_S("3)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
						for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) {
							/* pup_num = Q or 1 for ECC */
							if (dram_info->
							    rl_val[cs][idx][S] == 0) {
								DEBUG_RL_C("Failed byte is = ",
									   pup, 1);
							}
						}
						return MV_DDR3_TRAINING_ERR_RD_LVL_PUP_UNLOCK;
					}

					/* Set current rd_sample_delay  */
					reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR);
					reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK
					      << (REG_READ_DATA_SAMPLE_DELAYS_OFFS
						  * cs));
					reg |= (rd_sample_delay <<
						(REG_READ_DATA_SAMPLE_DELAYS_OFFS *
						 cs));
					reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR,
						  reg);
				}

				/*
				 * Set current rdReadyDelay according to the
				 * hash table (Need to do this in every phase
				 * change)
				 */
				if (!ratio_2to1) {
					/* 1:1 mode */
					add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
					switch (phase) {
					case 0:
						add = (add >>
						       REG_TRAINING_DEBUG_2_OFFS);
						break;
					case 1:
						add = (add >>
						       (REG_TRAINING_DEBUG_2_OFFS
							+ 3));
						break;
					case 4:
						add = (add >>
						       (REG_TRAINING_DEBUG_2_OFFS
							+ 6));
						break;
					case 5:
						add = (add >>
						       (REG_TRAINING_DEBUG_2_OFFS
							+ 9));
						break;
					}
					add &= REG_TRAINING_DEBUG_2_MASK;
				} else {
					/* 2:1 mode */
					add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
					add = (add >>
					       (phase *
						REG_TRAINING_DEBUG_3_OFFS));
					add &= REG_TRAINING_DEBUG_3_MASK;
				}

				reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
				reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
					 (REG_READ_DATA_READY_DELAYS_OFFS * cs));
				reg |= ((rd_sample_delay + add) <<
					(REG_READ_DATA_READY_DELAYS_OFFS * cs));
				reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
				dram_info->rd_smpl_dly = rd_sample_delay;
				dram_info->rd_rdy_dly = rd_sample_delay + add;
			}

			/* Reset counters for pups with states<RD_STATE_COUNT */
			for (pup = 0; pup <
				     (dram_info->num_of_std_pups * (1 - ecc) + ecc);
			     pup++) {
				if (dram_info->rl_val[cs][idx][C] < RL_RETRY_COUNT)
					dram_info->rl_val[cs][idx][C] = 0;
			}
		}
	}

	phase_min = 10;

	for (pup = 0; pup < (dram_info->num_of_std_pups); pup++) {
		if (dram_info->rl_val[cs][pup][PS] < phase_min)
			phase_min = dram_info->rl_val[cs][pup][PS];
	}

	/*
	 * Set current rdReadyDelay according to the hash table (Need to
	 * do this in every phase change)
	 */
	if (!ratio_2to1) {
		/* 1:1 mode */
		add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
		switch (phase_min) {
		case 0:
			add = (add >> REG_TRAINING_DEBUG_2_OFFS);
			break;
		case 1:
			add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 3));
			break;
		case 4:
			add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 6));
			break;
		case 5:
			add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 9));
			break;
		}
		add &= REG_TRAINING_DEBUG_2_MASK;
	} else {
		/* 2:1 mode */
		add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
		add = (add >> (phase_min * REG_TRAINING_DEBUG_3_OFFS));
		add &= REG_TRAINING_DEBUG_3_MASK;
	}

	reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
	reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
		 (REG_READ_DATA_READY_DELAYS_OFFS * cs));
	reg |= ((rd_sample_delay + add) << (REG_READ_DATA_READY_DELAYS_OFFS * cs));
	reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
	dram_info->rd_rdy_dly = rd_sample_delay + add;

	for (cs = 0; cs < dram_info->num_cs; cs++) {
		for (pup = 0; pup < dram_info->num_of_total_pups; pup++) {
			reg = ddr3_read_pup_reg(PUP_RL_MODE + 0x1, cs, pup);
			dram_info->rl_val[cs][pup][DQS] = (reg & 0x3F);
		}
	}

	return MV_OK;
}

#else

/*
 * Name:     ddr3_read_leveling_single_cs_window_mode
 * Desc:     Execute Read leveling for single Chip select
 * Args:     cs        - current chip select
 *           freq      - current sequence frequency
 *           ecc       - ecc iteration indication
 *           dram_info - main struct
 * Notes:
 * Returns:  MV_OK if success, MV_FAIL if fail.
 */
static int ddr3_read_leveling_single_cs_window_mode(u32 cs, u32 freq,
						    int ratio_2to1, u32 ecc,
						    MV_DRAM_INFO *dram_info)
{
	u32 reg, delay, phase, sum, pup, rd_sample_delay, add, locked_pups,
	    repeat_max_cnt, sdram_offset, final_sum, locked_sum;
	u32 delay_s, delay_e, tmp, phase_min, ui_max_delay;
	int all_locked, first_octet_locked, counter_in_progress;
	int final_delay = 0;

	DEBUG_RL_FULL_C("DDR3 - Read Leveling - Single CS - ", (u32) cs, 1);

	/* Init values */
	phase = 0;
	delay = 0;
	rd_sample_delay = dram_info->cl;
	all_locked = 0;
	first_octet_locked = 0;
	repeat_max_cnt = 0;
	sum = 0;
	final_sum = 0;
	locked_sum = 0;

	for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc);
	     pup++)
		dram_info->rl_val[cs][pup + ecc * ECC_BIT][S] = 0;

	/* Main loop */
	while (!all_locked) {
		counter_in_progress = 0;

		DEBUG_RL_FULL_S("DDR3 - Read Leveling - RdSmplDly = ");
		DEBUG_RL_FULL_D(rd_sample_delay, 2);
		DEBUG_RL_FULL_S(", RdRdyDly = ");
		DEBUG_RL_FULL_D(dram_info->rd_rdy_dly, 2);
		DEBUG_RL_FULL_S(", Phase = ");
		DEBUG_RL_FULL_D(phase, 1);
		DEBUG_RL_FULL_S(", Delay = ");
		DEBUG_RL_FULL_D(delay, 2);
		DEBUG_RL_FULL_S("\n");

		/*
		 * Broadcast to all PUPs current RL delays: DQS phase,leveling
		 * delay
		 */
		ddr3_write_pup_reg(PUP_RL_MODE, cs, PUP_BC, phase, delay);

		/* Reset PHY read FIFO */
		reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
			(1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
		/* 0x15B8 - Training SW 2 Register */
		reg_write(REG_DRAM_TRAINING_2_ADDR, reg);

		do {
			reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) &
				(1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
		} while (reg);	/* Wait for '0' */

		/* Read pattern from SDRAM */
		sdram_offset = cs * (SDRAM_CS_SIZE + 1) + SDRAM_RL_OFFS;
		locked_pups = 0;
		if (MV_OK !=
		    ddr3_sdram_compare(dram_info, 0xFF, &locked_pups,
				       rl_pattern, LEN_STD_PATTERN,
				       sdram_offset, 0, 0, NULL, 0))
			return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PATTERN;

		/* Octet evaluation */
		for (pup = 0; pup < (dram_info->num_of_std_pups *
				     (1 - ecc) + ecc); pup++) {
			/* pup_num = Q or 1 for ECC */
			int idx;

			idx = pup + ecc * ECC_BIT;

			/* Check Overrun */
			if (!((reg_read(REG_DRAM_TRAINING_2_ADDR) >>
			      (REG_DRAM_TRAINING_2_OVERRUN_OFFS +
			       pup)) & 0x1)) {
				/* If no OverRun */

				/* Inside the window */
				if (dram_info->rl_val[cs][idx][S] == RL_WINDOW_STATE) {
					/*
					 * Match expected value ? - Update
					 * State Machine
					 */
					if (((~locked_pups >> pup) & 0x1)
					    && (final_delay == 0)) {
						/* Match - Still inside the Window */
						DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got another match inside the window  for pup: ",
								(u32)pup, 1);

					} else {
						/* We got fail -> this is the end of the window */
						dram_info->rl_val[cs][idx][DE] = delay;
						dram_info->rl_val[cs][idx][PE] = phase;
						/* Go to Final State */
						dram_info->rl_val[cs][idx][S]++;
						final_sum++;
						DEBUG_RL_FULL_C("DDR3 - Read Leveling - We finished the window for pup: ",
								(u32)pup, 1);
					}

					/* Before the start of the window */
				} else if (dram_info->rl_val[cs][idx][S] ==
					   RL_UNLOCK_STATE) {
					/* Must be RL_UNLOCK_STATE */
					/*
					 * Match expected value ? - Update
					 * State Machine
					 */
					if (dram_info->rl_val[cs][idx][C] <
					    RL_RETRY_COUNT) {
						if (((~locked_pups >> pup) & 0x1)) {
							/* Match */
							DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have no overrun and a match on pup: ",
									(u32)pup, 1);
							dram_info->rl_val[cs][idx][C]++;

							/* If pup got to last state - lock the delays */
							if (dram_info->rl_val[cs][idx][C] ==
							    RL_RETRY_COUNT) {
								dram_info->rl_val[cs][idx][C] = 0;
								dram_info->rl_val[cs][idx][DS] =
									delay;
								dram_info->rl_val[cs][idx][PS] =
									phase;
								dram_info->rl_val[cs][idx][S]++;	/* Go to Window State */
								locked_sum++;
								/* Will count the pups that got locked */

								/* IF First lock - need to lock delays */
								if (first_octet_locked == 0) {
									DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got first lock on pup: ",
											(u32)pup, 1);
									first_octet_locked
									    =
									    1;
								}
							}

							/* if pup is in not in final state but there was match - dont increment counter */
							else {
								counter_in_progress
								    = 1;
							}
						}
					}
				}
			} else {
				DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got overrun on pup: ",
						(u32)pup, 1);
				counter_in_progress = 1;
			}
		}

		if (final_sum == (dram_info->num_of_std_pups * (1 - ecc) + ecc)) {
			all_locked = 1;
			DEBUG_RL_FULL_S("DDR3 - Read Leveling - Single Cs - All pups locked\n");
		}

		/*
		 * This is a fix for unstable condition where pups are
		 * toggling between match and no match
		 */
		/*
		 * If some of the pups is >1 <3, check if we did it too many
		 * times
		 */
		if (counter_in_progress == 1) {
			if (repeat_max_cnt < RL_RETRY_COUNT) {
				/* Notify at least one Counter is >=1 and < 3 */
				repeat_max_cnt++;
				counter_in_progress = 1;
				DEBUG_RL_FULL_S("DDR3 - Read Leveling - Counter is >=1 and <3\n");
				DEBUG_RL_FULL_S("DDR3 - Read Leveling - So we will not increment the delay to see if locked again\n");
			} else {
				DEBUG_RL_FULL_S("DDR3 - Read Leveling - repeat_max_cnt reached max so now we will increment the delay\n");
				counter_in_progress = 0;
			}
		}

		/*
		 * Check some of the pups are in the middle of state machine
		 * and don't increment the delays
		 */
		if (!counter_in_progress && !all_locked) {
			repeat_max_cnt = 0;
			if (!ratio_2to1)
				ui_max_delay = MAX_DELAY_INV;
			else
				ui_max_delay = MAX_DELAY;

			/* Increment Delay */
			if (delay < ui_max_delay) {
				/* Delay Incrementation */
				delay++;
				if (delay == ui_max_delay) {
					/*
					 * Mark the last delay/pahse place
					 * for window final place
					 */
					if ((!ratio_2to1
					     && phase == MAX_PHASE_RL_L_1TO1)
					    || (ratio_2to1
						&& phase ==
						MAX_PHASE_RL_L_2TO1))
						final_delay = 1;
				}
			} else {
				/* Phase+CL Incrementation */
				delay = 0;
				if (!ratio_2to1) {
					/* 1:1 mode */
					if (first_octet_locked) {
						/* some pupet was Locked */
						if (phase < MAX_PHASE_RL_L_1TO1) {
#ifdef RL_WINDOW_WA
							if (phase == 0)
#else
							if (phase == 1)
#endif
								phase = 4;
							else
								phase++;
						} else {
							DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
							return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PUP_UNLOCK;
						}
					} else {
						/* No Pup was Locked */
						if (phase < MAX_PHASE_RL_UL_1TO1) {
#ifdef RL_WINDOW_WA
							if (phase == 0)
								phase = 4;
#else
							phase++;
#endif
						} else
							phase = 0;
					}
				} else {
					/* 2:1 mode */
					if (first_octet_locked) {
						/* Some Pup was Locked */
						if (phase < MAX_PHASE_RL_L_2TO1) {
							phase++;
						} else {
							DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
							return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PUP_UNLOCK;
						}
					} else {
						/* No Pup was Locked */
						if (phase < MAX_PHASE_RL_UL_2TO1)
							phase++;
						else
							phase = 0;
					}
				}

				/*
				 * If we finished a full Phases cycle (so
				 * now phase = 0, need to increment
				 * rd_sample_dly
				 */
				if (phase == 0 && first_octet_locked == 0) {
					rd_sample_delay++;

					/* Set current rd_sample_delay  */
					reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR);
					reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK <<
						 (REG_READ_DATA_SAMPLE_DELAYS_OFFS
						  * cs));
					reg |= (rd_sample_delay <<
						(REG_READ_DATA_SAMPLE_DELAYS_OFFS *
						 cs));
					reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR,
						  reg);
				}

				/*
				 * Set current rdReadyDelay according to the
				 * hash table (Need to do this in every phase
				 * change)
				 */
				if (!ratio_2to1) {
					/* 1:1 mode */
					add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
					switch (phase) {
					case 0:
						add = add >>
							REG_TRAINING_DEBUG_2_OFFS;
						break;
					case 1:
						add = add >>
							(REG_TRAINING_DEBUG_2_OFFS
							 + 3);
						break;
					case 4:
						add = add >>
							(REG_TRAINING_DEBUG_2_OFFS
							 + 6);
						break;
					case 5:
						add = add >>
							(REG_TRAINING_DEBUG_2_OFFS
							 + 9);
						break;
					}
				} else {
					/* 2:1 mode */
					add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
					add = (add >> phase *
					       REG_TRAINING_DEBUG_3_OFFS);
				}
				add &= REG_TRAINING_DEBUG_2_MASK;
				reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
				reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
					 (REG_READ_DATA_READY_DELAYS_OFFS * cs));
				reg |= ((rd_sample_delay + add) <<
					(REG_READ_DATA_READY_DELAYS_OFFS * cs));
				reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
				dram_info->rd_smpl_dly = rd_sample_delay;
				dram_info->rd_rdy_dly = rd_sample_delay + add;
			}

			/* Reset counters for pups with states<RD_STATE_COUNT */
			for (pup = 0;
			     pup <
			     (dram_info->num_of_std_pups * (1 - ecc) + ecc);
			     pup++) {
				if (dram_info->rl_val[cs][idx][C] < RL_RETRY_COUNT)
					dram_info->rl_val[cs][idx][C] = 0;
			}
		}
	}

	phase_min = 10;

	for (pup = 0; pup < (dram_info->num_of_std_pups); pup++) {
		DEBUG_RL_S("DDR3 - Read Leveling - Window info - PUP: ");
		DEBUG_RL_D((u32) pup, 1);
		DEBUG_RL_S(", PS: ");
		DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][PS], 1);
		DEBUG_RL_S(", DS: ");
		DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][DS], 2);
		DEBUG_RL_S(", PE: ");
		DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][PE], 1);
		DEBUG_RL_S(", DE: ");
		DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][DE], 2);
		DEBUG_RL_S("\n");
	}

	/* Find center of the window procedure */
	for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc);
	     pup++) {
#ifdef RL_WINDOW_WA
		if (!ratio_2to1) {	/* 1:1 mode */
			if (dram_info->rl_val[cs][idx][PS] == 4)
				dram_info->rl_val[cs][idx][PS] = 1;
			if (dram_info->rl_val[cs][idx][PE] == 4)
				dram_info->rl_val[cs][idx][PE] = 1;

			delay_s = dram_info->rl_val[cs][idx][PS] *
				MAX_DELAY_INV + dram_info->rl_val[cs][idx][DS];
			delay_e = dram_info->rl_val[cs][idx][PE] *
				MAX_DELAY_INV + dram_info->rl_val[cs][idx][DE];

			tmp = (delay_e - delay_s) / 2 + delay_s;
			phase = tmp / MAX_DELAY_INV;
			if (phase == 1)	/* 1:1 mode */
				phase = 4;

			if (phase < phase_min)	/* for the read ready delay */
				phase_min = phase;

			dram_info->rl_val[cs][idx][P] = phase;
			dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY_INV;

		} else {
			delay_s = dram_info->rl_val[cs][idx][PS] *
				MAX_DELAY + dram_info->rl_val[cs][idx][DS];
			delay_e = dram_info->rl_val[cs][idx][PE] *
				MAX_DELAY + dram_info->rl_val[cs][idx][DE];

			tmp = (delay_e - delay_s) / 2 + delay_s;
			phase = tmp / MAX_DELAY;

			if (phase < phase_min)	/* for the read ready delay */
				phase_min = phase;

			dram_info->rl_val[cs][idx][P] = phase;
			dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY;
		}
#else
		if (!ratio_2to1) {	/* 1:1 mode */
			if (dram_info->rl_val[cs][idx][PS] > 1)
				dram_info->rl_val[cs][idx][PS] -= 2;
			if (dram_info->rl_val[cs][idx][PE] > 1)
				dram_info->rl_val[cs][idx][PE] -= 2;
		}

		delay_s = dram_info->rl_val[cs][idx][PS] * MAX_DELAY +
			dram_info->rl_val[cs][idx][DS];
		delay_e = dram_info->rl_val[cs][idx][PE] * MAX_DELAY +
			dram_info->rl_val[cs][idx][DE];

		tmp = (delay_e - delay_s) / 2 + delay_s;
		phase = tmp / MAX_DELAY;
		if (!ratio_2to1 && phase > 1)	/* 1:1 mode */
			phase += 2;

		if (phase < phase_min)	/* for the read ready delay */
			phase_min = phase;

		dram_info->rl_val[cs][idx][P] = phase;
		dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY;
#endif
	}

	/* Set current rdReadyDelay according to the hash table (Need to do this in every phase change) */
	if (!ratio_2to1) {	/* 1:1 mode */
		add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
		switch (phase_min) {
		case 0:
			add = (add >> REG_TRAINING_DEBUG_2_OFFS);
			break;
		case 1:
			add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 3));
			break;
		case 4:
			add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 6));
			break;
		case 5:
			add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 9));
			break;
		}
	} else {		/* 2:1 mode */
		add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
		add = (add >> phase_min * REG_TRAINING_DEBUG_3_OFFS);
	}

	add &= REG_TRAINING_DEBUG_2_MASK;
	reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
	reg &=
	    ~(REG_READ_DATA_READY_DELAYS_MASK <<
	      (REG_READ_DATA_READY_DELAYS_OFFS * cs));
	reg |=
	    ((rd_sample_delay + add) << (REG_READ_DATA_READY_DELAYS_OFFS * cs));
	reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
	dram_info->rd_rdy_dly = rd_sample_delay + add;

	for (cs = 0; cs < dram_info->num_cs; cs++) {
		for (pup = 0; pup < dram_info->num_of_total_pups; pup++) {
			reg = ddr3_read_pup_reg(PUP_RL_MODE + 0x1, cs, pup);
			dram_info->rl_val[cs][pup][DQS] = (reg & 0x3F);
		}
	}

	return MV_OK;
}
#endif