drivers/ddr/marvell/a38x/ddr3_training_pbs.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) Marvell International Ltd. and its affiliates
 */

#include "ddr3_init.h"
#include "mv_ddr_training_db.h"
#include "mv_ddr_common.h"
#include "mv_ddr_regs.h"

#define TYPICAL_PBS_VALUE	12

u32 nominal_adll[MAX_INTERFACE_NUM * MAX_BUS_NUM];
enum hws_training_ip_stat train_status[MAX_INTERFACE_NUM];
u8 result_mat[MAX_INTERFACE_NUM][MAX_BUS_NUM][BUS_WIDTH_IN_BITS];
u8 result_mat_rx_dqs[MAX_INTERFACE_NUM][MAX_BUS_NUM][MAX_CS_NUM];
/* 4-EEWA, 3-EWA, 2-SWA, 1-Fail, 0-Pass */
u8 result_all_bit[MAX_BUS_NUM * BUS_WIDTH_IN_BITS * MAX_INTERFACE_NUM];
u8 max_pbs_per_pup[MAX_INTERFACE_NUM][MAX_BUS_NUM];
u8 min_pbs_per_pup[MAX_INTERFACE_NUM][MAX_BUS_NUM];
u8 max_adll_per_pup[MAX_INTERFACE_NUM][MAX_BUS_NUM];
u8 min_adll_per_pup[MAX_INTERFACE_NUM][MAX_BUS_NUM];
u32 pbsdelay_per_pup[NUM_OF_PBS_MODES][MAX_INTERFACE_NUM][MAX_BUS_NUM][MAX_CS_NUM];
u8 adll_shift_lock[MAX_INTERFACE_NUM][MAX_BUS_NUM];
u8 adll_shift_val[MAX_INTERFACE_NUM][MAX_BUS_NUM];
enum hws_pattern pbs_pattern = PATTERN_VREF;
static u8 pup_state[MAX_INTERFACE_NUM][MAX_BUS_NUM];

/*
 * Name:     ddr3_tip_pbs
 * Desc:     PBS
 * Args:     TBD
 * Notes:
 * Returns:  OK if success, other error code if fail.
 */
int ddr3_tip_pbs(u32 dev_num, enum pbs_dir pbs_mode)
{
	u32 res0[MAX_INTERFACE_NUM];
	int adll_tap = MEGA / mv_ddr_freq_get(medium_freq) / 64;
	int pad_num = 0;
	enum hws_search_dir search_dir =
		(pbs_mode == PBS_RX_MODE) ? HWS_HIGH2LOW : HWS_LOW2HIGH;
	enum hws_dir dir = (pbs_mode == PBS_RX_MODE) ? OPER_READ : OPER_WRITE;
	int iterations = (pbs_mode == PBS_RX_MODE) ? 31 : 63;
	u32 res_valid_mask = (pbs_mode == PBS_RX_MODE) ? 0x1f : 0x3f;
	int init_val = (search_dir == HWS_LOW2HIGH) ? 0 : iterations;
	enum hws_edge_compare search_edge = EDGE_FP;
	u32 pup = 0, bit = 0, if_id = 0, all_lock = 0, cs_num = 0;
	u32 reg_addr = 0;
	u32 validation_val = 0;
	u32 cs_enable_reg_val[MAX_INTERFACE_NUM];
	u16 *mask_results_dq_reg_map = ddr3_tip_get_mask_results_dq_reg();
	u8 temp = 0;
	u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
	struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();

	/* save current cs enable reg val */
	for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
		VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);

		/* save current cs enable reg val */
		CHECK_STATUS(ddr3_tip_if_read
			     (dev_num, ACCESS_TYPE_UNICAST, if_id,
			      DUAL_DUNIT_CFG_REG, cs_enable_reg_val, MASK_ALL_BITS));

		/* enable single cs */
		CHECK_STATUS(ddr3_tip_if_write
			     (dev_num, ACCESS_TYPE_UNICAST, if_id,
			      DUAL_DUNIT_CFG_REG, (1 << 3), (1 << 3)));
	}

	reg_addr = (pbs_mode == PBS_RX_MODE) ?
		CRX_PHY_REG(effective_cs) :
		CTX_PHY_REG(effective_cs);
	ddr3_tip_read_adll_value(dev_num, nominal_adll, reg_addr, MASK_ALL_BITS);

	/* stage 1 shift ADLL */
	ddr3_tip_ip_training(dev_num, ACCESS_TYPE_MULTICAST,
			     PARAM_NOT_CARE, ACCESS_TYPE_MULTICAST,
			     PARAM_NOT_CARE, RESULT_PER_BIT,
			     HWS_CONTROL_ELEMENT_ADLL, search_dir, dir,
			     tm->if_act_mask, init_val, iterations,
			     pbs_pattern, search_edge, CS_SINGLE, cs_num,
			     train_status);
	validation_val = (pbs_mode == PBS_RX_MODE) ? 0x1f : 0;
	for (pup = 0; pup < octets_per_if_num; pup++) {
		VALIDATE_BUS_ACTIVE(tm->bus_act_mask, pup);
		for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
			VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
			min_adll_per_pup[if_id][pup] =
				(pbs_mode == PBS_RX_MODE) ? 0x1f : 0x3f;
			pup_state[if_id][pup] = 0x3;
			adll_shift_lock[if_id][pup] = 1;
			max_adll_per_pup[if_id][pup] = 0x0;
		}
	}

	/* EBA */
	for (pup = 0; pup < octets_per_if_num; pup++) {
		VALIDATE_BUS_ACTIVE(tm->bus_act_mask, pup);
		for (bit = 0; bit < BUS_WIDTH_IN_BITS; bit++) {
			CHECK_STATUS(ddr3_tip_if_read
				     (dev_num, ACCESS_TYPE_MULTICAST,
				      PARAM_NOT_CARE,
				      mask_results_dq_reg_map[
					      bit + pup * BUS_WIDTH_IN_BITS],
				      res0, MASK_ALL_BITS));
			for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1;
			     if_id++) {
				VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
				DEBUG_PBS_ENGINE(DEBUG_LEVEL_TRACE,
						 ("FP I/F %d, bit:%d, pup:%d res0 0x%x\n",
						  if_id, bit, pup,
						  res0[if_id]));
				if (pup_state[if_id][pup] != 3)
					continue;
				/* if not EBA state than move to next pup */

				if ((res0[if_id] & 0x2000000) == 0) {
					DEBUG_PBS_ENGINE(DEBUG_LEVEL_TRACE,
							 ("-- Fail Training IP\n"));
					/* training machine failed */
					pup_state[if_id][pup] = 1;
					adll_shift_lock[if_id][pup] = 0;
					continue;
				}

				else if ((res0[if_id] & res_valid_mask) ==
					 validation_val) {
					DEBUG_PBS_ENGINE(DEBUG_LEVEL_TRACE,
							 ("-- FAIL EBA %d %d %d %d\n",
							  if_id, bit, pup,
							  res0[if_id]));
					pup_state[if_id][pup] = 4;
					/* this pup move to EEBA */
					adll_shift_lock[if_id][pup] = 0;
					continue;
				} else {
					/*
					 * The search ended in Pass we need
					 * Fail
					 */
					res0[if_id] =
						(pbs_mode == PBS_RX_MODE) ?
						((res0[if_id] &
						  res_valid_mask) + 1) :
						((res0[if_id] &
						  res_valid_mask) - 1);
					max_adll_per_pup[if_id][pup] =
						(max_adll_per_pup[if_id][pup] <
						 res0[if_id]) ?
						(u8)res0[if_id] :
						max_adll_per_pup[if_id][pup];
					min_adll_per_pup[if_id][pup] =
						(res0[if_id] >
						 min_adll_per_pup[if_id][pup]) ?
						min_adll_per_pup[if_id][pup] :
						(u8)
						res0[if_id];
					/*
					 * vs the Rx we are searching for the
					 * smallest value of DQ shift so all
					 * Bus would fail
					 */
					adll_shift_val[if_id][pup] =
						(pbs_mode == PBS_RX_MODE) ?
						max_adll_per_pup[if_id][pup] :
						min_adll_per_pup[if_id][pup];
				}
			}
		}
	}

	/* EEBA */
	for (pup = 0; pup < octets_per_if_num; pup++) {
		VALIDATE_BUS_ACTIVE(tm->bus_act_mask, pup);
		for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
			VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);

			if (pup_state[if_id][pup] != 4)
				continue;
			/*
			 * if pup state different from EEBA than move to
			 * next pup
			 */
			reg_addr = (pbs_mode == PBS_RX_MODE) ?
				(0x54 + effective_cs * 0x10) :
				(0x14 + effective_cs * 0x10);
			CHECK_STATUS(ddr3_tip_bus_write
				     (dev_num, ACCESS_TYPE_UNICAST, if_id,
				      ACCESS_TYPE_UNICAST, pup, DDR_PHY_DATA,
				      reg_addr, 0x1f));
			reg_addr = (pbs_mode == PBS_RX_MODE) ?
				(0x55 + effective_cs * 0x10) :
				(0x15 + effective_cs * 0x10);
			CHECK_STATUS(ddr3_tip_bus_write
				     (dev_num, ACCESS_TYPE_UNICAST, if_id,
				      ACCESS_TYPE_UNICAST, pup, DDR_PHY_DATA,
				      reg_addr, 0x1f));
			/* initialize the Edge2 Max. */
			adll_shift_val[if_id][pup] = 0;
			min_adll_per_pup[if_id][pup] =
				(pbs_mode == PBS_RX_MODE) ? 0x1f : 0x3f;
			max_adll_per_pup[if_id][pup] = 0x0;

			ddr3_tip_ip_training(dev_num, ACCESS_TYPE_MULTICAST,
					     PARAM_NOT_CARE,
					     ACCESS_TYPE_MULTICAST,
					     PARAM_NOT_CARE, RESULT_PER_BIT,
					     HWS_CONTROL_ELEMENT_ADLL,
					     search_dir, dir,
					     tm->if_act_mask, init_val,
					     iterations, pbs_pattern,
					     search_edge, CS_SINGLE, cs_num,
					     train_status);
			DEBUG_PBS_ENGINE(DEBUG_LEVEL_INFO,
					 ("ADLL shift results:\n"));

			for (bit = 0; bit < BUS_WIDTH_IN_BITS; bit++) {
				CHECK_STATUS(ddr3_tip_if_read
					     (dev_num, ACCESS_TYPE_MULTICAST,
					      PARAM_NOT_CARE,
					      mask_results_dq_reg_map[
						      bit + pup *
						      BUS_WIDTH_IN_BITS],
					      res0, MASK_ALL_BITS));
				DEBUG_PBS_ENGINE(DEBUG_LEVEL_TRACE,
						 ("FP I/F %d, bit:%d, pup:%d res0 0x%x\n",
						  if_id, bit, pup,
						  res0[if_id]));

				if ((res0[if_id] & 0x2000000) == 0) {
					DEBUG_PBS_ENGINE(DEBUG_LEVEL_TRACE,
							 (" -- EEBA Fail\n"));
					bit = BUS_WIDTH_IN_BITS;
					/* exit bit loop */
					DEBUG_PBS_ENGINE(DEBUG_LEVEL_TRACE,
							 ("-- EEBA Fail Training IP\n"));
					/*
					 * training machine failed but pass
					 * before in the EBA so maybe the DQS
					 * shift change env.
					 */
					pup_state[if_id][pup] = 2;
					adll_shift_lock[if_id][pup] = 0;
					reg_addr = (pbs_mode == PBS_RX_MODE) ?
						(0x54 + effective_cs * 0x10) :
						(0x14 + effective_cs * 0x10);
					CHECK_STATUS(ddr3_tip_bus_write
						     (dev_num,
						      ACCESS_TYPE_UNICAST,
						      if_id,
						      ACCESS_TYPE_UNICAST, pup,
						      DDR_PHY_DATA, reg_addr,
						      0x0));
					reg_addr = (pbs_mode == PBS_RX_MODE) ?
						(0x55 + effective_cs * 0x10) :
						(0x15 + effective_cs * 0x10);
					CHECK_STATUS(ddr3_tip_bus_write
						     (dev_num,
						      ACCESS_TYPE_UNICAST,
						      if_id,
						      ACCESS_TYPE_UNICAST, pup,
						      DDR_PHY_DATA, reg_addr,
						      0x0));
					continue;
				} else if ((res0[if_id] & res_valid_mask) ==
					   validation_val) {
					/* exit bit loop */
					bit = BUS_WIDTH_IN_BITS;
					DEBUG_PBS_ENGINE(DEBUG_LEVEL_TRACE,
							 ("-- FAIL EEBA\n"));
					/* this pup move to SBA */
					pup_state[if_id][pup] = 2;
					adll_shift_lock[if_id][pup] = 0;
					reg_addr = (pbs_mode == PBS_RX_MODE) ?
						(0x54 + effective_cs * 0x10) :
						(0x14 + effective_cs * 0x10);
					CHECK_STATUS(ddr3_tip_bus_write
						     (dev_num,
						      ACCESS_TYPE_UNICAST,
						      if_id,
						      ACCESS_TYPE_UNICAST, pup,
						      DDR_PHY_DATA, reg_addr,
						      0x0));
					reg_addr = (pbs_mode == PBS_RX_MODE) ?
						(0x55 + effective_cs * 0x10) :
						(0x15 + effective_cs * 0x10);
					CHECK_STATUS(ddr3_tip_bus_write
						     (dev_num,
						      ACCESS_TYPE_UNICAST,
						      if_id,
						      ACCESS_TYPE_UNICAST, pup,
						      DDR_PHY_DATA, reg_addr,
						      0x0));
					continue;
				} else {
					adll_shift_lock[if_id][pup] = 1;
					/*
					 * The search ended in Pass we need
					 * Fail
					 */
					res0[if_id] =
						(pbs_mode == PBS_RX_MODE) ?
						((res0[if_id] &
						  res_valid_mask) + 1) :
						((res0[if_id] &
						  res_valid_mask) - 1);
					max_adll_per_pup[if_id][pup] =
						(max_adll_per_pup[if_id][pup] <
						 res0[if_id]) ?
						(u8)res0[if_id] :
						max_adll_per_pup[if_id][pup];
					min_adll_per_pup[if_id][pup] =
						(res0[if_id] >
						 min_adll_per_pup[if_id][pup]) ?
						min_adll_per_pup[if_id][pup] :
						(u8)res0[if_id];
					/*
					 * vs the Rx we are searching for the
					 * smallest value of DQ shift so all Bus
					 * would fail
					 */
					adll_shift_val[if_id][pup] =
						(pbs_mode == PBS_RX_MODE) ?
						max_adll_per_pup[if_id][pup] :
						min_adll_per_pup[if_id][pup];
				}
			}
		}
	}

	/* Print Stage result */
	for (pup = 0; pup < octets_per_if_num; pup++) {
		VALIDATE_BUS_ACTIVE(tm->bus_act_mask, pup);
		for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
			VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
			DEBUG_PBS_ENGINE(DEBUG_LEVEL_TRACE,
					 ("FP I/F %d, ADLL Shift for EBA: pup[%d] Lock status = %d Lock Val = %d,%d\n",
					  if_id, pup,
					  adll_shift_lock[if_id][pup],
					  max_adll_per_pup[if_id][pup],
					  min_adll_per_pup[if_id][pup]));
		}
	}
	DEBUG_PBS_ENGINE(DEBUG_LEVEL_INFO,
			 ("Update ADLL Shift of all pups:\n"));

	for (pup = 0; pup < octets_per_if_num; pup++) {
		VALIDATE_BUS_ACTIVE(tm->bus_act_mask, pup);
		for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
			VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
			if (adll_shift_lock[if_id][pup] != 1)
				continue;
			/* if pup not locked continue to next pup */

			reg_addr = (pbs_mode == PBS_RX_MODE) ?
				(0x3 + effective_cs * 4) :
				(0x1 + effective_cs * 4);
			CHECK_STATUS(ddr3_tip_bus_write
				     (dev_num, ACCESS_TYPE_UNICAST, if_id,
				      ACCESS_TYPE_UNICAST, pup, DDR_PHY_DATA,
				      reg_addr, adll_shift_val[if_id][pup]));
			DEBUG_PBS_ENGINE(DEBUG_LEVEL_TRACE,
					 ("FP I/F %d, Pup[%d] = %d\n", if_id,
					  pup, adll_shift_val[if_id][pup]));
		}
	}

	/* PBS EEBA&EBA */
	/* Start the Per Bit Skew search */
	for (pup = 0; pup < octets_per_if_num; pup++) {
		VALIDATE_BUS_ACTIVE(tm->bus_act_mask, pup);
		for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
			VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
			max_pbs_per_pup[if_id][pup] = 0x0;
			min_pbs_per_pup[if_id][pup] = 0x1f;
			for (bit = 0; bit < BUS_WIDTH_IN_BITS; bit++) {
				/* reset result for PBS */
				result_all_bit[bit + pup * BUS_WIDTH_IN_BITS +
					       if_id * MAX_BUS_NUM *
					       BUS_WIDTH_IN_BITS] = 0;
			}
		}
	}

	iterations = 31;
	search_dir = HWS_LOW2HIGH;
	/* !!!!! ran sh (search_dir == HWS_LOW2HIGH)?0:iterations; */
	init_val = 0;

	ddr3_tip_ip_training(dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
			     ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
			     RESULT_PER_BIT, HWS_CONTROL_ELEMENT_DQ_SKEW,
			     search_dir, dir, tm->if_act_mask, init_val,
			     iterations, pbs_pattern, search_edge,
			     CS_SINGLE, cs_num, train_status);

	for (pup = 0; pup < octets_per_if_num; pup++) {
		VALIDATE_BUS_ACTIVE(tm->bus_act_mask, pup);
		for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
			VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
			if (adll_shift_lock[if_id][pup] != 1) {
				/* if pup not lock continue to next pup */
				continue;
			}

			for (bit = 0; bit < BUS_WIDTH_IN_BITS; bit++) {
				CHECK_STATUS(ddr3_tip_if_read
					     (dev_num, ACCESS_TYPE_MULTICAST,
					      PARAM_NOT_CARE,
					      mask_results_dq_reg_map[
						      bit +
						      pup * BUS_WIDTH_IN_BITS],
					      res0, MASK_ALL_BITS));
				DEBUG_PBS_ENGINE(DEBUG_LEVEL_INFO,
						 ("Per Bit Skew search, FP I/F %d, bit:%d, pup:%d res0 0x%x\n",
						  if_id, bit, pup,
						  res0[if_id]));
				if ((res0[if_id] & 0x2000000) == 0) {
					DEBUG_PBS_ENGINE(DEBUG_LEVEL_INFO,
							 ("--EBA PBS Fail - Training IP machine\n"));
					/* exit the bit loop */
					bit = BUS_WIDTH_IN_BITS;
					/*
					 * ADLL is no long in lock need new
					 * search
					 */
					adll_shift_lock[if_id][pup] = 0;
					/* Move to SBA */
					pup_state[if_id][pup] = 2;
					max_pbs_per_pup[if_id][pup] = 0x0;
					min_pbs_per_pup[if_id][pup] = 0x1f;
					continue;
				} else {
					temp = (u8)(res0[if_id] &
						    res_valid_mask);
					max_pbs_per_pup[if_id][pup] =
						(temp >
						 max_pbs_per_pup[if_id][pup]) ?
						temp :
						max_pbs_per_pup[if_id][pup];
					min_pbs_per_pup[if_id][pup] =
						(temp <
						 min_pbs_per_pup[if_id][pup]) ?
						temp :
						min_pbs_per_pup[if_id][pup];
					result_all_bit[bit +
						       pup * BUS_WIDTH_IN_BITS +
						       if_id * MAX_BUS_NUM *
						       BUS_WIDTH_IN_BITS] =
						temp;
				}
			}
		}
	}

	/* Check all Pup lock */
	all_lock = 1;
	for (pup = 0; pup < octets_per_if_num; pup++) {
		VALIDATE_BUS_ACTIVE(tm->bus_act_mask, pup);
		for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
			VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
			all_lock = all_lock * adll_shift_lock[if_id][pup];
		}
	}

	/* Only if not all Pups Lock */
	if (all_lock == 0) {
		DEBUG_PBS_ENGINE(DEBUG_LEVEL_INFO,
				 ("##########ADLL shift for SBA###########\n"));

		/* ADLL shift for SBA */
		search_dir = (pbs_mode == PBS_RX_MODE) ? HWS_LOW2HIGH :
			HWS_HIGH2LOW;
		init_val = (search_dir == HWS_LOW2HIGH) ? 0 : iterations;
		for (pup = 0; pup < octets_per_if_num; pup++) {
			VALIDATE_BUS_ACTIVE(tm->bus_act_mask, pup);
			for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1;
			     if_id++) {
				VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
				if (adll_shift_lock[if_id][pup] == 1) {
					/*if pup lock continue to next pup */
					continue;
				}
				/*init the var altogth init before */
				adll_shift_lock[if_id][pup] = 0;
				reg_addr = (pbs_mode == PBS_RX_MODE) ?
					(0x54 + effective_cs * 0x10) :
					(0x14 + effective_cs * 0x10);
				CHECK_STATUS(ddr3_tip_bus_write
					     (dev_num, ACCESS_TYPE_UNICAST,
					      if_id, ACCESS_TYPE_UNICAST, pup,
					      DDR_PHY_DATA, reg_addr, 0));
				reg_addr = (pbs_mode == PBS_RX_MODE) ?
					(0x55 + effective_cs * 0x10) :
					(0x15 + effective_cs * 0x10);
				CHECK_STATUS(ddr3_tip_bus_write
					     (dev_num, ACCESS_TYPE_UNICAST,
					      if_id, ACCESS_TYPE_UNICAST, pup,
					      DDR_PHY_DATA, reg_addr, 0));
				reg_addr = (pbs_mode == PBS_RX_MODE) ?
					(0x5f + effective_cs * 0x10) :
					(0x1f + effective_cs * 0x10);
				CHECK_STATUS(ddr3_tip_bus_write
					     (dev_num, ACCESS_TYPE_UNICAST,
					      if_id, ACCESS_TYPE_UNICAST, pup,
					      DDR_PHY_DATA, reg_addr, 0));
				/* initilaze the Edge2 Max. */
				adll_shift_val[if_id][pup] = 0;
				min_adll_per_pup[if_id][pup] = 0x1f;
				max_adll_per_pup[if_id][pup] = 0x0;

				ddr3_tip_ip_training(dev_num,
						     ACCESS_TYPE_MULTICAST,
						     PARAM_NOT_CARE,
						     ACCESS_TYPE_MULTICAST,
						     PARAM_NOT_CARE,
						     RESULT_PER_BIT,
						     HWS_CONTROL_ELEMENT_ADLL,
						     search_dir, dir,
						     tm->if_act_mask,
						     init_val, iterations,
						     pbs_pattern,
						     search_edge, CS_SINGLE,
						     cs_num, train_status);

				for (bit = 0; bit < BUS_WIDTH_IN_BITS; bit++) {
					CHECK_STATUS(ddr3_tip_if_read
						     (dev_num,
						      ACCESS_TYPE_MULTICAST,
						      PARAM_NOT_CARE,
						      mask_results_dq_reg_map
						      [bit +
						       pup *
						       BUS_WIDTH_IN_BITS],
						      res0, MASK_ALL_BITS));
					DEBUG_PBS_ENGINE(
						DEBUG_LEVEL_INFO,
						("FP I/F %d, bit:%d, pup:%d res0 0x%x\n",
						 if_id, bit, pup, res0[if_id]));
					if ((res0[if_id] & 0x2000000) == 0) {
						/* exit the bit loop */
						bit = BUS_WIDTH_IN_BITS;
						/* Fail SBA --> Fail PBS */
						pup_state[if_id][pup] = 1;
						DEBUG_PBS_ENGINE
							(DEBUG_LEVEL_INFO,
							 (" SBA Fail\n"));
						continue;
					} else {
						/*
						 * - increment to get all
						 * 8 bit lock.
						 */
						adll_shift_lock[if_id][pup]++;
						/*
						 * The search ended in Pass
						 * we need Fail
						 */
						res0[if_id] =
							(pbs_mode == PBS_RX_MODE) ?
							((res0[if_id] & res_valid_mask) + 1) :
							((res0[if_id] & res_valid_mask) - 1);
						max_adll_per_pup[if_id][pup] =
							(max_adll_per_pup[if_id]
							 [pup] < res0[if_id]) ?
							(u8)res0[if_id] :
							max_adll_per_pup[if_id][pup];
						min_adll_per_pup[if_id][pup] =
							(res0[if_id] >
							 min_adll_per_pup[if_id]
							 [pup]) ?
							min_adll_per_pup[if_id][pup] :
							(u8)res0[if_id];
						/*
						 * vs the Rx we are searching for
						 * the smallest value of DQ shift
						 * so all Bus would fail
						 */
						adll_shift_val[if_id][pup] =
							(pbs_mode == PBS_RX_MODE) ?
							max_adll_per_pup[if_id][pup] :
							min_adll_per_pup[if_id][pup];
					}
				}
				/* 1 is lock */
				adll_shift_lock[if_id][pup] =
					(adll_shift_lock[if_id][pup] == 8) ?
					1 : 0;
				reg_addr = (pbs_mode == PBS_RX_MODE) ?
					(0x3 + effective_cs * 4) :
					(0x1 + effective_cs * 4);
				CHECK_STATUS(ddr3_tip_bus_write
					     (dev_num, ACCESS_TYPE_UNICAST,
					      if_id, ACCESS_TYPE_UNICAST, pup,
					      DDR_PHY_DATA, reg_addr,
					      adll_shift_val[if_id][pup]));
				DEBUG_PBS_ENGINE(
					DEBUG_LEVEL_INFO,
					("adll_shift_lock[%x][%x] = %x\n",
					 if_id, pup,
					 adll_shift_lock[if_id][pup]));
			}
		}

		/* End ADLL Shift for SBA */
		/* Start the Per Bit Skew search */
		/* The ADLL shift finished with a Pass */
		search_edge = (pbs_mode == PBS_RX_MODE) ? EDGE_PF : EDGE_FP;
		search_dir = (pbs_mode == PBS_RX_MODE) ?
			HWS_LOW2HIGH : HWS_HIGH2LOW;
		iterations = 0x1f;
		/* - The initial value is different in Rx and Tx mode */
		init_val = (pbs_mode == PBS_RX_MODE) ? 0 : iterations;

		ddr3_tip_ip_training(dev_num, ACCESS_TYPE_MULTICAST,
				     PARAM_NOT_CARE, ACCESS_TYPE_MULTICAST,
				     PARAM_NOT_CARE, RESULT_PER_BIT,
				     HWS_CONTROL_ELEMENT_DQ_SKEW,
				     search_dir, dir, tm->if_act_mask,
				     init_val, iterations, pbs_pattern,
				     search_edge, CS_SINGLE, cs_num,
				     train_status);

		for (pup = 0; pup < octets_per_if_num; pup++) {
			VALIDATE_BUS_ACTIVE(tm->bus_act_mask, pup);
			for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1;
			     if_id++) {
				VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
				for (bit = 0; bit < BUS_WIDTH_IN_BITS; bit++) {
					CHECK_STATUS(ddr3_tip_if_read
						     (dev_num,
						      ACCESS_TYPE_MULTICAST,
						      PARAM_NOT_CARE,
						      mask_results_dq_reg_map
						      [bit +
						       pup *
						       BUS_WIDTH_IN_BITS],
						      res0, MASK_ALL_BITS));
					if (pup_state[if_id][pup] != 2) {
						/*
						 * if pup is not SBA continue
						 * to next pup
						 */
						bit = BUS_WIDTH_IN_BITS;
						continue;
					}
					DEBUG_PBS_ENGINE(
						DEBUG_LEVEL_INFO,
						("Per Bit Skew search, PF I/F %d, bit:%d, pup:%d res0 0x%x\n",
						 if_id, bit, pup, res0[if_id]));
					if ((res0[if_id] & 0x2000000) == 0) {
						DEBUG_PBS_ENGINE
							(DEBUG_LEVEL_INFO,
							 ("SBA Fail\n"));

						max_pbs_per_pup[if_id][pup] =
							0x1f;
						result_all_bit[
							bit + pup *
							BUS_WIDTH_IN_BITS +
							if_id * MAX_BUS_NUM *
							BUS_WIDTH_IN_BITS] =
							0x1f;
					} else {
						temp = (u8)(res0[if_id] &
							    res_valid_mask);
						max_pbs_per_pup[if_id][pup] =
							(temp >
							 max_pbs_per_pup[if_id]
							 [pup]) ? temp :
							max_pbs_per_pup
							[if_id][pup];
						min_pbs_per_pup[if_id][pup] =
							(temp <
							 min_pbs_per_pup[if_id]
							 [pup]) ? temp :
							min_pbs_per_pup
							[if_id][pup];
						result_all_bit[
							bit + pup *
							BUS_WIDTH_IN_BITS +
							if_id * MAX_BUS_NUM *
							BUS_WIDTH_IN_BITS] =
							temp;
						adll_shift_lock[if_id][pup] = 1;
					}
				}
			}
		}

		/* Check all Pup state */
		all_lock = 1;
		for (pup = 0; pup < octets_per_if_num; pup++) {
			/*
			 * DEBUG_PBS_ENGINE(DEBUG_LEVEL_INFO,
			 * ("pup_state[%d][%d] = %d\n",if_id,pup,pup_state
			 * [if_id][pup]));
			*/
		}
	}

	/* END OF SBA */
	/* Norm */
	for (pup = 0; pup < octets_per_if_num; pup++) {
		VALIDATE_BUS_ACTIVE(tm->bus_act_mask, pup);
		for (bit = 0; bit < BUS_WIDTH_IN_BITS; bit++) {
			for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1;
			     if_id++) {
				VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
				/* if pup not lock continue to next pup */
				if (adll_shift_lock[if_id][pup] != 1) {
					DEBUG_PBS_ENGINE(
						DEBUG_LEVEL_ERROR,
						("PBS failed for IF #%d\n",
						 if_id));
					training_result[training_stage][if_id]
						= TEST_FAILED;

					result_mat[if_id][pup][bit] = 0;
					max_pbs_per_pup[if_id][pup] = 0;
					min_pbs_per_pup[if_id][pup] = 0;
				} else {
					training_result[
						training_stage][if_id] =
						(training_result[training_stage]
						 [if_id] == TEST_FAILED) ?
						TEST_FAILED : TEST_SUCCESS;
					result_mat[if_id][pup][bit] =
						result_all_bit[
							bit + pup *
							BUS_WIDTH_IN_BITS +
							if_id * MAX_BUS_NUM *
							BUS_WIDTH_IN_BITS] -
						min_pbs_per_pup[if_id][pup];
				}
				DEBUG_PBS_ENGINE(
					DEBUG_LEVEL_INFO,
					("The abs min_pbs[%d][%d] = %d\n",
					 if_id, pup,
					 min_pbs_per_pup[if_id][pup]));
			}
		}
	}

	/* Clean all results */
	ddr3_tip_clean_pbs_result(dev_num, pbs_mode);

	/* DQ PBS register update with the final result */
	for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
		VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
		for (pup = 0; pup < octets_per_if_num; pup++) {
			VALIDATE_BUS_ACTIVE(tm->bus_act_mask, pup);

			DEBUG_PBS_ENGINE(
				DEBUG_LEVEL_INFO,
				("Final Results: if_id %d, pup %d, Pup State: %d\n",
				 if_id, pup, pup_state[if_id][pup]));
			for (bit = 0; bit < BUS_WIDTH_IN_BITS; bit++) {
				if (dq_map_table == NULL) {
					DEBUG_PBS_ENGINE(
						DEBUG_LEVEL_ERROR,
						("dq_map_table not initialized\n"));
					return MV_FAIL;
				}
				pad_num = dq_map_table[
					bit + pup * BUS_WIDTH_IN_BITS +
					if_id * BUS_WIDTH_IN_BITS *
					MAX_BUS_NUM];
				DEBUG_PBS_ENGINE(DEBUG_LEVEL_INFO,
						 ("result_mat: %d ",
						  result_mat[if_id][pup]
						  [bit]));
				reg_addr = (pbs_mode == PBS_RX_MODE) ?
					PBS_RX_PHY_REG(effective_cs, 0) :
					PBS_TX_PHY_REG(effective_cs, 0);
				CHECK_STATUS(ddr3_tip_bus_write
					     (dev_num, ACCESS_TYPE_UNICAST,
					      if_id, ACCESS_TYPE_UNICAST, pup,
					      DDR_PHY_DATA, reg_addr + pad_num,
					      result_mat[if_id][pup][bit]));
			}

			if (max_pbs_per_pup[if_id][pup] == min_pbs_per_pup[if_id][pup]) {
				temp = TYPICAL_PBS_VALUE;
			} else {
				temp = ((max_adll_per_pup[if_id][pup] -
					 min_adll_per_pup[if_id][pup]) *
					adll_tap /
					(max_pbs_per_pup[if_id][pup] -
					 min_pbs_per_pup[if_id][pup]));
			}
			pbsdelay_per_pup[pbs_mode]
			[if_id][pup][effective_cs] = temp;

			/* RX results ready, write RX also */
			if (pbs_mode == PBS_TX_MODE) {
				/* Write TX results */
				reg_addr = (0x14 + effective_cs * 0x10);
				CHECK_STATUS(ddr3_tip_bus_write
					     (dev_num, ACCESS_TYPE_UNICAST,
					      if_id, ACCESS_TYPE_UNICAST, pup,
					      DDR_PHY_DATA, reg_addr,
					      (max_pbs_per_pup[if_id][pup] -
					       min_pbs_per_pup[if_id][pup]) /
					      2));
				reg_addr = (0x15 + effective_cs * 0x10);
				CHECK_STATUS(ddr3_tip_bus_write
					     (dev_num, ACCESS_TYPE_UNICAST,
					      if_id, ACCESS_TYPE_UNICAST, pup,
					      DDR_PHY_DATA, reg_addr,
					      (max_pbs_per_pup[if_id][pup] -
					       min_pbs_per_pup[if_id][pup]) /
					      2));

				/* Write previously stored RX results */
				reg_addr = (0x54 + effective_cs * 0x10);
				CHECK_STATUS(ddr3_tip_bus_write
					     (dev_num, ACCESS_TYPE_UNICAST,
					      if_id, ACCESS_TYPE_UNICAST, pup,
					      DDR_PHY_DATA, reg_addr,
					      result_mat_rx_dqs[if_id][pup]
					      [effective_cs]));
				reg_addr = (0x55 + effective_cs * 0x10);
				CHECK_STATUS(ddr3_tip_bus_write
					     (dev_num, ACCESS_TYPE_UNICAST,
					      if_id, ACCESS_TYPE_UNICAST, pup,
					      DDR_PHY_DATA, reg_addr,
					      result_mat_rx_dqs[if_id][pup]
					      [effective_cs]));
			} else {
				/*
				 * RX results may affect RL results correctess,
				 * so just store the results that will written
				 * in TX stage
				 */
				result_mat_rx_dqs[if_id][pup][effective_cs] =
					(max_pbs_per_pup[if_id][pup] -
					 min_pbs_per_pup[if_id][pup]) / 2;
			}
			DEBUG_PBS_ENGINE(
				DEBUG_LEVEL_INFO,
				(", PBS tap=%d [psec] ==> skew observed = %d\n",
				 temp,
				 ((max_pbs_per_pup[if_id][pup] -
				   min_pbs_per_pup[if_id][pup]) *
				 temp)));
		}
	}

	/* Write back to the phy the default values */
	reg_addr = (pbs_mode == PBS_RX_MODE) ?
		CRX_PHY_REG(effective_cs) :
		CTX_PHY_REG(effective_cs);
	ddr3_tip_write_adll_value(dev_num, nominal_adll, reg_addr);

	for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
		reg_addr = (pbs_mode == PBS_RX_MODE) ?
			(0x5a + effective_cs * 0x10) :
			(0x1a + effective_cs * 0x10);
		CHECK_STATUS(ddr3_tip_bus_write
			     (dev_num, ACCESS_TYPE_UNICAST, if_id,
			      ACCESS_TYPE_UNICAST, pup, DDR_PHY_DATA, reg_addr,
			      0));

		/* restore cs enable value */
		VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
		CHECK_STATUS(ddr3_tip_if_write
			     (dev_num, ACCESS_TYPE_UNICAST, if_id,
			      DUAL_DUNIT_CFG_REG, cs_enable_reg_val[if_id],
			      MASK_ALL_BITS));
	}

	/* exit test mode */
	CHECK_STATUS(ddr3_tip_if_write
		     (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
		      ODPG_WR_RD_MODE_ENA_REG, 0xffff, MASK_ALL_BITS));

	for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
		VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
		for (pup = 0; pup < octets_per_if_num; pup++) {
			VALIDATE_BUS_ACTIVE(tm->bus_act_mask, pup);
			/*
			 * no valid window found
			 * (no lock at EBA ADLL shift at EBS)
			 */
			if (pup_state[if_id][pup] == 1)
				return MV_FAIL;
		}
	}

	return MV_OK;
}

/*
 * Name:     ddr3_tip_pbs_rx.
 * Desc:     PBS TX
 * Args:     TBD
 * Notes:
 * Returns:  OK if success, other error code if fail.
 */
int ddr3_tip_pbs_rx(u32 uidev_num)
{
	return ddr3_tip_pbs(uidev_num, PBS_RX_MODE);
}

/*
 * Name:     ddr3_tip_pbs_tx.
 * Desc:     PBS TX
 * Args:     TBD
 * Notes:
 * Returns:  OK if success, other error code if fail.
 */
int ddr3_tip_pbs_tx(u32 uidev_num)
{
	return ddr3_tip_pbs(uidev_num, PBS_TX_MODE);
}

#ifdef DDR_VIEWER_TOOL
/*
 * Print PBS Result
 */
int ddr3_tip_print_all_pbs_result(u32 dev_num)
{
	u32 curr_cs;
	unsigned int max_cs = mv_ddr_cs_num_get();

	for (curr_cs = 0; curr_cs < max_cs; curr_cs++) {
		ddr3_tip_print_pbs_result(dev_num, curr_cs, PBS_RX_MODE);
		ddr3_tip_print_pbs_result(dev_num, curr_cs, PBS_TX_MODE);
	}

	return MV_OK;
}

/*
 * Print PBS Result
 */
int ddr3_tip_print_pbs_result(u32 dev_num, u32 cs_num, enum pbs_dir pbs_mode)
{
	u32 data_value = 0, bit = 0, if_id = 0, pup = 0;
	u32 reg_addr = (pbs_mode == PBS_RX_MODE) ?
		PBS_RX_PHY_REG(cs_num, 0) :
		PBS_TX_PHY_REG(cs_num , 0);
	u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
	struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();

	printf("%s,CS%d,PBS,ADLLRATIO,,,",
	       (pbs_mode == PBS_RX_MODE) ? "Rx" : "Tx", cs_num);

	for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
		VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
		for (pup = 0; pup < octets_per_if_num; pup++) {
			VALIDATE_BUS_ACTIVE(tm->bus_act_mask, pup);
			printf("%d,",
			       pbsdelay_per_pup[pbs_mode][if_id][pup][cs_num]);
		}
	}
	printf("CS%d, %s ,PBS\n", cs_num,
	       (pbs_mode == PBS_RX_MODE) ? "Rx" : "Tx");

	for (bit = 0; bit < BUS_WIDTH_IN_BITS; bit++) {
		printf("%s, DQ", (pbs_mode == PBS_RX_MODE) ? "Rx" : "Tx");
		for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
			VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
			printf("%d ,PBS,,, ", bit);
			for (pup = 0; pup <= octets_per_if_num;
			     pup++) {
				VALIDATE_BUS_ACTIVE(tm->bus_act_mask, pup);
				CHECK_STATUS(ddr3_tip_bus_read
					     (dev_num, if_id,
					      ACCESS_TYPE_UNICAST, pup,
					      DDR_PHY_DATA, reg_addr + bit,
					      &data_value));
				printf("%d , ", data_value);
			}
		}
		printf("\n");
	}
	printf("\n");

	return MV_OK;
}
#endif /* DDR_VIEWER_TOOL */

/*
 * Fixup PBS Result
 */
int ddr3_tip_clean_pbs_result(u32 dev_num, enum pbs_dir pbs_mode)
{
	u32 if_id, pup, bit;
	u32 reg_addr = (pbs_mode == PBS_RX_MODE) ?
		PBS_RX_PHY_REG(effective_cs, 0) :
		PBS_TX_PHY_REG(effective_cs, 0);
	u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
	struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();

	for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
		VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
		for (pup = 0; pup <= octets_per_if_num; pup++) {
			for (bit = 0; bit <= BUS_WIDTH_IN_BITS + 3; bit++) {
				CHECK_STATUS(ddr3_tip_bus_write
					     (dev_num, ACCESS_TYPE_UNICAST,
					      if_id, ACCESS_TYPE_UNICAST, pup,
					      DDR_PHY_DATA, reg_addr + bit, 0));
			}
		}
	}

	return MV_OK;
}