sequencer.c

	int bgn_best = IO_IO_OUT1_DELAY_MAX + 1;
	int end_best = IO_IO_OUT1_DELAY_MAX + 1;
	int win_best = 0;

	int ret;

	debug("%s:%d %u %u", __func__, __LINE__, write_group, test_bgn);

	dm_margin = 0;

	start_dqs = readl((SDR_PHYGRP_SCCGRP_ADDRESS |
			  SCC_MGR_IO_OUT1_DELAY_OFFSET) +
			  (RW_MGR_MEM_DQ_PER_WRITE_DQS << 2));

	/* Per-bit deskew. */

	/*
	 * Set the left and right edge of each bit to an illegal value.
	 * Use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value.
	 */
	sticky_bit_chk = 0;
	for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
		left_edge[i]  = IO_IO_OUT1_DELAY_MAX + 1;
		right_edge[i] = IO_IO_OUT1_DELAY_MAX + 1;
	}

	/* Search for the left edge of the window for each bit. */
	search_left_edge(1, rank_bgn, write_group, 0, test_bgn,
			 &sticky_bit_chk,
			 left_edge, right_edge, 0);

	/* Search for the right edge of the window for each bit. */
	ret = search_right_edge(1, rank_bgn, write_group, 0,
				start_dqs, 0,
				&sticky_bit_chk,
				left_edge, right_edge, 0);
	if (ret) {
		set_failing_group_stage(test_bgn + ret - 1, CAL_STAGE_WRITES,
					CAL_SUBSTAGE_WRITES_CENTER);
		return -EINVAL;
	}

	min_index = get_window_mid_index(1, left_edge, right_edge, &mid_min);

	/* Determine the amount we can change DQS (which is -mid_min). */
	orig_mid_min = mid_min;
	new_dqs = start_dqs;
	mid_min = 0;
	debug_cond(DLEVEL == 1,
		   "%s:%d write_center: start_dqs=%d new_dqs=%d mid_min=%d\n",
		   __func__, __LINE__, start_dqs, new_dqs, mid_min);

	/* Add delay to bring centre of all DQ windows to the same "level". */
	center_dq_windows(1, left_edge, right_edge, mid_min, orig_mid_min,
			  min_index, 0, &dq_margin, &dqs_margin);

	/* Move DQS */
	scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, new_dqs);
	writel(0, &sdr_scc_mgr->update);

	/* Centre DM */
	debug_cond(DLEVEL == 2, "%s:%d write_center: DM\n", __func__, __LINE__);

	/*
	 * Set the left and right edge of each bit to an illegal value.
	 * Use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value.
	 */
	left_edge[0]  = IO_IO_OUT1_DELAY_MAX + 1;
	right_edge[0] = IO_IO_OUT1_DELAY_MAX + 1;

	/* Search for the/part of the window with DM shift. */
	search_window(1, rank_bgn, write_group, &bgn_curr, &end_curr,
		      &bgn_best, &end_best, &win_best, 0);

	/* Reset DM delay chains to 0. */
	scc_mgr_apply_group_dm_out1_delay(0);

	/*
	 * Check to see if the current window nudges up aganist 0 delay.
	 * If so we need to continue the search by shifting DQS otherwise DQS
	 * search begins as a new search.
	 */
	if (end_curr != 0) {
		bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
		end_curr = IO_IO_OUT1_DELAY_MAX + 1;
	}

	/* Search for the/part of the window with DQS shifts. */
	search_window(0, rank_bgn, write_group, &bgn_curr, &end_curr,
		      &bgn_best, &end_best, &win_best, new_dqs);

	/* Assign left and right edge for cal and reporting. */
	left_edge[0] = -1 * bgn_best;
	right_edge[0] = end_best;

	debug_cond(DLEVEL == 2, "%s:%d dm_calib: left=%d right=%d\n",
		   __func__, __LINE__, left_edge[0], right_edge[0]);

	/* Move DQS (back to orig). */
	scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, new_dqs);

	/* Move DM */

	/* Find middle of window for the DM bit. */
	mid = (left_edge[0] - right_edge[0]) / 2;

	/* Only move right, since we are not moving DQS/DQ. */
	if (mid < 0)
		mid = 0;

	/* dm_marign should fail if we never find a window. */
	if (win_best == 0)
		dm_margin = -1;
	else
		dm_margin = left_edge[0] - mid;

	scc_mgr_apply_group_dm_out1_delay(mid);
	writel(0, &sdr_scc_mgr->update);

	debug_cond(DLEVEL == 2,
		   "%s:%d dm_calib: left=%d right=%d mid=%d dm_margin=%d\n",
		   __func__, __LINE__, left_edge[0], right_edge[0],
		   mid, dm_margin);
	/* Export values. */
	gbl->fom_out += dq_margin + dqs_margin;

	debug_cond(DLEVEL == 2,
		   "%s:%d write_center: dq_margin=%d dqs_margin=%d dm_margin=%d\n",
		   __func__, __LINE__, dq_margin, dqs_margin, dm_margin);

	/*
	 * Do not remove this line as it makes sure all of our
	 * decisions have been applied.
	 */
	writel(0, &sdr_scc_mgr->update);

	if ((dq_margin < 0) || (dqs_margin < 0) || (dm_margin < 0))
		return -EINVAL;

	return 0;
}

/**
 * rw_mgr_mem_calibrate_writes() - Write Calibration Part One
 * @rank_bgn:		Rank number
 * @group:		Read/Write Group
 * @test_bgn:		Rank at which the test begins
 *
 * Stage 2: Write Calibration Part One.
 *
 * This function implements UniPHY calibration Stage 2, as explained in
 * detail in Altera EMI_RM 2015.05.04 , "UniPHY Calibration Stages".
 */
static int rw_mgr_mem_calibrate_writes(const u32 rank_bgn, const u32 group,
				       const u32 test_bgn)
{
	int ret;

	/* Update info for sims */
	debug("%s:%d %u %u\n", __func__, __LINE__, group, test_bgn);

	reg_file_set_group(group);
	reg_file_set_stage(CAL_STAGE_WRITES);
	reg_file_set_sub_stage(CAL_SUBSTAGE_WRITES_CENTER);

	ret = rw_mgr_mem_calibrate_writes_center(rank_bgn, group, test_bgn);
	if (ret)
		set_failing_group_stage(group, CAL_STAGE_WRITES,
					CAL_SUBSTAGE_WRITES_CENTER);

	return ret;
}

/**
 * mem_precharge_and_activate() - Precharge all banks and activate
 *
 * Precharge all banks and activate row 0 in bank "000..." and bank "111...".
 */
static void mem_precharge_and_activate(void)
{
	int r;

	for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) {
		/* Test if the rank should be skipped. */
		if (param->skip_ranks[r])
			continue;

		/* Set rank. */
		set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF);

		/* Precharge all banks. */
		writel(RW_MGR_PRECHARGE_ALL, SDR_PHYGRP_RWMGRGRP_ADDRESS |
					     RW_MGR_RUN_SINGLE_GROUP_OFFSET);

		writel(0x0F, &sdr_rw_load_mgr_regs->load_cntr0);
		writel(RW_MGR_ACTIVATE_0_AND_1_WAIT1,
			&sdr_rw_load_jump_mgr_regs->load_jump_add0);

		writel(0x0F, &sdr_rw_load_mgr_regs->load_cntr1);
		writel(RW_MGR_ACTIVATE_0_AND_1_WAIT2,
			&sdr_rw_load_jump_mgr_regs->load_jump_add1);

		/* Activate rows. */
		writel(RW_MGR_ACTIVATE_0_AND_1, SDR_PHYGRP_RWMGRGRP_ADDRESS |
						RW_MGR_RUN_SINGLE_GROUP_OFFSET);
	}
}

/**
 * mem_init_latency() - Configure memory RLAT and WLAT settings
 *
 * Configure memory RLAT and WLAT parameters.
 */
static void mem_init_latency(void)
{
	/*
	 * For AV/CV, LFIFO is hardened and always runs at full rate
	 * so max latency in AFI clocks, used here, is correspondingly
	 * smaller.
	 */
	const u32 max_latency = (1 << MAX_LATENCY_COUNT_WIDTH) - 1;
	u32 rlat, wlat;

	debug("%s:%d\n", __func__, __LINE__);

	/*
	 * Read in write latency.
	 * WL for Hard PHY does not include additive latency.
	 */
	wlat = readl(&data_mgr->t_wl_add);
	wlat += readl(&data_mgr->mem_t_add);

	gbl->rw_wl_nop_cycles = wlat - 1;

	/* Read in readl latency. */
	rlat = readl(&data_mgr->t_rl_add);

	/* Set a pretty high read latency initially. */
	gbl->curr_read_lat = rlat + 16;
	if (gbl->curr_read_lat > max_latency)
		gbl->curr_read_lat = max_latency;

	writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat);

	/* Advertise write latency. */
	writel(wlat, &phy_mgr_cfg->afi_wlat);
}

/**
 * @mem_skip_calibrate() - Set VFIFO and LFIFO to instant-on settings
 *
 * Set VFIFO and LFIFO to instant-on settings in skip calibration mode.
 */
static void mem_skip_calibrate(void)
{
	uint32_t vfifo_offset;
	uint32_t i, j, r;

	debug("%s:%d\n", __func__, __LINE__);
	/* Need to update every shadow register set used by the interface */
	for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
	     r += NUM_RANKS_PER_SHADOW_REG) {
		/*
		 * Set output phase alignment settings appropriate for
		 * skip calibration.
		 */
		for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
			scc_mgr_set_dqs_en_phase(i, 0);
#if IO_DLL_CHAIN_LENGTH == 6
			scc_mgr_set_dqdqs_output_phase(i, 6);
#else
			scc_mgr_set_dqdqs_output_phase(i, 7);
#endif
			/*
			 * Case:33398
			 *
			 * Write data arrives to the I/O two cycles before write
			 * latency is reached (720 deg).
			 *   -> due to bit-slip in a/c bus
			 *   -> to allow board skew where dqs is longer than ck
			 *      -> how often can this happen!?
			 *      -> can claim back some ptaps for high freq
			 *       support if we can relax this, but i digress...
			 *
			 * The write_clk leads mem_ck by 90 deg
			 * The minimum ptap of the OPA is 180 deg
			 * Each ptap has (360 / IO_DLL_CHAIN_LENGH) deg of delay
			 * The write_clk is always delayed by 2 ptaps
			 *
			 * Hence, to make DQS aligned to CK, we need to delay
			 * DQS by:
			 *    (720 - 90 - 180 - 2 * (360 / IO_DLL_CHAIN_LENGTH))
			 *
			 * Dividing the above by (360 / IO_DLL_CHAIN_LENGTH)
			 * gives us the number of ptaps, which simplies to:
			 *
			 *    (1.25 * IO_DLL_CHAIN_LENGTH - 2)
			 */
			scc_mgr_set_dqdqs_output_phase(i,
					1.25 * IO_DLL_CHAIN_LENGTH - 2);
		}
		writel(0xff, &sdr_scc_mgr->dqs_ena);
		writel(0xff, &sdr_scc_mgr->dqs_io_ena);

		for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) {
			writel(i, SDR_PHYGRP_SCCGRP_ADDRESS |
				  SCC_MGR_GROUP_COUNTER_OFFSET);
		}
		writel(0xff, &sdr_scc_mgr->dq_ena);
		writel(0xff, &sdr_scc_mgr->dm_ena);
		writel(0, &sdr_scc_mgr->update);
	}

	/* Compensate for simulation model behaviour */
	for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
		scc_mgr_set_dqs_bus_in_delay(i, 10);
		scc_mgr_load_dqs(i);
	}
	writel(0, &sdr_scc_mgr->update);

	/*
	 * ArriaV has hard FIFOs that can only be initialized by incrementing
	 * in sequencer.
	 */
	vfifo_offset = CALIB_VFIFO_OFFSET;
	for (j = 0; j < vfifo_offset; j++)
		writel(0xff, &phy_mgr_cmd->inc_vfifo_hard_phy);
	writel(0, &phy_mgr_cmd->fifo_reset);

	/*
	 * For Arria V and Cyclone V with hard LFIFO, we get the skip-cal
	 * setting from generation-time constant.
	 */
	gbl->curr_read_lat = CALIB_LFIFO_OFFSET;
	writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat);
}

/**
 * mem_calibrate() - Memory calibration entry point.
 *
 * Perform memory calibration.
 */
static uint32_t mem_calibrate(void)
{
	uint32_t i;
	uint32_t rank_bgn, sr;
	uint32_t write_group, write_test_bgn;
	uint32_t read_group, read_test_bgn;
	uint32_t run_groups, current_run;
	uint32_t failing_groups = 0;
	uint32_t group_failed = 0;

	const u32 rwdqs_ratio = RW_MGR_MEM_IF_READ_DQS_WIDTH /
				RW_MGR_MEM_IF_WRITE_DQS_WIDTH;

	debug("%s:%d\n", __func__, __LINE__);

	/* Initialize the data settings */
	gbl->error_substage = CAL_SUBSTAGE_NIL;
	gbl->error_stage = CAL_STAGE_NIL;
	gbl->error_group = 0xff;
	gbl->fom_in = 0;
	gbl->fom_out = 0;

	/* Initialize WLAT and RLAT. */
	mem_init_latency();

	/* Initialize bit slips. */
	mem_precharge_and_activate();

	for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
		writel(i, SDR_PHYGRP_SCCGRP_ADDRESS |
			  SCC_MGR_GROUP_COUNTER_OFFSET);
		/* Only needed once to set all groups, pins, DQ, DQS, DM. */
		if (i == 0)
			scc_mgr_set_hhp_extras();

		scc_set_bypass_mode(i);
	}

	/* Calibration is skipped. */
	if ((dyn_calib_steps & CALIB_SKIP_ALL) == CALIB_SKIP_ALL) {
		/*
		 * Set VFIFO and LFIFO to instant-on settings in skip
		 * calibration mode.
		 */
		mem_skip_calibrate();

		/*
		 * Do not remove this line as it makes sure all of our
		 * decisions have been applied.
		 */
		writel(0, &sdr_scc_mgr->update);
		return 1;
	}

	/* Calibration is not skipped. */
	for (i = 0; i < NUM_CALIB_REPEAT; i++) {
		/*
		 * Zero all delay chain/phase settings for all
		 * groups and all shadow register sets.
		 */
		scc_mgr_zero_all();

		run_groups = ~param->skip_groups;

		for (write_group = 0, write_test_bgn = 0; write_group
			< RW_MGR_MEM_IF_WRITE_DQS_WIDTH; write_group++,
			write_test_bgn += RW_MGR_MEM_DQ_PER_WRITE_DQS) {

			/* Initialize the group failure */
			group_failed = 0;

			current_run = run_groups & ((1 <<
				RW_MGR_NUM_DQS_PER_WRITE_GROUP) - 1);
			run_groups = run_groups >>
				RW_MGR_NUM_DQS_PER_WRITE_GROUP;

			if (current_run == 0)
				continue;

			writel(write_group, SDR_PHYGRP_SCCGRP_ADDRESS |
					    SCC_MGR_GROUP_COUNTER_OFFSET);
			scc_mgr_zero_group(write_group, 0);

			for (read_group = write_group * rwdqs_ratio,
			     read_test_bgn = 0;
			     read_group < (write_group + 1) * rwdqs_ratio;
			     read_group++,
			     read_test_bgn += RW_MGR_MEM_DQ_PER_READ_DQS) {
				if (STATIC_CALIB_STEPS & CALIB_SKIP_VFIFO)
					continue;

				/* Calibrate the VFIFO */
				if (rw_mgr_mem_calibrate_vfifo(read_group,
							       read_test_bgn))
					continue;

				if (!(gbl->phy_debug_mode_flags & PHY_DEBUG_SWEEP_ALL_GROUPS))
					return 0;

				/* The group failed, we're done. */
				goto grp_failed;
			}

			/* Calibrate the output side */
			for (rank_bgn = 0, sr = 0;
			     rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS;
			     rank_bgn += NUM_RANKS_PER_SHADOW_REG, sr++) {
				if (STATIC_CALIB_STEPS & CALIB_SKIP_WRITES)
					continue;

				/* Not needed in quick mode! */
				if (STATIC_CALIB_STEPS & CALIB_SKIP_DELAY_SWEEPS)
					continue;

				/*
				 * Determine if this set of ranks
				 * should be skipped entirely.
				 */
				if (param->skip_shadow_regs[sr])
					continue;

				/* Calibrate WRITEs */
				if (!rw_mgr_mem_calibrate_writes(rank_bgn,
						write_group, write_test_bgn))
					continue;

				group_failed = 1;
				if (!(gbl->phy_debug_mode_flags & PHY_DEBUG_SWEEP_ALL_GROUPS))
					return 0;
			}

			/* Some group failed, we're done. */
			if (group_failed)
				goto grp_failed;

			for (read_group = write_group * rwdqs_ratio,
			     read_test_bgn = 0;
			     read_group < (write_group + 1) * rwdqs_ratio;
			     read_group++,
			     read_test_bgn += RW_MGR_MEM_DQ_PER_READ_DQS) {
				if (STATIC_CALIB_STEPS & CALIB_SKIP_WRITES)
					continue;

				if (rw_mgr_mem_calibrate_vfifo_end(read_group,
								read_test_bgn))
					continue;

				if (!(gbl->phy_debug_mode_flags & PHY_DEBUG_SWEEP_ALL_GROUPS))
					return 0;

				/* The group failed, we're done. */
				goto grp_failed;
			}

			/* No group failed, continue as usual. */
			continue;

grp_failed:		/* A group failed, increment the counter. */
			failing_groups++;
		}

		/*
		 * USER If there are any failing groups then report
		 * the failure.
		 */
		if (failing_groups != 0)
			return 0;

		if (STATIC_CALIB_STEPS & CALIB_SKIP_LFIFO)
			continue;

		/*
		 * If we're skipping groups as part of debug,
		 * don't calibrate LFIFO.
		 */
		if (param->skip_groups != 0)
			continue;

		/* Calibrate the LFIFO */
		if (!rw_mgr_mem_calibrate_lfifo())
			return 0;
	}

	/*
	 * Do not remove this line as it makes sure all of our decisions
	 * have been applied.
	 */
	writel(0, &sdr_scc_mgr->update);
	return 1;
}

/**
 * run_mem_calibrate() - Perform memory calibration
 *
 * This function triggers the entire memory calibration procedure.
 */
static int run_mem_calibrate(void)
{
	int pass;

	debug("%s:%d\n", __func__, __LINE__);

	/* Reset pass/fail status shown on afi_cal_success/fail */
	writel(PHY_MGR_CAL_RESET, &phy_mgr_cfg->cal_status);

	/* Stop tracking manager. */
	clrbits_le32(&sdr_ctrl->ctrl_cfg, 1 << 22);

	phy_mgr_initialize();
	rw_mgr_mem_initialize();

	/* Perform the actual memory calibration. */
	pass = mem_calibrate();

	mem_precharge_and_activate();
	writel(0, &phy_mgr_cmd->fifo_reset);

	/* Handoff. */
	rw_mgr_mem_handoff();
	/*
	 * In Hard PHY this is a 2-bit control:
	 * 0: AFI Mux Select
	 * 1: DDIO Mux Select
	 */
	writel(0x2, &phy_mgr_cfg->mux_sel);

	/* Start tracking manager. */
	setbits_le32(&sdr_ctrl->ctrl_cfg, 1 << 22);

	return pass;
}

/**
 * debug_mem_calibrate() - Report result of memory calibration
 * @pass:	Value indicating whether calibration passed or failed
 *
 * This function reports the results of the memory calibration
 * and writes debug information into the register file.
 */
static void debug_mem_calibrate(int pass)
{
	uint32_t debug_info;

	if (pass) {
		printf("%s: CALIBRATION PASSED\n", __FILE__);

		gbl->fom_in /= 2;
		gbl->fom_out /= 2;

		if (gbl->fom_in > 0xff)
			gbl->fom_in = 0xff;

		if (gbl->fom_out > 0xff)
			gbl->fom_out = 0xff;

		/* Update the FOM in the register file */
		debug_info = gbl->fom_in;
		debug_info |= gbl->fom_out << 8;
		writel(debug_info, &sdr_reg_file->fom);

		writel(debug_info, &phy_mgr_cfg->cal_debug_info);
		writel(PHY_MGR_CAL_SUCCESS, &phy_mgr_cfg->cal_status);
	} else {
		printf("%s: CALIBRATION FAILED\n", __FILE__);

		debug_info = gbl->error_stage;
		debug_info |= gbl->error_substage << 8;
		debug_info |= gbl->error_group << 16;

		writel(debug_info, &sdr_reg_file->failing_stage);
		writel(debug_info, &phy_mgr_cfg->cal_debug_info);
		writel(PHY_MGR_CAL_FAIL, &phy_mgr_cfg->cal_status);

		/* Update the failing group/stage in the register file */
		debug_info = gbl->error_stage;
		debug_info |= gbl->error_substage << 8;
		debug_info |= gbl->error_group << 16;
		writel(debug_info, &sdr_reg_file->failing_stage);
	}

	printf("%s: Calibration complete\n", __FILE__);
}

/**
 * hc_initialize_rom_data() - Initialize ROM data
 *
 * Initialize ROM data.
 */
static void hc_initialize_rom_data(void)
{
	u32 i, addr;

	addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_INST_ROM_WRITE_OFFSET;
	for (i = 0; i < ARRAY_SIZE(inst_rom_init); i++)
		writel(inst_rom_init[i], addr + (i << 2));

	addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_AC_ROM_WRITE_OFFSET;
	for (i = 0; i < ARRAY_SIZE(ac_rom_init); i++)
		writel(ac_rom_init[i], addr + (i << 2));
}

/**
 * initialize_reg_file() - Initialize SDR register file
 *
 * Initialize SDR register file.
 */
static void initialize_reg_file(void)
{
	/* Initialize the register file with the correct data */
	writel(REG_FILE_INIT_SEQ_SIGNATURE, &sdr_reg_file->signature);
	writel(0, &sdr_reg_file->debug_data_addr);
	writel(0, &sdr_reg_file->cur_stage);
	writel(0, &sdr_reg_file->fom);
	writel(0, &sdr_reg_file->failing_stage);
	writel(0, &sdr_reg_file->debug1);
	writel(0, &sdr_reg_file->debug2);
}

/**
 * initialize_hps_phy() - Initialize HPS PHY
 *
 * Initialize HPS PHY.
 */
static void initialize_hps_phy(void)
{
	uint32_t reg;
	/*
	 * Tracking also gets configured here because it's in the
	 * same register.
	 */
	uint32_t trk_sample_count = 7500;
	uint32_t trk_long_idle_sample_count = (10 << 16) | 100;
	/*
	 * Format is number of outer loops in the 16 MSB, sample
	 * count in 16 LSB.
	 */

	reg = 0;
	reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ACDELAYEN_SET(2);
	reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQDELAYEN_SET(1);
	reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSDELAYEN_SET(1);
	reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSLOGICDELAYEN_SET(1);
	reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_RESETDELAYEN_SET(0);
	reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_LPDDRDIS_SET(1);
	/*
	 * This field selects the intrinsic latency to RDATA_EN/FULL path.
	 * 00-bypass, 01- add 5 cycles, 10- add 10 cycles, 11- add 15 cycles.
	 */
	reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ADDLATSEL_SET(0);
	reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_SET(
		trk_sample_count);
	writel(reg, &sdr_ctrl->phy_ctrl0);

	reg = 0;
	reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_SAMPLECOUNT_31_20_SET(
		trk_sample_count >>
		SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_WIDTH);
	reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_SET(
		trk_long_idle_sample_count);
	writel(reg, &sdr_ctrl->phy_ctrl1);

	reg = 0;
	reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_2_LONGIDLESAMPLECOUNT_31_20_SET(
		trk_long_idle_sample_count >>
		SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_WIDTH);
	writel(reg, &sdr_ctrl->phy_ctrl2);
}

/**
 * initialize_tracking() - Initialize tracking
 *
 * Initialize the register file with usable initial data.
 */
static void initialize_tracking(void)
{
	/*
	 * Initialize the register file with the correct data.
	 * Compute usable version of value in case we skip full
	 * computation later.
	 */
	writel(DIV_ROUND_UP(IO_DELAY_PER_OPA_TAP, IO_DELAY_PER_DCHAIN_TAP) - 1,
	       &sdr_reg_file->dtaps_per_ptap);

	/* trk_sample_count */
	writel(7500, &sdr_reg_file->trk_sample_count);

	/* longidle outer loop [15:0] */
	writel((10 << 16) | (100 << 0), &sdr_reg_file->trk_longidle);

	/*
	 * longidle sample count [31:24]
	 * trfc, worst case of 933Mhz 4Gb [23:16]
	 * trcd, worst case [15:8]
	 * vfifo wait [7:0]
	 */
	writel((243 << 24) | (14 << 16) | (10 << 8) | (4 << 0),
	       &sdr_reg_file->delays);

	/* mux delay */
	writel((RW_MGR_IDLE << 24) | (RW_MGR_ACTIVATE_1 << 16) |
	       (RW_MGR_SGLE_READ << 8) | (RW_MGR_PRECHARGE_ALL << 0),
	       &sdr_reg_file->trk_rw_mgr_addr);

	writel(RW_MGR_MEM_IF_READ_DQS_WIDTH,
	       &sdr_reg_file->trk_read_dqs_width);

	/* trefi [7:0] */
	writel((RW_MGR_REFRESH_ALL << 24) | (1000 << 0),
	       &sdr_reg_file->trk_rfsh);
}

int sdram_calibration_full(void)
{
	struct param_type my_param;
	struct gbl_type my_gbl;
	uint32_t pass;

	memset(&my_param, 0, sizeof(my_param));
	memset(&my_gbl, 0, sizeof(my_gbl));

	param = &my_param;
	gbl = &my_gbl;

	/* Set the calibration enabled by default */
	gbl->phy_debug_mode_flags |= PHY_DEBUG_ENABLE_CAL_RPT;
	/*
	 * Only sweep all groups (regardless of fail state) by default
	 * Set enabled read test by default.
	 */
#if DISABLE_GUARANTEED_READ
	gbl->phy_debug_mode_flags |= PHY_DEBUG_DISABLE_GUARANTEED_READ;
#endif
	/* Initialize the register file */
	initialize_reg_file();

	/* Initialize any PHY CSR */
	initialize_hps_phy();

	scc_mgr_initialize();

	initialize_tracking();

	printf("%s: Preparing to start memory calibration\n", __FILE__);

	debug("%s:%d\n", __func__, __LINE__);
	debug_cond(DLEVEL == 1,
		   "DDR3 FULL_RATE ranks=%u cs/dimm=%u dq/dqs=%u,%u vg/dqs=%u,%u ",
		   RW_MGR_MEM_NUMBER_OF_RANKS, RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM,
		   RW_MGR_MEM_DQ_PER_READ_DQS, RW_MGR_MEM_DQ_PER_WRITE_DQS,
		   RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS,
		   RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS);
	debug_cond(DLEVEL == 1,
		   "dqs=%u,%u dq=%u dm=%u ptap_delay=%u dtap_delay=%u ",
		   RW_MGR_MEM_IF_READ_DQS_WIDTH, RW_MGR_MEM_IF_WRITE_DQS_WIDTH,
		   RW_MGR_MEM_DATA_WIDTH, RW_MGR_MEM_DATA_MASK_WIDTH,
		   IO_DELAY_PER_OPA_TAP, IO_DELAY_PER_DCHAIN_TAP);
	debug_cond(DLEVEL == 1, "dtap_dqsen_delay=%u, dll=%u",
		   IO_DELAY_PER_DQS_EN_DCHAIN_TAP, IO_DLL_CHAIN_LENGTH);
	debug_cond(DLEVEL == 1, "max values: en_p=%u dqdqs_p=%u en_d=%u dqs_in_d=%u ",
		   IO_DQS_EN_PHASE_MAX, IO_DQDQS_OUT_PHASE_MAX,
		   IO_DQS_EN_DELAY_MAX, IO_DQS_IN_DELAY_MAX);
	debug_cond(DLEVEL == 1, "io_in_d=%u io_out1_d=%u io_out2_d=%u ",
		   IO_IO_IN_DELAY_MAX, IO_IO_OUT1_DELAY_MAX,
		   IO_IO_OUT2_DELAY_MAX);
	debug_cond(DLEVEL == 1, "dqs_in_reserve=%u dqs_out_reserve=%u\n",
		   IO_DQS_IN_RESERVE, IO_DQS_OUT_RESERVE);

	hc_initialize_rom_data();

	/* update info for sims */
	reg_file_set_stage(CAL_STAGE_NIL);
	reg_file_set_group(0);

	/*
	 * Load global needed for those actions that require
	 * some dynamic calibration support.
	 */
	dyn_calib_steps = STATIC_CALIB_STEPS;
	/*
	 * Load global to allow dynamic selection of delay loop settings
	 * based on calibration mode.
	 */
	if (!(dyn_calib_steps & CALIB_SKIP_DELAY_LOOPS))
		skip_delay_mask = 0xff;
	else
		skip_delay_mask = 0x0;

	pass = run_mem_calibrate();
	debug_mem_calibrate(pass);
	return pass;
}