sequencer.c

						    left_edge[i] !=
						    IO_IO_OUT1_DELAY_MAX + 1)
							right_edge[i] = -1;
						/*
						 * If a right edge has not been
						 * seen yet, then a future
						 * passing test will mark this
						 * edge as the left edge.
						 */
						else if (right_edge[i] ==
							IO_IO_OUT1_DELAY_MAX +
							1)
							left_edge[i] = -(d + 1);
					}
				}
				debug_cond(DLEVEL == 2, "write_center[r,d=%d):", d);
				debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d",
					   (int)(bit_chk & 1), i, left_edge[i]);
				debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
					   right_edge[i]);
				bit_chk = bit_chk >> 1;
			}
		}
	}

	/* Check that all bits have a window */
	for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
		debug_cond(DLEVEL == 2, "%s:%d write_center: left_edge[%u]: \
			   %d right_edge[%u]: %d", __func__, __LINE__,
			   i, left_edge[i], i, right_edge[i]);
		if ((left_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) ||
		    (right_edge[i] == IO_IO_OUT1_DELAY_MAX + 1)) {
			set_failing_group_stage(test_bgn + i,
						CAL_STAGE_WRITES,
						CAL_SUBSTAGE_WRITES_CENTER);
			return 0;
		}
	}

	/* Find middle of window for each DQ bit */
	mid_min = left_edge[0] - right_edge[0];
	min_index = 0;
	for (i = 1; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
		mid = left_edge[i] - right_edge[i];
		if (mid < mid_min) {
			mid_min = mid;
			min_index = i;
		}
	}

	/*
	 * -mid_min/2 represents the amount that we need to move DQS.
	 * If mid_min is odd and positive we'll need to add one to
	 * make sure the rounding in further calculations is correct
	 * (always bias to the right), so just add 1 for all positive values.
	 */
	if (mid_min > 0)
		mid_min++;
	mid_min = mid_min / 2;
	debug_cond(DLEVEL == 1, "%s:%d write_center: mid_min=%d\n", __func__,
		   __LINE__, mid_min);

	/* Determine the amount we can change DQS (which is -mid_min) */
	orig_mid_min = mid_min;
	new_dqs = start_dqs;
	mid_min = 0;
	debug_cond(DLEVEL == 1, "%s:%d write_center: start_dqs=%d new_dqs=%d \
		   mid_min=%d\n", __func__, __LINE__, start_dqs, new_dqs, mid_min);
	/* Initialize data for export structures */
	dqs_margin = IO_IO_OUT1_DELAY_MAX + 1;
	dq_margin  = IO_IO_OUT1_DELAY_MAX + 1;

	/* add delay to bring centre of all DQ windows to the same "level" */
	addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_OUT1_DELAY_OFFSET;
	for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) {
		/* Use values before divide by 2 to reduce round off error */
		shift_dq = (left_edge[i] - right_edge[i] -
			(left_edge[min_index] - right_edge[min_index]))/2  +
		(orig_mid_min - mid_min);

		debug_cond(DLEVEL == 2, "%s:%d write_center: before: shift_dq \
			   [%u]=%d\n", __func__, __LINE__, i, shift_dq);

		temp_dq_out1_delay = readl(addr + (i << 2));
		if (shift_dq + (int32_t)temp_dq_out1_delay >
			(int32_t)IO_IO_OUT1_DELAY_MAX) {
			shift_dq = (int32_t)IO_IO_OUT1_DELAY_MAX - temp_dq_out1_delay;
		} else if (shift_dq + (int32_t)temp_dq_out1_delay < 0) {
			shift_dq = -(int32_t)temp_dq_out1_delay;
		}
		debug_cond(DLEVEL == 2, "write_center: after: shift_dq[%u]=%d\n",
			   i, shift_dq);
		scc_mgr_set_dq_out1_delay(write_group, i, temp_dq_out1_delay +
					  shift_dq);
		scc_mgr_load_dq(i);

		debug_cond(DLEVEL == 2, "write_center: margin[%u]=[%d,%d]\n", i,
			   left_edge[i] - shift_dq + (-mid_min),
			   right_edge[i] + shift_dq - (-mid_min));
		/* To determine values for export structures */
		if (left_edge[i] - shift_dq + (-mid_min) < dq_margin)
			dq_margin = left_edge[i] - shift_dq + (-mid_min);

		if (right_edge[i] + shift_dq - (-mid_min) < dqs_margin)
			dqs_margin = right_edge[i] + shift_dq - (-mid_min);
	}

	/* Move DQS */
	scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, new_dqs);
	addr = (u32)&sdr_scc_mgr->update;
	writel(0, addr);

	/* Centre DM */
	debug_cond(DLEVEL == 2, "%s:%d write_center: DM\n", __func__, __LINE__);

	/*
	 * set the left and right edge of each bit to an illegal value,
	 * use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value,
	 */
	left_edge[0]  = IO_IO_OUT1_DELAY_MAX + 1;
	right_edge[0] = IO_IO_OUT1_DELAY_MAX + 1;
	int32_t bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
	int32_t end_curr = IO_IO_OUT1_DELAY_MAX + 1;
	int32_t bgn_best = IO_IO_OUT1_DELAY_MAX + 1;
	int32_t end_best = IO_IO_OUT1_DELAY_MAX + 1;
	int32_t win_best = 0;

	/* Search for the/part of the window with DM shift */
	addr = (u32)&sdr_scc_mgr->update;
	for (d = IO_IO_OUT1_DELAY_MAX; d >= 0; d -= DELTA_D) {
		scc_mgr_apply_group_dm_out1_delay(write_group, d);
		writel(0, addr);

		if (rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 1,
						    PASS_ALL_BITS, &bit_chk,
						    0)) {
			/* USE Set current end of the window */
			end_curr = -d;
			/*
			 * If a starting edge of our window has not been seen
			 * this is our current start of the DM window.
			 */
			if (bgn_curr == IO_IO_OUT1_DELAY_MAX + 1)
				bgn_curr = -d;

			/*
			 * If current window is bigger than best seen.
			 * Set best seen to be current window.
			 */
			if ((end_curr-bgn_curr+1) > win_best) {
				win_best = end_curr-bgn_curr+1;
				bgn_best = bgn_curr;
				end_best = end_curr;
			}
		} else {
			/* We just saw a failing test. Reset temp edge */
			bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
			end_curr = IO_IO_OUT1_DELAY_MAX + 1;
			}
		}


	/* Reset DM delay chains to 0 */
	scc_mgr_apply_group_dm_out1_delay(write_group, 0);

	/*
	 * Check to see if the current window nudges up aganist 0 delay.
	 * If so we need to continue the search by shifting DQS otherwise DQS
	 * search begins as a new search. */
	if (end_curr != 0) {
		bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
		end_curr = IO_IO_OUT1_DELAY_MAX + 1;
	}

	/* Search for the/part of the window with DQS shifts */
	addr = (u32)&sdr_scc_mgr->update;
	for (d = 0; d <= IO_IO_OUT1_DELAY_MAX - new_dqs; d += DELTA_D) {
		/*
		 * Note: This only shifts DQS, so are we limiting ourselve to
		 * width of DQ unnecessarily.
		 */
		scc_mgr_apply_group_dqs_io_and_oct_out1(write_group,
							d + new_dqs);

		writel(0, addr);
		if (rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 1,
						    PASS_ALL_BITS, &bit_chk,
						    0)) {
			/* USE Set current end of the window */
			end_curr = d;
			/*
			 * If a beginning edge of our window has not been seen
			 * this is our current begin of the DM window.
			 */
			if (bgn_curr == IO_IO_OUT1_DELAY_MAX + 1)
				bgn_curr = d;

			/*
			 * If current window is bigger than best seen. Set best
			 * seen to be current window.
			 */
			if ((end_curr-bgn_curr+1) > win_best) {
				win_best = end_curr-bgn_curr+1;
				bgn_best = bgn_curr;
				end_best = end_curr;
			}
		} else {
			/* We just saw a failing test. Reset temp edge */
			bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
			end_curr = IO_IO_OUT1_DELAY_MAX + 1;

			/* Early exit optimization: if ther remaining delay
			chain space is less than already seen largest window
			we can exit */
			if ((win_best-1) >
				(IO_IO_OUT1_DELAY_MAX - new_dqs - d)) {
					break;
				}
			}
		}

	/* assign left and right edge for cal and reporting; */
	left_edge[0] = -1*bgn_best;
	right_edge[0] = end_best;

	debug_cond(DLEVEL == 2, "%s:%d dm_calib: left=%d right=%d\n", __func__,
		   __LINE__, left_edge[0], right_edge[0]);

	/* Move DQS (back to orig) */
	scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, new_dqs);

	/* Move DM */

	/* Find middle of window for the DM bit */
	mid = (left_edge[0] - right_edge[0]) / 2;

	/* only move right, since we are not moving DQS/DQ */
	if (mid < 0)
		mid = 0;

	/* dm_marign should fail if we never find a window */
	if (win_best == 0)
		dm_margin = -1;
	else
		dm_margin = left_edge[0] - mid;

	scc_mgr_apply_group_dm_out1_delay(write_group, mid);
	addr = (u32)&sdr_scc_mgr->update;
	writel(0, addr);

	debug_cond(DLEVEL == 2, "%s:%d dm_calib: left=%d right=%d mid=%d \
		   dm_margin=%d\n", __func__, __LINE__, left_edge[0],
		   right_edge[0], mid, dm_margin);
	/* Export values */
	gbl->fom_out += dq_margin + dqs_margin;

	debug_cond(DLEVEL == 2, "%s:%d write_center: dq_margin=%d \
		   dqs_margin=%d dm_margin=%d\n", __func__, __LINE__,
		   dq_margin, dqs_margin, dm_margin);

	/*
	 * Do not remove this line as it makes sure all of our
	 * decisions have been applied.
	 */
	addr = (u32)&sdr_scc_mgr->update;
	writel(0, addr);
	return (dq_margin >= 0) && (dqs_margin >= 0) && (dm_margin >= 0);
}

/* calibrate the write operations */
static uint32_t rw_mgr_mem_calibrate_writes(uint32_t rank_bgn, uint32_t g,
	uint32_t test_bgn)
{
	/* update info for sims */
	debug("%s:%d %u %u\n", __func__, __LINE__, g, test_bgn);

	reg_file_set_stage(CAL_STAGE_WRITES);
	reg_file_set_sub_stage(CAL_SUBSTAGE_WRITES_CENTER);

	reg_file_set_group(g);

	if (!rw_mgr_mem_calibrate_writes_center(rank_bgn, g, test_bgn)) {
		set_failing_group_stage(g, CAL_STAGE_WRITES,
					CAL_SUBSTAGE_WRITES_CENTER);
		return 0;
	}

	return 1;
}

/* precharge all banks and activate row 0 in bank "000..." and bank "111..." */
static void mem_precharge_and_activate(void)
{
	uint32_t r;
	uint32_t addr;

	for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) {
		if (param->skip_ranks[r]) {
			/* request to skip the rank */
			continue;
		}

		/* set rank */
		set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF);

		/* precharge all banks ... */
		addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
		writel(RW_MGR_PRECHARGE_ALL, addr);

		addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0;
		writel(0x0F, addr);
		addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0;
		writel(RW_MGR_ACTIVATE_0_AND_1_WAIT1, addr);

		addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1;
		writel(0x0F, addr);
		addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1;
		writel(RW_MGR_ACTIVATE_0_AND_1_WAIT2, addr);

		/* activate rows */
		addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
		writel(RW_MGR_ACTIVATE_0_AND_1, addr);
	}
}

/* Configure various memory related parameters. */
static void mem_config(void)
{
	uint32_t rlat, wlat;
	uint32_t rw_wl_nop_cycles;
	uint32_t max_latency;
	uint32_t addr;

	debug("%s:%d\n", __func__, __LINE__);
	/* read in write and read latency */
	addr = (u32)&data_mgr->t_wl_add;
	wlat = readl(addr);

	addr = (u32)&data_mgr->mem_t_add;
	wlat += readl(addr);
	/* WL for hard phy does not include additive latency */

	/*
	 * add addtional write latency to offset the address/command extra
	 * clock cycle. We change the AC mux setting causing AC to be delayed
	 * by one mem clock cycle. Only do this for DDR3
	 */
	wlat = wlat + 1;

	addr = (u32)&data_mgr->t_rl_add;
	rlat = readl(addr);

	rw_wl_nop_cycles = wlat - 2;
	gbl->rw_wl_nop_cycles = rw_wl_nop_cycles;

	/*
	 * For AV/CV, lfifo is hardened and always runs at full rate so
	 * max latency in AFI clocks, used here, is correspondingly smaller.
	 */
	max_latency = (1<<MAX_LATENCY_COUNT_WIDTH)/1 - 1;
	/* configure for a burst length of 8 */

	/* write latency */
	/* Adjust Write Latency for Hard PHY */
	wlat = wlat + 1;

	/* set a pretty high read latency initially */
	gbl->curr_read_lat = rlat + 16;

	if (gbl->curr_read_lat > max_latency)
		gbl->curr_read_lat = max_latency;

	addr = (u32)&phy_mgr_cfg->phy_rlat;
	writel(gbl->curr_read_lat, addr);

	/* advertise write latency */
	gbl->curr_write_lat = wlat;
	addr = (u32)&phy_mgr_cfg->afi_wlat;
	writel(wlat - 2, addr);

	/* initialize bit slips */
	mem_precharge_and_activate();
}

/* Set VFIFO and LFIFO to instant-on settings in skip calibration mode */
static void mem_skip_calibrate(void)
{
	uint32_t vfifo_offset;
	uint32_t i, j, r;
	uint32_t addr;

	debug("%s:%d\n", __func__, __LINE__);
	/* Need to update every shadow register set used by the interface */
	for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
		r += NUM_RANKS_PER_SHADOW_REG) {
		/*
		 * Set output phase alignment settings appropriate for
		 * skip calibration.
		 */
		for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
			scc_mgr_set_dqs_en_phase(i, 0);
#if IO_DLL_CHAIN_LENGTH == 6
			scc_mgr_set_dqdqs_output_phase(i, 6);
#else
			scc_mgr_set_dqdqs_output_phase(i, 7);
#endif
			/*
			 * Case:33398
			 *
			 * Write data arrives to the I/O two cycles before write
			 * latency is reached (720 deg).
			 *   -> due to bit-slip in a/c bus
			 *   -> to allow board skew where dqs is longer than ck
			 *      -> how often can this happen!?
			 *      -> can claim back some ptaps for high freq
			 *       support if we can relax this, but i digress...
			 *
			 * The write_clk leads mem_ck by 90 deg
			 * The minimum ptap of the OPA is 180 deg
			 * Each ptap has (360 / IO_DLL_CHAIN_LENGH) deg of delay
			 * The write_clk is always delayed by 2 ptaps
			 *
			 * Hence, to make DQS aligned to CK, we need to delay
			 * DQS by:
			 *    (720 - 90 - 180 - 2 * (360 / IO_DLL_CHAIN_LENGTH))
			 *
			 * Dividing the above by (360 / IO_DLL_CHAIN_LENGTH)
			 * gives us the number of ptaps, which simplies to:
			 *
			 *    (1.25 * IO_DLL_CHAIN_LENGTH - 2)
			 */
			scc_mgr_set_dqdqs_output_phase(i, (1.25 *
				IO_DLL_CHAIN_LENGTH - 2));
		}
		addr = (u32)&sdr_scc_mgr->dqs_ena;
		writel(0xff, addr);
		addr = (u32)&sdr_scc_mgr->dqs_io_ena;
		writel(0xff, addr);

		addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_GROUP_COUNTER_OFFSET;
		for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) {
			writel(i, addr);
		}
		addr = (u32)&sdr_scc_mgr->dq_ena;
		writel(0xff, addr);
		addr = (u32)&sdr_scc_mgr->dm_ena;
		writel(0xff, addr);
		addr = (u32)&sdr_scc_mgr->update;
		writel(0, addr);
	}

	/* Compensate for simulation model behaviour */
	for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
		scc_mgr_set_dqs_bus_in_delay(i, 10);
		scc_mgr_load_dqs(i);
	}
	addr = (u32)&sdr_scc_mgr->update;
	writel(0, addr);

	/*
	 * ArriaV has hard FIFOs that can only be initialized by incrementing
	 * in sequencer.
	 */
	vfifo_offset = CALIB_VFIFO_OFFSET;
	addr = (u32)&phy_mgr_cmd->inc_vfifo_hard_phy;
	for (j = 0; j < vfifo_offset; j++) {
		writel(0xff, addr);
	}
	addr = (u32)&phy_mgr_cmd->fifo_reset;
	writel(0, addr);

	/*
	 * For ACV with hard lfifo, we get the skip-cal setting from
	 * generation-time constant.
	 */
	gbl->curr_read_lat = CALIB_LFIFO_OFFSET;
	addr = (u32)&phy_mgr_cfg->phy_rlat;
	writel(gbl->curr_read_lat, addr);
}

/* Memory calibration entry point */
static uint32_t mem_calibrate(void)
{
	uint32_t i;
	uint32_t rank_bgn, sr;
	uint32_t write_group, write_test_bgn;
	uint32_t read_group, read_test_bgn;
	uint32_t run_groups, current_run;
	uint32_t failing_groups = 0;
	uint32_t group_failed = 0;
	uint32_t sr_failed = 0;
	uint32_t addr;

	debug("%s:%d\n", __func__, __LINE__);
	/* Initialize the data settings */

	gbl->error_substage = CAL_SUBSTAGE_NIL;
	gbl->error_stage = CAL_STAGE_NIL;
	gbl->error_group = 0xff;
	gbl->fom_in = 0;
	gbl->fom_out = 0;

	mem_config();

	uint32_t bypass_mode = 0x1;
	addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_GROUP_COUNTER_OFFSET;
	for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
		writel(i, addr);
		scc_set_bypass_mode(i, bypass_mode);
	}

	if ((dyn_calib_steps & CALIB_SKIP_ALL) == CALIB_SKIP_ALL) {
		/*
		 * Set VFIFO and LFIFO to instant-on settings in skip
		 * calibration mode.
		 */
		mem_skip_calibrate();
	} else {
		for (i = 0; i < NUM_CALIB_REPEAT; i++) {
			/*
			 * Zero all delay chain/phase settings for all
			 * groups and all shadow register sets.
			 */
			scc_mgr_zero_all();

			run_groups = ~param->skip_groups;

			for (write_group = 0, write_test_bgn = 0; write_group
				< RW_MGR_MEM_IF_WRITE_DQS_WIDTH; write_group++,
				write_test_bgn += RW_MGR_MEM_DQ_PER_WRITE_DQS) {
				/* Initialized the group failure */
				group_failed = 0;

				current_run = run_groups & ((1 <<
					RW_MGR_NUM_DQS_PER_WRITE_GROUP) - 1);
				run_groups = run_groups >>
					RW_MGR_NUM_DQS_PER_WRITE_GROUP;

				if (current_run == 0)
					continue;

				addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_GROUP_COUNTER_OFFSET;
				writel(write_group, addr);
				scc_mgr_zero_group(write_group, write_test_bgn,
						   0);

				for (read_group = write_group *
					RW_MGR_MEM_IF_READ_DQS_WIDTH /
					RW_MGR_MEM_IF_WRITE_DQS_WIDTH,
					read_test_bgn = 0;
					read_group < (write_group + 1) *
					RW_MGR_MEM_IF_READ_DQS_WIDTH /
					RW_MGR_MEM_IF_WRITE_DQS_WIDTH &&
					group_failed == 0;
					read_group++, read_test_bgn +=
					RW_MGR_MEM_DQ_PER_READ_DQS) {
					/* Calibrate the VFIFO */
					if (!((STATIC_CALIB_STEPS) &
						CALIB_SKIP_VFIFO)) {
						if (!rw_mgr_mem_calibrate_vfifo
							(read_group,
							read_test_bgn)) {
							group_failed = 1;

							if (!(gbl->
							phy_debug_mode_flags &
						PHY_DEBUG_SWEEP_ALL_GROUPS)) {
								return 0;
							}
						}
					}
				}

				/* Calibrate the output side */
				if (group_failed == 0)	{
					for (rank_bgn = 0, sr = 0; rank_bgn
						< RW_MGR_MEM_NUMBER_OF_RANKS;
						rank_bgn +=
						NUM_RANKS_PER_SHADOW_REG,
						++sr) {
						sr_failed = 0;
						if (!((STATIC_CALIB_STEPS) &
						CALIB_SKIP_WRITES)) {
							if ((STATIC_CALIB_STEPS)
						& CALIB_SKIP_DELAY_SWEEPS) {
						/* not needed in quick mode! */
							} else {
						/*
						 * Determine if this set of
						 * ranks should be skipped
						 * entirely.
						 */
					if (!param->skip_shadow_regs[sr]) {
						if (!rw_mgr_mem_calibrate_writes
						(rank_bgn, write_group,
						write_test_bgn)) {
							sr_failed = 1;
							if (!(gbl->
							phy_debug_mode_flags &
						PHY_DEBUG_SWEEP_ALL_GROUPS)) {
								return 0;
									}
									}
								}
							}
						}
						if (sr_failed != 0)
							group_failed = 1;
					}
				}

				if (group_failed == 0) {
					for (read_group = write_group *
					RW_MGR_MEM_IF_READ_DQS_WIDTH /
					RW_MGR_MEM_IF_WRITE_DQS_WIDTH,
					read_test_bgn = 0;
						read_group < (write_group + 1)
						* RW_MGR_MEM_IF_READ_DQS_WIDTH
						/ RW_MGR_MEM_IF_WRITE_DQS_WIDTH &&
						group_failed == 0;
						read_group++, read_test_bgn +=
						RW_MGR_MEM_DQ_PER_READ_DQS) {
						if (!((STATIC_CALIB_STEPS) &
							CALIB_SKIP_WRITES)) {
					if (!rw_mgr_mem_calibrate_vfifo_end
						(read_group, read_test_bgn)) {
							group_failed = 1;

						if (!(gbl->phy_debug_mode_flags
						& PHY_DEBUG_SWEEP_ALL_GROUPS)) {
								return 0;
								}
							}
						}
					}
				}

				if (group_failed != 0)
					failing_groups++;
			}

			/*
			 * USER If there are any failing groups then report
			 * the failure.
			 */
			if (failing_groups != 0)
				return 0;

			/* Calibrate the LFIFO */
			if (!((STATIC_CALIB_STEPS) & CALIB_SKIP_LFIFO)) {
				/*
				 * If we're skipping groups as part of debug,
				 * don't calibrate LFIFO.
				 */
				if (param->skip_groups == 0) {
					if (!rw_mgr_mem_calibrate_lfifo())
						return 0;
				}
			}
		}
	}

	/*
	 * Do not remove this line as it makes sure all of our decisions
	 * have been applied.
	 */
	addr = (u32)&sdr_scc_mgr->update;
	writel(0, addr);
	return 1;
}

static uint32_t run_mem_calibrate(void)
{
	uint32_t pass;
	uint32_t debug_info;
	uint32_t addr;

	debug("%s:%d\n", __func__, __LINE__);

	/* Reset pass/fail status shown on afi_cal_success/fail */
	addr = (u32)&phy_mgr_cfg->cal_status;
	writel(PHY_MGR_CAL_RESET, addr);

	/* stop tracking manger */
	uint32_t ctrlcfg = readl(&sdr_ctrl->ctrl_cfg);

	writel(ctrlcfg & 0xFFBFFFFF, &sdr_ctrl->ctrl_cfg);

	initialize();
	rw_mgr_mem_initialize();

	pass = mem_calibrate();

	mem_precharge_and_activate();
	addr = (u32)&phy_mgr_cmd->fifo_reset;
	writel(0, addr);

	/*
	 * Handoff:
	 * Don't return control of the PHY back to AFI when in debug mode.
	 */
	if ((gbl->phy_debug_mode_flags & PHY_DEBUG_IN_DEBUG_MODE) == 0) {
		rw_mgr_mem_handoff();
		/*
		 * In Hard PHY this is a 2-bit control:
		 * 0: AFI Mux Select
		 * 1: DDIO Mux Select
		 */
		addr = (u32)&phy_mgr_cfg->mux_sel;
		writel(0x2, addr);
	}

	writel(ctrlcfg, &sdr_ctrl->ctrl_cfg);

	if (pass) {
		printf("%s: CALIBRATION PASSED\n", __FILE__);

		gbl->fom_in /= 2;
		gbl->fom_out /= 2;

		if (gbl->fom_in > 0xff)
			gbl->fom_in = 0xff;

		if (gbl->fom_out > 0xff)
			gbl->fom_out = 0xff;

		/* Update the FOM in the register file */
		debug_info = gbl->fom_in;
		debug_info |= gbl->fom_out << 8;
		addr = (u32)&sdr_reg_file->fom;
		writel(debug_info, addr);

		addr = (u32)&phy_mgr_cfg->cal_debug_info;
		writel(debug_info, addr);
		addr = (u32)&phy_mgr_cfg->cal_status;
		writel(PHY_MGR_CAL_SUCCESS, addr);
	} else {
		printf("%s: CALIBRATION FAILED\n", __FILE__);

		debug_info = gbl->error_stage;
		debug_info |= gbl->error_substage << 8;
		debug_info |= gbl->error_group << 16;

		addr = (u32)&sdr_reg_file->failing_stage;
		writel(debug_info, addr);
		addr = (u32)&phy_mgr_cfg->cal_debug_info;
		writel(debug_info, addr);
		addr = (u32)&phy_mgr_cfg->cal_status;
		writel(PHY_MGR_CAL_FAIL, addr);

		/* Update the failing group/stage in the register file */
		debug_info = gbl->error_stage;
		debug_info |= gbl->error_substage << 8;
		debug_info |= gbl->error_group << 16;
		addr = (u32)&sdr_reg_file->failing_stage;
		writel(debug_info, addr);
	}

	return pass;
}

static void hc_initialize_rom_data(void)
{
	uint32_t i;
	uint32_t addr;

	addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_INST_ROM_WRITE_OFFSET;
	for (i = 0; i < ARRAY_SIZE(inst_rom_init); i++) {
		uint32_t data = inst_rom_init[i];
		writel(data, addr + (i << 2));
	}

	addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_AC_ROM_WRITE_OFFSET;
	for (i = 0; i < ARRAY_SIZE(ac_rom_init); i++) {
		uint32_t data = ac_rom_init[i];
		writel(data, addr + (i << 2));
	}
}

static void initialize_reg_file(void)
{
	uint32_t addr;

	/* Initialize the register file with the correct data */
	addr = (u32)&sdr_reg_file->signature;
	writel(REG_FILE_INIT_SEQ_SIGNATURE, addr);

	addr = (u32)&sdr_reg_file->debug_data_addr;
	writel(0, addr);

	addr = (u32)&sdr_reg_file->cur_stage;
	writel(0, addr);

	addr = (u32)&sdr_reg_file->fom;
	writel(0, addr);

	addr = (u32)&sdr_reg_file->failing_stage;
	writel(0, addr);

	addr = (u32)&sdr_reg_file->debug1;
	writel(0, addr);

	addr = (u32)&sdr_reg_file->debug2;
	writel(0, addr);
}

static void initialize_hps_phy(void)
{
	uint32_t reg;
	/*
	 * Tracking also gets configured here because it's in the
	 * same register.
	 */
	uint32_t trk_sample_count = 7500;
	uint32_t trk_long_idle_sample_count = (10 << 16) | 100;
	/*
	 * Format is number of outer loops in the 16 MSB, sample
	 * count in 16 LSB.
	 */

	reg = 0;
	reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ACDELAYEN_SET(2);
	reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQDELAYEN_SET(1);
	reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSDELAYEN_SET(1);
	reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSLOGICDELAYEN_SET(1);
	reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_RESETDELAYEN_SET(0);
	reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_LPDDRDIS_SET(1);
	/*
	 * This field selects the intrinsic latency to RDATA_EN/FULL path.
	 * 00-bypass, 01- add 5 cycles, 10- add 10 cycles, 11- add 15 cycles.
	 */
	reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ADDLATSEL_SET(0);
	reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_SET(
		trk_sample_count);
	writel(reg, &sdr_ctrl->phy_ctrl0);

	reg = 0;
	reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_SAMPLECOUNT_31_20_SET(
		trk_sample_count >>
		SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_WIDTH);
	reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_SET(
		trk_long_idle_sample_count);
	writel(reg, &sdr_ctrl->phy_ctrl1);

	reg = 0;
	reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_2_LONGIDLESAMPLECOUNT_31_20_SET(
		trk_long_idle_sample_count >>
		SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_WIDTH);
	writel(reg, &sdr_ctrl->phy_ctrl2);
}

static void initialize_tracking(void)
{
	uint32_t concatenated_longidle = 0x0;
	uint32_t concatenated_delays = 0x0;
	uint32_t concatenated_rw_addr = 0x0;
	uint32_t concatenated_refresh = 0x0;
	uint32_t trk_sample_count = 7500;
	uint32_t dtaps_per_ptap;
	uint32_t tmp_delay;
	uint32_t addr;

	/*
	 * compute usable version of value in case we skip full
	 * computation later
	 */
	dtaps_per_ptap = 0;
	tmp_delay = 0;
	while (tmp_delay < IO_DELAY_PER_OPA_TAP) {
		dtaps_per_ptap++;
		tmp_delay += IO_DELAY_PER_DCHAIN_TAP;
	}
	dtaps_per_ptap--;

	concatenated_longidle = concatenated_longidle ^ 10;
		/*longidle outer loop */
	concatenated_longidle = concatenated_longidle << 16;
	concatenated_longidle = concatenated_longidle ^ 100;
		/*longidle sample count */
	concatenated_delays = concatenated_delays ^ 243;
		/* trfc, worst case of 933Mhz 4Gb */
	concatenated_delays = concatenated_delays << 8;
	concatenated_delays = concatenated_delays ^ 14;
		/* trcd, worst case */
	concatenated_delays = concatenated_delays << 8;
	concatenated_delays = concatenated_delays ^ 10;
		/* vfifo wait */
	concatenated_delays = concatenated_delays << 8;
	concatenated_delays = concatenated_delays ^ 4;
		/* mux delay */

	concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_IDLE;
	concatenated_rw_addr = concatenated_rw_addr << 8;
	concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_ACTIVATE_1;
	concatenated_rw_addr = concatenated_rw_addr << 8;
	concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_SGLE_READ;
	concatenated_rw_addr = concatenated_rw_addr << 8;
	concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_PRECHARGE_ALL;

	concatenated_refresh = concatenated_refresh ^ RW_MGR_REFRESH_ALL;
	concatenated_refresh = concatenated_refresh << 24;
	concatenated_refresh = concatenated_refresh ^ 1000; /* trefi */

	/* Initialize the register file with the correct data */
	addr = (u32)&sdr_reg_file->dtaps_per_ptap;
	writel(dtaps_per_ptap, addr);

	addr = (u32)&sdr_reg_file->trk_sample_count;
	writel(trk_sample_count, addr);

	addr = (u32)&sdr_reg_file->trk_longidle;
	writel(concatenated_longidle, addr);

	addr = (u32)&sdr_reg_file->delays;
	writel(concatenated_delays, addr);

	addr = (u32)&sdr_reg_file->trk_rw_mgr_addr;
	writel(concatenated_rw_addr, addr);

	addr = (u32)&sdr_reg_file->trk_read_dqs_width;
	writel(RW_MGR_MEM_IF_READ_DQS_WIDTH, addr);

	addr = (u32)&sdr_reg_file->trk_rfsh;
	writel(concatenated_refresh, addr);
}

int sdram_calibration_full(void)
{
	struct param_type my_param;
	struct gbl_type my_gbl;
	uint32_t pass;
	uint32_t i;

	param = &my_param;
	gbl = &my_gbl;

	/* Initialize the debug mode flags */
	gbl->phy_debug_mode_flags = 0;
	/* Set the calibration enabled by default */
	gbl->phy_debug_mode_flags |= PHY_DEBUG_ENABLE_CAL_RPT;
	/*
	 * Only sweep all groups (regardless of fail state) by default
	 * Set enabled read test by default.
	 */
#if DISABLE_GUARANTEED_READ
	gbl->phy_debug_mode_flags |= PHY_DEBUG_DISABLE_GUARANTEED_READ;
#endif
	/* Initialize the register file */
	initialize_reg_file();

	/* Initialize any PHY CSR */
	initialize_hps_phy();

	scc_mgr_initialize();

	initialize_tracking();

	/* USER Enable all ranks, groups */
	for (i = 0; i < RW_MGR_MEM_NUMBER_OF_RANKS; i++)
		param->skip_ranks[i] = 0;
	for (i = 0; i < NUM_SHADOW_REGS; ++i)
		param->skip_shadow_regs[i] = 0;
	param->skip_groups = 0;

	printf("%s: Preparing to start memory calibration\n", __FILE__);

	debug("%s:%d\n", __func__, __LINE__);
	debug_cond(DLEVEL == 1,
		   "DDR3 FULL_RATE ranks=%u cs/dimm=%u dq/dqs=%u,%u vg/dqs=%u,%u ",
		   RW_MGR_MEM_NUMBER_OF_RANKS, RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM,
		   RW_MGR_MEM_DQ_PER_READ_DQS, RW_MGR_MEM_DQ_PER_WRITE_DQS,
		   RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS,
		   RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS);
	debug_cond(DLEVEL == 1,
		   "dqs=%u,%u dq=%u dm=%u ptap_delay=%u dtap_delay=%u ",
		   RW_MGR_MEM_IF_READ_DQS_WIDTH, RW_MGR_MEM_IF_WRITE_DQS_WIDTH,
		   RW_MGR_MEM_DATA_WIDTH, RW_MGR_MEM_DATA_MASK_WIDTH,
		   IO_DELAY_PER_OPA_TAP, IO_DELAY_PER_DCHAIN_TAP);
	debug_cond(DLEVEL == 1, "dtap_dqsen_delay=%u, dll=%u",
		   IO_DELAY_PER_DQS_EN_DCHAIN_TAP, IO_DLL_CHAIN_LENGTH);
	debug_cond(DLEVEL == 1, "max values: en_p=%u dqdqs_p=%u en_d=%u dqs_in_d=%u ",
		   IO_DQS_EN_PHASE_MAX, IO_DQDQS_OUT_PHASE_MAX,
		   IO_DQS_EN_DELAY_MAX, IO_DQS_IN_DELAY_MAX);
	debug_cond(DLEVEL == 1, "io_in_d=%u io_out1_d=%u io_out2_d=%u ",
		   IO_IO_IN_DELAY_MAX, IO_IO_OUT1_DELAY_MAX,
		   IO_IO_OUT2_DELAY_MAX);
	debug_cond(DLEVEL == 1, "dqs_in_reserve=%u dqs_out_reserve=%u\n",
		   IO_DQS_IN_RESERVE, IO_DQS_OUT_RESERVE);

	hc_initialize_rom_data();

	/* update info for sims */
	reg_file_set_stage(CAL_STAGE_NIL);
	reg_file_set_group(0);

	/*
	 * Load global needed for those actions that require
	 * some dynamic calibration support.
	 */
	dyn_calib_steps = STATIC_CALIB_STEPS;