Newer
Older
/*
* Copyright Altera Corporation (C) 2012-2015
*
* SPDX-License-Identifier: BSD-3-Clause
*/
#include <common.h>
#include <asm/io.h>
#include <asm/arch/sdram.h>
#include <errno.h>
#include "sequencer.h"
#include "sequencer_auto.h"
#include "sequencer_auto_ac_init.h"
#include "sequencer_auto_inst_init.h"
#include "sequencer_defines.h"
static struct socfpga_sdr_rw_load_manager *sdr_rw_load_mgr_regs =
(struct socfpga_sdr_rw_load_manager *)(SDR_PHYGRP_RWMGRGRP_ADDRESS | 0x800);
static struct socfpga_sdr_rw_load_jump_manager *sdr_rw_load_jump_mgr_regs =
(struct socfpga_sdr_rw_load_jump_manager *)(SDR_PHYGRP_RWMGRGRP_ADDRESS | 0xC00);
static struct socfpga_sdr_reg_file *sdr_reg_file =
(struct socfpga_sdr_reg_file *)SDR_PHYGRP_REGFILEGRP_ADDRESS;
static struct socfpga_sdr_scc_mgr *sdr_scc_mgr =
(struct socfpga_sdr_scc_mgr *)(SDR_PHYGRP_SCCGRP_ADDRESS | 0xe00);
static struct socfpga_phy_mgr_cmd *phy_mgr_cmd =
(struct socfpga_phy_mgr_cmd *)SDR_PHYGRP_PHYMGRGRP_ADDRESS;
static struct socfpga_phy_mgr_cfg *phy_mgr_cfg =
(struct socfpga_phy_mgr_cfg *)(SDR_PHYGRP_PHYMGRGRP_ADDRESS | 0x40);
static struct socfpga_data_mgr *data_mgr =
(struct socfpga_data_mgr *)SDR_PHYGRP_DATAMGRGRP_ADDRESS;
static struct socfpga_sdr_ctrl *sdr_ctrl =
(struct socfpga_sdr_ctrl *)SDR_CTRLGRP_ADDRESS;
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
#define DELTA_D 1
/*
* In order to reduce ROM size, most of the selectable calibration steps are
* decided at compile time based on the user's calibration mode selection,
* as captured by the STATIC_CALIB_STEPS selection below.
*
* However, to support simulation-time selection of fast simulation mode, where
* we skip everything except the bare minimum, we need a few of the steps to
* be dynamic. In those cases, we either use the DYNAMIC_CALIB_STEPS for the
* check, which is based on the rtl-supplied value, or we dynamically compute
* the value to use based on the dynamically-chosen calibration mode
*/
#define DLEVEL 0
#define STATIC_IN_RTL_SIM 0
#define STATIC_SKIP_DELAY_LOOPS 0
#define STATIC_CALIB_STEPS (STATIC_IN_RTL_SIM | CALIB_SKIP_FULL_TEST | \
STATIC_SKIP_DELAY_LOOPS)
/* calibration steps requested by the rtl */
uint16_t dyn_calib_steps;
/*
* To make CALIB_SKIP_DELAY_LOOPS a dynamic conditional option
* instead of static, we use boolean logic to select between
* non-skip and skip values
*
* The mask is set to include all bits when not-skipping, but is
* zero when skipping
*/
uint16_t skip_delay_mask; /* mask off bits when skipping/not-skipping */
#define SKIP_DELAY_LOOP_VALUE_OR_ZERO(non_skip_value) \
((non_skip_value) & skip_delay_mask)
struct gbl_type *gbl;
struct param_type *param;
uint32_t curr_shadow_reg;
static uint32_t rw_mgr_mem_calibrate_write_test(uint32_t rank_bgn,
uint32_t write_group, uint32_t use_dm,
uint32_t all_correct, uint32_t *bit_chk, uint32_t all_ranks);
static void set_failing_group_stage(uint32_t group, uint32_t stage,
uint32_t substage)
{
/*
* Only set the global stage if there was not been any other
* failing group
*/
if (gbl->error_stage == CAL_STAGE_NIL) {
gbl->error_substage = substage;
gbl->error_stage = stage;
gbl->error_group = group;
}
}
static void reg_file_set_group(u16 set_group)
clrsetbits_le32(&sdr_reg_file->cur_stage, 0xffff0000, set_group << 16);
static void reg_file_set_stage(u8 set_stage)
clrsetbits_le32(&sdr_reg_file->cur_stage, 0xffff, set_stage & 0xff);
static void reg_file_set_sub_stage(u8 set_sub_stage)
set_sub_stage &= 0xff;
clrsetbits_le32(&sdr_reg_file->cur_stage, 0xff00, set_sub_stage << 8);
/**
* phy_mgr_initialize() - Initialize PHY Manager
*
* Initialize PHY Manager.
*/
static void phy_mgr_initialize(void)
debug("%s:%d\n", __func__, __LINE__);
/* Calibration has control over path to memory */
/*
* In Hard PHY this is a 2-bit control:
* 0: AFI Mux Select
* 1: DDIO Mux Select
*/
writel(0x3, &phy_mgr_cfg->mux_sel);
/* USER memory clock is not stable we begin initialization */
writel(0, &phy_mgr_cfg->reset_mem_stbl);
/* USER calibration status all set to zero */
writel(0, &phy_mgr_cfg->cal_status);
writel(0, &phy_mgr_cfg->cal_debug_info);
/* Init params only if we do NOT skip calibration. */
if ((dyn_calib_steps & CALIB_SKIP_ALL) == CALIB_SKIP_ALL)
return;
ratio = RW_MGR_MEM_DQ_PER_READ_DQS /
RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS;
param->read_correct_mask_vg = (1 << ratio) - 1;
param->write_correct_mask_vg = (1 << ratio) - 1;
param->read_correct_mask = (1 << RW_MGR_MEM_DQ_PER_READ_DQS) - 1;
param->write_correct_mask = (1 << RW_MGR_MEM_DQ_PER_WRITE_DQS) - 1;
ratio = RW_MGR_MEM_DATA_WIDTH /
RW_MGR_MEM_DATA_MASK_WIDTH;
param->dm_correct_mask = (1 << ratio) - 1;
/**
* set_rank_and_odt_mask() - Set Rank and ODT mask
* @rank: Rank mask
* @odt_mode: ODT mode, OFF or READ_WRITE
*
* Set Rank and ODT mask (On-Die Termination).
*/
static void set_rank_and_odt_mask(const u32 rank, const u32 odt_mode)
u32 odt_mask_0 = 0;
u32 odt_mask_1 = 0;
u32 cs_and_odt_mask;
if (odt_mode == RW_MGR_ODT_MODE_OFF) {
odt_mask_0 = 0x0;
odt_mask_1 = 0x0;
} else { /* RW_MGR_ODT_MODE_READ_WRITE */
switch (RW_MGR_MEM_NUMBER_OF_RANKS) {
case 1: /* 1 Rank */
/* Read: ODT = 0 ; Write: ODT = 1 */
odt_mask_0 = 0x0;
odt_mask_1 = 0x1;
break;
case 2: /* 2 Ranks */
if (RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM == 1) {
/*
* - Dual-Slot , Single-Rank (1 CS per DIMM)
* OR
* - RDIMM, 4 total CS (2 CS per DIMM, 2 DIMM)
*
* Since MEM_NUMBER_OF_RANKS is 2, they
* are both single rank with 2 CS each
* (special for RDIMM).
*
* Read: Turn on ODT on the opposite rank
* Write: Turn on ODT on all ranks
*/
odt_mask_0 = 0x3 & ~(1 << rank);
odt_mask_1 = 0x3;
} else {
/*
* - Single-Slot , Dual-Rank (2 CS per DIMM)
*
* Read: Turn on ODT off on all ranks
* Write: Turn on ODT on active rank
*/
odt_mask_0 = 0x0;
odt_mask_1 = 0x3 & (1 << rank);
}
break;
case 4: /* 4 Ranks */
/* Read:
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
* ----------+-----------------------+
* | ODT |
* Read From +-----------------------+
* Rank | 3 | 2 | 1 | 0 |
* ----------+-----+-----+-----+-----+
* 0 | 0 | 1 | 0 | 0 |
* 1 | 1 | 0 | 0 | 0 |
* 2 | 0 | 0 | 0 | 1 |
* 3 | 0 | 0 | 1 | 0 |
* ----------+-----+-----+-----+-----+
*
* Write:
* ----------+-----------------------+
* | ODT |
* Write To +-----------------------+
* Rank | 3 | 2 | 1 | 0 |
* ----------+-----+-----+-----+-----+
* 0 | 0 | 1 | 0 | 1 |
* 1 | 1 | 0 | 1 | 0 |
* 2 | 0 | 1 | 0 | 1 |
* 3 | 1 | 0 | 1 | 0 |
* ----------+-----+-----+-----+-----+
*/
switch (rank) {
case 0:
odt_mask_0 = 0x4;
odt_mask_1 = 0x5;
break;
case 1:
odt_mask_0 = 0x8;
odt_mask_1 = 0xA;
break;
case 2:
odt_mask_0 = 0x1;
odt_mask_1 = 0x5;
break;
case 3:
odt_mask_0 = 0x2;
odt_mask_1 = 0xA;
break;
}
cs_and_odt_mask = (0xFF & ~(1 << rank)) |
((0xFF & odt_mask_0) << 8) |
((0xFF & odt_mask_1) << 16);
writel(cs_and_odt_mask, SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_SET_CS_AND_ODT_MASK_OFFSET);
/**
* scc_mgr_set() - Set SCC Manager register
* @off: Base offset in SCC Manager space
* @grp: Read/Write group
* @val: Value to be set
*
* This function sets the SCC Manager (Scan Chain Control Manager) register.
*/
static void scc_mgr_set(u32 off, u32 grp, u32 val)
writel(val, SDR_PHYGRP_SCCGRP_ADDRESS | off | (grp << 2));
}
/**
* scc_mgr_initialize() - Initialize SCC Manager registers
*
* Initialize SCC Manager registers.
*/
static void scc_mgr_initialize(void)
{
* Clear register file for HPS. 16 (2^4) is the size of the
* full register file in the scc mgr:
* RFILE_DEPTH = 1 + log2(MEM_DQ_PER_DQS + 1 + MEM_DM_PER_DQS +
* MEM_IF_READ_DQS_WIDTH - 1);
for (i = 0; i < 16; i++) {
debug_cond(DLEVEL == 1, "%s:%d: Clearing SCC RFILE index %u\n",
__func__, __LINE__, i);
scc_mgr_set(SCC_MGR_HHP_RFILE_OFFSET, 0, i);
static void scc_mgr_set_dqdqs_output_phase(uint32_t write_group, uint32_t phase)
{
scc_mgr_set(SCC_MGR_DQDQS_OUT_PHASE_OFFSET, write_group, phase);
}
static void scc_mgr_set_dqs_bus_in_delay(uint32_t read_group, uint32_t delay)
scc_mgr_set(SCC_MGR_DQS_IN_DELAY_OFFSET, read_group, delay);
static void scc_mgr_set_dqs_en_phase(uint32_t read_group, uint32_t phase)
{
scc_mgr_set(SCC_MGR_DQS_EN_PHASE_OFFSET, read_group, phase);
}
static void scc_mgr_set_dqs_en_delay(uint32_t read_group, uint32_t delay)
{
scc_mgr_set(SCC_MGR_DQS_EN_DELAY_OFFSET, read_group, delay);
static void scc_mgr_set_dqs_io_in_delay(uint32_t delay)
scc_mgr_set(SCC_MGR_IO_IN_DELAY_OFFSET, RW_MGR_MEM_DQ_PER_WRITE_DQS,
delay);
static void scc_mgr_set_dq_in_delay(uint32_t dq_in_group, uint32_t delay)
scc_mgr_set(SCC_MGR_IO_IN_DELAY_OFFSET, dq_in_group, delay);
}
static void scc_mgr_set_dq_out1_delay(uint32_t dq_in_group, uint32_t delay)
{
scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET, dq_in_group, delay);
static void scc_mgr_set_dqs_out1_delay(uint32_t delay)
scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET, RW_MGR_MEM_DQ_PER_WRITE_DQS,
delay);
}
static void scc_mgr_set_dm_out1_delay(uint32_t dm, uint32_t delay)
{
scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET,
RW_MGR_MEM_DQ_PER_WRITE_DQS + 1 + dm,
delay);
}
/* load up dqs config settings */
static void scc_mgr_load_dqs(uint32_t dqs)
{
writel(dqs, &sdr_scc_mgr->dqs_ena);
}
/* load up dqs io config settings */
static void scc_mgr_load_dqs_io(void)
{
writel(0, &sdr_scc_mgr->dqs_io_ena);
}
/* load up dq config settings */
static void scc_mgr_load_dq(uint32_t dq_in_group)
{
writel(dq_in_group, &sdr_scc_mgr->dq_ena);
}
/* load up dm config settings */
static void scc_mgr_load_dm(uint32_t dm)
{
writel(dm, &sdr_scc_mgr->dm_ena);
/**
* scc_mgr_set_all_ranks() - Set SCC Manager register for all ranks
* @off: Base offset in SCC Manager space
* @grp: Read/Write group
* @val: Value to be set
* @update: If non-zero, trigger SCC Manager update for all ranks
*
* This function sets the SCC Manager (Scan Chain Control Manager) register
* and optionally triggers the SCC update for all ranks.
*/
static void scc_mgr_set_all_ranks(const u32 off, const u32 grp, const u32 val,
const int update)
for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
r += NUM_RANKS_PER_SHADOW_REG) {
scc_mgr_set(off, grp, val);
if (update || (r == 0)) {
writel(grp, &sdr_scc_mgr->dqs_ena);
writel(0, &sdr_scc_mgr->update);
static void scc_mgr_set_dqs_en_phase_all_ranks(u32 read_group, u32 phase)
{
/*
* USER although the h/w doesn't support different phases per
* shadow register, for simplicity our scc manager modeling
* keeps different phase settings per shadow reg, and it's
* important for us to keep them in sync to match h/w.
* for efficiency, the scan chain update should occur only
* once to sr0.
*/
scc_mgr_set_all_ranks(SCC_MGR_DQS_EN_PHASE_OFFSET,
read_group, phase, 0);
}
static void scc_mgr_set_dqdqs_output_phase_all_ranks(uint32_t write_group,
uint32_t phase)
{
/*
* USER although the h/w doesn't support different phases per
* shadow register, for simplicity our scc manager modeling
* keeps different phase settings per shadow reg, and it's
* important for us to keep them in sync to match h/w.
* for efficiency, the scan chain update should occur only
* once to sr0.
*/
scc_mgr_set_all_ranks(SCC_MGR_DQDQS_OUT_PHASE_OFFSET,
write_group, phase, 0);
}
static void scc_mgr_set_dqs_en_delay_all_ranks(uint32_t read_group,
uint32_t delay)
{
/*
* In shadow register mode, the T11 settings are stored in
* registers in the core, which are updated by the DQS_ENA
* signals. Not issuing the SCC_MGR_UPD command allows us to
* save lots of rank switching overhead, by calling
* select_shadow_regs_for_update with update_scan_chains
* set to 0.
*/
scc_mgr_set_all_ranks(SCC_MGR_DQS_EN_DELAY_OFFSET,
read_group, delay, 1);
writel(0, &sdr_scc_mgr->update);
/**
* scc_mgr_set_oct_out1_delay() - Set OCT output delay
* @write_group: Write group
* @delay: Delay value
*
* This function sets the OCT output delay in SCC manager.
*/
static void scc_mgr_set_oct_out1_delay(const u32 write_group, const u32 delay)
const int ratio = RW_MGR_MEM_IF_READ_DQS_WIDTH /
RW_MGR_MEM_IF_WRITE_DQS_WIDTH;
const int base = write_group * ratio;
int i;
/*
* Load the setting in the SCC manager
* Although OCT affects only write data, the OCT delay is controlled
* by the DQS logic block which is instantiated once per read group.
* For protocols where a write group consists of multiple read groups,
* the setting must be set multiple times.
*/
for (i = 0; i < ratio; i++)
scc_mgr_set(SCC_MGR_OCT_OUT1_DELAY_OFFSET, base + i, delay);
/**
* scc_mgr_set_hhp_extras() - Set HHP extras.
*
* Load the fixed setting in the SCC manager HHP extras.
*/
static void scc_mgr_set_hhp_extras(void)
{
/*
* Load the fixed setting in the SCC manager
* bits: 0:0 = 1'b1 - DQS bypass
* bits: 1:1 = 1'b1 - DQ bypass
* bits: 4:2 = 3'b001 - rfifo_mode
* bits: 6:5 = 2'b01 - rfifo clock_select
* bits: 7:7 = 1'b0 - separate gating from ungating setting
* bits: 8:8 = 1'b0 - separate OE from Output delay setting
const u32 value = (0 << 8) | (0 << 7) | (1 << 5) |
(1 << 2) | (1 << 1) | (1 << 0);
const u32 addr = SDR_PHYGRP_SCCGRP_ADDRESS |
SCC_MGR_HHP_GLOBALS_OFFSET |
SCC_MGR_HHP_EXTRAS_OFFSET;
debug_cond(DLEVEL == 1, "%s:%d Setting HHP Extras\n",
__func__, __LINE__);
writel(value, addr);
debug_cond(DLEVEL == 1, "%s:%d Done Setting HHP Extras\n",
__func__, __LINE__);
/**
* scc_mgr_zero_all() - Zero all DQS config
*
* Zero all DQS config.
*/
static void scc_mgr_zero_all(void)
{
/*
* USER Zero all DQS config settings, across all groups and all
* shadow registers
*/
for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
r += NUM_RANKS_PER_SHADOW_REG) {
for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
/*
* The phases actually don't exist on a per-rank basis,
* but there's no harm updating them several times, so
* let's keep the code simple.
*/
scc_mgr_set_dqs_bus_in_delay(i, IO_DQS_IN_RESERVE);
scc_mgr_set_dqs_en_phase(i, 0);
scc_mgr_set_dqs_en_delay(i, 0);
}
for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) {
scc_mgr_set_dqdqs_output_phase(i, 0);
/* Arria V/Cyclone V don't have out2. */
scc_mgr_set_oct_out1_delay(i, IO_DQS_OUT_RESERVE);
}
}
/* Multicast to all DQS group enables. */
writel(0xff, &sdr_scc_mgr->dqs_ena);
writel(0, &sdr_scc_mgr->update);
/**
* scc_set_bypass_mode() - Set bypass mode and trigger SCC update
* @write_group: Write group
*
* Set bypass mode and trigger SCC update.
*/
static void scc_set_bypass_mode(const u32 write_group)
/* Multicast to all DQ enables. */
writel(0xff, &sdr_scc_mgr->dq_ena);
writel(0xff, &sdr_scc_mgr->dm_ena);
/* Update current DQS IO enable. */
writel(0, &sdr_scc_mgr->dqs_io_ena);
writel(write_group, &sdr_scc_mgr->dqs_ena);
writel(0, &sdr_scc_mgr->update);
/**
* scc_mgr_load_dqs_for_write_group() - Load DQS settings for Write Group
* @write_group: Write group
*
* Load DQS settings for Write Group, do not trigger SCC update.
*/
static void scc_mgr_load_dqs_for_write_group(const u32 write_group)
const int ratio = RW_MGR_MEM_IF_READ_DQS_WIDTH /
RW_MGR_MEM_IF_WRITE_DQS_WIDTH;
const int base = write_group * ratio;
int i;
* Load the setting in the SCC manager
* Although OCT affects only write data, the OCT delay is controlled
* by the DQS logic block which is instantiated once per read group.
* For protocols where a write group consists of multiple read groups,
* the setting must be set multiple times.
for (i = 0; i < ratio; i++)
writel(base + i, &sdr_scc_mgr->dqs_ena);
/**
* scc_mgr_zero_group() - Zero all configs for a group
*
* Zero DQ, DM, DQS and OCT configs for a group.
*/
static void scc_mgr_zero_group(const u32 write_group, const int out_only)
for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
r += NUM_RANKS_PER_SHADOW_REG) {
/* Zero all DQ config settings. */
for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
scc_mgr_set_dq_out1_delay(i, 0);
scc_mgr_set_dq_in_delay(i, 0);
/* Multicast to all DQ enables. */
writel(0xff, &sdr_scc_mgr->dq_ena);
/* Zero all DM config settings. */
for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++)
scc_mgr_set_dm_out1_delay(i, 0);
/* Multicast to all DM enables. */
writel(0xff, &sdr_scc_mgr->dm_ena);
/* Zero all DQS IO settings. */
scc_mgr_set_dqs_io_in_delay(0);
/* Arria V/Cyclone V don't have out2. */
scc_mgr_set_dqs_out1_delay(IO_DQS_OUT_RESERVE);
scc_mgr_set_oct_out1_delay(write_group, IO_DQS_OUT_RESERVE);
scc_mgr_load_dqs_for_write_group(write_group);
/* Multicast to all DQS IO enables (only 1 in total). */
writel(0, &sdr_scc_mgr->dqs_io_ena);
/* Hit update to zero everything. */
writel(0, &sdr_scc_mgr->update);
}
}
/*
* apply and load a particular input delay for the DQ pins in a group
* group_bgn is the index of the first dq pin (in the write group)
*/
static void scc_mgr_apply_group_dq_in_delay(uint32_t group_bgn, uint32_t delay)
{
uint32_t i, p;
for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) {
scc_mgr_set_dq_in_delay(p, delay);
scc_mgr_load_dq(p);
}
}
/**
* scc_mgr_apply_group_dq_out1_delay() - Apply and load an output delay for the DQ pins in a group
* @delay: Delay value
*
* Apply and load a particular output delay for the DQ pins in a group.
*/
static void scc_mgr_apply_group_dq_out1_delay(const u32 delay)
for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
scc_mgr_set_dq_out1_delay(i, delay);
scc_mgr_load_dq(i);
}
}
/* apply and load a particular output delay for the DM pins in a group */
static void scc_mgr_apply_group_dm_out1_delay(uint32_t delay1)
{
uint32_t i;
for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) {
scc_mgr_set_dm_out1_delay(i, delay1);
scc_mgr_load_dm(i);
}
}
/* apply and load delay on both DQS and OCT out1 */
static void scc_mgr_apply_group_dqs_io_and_oct_out1(uint32_t write_group,
uint32_t delay)
{
scc_mgr_set_dqs_out1_delay(delay);
scc_mgr_load_dqs_io();
scc_mgr_set_oct_out1_delay(write_group, delay);
scc_mgr_load_dqs_for_write_group(write_group);
}
/**
* scc_mgr_apply_group_all_out_delay_add() - Apply a delay to the entire output side: DQ, DM, DQS, OCT
* @write_group: Write group
* @delay: Delay value
*
* Apply a delay to the entire output side: DQ, DM, DQS, OCT.
*/
static void scc_mgr_apply_group_all_out_delay_add(const u32 write_group,
const u32 delay)
{
u32 i, new_delay;
/* DQ shift */
for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++)
scc_mgr_load_dq(i);
/* DM shift */
for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++)
scc_mgr_load_dm(i);
/* DQS shift */
new_delay = READ_SCC_DQS_IO_OUT2_DELAY + delay;
if (new_delay > IO_IO_OUT2_DELAY_MAX) {
debug_cond(DLEVEL == 1,
"%s:%d (%u, %u) DQS: %u > %d; adding %u to OUT1\n",
__func__, __LINE__, write_group, delay, new_delay,
IO_IO_OUT2_DELAY_MAX,
new_delay - IO_IO_OUT2_DELAY_MAX);
new_delay -= IO_IO_OUT2_DELAY_MAX;
scc_mgr_set_dqs_out1_delay(new_delay);
}
scc_mgr_load_dqs_io();
/* OCT shift */
new_delay = READ_SCC_OCT_OUT2_DELAY + delay;
if (new_delay > IO_IO_OUT2_DELAY_MAX) {
debug_cond(DLEVEL == 1,
"%s:%d (%u, %u) DQS: %u > %d; adding %u to OUT1\n",
__func__, __LINE__, write_group, delay,
new_delay, IO_IO_OUT2_DELAY_MAX,
new_delay - IO_IO_OUT2_DELAY_MAX);
new_delay -= IO_IO_OUT2_DELAY_MAX;
scc_mgr_set_oct_out1_delay(write_group, new_delay);
}
scc_mgr_load_dqs_for_write_group(write_group);
}
/**
* scc_mgr_apply_group_all_out_delay_add() - Apply a delay to the entire output side to all ranks
* @write_group: Write group
* @delay: Delay value
*
* Apply a delay to the entire output side (DQ, DM, DQS, OCT) to all ranks.
static void
scc_mgr_apply_group_all_out_delay_add_all_ranks(const u32 write_group,
const u32 delay)
int r;
for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
r += NUM_RANKS_PER_SHADOW_REG) {
scc_mgr_apply_group_all_out_delay_add(write_group, delay);
writel(0, &sdr_scc_mgr->update);
/**
* set_jump_as_return() - Return instruction optimization
*
* Optimization used to recover some slots in ddr3 inst_rom could be
* applied to other protocols if we wanted to
*/
static void set_jump_as_return(void)
{
/*
* To save space, we replace return with jump to special shared
* RETURN instruction so we set the counter to large value so that
writel(0xff, &sdr_rw_load_mgr_regs->load_cntr0);
writel(RW_MGR_RETURN, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
}
/*
* should always use constants as argument to ensure all computations are
* performed at compile time
*/
static void delay_for_n_mem_clocks(const uint32_t clocks)
{
uint32_t afi_clocks;
uint8_t inner = 0;
uint8_t outer = 0;
uint16_t c_loop = 0;
debug("%s:%d: clocks=%u ... start\n", __func__, __LINE__, clocks);
afi_clocks = (clocks + AFI_RATE_RATIO-1) / AFI_RATE_RATIO;
/* scale (rounding up) to get afi clocks */
/*
* Note, we don't bother accounting for being off a little bit
* because of a few extra instructions in outer loops
* Note, the loops have a test at the end, and do the test before
* the decrement, and so always perform the loop
* 1 time more than the counter value
*/
if (afi_clocks == 0) {
;
} else if (afi_clocks <= 0x100) {
inner = afi_clocks-1;
outer = 0;
c_loop = 0;
} else if (afi_clocks <= 0x10000) {
inner = 0xff;
outer = (afi_clocks-1) >> 8;
c_loop = 0;
} else {
inner = 0xff;
outer = 0xff;
c_loop = (afi_clocks-1) >> 16;
}
/*
* rom instructions are structured as follows:
*
* IDLE_LOOP2: jnz cntr0, TARGET_A
* IDLE_LOOP1: jnz cntr1, TARGET_B
* return
*
* so, when doing nested loops, TARGET_A is set to IDLE_LOOP2, and
* TARGET_B is set to IDLE_LOOP2 as well
*
* if we have no outer loop, though, then we can use IDLE_LOOP1 only,
* and set TARGET_B to IDLE_LOOP1 and we skip IDLE_LOOP2 entirely
*
* a little confusing, but it helps save precious space in the inst_rom
* and sequencer rom and keeps the delays more accurate and reduces
* overhead
*/
if (afi_clocks <= 0x100) {
writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner),
&sdr_rw_load_mgr_regs->load_cntr1);
writel(RW_MGR_IDLE_LOOP1,
&sdr_rw_load_jump_mgr_regs->load_jump_add1);
writel(RW_MGR_IDLE_LOOP1, SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_RUN_SINGLE_GROUP_OFFSET);
writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner),
&sdr_rw_load_mgr_regs->load_cntr0);
writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(outer),
&sdr_rw_load_mgr_regs->load_cntr1);
writel(RW_MGR_IDLE_LOOP2,
&sdr_rw_load_jump_mgr_regs->load_jump_add0);
writel(RW_MGR_IDLE_LOOP2,
&sdr_rw_load_jump_mgr_regs->load_jump_add1);
/* hack to get around compiler not being smart enough */
if (afi_clocks <= 0x10000) {
/* only need to run once */
writel(RW_MGR_IDLE_LOOP2, SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_RUN_SINGLE_GROUP_OFFSET);
writel(RW_MGR_IDLE_LOOP2,
SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_RUN_SINGLE_GROUP_OFFSET);
} while (c_loop-- != 0);
}
}
debug("%s:%d clocks=%u ... end\n", __func__, __LINE__, clocks);
}
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
/**
* rw_mgr_mem_init_load_regs() - Load instruction registers
* @cntr0: Counter 0 value
* @cntr1: Counter 1 value
* @cntr2: Counter 2 value
* @jump: Jump instruction value
*
* Load instruction registers.
*/
static void rw_mgr_mem_init_load_regs(u32 cntr0, u32 cntr1, u32 cntr2, u32 jump)
{
uint32_t grpaddr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_RUN_SINGLE_GROUP_OFFSET;
/* Load counters */
writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(cntr0),
&sdr_rw_load_mgr_regs->load_cntr0);
writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(cntr1),
&sdr_rw_load_mgr_regs->load_cntr1);
writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(cntr2),
&sdr_rw_load_mgr_regs->load_cntr2);
/* Load jump address */
writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add1);
writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add2);
/* Execute count instruction */
writel(jump, grpaddr);
}
/**
* rw_mgr_mem_load_user() - Load user calibration values
* @fin1: Final instruction 1
* @fin2: Final instruction 2
* @precharge: If 1, precharge the banks at the end
*
* Load user calibration values and optionally precharge the banks.
*/
static void rw_mgr_mem_load_user(const u32 fin1, const u32 fin2,
const int precharge)
u32 grpaddr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_RUN_SINGLE_GROUP_OFFSET;
u32 r;
for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) {
if (param->skip_ranks[r]) {
/* request to skip the rank */
continue;
}
/* set rank */
set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF);
/* precharge all banks ... */
if (precharge)
writel(RW_MGR_PRECHARGE_ALL, grpaddr);
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
/*
* USER Use Mirror-ed commands for odd ranks if address
* mirrorring is on
*/
if ((RW_MGR_MEM_ADDRESS_MIRRORING >> r) & 0x1) {
set_jump_as_return();
writel(RW_MGR_MRS2_MIRR, grpaddr);
delay_for_n_mem_clocks(4);
set_jump_as_return();
writel(RW_MGR_MRS3_MIRR, grpaddr);
delay_for_n_mem_clocks(4);
set_jump_as_return();
writel(RW_MGR_MRS1_MIRR, grpaddr);
delay_for_n_mem_clocks(4);
set_jump_as_return();
writel(fin1, grpaddr);
} else {
set_jump_as_return();
writel(RW_MGR_MRS2, grpaddr);
delay_for_n_mem_clocks(4);
set_jump_as_return();
writel(RW_MGR_MRS3, grpaddr);
delay_for_n_mem_clocks(4);
set_jump_as_return();
writel(RW_MGR_MRS1, grpaddr);
set_jump_as_return();
writel(fin2, grpaddr);
}
if (precharge)
continue;
set_jump_as_return();
writel(RW_MGR_ZQCL, grpaddr);
/* tZQinit = tDLLK = 512 ck cycles */
delay_for_n_mem_clocks(512);
}
}
/**
* rw_mgr_mem_initialize() - Initialize RW Manager
*
* Initialize RW Manager.
*/
static void rw_mgr_mem_initialize(void)
{
debug("%s:%d\n", __func__, __LINE__);
/* The reset / cke part of initialization is broadcasted to all ranks */
writel(RW_MGR_RANK_ALL, SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_SET_CS_AND_ODT_MASK_OFFSET);
/*
* Here's how you load register for a loop
* Counters are located @ 0x800
* Jump address are located @ 0xC00
* For both, registers 0 to 3 are selected using bits 3 and 2, like
* in 0x800, 0x804, 0x808, 0x80C and 0xC00, 0xC04, 0xC08, 0xC0C
* I know this ain't pretty, but Avalon bus throws away the 2 least
* significant bits
*/
/* Start with memory RESET activated */
/* tINIT = 200us */
/*
* 200us @ 266MHz (3.75 ns) ~ 54000 clock cycles
* If a and b are the number of iteration in 2 nested loops
* it takes the following number of cycles to complete the operation:
* number_of_cycles = ((2 + n) * a + 2) * b
* where n is the number of instruction in the inner loop
* One possible solution is n = 0 , a = 256 , b = 106 => a = FF,
* b = 6A
*/
rw_mgr_mem_init_load_regs(SEQ_TINIT_CNTR0_VAL, SEQ_TINIT_CNTR1_VAL,
SEQ_TINIT_CNTR2_VAL,
RW_MGR_INIT_RESET_0_CKE_0);
/* Indicate that memory is stable. */
writel(1, &phy_mgr_cfg->reset_mem_stbl);
/*
* transition the RESET to high
* Wait for 500us
*/
/*
* 500us @ 266MHz (3.75 ns) ~ 134000 clock cycles
* If a and b are the number of iteration in 2 nested loops
* it takes the following number of cycles to complete the operation
* number_of_cycles = ((2 + n) * a + 2) * b
* where n is the number of instruction in the inner loop
* One possible solution is n = 2 , a = 131 , b = 256 => a = 83,
* b = FF
*/
rw_mgr_mem_init_load_regs(SEQ_TRESET_CNTR0_VAL, SEQ_TRESET_CNTR1_VAL,
SEQ_TRESET_CNTR2_VAL,
RW_MGR_INIT_RESET_1_CKE_0);
/* Bring up clock enable. */
/* tXRP < 250 ck cycles */
delay_for_n_mem_clocks(250);
rw_mgr_mem_load_user(RW_MGR_MRS0_DLL_RESET_MIRR, RW_MGR_MRS0_DLL_RESET,
0);
}
/*
* At the end of calibration we have to program the user settings in, and
* USER hand off the memory to the user.
*/
static void rw_mgr_mem_handoff(void)
{
rw_mgr_mem_load_user(RW_MGR_MRS0_USER_MIRR, RW_MGR_MRS0_USER, 1);
/*
* USER need to wait tMOD (12CK or 15ns) time before issuing
* other commands, but we will have plenty of NIOS cycles before
* actual handoff so its okay.
*/
/**
* rw_mgr_mem_calibrate_read_test_patterns() - Read back test patterns
* @rank_bgn: Rank number
* @group: Read/Write Group
* @all_ranks: Test all ranks
*
* Performs a guaranteed read on the patterns we are going to use during a
* read test to ensure memory works.
static int
rw_mgr_mem_calibrate_read_test_patterns(const u32 rank_bgn, const u32 group,
const u32 all_ranks)
const u32 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_RUN_SINGLE_GROUP_OFFSET;
const u32 addr_offset =
(group * RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS) << 2;
const u32 rank_end = all_ranks ?
RW_MGR_MEM_NUMBER_OF_RANKS :
(rank_bgn + NUM_RANKS_PER_SHADOW_REG);
const u32 shift_ratio = RW_MGR_MEM_DQ_PER_READ_DQS /
RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS;
const u32 correct_mask_vg = param->read_correct_mask_vg;
u32 tmp_bit_chk, base_rw_mgr, bit_chk;
int vg, r;
int ret = 0;
bit_chk = param->read_correct_mask;
for (r = rank_bgn; r < rank_end; r++) {
/* Request to skip the rank */
if (param->skip_ranks[r])
continue;
/* Set rank */
set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
/* Load up a constant bursts of read commands */
writel(0x20, &sdr_rw_load_mgr_regs->load_cntr0);
writel(RW_MGR_GUARANTEED_READ,
&sdr_rw_load_jump_mgr_regs->load_jump_add0);
writel(0x20, &sdr_rw_load_mgr_regs->load_cntr1);
writel(RW_MGR_GUARANTEED_READ_CONT,
&sdr_rw_load_jump_mgr_regs->load_jump_add1);
tmp_bit_chk = 0;
for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS - 1;
vg >= 0; vg--) {
/* Reset the FIFOs to get pointers to known state. */
writel(0, &phy_mgr_cmd->fifo_reset);
writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_RESET_READ_DATAPATH_OFFSET);
writel(RW_MGR_GUARANTEED_READ,
addr + addr_offset + (vg << 2));
base_rw_mgr = readl(SDR_PHYGRP_RWMGRGRP_ADDRESS);
tmp_bit_chk <<= shift_ratio;
tmp_bit_chk |= correct_mask_vg & ~base_rw_mgr;
bit_chk &= tmp_bit_chk;
writel(RW_MGR_CLEAR_DQS_ENABLE, addr + (group << 2));
set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
if (bit_chk != param->read_correct_mask)
ret = -EIO;
debug_cond(DLEVEL == 1,
"%s:%d test_load_patterns(%u,ALL) => (%u == %u) => %i\n",
__func__, __LINE__, group, bit_chk,
param->read_correct_mask, ret);
return ret;
/**
* rw_mgr_mem_calibrate_read_load_patterns() - Load up the patterns for read test
* @rank_bgn: Rank number
* @all_ranks: Test all ranks
*
* Load up the patterns we are going to use during a read test.
*/
static void rw_mgr_mem_calibrate_read_load_patterns(const u32 rank_bgn,
const int all_ranks)
const u32 rank_end = all_ranks ?
RW_MGR_MEM_NUMBER_OF_RANKS :
(rank_bgn + NUM_RANKS_PER_SHADOW_REG);
u32 r;
debug("%s:%d\n", __func__, __LINE__);
for (r = rank_bgn; r < rank_end; r++) {
if (param->skip_ranks[r])
/* request to skip the rank */
continue;
/* set rank */
set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
/* Load up a constant bursts */
writel(0x20, &sdr_rw_load_mgr_regs->load_cntr0);
writel(RW_MGR_GUARANTEED_WRITE_WAIT0,
&sdr_rw_load_jump_mgr_regs->load_jump_add0);
writel(0x20, &sdr_rw_load_mgr_regs->load_cntr1);
writel(RW_MGR_GUARANTEED_WRITE_WAIT1,
&sdr_rw_load_jump_mgr_regs->load_jump_add1);
writel(0x04, &sdr_rw_load_mgr_regs->load_cntr2);
writel(RW_MGR_GUARANTEED_WRITE_WAIT2,
&sdr_rw_load_jump_mgr_regs->load_jump_add2);
writel(0x04, &sdr_rw_load_mgr_regs->load_cntr3);
writel(RW_MGR_GUARANTEED_WRITE_WAIT3,
&sdr_rw_load_jump_mgr_regs->load_jump_add3);
writel(RW_MGR_GUARANTEED_WRITE, SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_RUN_SINGLE_GROUP_OFFSET);
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
}
set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
}
/*
* try a read and see if it returns correct data back. has dummy reads
* inserted into the mix used to align dqs enable. has more thorough checks
* than the regular read test.
*/
static uint32_t rw_mgr_mem_calibrate_read_test(uint32_t rank_bgn, uint32_t group,
uint32_t num_tries, uint32_t all_correct, uint32_t *bit_chk,
uint32_t all_groups, uint32_t all_ranks)
{
uint32_t r, vg;
uint32_t correct_mask_vg;
uint32_t tmp_bit_chk;
uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
(rank_bgn + NUM_RANKS_PER_SHADOW_REG);
uint32_t addr;
uint32_t base_rw_mgr;
*bit_chk = param->read_correct_mask;
correct_mask_vg = param->read_correct_mask_vg;
int ret;
uint32_t quick_read_mode = (((STATIC_CALIB_STEPS) &
CALIB_SKIP_DELAY_SWEEPS) && ENABLE_SUPER_QUICK_CALIBRATION);
for (r = rank_bgn; r < rank_end; r++) {
if (param->skip_ranks[r])
/* request to skip the rank */
continue;
/* set rank */
set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
writel(0x10, &sdr_rw_load_mgr_regs->load_cntr1);
writel(RW_MGR_READ_B2B_WAIT1,
&sdr_rw_load_jump_mgr_regs->load_jump_add1);
writel(0x10, &sdr_rw_load_mgr_regs->load_cntr2);
writel(RW_MGR_READ_B2B_WAIT2,
&sdr_rw_load_jump_mgr_regs->load_jump_add2);
if (quick_read_mode)
writel(0x1, &sdr_rw_load_mgr_regs->load_cntr0);
/* need at least two (1+1) reads to capture failures */
else if (all_groups)
writel(0x06, &sdr_rw_load_mgr_regs->load_cntr0);
writel(0x32, &sdr_rw_load_mgr_regs->load_cntr0);
writel(RW_MGR_READ_B2B,
&sdr_rw_load_jump_mgr_regs->load_jump_add0);
if (all_groups)
writel(RW_MGR_MEM_IF_READ_DQS_WIDTH *
RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS - 1,
&sdr_rw_load_mgr_regs->load_cntr3);
writel(0x0, &sdr_rw_load_mgr_regs->load_cntr3);
writel(RW_MGR_READ_B2B,
&sdr_rw_load_jump_mgr_regs->load_jump_add3);
tmp_bit_chk = 0;
for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS-1; ; vg--) {
/* reset the fifos to get pointers to known state */
writel(0, &phy_mgr_cmd->fifo_reset);
writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_RESET_READ_DATAPATH_OFFSET);
tmp_bit_chk = tmp_bit_chk << (RW_MGR_MEM_DQ_PER_READ_DQS
/ RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS);
if (all_groups)
addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_ALL_GROUPS_OFFSET;
else
addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
writel(RW_MGR_READ_B2B, addr +
((group * RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS +
vg) << 2));
base_rw_mgr = readl(SDR_PHYGRP_RWMGRGRP_ADDRESS);
tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & ~(base_rw_mgr));
if (vg == 0)
break;
}
*bit_chk &= tmp_bit_chk;
}
addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
writel(RW_MGR_CLEAR_DQS_ENABLE, addr + (group << 2));
set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
ret = (*bit_chk == param->read_correct_mask);
debug_cond(DLEVEL == 2,
"%s:%d read_test(%u,ALL,%u) => (%u == %u) => %i\n",
__func__, __LINE__, group, all_groups, *bit_chk,
param->read_correct_mask, ret);
ret = (*bit_chk != 0x00);
debug_cond(DLEVEL == 2,
"%s:%d read_test(%u,ONE,%u) => (%u != %u) => %i\n",
__func__, __LINE__, group, all_groups, *bit_chk,
0, ret);
return ret;
/**
* rw_mgr_mem_calibrate_read_test_all_ranks() - Perform READ test on all ranks
* @grp: Read/Write group
* @num_tries: Number of retries of the test
* @all_correct: All bits must be correct in the mask
* @all_groups: Test all R/W groups
*
* Perform a READ test across all memory ranks.
*/
static int
rw_mgr_mem_calibrate_read_test_all_ranks(const u32 grp, const u32 num_tries,
const u32 all_correct,
const u32 all_groups)
u32 bit_chk;
return rw_mgr_mem_calibrate_read_test(0, grp, num_tries, all_correct,
&bit_chk, all_groups, 1);
/**
* rw_mgr_incr_vfifo() - Increase VFIFO value
* @grp: Read/Write group
*
* Increase VFIFO value.
*/
static void rw_mgr_incr_vfifo(const u32 grp)
writel(grp, &phy_mgr_cmd->inc_vfifo_hard_phy);
/**
* rw_mgr_decr_vfifo() - Decrease VFIFO value
* @grp: Read/Write group
*
* Decrease VFIFO value.
*/
static void rw_mgr_decr_vfifo(const u32 grp)
for (i = 0; i < VFIFO_SIZE - 1; i++)
/**
* find_vfifo_failing_read() - Push VFIFO to get a failing read
* @grp: Read/Write group
*
* Push VFIFO until a failing read happens.
*/
static int find_vfifo_failing_read(const u32 grp)
u32 v, ret, fail_cnt = 0;
for (v = 0; v < VFIFO_SIZE; v++) {
debug_cond(DLEVEL == 2, "%s:%d: vfifo %u\n",
__func__, __LINE__, v);
ret = rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,
PASS_ONE_BIT, 0);
fail_cnt++;
if (fail_cnt == 2)
/* No failing read found! Something must have gone wrong. */
debug_cond(DLEVEL == 2, "%s:%d: vfifo failed\n", __func__, __LINE__);
return 0;
/**
* sdr_find_phase_delay() - Find DQS enable phase or delay
* @working: If 1, look for working phase/delay, if 0, look for non-working
* @delay: If 1, look for delay, if 0, look for phase
* @grp: Read/Write group
* @work: Working window position
* @work_inc: Working window increment
* @pd: DQS Phase/Delay Iterator
*
* Find working or non-working DQS enable phase setting.
*/
static int sdr_find_phase_delay(int working, int delay, const u32 grp,
u32 *work, const u32 work_inc, u32 *pd)
{
const u32 max = delay ? IO_DQS_EN_DELAY_MAX : IO_DQS_EN_PHASE_MAX;
u32 ret;
for (; *pd <= max; (*pd)++) {
if (delay)
scc_mgr_set_dqs_en_delay_all_ranks(grp, *pd);
else
scc_mgr_set_dqs_en_phase_all_ranks(grp, *pd);
ret = rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,
PASS_ONE_BIT, 0);
if (!working)
ret = !ret;
if (ret)
return 0;
if (work)
*work += work_inc;
}
return -EINVAL;
}
/**
* sdr_find_phase() - Find DQS enable phase
* @working: If 1, look for working phase, if 0, look for non-working phase
* @grp: Read/Write group
* @work: Working window position
* @i: Iterator
* @p: DQS Phase Iterator
*
* Find working or non-working DQS enable phase setting.
*/
static int sdr_find_phase(int working, const u32 grp, u32 *work,
const u32 end = VFIFO_SIZE + (working ? 0 : 1);
int ret;
for (; *i < end; (*i)++) {
if (working)
*p = 0;
ret = sdr_find_phase_delay(working, 0, grp, work,
IO_DELAY_PER_OPA_TAP, p);
if (!ret)
return 0;
if (*p > IO_DQS_EN_PHASE_MAX) {
/* Fiddle with FIFO. */
/**
* sdr_working_phase() - Find working DQS enable phase
* @grp: Read/Write group
* @work_bgn: Working window start position
* @d: dtaps output value
* @p: DQS Phase Iterator
* @i: Iterator
*
* Find working DQS enable phase setting.
*/
static int sdr_working_phase(const u32 grp, u32 *work_bgn, u32 *d,
const u32 dtaps_per_ptap = IO_DELAY_PER_OPA_TAP /
IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
int ret;
*work_bgn = 0;
for (*d = 0; *d <= dtaps_per_ptap; (*d)++) {
*i = 0;
scc_mgr_set_dqs_en_delay_all_ranks(grp, *d);
ret = sdr_find_phase(1, grp, work_bgn, i, p);
if (!ret)
return 0;
*work_bgn += IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
}
/* Cannot find working solution */
debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: no vfifo/ptap/dtap\n",
__func__, __LINE__);
return -EINVAL;
/**
* sdr_backup_phase() - Find DQS enable backup phase
* @grp: Read/Write group
* @work_bgn: Working window start position
* @p: DQS Phase Iterator
*
* Find DQS enable backup phase setting.
*/
static void sdr_backup_phase(const u32 grp, u32 *work_bgn, u32 *p)
u32 tmp_delay, d;
/* Special case code for backing up a phase */
if (*p == 0) {
*p = IO_DQS_EN_PHASE_MAX;
} else {
(*p)--;
}
tmp_delay = *work_bgn - IO_DELAY_PER_OPA_TAP;
scc_mgr_set_dqs_en_phase_all_ranks(grp, *p);
for (d = 0; d <= IO_DQS_EN_DELAY_MAX && tmp_delay < *work_bgn; d++) {
scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
ret = rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,
PASS_ONE_BIT, 0);
*work_bgn = tmp_delay;
break;
}
tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
/* Restore VFIFO to old state before we decremented it (if needed). */
(*p)++;
if (*p > IO_DQS_EN_PHASE_MAX) {
*p = 0;
scc_mgr_set_dqs_en_delay_all_ranks(grp, 0);
/**
* sdr_nonworking_phase() - Find non-working DQS enable phase
* @grp: Read/Write group
* @work_end: Working window end position
* @p: DQS Phase Iterator
* @i: Iterator
*
* Find non-working DQS enable phase setting.
*/
static int sdr_nonworking_phase(const u32 grp, u32 *work_end, u32 *p, u32 *i)
(*p)++;
*work_end += IO_DELAY_PER_OPA_TAP;
if (*p > IO_DQS_EN_PHASE_MAX) {
ret = sdr_find_phase(0, grp, work_end, i, p);
if (ret) {
/* Cannot see edge of failing read. */
debug_cond(DLEVEL == 2, "%s:%d: end: failed\n",
__func__, __LINE__);
/**
* sdr_find_window_center() - Find center of the working DQS window.
* @grp: Read/Write group
* @work_bgn: First working settings
* @work_end: Last working settings
*
* Find center of the working DQS enable window.
*/
static int sdr_find_window_center(const u32 grp, const u32 work_bgn,
u32 work_mid;
work_mid = (work_bgn + work_end) / 2;
debug_cond(DLEVEL == 2, "work_bgn=%d work_end=%d work_mid=%d\n",
work_bgn, work_end, work_mid);
/* Get the middle delay to be less than a VFIFO delay */
tmp_delay = (IO_DQS_EN_PHASE_MAX + 1) * IO_DELAY_PER_OPA_TAP;
debug_cond(DLEVEL == 2, "vfifo ptap delay %d\n", tmp_delay);
work_mid %= tmp_delay;
debug_cond(DLEVEL == 2, "new work_mid %d\n", work_mid);
tmp_delay = rounddown(work_mid, IO_DELAY_PER_OPA_TAP);
if (tmp_delay > IO_DQS_EN_PHASE_MAX * IO_DELAY_PER_OPA_TAP)
tmp_delay = IO_DQS_EN_PHASE_MAX * IO_DELAY_PER_OPA_TAP;
p = tmp_delay / IO_DELAY_PER_OPA_TAP;
debug_cond(DLEVEL == 2, "new p %d, tmp_delay=%d\n", p, tmp_delay);
d = DIV_ROUND_UP(work_mid - tmp_delay, IO_DELAY_PER_DQS_EN_DCHAIN_TAP);
if (d > IO_DQS_EN_DELAY_MAX)
d = IO_DQS_EN_DELAY_MAX;
tmp_delay += d * IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
debug_cond(DLEVEL == 2, "new d %d, tmp_delay=%d\n", d, tmp_delay);
scc_mgr_set_dqs_en_phase_all_ranks(grp, p);
scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
/*
* push vfifo until we can successfully calibrate. We can do this
* because the largest possible margin in 1 VFIFO cycle.
*/
for (i = 0; i < VFIFO_SIZE; i++) {
debug_cond(DLEVEL == 2, "find_dqs_en_phase: center\n");
if (rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,
debug_cond(DLEVEL == 2,
"%s:%d center: found: ptap=%u dtap=%u\n",
__func__, __LINE__, p, d);
/* Fiddle with FIFO. */
debug_cond(DLEVEL == 2, "%s:%d center: failed.\n",
__func__, __LINE__);
return -EINVAL;
/**
* rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase() - Find a good DQS enable to use
* @grp: Read/Write Group
*
* Find a good DQS enable to use.
*/
static int rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(const u32 grp)
u32 d, p, i;
u32 dtaps_per_ptap;
u32 work_bgn, work_end;
u32 found_passing_read, found_failing_read, initial_failing_dtap;
int ret;
debug("%s:%d %u\n", __func__, __LINE__, grp);
reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER);
scc_mgr_set_dqs_en_delay_all_ranks(grp, 0);
scc_mgr_set_dqs_en_phase_all_ranks(grp, 0);
/* Step 0: Determine number of delay taps for each phase tap. */
dtaps_per_ptap = IO_DELAY_PER_OPA_TAP / IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
/* Step 1: First push vfifo until we get a failing read. */
/* Step 2: Find first working phase, increment in ptaps. */
ret = sdr_working_phase(grp, &work_bgn, &d, &p, &i);
if (ret)
return ret;
work_end = work_bgn;
/*
* If d is 0 then the working window covers a phase tap and we can
* follow the old procedure. Otherwise, we've found the beginning
* and we need to increment the dtaps until we find the end.
*/
if (d == 0) {
/*
* Step 3a: If we have room, back off by one and
* increment in dtaps.
*/
sdr_backup_phase(grp, &work_bgn, &p);
/*
* Step 4a: go forward from working phase to non working
* phase, increment in ptaps.
*/
ret = sdr_nonworking_phase(grp, &work_end, &p, &i);
if (ret)
return ret;
/* Step 5a: Back off one from last, increment in dtaps. */
/* Special case code for backing up a phase */
if (p == 0) {
p = IO_DQS_EN_PHASE_MAX;
} else {
p = p - 1;
}
work_end -= IO_DELAY_PER_OPA_TAP;
scc_mgr_set_dqs_en_phase_all_ranks(grp, p);
d = 0;
debug_cond(DLEVEL == 2, "%s:%d p: ptap=%u\n",
__func__, __LINE__, p);
/* The dtap increment to find the failing edge is done here. */
sdr_find_phase_delay(0, 1, grp, &work_end,
IO_DELAY_PER_DQS_EN_DCHAIN_TAP, &d);
/* Go back to working dtap */
if (d != 0)
work_end -= IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
debug_cond(DLEVEL == 2,
"%s:%d p/d: ptap=%u dtap=%u end=%u\n",
__func__, __LINE__, p, d - 1, work_end);
if (work_end < work_bgn) {
/* nil range */
debug_cond(DLEVEL == 2, "%s:%d end-2: failed\n",
__func__, __LINE__);
return -EINVAL;
debug_cond(DLEVEL == 2, "%s:%d found range [%u,%u]\n",
__func__, __LINE__, work_bgn, work_end);
/*
* We need to calculate the number of dtaps that equal a ptap.
* To do that we'll back up a ptap and re-find the edge of the
* window using dtaps
debug_cond(DLEVEL == 2, "%s:%d calculate dtaps_per_ptap for tracking\n",
__func__, __LINE__);
/* Special case code for backing up a phase */
if (p == 0) {
p = IO_DQS_EN_PHASE_MAX;
debug_cond(DLEVEL == 2, "%s:%d backedup cycle/phase: p=%u\n",
__func__, __LINE__, p);
} else {
p = p - 1;
debug_cond(DLEVEL == 2, "%s:%d backedup phase only: p=%u",
__func__, __LINE__, p);
}
scc_mgr_set_dqs_en_phase_all_ranks(grp, p);
/*
* Increase dtap until we first see a passing read (in case the
* window is smaller than a ptap), and then a failing read to
* mark the edge of the window again.
/* Find a passing read. */
debug_cond(DLEVEL == 2, "%s:%d find passing read\n",
__func__, __LINE__);
initial_failing_dtap = d;
found_passing_read = !sdr_find_phase_delay(1, 1, grp, NULL, 0, &d);
if (found_passing_read) {
/* Find a failing read. */
debug_cond(DLEVEL == 2, "%s:%d find failing read\n",
__func__, __LINE__);
d++;
found_failing_read = !sdr_find_phase_delay(0, 1, grp, NULL, 0,
&d);
debug_cond(DLEVEL == 1,
"%s:%d failed to calculate dtaps per ptap. Fall back on static value\n",
__func__, __LINE__);
}
/*
* The dynamically calculated dtaps_per_ptap is only valid if we
* found a passing/failing read. If we didn't, it means d hit the max
* (IO_DQS_EN_DELAY_MAX). Otherwise, dtaps_per_ptap retains its
* statically calculated value.
*/
if (found_passing_read && found_failing_read)
dtaps_per_ptap = d - initial_failing_dtap;
writel(dtaps_per_ptap, &sdr_reg_file->dtaps_per_ptap);
debug_cond(DLEVEL == 2, "%s:%d dtaps_per_ptap=%u - %u = %u",
__func__, __LINE__, d, initial_failing_dtap, dtaps_per_ptap);
/* Step 6: Find the centre of the window. */
ret = sdr_find_window_center(grp, work_bgn, work_end);
return ret;
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
}
/* per-bit deskew DQ and center */
static uint32_t rw_mgr_mem_calibrate_vfifo_center(uint32_t rank_bgn,
uint32_t write_group, uint32_t read_group, uint32_t test_bgn,
uint32_t use_read_test, uint32_t update_fom)
{
uint32_t i, p, d, min_index;
/*
* Store these as signed since there are comparisons with
* signed numbers.
*/
uint32_t bit_chk;
uint32_t sticky_bit_chk;
int32_t left_edge[RW_MGR_MEM_DQ_PER_READ_DQS];
int32_t right_edge[RW_MGR_MEM_DQ_PER_READ_DQS];
int32_t final_dq[RW_MGR_MEM_DQ_PER_READ_DQS];
int32_t mid;
int32_t orig_mid_min, mid_min;
int32_t new_dqs, start_dqs, start_dqs_en, shift_dq, final_dqs,
final_dqs_en;
int32_t dq_margin, dqs_margin;
uint32_t stop;
uint32_t temp_dq_in_delay1, temp_dq_in_delay2;
uint32_t addr;
debug("%s:%d: %u %u", __func__, __LINE__, read_group, test_bgn);
addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_DQS_IN_DELAY_OFFSET;
start_dqs = readl(addr + (read_group << 2));
if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS)
start_dqs_en = readl(addr + ((read_group << 2)
- IO_DQS_EN_DELAY_OFFSET));
/* set the left and right edge of each bit to an illegal value */
/* use (IO_IO_IN_DELAY_MAX + 1) as an illegal value */
sticky_bit_chk = 0;
for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
left_edge[i] = IO_IO_IN_DELAY_MAX + 1;
right_edge[i] = IO_IO_IN_DELAY_MAX + 1;
}
/* Search for the left edge of the window for each bit */
for (d = 0; d <= IO_IO_IN_DELAY_MAX; d++) {
scc_mgr_apply_group_dq_in_delay(write_group, test_bgn, d);
writel(0, &sdr_scc_mgr->update);
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
/*
* Stop searching when the read test doesn't pass AND when
* we've seen a passing read on every bit.
*/
if (use_read_test) {
stop = !rw_mgr_mem_calibrate_read_test(rank_bgn,
read_group, NUM_READ_PB_TESTS, PASS_ONE_BIT,
&bit_chk, 0, 0);
} else {
rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
0, PASS_ONE_BIT,
&bit_chk, 0);
bit_chk = bit_chk >> (RW_MGR_MEM_DQ_PER_READ_DQS *
(read_group - (write_group *
RW_MGR_MEM_IF_READ_DQS_WIDTH /
RW_MGR_MEM_IF_WRITE_DQS_WIDTH)));
stop = (bit_chk == 0);
}
sticky_bit_chk = sticky_bit_chk | bit_chk;
stop = stop && (sticky_bit_chk == param->read_correct_mask);
debug_cond(DLEVEL == 2, "%s:%d vfifo_center(left): dtap=%u => %u == %u \
&& %u", __func__, __LINE__, d,
sticky_bit_chk,
param->read_correct_mask, stop);
if (stop == 1) {
break;
} else {
for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
if (bit_chk & 1) {
/* Remember a passing test as the
left_edge */
left_edge[i] = d;
} else {
/* If a left edge has not been seen yet,
then a future passing test will mark
this edge as the right edge */
if (left_edge[i] ==
IO_IO_IN_DELAY_MAX + 1) {
right_edge[i] = -(d + 1);
}
}
bit_chk = bit_chk >> 1;
}
}
}
/* Reset DQ delay chains to 0 */
scc_mgr_apply_group_dq_in_delay(test_bgn, 0);
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
sticky_bit_chk = 0;
for (i = RW_MGR_MEM_DQ_PER_READ_DQS - 1;; i--) {
debug_cond(DLEVEL == 2, "%s:%d vfifo_center: left_edge[%u]: \
%d right_edge[%u]: %d\n", __func__, __LINE__,
i, left_edge[i], i, right_edge[i]);
/*
* Check for cases where we haven't found the left edge,
* which makes our assignment of the the right edge invalid.
* Reset it to the illegal value.
*/
if ((left_edge[i] == IO_IO_IN_DELAY_MAX + 1) && (
right_edge[i] != IO_IO_IN_DELAY_MAX + 1)) {
right_edge[i] = IO_IO_IN_DELAY_MAX + 1;
debug_cond(DLEVEL == 2, "%s:%d vfifo_center: reset \
right_edge[%u]: %d\n", __func__, __LINE__,
i, right_edge[i]);
}
/*
* Reset sticky bit (except for bits where we have seen
* both the left and right edge).
*/
sticky_bit_chk = sticky_bit_chk << 1;
if ((left_edge[i] != IO_IO_IN_DELAY_MAX + 1) &&
(right_edge[i] != IO_IO_IN_DELAY_MAX + 1)) {
sticky_bit_chk = sticky_bit_chk | 1;
}
if (i == 0)
break;
}
/* Search for the right edge of the window for each bit */
for (d = 0; d <= IO_DQS_IN_DELAY_MAX - start_dqs; d++) {
scc_mgr_set_dqs_bus_in_delay(read_group, d + start_dqs);
if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
uint32_t delay = d + start_dqs_en;
if (delay > IO_DQS_EN_DELAY_MAX)
delay = IO_DQS_EN_DELAY_MAX;
scc_mgr_set_dqs_en_delay(read_group, delay);
}
scc_mgr_load_dqs(read_group);
writel(0, &sdr_scc_mgr->update);
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
/*
* Stop searching when the read test doesn't pass AND when
* we've seen a passing read on every bit.
*/
if (use_read_test) {
stop = !rw_mgr_mem_calibrate_read_test(rank_bgn,
read_group, NUM_READ_PB_TESTS, PASS_ONE_BIT,
&bit_chk, 0, 0);
} else {
rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
0, PASS_ONE_BIT,
&bit_chk, 0);
bit_chk = bit_chk >> (RW_MGR_MEM_DQ_PER_READ_DQS *
(read_group - (write_group *
RW_MGR_MEM_IF_READ_DQS_WIDTH /
RW_MGR_MEM_IF_WRITE_DQS_WIDTH)));
stop = (bit_chk == 0);
}
sticky_bit_chk = sticky_bit_chk | bit_chk;
stop = stop && (sticky_bit_chk == param->read_correct_mask);
debug_cond(DLEVEL == 2, "%s:%d vfifo_center(right): dtap=%u => %u == \
%u && %u", __func__, __LINE__, d,
sticky_bit_chk, param->read_correct_mask, stop);
if (stop == 1) {
break;
} else {
for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
if (bit_chk & 1) {
/* Remember a passing test as
the right_edge */
right_edge[i] = d;
} else {
if (d != 0) {
/* If a right edge has not been
seen yet, then a future passing
test will mark this edge as the
left edge */
if (right_edge[i] ==
IO_IO_IN_DELAY_MAX + 1) {
left_edge[i] = -(d + 1);
}
} else {
/* d = 0 failed, but it passed
when testing the left edge,
so it must be marginal,
set it to -1 */
if (right_edge[i] ==
IO_IO_IN_DELAY_MAX + 1 &&
left_edge[i] !=
IO_IO_IN_DELAY_MAX
+ 1) {
right_edge[i] = -1;
}
/* If a right edge has not been
seen yet, then a future passing
test will mark this edge as the
left edge */
else if (right_edge[i] ==
IO_IO_IN_DELAY_MAX +
1) {
left_edge[i] = -(d + 1);
}
}
}
debug_cond(DLEVEL == 2, "%s:%d vfifo_center[r,\
d=%u]: ", __func__, __LINE__, d);
debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d ",
(int)(bit_chk & 1), i, left_edge[i]);
debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
right_edge[i]);
bit_chk = bit_chk >> 1;
}
}
}
/* Check that all bits have a window */
for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
debug_cond(DLEVEL == 2, "%s:%d vfifo_center: left_edge[%u]: \
%d right_edge[%u]: %d", __func__, __LINE__,
i, left_edge[i], i, right_edge[i]);
if ((left_edge[i] == IO_IO_IN_DELAY_MAX + 1) || (right_edge[i]
== IO_IO_IN_DELAY_MAX + 1)) {
/*
* Restore delay chain settings before letting the loop
* in rw_mgr_mem_calibrate_vfifo to retry different
* dqs/ck relationships.
*/
scc_mgr_set_dqs_bus_in_delay(read_group, start_dqs);
if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
scc_mgr_set_dqs_en_delay(read_group,
start_dqs_en);
}
scc_mgr_load_dqs(read_group);
writel(0, &sdr_scc_mgr->update);
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
debug_cond(DLEVEL == 1, "%s:%d vfifo_center: failed to \
find edge [%u]: %d %d", __func__, __LINE__,
i, left_edge[i], right_edge[i]);
if (use_read_test) {
set_failing_group_stage(read_group *
RW_MGR_MEM_DQ_PER_READ_DQS + i,
CAL_STAGE_VFIFO,
CAL_SUBSTAGE_VFIFO_CENTER);
} else {
set_failing_group_stage(read_group *
RW_MGR_MEM_DQ_PER_READ_DQS + i,
CAL_STAGE_VFIFO_AFTER_WRITES,
CAL_SUBSTAGE_VFIFO_CENTER);
}
return 0;
}
}
/* Find middle of window for each DQ bit */
mid_min = left_edge[0] - right_edge[0];
min_index = 0;
for (i = 1; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
mid = left_edge[i] - right_edge[i];
if (mid < mid_min) {
mid_min = mid;
min_index = i;
}
}
/*
* -mid_min/2 represents the amount that we need to move DQS.
* If mid_min is odd and positive we'll need to add one to
* make sure the rounding in further calculations is correct
* (always bias to the right), so just add 1 for all positive values.
*/
if (mid_min > 0)
mid_min++;
mid_min = mid_min / 2;
debug_cond(DLEVEL == 1, "%s:%d vfifo_center: mid_min=%d (index=%u)\n",
__func__, __LINE__, mid_min, min_index);
/* Determine the amount we can change DQS (which is -mid_min) */
orig_mid_min = mid_min;
new_dqs = start_dqs - mid_min;
if (new_dqs > IO_DQS_IN_DELAY_MAX)
new_dqs = IO_DQS_IN_DELAY_MAX;
else if (new_dqs < 0)
new_dqs = 0;
mid_min = start_dqs - new_dqs;
debug_cond(DLEVEL == 1, "vfifo_center: new mid_min=%d new_dqs=%d\n",
mid_min, new_dqs);
if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
if (start_dqs_en - mid_min > IO_DQS_EN_DELAY_MAX)
mid_min += start_dqs_en - mid_min - IO_DQS_EN_DELAY_MAX;
else if (start_dqs_en - mid_min < 0)
mid_min += start_dqs_en - mid_min;
}
new_dqs = start_dqs - mid_min;
debug_cond(DLEVEL == 1, "vfifo_center: start_dqs=%d start_dqs_en=%d \
new_dqs=%d mid_min=%d\n", start_dqs,
IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS ? start_dqs_en : -1,
new_dqs, mid_min);
/* Initialize data for export structures */
dqs_margin = IO_IO_IN_DELAY_MAX + 1;
dq_margin = IO_IO_IN_DELAY_MAX + 1;
/* add delay to bring centre of all DQ windows to the same "level" */
for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) {
/* Use values before divide by 2 to reduce round off error */
shift_dq = (left_edge[i] - right_edge[i] -
(left_edge[min_index] - right_edge[min_index]))/2 +
(orig_mid_min - mid_min);
debug_cond(DLEVEL == 2, "vfifo_center: before: \
shift_dq[%u]=%d\n", i, shift_dq);
addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_IN_DELAY_OFFSET;
temp_dq_in_delay1 = readl(addr + (p << 2));
temp_dq_in_delay2 = readl(addr + (i << 2));
if (shift_dq + (int32_t)temp_dq_in_delay1 >
(int32_t)IO_IO_IN_DELAY_MAX) {
shift_dq = (int32_t)IO_IO_IN_DELAY_MAX - temp_dq_in_delay2;
} else if (shift_dq + (int32_t)temp_dq_in_delay1 < 0) {
shift_dq = -(int32_t)temp_dq_in_delay1;
}
debug_cond(DLEVEL == 2, "vfifo_center: after: \
shift_dq[%u]=%d\n", i, shift_dq);
final_dq[i] = temp_dq_in_delay1 + shift_dq;
scc_mgr_set_dq_in_delay(p, final_dq[i]);
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
scc_mgr_load_dq(p);
debug_cond(DLEVEL == 2, "vfifo_center: margin[%u]=[%d,%d]\n", i,
left_edge[i] - shift_dq + (-mid_min),
right_edge[i] + shift_dq - (-mid_min));
/* To determine values for export structures */
if (left_edge[i] - shift_dq + (-mid_min) < dq_margin)
dq_margin = left_edge[i] - shift_dq + (-mid_min);
if (right_edge[i] + shift_dq - (-mid_min) < dqs_margin)
dqs_margin = right_edge[i] + shift_dq - (-mid_min);
}
final_dqs = new_dqs;
if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS)
final_dqs_en = start_dqs_en - mid_min;
/* Move DQS-en */
if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
scc_mgr_set_dqs_en_delay(read_group, final_dqs_en);
scc_mgr_load_dqs(read_group);
}
/* Move DQS */
scc_mgr_set_dqs_bus_in_delay(read_group, final_dqs);
scc_mgr_load_dqs(read_group);
debug_cond(DLEVEL == 2, "%s:%d vfifo_center: dq_margin=%d \
dqs_margin=%d", __func__, __LINE__,
dq_margin, dqs_margin);
/*
* Do not remove this line as it makes sure all of our decisions
* have been applied. Apply the update bit.
*/
writel(0, &sdr_scc_mgr->update);
return (dq_margin >= 0) && (dqs_margin >= 0);
}
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
/**
* rw_mgr_mem_calibrate_guaranteed_write() - Perform guaranteed write into the device
* @rw_group: Read/Write Group
* @phase: DQ/DQS phase
*
* Because initially no communication ca be reliably performed with the memory
* device, the sequencer uses a guaranteed write mechanism to write data into
* the memory device.
*/
static int rw_mgr_mem_calibrate_guaranteed_write(const u32 rw_group,
const u32 phase)
{
int ret;
/* Set a particular DQ/DQS phase. */
scc_mgr_set_dqdqs_output_phase_all_ranks(rw_group, phase);
debug_cond(DLEVEL == 1, "%s:%d guaranteed write: g=%u p=%u\n",
__func__, __LINE__, rw_group, phase);
/*
* Altera EMI_RM 2015.05.04 :: Figure 1-25
* Load up the patterns used by read calibration using the
* current DQDQS phase.
*/
rw_mgr_mem_calibrate_read_load_patterns(0, 1);
if (gbl->phy_debug_mode_flags & PHY_DEBUG_DISABLE_GUARANTEED_READ)
return 0;
/*
* Altera EMI_RM 2015.05.04 :: Figure 1-26
* Back-to-Back reads of the patterns used for calibration.
*/
ret = rw_mgr_mem_calibrate_read_test_patterns(0, rw_group, 1);
if (ret)
debug_cond(DLEVEL == 1,
"%s:%d Guaranteed read test failed: g=%u p=%u\n",
__func__, __LINE__, rw_group, phase);
return ret;
}
/**
* rw_mgr_mem_calibrate_dqs_enable_calibration() - DQS Enable Calibration
* @rw_group: Read/Write Group
* @test_bgn: Rank at which the test begins
*
* DQS enable calibration ensures reliable capture of the DQ signal without
* glitches on the DQS line.
*/
static int rw_mgr_mem_calibrate_dqs_enable_calibration(const u32 rw_group,
const u32 test_bgn)
{
/*
Loading
Loading full blame…