Newer
Older
*/
if (right_edge[i] == delay_max + 1 &&
left_edge[i] != delay_max + 1)
right_edge[i] = -1;
}
}
break;
}
/* stop != 1 */
for (i = 0; i < per_dqs; i++) {
if (bit_chk & 1) {
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
/*
* Remember a passing test as
* the right_edge.
*/
right_edge[i] = d;
} else {
if (d != 0) {
/*
* If a right edge has not
* been seen yet, then a future
* passing test will mark this
* edge as the left edge.
*/
if (right_edge[i] == delay_max + 1)
left_edge[i] = -(d + 1);
} else {
/*
* d = 0 failed, but it passed
* when testing the left edge,
* so it must be marginal, set
* it to -1
*/
if (right_edge[i] == delay_max + 1 &&
left_edge[i] != delay_max + 1)
right_edge[i] = -1;
/*
* If a right edge has not been
* seen yet, then a future
* passing test will mark this
* edge as the left edge.
*/
else if (right_edge[i] == delay_max + 1)
left_edge[i] = -(d + 1);
}
}
debug_cond(DLEVEL == 2, "%s:%d center[r,d=%u]: ",
__func__, __LINE__, d);
debug_cond(DLEVEL == 2,
"bit_chk_test=%i left_edge[%u]: %d ",
bit_chk & 1, i, left_edge[i]);
debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
right_edge[i]);
bit_chk >>= 1;
}
}
/* Check that all bits have a window */
for (i = 0; i < per_dqs; i++) {
debug_cond(DLEVEL == 2,
"%s:%d write_center: left_edge[%u]: %d right_edge[%u]: %d",
__func__, __LINE__, i, left_edge[i],
i, right_edge[i]);
if ((left_edge[i] == dqs_max + 1) ||
(right_edge[i] == dqs_max + 1))
return i + 1; /* FIXME: If we fail, retval > 0 */
}
return 0;
}
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
/**
* get_window_mid_index() - Find the best middle setting of DQ/DQS phase
* @write: Perform read (Stage 2) or write (Stage 3) calibration
* @left_edge: Left edge of the DQ/DQS phase
* @right_edge: Right edge of the DQ/DQS phase
* @mid_min: Best DQ/DQS phase middle setting
*
* Find index and value of the middle of the DQ/DQS working phase.
*/
static int get_window_mid_index(const int write, int *left_edge,
int *right_edge, int *mid_min)
{
const u32 per_dqs = write ? RW_MGR_MEM_DQ_PER_WRITE_DQS :
RW_MGR_MEM_DQ_PER_READ_DQS;
int i, mid, min_index;
/* Find middle of window for each DQ bit */
*mid_min = left_edge[0] - right_edge[0];
min_index = 0;
for (i = 1; i < per_dqs; i++) {
mid = left_edge[i] - right_edge[i];
if (mid < *mid_min) {
*mid_min = mid;
min_index = i;
}
}
/*
* -mid_min/2 represents the amount that we need to move DQS.
* If mid_min is odd and positive we'll need to add one to make
* sure the rounding in further calculations is correct (always
* bias to the right), so just add 1 for all positive values.
*/
if (*mid_min > 0)
(*mid_min)++;
*mid_min = *mid_min / 2;
debug_cond(DLEVEL == 1, "%s:%d vfifo_center: *mid_min=%d (index=%u)\n",
__func__, __LINE__, *mid_min, min_index);
return min_index;
}
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
/**
* center_dq_windows() - Center the DQ/DQS windows
* @write: Perform read (Stage 2) or write (Stage 3) calibration
* @left_edge: Left edge of the DQ/DQS phase
* @right_edge: Right edge of the DQ/DQS phase
* @mid_min: Adjusted DQ/DQS phase middle setting
* @orig_mid_min: Original DQ/DQS phase middle setting
* @min_index: DQ/DQS phase middle setting index
* @test_bgn: Rank number to begin the test
* @dq_margin: Amount of shift for the DQ
* @dqs_margin: Amount of shift for the DQS
*
* Align the DQ/DQS windows in each group.
*/
static void center_dq_windows(const int write, int *left_edge, int *right_edge,
const int mid_min, const int orig_mid_min,
const int min_index, const int test_bgn,
int *dq_margin, int *dqs_margin)
{
const u32 delay_max = write ? IO_IO_OUT1_DELAY_MAX : IO_IO_IN_DELAY_MAX;
const u32 per_dqs = write ? RW_MGR_MEM_DQ_PER_WRITE_DQS :
RW_MGR_MEM_DQ_PER_READ_DQS;
const u32 delay_off = write ? SCC_MGR_IO_OUT1_DELAY_OFFSET :
SCC_MGR_IO_IN_DELAY_OFFSET;
const u32 addr = SDR_PHYGRP_SCCGRP_ADDRESS | delay_off;
u32 temp_dq_io_delay1, temp_dq_io_delay2;
int shift_dq, i, p;
/* Initialize data for export structures */
*dqs_margin = delay_max + 1;
*dq_margin = delay_max + 1;
/* add delay to bring centre of all DQ windows to the same "level" */
for (i = 0, p = test_bgn; i < per_dqs; i++, p++) {
/* Use values before divide by 2 to reduce round off error */
shift_dq = (left_edge[i] - right_edge[i] -
(left_edge[min_index] - right_edge[min_index]))/2 +
(orig_mid_min - mid_min);
debug_cond(DLEVEL == 2,
"vfifo_center: before: shift_dq[%u]=%d\n",
i, shift_dq);
temp_dq_io_delay1 = readl(addr + (p << 2));
temp_dq_io_delay2 = readl(addr + (i << 2));
if (shift_dq + temp_dq_io_delay1 > delay_max)
shift_dq = delay_max - temp_dq_io_delay2;
else if (shift_dq + temp_dq_io_delay1 < 0)
shift_dq = -temp_dq_io_delay1;
debug_cond(DLEVEL == 2,
"vfifo_center: after: shift_dq[%u]=%d\n",
i, shift_dq);
if (write)
scc_mgr_set_dq_out1_delay(i, temp_dq_io_delay1 + shift_dq);
else
scc_mgr_set_dq_in_delay(p, temp_dq_io_delay1 + shift_dq);
scc_mgr_load_dq(p);
debug_cond(DLEVEL == 2,
"vfifo_center: margin[%u]=[%d,%d]\n", i,
left_edge[i] - shift_dq + (-mid_min),
right_edge[i] + shift_dq - (-mid_min));
/* To determine values for export structures */
if (left_edge[i] - shift_dq + (-mid_min) < *dq_margin)
*dq_margin = left_edge[i] - shift_dq + (-mid_min);
if (right_edge[i] + shift_dq - (-mid_min) < *dqs_margin)
*dqs_margin = right_edge[i] + shift_dq - (-mid_min);
}
}
/**
* rw_mgr_mem_calibrate_vfifo_center() - Per-bit deskew DQ and centering
* @rank_bgn: Rank number
* @rw_group: Read/Write Group
* @test_bgn: Rank at which the test begins
* @use_read_test: Perform a read test
* @update_fom: Update FOM
*
* Per-bit deskew DQ and centering.
*/
static int rw_mgr_mem_calibrate_vfifo_center(const u32 rank_bgn,
const u32 rw_group, const u32 test_bgn,
const int use_read_test, const int update_fom)
const u32 addr =
SDR_PHYGRP_SCCGRP_ADDRESS + SCC_MGR_DQS_IN_DELAY_OFFSET +
(rw_group << 2);
/*
* Store these as signed since there are comparisons with
* signed numbers.
*/
uint32_t sticky_bit_chk;
int32_t left_edge[RW_MGR_MEM_DQ_PER_READ_DQS];
int32_t right_edge[RW_MGR_MEM_DQ_PER_READ_DQS];
int32_t orig_mid_min, mid_min;
int32_t new_dqs, start_dqs, start_dqs_en, final_dqs_en;
int32_t dq_margin, dqs_margin;
int i, min_index;
int ret;
debug("%s:%d: %u %u", __func__, __LINE__, rw_group, test_bgn);
start_dqs = readl(addr);
if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS)
start_dqs_en = readl(addr - IO_DQS_EN_DELAY_OFFSET);
/* set the left and right edge of each bit to an illegal value */
/* use (IO_IO_IN_DELAY_MAX + 1) as an illegal value */
sticky_bit_chk = 0;
for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
left_edge[i] = IO_IO_IN_DELAY_MAX + 1;
right_edge[i] = IO_IO_IN_DELAY_MAX + 1;
}
/* Search for the left edge of the window for each bit */
search_left_edge(0, rank_bgn, rw_group, rw_group, test_bgn,
&sticky_bit_chk,
left_edge, right_edge, use_read_test);
/* Search for the right edge of the window for each bit */
ret = search_right_edge(0, rank_bgn, rw_group, rw_group,
start_dqs, start_dqs_en,
&sticky_bit_chk,
left_edge, right_edge, use_read_test);
if (ret) {
* Restore delay chain settings before letting the loop
* in rw_mgr_mem_calibrate_vfifo to retry different
* dqs/ck relationships.
scc_mgr_set_dqs_bus_in_delay(rw_group, start_dqs);
if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS)
scc_mgr_set_dqs_en_delay(rw_group, start_dqs_en);
scc_mgr_load_dqs(rw_group);
writel(0, &sdr_scc_mgr->update);
debug_cond(DLEVEL == 1,
"%s:%d vfifo_center: failed to find edge [%u]: %d %d",
__func__, __LINE__, i, left_edge[i], right_edge[i]);
if (use_read_test) {
set_failing_group_stage(rw_group *
RW_MGR_MEM_DQ_PER_READ_DQS + i,
CAL_STAGE_VFIFO,
CAL_SUBSTAGE_VFIFO_CENTER);
set_failing_group_stage(rw_group *
RW_MGR_MEM_DQ_PER_READ_DQS + i,
CAL_STAGE_VFIFO_AFTER_WRITES,
CAL_SUBSTAGE_VFIFO_CENTER);
return -EIO;
min_index = get_window_mid_index(0, left_edge, right_edge, &mid_min);
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
/* Determine the amount we can change DQS (which is -mid_min) */
orig_mid_min = mid_min;
new_dqs = start_dqs - mid_min;
if (new_dqs > IO_DQS_IN_DELAY_MAX)
new_dqs = IO_DQS_IN_DELAY_MAX;
else if (new_dqs < 0)
new_dqs = 0;
mid_min = start_dqs - new_dqs;
debug_cond(DLEVEL == 1, "vfifo_center: new mid_min=%d new_dqs=%d\n",
mid_min, new_dqs);
if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
if (start_dqs_en - mid_min > IO_DQS_EN_DELAY_MAX)
mid_min += start_dqs_en - mid_min - IO_DQS_EN_DELAY_MAX;
else if (start_dqs_en - mid_min < 0)
mid_min += start_dqs_en - mid_min;
}
new_dqs = start_dqs - mid_min;
debug_cond(DLEVEL == 1,
"vfifo_center: start_dqs=%d start_dqs_en=%d new_dqs=%d mid_min=%d\n",
start_dqs,
IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS ? start_dqs_en : -1,
new_dqs, mid_min);
/* Add delay to bring centre of all DQ windows to the same "level". */
center_dq_windows(0, left_edge, right_edge, mid_min, orig_mid_min,
min_index, test_bgn, &dq_margin, &dqs_margin);
/* Move DQS-en */
if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
final_dqs_en = start_dqs_en - mid_min;
scc_mgr_set_dqs_en_delay(rw_group, final_dqs_en);
scc_mgr_load_dqs(rw_group);
}
/* Move DQS */
scc_mgr_set_dqs_bus_in_delay(rw_group, new_dqs);
scc_mgr_load_dqs(rw_group);
debug_cond(DLEVEL == 2,
"%s:%d vfifo_center: dq_margin=%d dqs_margin=%d",
__func__, __LINE__, dq_margin, dqs_margin);
/*
* Do not remove this line as it makes sure all of our decisions
* have been applied. Apply the update bit.
*/
writel(0, &sdr_scc_mgr->update);
if ((dq_margin < 0) || (dqs_margin < 0))
return -EINVAL;
return 0;
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
/**
* rw_mgr_mem_calibrate_guaranteed_write() - Perform guaranteed write into the device
* @rw_group: Read/Write Group
* @phase: DQ/DQS phase
*
* Because initially no communication ca be reliably performed with the memory
* device, the sequencer uses a guaranteed write mechanism to write data into
* the memory device.
*/
static int rw_mgr_mem_calibrate_guaranteed_write(const u32 rw_group,
const u32 phase)
{
int ret;
/* Set a particular DQ/DQS phase. */
scc_mgr_set_dqdqs_output_phase_all_ranks(rw_group, phase);
debug_cond(DLEVEL == 1, "%s:%d guaranteed write: g=%u p=%u\n",
__func__, __LINE__, rw_group, phase);
/*
* Altera EMI_RM 2015.05.04 :: Figure 1-25
* Load up the patterns used by read calibration using the
* current DQDQS phase.
*/
rw_mgr_mem_calibrate_read_load_patterns(0, 1);
if (gbl->phy_debug_mode_flags & PHY_DEBUG_DISABLE_GUARANTEED_READ)
return 0;
/*
* Altera EMI_RM 2015.05.04 :: Figure 1-26
* Back-to-Back reads of the patterns used for calibration.
*/
ret = rw_mgr_mem_calibrate_read_test_patterns(0, rw_group, 1);
if (ret)
debug_cond(DLEVEL == 1,
"%s:%d Guaranteed read test failed: g=%u p=%u\n",
__func__, __LINE__, rw_group, phase);
return ret;
}
/**
* rw_mgr_mem_calibrate_dqs_enable_calibration() - DQS Enable Calibration
* @rw_group: Read/Write Group
* @test_bgn: Rank at which the test begins
*
* DQS enable calibration ensures reliable capture of the DQ signal without
* glitches on the DQS line.
*/
static int rw_mgr_mem_calibrate_dqs_enable_calibration(const u32 rw_group,
const u32 test_bgn)
{
/*
* Altera EMI_RM 2015.05.04 :: Figure 1-27
* DQS and DQS Eanble Signal Relationships.
*/
Marek Vasut
committed
/* We start at zero, so have one less dq to devide among */
const u32 delay_step = IO_IO_IN_DELAY_MAX /
(RW_MGR_MEM_DQ_PER_READ_DQS - 1);
int ret;
Marek Vasut
committed
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
u32 i, p, d, r;
debug("%s:%d (%u,%u)\n", __func__, __LINE__, rw_group, test_bgn);
/* Try different dq_in_delays since the DQ path is shorter than DQS. */
for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
r += NUM_RANKS_PER_SHADOW_REG) {
for (i = 0, p = test_bgn, d = 0;
i < RW_MGR_MEM_DQ_PER_READ_DQS;
i++, p++, d += delay_step) {
debug_cond(DLEVEL == 1,
"%s:%d: g=%u r=%u i=%u p=%u d=%u\n",
__func__, __LINE__, rw_group, r, i, p, d);
scc_mgr_set_dq_in_delay(p, d);
scc_mgr_load_dq(p);
}
writel(0, &sdr_scc_mgr->update);
}
/*
* Try rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase across different
* dq_in_delay values
*/
ret = rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(rw_group);
Marek Vasut
committed
debug_cond(DLEVEL == 1,
"%s:%d: g=%u found=%u; Reseting delay chain to zero\n",
__func__, __LINE__, rw_group, !ret);
Marek Vasut
committed
for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
r += NUM_RANKS_PER_SHADOW_REG) {
scc_mgr_apply_group_dq_in_delay(test_bgn, 0);
writel(0, &sdr_scc_mgr->update);
}
return ret;
}
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
/**
* rw_mgr_mem_calibrate_dq_dqs_centering() - Centering DQ/DQS
* @rw_group: Read/Write Group
* @test_bgn: Rank at which the test begins
* @use_read_test: Perform a read test
* @update_fom: Update FOM
*
* The centerin DQ/DQS stage attempts to align DQ and DQS signals on reads
* within a group.
*/
static int
rw_mgr_mem_calibrate_dq_dqs_centering(const u32 rw_group, const u32 test_bgn,
const int use_read_test,
const int update_fom)
{
int ret, grp_calibrated;
u32 rank_bgn, sr;
/*
* Altera EMI_RM 2015.05.04 :: Figure 1-28
* Read per-bit deskew can be done on a per shadow register basis.
*/
grp_calibrated = 1;
for (rank_bgn = 0, sr = 0;
rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS;
rank_bgn += NUM_RANKS_PER_SHADOW_REG, sr++) {
/* Check if this set of ranks should be skipped entirely. */
if (param->skip_shadow_regs[sr])
continue;
ret = rw_mgr_mem_calibrate_vfifo_center(rank_bgn, rw_group,
test_bgn,
use_read_test,
update_fom);
if (!ret)
continue;
grp_calibrated = 0;
}
if (!grp_calibrated)
return -EIO;
return 0;
}
/**
* rw_mgr_mem_calibrate_vfifo() - Calibrate the read valid prediction FIFO
* @rw_group: Read/Write Group
* @test_bgn: Rank at which the test begins
*
* Stage 1: Calibrate the read valid prediction FIFO.
*
* This function implements UniPHY calibration Stage 1, as explained in
* detail in Altera EMI_RM 2015.05.04 , "UniPHY Calibration Stages".
* - read valid prediction will consist of finding:
* - DQS enable phase and DQS enable delay (DQS Enable Calibration)
* - DQS input phase and DQS input delay (DQ/DQS Centering)
* - we also do a per-bit deskew on the DQ lines.
*/
static int rw_mgr_mem_calibrate_vfifo(const u32 rw_group, const u32 test_bgn)
uint32_t p, d;
uint32_t dtaps_per_ptap;
uint32_t failed_substage;
int ret;
debug("%s:%d: %u %u\n", __func__, __LINE__, rw_group, test_bgn);
/* Update info for sims */
reg_file_set_group(rw_group);
reg_file_set_stage(CAL_STAGE_VFIFO);
reg_file_set_sub_stage(CAL_SUBSTAGE_GUARANTEED_READ);
failed_substage = CAL_SUBSTAGE_GUARANTEED_READ;
/* USER Determine number of delay taps for each phase tap. */
dtaps_per_ptap = DIV_ROUND_UP(IO_DELAY_PER_OPA_TAP,
IO_DELAY_PER_DQS_EN_DCHAIN_TAP) - 1;
for (d = 0; d <= dtaps_per_ptap; d += 2) {
/*
* In RLDRAMX we may be messing the delay of pins in
* the same write rw_group but outside of the current read
* the rw_group, but that's ok because we haven't calibrated
* output side yet.
*/
if (d > 0) {
scc_mgr_apply_group_all_out_delay_add_all_ranks(
rw_group, d);
for (p = 0; p <= IO_DQDQS_OUT_PHASE_MAX; p++) {
/* 1) Guaranteed Write */
ret = rw_mgr_mem_calibrate_guaranteed_write(rw_group, p);
if (ret)
break;
/* 2) DQS Enable Calibration */
ret = rw_mgr_mem_calibrate_dqs_enable_calibration(rw_group,
test_bgn);
if (ret) {
failed_substage = CAL_SUBSTAGE_DQS_EN_PHASE;
continue;
}
/* 3) Centering DQ/DQS */
* If doing read after write calibration, do not update
* FOM now. Do it then.
ret = rw_mgr_mem_calibrate_dq_dqs_centering(rw_group,
test_bgn, 1, 0);
if (ret) {
failed_substage = CAL_SUBSTAGE_VFIFO_CENTER;
continue;
/* All done. */
goto cal_done_ok;
/* Calibration Stage 1 failed. */
set_failing_group_stage(rw_group, CAL_STAGE_VFIFO, failed_substage);
return 0;
/* Calibration Stage 1 completed OK. */
cal_done_ok:
/*
* Reset the delay chains back to zero if they have moved > 1
* (check for > 1 because loop will increase d even when pass in
* first case).
*/
if (d > 2)
scc_mgr_zero_group(rw_group, 1);
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
return 1;
}
/* VFIFO Calibration -- Read Deskew Calibration after write deskew */
static uint32_t rw_mgr_mem_calibrate_vfifo_end(uint32_t read_group,
uint32_t test_bgn)
{
uint32_t rank_bgn, sr;
uint32_t grp_calibrated;
uint32_t write_group;
debug("%s:%d %u %u", __func__, __LINE__, read_group, test_bgn);
/* update info for sims */
reg_file_set_stage(CAL_STAGE_VFIFO_AFTER_WRITES);
reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER);
write_group = read_group;
/* update info for sims */
reg_file_set_group(read_group);
grp_calibrated = 1;
/* Read per-bit deskew can be done on a per shadow register basis */
for (rank_bgn = 0, sr = 0; rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS;
rank_bgn += NUM_RANKS_PER_SHADOW_REG, ++sr) {
/* Determine if this set of ranks should be skipped entirely */
if (!param->skip_shadow_regs[sr]) {
/* This is the last calibration round, update FOM here */
if (rw_mgr_mem_calibrate_vfifo_center(rank_bgn,
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
read_group,
test_bgn, 0,
1)) {
grp_calibrated = 0;
}
}
}
if (grp_calibrated == 0) {
set_failing_group_stage(write_group,
CAL_STAGE_VFIFO_AFTER_WRITES,
CAL_SUBSTAGE_VFIFO_CENTER);
return 0;
}
return 1;
}
/* Calibrate LFIFO to find smallest read latency */
static uint32_t rw_mgr_mem_calibrate_lfifo(void)
{
uint32_t found_one;
debug("%s:%d\n", __func__, __LINE__);
/* update info for sims */
reg_file_set_stage(CAL_STAGE_LFIFO);
reg_file_set_sub_stage(CAL_SUBSTAGE_READ_LATENCY);
/* Load up the patterns used by read calibration for all ranks */
rw_mgr_mem_calibrate_read_load_patterns(0, 1);
found_one = 0;
do {
writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat);
debug_cond(DLEVEL == 2, "%s:%d lfifo: read_lat=%u",
__func__, __LINE__, gbl->curr_read_lat);
if (!rw_mgr_mem_calibrate_read_test_all_ranks(0,
NUM_READ_TESTS,
PASS_ALL_BITS,
break;
}
found_one = 1;
/* reduce read latency and see if things are working */
/* correctly */
gbl->curr_read_lat--;
} while (gbl->curr_read_lat > 0);
/* reset the fifos to get pointers to known state */
writel(0, &phy_mgr_cmd->fifo_reset);
if (found_one) {
/* add a fudge factor to the read latency that was determined */
gbl->curr_read_lat += 2;
writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat);
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
debug_cond(DLEVEL == 2, "%s:%d lfifo: success: using \
read_lat=%u\n", __func__, __LINE__,
gbl->curr_read_lat);
return 1;
} else {
set_failing_group_stage(0xff, CAL_STAGE_LFIFO,
CAL_SUBSTAGE_READ_LATENCY);
debug_cond(DLEVEL == 2, "%s:%d lfifo: failed at initial \
read_lat=%u\n", __func__, __LINE__,
gbl->curr_read_lat);
return 0;
}
}
/*
* issue write test command.
* two variants are provided. one that just tests a write pattern and
* another that tests datamask functionality.
*/
static void rw_mgr_mem_calibrate_write_test_issue(uint32_t group,
uint32_t test_dm)
{
uint32_t mcc_instruction;
uint32_t quick_write_mode = (((STATIC_CALIB_STEPS) & CALIB_SKIP_WRITES) &&
ENABLE_SUPER_QUICK_CALIBRATION);
uint32_t rw_wl_nop_cycles;
uint32_t addr;
/*
* Set counter and jump addresses for the right
* number of NOP cycles.
* The number of supported NOP cycles can range from -1 to infinity
* Three different cases are handled:
*
* 1. For a number of NOP cycles greater than 0, the RW Mgr looping
* mechanism will be used to insert the right number of NOPs
*
* 2. For a number of NOP cycles equals to 0, the micro-instruction
* issuing the write command will jump straight to the
* micro-instruction that turns on DQS (for DDRx), or outputs write
* data (for RLD), skipping
* the NOP micro-instruction all together
*
* 3. A number of NOP cycles equal to -1 indicates that DQS must be
* turned on in the same micro-instruction that issues the write
* command. Then we need
* to directly jump to the micro-instruction that sends out the data
*
* NOTE: Implementing this mechanism uses 2 RW Mgr jump-counters
* (2 and 3). One jump-counter (0) is used to perform multiple
* write-read operations.
* one counter left to issue this command in "multiple-group" mode
*/
rw_wl_nop_cycles = gbl->rw_wl_nop_cycles;
if (rw_wl_nop_cycles == -1) {
/*
* CNTR 2 - We want to execute the special write operation that
* turns on DQS right away and then skip directly to the
* instruction that sends out the data. We set the counter to a
* large number so that the jump is always taken.
*/
writel(0xFF, &sdr_rw_load_mgr_regs->load_cntr2);
/* CNTR 3 - Not used */
if (test_dm) {
mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0_WL_1;
writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_DATA,
&sdr_rw_load_jump_mgr_regs->load_jump_add2);
writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_NOP,
&sdr_rw_load_jump_mgr_regs->load_jump_add3);
} else {
mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0_WL_1;
writel(RW_MGR_LFSR_WR_RD_BANK_0_DATA,
&sdr_rw_load_jump_mgr_regs->load_jump_add2);
writel(RW_MGR_LFSR_WR_RD_BANK_0_NOP,
&sdr_rw_load_jump_mgr_regs->load_jump_add3);
}
} else if (rw_wl_nop_cycles == 0) {
/*
* CNTR 2 - We want to skip the NOP operation and go straight
* to the DQS enable instruction. We set the counter to a large
* number so that the jump is always taken.
*/
writel(0xFF, &sdr_rw_load_mgr_regs->load_cntr2);
/* CNTR 3 - Not used */
if (test_dm) {
mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0;
writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_DQS,
&sdr_rw_load_jump_mgr_regs->load_jump_add2);
} else {
mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0;
writel(RW_MGR_LFSR_WR_RD_BANK_0_DQS,
&sdr_rw_load_jump_mgr_regs->load_jump_add2);
}
} else {
/*
* CNTR 2 - In this case we want to execute the next instruction
* and NOT take the jump. So we set the counter to 0. The jump
* address doesn't count.
*/
writel(0x0, &sdr_rw_load_mgr_regs->load_cntr2);
writel(0x0, &sdr_rw_load_jump_mgr_regs->load_jump_add2);
/*
* CNTR 3 - Set the nop counter to the number of cycles we
* need to loop for, minus 1.
*/
writel(rw_wl_nop_cycles - 1, &sdr_rw_load_mgr_regs->load_cntr3);
if (test_dm) {
mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0;
writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_NOP,
&sdr_rw_load_jump_mgr_regs->load_jump_add3);
} else {
mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0;
writel(RW_MGR_LFSR_WR_RD_BANK_0_NOP,
&sdr_rw_load_jump_mgr_regs->load_jump_add3);
writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_RESET_READ_DATAPATH_OFFSET);
if (quick_write_mode)
writel(0x08, &sdr_rw_load_mgr_regs->load_cntr0);
writel(0x40, &sdr_rw_load_mgr_regs->load_cntr0);
writel(mcc_instruction, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
/*
* CNTR 1 - This is used to ensure enough time elapses
* for read data to come back.
*/
writel(0x30, &sdr_rw_load_mgr_regs->load_cntr1);
writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_WAIT,
&sdr_rw_load_jump_mgr_regs->load_jump_add1);
writel(RW_MGR_LFSR_WR_RD_BANK_0_WAIT,
&sdr_rw_load_jump_mgr_regs->load_jump_add1);
addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
writel(mcc_instruction, addr + (group << 2));
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
}
/* Test writes, can check for a single bit pass or multiple bit pass */
static uint32_t rw_mgr_mem_calibrate_write_test(uint32_t rank_bgn,
uint32_t write_group, uint32_t use_dm, uint32_t all_correct,
uint32_t *bit_chk, uint32_t all_ranks)
{
uint32_t r;
uint32_t correct_mask_vg;
uint32_t tmp_bit_chk;
uint32_t vg;
uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
(rank_bgn + NUM_RANKS_PER_SHADOW_REG);
uint32_t addr_rw_mgr;
uint32_t base_rw_mgr;
*bit_chk = param->write_correct_mask;
correct_mask_vg = param->write_correct_mask_vg;
for (r = rank_bgn; r < rank_end; r++) {
if (param->skip_ranks[r]) {
/* request to skip the rank */
continue;
}
/* set rank */
set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
tmp_bit_chk = 0;
addr_rw_mgr = SDR_PHYGRP_RWMGRGRP_ADDRESS;
for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS-1; ; vg--) {
/* reset the fifos to get pointers to known state */
writel(0, &phy_mgr_cmd->fifo_reset);
tmp_bit_chk = tmp_bit_chk <<
(RW_MGR_MEM_DQ_PER_WRITE_DQS /
RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS);
rw_mgr_mem_calibrate_write_test_issue(write_group *
RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS+vg,
use_dm);
base_rw_mgr = readl(addr_rw_mgr);
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & ~(base_rw_mgr));
if (vg == 0)
break;
}
*bit_chk &= tmp_bit_chk;
}
if (all_correct) {
set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
debug_cond(DLEVEL == 2, "write_test(%u,%u,ALL) : %u == \
%u => %lu", write_group, use_dm,
*bit_chk, param->write_correct_mask,
(long unsigned int)(*bit_chk ==
param->write_correct_mask));
return *bit_chk == param->write_correct_mask;
} else {
set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
debug_cond(DLEVEL == 2, "write_test(%u,%u,ONE) : %u != ",
write_group, use_dm, *bit_chk);
debug_cond(DLEVEL == 2, "%lu" " => %lu", (long unsigned int)0,
(long unsigned int)(*bit_chk != 0));
return *bit_chk != 0x00;
}
}
/*
* center all windows. do per-bit-deskew to possibly increase size of
* certain windows.
*/
static uint32_t rw_mgr_mem_calibrate_writes_center(uint32_t rank_bgn,
uint32_t write_group, uint32_t test_bgn)
{
uint32_t i, min_index;
int32_t d;
/*
* Store these as signed since there are comparisons with
* signed numbers.
*/
uint32_t bit_chk;
uint32_t sticky_bit_chk;
int32_t left_edge[RW_MGR_MEM_DQ_PER_WRITE_DQS];
int32_t right_edge[RW_MGR_MEM_DQ_PER_WRITE_DQS];
int32_t mid;
int32_t mid_min, orig_mid_min;
int32_t new_dqs, start_dqs;
int32_t dq_margin, dqs_margin, dm_margin;
uint32_t addr;
int ret;
debug("%s:%d %u %u", __func__, __LINE__, write_group, test_bgn);
dm_margin = 0;
addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_OUT1_DELAY_OFFSET;
start_dqs = readl(addr +
(RW_MGR_MEM_DQ_PER_WRITE_DQS << 2));
/* per-bit deskew */
/*
* set the left and right edge of each bit to an illegal value
* use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value.
*/
sticky_bit_chk = 0;
for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
left_edge[i] = IO_IO_OUT1_DELAY_MAX + 1;
right_edge[i] = IO_IO_OUT1_DELAY_MAX + 1;
}
/* Search for the left edge of the window for each bit */
search_left_edge(1, rank_bgn, write_group, 0, test_bgn,
&sticky_bit_chk,
left_edge, right_edge, 0);
/* Search for the right edge of the window for each bit */
ret = search_right_edge(1, rank_bgn, write_group, 0,
start_dqs, 0,
&sticky_bit_chk,
left_edge, right_edge, 0);
if (ret) {
set_failing_group_stage(test_bgn + ret - 1, CAL_STAGE_WRITES,
CAL_SUBSTAGE_WRITES_CENTER);
return 0;
min_index = get_window_mid_index(1, left_edge, right_edge, &mid_min);
/* Determine the amount we can change DQS (which is -mid_min) */
orig_mid_min = mid_min;
new_dqs = start_dqs;
mid_min = 0;
debug_cond(DLEVEL == 1, "%s:%d write_center: start_dqs=%d new_dqs=%d \
mid_min=%d\n", __func__, __LINE__, start_dqs, new_dqs, mid_min);
/* Add delay to bring centre of all DQ windows to the same "level". */
center_dq_windows(1, left_edge, right_edge, mid_min, orig_mid_min,
min_index, 0, &dq_margin, &dqs_margin);
/* Move DQS */
scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, new_dqs);
writel(0, &sdr_scc_mgr->update);
/* Centre DM */
debug_cond(DLEVEL == 2, "%s:%d write_center: DM\n", __func__, __LINE__);
/*
* set the left and right edge of each bit to an illegal value,
* use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value,
*/
left_edge[0] = IO_IO_OUT1_DELAY_MAX + 1;
right_edge[0] = IO_IO_OUT1_DELAY_MAX + 1;
int32_t bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
int32_t end_curr = IO_IO_OUT1_DELAY_MAX + 1;
int32_t bgn_best = IO_IO_OUT1_DELAY_MAX + 1;
int32_t end_best = IO_IO_OUT1_DELAY_MAX + 1;
int32_t win_best = 0;
/* Search for the/part of the window with DM shift */
for (d = IO_IO_OUT1_DELAY_MAX; d >= 0; d -= DELTA_D) {
scc_mgr_apply_group_dm_out1_delay(d);
writel(0, &sdr_scc_mgr->update);
if (rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 1,
PASS_ALL_BITS, &bit_chk,
0)) {
/* USE Set current end of the window */
end_curr = -d;
/*
* If a starting edge of our window has not been seen
* this is our current start of the DM window.
*/
if (bgn_curr == IO_IO_OUT1_DELAY_MAX + 1)
bgn_curr = -d;
/*
* If current window is bigger than best seen.
* Set best seen to be current window.
*/
if ((end_curr-bgn_curr+1) > win_best) {