|
|
|
@ -57,7 +57,7 @@ const struct socfpga_sdram_misc_config *misccfg; |
|
|
|
|
STATIC_SKIP_DELAY_LOOPS) |
|
|
|
|
|
|
|
|
|
/* calibration steps requested by the rtl */ |
|
|
|
|
u16 dyn_calib_steps; |
|
|
|
|
static u16 dyn_calib_steps; |
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* To make CALIB_SKIP_DELAY_LOOPS a dynamic conditional option |
|
|
|
@ -68,13 +68,13 @@ u16 dyn_calib_steps; |
|
|
|
|
* zero when skipping |
|
|
|
|
*/ |
|
|
|
|
|
|
|
|
|
u16 skip_delay_mask; /* mask off bits when skipping/not-skipping */ |
|
|
|
|
static u16 skip_delay_mask; /* mask off bits when skipping/not-skipping */ |
|
|
|
|
|
|
|
|
|
#define SKIP_DELAY_LOOP_VALUE_OR_ZERO(non_skip_value) \ |
|
|
|
|
((non_skip_value) & skip_delay_mask) |
|
|
|
|
|
|
|
|
|
struct gbl_type *gbl; |
|
|
|
|
struct param_type *param; |
|
|
|
|
static struct gbl_type *gbl; |
|
|
|
|
static struct param_type *param; |
|
|
|
|
|
|
|
|
|
static void set_failing_group_stage(u32 group, u32 stage, |
|
|
|
|
u32 substage) |
|
|
|
@ -277,9 +277,9 @@ static void scc_mgr_initialize(void) |
|
|
|
|
int i; |
|
|
|
|
|
|
|
|
|
for (i = 0; i < 16; i++) { |
|
|
|
|
debug_cond(DLEVEL == 1, "%s:%d: Clearing SCC RFILE index %u\n", |
|
|
|
|
debug_cond(DLEVEL >= 1, "%s:%d: Clearing SCC RFILE index %u\n", |
|
|
|
|
__func__, __LINE__, i); |
|
|
|
|
scc_mgr_set(SCC_MGR_HHP_RFILE_OFFSET, 0, i); |
|
|
|
|
scc_mgr_set(SCC_MGR_HHP_RFILE_OFFSET, i, 0); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
@ -303,15 +303,22 @@ static void scc_mgr_set_dqs_en_delay(u32 read_group, u32 delay) |
|
|
|
|
scc_mgr_set(SCC_MGR_DQS_EN_DELAY_OFFSET, read_group, delay); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static void scc_mgr_set_dq_in_delay(u32 dq_in_group, u32 delay) |
|
|
|
|
{ |
|
|
|
|
scc_mgr_set(SCC_MGR_IO_IN_DELAY_OFFSET, dq_in_group, delay); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static void scc_mgr_set_dqs_io_in_delay(u32 delay) |
|
|
|
|
{ |
|
|
|
|
scc_mgr_set(SCC_MGR_IO_IN_DELAY_OFFSET, rwcfg->mem_dq_per_write_dqs, |
|
|
|
|
delay); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static void scc_mgr_set_dq_in_delay(u32 dq_in_group, u32 delay) |
|
|
|
|
static void scc_mgr_set_dm_in_delay(u32 dm, u32 delay) |
|
|
|
|
{ |
|
|
|
|
scc_mgr_set(SCC_MGR_IO_IN_DELAY_OFFSET, dq_in_group, delay); |
|
|
|
|
scc_mgr_set(SCC_MGR_IO_IN_DELAY_OFFSET, |
|
|
|
|
rwcfg->mem_dq_per_write_dqs + 1 + dm, |
|
|
|
|
delay); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static void scc_mgr_set_dq_out1_delay(u32 dq_in_group, u32 delay) |
|
|
|
@ -424,7 +431,6 @@ static void scc_mgr_set_dqs_en_delay_all_ranks(u32 read_group, |
|
|
|
|
*/ |
|
|
|
|
scc_mgr_set_all_ranks(SCC_MGR_DQS_EN_DELAY_OFFSET, |
|
|
|
|
read_group, delay, 1); |
|
|
|
|
writel(0, &sdr_scc_mgr->update); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
@ -473,10 +479,10 @@ static void scc_mgr_set_hhp_extras(void) |
|
|
|
|
SCC_MGR_HHP_GLOBALS_OFFSET | |
|
|
|
|
SCC_MGR_HHP_EXTRAS_OFFSET; |
|
|
|
|
|
|
|
|
|
debug_cond(DLEVEL == 1, "%s:%d Setting HHP Extras\n", |
|
|
|
|
debug_cond(DLEVEL >= 1, "%s:%d Setting HHP Extras\n", |
|
|
|
|
__func__, __LINE__); |
|
|
|
|
writel(value, addr); |
|
|
|
|
debug_cond(DLEVEL == 1, "%s:%d Done Setting HHP Extras\n", |
|
|
|
|
debug_cond(DLEVEL >= 1, "%s:%d Done Setting HHP Extras\n", |
|
|
|
|
__func__, __LINE__); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
@ -585,8 +591,11 @@ static void scc_mgr_zero_group(const u32 write_group, const int out_only) |
|
|
|
|
writel(0xff, &sdr_scc_mgr->dq_ena); |
|
|
|
|
|
|
|
|
|
/* Zero all DM config settings. */ |
|
|
|
|
for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) |
|
|
|
|
for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) { |
|
|
|
|
if (!out_only) |
|
|
|
|
scc_mgr_set_dm_in_delay(i, 0); |
|
|
|
|
scc_mgr_set_dm_out1_delay(i, 0); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/* Multicast to all DM enables. */ |
|
|
|
|
writel(0xff, &sdr_scc_mgr->dm_ena); |
|
|
|
@ -684,7 +693,7 @@ static void scc_mgr_apply_group_all_out_delay_add(const u32 write_group, |
|
|
|
|
/* DQS shift */ |
|
|
|
|
new_delay = READ_SCC_DQS_IO_OUT2_DELAY + delay; |
|
|
|
|
if (new_delay > iocfg->io_out2_delay_max) { |
|
|
|
|
debug_cond(DLEVEL == 1, |
|
|
|
|
debug_cond(DLEVEL >= 1, |
|
|
|
|
"%s:%d (%u, %u) DQS: %u > %d; adding %u to OUT1\n", |
|
|
|
|
__func__, __LINE__, write_group, delay, new_delay, |
|
|
|
|
iocfg->io_out2_delay_max, |
|
|
|
@ -698,7 +707,7 @@ static void scc_mgr_apply_group_all_out_delay_add(const u32 write_group, |
|
|
|
|
/* OCT shift */ |
|
|
|
|
new_delay = READ_SCC_OCT_OUT2_DELAY + delay; |
|
|
|
|
if (new_delay > iocfg->io_out2_delay_max) { |
|
|
|
|
debug_cond(DLEVEL == 1, |
|
|
|
|
debug_cond(DLEVEL >= 1, |
|
|
|
|
"%s:%d (%u, %u) DQS: %u > %d; adding %u to OUT1\n", |
|
|
|
|
__func__, __LINE__, write_group, delay, |
|
|
|
|
new_delay, iocfg->io_out2_delay_max, |
|
|
|
@ -1201,15 +1210,14 @@ rw_mgr_mem_calibrate_write_test(const u32 rank_bgn, const u32 write_group, |
|
|
|
|
|
|
|
|
|
set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF); |
|
|
|
|
if (all_correct) { |
|
|
|
|
debug_cond(DLEVEL == 2, |
|
|
|
|
debug_cond(DLEVEL >= 2, |
|
|
|
|
"write_test(%u,%u,ALL) : %u == %u => %i\n", |
|
|
|
|
write_group, use_dm, *bit_chk, |
|
|
|
|
param->write_correct_mask, |
|
|
|
|
*bit_chk == param->write_correct_mask); |
|
|
|
|
return *bit_chk == param->write_correct_mask; |
|
|
|
|
} else { |
|
|
|
|
set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF); |
|
|
|
|
debug_cond(DLEVEL == 2, |
|
|
|
|
debug_cond(DLEVEL >= 2, |
|
|
|
|
"write_test(%u,%u,ONE) : %u != %i => %i\n", |
|
|
|
|
write_group, use_dm, *bit_chk, 0, *bit_chk != 0); |
|
|
|
|
return *bit_chk != 0x00; |
|
|
|
@ -1284,7 +1292,7 @@ rw_mgr_mem_calibrate_read_test_patterns(const u32 rank_bgn, const u32 group, |
|
|
|
|
if (bit_chk != param->read_correct_mask) |
|
|
|
|
ret = -EIO; |
|
|
|
|
|
|
|
|
|
debug_cond(DLEVEL == 1, |
|
|
|
|
debug_cond(DLEVEL >= 1, |
|
|
|
|
"%s:%d test_load_patterns(%u,ALL) => (%u == %u) => %i\n", |
|
|
|
|
__func__, __LINE__, group, bit_chk, |
|
|
|
|
param->read_correct_mask, ret); |
|
|
|
@ -1445,13 +1453,13 @@ rw_mgr_mem_calibrate_read_test(const u32 rank_bgn, const u32 group, |
|
|
|
|
|
|
|
|
|
if (all_correct) { |
|
|
|
|
ret = (*bit_chk == param->read_correct_mask); |
|
|
|
|
debug_cond(DLEVEL == 2, |
|
|
|
|
debug_cond(DLEVEL >= 2, |
|
|
|
|
"%s:%d read_test(%u,ALL,%u) => (%u == %u) => %i\n", |
|
|
|
|
__func__, __LINE__, group, all_groups, *bit_chk, |
|
|
|
|
param->read_correct_mask, ret); |
|
|
|
|
} else { |
|
|
|
|
ret = (*bit_chk != 0x00); |
|
|
|
|
debug_cond(DLEVEL == 2, |
|
|
|
|
debug_cond(DLEVEL >= 2, |
|
|
|
|
"%s:%d read_test(%u,ONE,%u) => (%u != %u) => %i\n", |
|
|
|
|
__func__, __LINE__, group, all_groups, *bit_chk, |
|
|
|
|
0, ret); |
|
|
|
@ -1515,7 +1523,7 @@ static int find_vfifo_failing_read(const u32 grp) |
|
|
|
|
u32 v, ret, fail_cnt = 0; |
|
|
|
|
|
|
|
|
|
for (v = 0; v < misccfg->read_valid_fifo_size; v++) { |
|
|
|
|
debug_cond(DLEVEL == 2, "%s:%d: vfifo %u\n", |
|
|
|
|
debug_cond(DLEVEL >= 2, "%s:%d: vfifo %u\n", |
|
|
|
|
__func__, __LINE__, v); |
|
|
|
|
ret = rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1, |
|
|
|
|
PASS_ONE_BIT, 0); |
|
|
|
@ -1531,7 +1539,7 @@ static int find_vfifo_failing_read(const u32 grp) |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/* No failing read found! Something must have gone wrong. */ |
|
|
|
|
debug_cond(DLEVEL == 2, "%s:%d: vfifo failed\n", __func__, __LINE__); |
|
|
|
|
debug_cond(DLEVEL >= 2, "%s:%d: vfifo failed\n", __func__, __LINE__); |
|
|
|
|
return 0; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
@ -1638,7 +1646,7 @@ static int sdr_working_phase(const u32 grp, u32 *work_bgn, u32 *d, |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/* Cannot find working solution */ |
|
|
|
|
debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: no vfifo/ptap/dtap\n", |
|
|
|
|
debug_cond(DLEVEL >= 2, "%s:%d find_dqs_en_phase: no vfifo/ptap/dtap\n", |
|
|
|
|
__func__, __LINE__); |
|
|
|
|
return -EINVAL; |
|
|
|
|
} |
|
|
|
@ -1714,7 +1722,7 @@ static int sdr_nonworking_phase(const u32 grp, u32 *work_end, u32 *p, u32 *i) |
|
|
|
|
ret = sdr_find_phase(0, grp, work_end, i, p); |
|
|
|
|
if (ret) { |
|
|
|
|
/* Cannot see edge of failing read. */ |
|
|
|
|
debug_cond(DLEVEL == 2, "%s:%d: end: failed\n", |
|
|
|
|
debug_cond(DLEVEL >= 2, "%s:%d: end: failed\n", |
|
|
|
|
__func__, __LINE__); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
@ -1738,21 +1746,21 @@ static int sdr_find_window_center(const u32 grp, const u32 work_bgn, |
|
|
|
|
|
|
|
|
|
work_mid = (work_bgn + work_end) / 2; |
|
|
|
|
|
|
|
|
|
debug_cond(DLEVEL == 2, "work_bgn=%d work_end=%d work_mid=%d\n", |
|
|
|
|
debug_cond(DLEVEL >= 2, "work_bgn=%d work_end=%d work_mid=%d\n", |
|
|
|
|
work_bgn, work_end, work_mid); |
|
|
|
|
/* Get the middle delay to be less than a VFIFO delay */ |
|
|
|
|
tmp_delay = (iocfg->dqs_en_phase_max + 1) * iocfg->delay_per_opa_tap; |
|
|
|
|
|
|
|
|
|
debug_cond(DLEVEL == 2, "vfifo ptap delay %d\n", tmp_delay); |
|
|
|
|
debug_cond(DLEVEL >= 2, "vfifo ptap delay %d\n", tmp_delay); |
|
|
|
|
work_mid %= tmp_delay; |
|
|
|
|
debug_cond(DLEVEL == 2, "new work_mid %d\n", work_mid); |
|
|
|
|
debug_cond(DLEVEL >= 2, "new work_mid %d\n", work_mid); |
|
|
|
|
|
|
|
|
|
tmp_delay = rounddown(work_mid, iocfg->delay_per_opa_tap); |
|
|
|
|
if (tmp_delay > iocfg->dqs_en_phase_max * iocfg->delay_per_opa_tap) |
|
|
|
|
tmp_delay = iocfg->dqs_en_phase_max * iocfg->delay_per_opa_tap; |
|
|
|
|
p = tmp_delay / iocfg->delay_per_opa_tap; |
|
|
|
|
|
|
|
|
|
debug_cond(DLEVEL == 2, "new p %d, tmp_delay=%d\n", p, tmp_delay); |
|
|
|
|
debug_cond(DLEVEL >= 2, "new p %d, tmp_delay=%d\n", p, tmp_delay); |
|
|
|
|
|
|
|
|
|
d = DIV_ROUND_UP(work_mid - tmp_delay, |
|
|
|
|
iocfg->delay_per_dqs_en_dchain_tap); |
|
|
|
@ -1760,7 +1768,7 @@ static int sdr_find_window_center(const u32 grp, const u32 work_bgn, |
|
|
|
|
d = iocfg->dqs_en_delay_max; |
|
|
|
|
tmp_delay += d * iocfg->delay_per_dqs_en_dchain_tap; |
|
|
|
|
|
|
|
|
|
debug_cond(DLEVEL == 2, "new d %d, tmp_delay=%d\n", d, tmp_delay); |
|
|
|
|
debug_cond(DLEVEL >= 2, "new d %d, tmp_delay=%d\n", d, tmp_delay); |
|
|
|
|
|
|
|
|
|
scc_mgr_set_dqs_en_phase_all_ranks(grp, p); |
|
|
|
|
scc_mgr_set_dqs_en_delay_all_ranks(grp, d); |
|
|
|
@ -1770,11 +1778,11 @@ static int sdr_find_window_center(const u32 grp, const u32 work_bgn, |
|
|
|
|
* because the largest possible margin in 1 VFIFO cycle. |
|
|
|
|
*/ |
|
|
|
|
for (i = 0; i < misccfg->read_valid_fifo_size; i++) { |
|
|
|
|
debug_cond(DLEVEL == 2, "find_dqs_en_phase: center\n"); |
|
|
|
|
debug_cond(DLEVEL >= 2, "find_dqs_en_phase: center\n"); |
|
|
|
|
if (rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1, |
|
|
|
|
PASS_ONE_BIT, |
|
|
|
|
0)) { |
|
|
|
|
debug_cond(DLEVEL == 2, |
|
|
|
|
debug_cond(DLEVEL >= 2, |
|
|
|
|
"%s:%d center: found: ptap=%u dtap=%u\n", |
|
|
|
|
__func__, __LINE__, p, d); |
|
|
|
|
return 0; |
|
|
|
@ -1784,7 +1792,7 @@ static int sdr_find_window_center(const u32 grp, const u32 work_bgn, |
|
|
|
|
rw_mgr_incr_vfifo(grp); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
debug_cond(DLEVEL == 2, "%s:%d center: failed.\n", |
|
|
|
|
debug_cond(DLEVEL >= 2, "%s:%d center: failed.\n", |
|
|
|
|
__func__, __LINE__); |
|
|
|
|
return -EINVAL; |
|
|
|
|
} |
|
|
|
@ -1860,7 +1868,7 @@ static int rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(const u32 grp) |
|
|
|
|
|
|
|
|
|
d = 0; |
|
|
|
|
|
|
|
|
|
debug_cond(DLEVEL == 2, "%s:%d p: ptap=%u\n", |
|
|
|
|
debug_cond(DLEVEL >= 2, "%s:%d p: ptap=%u\n", |
|
|
|
|
__func__, __LINE__, p); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
@ -1872,18 +1880,18 @@ static int rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(const u32 grp) |
|
|
|
|
if (d != 0) |
|
|
|
|
work_end -= iocfg->delay_per_dqs_en_dchain_tap; |
|
|
|
|
|
|
|
|
|
debug_cond(DLEVEL == 2, |
|
|
|
|
debug_cond(DLEVEL >= 2, |
|
|
|
|
"%s:%d p/d: ptap=%u dtap=%u end=%u\n", |
|
|
|
|
__func__, __LINE__, p, d - 1, work_end); |
|
|
|
|
|
|
|
|
|
if (work_end < work_bgn) { |
|
|
|
|
/* nil range */ |
|
|
|
|
debug_cond(DLEVEL == 2, "%s:%d end-2: failed\n", |
|
|
|
|
debug_cond(DLEVEL >= 2, "%s:%d end-2: failed\n", |
|
|
|
|
__func__, __LINE__); |
|
|
|
|
return -EINVAL; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
debug_cond(DLEVEL == 2, "%s:%d found range [%u,%u]\n", |
|
|
|
|
debug_cond(DLEVEL >= 2, "%s:%d found range [%u,%u]\n", |
|
|
|
|
__func__, __LINE__, work_bgn, work_end); |
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
@ -1891,18 +1899,18 @@ static int rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(const u32 grp) |
|
|
|
|
* To do that we'll back up a ptap and re-find the edge of the |
|
|
|
|
* window using dtaps |
|
|
|
|
*/ |
|
|
|
|
debug_cond(DLEVEL == 2, "%s:%d calculate dtaps_per_ptap for tracking\n", |
|
|
|
|
debug_cond(DLEVEL >= 2, "%s:%d calculate dtaps_per_ptap for tracking\n", |
|
|
|
|
__func__, __LINE__); |
|
|
|
|
|
|
|
|
|
/* Special case code for backing up a phase */ |
|
|
|
|
if (p == 0) { |
|
|
|
|
p = iocfg->dqs_en_phase_max; |
|
|
|
|
rw_mgr_decr_vfifo(grp); |
|
|
|
|
debug_cond(DLEVEL == 2, "%s:%d backedup cycle/phase: p=%u\n", |
|
|
|
|
debug_cond(DLEVEL >= 2, "%s:%d backedup cycle/phase: p=%u\n", |
|
|
|
|
__func__, __LINE__, p); |
|
|
|
|
} else { |
|
|
|
|
p = p - 1; |
|
|
|
|
debug_cond(DLEVEL == 2, "%s:%d backedup phase only: p=%u", |
|
|
|
|
debug_cond(DLEVEL >= 2, "%s:%d backedup phase only: p=%u", |
|
|
|
|
__func__, __LINE__, p); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
@ -1915,7 +1923,7 @@ static int rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(const u32 grp) |
|
|
|
|
*/ |
|
|
|
|
|
|
|
|
|
/* Find a passing read. */ |
|
|
|
|
debug_cond(DLEVEL == 2, "%s:%d find passing read\n", |
|
|
|
|
debug_cond(DLEVEL >= 2, "%s:%d find passing read\n", |
|
|
|
|
__func__, __LINE__); |
|
|
|
|
|
|
|
|
|
initial_failing_dtap = d; |
|
|
|
@ -1923,13 +1931,13 @@ static int rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(const u32 grp) |
|
|
|
|
found_passing_read = !sdr_find_phase_delay(1, 1, grp, NULL, 0, &d); |
|
|
|
|
if (found_passing_read) { |
|
|
|
|
/* Find a failing read. */ |
|
|
|
|
debug_cond(DLEVEL == 2, "%s:%d find failing read\n", |
|
|
|
|
debug_cond(DLEVEL >= 2, "%s:%d find failing read\n", |
|
|
|
|
__func__, __LINE__); |
|
|
|
|
d++; |
|
|
|
|
found_failing_read = !sdr_find_phase_delay(0, 1, grp, NULL, 0, |
|
|
|
|
&d); |
|
|
|
|
} else { |
|
|
|
|
debug_cond(DLEVEL == 1, |
|
|
|
|
debug_cond(DLEVEL >= 1, |
|
|
|
|
"%s:%d failed to calculate dtaps per ptap. Fall back on static value\n", |
|
|
|
|
__func__, __LINE__); |
|
|
|
|
} |
|
|
|
@ -1944,7 +1952,7 @@ static int rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(const u32 grp) |
|
|
|
|
dtaps_per_ptap = d - initial_failing_dtap; |
|
|
|
|
|
|
|
|
|
writel(dtaps_per_ptap, &sdr_reg_file->dtaps_per_ptap); |
|
|
|
|
debug_cond(DLEVEL == 2, "%s:%d dtaps_per_ptap=%u - %u = %u", |
|
|
|
|
debug_cond(DLEVEL >= 2, "%s:%d dtaps_per_ptap=%u - %u = %u", |
|
|
|
|
__func__, __LINE__, d, initial_failing_dtap, dtaps_per_ptap); |
|
|
|
|
|
|
|
|
|
/* Step 6: Find the centre of the window. */ |
|
|
|
@ -2000,7 +2008,7 @@ static u32 search_stop_check(const int write, const int d, const int rank_bgn, |
|
|
|
|
} |
|
|
|
|
*sticky_bit_chk = *sticky_bit_chk | *bit_chk; |
|
|
|
|
ret = ret && (*sticky_bit_chk == correct_mask); |
|
|
|
|
debug_cond(DLEVEL == 2, |
|
|
|
|
debug_cond(DLEVEL >= 2, |
|
|
|
|
"%s:%d center(left): dtap=%u => %u == %u && %u", |
|
|
|
|
__func__, __LINE__, d, |
|
|
|
|
*sticky_bit_chk, correct_mask, ret); |
|
|
|
@ -2079,7 +2087,7 @@ static void search_left_edge(const int write, const int rank_bgn, |
|
|
|
|
|
|
|
|
|
*sticky_bit_chk = 0; |
|
|
|
|
for (i = per_dqs - 1; i >= 0; i--) { |
|
|
|
|
debug_cond(DLEVEL == 2, |
|
|
|
|
debug_cond(DLEVEL >= 2, |
|
|
|
|
"%s:%d vfifo_center: left_edge[%u]: %d right_edge[%u]: %d\n", |
|
|
|
|
__func__, __LINE__, i, left_edge[i], |
|
|
|
|
i, right_edge[i]); |
|
|
|
@ -2092,7 +2100,7 @@ static void search_left_edge(const int write, const int rank_bgn, |
|
|
|
|
if ((left_edge[i] == delay_max + 1) && |
|
|
|
|
(right_edge[i] != delay_max + 1)) { |
|
|
|
|
right_edge[i] = delay_max + 1; |
|
|
|
|
debug_cond(DLEVEL == 2, |
|
|
|
|
debug_cond(DLEVEL >= 2, |
|
|
|
|
"%s:%d vfifo_center: reset right_edge[%u]: %d\n", |
|
|
|
|
__func__, __LINE__, i, right_edge[i]); |
|
|
|
|
} |
|
|
|
@ -2222,12 +2230,12 @@ static int search_right_edge(const int write, const int rank_bgn, |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
debug_cond(DLEVEL == 2, "%s:%d center[r,d=%u]: ", |
|
|
|
|
debug_cond(DLEVEL >= 2, "%s:%d center[r,d=%u]: ", |
|
|
|
|
__func__, __LINE__, d); |
|
|
|
|
debug_cond(DLEVEL == 2, |
|
|
|
|
debug_cond(DLEVEL >= 2, |
|
|
|
|
"bit_chk_test=%i left_edge[%u]: %d ", |
|
|
|
|
bit_chk & 1, i, left_edge[i]); |
|
|
|
|
debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i, |
|
|
|
|
debug_cond(DLEVEL >= 2, "right_edge[%u]: %d\n", i, |
|
|
|
|
right_edge[i]); |
|
|
|
|
bit_chk >>= 1; |
|
|
|
|
} |
|
|
|
@ -2235,7 +2243,7 @@ static int search_right_edge(const int write, const int rank_bgn, |
|
|
|
|
|
|
|
|
|
/* Check that all bits have a window */ |
|
|
|
|
for (i = 0; i < per_dqs; i++) { |
|
|
|
|
debug_cond(DLEVEL == 2, |
|
|
|
|
debug_cond(DLEVEL >= 2, |
|
|
|
|
"%s:%d write_center: left_edge[%u]: %d right_edge[%u]: %d", |
|
|
|
|
__func__, __LINE__, i, left_edge[i], |
|
|
|
|
i, right_edge[i]); |
|
|
|
@ -2284,7 +2292,7 @@ static int get_window_mid_index(const int write, int *left_edge, |
|
|
|
|
(*mid_min)++; |
|
|
|
|
*mid_min = *mid_min / 2; |
|
|
|
|
|
|
|
|
|
debug_cond(DLEVEL == 1, "%s:%d vfifo_center: *mid_min=%d (index=%u)\n", |
|
|
|
|
debug_cond(DLEVEL >= 1, "%s:%d vfifo_center: *mid_min=%d (index=%u)\n", |
|
|
|
|
__func__, __LINE__, *mid_min, min_index); |
|
|
|
|
return min_index; |
|
|
|
|
} |
|
|
|
@ -2308,15 +2316,15 @@ static void center_dq_windows(const int write, int *left_edge, int *right_edge, |
|
|
|
|
const int min_index, const int test_bgn, |
|
|
|
|
int *dq_margin, int *dqs_margin) |
|
|
|
|
{ |
|
|
|
|
const u32 delay_max = write ? iocfg->io_out1_delay_max : |
|
|
|
|
const s32 delay_max = write ? iocfg->io_out1_delay_max : |
|
|
|
|
iocfg->io_in_delay_max; |
|
|
|
|
const u32 per_dqs = write ? rwcfg->mem_dq_per_write_dqs : |
|
|
|
|
const s32 per_dqs = write ? rwcfg->mem_dq_per_write_dqs : |
|
|
|
|
rwcfg->mem_dq_per_read_dqs; |
|
|
|
|
const u32 delay_off = write ? SCC_MGR_IO_OUT1_DELAY_OFFSET : |
|
|
|
|
const s32 delay_off = write ? SCC_MGR_IO_OUT1_DELAY_OFFSET : |
|
|
|
|
SCC_MGR_IO_IN_DELAY_OFFSET; |
|
|
|
|
const u32 addr = SDR_PHYGRP_SCCGRP_ADDRESS | delay_off; |
|
|
|
|
const s32 addr = SDR_PHYGRP_SCCGRP_ADDRESS | delay_off; |
|
|
|
|
|
|
|
|
|
u32 temp_dq_io_delay1, temp_dq_io_delay2; |
|
|
|
|
s32 temp_dq_io_delay1; |
|
|
|
|
int shift_dq, i, p; |
|
|
|
|
|
|
|
|
|
/* Initialize data for export structures */ |
|
|
|
@ -2330,19 +2338,18 @@ static void center_dq_windows(const int write, int *left_edge, int *right_edge, |
|
|
|
|
(left_edge[min_index] - right_edge[min_index]))/2 + |
|
|
|
|
(orig_mid_min - mid_min); |
|
|
|
|
|
|
|
|
|
debug_cond(DLEVEL == 2, |
|
|
|
|
debug_cond(DLEVEL >= 2, |
|
|
|
|
"vfifo_center: before: shift_dq[%u]=%d\n", |
|
|
|
|
i, shift_dq); |
|
|
|
|
|
|
|
|
|
temp_dq_io_delay1 = readl(addr + (p << 2)); |
|
|
|
|
temp_dq_io_delay2 = readl(addr + (i << 2)); |
|
|
|
|
temp_dq_io_delay1 = readl(addr + (i << 2)); |
|
|
|
|
|
|
|
|
|
if (shift_dq + temp_dq_io_delay1 > delay_max) |
|
|
|
|
shift_dq = delay_max - temp_dq_io_delay2; |
|
|
|
|
shift_dq = delay_max - temp_dq_io_delay1; |
|
|
|
|
else if (shift_dq + temp_dq_io_delay1 < 0) |
|
|
|
|
shift_dq = -temp_dq_io_delay1; |
|
|
|
|
|
|
|
|
|
debug_cond(DLEVEL == 2, |
|
|
|
|
debug_cond(DLEVEL >= 2, |
|
|
|
|
"vfifo_center: after: shift_dq[%u]=%d\n", |
|
|
|
|
i, shift_dq); |
|
|
|
|
|
|
|
|
@ -2355,7 +2362,7 @@ static void center_dq_windows(const int write, int *left_edge, int *right_edge, |
|
|
|
|
|
|
|
|
|
scc_mgr_load_dq(p); |
|
|
|
|
|
|
|
|
|
debug_cond(DLEVEL == 2, |
|
|
|
|
debug_cond(DLEVEL >= 2, |
|
|
|
|
"vfifo_center: margin[%u]=[%d,%d]\n", i, |
|
|
|
|
left_edge[i] - shift_dq + (-mid_min), |
|
|
|
|
right_edge[i] + shift_dq - (-mid_min)); |
|
|
|
@ -2437,7 +2444,7 @@ static int rw_mgr_mem_calibrate_vfifo_center(const u32 rank_bgn, |
|
|
|
|
scc_mgr_load_dqs(rw_group); |
|
|
|
|
writel(0, &sdr_scc_mgr->update); |
|
|
|
|
|
|
|
|
|
debug_cond(DLEVEL == 1, |
|
|
|
|
debug_cond(DLEVEL >= 1, |
|
|
|
|
"%s:%d vfifo_center: failed to find edge [%u]: %d %d", |
|
|
|
|
__func__, __LINE__, i, left_edge[i], right_edge[i]); |
|
|
|
|
if (use_read_test) { |
|
|
|
@ -2465,7 +2472,7 @@ static int rw_mgr_mem_calibrate_vfifo_center(const u32 rank_bgn, |
|
|
|
|
new_dqs = 0; |
|
|
|
|
|
|
|
|
|
mid_min = start_dqs - new_dqs; |
|
|
|
|
debug_cond(DLEVEL == 1, "vfifo_center: new mid_min=%d new_dqs=%d\n", |
|
|
|
|
debug_cond(DLEVEL >= 1, "vfifo_center: new mid_min=%d new_dqs=%d\n", |
|
|
|
|
mid_min, new_dqs); |
|
|
|
|
|
|
|
|
|
if (iocfg->shift_dqs_en_when_shift_dqs) { |
|
|
|
@ -2477,7 +2484,7 @@ static int rw_mgr_mem_calibrate_vfifo_center(const u32 rank_bgn, |
|
|
|
|
} |
|
|
|
|
new_dqs = start_dqs - mid_min; |
|
|
|
|
|
|
|
|
|
debug_cond(DLEVEL == 1, |
|
|
|
|
debug_cond(DLEVEL >= 1, |
|
|
|
|
"vfifo_center: start_dqs=%d start_dqs_en=%d new_dqs=%d mid_min=%d\n", |
|
|
|
|
start_dqs, |
|
|
|
|
iocfg->shift_dqs_en_when_shift_dqs ? start_dqs_en : -1, |
|
|
|
@ -2497,7 +2504,7 @@ static int rw_mgr_mem_calibrate_vfifo_center(const u32 rank_bgn, |
|
|
|
|
/* Move DQS */ |
|
|
|
|
scc_mgr_set_dqs_bus_in_delay(rw_group, new_dqs); |
|
|
|
|
scc_mgr_load_dqs(rw_group); |
|
|
|
|
debug_cond(DLEVEL == 2, |
|
|
|
|
debug_cond(DLEVEL >= 2, |
|
|
|
|
"%s:%d vfifo_center: dq_margin=%d dqs_margin=%d", |
|
|
|
|
__func__, __LINE__, dq_margin, dqs_margin); |
|
|
|
|
|
|
|
|
@ -2530,7 +2537,7 @@ static int rw_mgr_mem_calibrate_guaranteed_write(const u32 rw_group, |
|
|
|
|
/* Set a particular DQ/DQS phase. */ |
|
|
|
|
scc_mgr_set_dqdqs_output_phase_all_ranks(rw_group, phase); |
|
|
|
|
|
|
|
|
|
debug_cond(DLEVEL == 1, "%s:%d guaranteed write: g=%u p=%u\n", |
|
|
|
|
debug_cond(DLEVEL >= 1, "%s:%d guaranteed write: g=%u p=%u\n", |
|
|
|
|
__func__, __LINE__, rw_group, phase); |
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
@ -2549,7 +2556,7 @@ static int rw_mgr_mem_calibrate_guaranteed_write(const u32 rw_group, |
|
|
|
|
*/ |
|
|
|
|
ret = rw_mgr_mem_calibrate_read_test_patterns(0, rw_group, 1); |
|
|
|
|
if (ret) |
|
|
|
|
debug_cond(DLEVEL == 1, |
|
|
|
|
debug_cond(DLEVEL >= 1, |
|
|
|
|
"%s:%d Guaranteed read test failed: g=%u p=%u\n", |
|
|
|
|
__func__, __LINE__, rw_group, phase); |
|
|
|
|
return ret; |
|
|
|
@ -2585,7 +2592,7 @@ static int rw_mgr_mem_calibrate_dqs_enable_calibration(const u32 rw_group, |
|
|
|
|
for (i = 0, p = test_bgn, d = 0; |
|
|
|
|
i < rwcfg->mem_dq_per_read_dqs; |
|
|
|
|
i++, p++, d += delay_step) { |
|
|
|
|
debug_cond(DLEVEL == 1, |
|
|
|
|
debug_cond(DLEVEL >= 1, |
|
|
|
|
"%s:%d: g=%u r=%u i=%u p=%u d=%u\n", |
|
|
|
|
__func__, __LINE__, rw_group, r, i, p, d); |
|
|
|
|
|
|
|
|
@ -2602,7 +2609,7 @@ static int rw_mgr_mem_calibrate_dqs_enable_calibration(const u32 rw_group, |
|
|
|
|
*/ |
|
|
|
|
ret = rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(rw_group); |
|
|
|
|
|
|
|
|
|
debug_cond(DLEVEL == 1, |
|
|
|
|
debug_cond(DLEVEL >= 1, |
|
|
|
|
"%s:%d: g=%u found=%u; Reseting delay chain to zero\n", |
|
|
|
|
__func__, __LINE__, rw_group, !ret); |
|
|
|
|
|
|
|
|
@ -2808,7 +2815,7 @@ static u32 rw_mgr_mem_calibrate_lfifo(void) |
|
|
|
|
|
|
|
|
|
do { |
|
|
|
|
writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat); |
|
|
|
|
debug_cond(DLEVEL == 2, "%s:%d lfifo: read_lat=%u", |
|
|
|
|
debug_cond(DLEVEL >= 2, "%s:%d lfifo: read_lat=%u", |
|
|
|
|
__func__, __LINE__, gbl->curr_read_lat); |
|
|
|
|
|
|
|
|
|
if (!rw_mgr_mem_calibrate_read_test_all_ranks(0, NUM_READ_TESTS, |
|
|
|
@ -2830,14 +2837,14 @@ static u32 rw_mgr_mem_calibrate_lfifo(void) |
|
|
|
|
/* Add a fudge factor to the read latency that was determined */ |
|
|
|
|
gbl->curr_read_lat += 2; |
|
|
|
|
writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat); |
|
|
|
|
debug_cond(DLEVEL == 2, |
|
|
|
|
debug_cond(DLEVEL >= 2, |
|
|
|
|
"%s:%d lfifo: success: using read_lat=%u\n", |
|
|
|
|
__func__, __LINE__, gbl->curr_read_lat); |
|
|
|
|
} else { |
|
|
|
|
set_failing_group_stage(0xff, CAL_STAGE_LFIFO, |
|
|
|
|
CAL_SUBSTAGE_READ_LATENCY); |
|
|
|
|
|
|
|
|
|
debug_cond(DLEVEL == 2, |
|
|
|
|
debug_cond(DLEVEL >= 2, |
|
|
|
|
"%s:%d lfifo: failed at initial read_lat=%u\n", |
|
|
|
|
__func__, __LINE__, gbl->curr_read_lat); |
|
|
|
|
} |
|
|
|
@ -3000,7 +3007,7 @@ rw_mgr_mem_calibrate_writes_center(const u32 rank_bgn, const u32 write_group, |
|
|
|
|
orig_mid_min = mid_min; |
|
|
|
|
new_dqs = start_dqs; |
|
|
|
|
mid_min = 0; |
|
|
|
|
debug_cond(DLEVEL == 1, |
|
|
|
|
debug_cond(DLEVEL >= 1, |
|
|
|
|
"%s:%d write_center: start_dqs=%d new_dqs=%d mid_min=%d\n", |
|
|
|
|
__func__, __LINE__, start_dqs, new_dqs, mid_min); |
|
|
|
|
|
|
|
|
@ -3013,7 +3020,7 @@ rw_mgr_mem_calibrate_writes_center(const u32 rank_bgn, const u32 write_group, |
|
|
|
|
writel(0, &sdr_scc_mgr->update); |
|
|
|
|
|
|
|
|
|
/* Centre DM */ |
|
|
|
|
debug_cond(DLEVEL == 2, "%s:%d write_center: DM\n", __func__, __LINE__); |
|
|
|
|
debug_cond(DLEVEL >= 2, "%s:%d write_center: DM\n", __func__, __LINE__); |
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Set the left and right edge of each bit to an illegal value. |
|
|
|
@ -3047,7 +3054,7 @@ rw_mgr_mem_calibrate_writes_center(const u32 rank_bgn, const u32 write_group, |
|
|
|
|
left_edge[0] = -1 * bgn_best; |
|
|
|
|
right_edge[0] = end_best; |
|
|
|
|
|
|
|
|
|
debug_cond(DLEVEL == 2, "%s:%d dm_calib: left=%d right=%d\n", |
|
|
|
|
debug_cond(DLEVEL >= 2, "%s:%d dm_calib: left=%d right=%d\n", |
|
|
|
|
__func__, __LINE__, left_edge[0], right_edge[0]); |
|
|
|
|
|
|
|
|
|
/* Move DQS (back to orig). */ |
|
|
|
@ -3071,14 +3078,14 @@ rw_mgr_mem_calibrate_writes_center(const u32 rank_bgn, const u32 write_group, |
|
|
|
|
scc_mgr_apply_group_dm_out1_delay(mid); |
|
|
|
|
writel(0, &sdr_scc_mgr->update); |
|
|
|
|
|
|
|
|
|
debug_cond(DLEVEL == 2, |
|
|
|
|
debug_cond(DLEVEL >= 2, |
|
|
|
|
"%s:%d dm_calib: left=%d right=%d mid=%d dm_margin=%d\n", |
|
|
|
|
__func__, __LINE__, left_edge[0], right_edge[0], |
|
|
|
|
mid, dm_margin); |
|
|
|
|
/* Export values. */ |
|
|
|
|
gbl->fom_out += dq_margin + dqs_margin; |
|
|
|
|
|
|
|
|
|
debug_cond(DLEVEL == 2, |
|
|
|
|
debug_cond(DLEVEL >= 2, |
|
|
|
|
"%s:%d write_center: dq_margin=%d dqs_margin=%d dm_margin=%d\n", |
|
|
|
|
__func__, __LINE__, dq_margin, dqs_margin, dm_margin); |
|
|
|
|
|
|
|
|
@ -3479,6 +3486,7 @@ grp_failed: /* A group failed, increment the counter. */ |
|
|
|
|
static int run_mem_calibrate(void) |
|
|
|
|
{ |
|
|
|
|
int pass; |
|
|
|
|
u32 ctrl_cfg; |
|
|
|
|
|
|
|
|
|
debug("%s:%d\n", __func__, __LINE__); |
|
|
|
|
|
|
|
|
@ -3486,7 +3494,9 @@ static int run_mem_calibrate(void) |
|
|
|
|
writel(PHY_MGR_CAL_RESET, &phy_mgr_cfg->cal_status); |
|
|
|
|
|
|
|
|
|
/* Stop tracking manager. */ |
|
|
|
|
clrbits_le32(&sdr_ctrl->ctrl_cfg, 1 << 22); |
|
|
|
|
ctrl_cfg = readl(&sdr_ctrl->ctrl_cfg); |
|
|
|
|
writel(ctrl_cfg & ~SDR_CTRLGRP_CTRLCFG_DQSTRKEN_MASK, |
|
|
|
|
&sdr_ctrl->ctrl_cfg); |
|
|
|
|
|
|
|
|
|
phy_mgr_initialize(); |
|
|
|
|
rw_mgr_mem_initialize(); |
|
|
|
@ -3507,7 +3517,7 @@ static int run_mem_calibrate(void) |
|
|
|
|
writel(0x2, &phy_mgr_cfg->mux_sel); |
|
|
|
|
|
|
|
|
|
/* Start tracking manager. */ |
|
|
|
|
setbits_le32(&sdr_ctrl->ctrl_cfg, 1 << 22); |
|
|
|
|
writel(ctrl_cfg, &sdr_ctrl->ctrl_cfg); |
|
|
|
|
|
|
|
|
|
return pass; |
|
|
|
|
} |
|
|
|
@ -3734,27 +3744,27 @@ int sdram_calibration_full(void) |
|
|
|
|
printf("%s: Preparing to start memory calibration\n", __FILE__); |
|
|
|
|
|
|
|
|
|
debug("%s:%d\n", __func__, __LINE__); |
|
|
|
|
debug_cond(DLEVEL == 1, |
|
|
|
|
debug_cond(DLEVEL >= 1, |
|
|
|
|
"DDR3 FULL_RATE ranks=%u cs/dimm=%u dq/dqs=%u,%u vg/dqs=%u,%u ", |
|
|
|
|
rwcfg->mem_number_of_ranks, rwcfg->mem_number_of_cs_per_dimm, |
|
|
|
|
rwcfg->mem_dq_per_read_dqs, rwcfg->mem_dq_per_write_dqs, |
|
|
|
|
rwcfg->mem_virtual_groups_per_read_dqs, |
|
|
|
|
rwcfg->mem_virtual_groups_per_write_dqs); |
|
|
|
|
debug_cond(DLEVEL == 1, |
|
|
|
|
debug_cond(DLEVEL >= 1, |
|
|
|
|
"dqs=%u,%u dq=%u dm=%u ptap_delay=%u dtap_delay=%u ", |
|
|
|
|
rwcfg->mem_if_read_dqs_width, rwcfg->mem_if_write_dqs_width, |
|
|
|
|
rwcfg->mem_data_width, rwcfg->mem_data_mask_width, |
|
|
|
|
iocfg->delay_per_opa_tap, iocfg->delay_per_dchain_tap); |
|
|
|
|
debug_cond(DLEVEL == 1, "dtap_dqsen_delay=%u, dll=%u", |
|
|
|
|
debug_cond(DLEVEL >= 1, "dtap_dqsen_delay=%u, dll=%u", |
|
|
|
|
iocfg->delay_per_dqs_en_dchain_tap, iocfg->dll_chain_length); |
|
|
|
|
debug_cond(DLEVEL == 1, |
|
|
|
|
debug_cond(DLEVEL >= 1, |
|
|
|
|
"max values: en_p=%u dqdqs_p=%u en_d=%u dqs_in_d=%u ", |
|
|
|
|
iocfg->dqs_en_phase_max, iocfg->dqdqs_out_phase_max, |
|
|
|
|
iocfg->dqs_en_delay_max, iocfg->dqs_in_delay_max); |
|
|
|
|
debug_cond(DLEVEL == 1, "io_in_d=%u io_out1_d=%u io_out2_d=%u ", |
|
|
|
|
debug_cond(DLEVEL >= 1, "io_in_d=%u io_out1_d=%u io_out2_d=%u ", |
|
|
|
|
iocfg->io_in_delay_max, iocfg->io_out1_delay_max, |
|
|
|
|
iocfg->io_out2_delay_max); |
|
|
|
|
debug_cond(DLEVEL == 1, "dqs_in_reserve=%u dqs_out_reserve=%u\n", |
|
|
|
|
debug_cond(DLEVEL >= 1, "dqs_in_reserve=%u dqs_out_reserve=%u\n", |
|
|
|
|
iocfg->dqs_in_reserve, iocfg->dqs_out_reserve); |
|
|
|
|
|
|
|
|
|
hc_initialize_rom_data(); |
|
|
|
|