2 * Copyright 2014-2015 Freescale Semiconductor, Inc.
4 * SPDX-License-Identifier: GPL-2.0+
9 #include <fsl_ddr_sdram.h>
10 #include <asm/processor.h>
11 #include <fsl_immap.h>
13 #include <fsl_errata.h>
15 #ifdef CONFIG_SYS_FSL_ERRATUM_A008511
16 static void set_wait_for_bits_clear(void *ptr
, u32 value
, u32 bits
)
20 ddr_out32(ptr
, value
);
22 while (ddr_in32(ptr
) & bits
) {
27 puts("Error: A007865 wait for clear timeout.\n");
29 #endif /* CONFIG_SYS_FSL_ERRATUM_A008511 */
31 #if (CONFIG_CHIP_SELECTS_PER_CTRL > 4)
32 #error Invalid setting for CONFIG_CHIP_SELECTS_PER_CTRL
36 * regs has the to-be-set values for DDR controller registers
37 * ctrl_num is the DDR controller number
38 * step: 0 goes through the initialization in one pass
39 * 1 sets registers and returns before enabling controller
40 * 2 resumes from step 1 and continues to initialize
41 * Dividing the initialization to two steps to deassert DDR reset signal
42 * to comply with JEDEC specs for RDIMMs.
44 void fsl_ddr_set_memctl_regs(const fsl_ddr_cfg_regs_t
*regs
,
45 unsigned int ctrl_num
, int step
)
47 unsigned int i
, bus_width
;
48 struct ccsr_ddr __iomem
*ddr
;
50 u32 total_gb_size_per_controller
;
52 #ifdef CONFIG_SYS_FSL_ERRATUM_A008511
54 u32 vref_seq1
[3] = {0x80, 0x96, 0x16}; /* for range 1 */
55 u32 vref_seq2
[3] = {0xc0, 0xf0, 0x70}; /* for range 2 */
56 u32
*vref_seq
= vref_seq1
;
58 #ifdef CONFIG_SYS_FSL_ERRATUM_A009942
62 #ifdef CONFIG_FSL_DDR_BIST
63 u32 mtcr
, err_detect
, err_sbe
;
64 u32 cs0_bnds
, cs1_bnds
, cs2_bnds
, cs3_bnds
, cs0_config
;
66 #ifdef CONFIG_FSL_DDR_BIST
67 char buffer
[CONFIG_SYS_CBSIZE
];
72 ddr
= (void *)CONFIG_SYS_FSL_DDR_ADDR
;
74 #if defined(CONFIG_SYS_FSL_DDR2_ADDR) && (CONFIG_NUM_DDR_CONTROLLERS > 1)
76 ddr
= (void *)CONFIG_SYS_FSL_DDR2_ADDR
;
79 #if defined(CONFIG_SYS_FSL_DDR3_ADDR) && (CONFIG_NUM_DDR_CONTROLLERS > 2)
81 ddr
= (void *)CONFIG_SYS_FSL_DDR3_ADDR
;
84 #if defined(CONFIG_SYS_FSL_DDR4_ADDR) && (CONFIG_NUM_DDR_CONTROLLERS > 3)
86 ddr
= (void *)CONFIG_SYS_FSL_DDR4_ADDR
;
90 printf("%s unexpected ctrl_num = %u\n", __func__
, ctrl_num
);
98 ddr_out32(&ddr
->eor
, regs
->ddr_eor
);
100 ddr_out32(&ddr
->sdram_clk_cntl
, regs
->ddr_sdram_clk_cntl
);
102 for (i
= 0; i
< CONFIG_CHIP_SELECTS_PER_CTRL
; i
++) {
104 ddr_out32(&ddr
->cs0_bnds
, regs
->cs
[i
].bnds
);
105 ddr_out32(&ddr
->cs0_config
, regs
->cs
[i
].config
);
106 ddr_out32(&ddr
->cs0_config_2
, regs
->cs
[i
].config_2
);
109 ddr_out32(&ddr
->cs1_bnds
, regs
->cs
[i
].bnds
);
110 ddr_out32(&ddr
->cs1_config
, regs
->cs
[i
].config
);
111 ddr_out32(&ddr
->cs1_config_2
, regs
->cs
[i
].config_2
);
114 ddr_out32(&ddr
->cs2_bnds
, regs
->cs
[i
].bnds
);
115 ddr_out32(&ddr
->cs2_config
, regs
->cs
[i
].config
);
116 ddr_out32(&ddr
->cs2_config_2
, regs
->cs
[i
].config_2
);
119 ddr_out32(&ddr
->cs3_bnds
, regs
->cs
[i
].bnds
);
120 ddr_out32(&ddr
->cs3_config
, regs
->cs
[i
].config
);
121 ddr_out32(&ddr
->cs3_config_2
, regs
->cs
[i
].config_2
);
125 ddr_out32(&ddr
->timing_cfg_3
, regs
->timing_cfg_3
);
126 ddr_out32(&ddr
->timing_cfg_0
, regs
->timing_cfg_0
);
127 ddr_out32(&ddr
->timing_cfg_1
, regs
->timing_cfg_1
);
128 ddr_out32(&ddr
->timing_cfg_2
, regs
->timing_cfg_2
);
129 ddr_out32(&ddr
->timing_cfg_4
, regs
->timing_cfg_4
);
130 ddr_out32(&ddr
->timing_cfg_5
, regs
->timing_cfg_5
);
131 ddr_out32(&ddr
->timing_cfg_6
, regs
->timing_cfg_6
);
132 ddr_out32(&ddr
->timing_cfg_7
, regs
->timing_cfg_7
);
133 ddr_out32(&ddr
->timing_cfg_8
, regs
->timing_cfg_8
);
134 ddr_out32(&ddr
->timing_cfg_9
, regs
->timing_cfg_9
);
135 ddr_out32(&ddr
->ddr_zq_cntl
, regs
->ddr_zq_cntl
);
136 ddr_out32(&ddr
->dq_map_0
, regs
->dq_map_0
);
137 ddr_out32(&ddr
->dq_map_1
, regs
->dq_map_1
);
138 ddr_out32(&ddr
->dq_map_2
, regs
->dq_map_2
);
139 ddr_out32(&ddr
->dq_map_3
, regs
->dq_map_3
);
140 ddr_out32(&ddr
->sdram_cfg_3
, regs
->ddr_sdram_cfg_3
);
141 ddr_out32(&ddr
->sdram_mode
, regs
->ddr_sdram_mode
);
142 ddr_out32(&ddr
->sdram_mode_2
, regs
->ddr_sdram_mode_2
);
143 ddr_out32(&ddr
->sdram_mode_3
, regs
->ddr_sdram_mode_3
);
144 ddr_out32(&ddr
->sdram_mode_4
, regs
->ddr_sdram_mode_4
);
145 ddr_out32(&ddr
->sdram_mode_5
, regs
->ddr_sdram_mode_5
);
146 ddr_out32(&ddr
->sdram_mode_6
, regs
->ddr_sdram_mode_6
);
147 ddr_out32(&ddr
->sdram_mode_7
, regs
->ddr_sdram_mode_7
);
148 ddr_out32(&ddr
->sdram_mode_8
, regs
->ddr_sdram_mode_8
);
149 ddr_out32(&ddr
->sdram_mode_9
, regs
->ddr_sdram_mode_9
);
150 ddr_out32(&ddr
->sdram_mode_10
, regs
->ddr_sdram_mode_10
);
151 ddr_out32(&ddr
->sdram_mode_11
, regs
->ddr_sdram_mode_11
);
152 ddr_out32(&ddr
->sdram_mode_12
, regs
->ddr_sdram_mode_12
);
153 ddr_out32(&ddr
->sdram_mode_13
, regs
->ddr_sdram_mode_13
);
154 ddr_out32(&ddr
->sdram_mode_14
, regs
->ddr_sdram_mode_14
);
155 ddr_out32(&ddr
->sdram_mode_15
, regs
->ddr_sdram_mode_15
);
156 ddr_out32(&ddr
->sdram_mode_16
, regs
->ddr_sdram_mode_16
);
157 ddr_out32(&ddr
->sdram_md_cntl
, regs
->ddr_sdram_md_cntl
);
158 ddr_out32(&ddr
->sdram_interval
, regs
->ddr_sdram_interval
);
159 ddr_out32(&ddr
->sdram_data_init
, regs
->ddr_data_init
);
160 ddr_out32(&ddr
->ddr_wrlvl_cntl
, regs
->ddr_wrlvl_cntl
);
161 #ifndef CONFIG_SYS_FSL_DDR_EMU
163 * Skip these two registers if running on emulator
164 * because emulator doesn't have skew between bytes.
167 if (regs
->ddr_wrlvl_cntl_2
)
168 ddr_out32(&ddr
->ddr_wrlvl_cntl_2
, regs
->ddr_wrlvl_cntl_2
);
169 if (regs
->ddr_wrlvl_cntl_3
)
170 ddr_out32(&ddr
->ddr_wrlvl_cntl_3
, regs
->ddr_wrlvl_cntl_3
);
173 ddr_out32(&ddr
->ddr_sr_cntr
, regs
->ddr_sr_cntr
);
174 ddr_out32(&ddr
->ddr_sdram_rcw_1
, regs
->ddr_sdram_rcw_1
);
175 ddr_out32(&ddr
->ddr_sdram_rcw_2
, regs
->ddr_sdram_rcw_2
);
176 ddr_out32(&ddr
->ddr_sdram_rcw_3
, regs
->ddr_sdram_rcw_3
);
177 ddr_out32(&ddr
->ddr_sdram_rcw_4
, regs
->ddr_sdram_rcw_4
);
178 ddr_out32(&ddr
->ddr_sdram_rcw_5
, regs
->ddr_sdram_rcw_5
);
179 ddr_out32(&ddr
->ddr_sdram_rcw_6
, regs
->ddr_sdram_rcw_6
);
180 ddr_out32(&ddr
->ddr_cdr1
, regs
->ddr_cdr1
);
181 #ifdef CONFIG_DEEP_SLEEP
182 if (is_warm_boot()) {
183 ddr_out32(&ddr
->sdram_cfg_2
,
184 regs
->ddr_sdram_cfg_2
& ~SDRAM_CFG2_D_INIT
);
185 ddr_out32(&ddr
->init_addr
, CONFIG_SYS_SDRAM_BASE
);
186 ddr_out32(&ddr
->init_ext_addr
, DDR_INIT_ADDR_EXT_UIA
);
188 /* DRAM VRef will not be trained */
189 ddr_out32(&ddr
->ddr_cdr2
,
190 regs
->ddr_cdr2
& ~DDR_CDR2_VREF_TRAIN_EN
);
194 ddr_out32(&ddr
->sdram_cfg_2
, regs
->ddr_sdram_cfg_2
);
195 ddr_out32(&ddr
->init_addr
, regs
->ddr_init_addr
);
196 ddr_out32(&ddr
->init_ext_addr
, regs
->ddr_init_ext_addr
);
197 ddr_out32(&ddr
->ddr_cdr2
, regs
->ddr_cdr2
);
199 ddr_out32(&ddr
->err_disable
, regs
->err_disable
);
200 ddr_out32(&ddr
->err_int_en
, regs
->err_int_en
);
201 for (i
= 0; i
< 32; i
++) {
202 if (regs
->debug
[i
]) {
203 debug("Write to debug_%d as %08x\n",
204 i
+1, regs
->debug
[i
]);
205 ddr_out32(&ddr
->debug
[i
], regs
->debug
[i
]);
208 #ifdef CONFIG_SYS_FSL_ERRATUM_A008378
209 /* Erratum applies when accumulated ECC is used, or DBI is enabled */
210 #define IS_ACC_ECC_EN(v) ((v) & 0x4)
211 #define IS_DBI(v) ((((v) >> 12) & 0x3) == 0x2)
212 if (has_erratum_a008378()) {
213 if (IS_ACC_ECC_EN(regs
->ddr_sdram_cfg
) ||
214 IS_DBI(regs
->ddr_sdram_cfg_3
))
215 ddr_setbits32(&ddr
->debug
[28], 0x9 << 20);
219 #ifdef CONFIG_SYS_FSL_ERRATUM_A008511
221 /* This erraum only applies to verion 5.2.0 */
222 if (fsl_ddr_get_version(ctrl_num
) == 0x50200) {
223 /* Disable DRAM VRef training */
224 ddr_out32(&ddr
->ddr_cdr2
,
225 regs
->ddr_cdr2
& ~DDR_CDR2_VREF_TRAIN_EN
);
227 ddr_out32(&ddr
->debug
[28], 0x400);
229 ddr_out32(&ddr
->sdram_cfg_2
,
230 regs
->ddr_sdram_cfg_2
& ~SDRAM_CFG2_D_INIT
);
231 ddr_out32(&ddr
->debug
[25], 0x9000);
235 #ifdef CONFIG_SYS_FSL_ERRATUM_A009942
236 ddr_freq
= get_ddr_freq(ctrl_num
) / 1000000;
237 tmp
= ddr_in32(&ddr
->debug
[28]);
238 if (ddr_freq
<= 1333)
239 ddr_out32(&ddr
->debug
[28], tmp
| 0x0080006a);
240 else if (ddr_freq
<= 1600)
241 ddr_out32(&ddr
->debug
[28], tmp
| 0x0070006f);
242 else if (ddr_freq
<= 1867)
243 ddr_out32(&ddr
->debug
[28], tmp
| 0x00700076);
244 else if (ddr_freq
<= 2133)
245 ddr_out32(&ddr
->debug
[28], tmp
| 0x0060007b);
249 * For RDIMMs, JEDEC spec requires clocks to be stable before reset is
250 * deasserted. Clocks start when any chip select is enabled and clock
251 * control register is set. Because all DDR components are connected to
252 * one reset signal, this needs to be done in two steps. Step 1 is to
253 * get the clocks started. Step 2 resumes after reset signal is
262 /* Set, but do not enable the memory */
263 temp_sdram_cfg
= regs
->ddr_sdram_cfg
;
264 temp_sdram_cfg
&= ~(SDRAM_CFG_MEM_EN
);
265 ddr_out32(&ddr
->sdram_cfg
, temp_sdram_cfg
);
268 * 500 painful micro-seconds must elapse between
269 * the DDR clock setup and the DDR config enable.
270 * DDR2 need 200 us, and DDR3 need 500 us from spec,
271 * we choose the max, that is 500 us for all of case.
277 #ifdef CONFIG_DEEP_SLEEP
278 if (is_warm_boot()) {
279 /* enter self-refresh */
280 temp_sdram_cfg
= ddr_in32(&ddr
->sdram_cfg_2
);
281 temp_sdram_cfg
|= SDRAM_CFG2_FRC_SR
;
282 ddr_out32(&ddr
->sdram_cfg_2
, temp_sdram_cfg
);
283 /* do board specific memory setup */
284 board_mem_sleep_setup();
286 temp_sdram_cfg
= (ddr_in32(&ddr
->sdram_cfg
) | SDRAM_CFG_BI
);
289 temp_sdram_cfg
= ddr_in32(&ddr
->sdram_cfg
) & ~SDRAM_CFG_BI
;
290 /* Let the controller go */
291 ddr_out32(&ddr
->sdram_cfg
, temp_sdram_cfg
| SDRAM_CFG_MEM_EN
);
295 #ifdef CONFIG_SYS_FSL_ERRATUM_A008511
297 /* This erraum only applies to verion 5.2.0 */
298 if (fsl_ddr_get_version(ctrl_num
) == 0x50200) {
301 while (!(ddr_in32(&ddr
->debug
[1]) & 0x2) &&
307 printf("Controler %d timeout, debug_2 = %x\n",
308 ctrl_num
, ddr_in32(&ddr
->debug
[1]));
311 /* The vref setting sequence is different for range 2 */
312 if (regs
->ddr_cdr2
& DDR_CDR2_VREF_RANGE_2
)
313 vref_seq
= vref_seq2
;
316 for (i
= 0; i
< CONFIG_CHIP_SELECTS_PER_CTRL
; i
++) {
317 if (!(regs
->cs
[i
].config
& SDRAM_CS_CONFIG_EN
))
320 mr6
= (regs
->ddr_sdram_mode_10
>> 16) |
325 temp32
= mr6
| vref_seq
[0];
326 set_wait_for_bits_clear(&ddr
->sdram_md_cntl
,
327 temp32
, MD_CNTL_MD_EN
);
329 debug("MR6 = 0x%08x\n", temp32
);
330 temp32
= mr6
| vref_seq
[1];
331 set_wait_for_bits_clear(&ddr
->sdram_md_cntl
,
332 temp32
, MD_CNTL_MD_EN
);
334 debug("MR6 = 0x%08x\n", temp32
);
335 temp32
= mr6
| vref_seq
[2];
336 set_wait_for_bits_clear(&ddr
->sdram_md_cntl
,
337 temp32
, MD_CNTL_MD_EN
);
339 debug("MR6 = 0x%08x\n", temp32
);
341 ddr_out32(&ddr
->sdram_md_cntl
, 0);
342 ddr_out32(&ddr
->debug
[28], 0); /* Enable deskew */
343 ddr_out32(&ddr
->debug
[1], 0x400); /* restart deskew */
346 while (!(ddr_in32(&ddr
->debug
[1]) & 0x2) &&
352 printf("Controler %d timeout, debug_2 = %x\n",
353 ctrl_num
, ddr_in32(&ddr
->debug
[1]));
356 ddr_out32(&ddr
->sdram_cfg_2
, regs
->ddr_sdram_cfg_2
);
358 #endif /* CONFIG_SYS_FSL_ERRATUM_A008511 */
360 total_gb_size_per_controller
= 0;
361 for (i
= 0; i
< CONFIG_CHIP_SELECTS_PER_CTRL
; i
++) {
362 if (!(regs
->cs
[i
].config
& 0x80000000))
364 total_gb_size_per_controller
+= 1 << (
365 ((regs
->cs
[i
].config
>> 14) & 0x3) + 2 +
366 ((regs
->cs
[i
].config
>> 8) & 0x7) + 12 +
367 ((regs
->cs
[i
].config
>> 4) & 0x3) + 0 +
368 ((regs
->cs
[i
].config
>> 0) & 0x7) + 8 +
369 3 - ((regs
->ddr_sdram_cfg
>> 19) & 0x3) -
370 26); /* minus 26 (count of 64M) */
372 if (fsl_ddr_get_intl3r() & 0x80000000) /* 3-way interleaving */
373 total_gb_size_per_controller
*= 3;
374 else if (regs
->cs
[0].config
& 0x20000000) /* 2-way interleaving */
375 total_gb_size_per_controller
<<= 1;
377 * total memory / bus width = transactions needed
378 * transactions needed / data rate = seconds
379 * to add plenty of buffer, double the time
380 * For example, 2GB on 666MT/s 64-bit bus takes about 402ms
381 * Let's wait for 800ms
383 bus_width
= 3 - ((ddr_in32(&ddr
->sdram_cfg
) & SDRAM_CFG_DBW_MASK
)
384 >> SDRAM_CFG_DBW_SHIFT
);
385 timeout
= ((total_gb_size_per_controller
<< (6 - bus_width
)) * 100 /
386 (get_ddr_freq(ctrl_num
) >> 20)) << 2;
387 total_gb_size_per_controller
>>= 4; /* shift down to gb size */
388 debug("total %d GB\n", total_gb_size_per_controller
);
389 debug("Need to wait up to %d * 10ms\n", timeout
);
391 /* Poll DDR_SDRAM_CFG_2[D_INIT] bit until auto-data init is done. */
392 while ((ddr_in32(&ddr
->sdram_cfg_2
) & SDRAM_CFG2_D_INIT
) &&
394 udelay(10000); /* throttle polling rate */
399 printf("Waiting for D_INIT timeout. Memory may not work.\n");
400 #ifdef CONFIG_DEEP_SLEEP
401 if (is_warm_boot()) {
402 /* exit self-refresh */
403 temp_sdram_cfg
= ddr_in32(&ddr
->sdram_cfg_2
);
404 temp_sdram_cfg
&= ~SDRAM_CFG2_FRC_SR
;
405 ddr_out32(&ddr
->sdram_cfg_2
, temp_sdram_cfg
);
409 #ifdef CONFIG_FSL_DDR_BIST
410 #define BIST_PATTERN1 0xFFFFFFFF
411 #define BIST_PATTERN2 0x0
412 #define BIST_CR 0x80010000
413 #define BIST_CR_EN 0x80000000
414 #define BIST_CR_STAT 0x00000001
415 #define CTLR_INTLV_MASK 0x20000000
416 /* Perform build-in test on memory. Three-way interleaving is not yet
417 * supported by this code. */
418 if (getenv_f("ddr_bist", buffer
, CONFIG_SYS_CBSIZE
) >= 0) {
419 puts("Running BIST test. This will take a while...");
420 cs0_config
= ddr_in32(&ddr
->cs0_config
);
421 cs0_bnds
= ddr_in32(&ddr
->cs0_bnds
);
422 cs1_bnds
= ddr_in32(&ddr
->cs1_bnds
);
423 cs2_bnds
= ddr_in32(&ddr
->cs2_bnds
);
424 cs3_bnds
= ddr_in32(&ddr
->cs3_bnds
);
425 if (cs0_config
& CTLR_INTLV_MASK
) {
426 /* set bnds to non-interleaving */
427 ddr_out32(&ddr
->cs0_bnds
, (cs0_bnds
& 0xfffefffe) >> 1);
428 ddr_out32(&ddr
->cs1_bnds
, (cs1_bnds
& 0xfffefffe) >> 1);
429 ddr_out32(&ddr
->cs2_bnds
, (cs2_bnds
& 0xfffefffe) >> 1);
430 ddr_out32(&ddr
->cs3_bnds
, (cs3_bnds
& 0xfffefffe) >> 1);
432 ddr_out32(&ddr
->mtp1
, BIST_PATTERN1
);
433 ddr_out32(&ddr
->mtp2
, BIST_PATTERN1
);
434 ddr_out32(&ddr
->mtp3
, BIST_PATTERN2
);
435 ddr_out32(&ddr
->mtp4
, BIST_PATTERN2
);
436 ddr_out32(&ddr
->mtp5
, BIST_PATTERN1
);
437 ddr_out32(&ddr
->mtp6
, BIST_PATTERN1
);
438 ddr_out32(&ddr
->mtp7
, BIST_PATTERN2
);
439 ddr_out32(&ddr
->mtp8
, BIST_PATTERN2
);
440 ddr_out32(&ddr
->mtp9
, BIST_PATTERN1
);
441 ddr_out32(&ddr
->mtp10
, BIST_PATTERN2
);
443 ddr_out32(&ddr
->mtcr
, mtcr
);
445 while (timeout
> 0 && (mtcr
& BIST_CR_EN
)) {
448 mtcr
= ddr_in32(&ddr
->mtcr
);
454 err_detect
= ddr_in32(&ddr
->err_detect
);
455 err_sbe
= ddr_in32(&ddr
->err_sbe
);
456 if (mtcr
& BIST_CR_STAT
) {
457 printf("BIST test failed on controller %d.\n",
460 if (err_detect
|| (err_sbe
& 0xffff)) {
461 printf("ECC error detected on controller %d.\n",
465 if (cs0_config
& CTLR_INTLV_MASK
) {
466 /* restore bnds registers */
467 ddr_out32(&ddr
->cs0_bnds
, cs0_bnds
);
468 ddr_out32(&ddr
->cs1_bnds
, cs1_bnds
);
469 ddr_out32(&ddr
->cs2_bnds
, cs2_bnds
);
470 ddr_out32(&ddr
->cs3_bnds
, cs3_bnds
);