blob: aca04504f6ca6f9ce5cb54a5faedf091677954fa [file] [log] [blame]
Yann Gautiercaf575b2018-07-24 17:18:19 +02001/*
2 * Copyright (C) 2018, STMicroelectronics - All Rights Reserved
3 *
4 * SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
5 */
6
Antonio Nino Diaze0f90632018-12-14 00:18:21 +00007#include <stddef.h>
8
Yann Gautier57e282b2019-01-07 11:17:24 +01009#include <platform_def.h>
10
Yann Gautiercaf575b2018-07-24 17:18:19 +020011#include <arch.h>
12#include <arch_helpers.h>
Antonio Nino Diaze0f90632018-12-14 00:18:21 +000013#include <common/debug.h>
14#include <drivers/delay_timer.h>
15#include <drivers/st/stm32mp1_clk.h>
16#include <drivers/st/stm32mp1_ddr.h>
17#include <drivers/st/stm32mp1_ddr_regs.h>
18#include <drivers/st/stm32mp1_pmic.h>
19#include <drivers/st/stm32mp1_pwr.h>
20#include <drivers/st/stm32mp1_ram.h>
21#include <drivers/st/stm32mp1_rcc.h>
Yann Gautiercaf575b2018-07-24 17:18:19 +020022#include <dt-bindings/clock/stm32mp1-clks.h>
Antonio Nino Diaze0f90632018-12-14 00:18:21 +000023#include <lib/mmio.h>
24#include <plat/common/platform.h>
25
Yann Gautiercaf575b2018-07-24 17:18:19 +020026struct reg_desc {
27 const char *name;
28 uint16_t offset; /* Offset for base address */
29 uint8_t par_offset; /* Offset for parameter array */
30};
31
32#define INVALID_OFFSET 0xFFU
33
34#define TIMESLOT_1US (plat_get_syscnt_freq2() / 1000000U)
35
36#define DDRCTL_REG(x, y) \
37 { \
38 .name = #x, \
39 .offset = offsetof(struct stm32mp1_ddrctl, x), \
40 .par_offset = offsetof(struct y, x) \
41 }
42
43#define DDRPHY_REG(x, y) \
44 { \
45 .name = #x, \
46 .offset = offsetof(struct stm32mp1_ddrphy, x), \
47 .par_offset = offsetof(struct y, x) \
48 }
49
50#define DDRCTL_REG_REG(x) DDRCTL_REG(x, stm32mp1_ddrctrl_reg)
51static const struct reg_desc ddr_reg[] = {
52 DDRCTL_REG_REG(mstr),
53 DDRCTL_REG_REG(mrctrl0),
54 DDRCTL_REG_REG(mrctrl1),
55 DDRCTL_REG_REG(derateen),
56 DDRCTL_REG_REG(derateint),
57 DDRCTL_REG_REG(pwrctl),
58 DDRCTL_REG_REG(pwrtmg),
59 DDRCTL_REG_REG(hwlpctl),
60 DDRCTL_REG_REG(rfshctl0),
61 DDRCTL_REG_REG(rfshctl3),
62 DDRCTL_REG_REG(crcparctl0),
63 DDRCTL_REG_REG(zqctl0),
64 DDRCTL_REG_REG(dfitmg0),
65 DDRCTL_REG_REG(dfitmg1),
66 DDRCTL_REG_REG(dfilpcfg0),
67 DDRCTL_REG_REG(dfiupd0),
68 DDRCTL_REG_REG(dfiupd1),
69 DDRCTL_REG_REG(dfiupd2),
70 DDRCTL_REG_REG(dfiphymstr),
71 DDRCTL_REG_REG(odtmap),
72 DDRCTL_REG_REG(dbg0),
73 DDRCTL_REG_REG(dbg1),
74 DDRCTL_REG_REG(dbgcmd),
75 DDRCTL_REG_REG(poisoncfg),
76 DDRCTL_REG_REG(pccfg),
77};
78
79#define DDRCTL_REG_TIMING(x) DDRCTL_REG(x, stm32mp1_ddrctrl_timing)
80static const struct reg_desc ddr_timing[] = {
81 DDRCTL_REG_TIMING(rfshtmg),
82 DDRCTL_REG_TIMING(dramtmg0),
83 DDRCTL_REG_TIMING(dramtmg1),
84 DDRCTL_REG_TIMING(dramtmg2),
85 DDRCTL_REG_TIMING(dramtmg3),
86 DDRCTL_REG_TIMING(dramtmg4),
87 DDRCTL_REG_TIMING(dramtmg5),
88 DDRCTL_REG_TIMING(dramtmg6),
89 DDRCTL_REG_TIMING(dramtmg7),
90 DDRCTL_REG_TIMING(dramtmg8),
91 DDRCTL_REG_TIMING(dramtmg14),
92 DDRCTL_REG_TIMING(odtcfg),
93};
94
95#define DDRCTL_REG_MAP(x) DDRCTL_REG(x, stm32mp1_ddrctrl_map)
96static const struct reg_desc ddr_map[] = {
97 DDRCTL_REG_MAP(addrmap1),
98 DDRCTL_REG_MAP(addrmap2),
99 DDRCTL_REG_MAP(addrmap3),
100 DDRCTL_REG_MAP(addrmap4),
101 DDRCTL_REG_MAP(addrmap5),
102 DDRCTL_REG_MAP(addrmap6),
103 DDRCTL_REG_MAP(addrmap9),
104 DDRCTL_REG_MAP(addrmap10),
105 DDRCTL_REG_MAP(addrmap11),
106};
107
108#define DDRCTL_REG_PERF(x) DDRCTL_REG(x, stm32mp1_ddrctrl_perf)
109static const struct reg_desc ddr_perf[] = {
110 DDRCTL_REG_PERF(sched),
111 DDRCTL_REG_PERF(sched1),
112 DDRCTL_REG_PERF(perfhpr1),
113 DDRCTL_REG_PERF(perflpr1),
114 DDRCTL_REG_PERF(perfwr1),
115 DDRCTL_REG_PERF(pcfgr_0),
116 DDRCTL_REG_PERF(pcfgw_0),
117 DDRCTL_REG_PERF(pcfgqos0_0),
118 DDRCTL_REG_PERF(pcfgqos1_0),
119 DDRCTL_REG_PERF(pcfgwqos0_0),
120 DDRCTL_REG_PERF(pcfgwqos1_0),
121 DDRCTL_REG_PERF(pcfgr_1),
122 DDRCTL_REG_PERF(pcfgw_1),
123 DDRCTL_REG_PERF(pcfgqos0_1),
124 DDRCTL_REG_PERF(pcfgqos1_1),
125 DDRCTL_REG_PERF(pcfgwqos0_1),
126 DDRCTL_REG_PERF(pcfgwqos1_1),
127};
128
129#define DDRPHY_REG_REG(x) DDRPHY_REG(x, stm32mp1_ddrphy_reg)
130static const struct reg_desc ddrphy_reg[] = {
131 DDRPHY_REG_REG(pgcr),
132 DDRPHY_REG_REG(aciocr),
133 DDRPHY_REG_REG(dxccr),
134 DDRPHY_REG_REG(dsgcr),
135 DDRPHY_REG_REG(dcr),
136 DDRPHY_REG_REG(odtcr),
137 DDRPHY_REG_REG(zq0cr1),
138 DDRPHY_REG_REG(dx0gcr),
139 DDRPHY_REG_REG(dx1gcr),
140 DDRPHY_REG_REG(dx2gcr),
141 DDRPHY_REG_REG(dx3gcr),
142};
143
144#define DDRPHY_REG_TIMING(x) DDRPHY_REG(x, stm32mp1_ddrphy_timing)
145static const struct reg_desc ddrphy_timing[] = {
146 DDRPHY_REG_TIMING(ptr0),
147 DDRPHY_REG_TIMING(ptr1),
148 DDRPHY_REG_TIMING(ptr2),
149 DDRPHY_REG_TIMING(dtpr0),
150 DDRPHY_REG_TIMING(dtpr1),
151 DDRPHY_REG_TIMING(dtpr2),
152 DDRPHY_REG_TIMING(mr0),
153 DDRPHY_REG_TIMING(mr1),
154 DDRPHY_REG_TIMING(mr2),
155 DDRPHY_REG_TIMING(mr3),
156};
157
158#define DDRPHY_REG_CAL(x) DDRPHY_REG(x, stm32mp1_ddrphy_cal)
159static const struct reg_desc ddrphy_cal[] = {
160 DDRPHY_REG_CAL(dx0dllcr),
161 DDRPHY_REG_CAL(dx0dqtr),
162 DDRPHY_REG_CAL(dx0dqstr),
163 DDRPHY_REG_CAL(dx1dllcr),
164 DDRPHY_REG_CAL(dx1dqtr),
165 DDRPHY_REG_CAL(dx1dqstr),
166 DDRPHY_REG_CAL(dx2dllcr),
167 DDRPHY_REG_CAL(dx2dqtr),
168 DDRPHY_REG_CAL(dx2dqstr),
169 DDRPHY_REG_CAL(dx3dllcr),
170 DDRPHY_REG_CAL(dx3dqtr),
171 DDRPHY_REG_CAL(dx3dqstr),
172};
173
174#define DDR_REG_DYN(x) \
175 { \
176 .name = #x, \
177 .offset = offsetof(struct stm32mp1_ddrctl, x), \
178 .par_offset = INVALID_OFFSET \
179 }
180
181static const struct reg_desc ddr_dyn[] = {
182 DDR_REG_DYN(stat),
183 DDR_REG_DYN(init0),
184 DDR_REG_DYN(dfimisc),
185 DDR_REG_DYN(dfistat),
186 DDR_REG_DYN(swctl),
187 DDR_REG_DYN(swstat),
188 DDR_REG_DYN(pctrl_0),
189 DDR_REG_DYN(pctrl_1),
190};
191
192#define DDRPHY_REG_DYN(x) \
193 { \
194 .name = #x, \
195 .offset = offsetof(struct stm32mp1_ddrphy, x), \
196 .par_offset = INVALID_OFFSET \
197 }
198
199static const struct reg_desc ddrphy_dyn[] = {
200 DDRPHY_REG_DYN(pir),
201 DDRPHY_REG_DYN(pgsr),
202};
203
204enum reg_type {
205 REG_REG,
206 REG_TIMING,
207 REG_PERF,
208 REG_MAP,
209 REGPHY_REG,
210 REGPHY_TIMING,
211 REGPHY_CAL,
212/*
213 * Dynamic registers => managed in driver or not changed,
214 * can be dumped in interactive mode.
215 */
216 REG_DYN,
217 REGPHY_DYN,
218 REG_TYPE_NB
219};
220
221enum base_type {
222 DDR_BASE,
223 DDRPHY_BASE,
224 NONE_BASE
225};
226
227struct ddr_reg_info {
228 const char *name;
229 const struct reg_desc *desc;
230 uint8_t size;
231 enum base_type base;
232};
233
234static const struct ddr_reg_info ddr_registers[REG_TYPE_NB] = {
235 [REG_REG] = {
236 "static", ddr_reg, ARRAY_SIZE(ddr_reg), DDR_BASE
237 },
238 [REG_TIMING] = {
239 "timing", ddr_timing, ARRAY_SIZE(ddr_timing), DDR_BASE
240 },
241 [REG_PERF] = {
242 "perf", ddr_perf, ARRAY_SIZE(ddr_perf), DDR_BASE
243 },
244 [REG_MAP] = {
245 "map", ddr_map, ARRAY_SIZE(ddr_map), DDR_BASE
246 },
247 [REGPHY_REG] = {
248 "static", ddrphy_reg, ARRAY_SIZE(ddrphy_reg), DDRPHY_BASE
249 },
250 [REGPHY_TIMING] = {
251 "timing", ddrphy_timing, ARRAY_SIZE(ddrphy_timing), DDRPHY_BASE
252 },
253 [REGPHY_CAL] = {
254 "cal", ddrphy_cal, ARRAY_SIZE(ddrphy_cal), DDRPHY_BASE
255 },
256 [REG_DYN] = {
257 "dyn", ddr_dyn, ARRAY_SIZE(ddr_dyn), DDR_BASE
258 },
259 [REGPHY_DYN] = {
260 "dyn", ddrphy_dyn, ARRAY_SIZE(ddrphy_dyn), DDRPHY_BASE
261 },
262};
263
264static uint32_t get_base_addr(const struct ddr_info *priv, enum base_type base)
265{
266 if (base == DDRPHY_BASE) {
267 return (uint32_t)priv->phy;
268 } else {
269 return (uint32_t)priv->ctl;
270 }
271}
272
273static void set_reg(const struct ddr_info *priv,
274 enum reg_type type,
275 const void *param)
276{
277 unsigned int i;
278 unsigned int *ptr, value;
279 enum base_type base = ddr_registers[type].base;
280 uint32_t base_addr = get_base_addr(priv, base);
281 const struct reg_desc *desc = ddr_registers[type].desc;
282
283 VERBOSE("init %s\n", ddr_registers[type].name);
284 for (i = 0; i < ddr_registers[type].size; i++) {
285 ptr = (unsigned int *)(base_addr + desc[i].offset);
286 if (desc[i].par_offset == INVALID_OFFSET) {
287 ERROR("invalid parameter offset for %s", desc[i].name);
288 panic();
289 } else {
290 value = *((uint32_t *)((uint32_t)param +
291 desc[i].par_offset));
292 mmio_write_32((uint32_t)ptr, value);
293 }
294 }
295}
296
297static void stm32mp1_ddrphy_idone_wait(struct stm32mp1_ddrphy *phy)
298{
299 uint32_t pgsr;
300 int error = 0;
301 unsigned long start;
302 unsigned long time0, time;
303
304 start = get_timer(0);
305 time0 = start;
306
307 do {
308 pgsr = mmio_read_32((uint32_t)&phy->pgsr);
309 time = get_timer(start);
310 if (time != time0) {
311 VERBOSE(" > [0x%x] pgsr = 0x%x &\n",
312 (uint32_t)&phy->pgsr, pgsr);
313 VERBOSE(" [0x%x] pir = 0x%x (time=%x)\n",
314 (uint32_t)&phy->pir,
315 mmio_read_32((uint32_t)&phy->pir),
316 (uint32_t)time);
317 }
318
319 time0 = time;
320 if (time > plat_get_syscnt_freq2()) {
321 panic();
322 }
323 if ((pgsr & DDRPHYC_PGSR_DTERR) != 0U) {
324 VERBOSE("DQS Gate Trainig Error\n");
325 error++;
326 }
327 if ((pgsr & DDRPHYC_PGSR_DTIERR) != 0U) {
328 VERBOSE("DQS Gate Trainig Intermittent Error\n");
329 error++;
330 }
331 if ((pgsr & DDRPHYC_PGSR_DFTERR) != 0U) {
332 VERBOSE("DQS Drift Error\n");
333 error++;
334 }
335 if ((pgsr & DDRPHYC_PGSR_RVERR) != 0U) {
336 VERBOSE("Read Valid Training Error\n");
337 error++;
338 }
339 if ((pgsr & DDRPHYC_PGSR_RVEIRR) != 0U) {
340 VERBOSE("Read Valid Training Intermittent Error\n");
341 error++;
342 }
343 } while ((pgsr & DDRPHYC_PGSR_IDONE) == 0U && error == 0);
344 VERBOSE("\n[0x%x] pgsr = 0x%x\n",
345 (uint32_t)&phy->pgsr, pgsr);
346}
347
348static void stm32mp1_ddrphy_init(struct stm32mp1_ddrphy *phy, uint32_t pir)
349{
350 uint32_t pir_init = pir | DDRPHYC_PIR_INIT;
351
352 mmio_write_32((uint32_t)&phy->pir, pir_init);
353 VERBOSE("[0x%x] pir = 0x%x -> 0x%x\n",
354 (uint32_t)&phy->pir, pir_init,
355 mmio_read_32((uint32_t)&phy->pir));
356
357 /* Need to wait 10 configuration clock before start polling */
358 udelay(10);
359
360 /* Wait DRAM initialization and Gate Training Evaluation complete */
361 stm32mp1_ddrphy_idone_wait(phy);
362}
363
364/* Start quasi dynamic register update */
365static void stm32mp1_start_sw_done(struct stm32mp1_ddrctl *ctl)
366{
367 mmio_clrbits_32((uint32_t)&ctl->swctl, DDRCTRL_SWCTL_SW_DONE);
368 VERBOSE("[0x%x] swctl = 0x%x\n",
369 (uint32_t)&ctl->swctl, mmio_read_32((uint32_t)&ctl->swctl));
370}
371
372/* Wait quasi dynamic register update */
373static void stm32mp1_wait_sw_done_ack(struct stm32mp1_ddrctl *ctl)
374{
375 unsigned long start;
376 uint32_t swstat;
377
378 mmio_setbits_32((uint32_t)&ctl->swctl, DDRCTRL_SWCTL_SW_DONE);
379 VERBOSE("[0x%x] swctl = 0x%x\n",
380 (uint32_t)&ctl->swctl, mmio_read_32((uint32_t)&ctl->swctl));
381
382 start = get_timer(0);
383 do {
384 swstat = mmio_read_32((uint32_t)&ctl->swstat);
385 VERBOSE("[0x%x] swstat = 0x%x ",
386 (uint32_t)&ctl->swstat, swstat);
387 VERBOSE("timer in ms 0x%x = start 0x%lx\r",
388 get_timer(0), start);
389 if (get_timer(start) > plat_get_syscnt_freq2()) {
390 panic();
391 }
392 } while ((swstat & DDRCTRL_SWSTAT_SW_DONE_ACK) == 0U);
393
394 VERBOSE("[0x%x] swstat = 0x%x\n",
395 (uint32_t)&ctl->swstat, swstat);
396}
397
398/* Wait quasi dynamic register update */
399static void stm32mp1_wait_operating_mode(struct ddr_info *priv, uint32_t mode)
400{
401 unsigned long start;
402 uint32_t stat;
403 uint32_t operating_mode;
404 uint32_t selref_type;
405 int break_loop = 0;
406
407 start = get_timer(0);
408 for ( ; ; ) {
409 stat = mmio_read_32((uint32_t)&priv->ctl->stat);
410 operating_mode = stat & DDRCTRL_STAT_OPERATING_MODE_MASK;
411 selref_type = stat & DDRCTRL_STAT_SELFREF_TYPE_MASK;
412 VERBOSE("[0x%x] stat = 0x%x\n",
413 (uint32_t)&priv->ctl->stat, stat);
414 VERBOSE("timer in ms 0x%x = start 0x%lx\r",
415 get_timer(0), start);
416 if (get_timer(start) > plat_get_syscnt_freq2()) {
417 panic();
418 }
419
420 if (mode == DDRCTRL_STAT_OPERATING_MODE_SR) {
421 /*
422 * Self-refresh due to software
423 * => checking also STAT.selfref_type.
424 */
425 if ((operating_mode ==
426 DDRCTRL_STAT_OPERATING_MODE_SR) &&
427 (selref_type == DDRCTRL_STAT_SELFREF_TYPE_SR)) {
428 break_loop = 1;
429 }
430 } else if (operating_mode == mode) {
431 break_loop = 1;
432 } else if ((mode == DDRCTRL_STAT_OPERATING_MODE_NORMAL) &&
433 (operating_mode == DDRCTRL_STAT_OPERATING_MODE_SR) &&
434 (selref_type == DDRCTRL_STAT_SELFREF_TYPE_ASR)) {
435 /* Normal mode: handle also automatic self refresh */
436 break_loop = 1;
437 }
438
439 if (break_loop == 1) {
440 break;
441 }
442 }
443
444 VERBOSE("[0x%x] stat = 0x%x\n",
445 (uint32_t)&priv->ctl->stat, stat);
446}
447
448/* Mode Register Writes (MRW or MRS) */
449static void stm32mp1_mode_register_write(struct ddr_info *priv, uint8_t addr,
450 uint32_t data)
451{
452 uint32_t mrctrl0;
453
454 VERBOSE("MRS: %d = %x\n", addr, data);
455
456 /*
457 * 1. Poll MRSTAT.mr_wr_busy until it is '0'.
458 * This checks that there is no outstanding MR transaction.
459 * No write should be performed to MRCTRL0 and MRCTRL1
460 * if MRSTAT.mr_wr_busy = 1.
461 */
462 while ((mmio_read_32((uint32_t)&priv->ctl->mrstat) &
463 DDRCTRL_MRSTAT_MR_WR_BUSY) != 0U) {
464 ;
465 }
466
467 /*
468 * 2. Write the MRCTRL0.mr_type, MRCTRL0.mr_addr, MRCTRL0.mr_rank
469 * and (for MRWs) MRCTRL1.mr_data to define the MR transaction.
470 */
471 mrctrl0 = DDRCTRL_MRCTRL0_MR_TYPE_WRITE |
472 DDRCTRL_MRCTRL0_MR_RANK_ALL |
473 (((uint32_t)addr << DDRCTRL_MRCTRL0_MR_ADDR_SHIFT) &
474 DDRCTRL_MRCTRL0_MR_ADDR_MASK);
475 mmio_write_32((uint32_t)&priv->ctl->mrctrl0, mrctrl0);
476 VERBOSE("[0x%x] mrctrl0 = 0x%x (0x%x)\n",
477 (uint32_t)&priv->ctl->mrctrl0,
478 mmio_read_32((uint32_t)&priv->ctl->mrctrl0), mrctrl0);
479 mmio_write_32((uint32_t)&priv->ctl->mrctrl1, data);
480 VERBOSE("[0x%x] mrctrl1 = 0x%x\n",
481 (uint32_t)&priv->ctl->mrctrl1,
482 mmio_read_32((uint32_t)&priv->ctl->mrctrl1));
483
484 /*
485 * 3. In a separate APB transaction, write the MRCTRL0.mr_wr to 1. This
486 * bit is self-clearing, and triggers the MR transaction.
487 * The uMCTL2 then asserts the MRSTAT.mr_wr_busy while it performs
488 * the MR transaction to SDRAM, and no further access can be
489 * initiated until it is deasserted.
490 */
491 mrctrl0 |= DDRCTRL_MRCTRL0_MR_WR;
492 mmio_write_32((uint32_t)&priv->ctl->mrctrl0, mrctrl0);
493
494 while ((mmio_read_32((uint32_t)&priv->ctl->mrstat) &
495 DDRCTRL_MRSTAT_MR_WR_BUSY) != 0U) {
496 ;
497 }
498
499 VERBOSE("[0x%x] mrctrl0 = 0x%x\n",
500 (uint32_t)&priv->ctl->mrctrl0, mrctrl0);
501}
502
503/* Switch DDR3 from DLL-on to DLL-off */
504static void stm32mp1_ddr3_dll_off(struct ddr_info *priv)
505{
506 uint32_t mr1 = mmio_read_32((uint32_t)&priv->phy->mr1);
507 uint32_t mr2 = mmio_read_32((uint32_t)&priv->phy->mr2);
508 uint32_t dbgcam;
509
510 VERBOSE("mr1: 0x%x\n", mr1);
511 VERBOSE("mr2: 0x%x\n", mr2);
512
513 /*
514 * 1. Set the DBG1.dis_hif = 1.
515 * This prevents further reads/writes being received on the HIF.
516 */
517 mmio_setbits_32((uint32_t)&priv->ctl->dbg1, DDRCTRL_DBG1_DIS_HIF);
518 VERBOSE("[0x%x] dbg1 = 0x%x\n",
519 (uint32_t)&priv->ctl->dbg1,
520 mmio_read_32((uint32_t)&priv->ctl->dbg1));
521
522 /*
523 * 2. Ensure all commands have been flushed from the uMCTL2 by polling
524 * DBGCAM.wr_data_pipeline_empty = 1,
525 * DBGCAM.rd_data_pipeline_empty = 1,
526 * DBGCAM.dbg_wr_q_depth = 0 ,
527 * DBGCAM.dbg_lpr_q_depth = 0, and
528 * DBGCAM.dbg_hpr_q_depth = 0.
529 */
530 do {
531 dbgcam = mmio_read_32((uint32_t)&priv->ctl->dbgcam);
532 VERBOSE("[0x%x] dbgcam = 0x%x\n",
533 (uint32_t)&priv->ctl->dbgcam, dbgcam);
534 } while ((((dbgcam & DDRCTRL_DBGCAM_DATA_PIPELINE_EMPTY) ==
535 DDRCTRL_DBGCAM_DATA_PIPELINE_EMPTY)) &&
536 ((dbgcam & DDRCTRL_DBGCAM_DBG_Q_DEPTH) == 0U));
537
538 /*
539 * 3. Perform an MRS command (using MRCTRL0 and MRCTRL1 registers)
540 * to disable RTT_NOM:
541 * a. DDR3: Write to MR1[9], MR1[6] and MR1[2]
542 * b. DDR4: Write to MR1[10:8]
543 */
544 mr1 &= ~(BIT(9) | BIT(6) | BIT(2));
545 stm32mp1_mode_register_write(priv, 1, mr1);
546
547 /*
548 * 4. For DDR4 only: Perform an MRS command
549 * (using MRCTRL0 and MRCTRL1 registers) to write to MR5[8:6]
550 * to disable RTT_PARK
551 */
552
553 /*
554 * 5. Perform an MRS command (using MRCTRL0 and MRCTRL1 registers)
555 * to write to MR2[10:9], to disable RTT_WR
556 * (and therefore disable dynamic ODT).
557 * This applies for both DDR3 and DDR4.
558 */
559 mr2 &= ~GENMASK(10, 9);
560 stm32mp1_mode_register_write(priv, 2, mr2);
561
562 /*
563 * 6. Perform an MRS command (using MRCTRL0 and MRCTRL1 registers)
564 * to disable the DLL. The timing of this MRS is automatically
565 * handled by the uMCTL2.
566 * a. DDR3: Write to MR1[0]
567 * b. DDR4: Write to MR1[0]
568 */
569 mr1 |= BIT(0);
570 stm32mp1_mode_register_write(priv, 1, mr1);
571
572 /*
573 * 7. Put the SDRAM into self-refresh mode by setting
574 * PWRCTL.selfref_sw = 1, and polling STAT.operating_mode to ensure
575 * the DDRC has entered self-refresh.
576 */
577 mmio_setbits_32((uint32_t)&priv->ctl->pwrctl,
578 DDRCTRL_PWRCTL_SELFREF_SW);
579 VERBOSE("[0x%x] pwrctl = 0x%x\n",
580 (uint32_t)&priv->ctl->pwrctl,
581 mmio_read_32((uint32_t)&priv->ctl->pwrctl));
582
583 /*
584 * 8. Wait until STAT.operating_mode[1:0]==11 indicating that the
585 * DWC_ddr_umctl2 core is in self-refresh mode.
586 * Ensure transition to self-refresh was due to software
587 * by checking that STAT.selfref_type[1:0]=2.
588 */
589 stm32mp1_wait_operating_mode(priv, DDRCTRL_STAT_OPERATING_MODE_SR);
590
591 /*
592 * 9. Set the MSTR.dll_off_mode = 1.
593 * warning: MSTR.dll_off_mode is a quasi-dynamic type 2 field
594 */
595 stm32mp1_start_sw_done(priv->ctl);
596
597 mmio_setbits_32((uint32_t)&priv->ctl->mstr, DDRCTRL_MSTR_DLL_OFF_MODE);
598 VERBOSE("[0x%x] mstr = 0x%x\n",
599 (uint32_t)&priv->ctl->mstr,
600 mmio_read_32((uint32_t)&priv->ctl->mstr));
601
602 stm32mp1_wait_sw_done_ack(priv->ctl);
603
604 /* 10. Change the clock frequency to the desired value. */
605
606 /*
607 * 11. Update any registers which may be required to change for the new
608 * frequency. This includes static and dynamic registers.
609 * This includes both uMCTL2 registers and PHY registers.
610 */
611
612 /* Change Bypass Mode Frequency Range */
613 if (stm32mp1_clk_get_rate(DDRPHYC) < 100000000U) {
614 mmio_clrbits_32((uint32_t)&priv->phy->dllgcr,
615 DDRPHYC_DLLGCR_BPS200);
616 } else {
617 mmio_setbits_32((uint32_t)&priv->phy->dllgcr,
618 DDRPHYC_DLLGCR_BPS200);
619 }
620
621 mmio_setbits_32((uint32_t)&priv->phy->acdllcr, DDRPHYC_ACDLLCR_DLLDIS);
622
623 mmio_setbits_32((uint32_t)&priv->phy->dx0dllcr,
624 DDRPHYC_DXNDLLCR_DLLDIS);
625 mmio_setbits_32((uint32_t)&priv->phy->dx1dllcr,
626 DDRPHYC_DXNDLLCR_DLLDIS);
627 mmio_setbits_32((uint32_t)&priv->phy->dx2dllcr,
628 DDRPHYC_DXNDLLCR_DLLDIS);
629 mmio_setbits_32((uint32_t)&priv->phy->dx3dllcr,
630 DDRPHYC_DXNDLLCR_DLLDIS);
631
632 /* 12. Exit the self-refresh state by setting PWRCTL.selfref_sw = 0. */
633 mmio_clrbits_32((uint32_t)&priv->ctl->pwrctl,
634 DDRCTRL_PWRCTL_SELFREF_SW);
635 stm32mp1_wait_operating_mode(priv, DDRCTRL_STAT_OPERATING_MODE_NORMAL);
636
637 /*
638 * 13. If ZQCTL0.dis_srx_zqcl = 0, the uMCTL2 performs a ZQCL command
639 * at this point.
640 */
641
642 /*
643 * 14. Perform MRS commands as required to re-program timing registers
644 * in the SDRAM for the new frequency
645 * (in particular, CL, CWL and WR may need to be changed).
646 */
647
648 /* 15. Write DBG1.dis_hif = 0 to re-enable reads and writes. */
649 mmio_clrbits_32((uint32_t)&priv->ctl->dbg1, DDRCTRL_DBG1_DIS_HIF);
650 VERBOSE("[0x%x] dbg1 = 0x%x\n",
651 (uint32_t)&priv->ctl->dbg1,
652 mmio_read_32((uint32_t)&priv->ctl->dbg1));
653}
654
655static void stm32mp1_refresh_disable(struct stm32mp1_ddrctl *ctl)
656{
657 stm32mp1_start_sw_done(ctl);
658 /* Quasi-dynamic register update*/
659 mmio_setbits_32((uint32_t)&ctl->rfshctl3,
660 DDRCTRL_RFSHCTL3_DIS_AUTO_REFRESH);
661 mmio_clrbits_32((uint32_t)&ctl->pwrctl, DDRCTRL_PWRCTL_POWERDOWN_EN);
662 mmio_clrbits_32((uint32_t)&ctl->dfimisc,
663 DDRCTRL_DFIMISC_DFI_INIT_COMPLETE_EN);
664 stm32mp1_wait_sw_done_ack(ctl);
665}
666
667static void stm32mp1_refresh_restore(struct stm32mp1_ddrctl *ctl,
668 uint32_t rfshctl3, uint32_t pwrctl)
669{
670 stm32mp1_start_sw_done(ctl);
671 if ((rfshctl3 & DDRCTRL_RFSHCTL3_DIS_AUTO_REFRESH) == 0U) {
672 mmio_clrbits_32((uint32_t)&ctl->rfshctl3,
673 DDRCTRL_RFSHCTL3_DIS_AUTO_REFRESH);
674 }
675 if ((pwrctl & DDRCTRL_PWRCTL_POWERDOWN_EN) != 0U) {
676 mmio_setbits_32((uint32_t)&ctl->pwrctl,
677 DDRCTRL_PWRCTL_POWERDOWN_EN);
678 }
679 mmio_setbits_32((uint32_t)&ctl->dfimisc,
680 DDRCTRL_DFIMISC_DFI_INIT_COMPLETE_EN);
681 stm32mp1_wait_sw_done_ack(ctl);
682}
683
684static int board_ddr_power_init(enum ddr_type ddr_type)
685{
686 if (dt_check_pmic()) {
687 return pmic_ddr_power_init(ddr_type);
688 }
689
690 return 0;
691}
692
693void stm32mp1_ddr_init(struct ddr_info *priv,
694 struct stm32mp1_ddr_config *config)
695{
696 uint32_t pir;
697 int ret;
698
699 if ((config->c_reg.mstr & DDRCTRL_MSTR_DDR3) != 0U) {
700 ret = board_ddr_power_init(STM32MP_DDR3);
701 } else {
702 ret = board_ddr_power_init(STM32MP_LPDDR2);
703 }
704
705 if (ret != 0) {
706 panic();
707 }
708
709 VERBOSE("name = %s\n", config->info.name);
710 VERBOSE("speed = %d MHz\n", config->info.speed);
711 VERBOSE("size = 0x%x\n", config->info.size);
712
713 /* DDR INIT SEQUENCE */
714
715 /*
716 * 1. Program the DWC_ddr_umctl2 registers
717 * nota: check DFIMISC.dfi_init_complete = 0
718 */
719
720 /* 1.1 RESETS: presetn, core_ddrc_rstn, aresetn */
721 mmio_setbits_32(priv->rcc + RCC_DDRITFCR, RCC_DDRITFCR_DDRCAPBRST);
722 mmio_setbits_32(priv->rcc + RCC_DDRITFCR, RCC_DDRITFCR_DDRCAXIRST);
723 mmio_setbits_32(priv->rcc + RCC_DDRITFCR, RCC_DDRITFCR_DDRCORERST);
724 mmio_setbits_32(priv->rcc + RCC_DDRITFCR, RCC_DDRITFCR_DPHYAPBRST);
725 mmio_setbits_32(priv->rcc + RCC_DDRITFCR, RCC_DDRITFCR_DPHYRST);
726 mmio_setbits_32(priv->rcc + RCC_DDRITFCR, RCC_DDRITFCR_DPHYCTLRST);
727
728 /* 1.2. start CLOCK */
729 if (stm32mp1_ddr_clk_enable(priv, config->info.speed) != 0) {
730 panic();
731 }
732
733 /* 1.3. deassert reset */
734 /* De-assert PHY rstn and ctl_rstn via DPHYRST and DPHYCTLRST. */
735 mmio_clrbits_32(priv->rcc + RCC_DDRITFCR, RCC_DDRITFCR_DPHYRST);
736 mmio_clrbits_32(priv->rcc + RCC_DDRITFCR, RCC_DDRITFCR_DPHYCTLRST);
737 /*
738 * De-assert presetn once the clocks are active
739 * and stable via DDRCAPBRST bit.
740 */
741 mmio_clrbits_32(priv->rcc + RCC_DDRITFCR, RCC_DDRITFCR_DDRCAPBRST);
742
743 /* 1.4. wait 128 cycles to permit initialization of end logic */
744 udelay(2);
745 /* For PCLK = 133MHz => 1 us is enough, 2 to allow lower frequency */
746
747 /* 1.5. initialize registers ddr_umctl2 */
748 /* Stop uMCTL2 before PHY is ready */
749 mmio_clrbits_32((uint32_t)&priv->ctl->dfimisc,
750 DDRCTRL_DFIMISC_DFI_INIT_COMPLETE_EN);
751 VERBOSE("[0x%x] dfimisc = 0x%x\n",
752 (uint32_t)&priv->ctl->dfimisc,
753 mmio_read_32((uint32_t)&priv->ctl->dfimisc));
754
755 set_reg(priv, REG_REG, &config->c_reg);
756
757 /* DDR3 = don't set DLLOFF for init mode */
758 if ((config->c_reg.mstr &
759 (DDRCTRL_MSTR_DDR3 | DDRCTRL_MSTR_DLL_OFF_MODE))
760 == (DDRCTRL_MSTR_DDR3 | DDRCTRL_MSTR_DLL_OFF_MODE)) {
761 VERBOSE("deactivate DLL OFF in mstr\n");
762 mmio_clrbits_32((uint32_t)&priv->ctl->mstr,
763 DDRCTRL_MSTR_DLL_OFF_MODE);
764 VERBOSE("[0x%x] mstr = 0x%x\n",
765 (uint32_t)&priv->ctl->mstr,
766 mmio_read_32((uint32_t)&priv->ctl->mstr));
767 }
768
769 set_reg(priv, REG_TIMING, &config->c_timing);
770 set_reg(priv, REG_MAP, &config->c_map);
771
772 /* Skip CTRL init, SDRAM init is done by PHY PUBL */
773 mmio_clrsetbits_32((uint32_t)&priv->ctl->init0,
774 DDRCTRL_INIT0_SKIP_DRAM_INIT_MASK,
775 DDRCTRL_INIT0_SKIP_DRAM_INIT_NORMAL);
776 VERBOSE("[0x%x] init0 = 0x%x\n",
777 (uint32_t)&priv->ctl->init0,
778 mmio_read_32((uint32_t)&priv->ctl->init0));
779
780 set_reg(priv, REG_PERF, &config->c_perf);
781
782 /* 2. deassert reset signal core_ddrc_rstn, aresetn and presetn */
783 mmio_clrbits_32(priv->rcc + RCC_DDRITFCR, RCC_DDRITFCR_DDRCORERST);
784 mmio_clrbits_32(priv->rcc + RCC_DDRITFCR, RCC_DDRITFCR_DDRCAXIRST);
785 mmio_clrbits_32(priv->rcc + RCC_DDRITFCR, RCC_DDRITFCR_DPHYAPBRST);
786
787 /*
788 * 3. start PHY init by accessing relevant PUBL registers
789 * (DXGCR, DCR, PTR*, MR*, DTPR*)
790 */
791 set_reg(priv, REGPHY_REG, &config->p_reg);
792 set_reg(priv, REGPHY_TIMING, &config->p_timing);
793 set_reg(priv, REGPHY_CAL, &config->p_cal);
794
795 /* DDR3 = don't set DLLOFF for init mode */
796 if ((config->c_reg.mstr &
797 (DDRCTRL_MSTR_DDR3 | DDRCTRL_MSTR_DLL_OFF_MODE))
798 == (DDRCTRL_MSTR_DDR3 | DDRCTRL_MSTR_DLL_OFF_MODE)) {
799 VERBOSE("deactivate DLL OFF in mr1\n");
800 mmio_clrbits_32((uint32_t)&priv->phy->mr1, BIT(0));
801 VERBOSE("[0x%x] mr1 = 0x%x\n",
802 (uint32_t)&priv->phy->mr1,
803 mmio_read_32((uint32_t)&priv->phy->mr1));
804 }
805
806 /*
807 * 4. Monitor PHY init status by polling PUBL register PGSR.IDONE
808 * Perform DDR PHY DRAM initialization and Gate Training Evaluation
809 */
810 stm32mp1_ddrphy_idone_wait(priv->phy);
811
812 /*
813 * 5. Indicate to PUBL that controller performs SDRAM initialization
814 * by setting PIR.INIT and PIR CTLDINIT and pool PGSR.IDONE
815 * DRAM init is done by PHY, init0.skip_dram.init = 1
816 */
817
818 pir = DDRPHYC_PIR_DLLSRST | DDRPHYC_PIR_DLLLOCK | DDRPHYC_PIR_ZCAL |
819 DDRPHYC_PIR_ITMSRST | DDRPHYC_PIR_DRAMINIT | DDRPHYC_PIR_ICPC;
820
821 if ((config->c_reg.mstr & DDRCTRL_MSTR_DDR3) != 0U) {
822 pir |= DDRPHYC_PIR_DRAMRST; /* Only for DDR3 */
823 }
824
825 stm32mp1_ddrphy_init(priv->phy, pir);
826
827 /*
828 * 6. SET DFIMISC.dfi_init_complete_en to 1
829 * Enable quasi-dynamic register programming.
830 */
831 stm32mp1_start_sw_done(priv->ctl);
832
833 mmio_setbits_32((uint32_t)&priv->ctl->dfimisc,
834 DDRCTRL_DFIMISC_DFI_INIT_COMPLETE_EN);
835 VERBOSE("[0x%x] dfimisc = 0x%x\n",
836 (uint32_t)&priv->ctl->dfimisc,
837 mmio_read_32((uint32_t)&priv->ctl->dfimisc));
838
839 stm32mp1_wait_sw_done_ack(priv->ctl);
840
841 /*
842 * 7. Wait for DWC_ddr_umctl2 to move to normal operation mode
843 * by monitoring STAT.operating_mode signal
844 */
845
846 /* Wait uMCTL2 ready */
847 stm32mp1_wait_operating_mode(priv, DDRCTRL_STAT_OPERATING_MODE_NORMAL);
848
849 /* Switch to DLL OFF mode */
850 if ((config->c_reg.mstr & DDRCTRL_MSTR_DLL_OFF_MODE) != 0U) {
851 stm32mp1_ddr3_dll_off(priv);
852 }
853
854 VERBOSE("DDR DQS training : ");
855
856 /*
857 * 8. Disable Auto refresh and power down by setting
858 * - RFSHCTL3.dis_au_refresh = 1
859 * - PWRCTL.powerdown_en = 0
860 * - DFIMISC.dfiinit_complete_en = 0
861 */
862 stm32mp1_refresh_disable(priv->ctl);
863
864 /*
865 * 9. Program PUBL PGCR to enable refresh during training
866 * and rank to train
867 * not done => keep the programed value in PGCR
868 */
869
870 /*
871 * 10. configure PUBL PIR register to specify which training step
872 * to run
873 * Warning : RVTRN is not supported by this PUBL
874 */
875 stm32mp1_ddrphy_init(priv->phy, DDRPHYC_PIR_QSTRN);
876
877 /* 11. monitor PUB PGSR.IDONE to poll cpmpletion of training sequence */
878 stm32mp1_ddrphy_idone_wait(priv->phy);
879
880 /*
881 * 12. set back registers in step 8 to the orginal values if desidered
882 */
883 stm32mp1_refresh_restore(priv->ctl, config->c_reg.rfshctl3,
884 config->c_reg.pwrctl);
885
886 /* Enable uMCTL2 AXI port 0 */
887 mmio_setbits_32((uint32_t)&priv->ctl->pctrl_0, DDRCTRL_PCTRL_N_PORT_EN);
888 VERBOSE("[0x%x] pctrl_0 = 0x%x\n",
889 (uint32_t)&priv->ctl->pctrl_0,
890 mmio_read_32((uint32_t)&priv->ctl->pctrl_0));
891
892 /* Enable uMCTL2 AXI port 1 */
893 mmio_setbits_32((uint32_t)&priv->ctl->pctrl_1, DDRCTRL_PCTRL_N_PORT_EN);
894 VERBOSE("[0x%x] pctrl_1 = 0x%x\n",
895 (uint32_t)&priv->ctl->pctrl_1,
896 mmio_read_32((uint32_t)&priv->ctl->pctrl_1));
897}