blob: b5f697334e9187b656c5ffe62e469cb21227c9ed [file] [log] [blame]
Jacky Bai9a6f62f2019-11-25 14:43:26 +08001/*
Jacky Bai95ec94c2020-04-13 17:44:50 +08002 * Copyright 2019-2023 NXP
Jacky Bai9a6f62f2019-11-25 14:43:26 +08003 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
Jacky Baid746daa12019-11-25 13:19:37 +08007#include <bl31/interrupt_mgmt.h>
8#include <common/runtime_svc.h>
Jacky Bai9a6f62f2019-11-25 14:43:26 +08009#include <lib/mmio.h>
Jacky Baid746daa12019-11-25 13:19:37 +080010#include <lib/spinlock.h>
11#include <plat/common/platform.h>
Jacky Bai9a6f62f2019-11-25 14:43:26 +080012
13#include <dram.h>
Jacky Baia51b11a2020-01-14 14:19:05 +080014#include <gpc.h>
Jacky Bai9a6f62f2019-11-25 14:43:26 +080015
Jacky Baid746daa12019-11-25 13:19:37 +080016#define IMX_SIP_DDR_DVFS_GET_FREQ_COUNT 0x10
17#define IMX_SIP_DDR_DVFS_GET_FREQ_INFO 0x11
18
Jacky Bai9a6f62f2019-11-25 14:43:26 +080019struct dram_info dram_info;
20
Jacky Baid746daa12019-11-25 13:19:37 +080021/* lock used for DDR DVFS */
22spinlock_t dfs_lock;
23
Jacky Bai0d079202020-01-07 16:44:46 +080024#if defined(PLAT_imx8mq)
25/* ocram used to dram timing */
26static uint8_t dram_timing_saved[13 * 1024] __aligned(8);
27#endif
28
Jacky Baid746daa12019-11-25 13:19:37 +080029static volatile uint32_t wfe_done;
30static volatile bool wait_ddrc_hwffc_done = true;
31static unsigned int dev_fsp = 0x1;
32
33static uint32_t fsp_init_reg[3][4] = {
34 { DDRC_INIT3(0), DDRC_INIT4(0), DDRC_INIT6(0), DDRC_INIT7(0) },
35 { DDRC_FREQ1_INIT3(0), DDRC_FREQ1_INIT4(0), DDRC_FREQ1_INIT6(0), DDRC_FREQ1_INIT7(0) },
36 { DDRC_FREQ2_INIT3(0), DDRC_FREQ2_INIT4(0), DDRC_FREQ2_INIT6(0), DDRC_FREQ2_INIT7(0) },
37};
38
Jacky Bai0d079202020-01-07 16:44:46 +080039#if defined(PLAT_imx8mq)
40static inline struct dram_cfg_param *get_cfg_ptr(void *ptr,
41 void *old_base, void *new_base)
42{
43 uintptr_t offset = (uintptr_t)ptr & ~((uintptr_t)old_base);
44
45 return (struct dram_cfg_param *)(offset + new_base);
46}
47
48/* copy the dram timing info from DRAM to OCRAM */
49void imx8mq_dram_timing_copy(struct dram_timing_info *from)
50{
51 struct dram_timing_info *info = (struct dram_timing_info *)dram_timing_saved;
52
53 /* copy the whole 13KB content used for dram timing info */
54 memcpy(dram_timing_saved, from, sizeof(dram_timing_saved));
55
56 /* correct the header after copied into ocram */
57 info->ddrc_cfg = get_cfg_ptr(info->ddrc_cfg, from, dram_timing_saved);
58 info->ddrphy_cfg = get_cfg_ptr(info->ddrphy_cfg, from, dram_timing_saved);
59 info->ddrphy_trained_csr = get_cfg_ptr(info->ddrphy_trained_csr, from, dram_timing_saved);
60 info->ddrphy_pie = get_cfg_ptr(info->ddrphy_pie, from, dram_timing_saved);
61}
62#endif
63
Jacky Baie2507bb2021-12-20 17:56:08 +080064#if defined(PLAT_imx8mp)
65static uint32_t lpddr4_mr_read(unsigned int mr_rank, unsigned int mr_addr)
66{
67 unsigned int tmp, drate_byte;
68
69 tmp = mmio_read_32(DRC_PERF_MON_MRR0_DAT(0));
70 mmio_write_32(DRC_PERF_MON_MRR0_DAT(0), tmp | 0x1);
71 do {
72 tmp = mmio_read_32(DDRC_MRSTAT(0));
73 } while (tmp & 0x1);
74
75 mmio_write_32(DDRC_MRCTRL0(0), (mr_rank << 4) | 0x1);
76 mmio_write_32(DDRC_MRCTRL1(0), (mr_addr << 8));
77 mmio_write_32(DDRC_MRCTRL0(0), (mr_rank << 4) | BIT(31) | 0x1);
78
79 /* Workaround for SNPS STAR 9001549457 */
80 do {
81 tmp = mmio_read_32(DDRC_MRSTAT(0));
82 } while (tmp & 0x1);
83
84 do {
85 tmp = mmio_read_32(DRC_PERF_MON_MRR0_DAT(0));
86 } while (!(tmp & 0x8));
87 tmp = mmio_read_32(DRC_PERF_MON_MRR1_DAT(0));
88
89 drate_byte = (mmio_read_32(DDRC_DERATEEN(0)) >> 4) & 0xff;
90 tmp = (tmp >> (drate_byte * 8)) & 0xff;
91 mmio_write_32(DRC_PERF_MON_MRR0_DAT(0), 0x4);
92
93 return tmp;
94}
95#endif
96
Jacky Baid746daa12019-11-25 13:19:37 +080097static void get_mr_values(uint32_t (*mr_value)[8])
98{
99 uint32_t init_val;
100 unsigned int i, fsp_index;
101
102 for (fsp_index = 0U; fsp_index < 3U; fsp_index++) {
103 for (i = 0U; i < 4U; i++) {
104 init_val = mmio_read_32(fsp_init_reg[fsp_index][i]);
105 mr_value[fsp_index][2*i] = init_val >> 16;
106 mr_value[fsp_index][2*i + 1] = init_val & 0xFFFF;
107 }
Jacky Baie2507bb2021-12-20 17:56:08 +0800108
109#if defined(PLAT_imx8mp)
110 if (dram_info.dram_type == DDRC_LPDDR4) {
111 mr_value[fsp_index][5] = lpddr4_mr_read(1, MR12); /* read MR12 from DRAM */
112 mr_value[fsp_index][7] = lpddr4_mr_read(1, MR14); /* read MR14 from DRAM */
113 }
114#endif
Jacky Baid746daa12019-11-25 13:19:37 +0800115 }
116}
117
Jacky Baidde85e02020-05-08 17:37:24 +0800118static void save_rank_setting(void)
119{
120 uint32_t i, offset;
121 uint32_t pstate_num = dram_info.num_fsp;
122
Jacky Baiff820ba2020-09-08 09:55:59 +0800123 /* only support maximum 3 setpoints */
124 pstate_num = (pstate_num > MAX_FSP_NUM) ? MAX_FSP_NUM : pstate_num;
125
Jacky Baidde85e02020-05-08 17:37:24 +0800126 for (i = 0U; i < pstate_num; i++) {
127 offset = i ? (i + 1) * 0x1000 : 0U;
128 dram_info.rank_setting[i][0] = mmio_read_32(DDRC_DRAMTMG2(0) + offset);
129 if (dram_info.dram_type != DDRC_LPDDR4) {
130 dram_info.rank_setting[i][1] = mmio_read_32(DDRC_DRAMTMG9(0) + offset);
131 }
132#if !defined(PLAT_imx8mq)
133 dram_info.rank_setting[i][2] = mmio_read_32(DDRC_RANKCTL(0) + offset);
134#endif
135 }
136#if defined(PLAT_imx8mq)
137 dram_info.rank_setting[0][2] = mmio_read_32(DDRC_RANKCTL(0));
138#endif
139}
Jacky Bai9a6f62f2019-11-25 14:43:26 +0800140/* Restore the ddrc configs */
141void dram_umctl2_init(struct dram_timing_info *timing)
142{
143 struct dram_cfg_param *ddrc_cfg = timing->ddrc_cfg;
144 unsigned int i;
145
146 for (i = 0U; i < timing->ddrc_cfg_num; i++) {
147 mmio_write_32(ddrc_cfg->reg, ddrc_cfg->val);
148 ddrc_cfg++;
149 }
150
151 /* set the default fsp to P0 */
152 mmio_write_32(DDRC_MSTR2(0), 0x0);
153}
154
155/* Restore the dram PHY config */
156void dram_phy_init(struct dram_timing_info *timing)
157{
158 struct dram_cfg_param *cfg = timing->ddrphy_cfg;
159 unsigned int i;
160
161 /* Restore the PHY init config */
162 cfg = timing->ddrphy_cfg;
163 for (i = 0U; i < timing->ddrphy_cfg_num; i++) {
164 dwc_ddrphy_apb_wr(cfg->reg, cfg->val);
165 cfg++;
166 }
167
168 /* Restore the DDR PHY CSRs */
169 cfg = timing->ddrphy_trained_csr;
170 for (i = 0U; i < timing->ddrphy_trained_csr_num; i++) {
171 dwc_ddrphy_apb_wr(cfg->reg, cfg->val);
172 cfg++;
173 }
174
175 /* Load the PIE image */
176 cfg = timing->ddrphy_pie;
177 for (i = 0U; i < timing->ddrphy_pie_num; i++) {
178 dwc_ddrphy_apb_wr(cfg->reg, cfg->val);
179 cfg++;
180 }
Jacky Baid746daa12019-11-25 13:19:37 +0800181}
182
183/* EL3 SGI-8 IPI handler for DDR Dynamic frequency scaling */
184static uint64_t waiting_dvfs(uint32_t id, uint32_t flags,
185 void *handle, void *cookie)
186{
187 uint64_t mpidr = read_mpidr_el1();
188 unsigned int cpu_id = MPIDR_AFFLVL0_VAL(mpidr);
189 uint32_t irq;
190
191 irq = plat_ic_acknowledge_interrupt();
192 if (irq < 1022U) {
193 plat_ic_end_of_interrupt(irq);
194 }
195
196 /* set the WFE done status */
197 spin_lock(&dfs_lock);
198 wfe_done |= (1 << cpu_id * 8);
199 dsb();
200 spin_unlock(&dfs_lock);
201
202 while (1) {
203 /* ddr frequency change done */
204 if (!wait_ddrc_hwffc_done)
205 break;
206
207 wfe();
208 }
209
210 return 0;
Jacky Bai9a6f62f2019-11-25 14:43:26 +0800211}
212
213void dram_info_init(unsigned long dram_timing_base)
214{
215 uint32_t ddrc_mstr, current_fsp;
Marco Felsch753f6322022-09-21 17:48:35 +0200216 unsigned int idx = 0;
Jacky Baid746daa12019-11-25 13:19:37 +0800217 uint32_t flags = 0;
218 uint32_t rc;
219 unsigned int i;
Jacky Bai9a6f62f2019-11-25 14:43:26 +0800220
221 /* Get the dram type & rank */
222 ddrc_mstr = mmio_read_32(DDRC_MSTR(0));
223
224 dram_info.dram_type = ddrc_mstr & DDR_TYPE_MASK;
Jacky Bai95ec94c2020-04-13 17:44:50 +0800225 dram_info.num_rank = ((ddrc_mstr >> 24) & ACTIVE_RANK_MASK) == 0x3 ?
226 DDRC_ACTIVE_TWO_RANK : DDRC_ACTIVE_ONE_RANK;
Jacky Bai9a6f62f2019-11-25 14:43:26 +0800227
228 /* Get current fsp info */
Jacky Bai8e51c262020-08-03 13:31:26 +0800229 current_fsp = mmio_read_32(DDRC_DFIMISC(0));
230 current_fsp = (current_fsp >> 8) & 0xf;
Jacky Bai9a6f62f2019-11-25 14:43:26 +0800231 dram_info.boot_fsp = current_fsp;
232 dram_info.current_fsp = current_fsp;
233
Jacky Bai0d079202020-01-07 16:44:46 +0800234#if defined(PLAT_imx8mq)
235 imx8mq_dram_timing_copy((struct dram_timing_info *)dram_timing_base);
236 dram_timing_base = (unsigned long) dram_timing_saved;
237#endif
Jacky Baid746daa12019-11-25 13:19:37 +0800238 get_mr_values(dram_info.mr_table);
239
Jacky Bai9a6f62f2019-11-25 14:43:26 +0800240 dram_info.timing_info = (struct dram_timing_info *)dram_timing_base;
Jacky Baid746daa12019-11-25 13:19:37 +0800241
242 /* get the num of supported fsp */
243 for (i = 0U; i < 4U; ++i) {
244 if (!dram_info.timing_info->fsp_table[i]) {
245 break;
246 }
Marco Felsch753f6322022-09-21 17:48:35 +0200247 idx = i;
Jacky Baid746daa12019-11-25 13:19:37 +0800248 }
Jacky Baiff820ba2020-09-08 09:55:59 +0800249
250 /* only support maximum 3 setpoints */
251 dram_info.num_fsp = (i > MAX_FSP_NUM) ? MAX_FSP_NUM : i;
252
253 /* no valid fsp table, return directly */
254 if (i == 0U) {
255 return;
256 }
Jacky Baid746daa12019-11-25 13:19:37 +0800257
Jacky Baidde85e02020-05-08 17:37:24 +0800258 /* save the DRAMTMG2/9 for rank to rank workaround */
259 save_rank_setting();
260
Jacky Baid746daa12019-11-25 13:19:37 +0800261 /* check if has bypass mode support */
Marco Felsch753f6322022-09-21 17:48:35 +0200262 if (dram_info.timing_info->fsp_table[idx] < 666) {
Jacky Baid746daa12019-11-25 13:19:37 +0800263 dram_info.bypass_mode = true;
264 } else {
265 dram_info.bypass_mode = false;
266 }
267
268 /* Register the EL3 handler for DDR DVFS */
269 set_interrupt_rm_flag(flags, NON_SECURE);
270 rc = register_interrupt_type_handler(INTR_TYPE_EL3, waiting_dvfs, flags);
271 if (rc != 0) {
272 panic();
273 }
Jacky Baid746daa12019-11-25 13:19:37 +0800274
Jacky Baid148bae2020-04-22 21:26:13 +0800275 if (dram_info.dram_type == DDRC_LPDDR4 && current_fsp != 0x0) {
276 /* flush the L1/L2 cache */
277 dcsw_op_all(DCCSW);
278 lpddr4_swffc(&dram_info, dev_fsp, 0x0);
279 dev_fsp = (~dev_fsp) & 0x1;
280 } else if (current_fsp != 0x0) {
281 /* flush the L1/L2 cache */
282 dcsw_op_all(DCCSW);
283 ddr4_swffc(&dram_info, 0x0);
284 }
285}
Jacky Baid746daa12019-11-25 13:19:37 +0800286
287/*
288 * For each freq return the following info:
289 *
290 * r1: data rate
291 * r2: 1 + dram_core parent
292 * r3: 1 + dram_alt parent index
293 * r4: 1 + dram_apb parent index
294 *
295 * The parent indices can be used by an OS who manages source clocks to enabled
296 * them ahead of the switch.
297 *
298 * A parent value of "0" means "don't care".
299 *
300 * Current implementation of freq switch is hardcoded in
301 * plat/imx/common/imx8m/clock.c but in theory this can be enhanced to support
302 * a wide variety of rates.
303 */
304int dram_dvfs_get_freq_info(void *handle, u_register_t index)
305{
306 switch (index) {
307 case 0:
308 SMC_RET4(handle, dram_info.timing_info->fsp_table[0],
309 1, 0, 5);
310 case 1:
311 if (!dram_info.bypass_mode) {
312 SMC_RET4(handle, dram_info.timing_info->fsp_table[1],
313 1, 0, 0);
314 }
315 SMC_RET4(handle, dram_info.timing_info->fsp_table[1],
316 2, 2, 4);
317 case 2:
318 if (!dram_info.bypass_mode) {
319 SMC_RET4(handle, dram_info.timing_info->fsp_table[2],
320 1, 0, 0);
321 }
322 SMC_RET4(handle, dram_info.timing_info->fsp_table[2],
323 2, 3, 3);
324 case 3:
325 SMC_RET4(handle, dram_info.timing_info->fsp_table[3],
326 1, 0, 0);
327 default:
328 SMC_RET1(handle, -3);
329 }
330}
331
332int dram_dvfs_handler(uint32_t smc_fid, void *handle,
333 u_register_t x1, u_register_t x2, u_register_t x3)
334{
335 uint64_t mpidr = read_mpidr_el1();
336 unsigned int cpu_id = MPIDR_AFFLVL0_VAL(mpidr);
337 unsigned int fsp_index = x1;
338 uint32_t online_cores = x2;
339
340 if (x1 == IMX_SIP_DDR_DVFS_GET_FREQ_COUNT) {
341 SMC_RET1(handle, dram_info.num_fsp);
342 } else if (x1 == IMX_SIP_DDR_DVFS_GET_FREQ_INFO) {
343 return dram_dvfs_get_freq_info(handle, x2);
Jacky Baiff820ba2020-09-08 09:55:59 +0800344 } else if (x1 < 3U) {
Jacky Baid746daa12019-11-25 13:19:37 +0800345 wait_ddrc_hwffc_done = true;
346 dsb();
347
348 /* trigger the SGI IPI to info other cores */
349 for (int i = 0; i < PLATFORM_CORE_COUNT; i++) {
350 if (cpu_id != i && (online_cores & (0x1 << (i * 8)))) {
351 plat_ic_raise_el3_sgi(0x8, i);
352 }
353 }
Jacky Baia51b11a2020-01-14 14:19:05 +0800354#if defined(PLAT_imx8mq)
355 for (unsigned int i = 0; i < PLATFORM_CORE_COUNT; i++) {
356 if (i != cpu_id && online_cores & (1 << (i * 8))) {
357 imx_gpc_core_wake(1 << i);
358 }
359 }
360#endif
Jacky Baid746daa12019-11-25 13:19:37 +0800361 /* make sure all the core in WFE */
362 online_cores &= ~(0x1 << (cpu_id * 8));
363 while (1) {
364 if (online_cores == wfe_done) {
365 break;
366 }
367 }
368
369 /* flush the L1/L2 cache */
370 dcsw_op_all(DCCSW);
371
372 if (dram_info.dram_type == DDRC_LPDDR4) {
373 lpddr4_swffc(&dram_info, dev_fsp, fsp_index);
374 dev_fsp = (~dev_fsp) & 0x1;
Jacky Baid148bae2020-04-22 21:26:13 +0800375 } else {
Jacky Baid746daa12019-11-25 13:19:37 +0800376 ddr4_swffc(&dram_info, fsp_index);
377 }
378
379 dram_info.current_fsp = fsp_index;
380 wait_ddrc_hwffc_done = false;
381 wfe_done = 0;
382 dsb();
383 sev();
384 isb();
385 }
386
387 SMC_RET1(handle, 0);
Jacky Bai9a6f62f2019-11-25 14:43:26 +0800388}