blob: ff9a4e6bea9b61f6fa29e85eadd11823d414a772 [file] [log] [blame]
Achin Gupta4f6ad662013-10-25 09:08:21 +01001/*
Robert Wakim48e6b572021-10-21 15:39:56 +01002 * Copyright (c) 2013-2022, Arm Limited and Contributors. All rights reserved.
Achin Gupta4f6ad662013-10-25 09:08:21 +01003 *
dp-armfa3cf0b2017-05-03 09:38:09 +01004 * SPDX-License-Identifier: BSD-3-Clause
Achin Gupta4f6ad662013-10-25 09:08:21 +01005 */
6
Dan Handley2bd4ef22014-04-09 13:14:54 +01007#include <arch.h>
Achin Gupta4f6ad662013-10-25 09:08:21 +01008#include <asm_macros.S>
9
Achin Gupta4f6ad662013-10-25 09:08:21 +010010 .globl flush_dcache_range
Robert Wakim48e6b572021-10-21 15:39:56 +010011 .globl flush_dcache_to_popa_range
Olivier Deprezc80d0de2024-01-17 15:12:04 +010012 .globl flush_dcache_to_popa_range_mte2
Achin Guptae9c4a642015-09-11 16:03:13 +010013 .globl clean_dcache_range
Achin Gupta4f6ad662013-10-25 09:08:21 +010014 .globl inv_dcache_range
15 .globl dcsw_op_louis
16 .globl dcsw_op_all
Soby Mathew42aa5eb2014-09-02 10:47:33 +010017 .globl dcsw_op_level1
18 .globl dcsw_op_level2
19 .globl dcsw_op_level3
Achin Gupta4f6ad662013-10-25 09:08:21 +010020
Olivier Deprezc80d0de2024-01-17 15:12:04 +010021/* Opcodes for data cache maintenance by PA instructions. */
22
23/*
24 * sys #6, c7, c14, #1, x0
25 * DC CIPAPA, X0
26 */
27#define dc_cipapa_x0 0xd50e7e20
28
29/*
30 * sys #6, c7, c14, #3, x0
31 * DC CIDGPAPA, X0
32 */
33#define dc_cigdpapa_x0 0xd50e7ea0
34
Achin Guptae9c4a642015-09-11 16:03:13 +010035/*
36 * This macro can be used for implementing various data cache operations `op`
37 */
38.macro do_dcache_maintenance_by_mva op
Soby Mathewf7e8aee2017-06-15 16:18:45 +010039 /* Exit early if size is zero */
40 cbz x1, exit_loop_\op
Achin Gupta4f6ad662013-10-25 09:08:21 +010041 dcache_line_size x2, x3
42 add x1, x0, x1
43 sub x3, x2, #1
44 bic x0, x0, x3
Achin Guptae9c4a642015-09-11 16:03:13 +010045loop_\op:
46 dc \op, x0
Achin Gupta4f6ad662013-10-25 09:08:21 +010047 add x0, x0, x2
48 cmp x0, x1
Petre-Ionut Tudore5a6fef2019-11-07 15:18:03 +000049 b.lo loop_\op
Achin Gupta4f6ad662013-10-25 09:08:21 +010050 dsb sy
Soby Mathewf7e8aee2017-06-15 16:18:45 +010051exit_loop_\op:
Achin Gupta4f6ad662013-10-25 09:08:21 +010052 ret
Achin Guptae9c4a642015-09-11 16:03:13 +010053.endm
Okash Khawaja037b56e2022-11-04 12:38:01 +000054
Olivier Deprezc80d0de2024-01-17 15:12:04 +010055/* op: the hexadecimal instruction opcode for the cache operation */
56.macro do_dcache_maintenance_instr op
57 /* Exit early if size is zero */
58 cbz x1, exit_loop_\op
59 dcache_line_size x2, x3
60 sub x3, x2, #1
61 bic x0, x0, x3
62 add x1, x1, x0
63loop_\op:
64 .inst \op
65 add x0, x0, x2
66 cmp x0, x1
67 b.lo loop_\op
68 dsb osh
69exit_loop_\op:
70 ret
71.endm
72
Okash Khawaja037b56e2022-11-04 12:38:01 +000073.macro check_plat_can_cmo
74#if CONDITIONAL_CMO
75 mov x3, x30
76 mov x2, x0
77 bl plat_can_cmo
78 mov x30, x3
79 cbnz x0, 1f
80 ret
811:
82 mov x0, x2
83#endif
84.endm
Olivier Deprezc80d0de2024-01-17 15:12:04 +010085
86 /* -------------------------------------------
87 * DCache Clean+Invalidate by MVA from base
88 * address till size. 'x0' = addr, 'x1' = size
89 * -------------------------------------------
Achin Guptae9c4a642015-09-11 16:03:13 +010090 */
91func flush_dcache_range
Okash Khawaja037b56e2022-11-04 12:38:01 +000092 check_plat_can_cmo
Achin Guptae9c4a642015-09-11 16:03:13 +010093 do_dcache_maintenance_by_mva civac
Kévin Petita877c252015-03-24 14:03:57 +000094endfunc flush_dcache_range
Achin Gupta4f6ad662013-10-25 09:08:21 +010095
Achin Guptae9c4a642015-09-11 16:03:13 +010096 /* ------------------------------------------
Olivier Deprezc80d0de2024-01-17 15:12:04 +010097 * DCache Clean by MVA from base address till
98 * size. 'x0' = addr, 'x1' = size
Achin Guptae9c4a642015-09-11 16:03:13 +010099 * ------------------------------------------
100 */
101func clean_dcache_range
Okash Khawaja037b56e2022-11-04 12:38:01 +0000102 check_plat_can_cmo
Achin Guptae9c4a642015-09-11 16:03:13 +0100103 do_dcache_maintenance_by_mva cvac
104endfunc clean_dcache_range
Achin Gupta4f6ad662013-10-25 09:08:21 +0100105
106 /* ------------------------------------------
Olivier Deprezc80d0de2024-01-17 15:12:04 +0100107 * DCache Invalidate by MVA from base address
108 * till size. 'x0' = addr, 'x1' = size
Achin Gupta4f6ad662013-10-25 09:08:21 +0100109 * ------------------------------------------
110 */
Andrew Thoelke38bde412014-03-18 13:46:55 +0000111func inv_dcache_range
Okash Khawaja037b56e2022-11-04 12:38:01 +0000112 check_plat_can_cmo
Achin Guptae9c4a642015-09-11 16:03:13 +0100113 do_dcache_maintenance_by_mva ivac
Kévin Petita877c252015-03-24 14:03:57 +0000114endfunc inv_dcache_range
Achin Gupta4f6ad662013-10-25 09:08:21 +0100115
Robert Wakim48e6b572021-10-21 15:39:56 +0100116 /*
Robert Wakim48e6b572021-10-21 15:39:56 +0100117 * ------------------------------------------
Olivier Deprezc80d0de2024-01-17 15:12:04 +0100118 * DCache Clean+Invalidate by PA to POPA from
119 * base address till size.
Robert Wakim48e6b572021-10-21 15:39:56 +0100120 * 'x0' = addr, 'x1' = size
121 * ------------------------------------------
122 */
123func flush_dcache_to_popa_range
Okash Khawaja037b56e2022-11-04 12:38:01 +0000124 check_plat_can_cmo
Olivier Deprezc80d0de2024-01-17 15:12:04 +0100125 /* dc cipapa, x0 */
126 do_dcache_maintenance_instr dc_cipapa_x0
Robert Wakim48e6b572021-10-21 15:39:56 +0100127endfunc flush_dcache_to_popa_range
128
Olivier Deprezc80d0de2024-01-17 15:12:04 +0100129 /*
130 * ------------------------------------------
131 * Clean+Invalidate by PA to POPA (MTE2)
132 * from base address till size.
133 * 'x0' = addr, 'x1' = size
134 * ------------------------------------------
135 * On implementations with FEAT_MTE2, Root firmware must issue
136 * DC_CIGDPAPA instead of DC_CIPAPA, in order to additionally
137 * clean and invalidate Allocation Tags associated with the
138 * affected locations.
139 */
140func flush_dcache_to_popa_range_mte2
141 check_plat_can_cmo
142 /* dc cigdpapa, x0 */
143 do_dcache_maintenance_instr dc_cigdpapa_x0
144endfunc flush_dcache_to_popa_range_mte2
145
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100146 /* ---------------------------------------------------------------
147 * Data cache operations by set/way to the level specified
148 *
149 * The main function, do_dcsw_op requires:
150 * x0: The operation type (0-2), as defined in arch.h
151 * x3: The last cache level to operate on
152 * x9: clidr_el1
Soby Mathew42aa5eb2014-09-02 10:47:33 +0100153 * x10: The cache level to begin operation from
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100154 * and will carry out the operation on each data cache from level 0
155 * to the level in x3 in sequence
156 *
157 * The dcsw_op macro sets up the x3 and x9 parameters based on
158 * clidr_el1 cache information before invoking the main function
159 * ---------------------------------------------------------------
Achin Gupta4f6ad662013-10-25 09:08:21 +0100160 */
Achin Gupta4f6ad662013-10-25 09:08:21 +0100161
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100162 .macro dcsw_op shift, fw, ls
163 mrs x9, clidr_el1
164 ubfx x3, x9, \shift, \fw
165 lsl x3, x3, \ls
Soby Mathew42aa5eb2014-09-02 10:47:33 +0100166 mov x10, xzr
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100167 b do_dcsw_op
168 .endm
Achin Gupta4f6ad662013-10-25 09:08:21 +0100169
Andrew Thoelke38bde412014-03-18 13:46:55 +0000170func do_dcsw_op
Achin Gupta4f6ad662013-10-25 09:08:21 +0100171 cbz x3, exit
johpow0174b7e442021-12-01 13:18:30 -0600172 mrs x12, ID_AA64MMFR2_EL1 // stash FEAT_CCIDX identifier in x12
173 ubfx x12, x12, #ID_AA64MMFR2_EL1_CCIDX_SHIFT, #ID_AA64MMFR2_EL1_CCIDX_LENGTH
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100174 adr x14, dcsw_loop_table // compute inner loop address
175 add x14, x14, x0, lsl #5 // inner loop is 8x32-bit instructions
Alexei Fedorov90f2e882019-05-24 12:17:09 +0100176#if ENABLE_BTI
177 add x14, x14, x0, lsl #2 // inner loop is + "bti j" instruction
178#endif
Achin Gupta4f6ad662013-10-25 09:08:21 +0100179 mov x0, x9
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100180 mov w8, #1
181loop1:
182 add x2, x10, x10, lsr #1 // work out 3x current cache level
183 lsr x1, x0, x2 // extract cache type bits from clidr
184 and x1, x1, #7 // mask the bits for current cache only
185 cmp x1, #2 // see what cache we have at this level
Douglas Raillard9d92e8c2017-03-07 16:36:14 +0000186 b.lo level_done // nothing to do if no cache or icache
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100187
188 msr csselr_el1, x10 // select current cache level in csselr
189 isb // isb to sych the new cssr&csidr
190 mrs x1, ccsidr_el1 // read the new ccsidr
191 and x2, x1, #7 // extract the length of the cache lines
192 add x2, x2, #4 // add 4 (line length offset)
johpow0174b7e442021-12-01 13:18:30 -0600193
194 cbz x12, 1f // check for FEAT_CCIDX for Associativity
195 ubfx x4, x1, #3, #21 // x4 = associativity CCSIDR_EL1[23:3]
196 b 2f
1971:
198 ubfx x4, x1, #3, #10 // x4 = associativity CCSIDR_EL1[12:3]
1992:
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100200 clz w5, w4 // bit position of way size increment
201 lsl w9, w4, w5 // w9 = aligned max way number
202 lsl w16, w8, w5 // w16 = way number loop decrement
203 orr w9, w10, w9 // w9 = combine way and cache number
johpow0174b7e442021-12-01 13:18:30 -0600204
205 cbz x12, 3f // check for FEAT_CCIDX for NumSets
206 ubfx x6, x1, #32, #24 // x6 (w6) = numsets CCSIDR_EL1[55:32]
207 // ISA will not allow x->w ubfx
208 b 4f
2093:
210 ubfx w6, w1, #13, #15 // w6 = numsets CCSIDR_EL1[27:13]
2114:
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100212 lsl w17, w8, w2 // w17 = set number loop decrement
213 dsb sy // barrier before we start this level
214 br x14 // jump to DC operation specific loop
215
216 .macro dcsw_loop _op
Alexei Fedorov90f2e882019-05-24 12:17:09 +0100217#if ENABLE_BTI
218 bti j
219#endif
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100220loop2_\_op:
221 lsl w7, w6, w2 // w7 = aligned max set number
222
223loop3_\_op:
224 orr w11, w9, w7 // combine cache, way and set number
225 dc \_op, x11
226 subs w7, w7, w17 // decrement set number
Douglas Raillard9d92e8c2017-03-07 16:36:14 +0000227 b.hs loop3_\_op
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100228
229 subs x9, x9, x16 // decrement way number
Douglas Raillard9d92e8c2017-03-07 16:36:14 +0000230 b.hs loop2_\_op
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100231
232 b level_done
233 .endm
234
235level_done:
236 add x10, x10, #2 // increment cache number
237 cmp x3, x10
Petre-Ionut Tudore5a6fef2019-11-07 15:18:03 +0000238 b.hi loop1
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100239 msr csselr_el1, xzr // select cache level 0 in csselr
240 dsb sy // barrier to complete final cache operation
241 isb
Achin Gupta4f6ad662013-10-25 09:08:21 +0100242exit:
243 ret
Kévin Petita877c252015-03-24 14:03:57 +0000244endfunc do_dcsw_op
Achin Gupta4f6ad662013-10-25 09:08:21 +0100245
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100246dcsw_loop_table:
247 dcsw_loop isw
248 dcsw_loop cisw
249 dcsw_loop csw
250
Achin Gupta4f6ad662013-10-25 09:08:21 +0100251
Andrew Thoelke38bde412014-03-18 13:46:55 +0000252func dcsw_op_louis
Okash Khawaja94532202022-11-14 12:50:30 +0000253 check_plat_can_cmo
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100254 dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
Kévin Petita877c252015-03-24 14:03:57 +0000255endfunc dcsw_op_louis
Achin Gupta4f6ad662013-10-25 09:08:21 +0100256
257
Andrew Thoelke38bde412014-03-18 13:46:55 +0000258func dcsw_op_all
Okash Khawaja037b56e2022-11-04 12:38:01 +0000259 check_plat_can_cmo
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100260 dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
Kévin Petita877c252015-03-24 14:03:57 +0000261endfunc dcsw_op_all
Soby Mathew42aa5eb2014-09-02 10:47:33 +0100262
263 /* ---------------------------------------------------------------
264 * Helper macro for data cache operations by set/way for the
265 * level specified
266 * ---------------------------------------------------------------
267 */
268 .macro dcsw_op_level level
269 mrs x9, clidr_el1
270 mov x3, \level
271 sub x10, x3, #2
272 b do_dcsw_op
273 .endm
274
275 /* ---------------------------------------------------------------
276 * Data cache operations by set/way for level 1 cache
277 *
278 * The main function, do_dcsw_op requires:
279 * x0: The operation type (0-2), as defined in arch.h
280 * ---------------------------------------------------------------
281 */
282func dcsw_op_level1
Okash Khawaja037b56e2022-11-04 12:38:01 +0000283 check_plat_can_cmo
Soby Mathew42aa5eb2014-09-02 10:47:33 +0100284 dcsw_op_level #(1 << LEVEL_SHIFT)
Kévin Petita877c252015-03-24 14:03:57 +0000285endfunc dcsw_op_level1
Soby Mathew42aa5eb2014-09-02 10:47:33 +0100286
287 /* ---------------------------------------------------------------
288 * Data cache operations by set/way for level 2 cache
289 *
290 * The main function, do_dcsw_op requires:
291 * x0: The operation type (0-2), as defined in arch.h
292 * ---------------------------------------------------------------
293 */
294func dcsw_op_level2
Okash Khawaja037b56e2022-11-04 12:38:01 +0000295 check_plat_can_cmo
Soby Mathew42aa5eb2014-09-02 10:47:33 +0100296 dcsw_op_level #(2 << LEVEL_SHIFT)
Kévin Petita877c252015-03-24 14:03:57 +0000297endfunc dcsw_op_level2
Soby Mathew42aa5eb2014-09-02 10:47:33 +0100298
299 /* ---------------------------------------------------------------
300 * Data cache operations by set/way for level 3 cache
301 *
302 * The main function, do_dcsw_op requires:
303 * x0: The operation type (0-2), as defined in arch.h
304 * ---------------------------------------------------------------
305 */
306func dcsw_op_level3
Okash Khawaja037b56e2022-11-04 12:38:01 +0000307 check_plat_can_cmo
Soby Mathew42aa5eb2014-09-02 10:47:33 +0100308 dcsw_op_level #(3 << LEVEL_SHIFT)
Kévin Petita877c252015-03-24 14:03:57 +0000309endfunc dcsw_op_level3