blob: 314ed6efd7d04c84195df588be1361433f56c0e3 [file] [log] [blame]
Achin Gupta4f6ad662013-10-25 09:08:21 +01001/*
Robert Wakim48e6b572021-10-21 15:39:56 +01002 * Copyright (c) 2013-2022, Arm Limited and Contributors. All rights reserved.
Achin Gupta4f6ad662013-10-25 09:08:21 +01003 *
dp-armfa3cf0b2017-05-03 09:38:09 +01004 * SPDX-License-Identifier: BSD-3-Clause
Achin Gupta4f6ad662013-10-25 09:08:21 +01005 */
6
Dan Handley2bd4ef22014-04-09 13:14:54 +01007#include <arch.h>
Achin Gupta4f6ad662013-10-25 09:08:21 +01008#include <asm_macros.S>
9
Achin Gupta4f6ad662013-10-25 09:08:21 +010010 .globl flush_dcache_range
Robert Wakim48e6b572021-10-21 15:39:56 +010011 .globl flush_dcache_to_popa_range
Achin Guptae9c4a642015-09-11 16:03:13 +010012 .globl clean_dcache_range
Achin Gupta4f6ad662013-10-25 09:08:21 +010013 .globl inv_dcache_range
14 .globl dcsw_op_louis
15 .globl dcsw_op_all
Soby Mathew42aa5eb2014-09-02 10:47:33 +010016 .globl dcsw_op_level1
17 .globl dcsw_op_level2
18 .globl dcsw_op_level3
Achin Gupta4f6ad662013-10-25 09:08:21 +010019
Achin Guptae9c4a642015-09-11 16:03:13 +010020/*
21 * This macro can be used for implementing various data cache operations `op`
22 */
23.macro do_dcache_maintenance_by_mva op
Soby Mathewf7e8aee2017-06-15 16:18:45 +010024 /* Exit early if size is zero */
25 cbz x1, exit_loop_\op
Achin Gupta4f6ad662013-10-25 09:08:21 +010026 dcache_line_size x2, x3
27 add x1, x0, x1
28 sub x3, x2, #1
29 bic x0, x0, x3
Achin Guptae9c4a642015-09-11 16:03:13 +010030loop_\op:
31 dc \op, x0
Achin Gupta4f6ad662013-10-25 09:08:21 +010032 add x0, x0, x2
33 cmp x0, x1
Petre-Ionut Tudore5a6fef2019-11-07 15:18:03 +000034 b.lo loop_\op
Achin Gupta4f6ad662013-10-25 09:08:21 +010035 dsb sy
Soby Mathewf7e8aee2017-06-15 16:18:45 +010036exit_loop_\op:
Achin Gupta4f6ad662013-10-25 09:08:21 +010037 ret
Achin Guptae9c4a642015-09-11 16:03:13 +010038.endm
Okash Khawaja037b56e2022-11-04 12:38:01 +000039
40.macro check_plat_can_cmo
41#if CONDITIONAL_CMO
42 mov x3, x30
43 mov x2, x0
44 bl plat_can_cmo
45 mov x30, x3
46 cbnz x0, 1f
47 ret
481:
49 mov x0, x2
50#endif
51.endm
Achin Guptae9c4a642015-09-11 16:03:13 +010052 /* ------------------------------------------
53 * Clean+Invalidate from base address till
54 * size. 'x0' = addr, 'x1' = size
55 * ------------------------------------------
56 */
57func flush_dcache_range
Okash Khawaja037b56e2022-11-04 12:38:01 +000058 check_plat_can_cmo
Achin Guptae9c4a642015-09-11 16:03:13 +010059 do_dcache_maintenance_by_mva civac
Kévin Petita877c252015-03-24 14:03:57 +000060endfunc flush_dcache_range
Achin Gupta4f6ad662013-10-25 09:08:21 +010061
Achin Guptae9c4a642015-09-11 16:03:13 +010062 /* ------------------------------------------
63 * Clean from base address till size.
64 * 'x0' = addr, 'x1' = size
65 * ------------------------------------------
66 */
67func clean_dcache_range
Okash Khawaja037b56e2022-11-04 12:38:01 +000068 check_plat_can_cmo
Achin Guptae9c4a642015-09-11 16:03:13 +010069 do_dcache_maintenance_by_mva cvac
70endfunc clean_dcache_range
Achin Gupta4f6ad662013-10-25 09:08:21 +010071
72 /* ------------------------------------------
73 * Invalidate from base address till
74 * size. 'x0' = addr, 'x1' = size
75 * ------------------------------------------
76 */
Andrew Thoelke38bde412014-03-18 13:46:55 +000077func inv_dcache_range
Okash Khawaja037b56e2022-11-04 12:38:01 +000078 check_plat_can_cmo
Achin Guptae9c4a642015-09-11 16:03:13 +010079 do_dcache_maintenance_by_mva ivac
Kévin Petita877c252015-03-24 14:03:57 +000080endfunc inv_dcache_range
Achin Gupta4f6ad662013-10-25 09:08:21 +010081
82
Robert Wakim48e6b572021-10-21 15:39:56 +010083 /*
84 * On implementations with FEAT_MTE2,
85 * Root firmware must issue DC_CIGDPAPA instead of DC_CIPAPA ,
86 * in order to additionally clean and invalidate Allocation Tags
87 * associated with the affected locations.
88 *
89 * ------------------------------------------
90 * Clean+Invalidate by PA to POPA
91 * from base address till size.
92 * 'x0' = addr, 'x1' = size
93 * ------------------------------------------
94 */
95func flush_dcache_to_popa_range
96 /* Exit early if size is zero */
97 cbz x1, exit_loop_dc_cipapa
Okash Khawaja037b56e2022-11-04 12:38:01 +000098 check_plat_can_cmo
Robert Wakim48e6b572021-10-21 15:39:56 +010099 dcache_line_size x2, x3
100 sub x3, x2, #1
101 bic x0, x0, x3
102 add x1, x1, x0
103loop_dc_cipapa:
104 sys #6, c7, c14, #1, x0 /* DC CIPAPA,<Xt> */
105 add x0, x0, x2
106 cmp x0, x1
107 b.lo loop_dc_cipapa
108 dsb osh
109exit_loop_dc_cipapa:
110 ret
111endfunc flush_dcache_to_popa_range
112
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100113 /* ---------------------------------------------------------------
114 * Data cache operations by set/way to the level specified
115 *
116 * The main function, do_dcsw_op requires:
117 * x0: The operation type (0-2), as defined in arch.h
118 * x3: The last cache level to operate on
119 * x9: clidr_el1
Soby Mathew42aa5eb2014-09-02 10:47:33 +0100120 * x10: The cache level to begin operation from
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100121 * and will carry out the operation on each data cache from level 0
122 * to the level in x3 in sequence
123 *
124 * The dcsw_op macro sets up the x3 and x9 parameters based on
125 * clidr_el1 cache information before invoking the main function
126 * ---------------------------------------------------------------
Achin Gupta4f6ad662013-10-25 09:08:21 +0100127 */
Achin Gupta4f6ad662013-10-25 09:08:21 +0100128
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100129 .macro dcsw_op shift, fw, ls
130 mrs x9, clidr_el1
131 ubfx x3, x9, \shift, \fw
132 lsl x3, x3, \ls
Soby Mathew42aa5eb2014-09-02 10:47:33 +0100133 mov x10, xzr
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100134 b do_dcsw_op
135 .endm
Achin Gupta4f6ad662013-10-25 09:08:21 +0100136
Andrew Thoelke38bde412014-03-18 13:46:55 +0000137func do_dcsw_op
Achin Gupta4f6ad662013-10-25 09:08:21 +0100138 cbz x3, exit
johpow0174b7e442021-12-01 13:18:30 -0600139 mrs x12, ID_AA64MMFR2_EL1 // stash FEAT_CCIDX identifier in x12
140 ubfx x12, x12, #ID_AA64MMFR2_EL1_CCIDX_SHIFT, #ID_AA64MMFR2_EL1_CCIDX_LENGTH
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100141 adr x14, dcsw_loop_table // compute inner loop address
142 add x14, x14, x0, lsl #5 // inner loop is 8x32-bit instructions
Alexei Fedorov90f2e882019-05-24 12:17:09 +0100143#if ENABLE_BTI
144 add x14, x14, x0, lsl #2 // inner loop is + "bti j" instruction
145#endif
Achin Gupta4f6ad662013-10-25 09:08:21 +0100146 mov x0, x9
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100147 mov w8, #1
148loop1:
149 add x2, x10, x10, lsr #1 // work out 3x current cache level
150 lsr x1, x0, x2 // extract cache type bits from clidr
151 and x1, x1, #7 // mask the bits for current cache only
152 cmp x1, #2 // see what cache we have at this level
Douglas Raillard9d92e8c2017-03-07 16:36:14 +0000153 b.lo level_done // nothing to do if no cache or icache
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100154
155 msr csselr_el1, x10 // select current cache level in csselr
156 isb // isb to sych the new cssr&csidr
157 mrs x1, ccsidr_el1 // read the new ccsidr
158 and x2, x1, #7 // extract the length of the cache lines
159 add x2, x2, #4 // add 4 (line length offset)
johpow0174b7e442021-12-01 13:18:30 -0600160
161 cbz x12, 1f // check for FEAT_CCIDX for Associativity
162 ubfx x4, x1, #3, #21 // x4 = associativity CCSIDR_EL1[23:3]
163 b 2f
1641:
165 ubfx x4, x1, #3, #10 // x4 = associativity CCSIDR_EL1[12:3]
1662:
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100167 clz w5, w4 // bit position of way size increment
168 lsl w9, w4, w5 // w9 = aligned max way number
169 lsl w16, w8, w5 // w16 = way number loop decrement
170 orr w9, w10, w9 // w9 = combine way and cache number
johpow0174b7e442021-12-01 13:18:30 -0600171
172 cbz x12, 3f // check for FEAT_CCIDX for NumSets
173 ubfx x6, x1, #32, #24 // x6 (w6) = numsets CCSIDR_EL1[55:32]
174 // ISA will not allow x->w ubfx
175 b 4f
1763:
177 ubfx w6, w1, #13, #15 // w6 = numsets CCSIDR_EL1[27:13]
1784:
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100179 lsl w17, w8, w2 // w17 = set number loop decrement
180 dsb sy // barrier before we start this level
181 br x14 // jump to DC operation specific loop
182
183 .macro dcsw_loop _op
Alexei Fedorov90f2e882019-05-24 12:17:09 +0100184#if ENABLE_BTI
185 bti j
186#endif
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100187loop2_\_op:
188 lsl w7, w6, w2 // w7 = aligned max set number
189
190loop3_\_op:
191 orr w11, w9, w7 // combine cache, way and set number
192 dc \_op, x11
193 subs w7, w7, w17 // decrement set number
Douglas Raillard9d92e8c2017-03-07 16:36:14 +0000194 b.hs loop3_\_op
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100195
196 subs x9, x9, x16 // decrement way number
Douglas Raillard9d92e8c2017-03-07 16:36:14 +0000197 b.hs loop2_\_op
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100198
199 b level_done
200 .endm
201
202level_done:
203 add x10, x10, #2 // increment cache number
204 cmp x3, x10
Petre-Ionut Tudore5a6fef2019-11-07 15:18:03 +0000205 b.hi loop1
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100206 msr csselr_el1, xzr // select cache level 0 in csselr
207 dsb sy // barrier to complete final cache operation
208 isb
Achin Gupta4f6ad662013-10-25 09:08:21 +0100209exit:
210 ret
Kévin Petita877c252015-03-24 14:03:57 +0000211endfunc do_dcsw_op
Achin Gupta4f6ad662013-10-25 09:08:21 +0100212
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100213dcsw_loop_table:
214 dcsw_loop isw
215 dcsw_loop cisw
216 dcsw_loop csw
217
Achin Gupta4f6ad662013-10-25 09:08:21 +0100218
Andrew Thoelke38bde412014-03-18 13:46:55 +0000219func dcsw_op_louis
Okash Khawaja94532202022-11-14 12:50:30 +0000220 check_plat_can_cmo
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100221 dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
Kévin Petita877c252015-03-24 14:03:57 +0000222endfunc dcsw_op_louis
Achin Gupta4f6ad662013-10-25 09:08:21 +0100223
224
Andrew Thoelke38bde412014-03-18 13:46:55 +0000225func dcsw_op_all
Okash Khawaja037b56e2022-11-04 12:38:01 +0000226 check_plat_can_cmo
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100227 dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
Kévin Petita877c252015-03-24 14:03:57 +0000228endfunc dcsw_op_all
Soby Mathew42aa5eb2014-09-02 10:47:33 +0100229
230 /* ---------------------------------------------------------------
231 * Helper macro for data cache operations by set/way for the
232 * level specified
233 * ---------------------------------------------------------------
234 */
235 .macro dcsw_op_level level
236 mrs x9, clidr_el1
237 mov x3, \level
238 sub x10, x3, #2
239 b do_dcsw_op
240 .endm
241
242 /* ---------------------------------------------------------------
243 * Data cache operations by set/way for level 1 cache
244 *
245 * The main function, do_dcsw_op requires:
246 * x0: The operation type (0-2), as defined in arch.h
247 * ---------------------------------------------------------------
248 */
249func dcsw_op_level1
Okash Khawaja037b56e2022-11-04 12:38:01 +0000250 check_plat_can_cmo
Soby Mathew42aa5eb2014-09-02 10:47:33 +0100251 dcsw_op_level #(1 << LEVEL_SHIFT)
Kévin Petita877c252015-03-24 14:03:57 +0000252endfunc dcsw_op_level1
Soby Mathew42aa5eb2014-09-02 10:47:33 +0100253
254 /* ---------------------------------------------------------------
255 * Data cache operations by set/way for level 2 cache
256 *
257 * The main function, do_dcsw_op requires:
258 * x0: The operation type (0-2), as defined in arch.h
259 * ---------------------------------------------------------------
260 */
261func dcsw_op_level2
Okash Khawaja037b56e2022-11-04 12:38:01 +0000262 check_plat_can_cmo
Soby Mathew42aa5eb2014-09-02 10:47:33 +0100263 dcsw_op_level #(2 << LEVEL_SHIFT)
Kévin Petita877c252015-03-24 14:03:57 +0000264endfunc dcsw_op_level2
Soby Mathew42aa5eb2014-09-02 10:47:33 +0100265
266 /* ---------------------------------------------------------------
267 * Data cache operations by set/way for level 3 cache
268 *
269 * The main function, do_dcsw_op requires:
270 * x0: The operation type (0-2), as defined in arch.h
271 * ---------------------------------------------------------------
272 */
273func dcsw_op_level3
Okash Khawaja037b56e2022-11-04 12:38:01 +0000274 check_plat_can_cmo
Soby Mathew42aa5eb2014-09-02 10:47:33 +0100275 dcsw_op_level #(3 << LEVEL_SHIFT)
Kévin Petita877c252015-03-24 14:03:57 +0000276endfunc dcsw_op_level3