blob: 7cbefe6ead59c5a4d7c2a38a2ac9f43d3d4acc5d [file] [log] [blame]
Soby Mathewd29f67b2016-05-05 12:31:57 +01001/*
2 * Copyright (c) 2016, ARM Limited and Contributors. All rights reserved.
3 *
dp-armfa3cf0b2017-05-03 09:38:09 +01004 * SPDX-License-Identifier: BSD-3-Clause
Soby Mathewd29f67b2016-05-05 12:31:57 +01005 */
6
7#include <arch.h>
8#include <asm_macros.S>
9
10 .globl flush_dcache_range
11 .globl clean_dcache_range
12 .globl inv_dcache_range
13 .globl dcsw_op_louis
14 .globl dcsw_op_all
15 .globl dcsw_op_level1
16 .globl dcsw_op_level2
17 .globl dcsw_op_level3
18
19/*
20 * This macro can be used for implementing various data cache operations `op`
21 */
22.macro do_dcache_maintenance_by_mva op, coproc, opc1, CRn, CRm, opc2
Soby Mathewf7e8aee2017-06-15 16:18:45 +010023 /* Exit early if size is zero */
24 cmp r1, #0
25 beq exit_loop_\op
Soby Mathewd29f67b2016-05-05 12:31:57 +010026 dcache_line_size r2, r3
27 add r1, r0, r1
28 sub r3, r2, #1
29 bic r0, r0, r3
30loop_\op:
31 stcopr r0, \coproc, \opc1, \CRn, \CRm, \opc2
32 add r0, r0, r2
33 cmp r0, r1
34 blo loop_\op
35 dsb sy
Soby Mathewf7e8aee2017-06-15 16:18:45 +010036exit_loop_\op:
Soby Mathewd29f67b2016-05-05 12:31:57 +010037 bx lr
38.endm
39
40 /* ------------------------------------------
41 * Clean+Invalidate from base address till
42 * size. 'r0' = addr, 'r1' = size
43 * ------------------------------------------
44 */
45func flush_dcache_range
46 do_dcache_maintenance_by_mva cimvac, DCCIMVAC
47endfunc flush_dcache_range
48
49 /* ------------------------------------------
50 * Clean from base address till size.
51 * 'r0' = addr, 'r1' = size
52 * ------------------------------------------
53 */
54func clean_dcache_range
55 do_dcache_maintenance_by_mva cmvac, DCCMVAC
56endfunc clean_dcache_range
57
58 /* ------------------------------------------
59 * Invalidate from base address till
60 * size. 'r0' = addr, 'r1' = size
61 * ------------------------------------------
62 */
63func inv_dcache_range
64 do_dcache_maintenance_by_mva imvac, DCIMVAC
65endfunc inv_dcache_range
66
67 /* ----------------------------------------------------------------
68 * Data cache operations by set/way to the level specified
69 *
70 * The main function, do_dcsw_op requires:
71 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW),
72 * as defined in arch.h
73 * r1: The cache level to begin operation from
74 * r2: clidr_el1
75 * r3: The last cache level to operate on
76 * and will carry out the operation on each data cache from level 0
77 * to the level in r3 in sequence
78 *
79 * The dcsw_op macro sets up the r2 and r3 parameters based on
80 * clidr_el1 cache information before invoking the main function
81 * ----------------------------------------------------------------
82 */
83
84 .macro dcsw_op shift, fw, ls
85 ldcopr r2, CLIDR
86 ubfx r3, r2, \shift, \fw
87 lsl r3, r3, \ls
88 mov r1, #0
89 b do_dcsw_op
90 .endm
91
92func do_dcsw_op
Joel Huttond7590972019-04-09 14:52:59 +010093 push {r4-r12, lr}
Soby Mathewd29f67b2016-05-05 12:31:57 +010094 adr r11, dcsw_loop_table // compute cache op based on the operation type
95 add r6, r11, r0, lsl #3 // cache op is 2x32-bit instructions
96loop1:
97 add r10, r1, r1, LSR #1 // Work out 3x current cache level
98 mov r12, r2, LSR r10 // extract cache type bits from clidr
Joel Huttond7590972019-04-09 14:52:59 +010099 and r12, r12, #7 // mask the bits for current cache only
Soby Mathewd29f67b2016-05-05 12:31:57 +0100100 cmp r12, #2 // see what cache we have at this level
Joel Huttond7590972019-04-09 14:52:59 +0100101 blo level_done // no cache or only instruction cache at this level
Soby Mathewd29f67b2016-05-05 12:31:57 +0100102
103 stcopr r1, CSSELR // select current cache level in csselr
104 isb // isb to sych the new cssr&csidr
105 ldcopr r12, CCSIDR // read the new ccsidr
Joel Huttond7590972019-04-09 14:52:59 +0100106 and r10, r12, #7 // extract the length of the cache lines
107 add r10, r10, #4 // add 4 (r10 = line length offset)
Soby Mathewd29f67b2016-05-05 12:31:57 +0100108 ubfx r4, r12, #3, #10 // r4 = maximum way number (right aligned)
Joel Huttond7590972019-04-09 14:52:59 +0100109 clz r5, r4 // r5 = the bit position of the way size increment
Soby Mathewd29f67b2016-05-05 12:31:57 +0100110 mov r9, r4 // r9 working copy of the aligned max way number
111
112loop2:
113 ubfx r7, r12, #13, #15 // r7 = max set number (right aligned)
114
115loop3:
116 orr r0, r1, r9, LSL r5 // factor in the way number and cache level into r0
117 orr r0, r0, r7, LSL r10 // factor in the set number
118
119 blx r6
Joel Huttond7590972019-04-09 14:52:59 +0100120 subs r7, r7, #1 // decrement the set number
Douglas Raillard9d92e8c2017-03-07 16:36:14 +0000121 bhs loop3
Joel Huttond7590972019-04-09 14:52:59 +0100122 subs r9, r9, #1 // decrement the way number
Douglas Raillard9d92e8c2017-03-07 16:36:14 +0000123 bhs loop2
Soby Mathewd29f67b2016-05-05 12:31:57 +0100124level_done:
125 add r1, r1, #2 // increment the cache number
126 cmp r3, r1
Joel Hutton50ed13a2019-04-09 14:45:34 +0100127 // Ensure completion of previous cache maintenance instruction. Note
128 // this also mitigates erratum 814220 on Cortex-A7
129 dsb sy
Douglas Raillard9d92e8c2017-03-07 16:36:14 +0000130 bhi loop1
Soby Mathewd29f67b2016-05-05 12:31:57 +0100131
132 mov r6, #0
133 stcopr r6, CSSELR //select cache level 0 in csselr
134 dsb sy
135 isb
Joel Huttond7590972019-04-09 14:52:59 +0100136 pop {r4-r12, pc}
Soby Mathewd29f67b2016-05-05 12:31:57 +0100137
138dcsw_loop_table:
139 stcopr r0, DCISW
140 bx lr
141 stcopr r0, DCCISW
142 bx lr
143 stcopr r0, DCCSW
144 bx lr
145
146endfunc do_dcsw_op
147
148 /* ---------------------------------------------------------------
149 * Data cache operations by set/way till PoU.
150 *
151 * The function requires :
152 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW),
153 * as defined in arch.h
154 * ---------------------------------------------------------------
155 */
156func dcsw_op_louis
157 dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
158endfunc dcsw_op_louis
159
160 /* ---------------------------------------------------------------
161 * Data cache operations by set/way till PoC.
162 *
163 * The function requires :
164 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW),
165 * as defined in arch.h
166 * ---------------------------------------------------------------
167 */
168func dcsw_op_all
169 dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
170endfunc dcsw_op_all
171
172
173 /* ---------------------------------------------------------------
174 * Helper macro for data cache operations by set/way for the
175 * level specified
176 * ---------------------------------------------------------------
177 */
178 .macro dcsw_op_level level
179 ldcopr r2, CLIDR
180 mov r3, \level
181 sub r1, r3, #2
182 b do_dcsw_op
183 .endm
184
185 /* ---------------------------------------------------------------
186 * Data cache operations by set/way for level 1 cache
187 *
188 * The main function, do_dcsw_op requires:
189 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW),
190 * as defined in arch.h
191 * ---------------------------------------------------------------
192 */
193func dcsw_op_level1
194 dcsw_op_level #(1 << LEVEL_SHIFT)
195endfunc dcsw_op_level1
196
197 /* ---------------------------------------------------------------
198 * Data cache operations by set/way for level 2 cache
199 *
200 * The main function, do_dcsw_op requires:
201 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW),
202 * as defined in arch.h
203 * ---------------------------------------------------------------
204 */
205func dcsw_op_level2
206 dcsw_op_level #(2 << LEVEL_SHIFT)
207endfunc dcsw_op_level2
208
209 /* ---------------------------------------------------------------
210 * Data cache operations by set/way for level 3 cache
211 *
212 * The main function, do_dcsw_op requires:
213 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW),
214 * as defined in arch.h
215 * ---------------------------------------------------------------
216 */
217func dcsw_op_level3
218 dcsw_op_level #(3 << LEVEL_SHIFT)
219endfunc dcsw_op_level3