blob: de9c8e4f0154f4fcfaac96d7195b948c64542cd4 [file] [log] [blame]
Achin Gupta4f6ad662013-10-25 09:08:21 +01001/*
Petre-Ionut Tudore5a6fef2019-11-07 15:18:03 +00002 * Copyright (c) 2013-2020, ARM Limited and Contributors. All rights reserved.
Achin Gupta4f6ad662013-10-25 09:08:21 +01003 *
dp-armfa3cf0b2017-05-03 09:38:09 +01004 * SPDX-License-Identifier: BSD-3-Clause
Achin Gupta4f6ad662013-10-25 09:08:21 +01005 */
6
Dan Handley2bd4ef22014-04-09 13:14:54 +01007#include <arch.h>
Achin Gupta4f6ad662013-10-25 09:08:21 +01008#include <asm_macros.S>
9
Achin Gupta4f6ad662013-10-25 09:08:21 +010010 .globl flush_dcache_range
Achin Guptae9c4a642015-09-11 16:03:13 +010011 .globl clean_dcache_range
Achin Gupta4f6ad662013-10-25 09:08:21 +010012 .globl inv_dcache_range
13 .globl dcsw_op_louis
14 .globl dcsw_op_all
Soby Mathew42aa5eb2014-09-02 10:47:33 +010015 .globl dcsw_op_level1
16 .globl dcsw_op_level2
17 .globl dcsw_op_level3
Achin Gupta4f6ad662013-10-25 09:08:21 +010018
Achin Guptae9c4a642015-09-11 16:03:13 +010019/*
20 * This macro can be used for implementing various data cache operations `op`
21 */
22.macro do_dcache_maintenance_by_mva op
Soby Mathewf7e8aee2017-06-15 16:18:45 +010023 /* Exit early if size is zero */
24 cbz x1, exit_loop_\op
Achin Gupta4f6ad662013-10-25 09:08:21 +010025 dcache_line_size x2, x3
26 add x1, x0, x1
27 sub x3, x2, #1
28 bic x0, x0, x3
Achin Guptae9c4a642015-09-11 16:03:13 +010029loop_\op:
30 dc \op, x0
Achin Gupta4f6ad662013-10-25 09:08:21 +010031 add x0, x0, x2
32 cmp x0, x1
Petre-Ionut Tudore5a6fef2019-11-07 15:18:03 +000033 b.lo loop_\op
Achin Gupta4f6ad662013-10-25 09:08:21 +010034 dsb sy
Soby Mathewf7e8aee2017-06-15 16:18:45 +010035exit_loop_\op:
Achin Gupta4f6ad662013-10-25 09:08:21 +010036 ret
Achin Guptae9c4a642015-09-11 16:03:13 +010037.endm
38 /* ------------------------------------------
39 * Clean+Invalidate from base address till
40 * size. 'x0' = addr, 'x1' = size
41 * ------------------------------------------
42 */
43func flush_dcache_range
44 do_dcache_maintenance_by_mva civac
Kévin Petita877c252015-03-24 14:03:57 +000045endfunc flush_dcache_range
Achin Gupta4f6ad662013-10-25 09:08:21 +010046
Achin Guptae9c4a642015-09-11 16:03:13 +010047 /* ------------------------------------------
48 * Clean from base address till size.
49 * 'x0' = addr, 'x1' = size
50 * ------------------------------------------
51 */
52func clean_dcache_range
53 do_dcache_maintenance_by_mva cvac
54endfunc clean_dcache_range
Achin Gupta4f6ad662013-10-25 09:08:21 +010055
56 /* ------------------------------------------
57 * Invalidate from base address till
58 * size. 'x0' = addr, 'x1' = size
59 * ------------------------------------------
60 */
Andrew Thoelke38bde412014-03-18 13:46:55 +000061func inv_dcache_range
Achin Guptae9c4a642015-09-11 16:03:13 +010062 do_dcache_maintenance_by_mva ivac
Kévin Petita877c252015-03-24 14:03:57 +000063endfunc inv_dcache_range
Achin Gupta4f6ad662013-10-25 09:08:21 +010064
65
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +010066 /* ---------------------------------------------------------------
67 * Data cache operations by set/way to the level specified
68 *
69 * The main function, do_dcsw_op requires:
70 * x0: The operation type (0-2), as defined in arch.h
71 * x3: The last cache level to operate on
72 * x9: clidr_el1
Soby Mathew42aa5eb2014-09-02 10:47:33 +010073 * x10: The cache level to begin operation from
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +010074 * and will carry out the operation on each data cache from level 0
75 * to the level in x3 in sequence
76 *
77 * The dcsw_op macro sets up the x3 and x9 parameters based on
78 * clidr_el1 cache information before invoking the main function
79 * ---------------------------------------------------------------
Achin Gupta4f6ad662013-10-25 09:08:21 +010080 */
Achin Gupta4f6ad662013-10-25 09:08:21 +010081
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +010082 .macro dcsw_op shift, fw, ls
83 mrs x9, clidr_el1
84 ubfx x3, x9, \shift, \fw
85 lsl x3, x3, \ls
Soby Mathew42aa5eb2014-09-02 10:47:33 +010086 mov x10, xzr
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +010087 b do_dcsw_op
88 .endm
Achin Gupta4f6ad662013-10-25 09:08:21 +010089
Andrew Thoelke38bde412014-03-18 13:46:55 +000090func do_dcsw_op
Achin Gupta4f6ad662013-10-25 09:08:21 +010091 cbz x3, exit
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +010092 adr x14, dcsw_loop_table // compute inner loop address
93 add x14, x14, x0, lsl #5 // inner loop is 8x32-bit instructions
Alexei Fedorov90f2e882019-05-24 12:17:09 +010094#if ENABLE_BTI
95 add x14, x14, x0, lsl #2 // inner loop is + "bti j" instruction
96#endif
Achin Gupta4f6ad662013-10-25 09:08:21 +010097 mov x0, x9
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +010098 mov w8, #1
99loop1:
100 add x2, x10, x10, lsr #1 // work out 3x current cache level
101 lsr x1, x0, x2 // extract cache type bits from clidr
102 and x1, x1, #7 // mask the bits for current cache only
103 cmp x1, #2 // see what cache we have at this level
Douglas Raillard9d92e8c2017-03-07 16:36:14 +0000104 b.lo level_done // nothing to do if no cache or icache
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100105
106 msr csselr_el1, x10 // select current cache level in csselr
107 isb // isb to sych the new cssr&csidr
108 mrs x1, ccsidr_el1 // read the new ccsidr
109 and x2, x1, #7 // extract the length of the cache lines
110 add x2, x2, #4 // add 4 (line length offset)
111 ubfx x4, x1, #3, #10 // maximum way number
112 clz w5, w4 // bit position of way size increment
113 lsl w9, w4, w5 // w9 = aligned max way number
114 lsl w16, w8, w5 // w16 = way number loop decrement
115 orr w9, w10, w9 // w9 = combine way and cache number
116 ubfx w6, w1, #13, #15 // w6 = max set number
117 lsl w17, w8, w2 // w17 = set number loop decrement
118 dsb sy // barrier before we start this level
119 br x14 // jump to DC operation specific loop
120
121 .macro dcsw_loop _op
Alexei Fedorov90f2e882019-05-24 12:17:09 +0100122#if ENABLE_BTI
123 bti j
124#endif
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100125loop2_\_op:
126 lsl w7, w6, w2 // w7 = aligned max set number
127
128loop3_\_op:
129 orr w11, w9, w7 // combine cache, way and set number
130 dc \_op, x11
131 subs w7, w7, w17 // decrement set number
Douglas Raillard9d92e8c2017-03-07 16:36:14 +0000132 b.hs loop3_\_op
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100133
134 subs x9, x9, x16 // decrement way number
Douglas Raillard9d92e8c2017-03-07 16:36:14 +0000135 b.hs loop2_\_op
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100136
137 b level_done
138 .endm
139
140level_done:
141 add x10, x10, #2 // increment cache number
142 cmp x3, x10
Petre-Ionut Tudore5a6fef2019-11-07 15:18:03 +0000143 b.hi loop1
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100144 msr csselr_el1, xzr // select cache level 0 in csselr
145 dsb sy // barrier to complete final cache operation
146 isb
Achin Gupta4f6ad662013-10-25 09:08:21 +0100147exit:
148 ret
Kévin Petita877c252015-03-24 14:03:57 +0000149endfunc do_dcsw_op
Achin Gupta4f6ad662013-10-25 09:08:21 +0100150
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100151dcsw_loop_table:
152 dcsw_loop isw
153 dcsw_loop cisw
154 dcsw_loop csw
155
Achin Gupta4f6ad662013-10-25 09:08:21 +0100156
Andrew Thoelke38bde412014-03-18 13:46:55 +0000157func dcsw_op_louis
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100158 dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
Kévin Petita877c252015-03-24 14:03:57 +0000159endfunc dcsw_op_louis
Achin Gupta4f6ad662013-10-25 09:08:21 +0100160
161
Andrew Thoelke38bde412014-03-18 13:46:55 +0000162func dcsw_op_all
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100163 dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
Kévin Petita877c252015-03-24 14:03:57 +0000164endfunc dcsw_op_all
Soby Mathew42aa5eb2014-09-02 10:47:33 +0100165
166 /* ---------------------------------------------------------------
167 * Helper macro for data cache operations by set/way for the
168 * level specified
169 * ---------------------------------------------------------------
170 */
171 .macro dcsw_op_level level
172 mrs x9, clidr_el1
173 mov x3, \level
174 sub x10, x3, #2
175 b do_dcsw_op
176 .endm
177
178 /* ---------------------------------------------------------------
179 * Data cache operations by set/way for level 1 cache
180 *
181 * The main function, do_dcsw_op requires:
182 * x0: The operation type (0-2), as defined in arch.h
183 * ---------------------------------------------------------------
184 */
185func dcsw_op_level1
186 dcsw_op_level #(1 << LEVEL_SHIFT)
Kévin Petita877c252015-03-24 14:03:57 +0000187endfunc dcsw_op_level1
Soby Mathew42aa5eb2014-09-02 10:47:33 +0100188
189 /* ---------------------------------------------------------------
190 * Data cache operations by set/way for level 2 cache
191 *
192 * The main function, do_dcsw_op requires:
193 * x0: The operation type (0-2), as defined in arch.h
194 * ---------------------------------------------------------------
195 */
196func dcsw_op_level2
197 dcsw_op_level #(2 << LEVEL_SHIFT)
Kévin Petita877c252015-03-24 14:03:57 +0000198endfunc dcsw_op_level2
Soby Mathew42aa5eb2014-09-02 10:47:33 +0100199
200 /* ---------------------------------------------------------------
201 * Data cache operations by set/way for level 3 cache
202 *
203 * The main function, do_dcsw_op requires:
204 * x0: The operation type (0-2), as defined in arch.h
205 * ---------------------------------------------------------------
206 */
207func dcsw_op_level3
208 dcsw_op_level #(3 << LEVEL_SHIFT)
Kévin Petita877c252015-03-24 14:03:57 +0000209endfunc dcsw_op_level3