blob: acafea70b714e348be3cbc2db5a81bcb389b02ad [file] [log] [blame]
Achin Gupta4f6ad662013-10-25 09:08:21 +01001/*
Dan Handleye83b0ca2014-01-14 18:17:09 +00002 * Copyright (c) 2013-2014, ARM Limited and Contributors. All rights reserved.
Achin Gupta4f6ad662013-10-25 09:08:21 +01003 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions are met:
6 *
7 * Redistributions of source code must retain the above copyright notice, this
8 * list of conditions and the following disclaimer.
9 *
10 * Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 *
14 * Neither the name of ARM nor the names of its contributors may be used
15 * to endorse or promote products derived from this software without specific
16 * prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28 * POSSIBILITY OF SUCH DAMAGE.
29 */
30
Dan Handley2bd4ef22014-04-09 13:14:54 +010031#include <arch.h>
Achin Gupta4f6ad662013-10-25 09:08:21 +010032#include <asm_macros.S>
33
Achin Gupta4f6ad662013-10-25 09:08:21 +010034 .globl flush_dcache_range
Achin Guptae9c4a642015-09-11 16:03:13 +010035 .globl clean_dcache_range
Achin Gupta4f6ad662013-10-25 09:08:21 +010036 .globl inv_dcache_range
37 .globl dcsw_op_louis
38 .globl dcsw_op_all
Soby Mathew42aa5eb2014-09-02 10:47:33 +010039 .globl dcsw_op_level1
40 .globl dcsw_op_level2
41 .globl dcsw_op_level3
Achin Gupta4f6ad662013-10-25 09:08:21 +010042
Achin Guptae9c4a642015-09-11 16:03:13 +010043/*
44 * This macro can be used for implementing various data cache operations `op`
45 */
46.macro do_dcache_maintenance_by_mva op
Achin Gupta4f6ad662013-10-25 09:08:21 +010047 dcache_line_size x2, x3
48 add x1, x0, x1
49 sub x3, x2, #1
50 bic x0, x0, x3
Achin Guptae9c4a642015-09-11 16:03:13 +010051loop_\op:
52 dc \op, x0
Achin Gupta4f6ad662013-10-25 09:08:21 +010053 add x0, x0, x2
54 cmp x0, x1
Achin Guptae9c4a642015-09-11 16:03:13 +010055 b.lo loop_\op
Achin Gupta4f6ad662013-10-25 09:08:21 +010056 dsb sy
57 ret
Achin Guptae9c4a642015-09-11 16:03:13 +010058.endm
59 /* ------------------------------------------
60 * Clean+Invalidate from base address till
61 * size. 'x0' = addr, 'x1' = size
62 * ------------------------------------------
63 */
64func flush_dcache_range
65 do_dcache_maintenance_by_mva civac
Kévin Petita877c252015-03-24 14:03:57 +000066endfunc flush_dcache_range
Achin Gupta4f6ad662013-10-25 09:08:21 +010067
Achin Guptae9c4a642015-09-11 16:03:13 +010068 /* ------------------------------------------
69 * Clean from base address till size.
70 * 'x0' = addr, 'x1' = size
71 * ------------------------------------------
72 */
73func clean_dcache_range
74 do_dcache_maintenance_by_mva cvac
75endfunc clean_dcache_range
Achin Gupta4f6ad662013-10-25 09:08:21 +010076
77 /* ------------------------------------------
78 * Invalidate from base address till
79 * size. 'x0' = addr, 'x1' = size
80 * ------------------------------------------
81 */
Andrew Thoelke38bde412014-03-18 13:46:55 +000082func inv_dcache_range
Achin Guptae9c4a642015-09-11 16:03:13 +010083 do_dcache_maintenance_by_mva ivac
Kévin Petita877c252015-03-24 14:03:57 +000084endfunc inv_dcache_range
Achin Gupta4f6ad662013-10-25 09:08:21 +010085
86
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +010087 /* ---------------------------------------------------------------
88 * Data cache operations by set/way to the level specified
89 *
90 * The main function, do_dcsw_op requires:
91 * x0: The operation type (0-2), as defined in arch.h
92 * x3: The last cache level to operate on
93 * x9: clidr_el1
Soby Mathew42aa5eb2014-09-02 10:47:33 +010094 * x10: The cache level to begin operation from
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +010095 * and will carry out the operation on each data cache from level 0
96 * to the level in x3 in sequence
97 *
98 * The dcsw_op macro sets up the x3 and x9 parameters based on
99 * clidr_el1 cache information before invoking the main function
100 * ---------------------------------------------------------------
Achin Gupta4f6ad662013-10-25 09:08:21 +0100101 */
Achin Gupta4f6ad662013-10-25 09:08:21 +0100102
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100103 .macro dcsw_op shift, fw, ls
104 mrs x9, clidr_el1
105 ubfx x3, x9, \shift, \fw
106 lsl x3, x3, \ls
Soby Mathew42aa5eb2014-09-02 10:47:33 +0100107 mov x10, xzr
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100108 b do_dcsw_op
109 .endm
Achin Gupta4f6ad662013-10-25 09:08:21 +0100110
Andrew Thoelke38bde412014-03-18 13:46:55 +0000111func do_dcsw_op
Achin Gupta4f6ad662013-10-25 09:08:21 +0100112 cbz x3, exit
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100113 adr x14, dcsw_loop_table // compute inner loop address
114 add x14, x14, x0, lsl #5 // inner loop is 8x32-bit instructions
Achin Gupta4f6ad662013-10-25 09:08:21 +0100115 mov x0, x9
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100116 mov w8, #1
117loop1:
118 add x2, x10, x10, lsr #1 // work out 3x current cache level
119 lsr x1, x0, x2 // extract cache type bits from clidr
120 and x1, x1, #7 // mask the bits for current cache only
121 cmp x1, #2 // see what cache we have at this level
Douglas Raillard9d92e8c2017-03-07 16:36:14 +0000122 b.lo level_done // nothing to do if no cache or icache
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100123
124 msr csselr_el1, x10 // select current cache level in csselr
125 isb // isb to sych the new cssr&csidr
126 mrs x1, ccsidr_el1 // read the new ccsidr
127 and x2, x1, #7 // extract the length of the cache lines
128 add x2, x2, #4 // add 4 (line length offset)
129 ubfx x4, x1, #3, #10 // maximum way number
130 clz w5, w4 // bit position of way size increment
131 lsl w9, w4, w5 // w9 = aligned max way number
132 lsl w16, w8, w5 // w16 = way number loop decrement
133 orr w9, w10, w9 // w9 = combine way and cache number
134 ubfx w6, w1, #13, #15 // w6 = max set number
135 lsl w17, w8, w2 // w17 = set number loop decrement
136 dsb sy // barrier before we start this level
137 br x14 // jump to DC operation specific loop
138
139 .macro dcsw_loop _op
140loop2_\_op:
141 lsl w7, w6, w2 // w7 = aligned max set number
142
143loop3_\_op:
144 orr w11, w9, w7 // combine cache, way and set number
145 dc \_op, x11
146 subs w7, w7, w17 // decrement set number
Douglas Raillard9d92e8c2017-03-07 16:36:14 +0000147 b.hs loop3_\_op
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100148
149 subs x9, x9, x16 // decrement way number
Douglas Raillard9d92e8c2017-03-07 16:36:14 +0000150 b.hs loop2_\_op
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100151
152 b level_done
153 .endm
154
155level_done:
156 add x10, x10, #2 // increment cache number
157 cmp x3, x10
Douglas Raillard9d92e8c2017-03-07 16:36:14 +0000158 b.hi loop1
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100159 msr csselr_el1, xzr // select cache level 0 in csselr
160 dsb sy // barrier to complete final cache operation
161 isb
Achin Gupta4f6ad662013-10-25 09:08:21 +0100162exit:
163 ret
Kévin Petita877c252015-03-24 14:03:57 +0000164endfunc do_dcsw_op
Achin Gupta4f6ad662013-10-25 09:08:21 +0100165
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100166dcsw_loop_table:
167 dcsw_loop isw
168 dcsw_loop cisw
169 dcsw_loop csw
170
Achin Gupta4f6ad662013-10-25 09:08:21 +0100171
Andrew Thoelke38bde412014-03-18 13:46:55 +0000172func dcsw_op_louis
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100173 dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
Kévin Petita877c252015-03-24 14:03:57 +0000174endfunc dcsw_op_louis
Achin Gupta4f6ad662013-10-25 09:08:21 +0100175
176
Andrew Thoelke38bde412014-03-18 13:46:55 +0000177func dcsw_op_all
Andrew Thoelke6a5b3a42014-04-25 10:49:30 +0100178 dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
Kévin Petita877c252015-03-24 14:03:57 +0000179endfunc dcsw_op_all
Soby Mathew42aa5eb2014-09-02 10:47:33 +0100180
181 /* ---------------------------------------------------------------
182 * Helper macro for data cache operations by set/way for the
183 * level specified
184 * ---------------------------------------------------------------
185 */
186 .macro dcsw_op_level level
187 mrs x9, clidr_el1
188 mov x3, \level
189 sub x10, x3, #2
190 b do_dcsw_op
191 .endm
192
193 /* ---------------------------------------------------------------
194 * Data cache operations by set/way for level 1 cache
195 *
196 * The main function, do_dcsw_op requires:
197 * x0: The operation type (0-2), as defined in arch.h
198 * ---------------------------------------------------------------
199 */
200func dcsw_op_level1
201 dcsw_op_level #(1 << LEVEL_SHIFT)
Kévin Petita877c252015-03-24 14:03:57 +0000202endfunc dcsw_op_level1
Soby Mathew42aa5eb2014-09-02 10:47:33 +0100203
204 /* ---------------------------------------------------------------
205 * Data cache operations by set/way for level 2 cache
206 *
207 * The main function, do_dcsw_op requires:
208 * x0: The operation type (0-2), as defined in arch.h
209 * ---------------------------------------------------------------
210 */
211func dcsw_op_level2
212 dcsw_op_level #(2 << LEVEL_SHIFT)
Kévin Petita877c252015-03-24 14:03:57 +0000213endfunc dcsw_op_level2
Soby Mathew42aa5eb2014-09-02 10:47:33 +0100214
215 /* ---------------------------------------------------------------
216 * Data cache operations by set/way for level 3 cache
217 *
218 * The main function, do_dcsw_op requires:
219 * x0: The operation type (0-2), as defined in arch.h
220 * ---------------------------------------------------------------
221 */
222func dcsw_op_level3
223 dcsw_op_level #(3 << LEVEL_SHIFT)
Kévin Petita877c252015-03-24 14:03:57 +0000224endfunc dcsw_op_level3