blob: 8c5ef0bb0b50cc5dc6f18678d738f4d3940a6ed7 [file] [log] [blame]
Dimitris Papastamose08005a2017-10-12 13:02:29 +01001/*
Govindraj Rajaeee28e72023-08-01 15:52:40 -05002 * Copyright (c) 2017-2021, Arm Limited and Contributors. All rights reserved.
Dimitris Papastamose08005a2017-10-12 13:02:29 +01003 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
Dimitris Papastamos525c37a2017-11-13 09:49:45 +00007#include <assert.h>
Chris Kaya5fde282021-05-26 11:58:23 +01008#include <cdefs.h>
Scott Brandene5dcf982020-08-25 13:49:32 -07009#include <inttypes.h>
Antonio Nino Diaz033b4bb2018-10-25 16:52:26 +010010#include <stdbool.h>
Scott Brandene5dcf982020-08-25 13:49:32 -070011#include <stdint.h>
Dimitris Papastamose08005a2017-10-12 13:02:29 +010012
Chris Kay26a79612021-05-24 20:35:26 +010013#include "../amu_private.h"
Antonio Nino Diaze0f90632018-12-14 00:18:21 +000014#include <arch.h>
johpow01fa59c6f2020-10-02 13:41:11 -050015#include <arch_features.h>
Antonio Nino Diaze0f90632018-12-14 00:18:21 +000016#include <arch_helpers.h>
Chris Kayf11909f2021-08-19 11:21:52 +010017#include <common/debug.h>
Antonio Nino Diaze0f90632018-12-14 00:18:21 +000018#include <lib/el3_runtime/pubsub_events.h>
19#include <lib/extensions/amu.h>
Antonio Nino Diaze0f90632018-12-14 00:18:21 +000020
Alexei Fedorov7e6306b2020-07-14 08:17:56 +010021#include <plat/common/platform.h>
Dimitris Papastamoseaf3e6d2017-11-28 13:47:06 +000022
Chris Kayf11909f2021-08-19 11:21:52 +010023#if ENABLE_AMU_FCONF
24# include <lib/fconf/fconf.h>
25# include <lib/fconf/fconf_amu_getter.h>
26#endif
27
Chris Kay03be39d2021-05-05 13:38:30 +010028#if ENABLE_MPMM
29# include <lib/mpmm/mpmm.h>
30#endif
31
Chris Kay26a79612021-05-24 20:35:26 +010032struct amu_ctx {
33 uint64_t group0_cnts[AMU_GROUP0_MAX_COUNTERS];
34#if ENABLE_AMU_AUXILIARY_COUNTERS
35 uint64_t group1_cnts[AMU_GROUP1_MAX_COUNTERS];
36#endif
37
38 /* Architected event counter 1 does not have an offset register */
39 uint64_t group0_voffsets[AMU_GROUP0_MAX_COUNTERS - 1U];
40#if ENABLE_AMU_AUXILIARY_COUNTERS
41 uint64_t group1_voffsets[AMU_GROUP1_MAX_COUNTERS];
42#endif
43
44 uint16_t group0_enable;
45#if ENABLE_AMU_AUXILIARY_COUNTERS
46 uint16_t group1_enable;
47#endif
48};
49
50static struct amu_ctx amu_ctxs_[PLATFORM_CORE_COUNT];
51
52CASSERT((sizeof(amu_ctxs_[0].group0_enable) * CHAR_BIT) <= AMU_GROUP0_MAX_COUNTERS,
53 amu_ctx_group0_enable_cannot_represent_all_group0_counters);
54
55#if ENABLE_AMU_AUXILIARY_COUNTERS
56CASSERT((sizeof(amu_ctxs_[0].group1_enable) * CHAR_BIT) <= AMU_GROUP1_MAX_COUNTERS,
57 amu_ctx_group1_enable_cannot_represent_all_group1_counters);
58#endif
Dimitris Papastamoseaf3e6d2017-11-28 13:47:06 +000059
Chris Kaya5fde282021-05-26 11:58:23 +010060static inline __unused uint64_t read_hcr_el2_amvoffen(void)
61{
62 return (read_hcr_el2() & HCR_AMVOFFEN_BIT) >>
63 HCR_AMVOFFEN_SHIFT;
64}
65
66static inline __unused void write_cptr_el2_tam(uint64_t value)
67{
68 write_cptr_el2((read_cptr_el2() & ~CPTR_EL2_TAM_BIT) |
69 ((value << CPTR_EL2_TAM_SHIFT) & CPTR_EL2_TAM_BIT));
70}
71
John Powellcc799272022-03-29 00:25:59 -050072static inline __unused void ctx_write_cptr_el3_tam(cpu_context_t *ctx, uint64_t tam)
Chris Kaya5fde282021-05-26 11:58:23 +010073{
74 uint64_t value = read_ctx_reg(get_el3state_ctx(ctx), CTX_CPTR_EL3);
75
76 value &= ~TAM_BIT;
77 value |= (tam << TAM_SHIFT) & TAM_BIT;
78
79 write_ctx_reg(get_el3state_ctx(ctx), CTX_CPTR_EL3, value);
80}
81
John Powellcc799272022-03-29 00:25:59 -050082static inline __unused void ctx_write_scr_el3_amvoffen(cpu_context_t *ctx, uint64_t amvoffen)
83{
84 uint64_t value = read_ctx_reg(get_el3state_ctx(ctx), CTX_SCR_EL3);
85
86 value &= ~SCR_AMVOFFEN_BIT;
87 value |= (amvoffen << SCR_AMVOFFEN_SHIFT) & SCR_AMVOFFEN_BIT;
88
89 write_ctx_reg(get_el3state_ctx(ctx), CTX_SCR_EL3, value);
90}
91
Chris Kaya5fde282021-05-26 11:58:23 +010092static inline __unused void write_hcr_el2_amvoffen(uint64_t value)
93{
94 write_hcr_el2((read_hcr_el2() & ~HCR_AMVOFFEN_BIT) |
95 ((value << HCR_AMVOFFEN_SHIFT) & HCR_AMVOFFEN_BIT));
96}
97
98static inline __unused void write_amcr_el0_cg1rz(uint64_t value)
99{
100 write_amcr_el0((read_amcr_el0() & ~AMCR_CG1RZ_BIT) |
101 ((value << AMCR_CG1RZ_SHIFT) & AMCR_CG1RZ_BIT));
102}
103
104static inline __unused uint64_t read_amcfgr_el0_ncg(void)
105{
106 return (read_amcfgr_el0() >> AMCFGR_EL0_NCG_SHIFT) &
107 AMCFGR_EL0_NCG_MASK;
108}
109
Chris Kay26a79612021-05-24 20:35:26 +0100110static inline __unused uint64_t read_amcgcr_el0_cg0nc(void)
Chris Kaya40141d2021-05-25 12:33:18 +0100111{
112 return (read_amcgcr_el0() >> AMCGCR_EL0_CG0NC_SHIFT) &
113 AMCGCR_EL0_CG0NC_MASK;
114}
115
Chris Kaya5fde282021-05-26 11:58:23 +0100116static inline __unused uint64_t read_amcg1idr_el0_voff(void)
Alexei Fedorov7e6306b2020-07-14 08:17:56 +0100117{
Chris Kaya5fde282021-05-26 11:58:23 +0100118 return (read_amcg1idr_el0() >> AMCG1IDR_VOFF_SHIFT) &
119 AMCG1IDR_VOFF_MASK;
120}
121
122static inline __unused uint64_t read_amcgcr_el0_cg1nc(void)
123{
124 return (read_amcgcr_el0() >> AMCGCR_EL0_CG1NC_SHIFT) &
125 AMCGCR_EL0_CG1NC_MASK;
126}
Dimitris Papastamose08005a2017-10-12 13:02:29 +0100127
Chris Kaya5fde282021-05-26 11:58:23 +0100128static inline __unused uint64_t read_amcntenset0_el0_px(void)
129{
130 return (read_amcntenset0_el0() >> AMCNTENSET0_EL0_Pn_SHIFT) &
131 AMCNTENSET0_EL0_Pn_MASK;
132}
133
134static inline __unused uint64_t read_amcntenset1_el0_px(void)
135{
136 return (read_amcntenset1_el0() >> AMCNTENSET1_EL0_Pn_SHIFT) &
137 AMCNTENSET1_EL0_Pn_MASK;
138}
139
140static inline __unused void write_amcntenset0_el0_px(uint64_t px)
141{
142 uint64_t value = read_amcntenset0_el0();
143
144 value &= ~AMCNTENSET0_EL0_Pn_MASK;
145 value |= (px << AMCNTENSET0_EL0_Pn_SHIFT) & AMCNTENSET0_EL0_Pn_MASK;
146
147 write_amcntenset0_el0(value);
148}
149
150static inline __unused void write_amcntenset1_el0_px(uint64_t px)
151{
152 uint64_t value = read_amcntenset1_el0();
153
154 value &= ~AMCNTENSET1_EL0_Pn_MASK;
155 value |= (px << AMCNTENSET1_EL0_Pn_SHIFT) & AMCNTENSET1_EL0_Pn_MASK;
156
157 write_amcntenset1_el0(value);
158}
159
160static inline __unused void write_amcntenclr0_el0_px(uint64_t px)
161{
162 uint64_t value = read_amcntenclr0_el0();
163
164 value &= ~AMCNTENCLR0_EL0_Pn_MASK;
165 value |= (px << AMCNTENCLR0_EL0_Pn_SHIFT) & AMCNTENCLR0_EL0_Pn_MASK;
166
167 write_amcntenclr0_el0(value);
168}
169
170static inline __unused void write_amcntenclr1_el0_px(uint64_t px)
171{
172 uint64_t value = read_amcntenclr1_el0();
173
174 value &= ~AMCNTENCLR1_EL0_Pn_MASK;
175 value |= (px << AMCNTENCLR1_EL0_Pn_SHIFT) & AMCNTENCLR1_EL0_Pn_MASK;
176
177 write_amcntenclr1_el0(value);
178}
179
Chris Kaya5fde282021-05-26 11:58:23 +0100180#if ENABLE_AMU_AUXILIARY_COUNTERS
Chris Kay26a79612021-05-24 20:35:26 +0100181static __unused bool amu_group1_supported(void)
Chris Kaya5fde282021-05-26 11:58:23 +0100182{
183 return read_amcfgr_el0_ncg() > 0U;
Dimitris Papastamos525c37a2017-11-13 09:49:45 +0000184}
Alexei Fedorov7e6306b2020-07-14 08:17:56 +0100185#endif
Dimitris Papastamos525c37a2017-11-13 09:49:45 +0000186
187/*
Chris Kay26a79612021-05-24 20:35:26 +0100188 * Enable counters. This function is meant to be invoked by the context
189 * management library before exiting from EL3.
Dimitris Papastamos525c37a2017-11-13 09:49:45 +0000190 */
Boyan Karatotev1e966f32023-03-27 17:02:43 +0100191void amu_enable(cpu_context_t *ctx)
Dimitris Papastamos525c37a2017-11-13 09:49:45 +0000192{
Dimitris Papastamos525c37a2017-11-13 09:49:45 +0000193 /*
Boyan Karatotev1e966f32023-03-27 17:02:43 +0100194 * Set CPTR_EL3.TAM to zero so that any accesses to the Activity Monitor
195 * registers do not trap to EL3.
Dimitris Papastamos525c37a2017-11-13 09:49:45 +0000196 */
John Powellcc799272022-03-29 00:25:59 -0500197 ctx_write_cptr_el3_tam(ctx, 0U);
Dimitris Papastamos525c37a2017-11-13 09:49:45 +0000198
Boyan Karatotev1e966f32023-03-27 17:02:43 +0100199 /* Initialize FEAT_AMUv1p1 features if present. */
200 if (is_feat_amuv1p1_supported()) {
201 /*
202 * Set SCR_EL3.AMVOFFEN to one so that accesses to virtual
203 * offset registers at EL2 do not trap to EL3
204 */
205 ctx_write_scr_el3_amvoffen(ctx, 1U);
206 }
207}
Alexei Fedorov7e6306b2020-07-14 08:17:56 +0100208
Boyan Karatotev1e966f32023-03-27 17:02:43 +0100209void amu_init_el3(void)
210{
211 uint64_t group0_impl_ctr = read_amcgcr_el0_cg0nc();
212 uint64_t group0_en_mask = (1 << (group0_impl_ctr)) - 1U;
213 uint64_t num_ctr_groups = read_amcfgr_el0_ncg();
Chris Kay26a79612021-05-24 20:35:26 +0100214
Boyan Karatotev1e966f32023-03-27 17:02:43 +0100215 /* Enable all architected counters by default */
216 write_amcntenset0_el0_px(group0_en_mask);
Chris Kayf11909f2021-08-19 11:21:52 +0100217
218#if ENABLE_AMU_AUXILIARY_COUNTERS
Boyan Karatotev1e966f32023-03-27 17:02:43 +0100219 if (num_ctr_groups > 0U) {
220 uint64_t amcntenset1_el0_px = 0x0; /* Group 1 enable mask */
221 const struct amu_topology *topology;
Chris Kayf11909f2021-08-19 11:21:52 +0100222
Boyan Karatotev1e966f32023-03-27 17:02:43 +0100223 /*
224 * The platform may opt to enable specific auxiliary counters.
225 * This can be done via the common FCONF getter, or via the
226 * platform-implemented function.
227 */
Chris Kayf11909f2021-08-19 11:21:52 +0100228#if ENABLE_AMU_FCONF
Boyan Karatotev1e966f32023-03-27 17:02:43 +0100229 topology = FCONF_GET_PROPERTY(amu, config, topology);
Chris Kayf11909f2021-08-19 11:21:52 +0100230#else
Boyan Karatotev1e966f32023-03-27 17:02:43 +0100231 topology = plat_amu_topology();
Chris Kayf11909f2021-08-19 11:21:52 +0100232#endif /* ENABLE_AMU_FCONF */
233
Boyan Karatotev1e966f32023-03-27 17:02:43 +0100234 if (topology != NULL) {
235 unsigned int core_pos = plat_my_core_pos();
Chris Kayf11909f2021-08-19 11:21:52 +0100236
Boyan Karatotev1e966f32023-03-27 17:02:43 +0100237 amcntenset1_el0_px = topology->cores[core_pos].enable;
238 } else {
239 ERROR("AMU: failed to generate AMU topology\n");
240 }
Chris Kay26a79612021-05-24 20:35:26 +0100241
Chris Kay26a79612021-05-24 20:35:26 +0100242 write_amcntenset1_el0_px(amcntenset1_el0_px);
Boyan Karatotev1e966f32023-03-27 17:02:43 +0100243 }
244#else /* ENABLE_AMU_AUXILIARY_COUNTERS */
245 if (num_ctr_groups > 0U) {
Chris Kayf11909f2021-08-19 11:21:52 +0100246 VERBOSE("AMU: auxiliary counters detected but support is disabled\n");
Chris Kay925fda42021-05-25 10:42:56 +0100247 }
Boyan Karatotev1e966f32023-03-27 17:02:43 +0100248#endif /* ENABLE_AMU_AUXILIARY_COUNTERS */
johpow01fa59c6f2020-10-02 13:41:11 -0500249
Andre Przywara906776e2023-03-03 10:30:06 +0000250 if (is_feat_amuv1p1_supported()) {
johpow01fa59c6f2020-10-02 13:41:11 -0500251#if AMU_RESTRICT_COUNTERS
Chris Kay03be39d2021-05-05 13:38:30 +0100252 /*
253 * FEAT_AMUv1p1 adds a register field to restrict access to
254 * group 1 counters at all but the highest implemented EL. This
255 * is controlled with the `AMU_RESTRICT_COUNTERS` compile time
256 * flag, when set, system register reads at lower ELs return
257 * zero. Reads from the memory mapped view are unaffected.
258 */
259 VERBOSE("AMU group 1 counter access restricted.\n");
260 write_amcr_el0_cg1rz(1U);
johpow01fa59c6f2020-10-02 13:41:11 -0500261#else
Chris Kay03be39d2021-05-05 13:38:30 +0100262 write_amcr_el0_cg1rz(0U);
263#endif
264 }
265
266#if ENABLE_MPMM
267 mpmm_enable();
johpow01fa59c6f2020-10-02 13:41:11 -0500268#endif
Dimitris Papastamos525c37a2017-11-13 09:49:45 +0000269}
270
Boyan Karatotev1e966f32023-03-27 17:02:43 +0100271void amu_init_el2_unused(void)
272{
273 /*
274 * CPTR_EL2.TAM: Set to zero so any accesses to the Activity Monitor
275 * registers do not trap to EL2.
276 */
277 write_cptr_el2_tam(0U);
278
279 /* Initialize FEAT_AMUv1p1 features if present. */
280 if (is_feat_amuv1p1_supported()) {
281 /* Make sure virtual offsets are disabled if EL2 not used. */
282 write_hcr_el2_amvoffen(0U);
283 }
284}
285
Dimitris Papastamos525c37a2017-11-13 09:49:45 +0000286/* Read the group 0 counter identified by the given `idx`. */
Chris Kayf13c6b52021-05-24 21:00:07 +0100287static uint64_t amu_group0_cnt_read(unsigned int idx)
Dimitris Papastamos525c37a2017-11-13 09:49:45 +0000288{
Andre Przywara906776e2023-03-03 10:30:06 +0000289 assert(is_feat_amu_supported());
Chris Kaya40141d2021-05-25 12:33:18 +0100290 assert(idx < read_amcgcr_el0_cg0nc());
Dimitris Papastamos525c37a2017-11-13 09:49:45 +0000291
292 return amu_group0_cnt_read_internal(idx);
293}
294
Alexei Fedorov7e6306b2020-07-14 08:17:56 +0100295/* Write the group 0 counter identified by the given `idx` with `val` */
Chris Kayf13c6b52021-05-24 21:00:07 +0100296static void amu_group0_cnt_write(unsigned int idx, uint64_t val)
Dimitris Papastamos525c37a2017-11-13 09:49:45 +0000297{
Andre Przywara906776e2023-03-03 10:30:06 +0000298 assert(is_feat_amu_supported());
Chris Kaya40141d2021-05-25 12:33:18 +0100299 assert(idx < read_amcgcr_el0_cg0nc());
Dimitris Papastamos525c37a2017-11-13 09:49:45 +0000300
301 amu_group0_cnt_write_internal(idx, val);
302 isb();
303}
304
johpow01fa59c6f2020-10-02 13:41:11 -0500305/*
Chris Kay26a79612021-05-24 20:35:26 +0100306 * Unlike with auxiliary counters, we cannot detect at runtime whether an
307 * architected counter supports a virtual offset. These are instead fixed
308 * according to FEAT_AMUv1p1, but this switch will need to be updated if later
309 * revisions of FEAT_AMU add additional architected counters.
310 */
311static bool amu_group0_voffset_supported(uint64_t idx)
312{
313 switch (idx) {
314 case 0U:
315 case 2U:
316 case 3U:
317 return true;
318
319 case 1U:
320 return false;
321
322 default:
323 ERROR("AMU: can't set up virtual offset for unknown "
Scott Brandene5dcf982020-08-25 13:49:32 -0700324 "architected counter %" PRIu64 "!\n", idx);
Chris Kay26a79612021-05-24 20:35:26 +0100325
326 panic();
327 }
328}
329
330/*
johpow01fa59c6f2020-10-02 13:41:11 -0500331 * Read the group 0 offset register for a given index. Index must be 0, 2,
332 * or 3, the register for 1 does not exist.
333 *
334 * Using this function requires FEAT_AMUv1p1 support.
335 */
Chris Kayf13c6b52021-05-24 21:00:07 +0100336static uint64_t amu_group0_voffset_read(unsigned int idx)
johpow01fa59c6f2020-10-02 13:41:11 -0500337{
Andre Przywara906776e2023-03-03 10:30:06 +0000338 assert(is_feat_amuv1p1_supported());
Chris Kaya40141d2021-05-25 12:33:18 +0100339 assert(idx < read_amcgcr_el0_cg0nc());
johpow01fa59c6f2020-10-02 13:41:11 -0500340 assert(idx != 1U);
341
342 return amu_group0_voffset_read_internal(idx);
343}
344
345/*
346 * Write the group 0 offset register for a given index. Index must be 0, 2, or
347 * 3, the register for 1 does not exist.
348 *
349 * Using this function requires FEAT_AMUv1p1 support.
350 */
Chris Kayf13c6b52021-05-24 21:00:07 +0100351static void amu_group0_voffset_write(unsigned int idx, uint64_t val)
johpow01fa59c6f2020-10-02 13:41:11 -0500352{
Andre Przywara906776e2023-03-03 10:30:06 +0000353 assert(is_feat_amuv1p1_supported());
Chris Kaya40141d2021-05-25 12:33:18 +0100354 assert(idx < read_amcgcr_el0_cg0nc());
johpow01fa59c6f2020-10-02 13:41:11 -0500355 assert(idx != 1U);
356
357 amu_group0_voffset_write_internal(idx, val);
358 isb();
359}
360
Chris Kay925fda42021-05-25 10:42:56 +0100361#if ENABLE_AMU_AUXILIARY_COUNTERS
Alexei Fedorov7e6306b2020-07-14 08:17:56 +0100362/* Read the group 1 counter identified by the given `idx` */
Chris Kayf13c6b52021-05-24 21:00:07 +0100363static uint64_t amu_group1_cnt_read(unsigned int idx)
Dimitris Papastamos525c37a2017-11-13 09:49:45 +0000364{
Andre Przywara906776e2023-03-03 10:30:06 +0000365 assert(is_feat_amu_supported());
Alexei Fedorov7e6306b2020-07-14 08:17:56 +0100366 assert(amu_group1_supported());
Chris Kayda819142021-05-25 15:24:18 +0100367 assert(idx < read_amcgcr_el0_cg1nc());
Dimitris Papastamos525c37a2017-11-13 09:49:45 +0000368
369 return amu_group1_cnt_read_internal(idx);
370}
371
Alexei Fedorov7e6306b2020-07-14 08:17:56 +0100372/* Write the group 1 counter identified by the given `idx` with `val` */
Chris Kayf13c6b52021-05-24 21:00:07 +0100373static void amu_group1_cnt_write(unsigned int idx, uint64_t val)
Dimitris Papastamos525c37a2017-11-13 09:49:45 +0000374{
Andre Przywara906776e2023-03-03 10:30:06 +0000375 assert(is_feat_amu_supported());
Alexei Fedorov7e6306b2020-07-14 08:17:56 +0100376 assert(amu_group1_supported());
Chris Kayda819142021-05-25 15:24:18 +0100377 assert(idx < read_amcgcr_el0_cg1nc());
Dimitris Papastamos525c37a2017-11-13 09:49:45 +0000378
379 amu_group1_cnt_write_internal(idx, val);
380 isb();
381}
382
383/*
johpow01fa59c6f2020-10-02 13:41:11 -0500384 * Read the group 1 offset register for a given index.
385 *
386 * Using this function requires FEAT_AMUv1p1 support.
387 */
Chris Kayf13c6b52021-05-24 21:00:07 +0100388static uint64_t amu_group1_voffset_read(unsigned int idx)
johpow01fa59c6f2020-10-02 13:41:11 -0500389{
Andre Przywara906776e2023-03-03 10:30:06 +0000390 assert(is_feat_amuv1p1_supported());
johpow01fa59c6f2020-10-02 13:41:11 -0500391 assert(amu_group1_supported());
Chris Kayda819142021-05-25 15:24:18 +0100392 assert(idx < read_amcgcr_el0_cg1nc());
Chris Kaya5fde282021-05-26 11:58:23 +0100393 assert((read_amcg1idr_el0_voff() & (UINT64_C(1) << idx)) != 0U);
johpow01fa59c6f2020-10-02 13:41:11 -0500394
395 return amu_group1_voffset_read_internal(idx);
396}
397
398/*
399 * Write the group 1 offset register for a given index.
400 *
401 * Using this function requires FEAT_AMUv1p1 support.
402 */
Chris Kayf13c6b52021-05-24 21:00:07 +0100403static void amu_group1_voffset_write(unsigned int idx, uint64_t val)
johpow01fa59c6f2020-10-02 13:41:11 -0500404{
Andre Przywara906776e2023-03-03 10:30:06 +0000405 assert(is_feat_amuv1p1_supported());
johpow01fa59c6f2020-10-02 13:41:11 -0500406 assert(amu_group1_supported());
Chris Kayda819142021-05-25 15:24:18 +0100407 assert(idx < read_amcgcr_el0_cg1nc());
Chris Kaya5fde282021-05-26 11:58:23 +0100408 assert((read_amcg1idr_el0_voff() & (UINT64_C(1) << idx)) != 0U);
johpow01fa59c6f2020-10-02 13:41:11 -0500409
410 amu_group1_voffset_write_internal(idx, val);
411 isb();
412}
Chris Kay925fda42021-05-25 10:42:56 +0100413#endif
Dimitris Papastamoseaf3e6d2017-11-28 13:47:06 +0000414
415static void *amu_context_save(const void *arg)
416{
Chris Kay26a79612021-05-24 20:35:26 +0100417 uint64_t i, j;
Dimitris Papastamoseaf3e6d2017-11-28 13:47:06 +0000418
Chris Kay26a79612021-05-24 20:35:26 +0100419 unsigned int core_pos;
420 struct amu_ctx *ctx;
Dimitris Papastamoseaf3e6d2017-11-28 13:47:06 +0000421
Andre Przywara906776e2023-03-03 10:30:06 +0000422 uint64_t hcr_el2_amvoffen = 0; /* AMU virtual offsets enabled */
Chris Kay26a79612021-05-24 20:35:26 +0100423 uint64_t amcgcr_el0_cg0nc; /* Number of group 0 counters */
Dimitris Papastamoseaf3e6d2017-11-28 13:47:06 +0000424
Chris Kay925fda42021-05-25 10:42:56 +0100425#if ENABLE_AMU_AUXILIARY_COUNTERS
Chris Kay26a79612021-05-24 20:35:26 +0100426 uint64_t amcg1idr_el0_voff; /* Auxiliary counters with virtual offsets */
427 uint64_t amcfgr_el0_ncg; /* Number of counter groups */
428 uint64_t amcgcr_el0_cg1nc; /* Number of group 1 counters */
429#endif
430
Andre Przywara906776e2023-03-03 10:30:06 +0000431 if (!is_feat_amu_supported()) {
Chris Kay26a79612021-05-24 20:35:26 +0100432 return (void *)0;
Chris Kay925fda42021-05-25 10:42:56 +0100433 }
Chris Kay26a79612021-05-24 20:35:26 +0100434
435 core_pos = plat_my_core_pos();
436 ctx = &amu_ctxs_[core_pos];
437
438 amcgcr_el0_cg0nc = read_amcgcr_el0_cg0nc();
Andre Przywara906776e2023-03-03 10:30:06 +0000439 if (is_feat_amuv1p1_supported()) {
440 hcr_el2_amvoffen = read_hcr_el2_amvoffen();
441 }
Chris Kay26a79612021-05-24 20:35:26 +0100442
443#if ENABLE_AMU_AUXILIARY_COUNTERS
444 amcfgr_el0_ncg = read_amcfgr_el0_ncg();
445 amcgcr_el0_cg1nc = (amcfgr_el0_ncg > 0U) ? read_amcgcr_el0_cg1nc() : 0U;
446 amcg1idr_el0_voff = (hcr_el2_amvoffen != 0U) ? read_amcg1idr_el0_voff() : 0U;
Alexei Fedorov7e6306b2020-07-14 08:17:56 +0100447#endif
Chris Kay925fda42021-05-25 10:42:56 +0100448
Dimitris Papastamoseaf3e6d2017-11-28 13:47:06 +0000449 /*
Chris Kay26a79612021-05-24 20:35:26 +0100450 * Disable all AMU counters.
Dimitris Papastamoseaf3e6d2017-11-28 13:47:06 +0000451 */
Chris Kay26a79612021-05-24 20:35:26 +0100452
453 ctx->group0_enable = read_amcntenset0_el0_px();
454 write_amcntenclr0_el0_px(ctx->group0_enable);
Alexei Fedorov7e6306b2020-07-14 08:17:56 +0100455
Chris Kay925fda42021-05-25 10:42:56 +0100456#if ENABLE_AMU_AUXILIARY_COUNTERS
Chris Kay26a79612021-05-24 20:35:26 +0100457 if (amcfgr_el0_ncg > 0U) {
458 ctx->group1_enable = read_amcntenset1_el0_px();
459 write_amcntenclr1_el0_px(ctx->group1_enable);
Chris Kay925fda42021-05-25 10:42:56 +0100460 }
Alexei Fedorov7e6306b2020-07-14 08:17:56 +0100461#endif
Chris Kay925fda42021-05-25 10:42:56 +0100462
Chris Kay26a79612021-05-24 20:35:26 +0100463 /*
464 * Save the counters to the local context.
465 */
Dimitris Papastamoseaf3e6d2017-11-28 13:47:06 +0000466
Chris Kay26a79612021-05-24 20:35:26 +0100467 isb(); /* Ensure counters have been stopped */
468
469 for (i = 0U; i < amcgcr_el0_cg0nc; i++) {
Dimitris Papastamoseaf3e6d2017-11-28 13:47:06 +0000470 ctx->group0_cnts[i] = amu_group0_cnt_read(i);
Alexei Fedorov7e6306b2020-07-14 08:17:56 +0100471 }
Dimitris Papastamoseaf3e6d2017-11-28 13:47:06 +0000472
Chris Kay26a79612021-05-24 20:35:26 +0100473#if ENABLE_AMU_AUXILIARY_COUNTERS
474 for (i = 0U; i < amcgcr_el0_cg1nc; i++) {
475 ctx->group1_cnts[i] = amu_group1_cnt_read(i);
johpow01fa59c6f2020-10-02 13:41:11 -0500476 }
Chris Kay26a79612021-05-24 20:35:26 +0100477#endif
johpow01fa59c6f2020-10-02 13:41:11 -0500478
Chris Kay26a79612021-05-24 20:35:26 +0100479 /*
480 * Save virtual offsets for counters that offer them.
481 */
482
483 if (hcr_el2_amvoffen != 0U) {
484 for (i = 0U, j = 0U; i < amcgcr_el0_cg0nc; i++) {
485 if (!amu_group0_voffset_supported(i)) {
486 continue; /* No virtual offset */
Chris Kay925fda42021-05-25 10:42:56 +0100487 }
johpow01fa59c6f2020-10-02 13:41:11 -0500488
Chris Kay26a79612021-05-24 20:35:26 +0100489 ctx->group0_voffsets[j++] = amu_group0_voffset_read(i);
490 }
johpow01fa59c6f2020-10-02 13:41:11 -0500491
Chris Kay26a79612021-05-24 20:35:26 +0100492#if ENABLE_AMU_AUXILIARY_COUNTERS
493 for (i = 0U, j = 0U; i < amcgcr_el0_cg1nc; i++) {
494 if ((amcg1idr_el0_voff >> i) & 1U) {
495 continue; /* No virtual offset */
johpow01fa59c6f2020-10-02 13:41:11 -0500496 }
Chris Kay26a79612021-05-24 20:35:26 +0100497
498 ctx->group1_voffsets[j++] = amu_group1_voffset_read(i);
johpow01fa59c6f2020-10-02 13:41:11 -0500499 }
Alexei Fedorov7e6306b2020-07-14 08:17:56 +0100500#endif
Chris Kay26a79612021-05-24 20:35:26 +0100501 }
Chris Kay925fda42021-05-25 10:42:56 +0100502
Antonio Nino Diaz033b4bb2018-10-25 16:52:26 +0100503 return (void *)0;
Dimitris Papastamoseaf3e6d2017-11-28 13:47:06 +0000504}
505
506static void *amu_context_restore(const void *arg)
507{
Chris Kay26a79612021-05-24 20:35:26 +0100508 uint64_t i, j;
Dimitris Papastamoseaf3e6d2017-11-28 13:47:06 +0000509
Chris Kay26a79612021-05-24 20:35:26 +0100510 unsigned int core_pos;
511 struct amu_ctx *ctx;
512
Andre Przywara906776e2023-03-03 10:30:06 +0000513 uint64_t hcr_el2_amvoffen = 0; /* AMU virtual offsets enabled */
Chris Kay26a79612021-05-24 20:35:26 +0100514
Chris Kay26a79612021-05-24 20:35:26 +0100515 uint64_t amcgcr_el0_cg0nc; /* Number of group 0 counters */
516
517#if ENABLE_AMU_AUXILIARY_COUNTERS
Boyan Karatotev1e966f32023-03-27 17:02:43 +0100518 uint64_t amcfgr_el0_ncg; /* Number of counter groups */
Chris Kay26a79612021-05-24 20:35:26 +0100519 uint64_t amcgcr_el0_cg1nc; /* Number of group 1 counters */
520 uint64_t amcg1idr_el0_voff; /* Auxiliary counters with virtual offsets */
521#endif
522
Andre Przywara906776e2023-03-03 10:30:06 +0000523 if (!is_feat_amu_supported()) {
Chris Kay26a79612021-05-24 20:35:26 +0100524 return (void *)0;
Alexei Fedorov7e6306b2020-07-14 08:17:56 +0100525 }
Dimitris Papastamoseaf3e6d2017-11-28 13:47:06 +0000526
Chris Kay26a79612021-05-24 20:35:26 +0100527 core_pos = plat_my_core_pos();
528 ctx = &amu_ctxs_[core_pos];
529
Chris Kay26a79612021-05-24 20:35:26 +0100530 amcgcr_el0_cg0nc = read_amcgcr_el0_cg0nc();
531
Andre Przywara906776e2023-03-03 10:30:06 +0000532 if (is_feat_amuv1p1_supported()) {
533 hcr_el2_amvoffen = read_hcr_el2_amvoffen();
534 }
Dimitris Papastamoseaf3e6d2017-11-28 13:47:06 +0000535
Chris Kay925fda42021-05-25 10:42:56 +0100536#if ENABLE_AMU_AUXILIARY_COUNTERS
Boyan Karatotev1e966f32023-03-27 17:02:43 +0100537 amcfgr_el0_ncg = read_amcfgr_el0_ncg();
Chris Kay26a79612021-05-24 20:35:26 +0100538 amcgcr_el0_cg1nc = (amcfgr_el0_ncg > 0U) ? read_amcgcr_el0_cg1nc() : 0U;
539 amcg1idr_el0_voff = (hcr_el2_amvoffen != 0U) ? read_amcg1idr_el0_voff() : 0U;
540#endif
541
542 /*
Chris Kay26a79612021-05-24 20:35:26 +0100543 * Restore the counter values from the local context.
544 */
545
546 for (i = 0U; i < amcgcr_el0_cg0nc; i++) {
Alexei Fedorov7e6306b2020-07-14 08:17:56 +0100547 amu_group0_cnt_write(i, ctx->group0_cnts[i]);
548 }
Dimitris Papastamoseaf3e6d2017-11-28 13:47:06 +0000549
Chris Kay26a79612021-05-24 20:35:26 +0100550#if ENABLE_AMU_AUXILIARY_COUNTERS
551 for (i = 0U; i < amcgcr_el0_cg1nc; i++) {
552 amu_group1_cnt_write(i, ctx->group1_cnts[i]);
johpow01fa59c6f2020-10-02 13:41:11 -0500553 }
Chris Kay26a79612021-05-24 20:35:26 +0100554#endif
johpow01fa59c6f2020-10-02 13:41:11 -0500555
Chris Kay26a79612021-05-24 20:35:26 +0100556 /*
557 * Restore virtual offsets for counters that offer them.
558 */
Alexei Fedorov7e6306b2020-07-14 08:17:56 +0100559
Chris Kay26a79612021-05-24 20:35:26 +0100560 if (hcr_el2_amvoffen != 0U) {
561 for (i = 0U, j = 0U; i < amcgcr_el0_cg0nc; i++) {
562 if (!amu_group0_voffset_supported(i)) {
563 continue; /* No virtual offset */
Chris Kay925fda42021-05-25 10:42:56 +0100564 }
Dimitris Papastamoseaf3e6d2017-11-28 13:47:06 +0000565
Chris Kay26a79612021-05-24 20:35:26 +0100566 amu_group0_voffset_write(i, ctx->group0_voffsets[j++]);
567 }
johpow01fa59c6f2020-10-02 13:41:11 -0500568
Chris Kay26a79612021-05-24 20:35:26 +0100569#if ENABLE_AMU_AUXILIARY_COUNTERS
570 for (i = 0U, j = 0U; i < amcgcr_el0_cg1nc; i++) {
571 if ((amcg1idr_el0_voff >> i) & 1U) {
572 continue; /* No virtual offset */
johpow01fa59c6f2020-10-02 13:41:11 -0500573 }
Chris Kay26a79612021-05-24 20:35:26 +0100574
575 amu_group1_voffset_write(i, ctx->group1_voffsets[j++]);
johpow01fa59c6f2020-10-02 13:41:11 -0500576 }
Chris Kay26a79612021-05-24 20:35:26 +0100577#endif
578 }
579
580 /*
581 * Re-enable counters that were disabled during context save.
582 */
583
584 write_amcntenset0_el0_px(ctx->group0_enable);
johpow01fa59c6f2020-10-02 13:41:11 -0500585
Chris Kay26a79612021-05-24 20:35:26 +0100586#if ENABLE_AMU_AUXILIARY_COUNTERS
587 if (amcfgr_el0_ncg > 0) {
588 write_amcntenset1_el0_px(ctx->group1_enable);
Chris Kay925fda42021-05-25 10:42:56 +0100589 }
Alexei Fedorov7e6306b2020-07-14 08:17:56 +0100590#endif
Dimitris Papastamoseaf3e6d2017-11-28 13:47:06 +0000591
Chris Kay03be39d2021-05-05 13:38:30 +0100592#if ENABLE_MPMM
593 mpmm_enable();
594#endif
595
Antonio Nino Diaz033b4bb2018-10-25 16:52:26 +0100596 return (void *)0;
Dimitris Papastamoseaf3e6d2017-11-28 13:47:06 +0000597}
598
599SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_start, amu_context_save);
600SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_finish, amu_context_restore);