blob: d028fcedec10950efe61b30b8cde51ed113496c5 [file] [log] [blame]
johpow019d134022021-06-16 17:57:28 -05001/*
AlexeiFedoroveb6f6cd2024-03-13 13:59:09 +00002 * Copyright (c) 2022-2024, Arm Limited. All rights reserved.
johpow019d134022021-06-16 17:57:28 -05003 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7#include <assert.h>
8#include <errno.h>
Manish Pandey9174a752021-11-09 20:49:56 +00009#include <inttypes.h>
johpow019d134022021-06-16 17:57:28 -050010#include <limits.h>
11#include <stdint.h>
12
13#include <arch.h>
Olivier Deprezc80d0de2024-01-17 15:12:04 +010014#include <arch_features.h>
johpow019d134022021-06-16 17:57:28 -050015#include <arch_helpers.h>
16#include <common/debug.h>
17#include "gpt_rme_private.h"
18#include <lib/gpt_rme/gpt_rme.h>
19#include <lib/smccc.h>
20#include <lib/spinlock.h>
21#include <lib/xlat_tables/xlat_tables_v2.h>
22
23#if !ENABLE_RME
AlexeiFedorov7eaaac72024-03-13 15:18:02 +000024#error "ENABLE_RME must be enabled to use the GPT library"
johpow019d134022021-06-16 17:57:28 -050025#endif
26
27/*
28 * Lookup T from PPS
29 *
30 * PPS Size T
31 * 0b000 4GB 32
32 * 0b001 64GB 36
33 * 0b010 1TB 40
34 * 0b011 4TB 42
35 * 0b100 16TB 44
36 * 0b101 256TB 48
37 * 0b110 4PB 52
38 *
39 * See section 15.1.27 of the RME specification.
40 */
41static const gpt_t_val_e gpt_t_lookup[] = {PPS_4GB_T, PPS_64GB_T,
42 PPS_1TB_T, PPS_4TB_T,
43 PPS_16TB_T, PPS_256TB_T,
44 PPS_4PB_T};
45
46/*
47 * Lookup P from PGS
48 *
49 * PGS Size P
50 * 0b00 4KB 12
51 * 0b10 16KB 14
52 * 0b01 64KB 16
53 *
54 * Note that pgs=0b10 is 16KB and pgs=0b01 is 64KB, this is not a typo.
55 *
56 * See section 15.1.27 of the RME specification.
57 */
58static const gpt_p_val_e gpt_p_lookup[] = {PGS_4KB_P, PGS_64KB_P, PGS_16KB_P};
59
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +000060static void shatter_2mb(uintptr_t base, const gpi_info_t *gpi_info,
61 uint64_t l1_desc);
62static void shatter_32mb(uintptr_t base, const gpi_info_t *gpi_info,
63 uint64_t l1_desc);
64static void shatter_512mb(uintptr_t base, const gpi_info_t *gpi_info,
65 uint64_t l1_desc);
66
johpow019d134022021-06-16 17:57:28 -050067/*
AlexeiFedorov7eaaac72024-03-13 15:18:02 +000068 * This structure contains GPT configuration data
johpow019d134022021-06-16 17:57:28 -050069 */
70typedef struct {
71 uintptr_t plat_gpt_l0_base;
72 gpccr_pps_e pps;
73 gpt_t_val_e t;
74 gpccr_pgs_e pgs;
75 gpt_p_val_e p;
76} gpt_config_t;
77
78static gpt_config_t gpt_config;
79
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +000080/*
81 * Number of L1 entries in 2MB, depending on GPCCR_EL3.PGS:
82 * +-------+------------+
83 * | PGS | L1 entries |
84 * +-------+------------+
85 * | 4KB | 32 |
86 * +-------+------------+
87 * | 16KB | 8 |
88 * +-------+------------+
89 * | 64KB | 2 |
90 * +-------+------------+
91 */
92static unsigned int gpt_l1_cnt_2mb;
93
94/*
95 * Mask for the L1 index field, depending on
96 * GPCCR_EL3.L0GPTSZ and GPCCR_EL3.PGS:
97 * +---------+-------------------------------+
98 * | | PGS |
99 * +---------+----------+----------+---------+
100 * | L0GPTSZ | 4KB | 16KB | 64KB |
101 * +---------+----------+----------+---------+
102 * | 1GB | 0x3FFF | 0xFFF | 0x3FF |
103 * +---------+----------+----------+---------+
104 * | 16GB | 0x3FFFF | 0xFFFF | 0x3FFF |
105 * +---------+----------+----------+---------+
106 * | 64GB | 0xFFFFF | 0x3FFFF | 0xFFFF |
107 * +---------+----------+----------+---------+
108 * | 512GB | 0x7FFFFF | 0x1FFFFF | 0x7FFFF |
109 * +---------+----------+----------+---------+
110 */
111static uint64_t gpt_l1_index_mask;
112
113/* Number of 128-bit L1 entries in 2MB, 32MB and 512MB */
114#define L1_QWORDS_2MB (gpt_l1_cnt_2mb / 2U)
115#define L1_QWORDS_32MB (L1_QWORDS_2MB * 16U)
116#define L1_QWORDS_512MB (L1_QWORDS_32MB * 16U)
117
118/* Size in bytes of L1 entries in 2MB, 32MB */
119#define L1_BYTES_2MB (gpt_l1_cnt_2mb * sizeof(uint64_t))
120#define L1_BYTES_32MB (L1_BYTES_2MB * 16U)
121
122/* Get the index into the L1 table from a physical address */
123#define GPT_L1_INDEX(_pa) \
124 (((_pa) >> (unsigned int)GPT_L1_IDX_SHIFT(gpt_config.p)) & gpt_l1_index_mask)
125
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000126/* These variables are used during initialization of the L1 tables */
johpow019d134022021-06-16 17:57:28 -0500127static uintptr_t gpt_l1_tbl;
128
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000129/* These variable is used during runtime */
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +0100130#if (RME_GPT_BITLOCK_BLOCK == 0)
131/*
132 * The GPTs are protected by a global spinlock to ensure
133 * that multiple CPUs do not attempt to change the descriptors at once.
134 */
135static spinlock_t gpt_lock;
136#else
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000137
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +0100138/* Bitlocks base address */
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000139static bitlock_t *gpt_bitlock_base;
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +0100140#endif
141
142/* Lock/unlock macros for GPT entries */
143#if (RME_GPT_BITLOCK_BLOCK == 0)
144/*
145 * Access to GPT is controlled by a global lock to ensure
146 * that no more than one CPU is allowed to make changes at any
147 * given time.
148 */
149#define GPT_LOCK spin_lock(&gpt_lock)
150#define GPT_UNLOCK spin_unlock(&gpt_lock)
151#else
152/*
153 * Access to a block of memory is controlled by a bitlock.
154 * Size of block = RME_GPT_BITLOCK_BLOCK * 512MB.
155 */
156#define GPT_LOCK bit_lock(gpi_info.lock, gpi_info.mask)
157#define GPT_UNLOCK bit_unlock(gpi_info.lock, gpi_info.mask)
158#endif
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000159
160static void tlbi_page_dsbosh(uintptr_t base)
161{
162 /* Look-up table for invalidation TLBs for 4KB, 16KB and 64KB pages */
163 static const gpt_tlbi_lookup_t tlbi_page_lookup[] = {
164 { tlbirpalos_4k, ~(SZ_4K - 1UL) },
165 { tlbirpalos_64k, ~(SZ_64K - 1UL) },
166 { tlbirpalos_16k, ~(SZ_16K - 1UL) }
167 };
168
169 tlbi_page_lookup[gpt_config.pgs].function(
170 base & tlbi_page_lookup[gpt_config.pgs].mask);
171 dsbosh();
172}
173
174/*
175 * Helper function to fill out GPI entries in a single L1 table
176 * with Granules or Contiguous descriptor.
177 *
178 * Parameters
179 * l1 Pointer to 2MB, 32MB or 512MB aligned L1 table entry to fill out
180 * l1_desc GPT Granules or Contiguous descriptor set this range to
181 * cnt Number of double 128-bit L1 entries to fill
182 *
183 */
184static void fill_desc(uint64_t *l1, uint64_t l1_desc, unsigned int cnt)
185{
186 uint128_t *l1_quad = (uint128_t *)l1;
187 uint128_t l1_quad_desc = (uint128_t)l1_desc | ((uint128_t)l1_desc << 64);
188
189 VERBOSE("GPT: %s(%p 0x%"PRIx64" %u)\n", __func__, l1, l1_desc, cnt);
190
191 for (unsigned int i = 0U; i < cnt; i++) {
192 *l1_quad++ = l1_quad_desc;
193 }
194}
195
196static void shatter_2mb(uintptr_t base, const gpi_info_t *gpi_info,
197 uint64_t l1_desc)
198{
199 unsigned long idx = GPT_L1_INDEX(ALIGN_2MB(base));
200
201 VERBOSE("GPT: %s(0x%"PRIxPTR" 0x%"PRIx64")\n",
202 __func__, base, l1_desc);
203
204 /* Convert 2MB Contiguous block to Granules */
205 fill_desc(&gpi_info->gpt_l1_addr[idx], l1_desc, L1_QWORDS_2MB);
206}
207
208static void shatter_32mb(uintptr_t base, const gpi_info_t *gpi_info,
209 uint64_t l1_desc)
210{
211 unsigned long idx = GPT_L1_INDEX(ALIGN_2MB(base));
212 const uint64_t *l1_gran = &gpi_info->gpt_l1_addr[idx];
213 uint64_t l1_cont_desc = GPT_L1_CONT_DESC(l1_desc, 2MB);
214 uint64_t *l1;
215
216 VERBOSE("GPT: %s(0x%"PRIxPTR" 0x%"PRIx64")\n",
217 __func__, base, l1_desc);
218
219 /* Get index corresponding to 32MB aligned address */
220 idx = GPT_L1_INDEX(ALIGN_32MB(base));
221 l1 = &gpi_info->gpt_l1_addr[idx];
222
223 /* 16 x 2MB blocks in 32MB */
224 for (unsigned int i = 0U; i < 16U; i++) {
225 /* Fill with Granules or Contiguous descriptors */
226 fill_desc(l1, (l1 == l1_gran) ? l1_desc : l1_cont_desc,
227 L1_QWORDS_2MB);
228 l1 = (uint64_t *)((uintptr_t)l1 + L1_BYTES_2MB);
229 }
230}
231
232static void shatter_512mb(uintptr_t base, const gpi_info_t *gpi_info,
233 uint64_t l1_desc)
234{
235 unsigned long idx = GPT_L1_INDEX(ALIGN_32MB(base));
236 const uint64_t *l1_32mb = &gpi_info->gpt_l1_addr[idx];
237 uint64_t l1_cont_desc = GPT_L1_CONT_DESC(l1_desc, 32MB);
238 uint64_t *l1;
239
240 VERBOSE("GPT: %s(0x%"PRIxPTR" 0x%"PRIx64")\n",
241 __func__, base, l1_desc);
242
243 /* Get index corresponding to 512MB aligned address */
244 idx = GPT_L1_INDEX(ALIGN_512MB(base));
245 l1 = &gpi_info->gpt_l1_addr[idx];
246
247 /* 16 x 32MB blocks in 512MB */
248 for (unsigned int i = 0U; i < 16U; i++) {
249 if (l1 == l1_32mb) {
250 /* Shatter this 32MB block */
251 shatter_32mb(base, gpi_info, l1_desc);
252 } else {
253 /* Fill 32MB with Contiguous descriptors */
254 fill_desc(l1, l1_cont_desc, L1_QWORDS_32MB);
255 }
256
257 l1 = (uint64_t *)((uintptr_t)l1 + L1_BYTES_32MB);
258 }
259}
260
johpow019d134022021-06-16 17:57:28 -0500261/*
262 * This function checks to see if a GPI value is valid.
263 *
264 * These are valid GPI values.
265 * GPT_GPI_NO_ACCESS U(0x0)
266 * GPT_GPI_SECURE U(0x8)
267 * GPT_GPI_NS U(0x9)
268 * GPT_GPI_ROOT U(0xA)
269 * GPT_GPI_REALM U(0xB)
270 * GPT_GPI_ANY U(0xF)
271 *
272 * Parameters
273 * gpi GPI to check for validity.
274 *
275 * Return
276 * true for a valid GPI, false for an invalid one.
277 */
AlexeiFedoroveb6f6cd2024-03-13 13:59:09 +0000278static bool is_gpi_valid(unsigned int gpi)
johpow019d134022021-06-16 17:57:28 -0500279{
280 if ((gpi == GPT_GPI_NO_ACCESS) || (gpi == GPT_GPI_ANY) ||
281 ((gpi >= GPT_GPI_SECURE) && (gpi <= GPT_GPI_REALM))) {
282 return true;
johpow019d134022021-06-16 17:57:28 -0500283 }
Robert Wakim48e6b572021-10-21 15:39:56 +0100284 return false;
johpow019d134022021-06-16 17:57:28 -0500285}
286
287/*
288 * This function checks to see if two PAS regions overlap.
289 *
290 * Parameters
291 * base_1: base address of first PAS
292 * size_1: size of first PAS
293 * base_2: base address of second PAS
294 * size_2: size of second PAS
295 *
296 * Return
297 * True if PAS regions overlap, false if they do not.
298 */
AlexeiFedoroveb6f6cd2024-03-13 13:59:09 +0000299static bool check_pas_overlap(uintptr_t base_1, size_t size_1,
300 uintptr_t base_2, size_t size_2)
johpow019d134022021-06-16 17:57:28 -0500301{
302 if (((base_1 + size_1) > base_2) && ((base_2 + size_2) > base_1)) {
303 return true;
johpow019d134022021-06-16 17:57:28 -0500304 }
Robert Wakim48e6b572021-10-21 15:39:56 +0100305 return false;
johpow019d134022021-06-16 17:57:28 -0500306}
307
308/*
309 * This helper function checks to see if a PAS region from index 0 to
310 * (pas_idx - 1) occupies the L0 region at index l0_idx in the L0 table.
311 *
312 * Parameters
313 * l0_idx: Index of the L0 entry to check
314 * pas_regions: PAS region array
315 * pas_idx: Upper bound of the PAS array index.
316 *
317 * Return
318 * True if a PAS region occupies the L0 region in question, false if not.
319 */
AlexeiFedoroveb6f6cd2024-03-13 13:59:09 +0000320static bool does_previous_pas_exist_here(unsigned int l0_idx,
321 pas_region_t *pas_regions,
322 unsigned int pas_idx)
johpow019d134022021-06-16 17:57:28 -0500323{
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000324 /* Iterate over PAS regions up to pas_idx */
johpow019d134022021-06-16 17:57:28 -0500325 for (unsigned int i = 0U; i < pas_idx; i++) {
AlexeiFedoroveb6f6cd2024-03-13 13:59:09 +0000326 if (check_pas_overlap((GPT_L0GPTSZ_ACTUAL_SIZE * l0_idx),
johpow019d134022021-06-16 17:57:28 -0500327 GPT_L0GPTSZ_ACTUAL_SIZE,
328 pas_regions[i].base_pa, pas_regions[i].size)) {
329 return true;
330 }
331 }
332 return false;
333}
334
335/*
336 * This function iterates over all of the PAS regions and checks them to ensure
337 * proper alignment of base and size, that the GPI is valid, and that no regions
338 * overlap. As a part of the overlap checks, this function checks existing L0
339 * mappings against the new PAS regions in the event that gpt_init_pas_l1_tables
340 * is called multiple times to place L1 tables in different areas of memory. It
341 * also counts the number of L1 tables needed and returns it on success.
342 *
343 * Parameters
344 * *pas_regions Pointer to array of PAS region structures.
345 * pas_region_cnt Total number of PAS regions in the array.
346 *
347 * Return
348 * Negative Linux error code in the event of a failure, number of L1 regions
349 * required when successful.
350 */
AlexeiFedoroveb6f6cd2024-03-13 13:59:09 +0000351static int validate_pas_mappings(pas_region_t *pas_regions,
352 unsigned int pas_region_cnt)
johpow019d134022021-06-16 17:57:28 -0500353{
354 unsigned int idx;
355 unsigned int l1_cnt = 0U;
356 unsigned int pas_l1_cnt;
357 uint64_t *l0_desc = (uint64_t *)gpt_config.plat_gpt_l0_base;
358
359 assert(pas_regions != NULL);
360 assert(pas_region_cnt != 0U);
361
362 for (idx = 0U; idx < pas_region_cnt; idx++) {
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000363 /* Check for arithmetic overflow in region */
johpow019d134022021-06-16 17:57:28 -0500364 if ((ULONG_MAX - pas_regions[idx].base_pa) <
365 pas_regions[idx].size) {
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000366 ERROR("GPT: Address overflow in PAS[%u]!\n", idx);
johpow019d134022021-06-16 17:57:28 -0500367 return -EOVERFLOW;
368 }
369
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000370 /* Initial checks for PAS validity */
johpow019d134022021-06-16 17:57:28 -0500371 if (((pas_regions[idx].base_pa + pas_regions[idx].size) >
372 GPT_PPS_ACTUAL_SIZE(gpt_config.t)) ||
AlexeiFedoroveb6f6cd2024-03-13 13:59:09 +0000373 !is_gpi_valid(GPT_PAS_ATTR_GPI(pas_regions[idx].attrs))) {
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000374 ERROR("GPT: PAS[%u] is invalid!\n", idx);
johpow019d134022021-06-16 17:57:28 -0500375 return -EFAULT;
376 }
377
378 /*
379 * Make sure this PAS does not overlap with another one. We
380 * start from idx + 1 instead of 0 since prior PAS mappings will
381 * have already checked themselves against this one.
382 */
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000383 for (unsigned int i = idx + 1U; i < pas_region_cnt; i++) {
AlexeiFedoroveb6f6cd2024-03-13 13:59:09 +0000384 if (check_pas_overlap(pas_regions[idx].base_pa,
johpow019d134022021-06-16 17:57:28 -0500385 pas_regions[idx].size,
386 pas_regions[i].base_pa,
387 pas_regions[i].size)) {
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000388 ERROR("GPT: PAS[%u] overlaps with PAS[%u]\n",
johpow019d134022021-06-16 17:57:28 -0500389 i, idx);
390 return -EFAULT;
391 }
392 }
393
394 /*
395 * Since this function can be called multiple times with
396 * separate L1 tables we need to check the existing L0 mapping
397 * to see if this PAS would fall into one that has already been
398 * initialized.
399 */
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000400 for (unsigned int i =
401 (unsigned int)GPT_L0_IDX(pas_regions[idx].base_pa);
402 i <= GPT_L0_IDX(pas_regions[idx].base_pa +
403 pas_regions[idx].size - 1UL);
404 i++) {
johpow019d134022021-06-16 17:57:28 -0500405 if ((GPT_L0_TYPE(l0_desc[i]) == GPT_L0_TYPE_BLK_DESC) &&
406 (GPT_L0_BLKD_GPI(l0_desc[i]) == GPT_GPI_ANY)) {
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000407 /* This descriptor is unused so continue */
johpow019d134022021-06-16 17:57:28 -0500408 continue;
409 }
410
411 /*
412 * This descriptor has been initialized in a previous
413 * call to this function so cannot be initialized again.
414 */
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000415 ERROR("GPT: PAS[%u] overlaps with previous L0[%u]!\n",
johpow019d134022021-06-16 17:57:28 -0500416 idx, i);
417 return -EFAULT;
418 }
419
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000420 /* Check for block mapping (L0) type */
johpow019d134022021-06-16 17:57:28 -0500421 if (GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs) ==
422 GPT_PAS_ATTR_MAP_TYPE_BLOCK) {
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000423 /* Make sure base and size are block-aligned */
johpow019d134022021-06-16 17:57:28 -0500424 if (!GPT_IS_L0_ALIGNED(pas_regions[idx].base_pa) ||
425 !GPT_IS_L0_ALIGNED(pas_regions[idx].size)) {
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000426 ERROR("GPT: PAS[%u] is not block-aligned!\n",
johpow019d134022021-06-16 17:57:28 -0500427 idx);
428 return -EFAULT;
429 }
430
431 continue;
432 }
433
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000434 /* Check for granule mapping (L1) type */
johpow019d134022021-06-16 17:57:28 -0500435 if (GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs) ==
436 GPT_PAS_ATTR_MAP_TYPE_GRANULE) {
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000437 /* Make sure base and size are granule-aligned */
johpow019d134022021-06-16 17:57:28 -0500438 if (!GPT_IS_L1_ALIGNED(gpt_config.p, pas_regions[idx].base_pa) ||
439 !GPT_IS_L1_ALIGNED(gpt_config.p, pas_regions[idx].size)) {
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000440 ERROR("GPT: PAS[%u] is not granule-aligned!\n",
johpow019d134022021-06-16 17:57:28 -0500441 idx);
442 return -EFAULT;
443 }
444
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000445 /* Find how many L1 tables this PAS occupies */
johpow019d134022021-06-16 17:57:28 -0500446 pas_l1_cnt = (GPT_L0_IDX(pas_regions[idx].base_pa +
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000447 pas_regions[idx].size - 1UL) -
448 GPT_L0_IDX(pas_regions[idx].base_pa) + 1U);
johpow019d134022021-06-16 17:57:28 -0500449
450 /*
451 * This creates a situation where, if multiple PAS
452 * regions occupy the same table descriptor, we can get
453 * an artificially high total L1 table count. The way we
454 * handle this is by checking each PAS against those
455 * before it in the array, and if they both occupy the
456 * same PAS we subtract from pas_l1_cnt and only the
457 * first PAS in the array gets to count it.
458 */
459
460 /*
461 * If L1 count is greater than 1 we know the start and
462 * end PAs are in different L0 regions so we must check
463 * both for overlap against other PAS.
464 */
465 if (pas_l1_cnt > 1) {
AlexeiFedoroveb6f6cd2024-03-13 13:59:09 +0000466 if (does_previous_pas_exist_here(
johpow019d134022021-06-16 17:57:28 -0500467 GPT_L0_IDX(pas_regions[idx].base_pa +
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000468 pas_regions[idx].size - 1UL),
johpow019d134022021-06-16 17:57:28 -0500469 pas_regions, idx)) {
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000470 pas_l1_cnt--;
johpow019d134022021-06-16 17:57:28 -0500471 }
472 }
473
AlexeiFedoroveb6f6cd2024-03-13 13:59:09 +0000474 if (does_previous_pas_exist_here(
johpow019d134022021-06-16 17:57:28 -0500475 GPT_L0_IDX(pas_regions[idx].base_pa),
476 pas_regions, idx)) {
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000477 pas_l1_cnt--;
johpow019d134022021-06-16 17:57:28 -0500478 }
479
480 l1_cnt += pas_l1_cnt;
481 continue;
482 }
483
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000484 /* If execution reaches this point, mapping type is invalid */
485 ERROR("GPT: PAS[%u] has invalid mapping type 0x%x.\n", idx,
johpow019d134022021-06-16 17:57:28 -0500486 GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs));
487 return -EINVAL;
488 }
489
490 return l1_cnt;
491}
492
493/*
494 * This function validates L0 initialization parameters.
495 *
496 * Parameters
497 * l0_mem_base Base address of memory used for L0 tables.
498 * l1_mem_size Size of memory available for L0 tables.
499 *
500 * Return
501 * Negative Linux error code in the event of a failure, 0 for success.
502 */
AlexeiFedoroveb6f6cd2024-03-13 13:59:09 +0000503static int validate_l0_params(gpccr_pps_e pps, uintptr_t l0_mem_base,
504 size_t l0_mem_size)
johpow019d134022021-06-16 17:57:28 -0500505{
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +0100506 size_t l0_alignment, locks_size = 0;
johpow019d134022021-06-16 17:57:28 -0500507
508 /*
509 * Make sure PPS is valid and then store it since macros need this value
510 * to work.
511 */
512 if (pps > GPT_PPS_MAX) {
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000513 ERROR("GPT: Invalid PPS: 0x%x\n", pps);
johpow019d134022021-06-16 17:57:28 -0500514 return -EINVAL;
515 }
516 gpt_config.pps = pps;
517 gpt_config.t = gpt_t_lookup[pps];
518
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000519 /* Alignment must be the greater of 4KB or l0 table size */
johpow019d134022021-06-16 17:57:28 -0500520 l0_alignment = PAGE_SIZE_4KB;
521 if (l0_alignment < GPT_L0_TABLE_SIZE(gpt_config.t)) {
522 l0_alignment = GPT_L0_TABLE_SIZE(gpt_config.t);
523 }
524
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000525 /* Check base address */
526 if ((l0_mem_base == 0UL) ||
527 ((l0_mem_base & (l0_alignment - 1UL)) != 0UL)) {
528 ERROR("GPT: Invalid L0 base address: 0x%lx\n", l0_mem_base);
johpow019d134022021-06-16 17:57:28 -0500529 return -EFAULT;
530 }
531
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +0100532#if (RME_GPT_BITLOCK_BLOCK != 0)
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000533 /*
534 * Size of bitlocks in bytes for the protected address space
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +0100535 * with RME_GPT_BITLOCK_BLOCK * 512MB per bitlock.
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000536 */
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +0100537 locks_size = GPT_PPS_ACTUAL_SIZE(gpt_config.t) /
538 (RME_GPT_BITLOCK_BLOCK * SZ_512M * 8U);
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000539
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +0100540 /*
541 * If protected space size is less than the size covered
542 * by 'bitlock' structure, check for a single bitlock.
543 */
544 if (locks_size < LOCK_SIZE) {
545 locks_size = LOCK_SIZE;
546 }
547#endif
548 /* Check size for L0 tables and bitlocks */
549 if (l0_mem_size < (GPT_L0_TABLE_SIZE(gpt_config.t) + locks_size)) {
550 ERROR("GPT: Inadequate L0 memory\n");
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000551 ERROR(" Expected 0x%lx bytes, got 0x%lx bytes\n",
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +0100552 GPT_L0_TABLE_SIZE(gpt_config.t) + locks_size,
553 l0_mem_size);
johpow019d134022021-06-16 17:57:28 -0500554 return -ENOMEM;
555 }
556
557 return 0;
558}
559
560/*
561 * In the event that L1 tables are needed, this function validates
562 * the L1 table generation parameters.
563 *
564 * Parameters
565 * l1_mem_base Base address of memory used for L1 table allocation.
566 * l1_mem_size Total size of memory available for L1 tables.
567 * l1_gpt_cnt Number of L1 tables needed.
568 *
569 * Return
570 * Negative Linux error code in the event of a failure, 0 for success.
571 */
AlexeiFedoroveb6f6cd2024-03-13 13:59:09 +0000572static int validate_l1_params(uintptr_t l1_mem_base, size_t l1_mem_size,
573 unsigned int l1_gpt_cnt)
johpow019d134022021-06-16 17:57:28 -0500574{
575 size_t l1_gpt_mem_sz;
576
577 /* Check if the granularity is supported */
578 if (!xlat_arch_is_granule_size_supported(
579 GPT_PGS_ACTUAL_SIZE(gpt_config.p))) {
580 return -EPERM;
581 }
582
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000583 /* Make sure L1 tables are aligned to their size */
584 if ((l1_mem_base & (GPT_L1_TABLE_SIZE(gpt_config.p) - 1UL)) != 0UL) {
585 ERROR("GPT: Unaligned L1 GPT base address: 0x%"PRIxPTR"\n",
johpow019d134022021-06-16 17:57:28 -0500586 l1_mem_base);
587 return -EFAULT;
588 }
589
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000590 /* Get total memory needed for L1 tables */
johpow019d134022021-06-16 17:57:28 -0500591 l1_gpt_mem_sz = l1_gpt_cnt * GPT_L1_TABLE_SIZE(gpt_config.p);
592
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000593 /* Check for overflow */
johpow019d134022021-06-16 17:57:28 -0500594 if ((l1_gpt_mem_sz / GPT_L1_TABLE_SIZE(gpt_config.p)) != l1_gpt_cnt) {
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000595 ERROR("GPT: Overflow calculating L1 memory size\n");
johpow019d134022021-06-16 17:57:28 -0500596 return -ENOMEM;
597 }
598
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000599 /* Make sure enough space was supplied */
johpow019d134022021-06-16 17:57:28 -0500600 if (l1_mem_size < l1_gpt_mem_sz) {
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000601 ERROR("%sL1 GPTs%s", (const char *)"GPT: Inadequate ",
602 (const char *)" memory\n");
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000603 ERROR(" Expected 0x%lx bytes, got 0x%lx bytes\n",
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000604 l1_gpt_mem_sz, l1_mem_size);
johpow019d134022021-06-16 17:57:28 -0500605 return -ENOMEM;
606 }
607
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000608 VERBOSE("GPT: Requested 0x%lx bytes for L1 GPTs\n", l1_gpt_mem_sz);
johpow019d134022021-06-16 17:57:28 -0500609 return 0;
610}
611
612/*
613 * This function initializes L0 block descriptors (regions that cannot be
614 * transitioned at the granule level) according to the provided PAS.
615 *
616 * Parameters
617 * *pas Pointer to the structure defining the PAS region to
618 * initialize.
619 */
AlexeiFedoroveb6f6cd2024-03-13 13:59:09 +0000620static void generate_l0_blk_desc(pas_region_t *pas)
johpow019d134022021-06-16 17:57:28 -0500621{
622 uint64_t gpt_desc;
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000623 unsigned long idx, end_idx;
johpow019d134022021-06-16 17:57:28 -0500624 uint64_t *l0_gpt_arr;
625
626 assert(gpt_config.plat_gpt_l0_base != 0U);
627 assert(pas != NULL);
628
629 /*
630 * Checking of PAS parameters has already been done in
AlexeiFedoroveb6f6cd2024-03-13 13:59:09 +0000631 * validate_pas_mappings so no need to check the same things again.
johpow019d134022021-06-16 17:57:28 -0500632 */
633
634 l0_gpt_arr = (uint64_t *)gpt_config.plat_gpt_l0_base;
635
636 /* Create the GPT Block descriptor for this PAS region */
637 gpt_desc = GPT_L0_BLK_DESC(GPT_PAS_ATTR_GPI(pas->attrs));
638
639 /* Start index of this region in L0 GPTs */
Robert Wakim48e6b572021-10-21 15:39:56 +0100640 idx = GPT_L0_IDX(pas->base_pa);
johpow019d134022021-06-16 17:57:28 -0500641
642 /*
643 * Determine number of L0 GPT descriptors covered by
644 * this PAS region and use the count to populate these
645 * descriptors.
646 */
Robert Wakim48e6b572021-10-21 15:39:56 +0100647 end_idx = GPT_L0_IDX(pas->base_pa + pas->size);
johpow019d134022021-06-16 17:57:28 -0500648
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000649 /* Generate the needed block descriptors */
johpow019d134022021-06-16 17:57:28 -0500650 for (; idx < end_idx; idx++) {
651 l0_gpt_arr[idx] = gpt_desc;
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000652 VERBOSE("GPT: L0 entry (BLOCK) index %lu [%p]: GPI = 0x%"PRIx64" (0x%"PRIx64")\n",
johpow019d134022021-06-16 17:57:28 -0500653 idx, &l0_gpt_arr[idx],
654 (gpt_desc >> GPT_L0_BLK_DESC_GPI_SHIFT) &
655 GPT_L0_BLK_DESC_GPI_MASK, l0_gpt_arr[idx]);
656 }
657}
658
659/*
660 * Helper function to determine if the end physical address lies in the same L0
661 * region as the current physical address. If true, the end physical address is
662 * returned else, the start address of the next region is returned.
663 *
664 * Parameters
665 * cur_pa Physical address of the current PA in the loop through
666 * the range.
667 * end_pa Physical address of the end PA in a PAS range.
668 *
669 * Return
670 * The PA of the end of the current range.
671 */
AlexeiFedoroveb6f6cd2024-03-13 13:59:09 +0000672static uintptr_t get_l1_end_pa(uintptr_t cur_pa, uintptr_t end_pa)
johpow019d134022021-06-16 17:57:28 -0500673{
674 uintptr_t cur_idx;
675 uintptr_t end_idx;
676
Robert Wakim48e6b572021-10-21 15:39:56 +0100677 cur_idx = GPT_L0_IDX(cur_pa);
678 end_idx = GPT_L0_IDX(end_pa);
johpow019d134022021-06-16 17:57:28 -0500679
680 assert(cur_idx <= end_idx);
681
682 if (cur_idx == end_idx) {
683 return end_pa;
684 }
685
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000686 return (cur_idx + 1UL) << GPT_L0_IDX_SHIFT;
johpow019d134022021-06-16 17:57:28 -0500687}
688
689/*
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000690 * Helper function to fill out GPI entries from 'first' granule address of
691 * the specified 'length' in a single L1 table with 'l1_desc' Contiguous
692 * descriptor.
johpow019d134022021-06-16 17:57:28 -0500693 *
694 * Parameters
johpow019d134022021-06-16 17:57:28 -0500695 * l1 Pointer to L1 table to fill out
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000696 * first Address of first granule in range
697 * length Length of the range in bytes
698 * gpi GPI set this range to
699 *
700 * Return
701 * Address of next granule in range.
johpow019d134022021-06-16 17:57:28 -0500702 */
AlexeiFedorov98fc2bc2024-06-06 11:48:44 +0100703__unused static uintptr_t fill_l1_cont_desc(uint64_t *l1, uintptr_t first,
704 size_t length, unsigned int gpi)
johpow019d134022021-06-16 17:57:28 -0500705{
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000706 /*
707 * Look up table for contiguous blocks and descriptors.
708 * Entries should be defined in descending block sizes:
709 * 512MB, 32MB and 2MB.
710 */
711 static const gpt_fill_lookup_t gpt_fill_lookup[] = {
712#if (RME_GPT_MAX_BLOCK == 512)
713 { SZ_512M, GPT_L1_CONT_DESC_512MB },
714#endif
715#if (RME_GPT_MAX_BLOCK >= 32)
716 { SZ_32M, GPT_L1_CONT_DESC_32MB },
717#endif
718#if (RME_GPT_MAX_BLOCK != 0)
719 { SZ_2M, GPT_L1_CONT_DESC_2MB }
720#endif
721 };
johpow019d134022021-06-16 17:57:28 -0500722
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000723 /*
724 * Iterate through all block sizes (512MB, 32MB and 2MB)
725 * starting with maximum supported.
726 */
727 for (unsigned long i = 0UL; i < ARRAY_SIZE(gpt_fill_lookup); i++) {
728 /* Calculate index */
729 unsigned long idx = GPT_L1_INDEX(first);
730
731 /* Contiguous block size */
732 size_t cont_size = gpt_fill_lookup[i].size;
733
734 if (GPT_REGION_IS_CONT(length, first, cont_size)) {
735
736 /* Generate Contiguous descriptor */
737 uint64_t l1_desc = GPT_L1_GPI_CONT_DESC(gpi,
738 gpt_fill_lookup[i].desc);
739
740 /* Number of 128-bit L1 entries in block */
741 unsigned int cnt;
742
743 switch (cont_size) {
744 case SZ_512M:
745 cnt = L1_QWORDS_512MB;
746 break;
747 case SZ_32M:
748 cnt = L1_QWORDS_32MB;
749 break;
750 default: /* SZ_2MB */
751 cnt = L1_QWORDS_2MB;
752 }
753
754 VERBOSE("GPT: Contiguous descriptor 0x%"PRIxPTR" %luMB\n",
755 first, cont_size / SZ_1M);
756
757 /* Fill Contiguous descriptors */
758 fill_desc(&l1[idx], l1_desc, cnt);
759 first += cont_size;
760 length -= cont_size;
761
762 if (length == 0UL) {
763 break;
764 }
765 }
766 }
767
768 return first;
769}
770
771/* Build Granules descriptor with the same 'gpi' for every GPI entry */
772static uint64_t build_l1_desc(unsigned int gpi)
773{
774 uint64_t l1_desc = (uint64_t)gpi | ((uint64_t)gpi << 4);
775
776 l1_desc |= (l1_desc << 8);
777 l1_desc |= (l1_desc << 16);
778 return (l1_desc | (l1_desc << 32));
779}
780
781/*
782 * Helper function to fill out GPI entries from 'first' to 'last' granule
783 * address in a single L1 table with 'l1_desc' Granules descriptor.
784 *
785 * Parameters
786 * l1 Pointer to L1 table to fill out
787 * first Address of first granule in range
788 * last Address of last granule in range (inclusive)
789 * gpi GPI set this range to
790 *
791 * Return
792 * Address of next granule in range.
793 */
794static uintptr_t fill_l1_gran_desc(uint64_t *l1, uintptr_t first,
795 uintptr_t last, unsigned int gpi)
796{
797 uint64_t gpi_mask;
798 unsigned long i;
799
800 /* Generate Granules descriptor */
801 uint64_t l1_desc = build_l1_desc(gpi);
johpow019d134022021-06-16 17:57:28 -0500802
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000803 /* Shift the mask if we're starting in the middle of an L1 entry */
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000804 gpi_mask = ULONG_MAX << (GPT_L1_GPI_IDX(gpt_config.p, first) << 2);
johpow019d134022021-06-16 17:57:28 -0500805
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000806 /* Fill out each L1 entry for this region */
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000807 for (i = GPT_L1_INDEX(first); i <= GPT_L1_INDEX(last); i++) {
808
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000809 /* Account for stopping in the middle of an L1 entry */
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000810 if (i == GPT_L1_INDEX(last)) {
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000811 gpi_mask &= (gpi_mask >> ((15U -
johpow019d134022021-06-16 17:57:28 -0500812 GPT_L1_GPI_IDX(gpt_config.p, last)) << 2));
813 }
814
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000815 assert((l1[i] & gpi_mask) == (GPT_L1_ANY_DESC & gpi_mask));
816
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000817 /* Write GPI values */
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000818 l1[i] = (l1[i] & ~gpi_mask) | (l1_desc & gpi_mask);
johpow019d134022021-06-16 17:57:28 -0500819
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000820 /* Reset mask */
821 gpi_mask = ULONG_MAX;
johpow019d134022021-06-16 17:57:28 -0500822 }
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000823
824 return last + GPT_PGS_ACTUAL_SIZE(gpt_config.p);
johpow019d134022021-06-16 17:57:28 -0500825}
826
827/*
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000828 * Helper function to fill out GPI entries in a single L1 table.
AlexeiFedorov98fc2bc2024-06-06 11:48:44 +0100829 * This function fills out an entire L1 table with either Granules or Contiguous
830 * (RME_GPT_MAX_BLOCK != 0) descriptors depending on region length and alignment.
831 * Note. If RME_GPT_MAX_BLOCK == 0, then the L1 tables are filled with regular
832 * Granules descriptors.
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000833 *
834 * Parameters
835 * l1 Pointer to L1 table to fill out
836 * first Address of first granule in range
837 * last Address of last granule in range (inclusive)
838 * gpi GPI set this range to
839 */
840static void fill_l1_tbl(uint64_t *l1, uintptr_t first, uintptr_t last,
841 unsigned int gpi)
842{
843 assert(l1 != NULL);
844 assert(first <= last);
845 assert((first & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1UL)) == 0UL);
846 assert((last & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1UL)) == 0UL);
847 assert(GPT_L0_IDX(first) == GPT_L0_IDX(last));
848
AlexeiFedorov98fc2bc2024-06-06 11:48:44 +0100849#if (RME_GPT_MAX_BLOCK != 0)
AlexeiFedorova2565512024-06-20 10:29:58 +0100850 while (first <= last) {
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000851 /* Region length */
852 size_t length = last - first + GPT_PGS_ACTUAL_SIZE(gpt_config.p);
853
854 if (length < SZ_2M) {
855 /*
AlexeiFedorov98fc2bc2024-06-06 11:48:44 +0100856 * Fill with Granule descriptors in case of
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000857 * region length < 2MB.
858 */
859 first = fill_l1_gran_desc(l1, first, last, gpi);
860
861 } else if ((first & (SZ_2M - UL(1))) == UL(0)) {
862 /*
863 * For region length >= 2MB and at least 2MB aligned
864 * call to fill_l1_cont_desc will iterate through
865 * all block sizes (512MB, 32MB and 2MB) supported and
866 * fill corresponding Contiguous descriptors.
867 */
868 first = fill_l1_cont_desc(l1, first, length, gpi);
869 } else {
870 /*
871 * For not aligned region >= 2MB fill with Granules
872 * descriptors up to the next 2MB aligned address.
873 */
874 uintptr_t new_last = ALIGN_2MB(first + SZ_2M) -
875 GPT_PGS_ACTUAL_SIZE(gpt_config.p);
876
877 first = fill_l1_gran_desc(l1, first, new_last, gpi);
878 }
879 }
AlexeiFedorov98fc2bc2024-06-06 11:48:44 +0100880#else
881 /* Fill with Granule descriptors */
882 first = fill_l1_gran_desc(l1, first, last, gpi);
883#endif
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000884 assert(first == (last + GPT_PGS_ACTUAL_SIZE(gpt_config.p)));
885}
886
887/*
johpow019d134022021-06-16 17:57:28 -0500888 * This function finds the next available unused L1 table and initializes all
889 * granules descriptor entries to GPI_ANY. This ensures that there are no chunks
890 * of GPI_NO_ACCESS (0b0000) memory floating around in the system in the
891 * event that a PAS region stops midway through an L1 table, thus guaranteeing
892 * that all memory not explicitly assigned is GPI_ANY. This function does not
893 * check for overflow conditions, that should be done by the caller.
894 *
895 * Return
896 * Pointer to the next available L1 table.
897 */
AlexeiFedoroveb6f6cd2024-03-13 13:59:09 +0000898static uint64_t *get_new_l1_tbl(void)
johpow019d134022021-06-16 17:57:28 -0500899{
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000900 /* Retrieve the next L1 table */
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000901 uint64_t *l1 = (uint64_t *)gpt_l1_tbl;
johpow019d134022021-06-16 17:57:28 -0500902
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000903 /* Increment L1 GPT address */
904 gpt_l1_tbl += GPT_L1_TABLE_SIZE(gpt_config.p);
johpow019d134022021-06-16 17:57:28 -0500905
906 /* Initialize all GPIs to GPT_GPI_ANY */
907 for (unsigned int i = 0U; i < GPT_L1_ENTRY_COUNT(gpt_config.p); i++) {
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000908 l1[i] = GPT_L1_ANY_DESC;
johpow019d134022021-06-16 17:57:28 -0500909 }
910
911 return l1;
912}
913
914/*
915 * When L1 tables are needed, this function creates the necessary L0 table
916 * descriptors and fills out the L1 table entries according to the supplied
917 * PAS range.
918 *
919 * Parameters
920 * *pas Pointer to the structure defining the PAS region.
921 */
AlexeiFedoroveb6f6cd2024-03-13 13:59:09 +0000922static void generate_l0_tbl_desc(pas_region_t *pas)
johpow019d134022021-06-16 17:57:28 -0500923{
924 uintptr_t end_pa;
925 uintptr_t cur_pa;
926 uintptr_t last_gran_pa;
927 uint64_t *l0_gpt_base;
928 uint64_t *l1_gpt_arr;
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000929 unsigned int l0_idx, gpi;
johpow019d134022021-06-16 17:57:28 -0500930
931 assert(gpt_config.plat_gpt_l0_base != 0U);
932 assert(pas != NULL);
933
934 /*
935 * Checking of PAS parameters has already been done in
AlexeiFedoroveb6f6cd2024-03-13 13:59:09 +0000936 * validate_pas_mappings so no need to check the same things again.
johpow019d134022021-06-16 17:57:28 -0500937 */
johpow019d134022021-06-16 17:57:28 -0500938 end_pa = pas->base_pa + pas->size;
939 l0_gpt_base = (uint64_t *)gpt_config.plat_gpt_l0_base;
940
941 /* We start working from the granule at base PA */
942 cur_pa = pas->base_pa;
943
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000944 /* Get GPI */
945 gpi = GPT_PAS_ATTR_GPI(pas->attrs);
946
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000947 /* Iterate over each L0 region in this memory range */
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000948 for (l0_idx = (unsigned int)GPT_L0_IDX(pas->base_pa);
949 l0_idx <= (unsigned int)GPT_L0_IDX(end_pa - 1UL);
johpow019d134022021-06-16 17:57:28 -0500950 l0_idx++) {
johpow019d134022021-06-16 17:57:28 -0500951 /*
952 * See if the L0 entry is already a table descriptor or if we
953 * need to create one.
954 */
955 if (GPT_L0_TYPE(l0_gpt_base[l0_idx]) == GPT_L0_TYPE_TBL_DESC) {
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000956 /* Get the L1 array from the L0 entry */
johpow019d134022021-06-16 17:57:28 -0500957 l1_gpt_arr = GPT_L0_TBLD_ADDR(l0_gpt_base[l0_idx]);
958 } else {
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000959 /* Get a new L1 table from the L1 memory space */
AlexeiFedoroveb6f6cd2024-03-13 13:59:09 +0000960 l1_gpt_arr = get_new_l1_tbl();
johpow019d134022021-06-16 17:57:28 -0500961
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000962 /* Fill out the L0 descriptor and flush it */
johpow019d134022021-06-16 17:57:28 -0500963 l0_gpt_base[l0_idx] = GPT_L0_TBL_DESC(l1_gpt_arr);
964 }
965
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000966 VERBOSE("GPT: L0 entry (TABLE) index %u [%p] ==> L1 Addr %p (0x%"PRIx64")\n",
967 l0_idx, &l0_gpt_base[l0_idx], l1_gpt_arr, l0_gpt_base[l0_idx]);
johpow019d134022021-06-16 17:57:28 -0500968
969 /*
970 * Determine the PA of the last granule in this L0 descriptor.
971 */
AlexeiFedoroveb6f6cd2024-03-13 13:59:09 +0000972 last_gran_pa = get_l1_end_pa(cur_pa, end_pa) -
johpow019d134022021-06-16 17:57:28 -0500973 GPT_PGS_ACTUAL_SIZE(gpt_config.p);
974
975 /*
976 * Fill up L1 GPT entries between these two addresses. This
977 * function needs the addresses of the first granule and last
978 * granule in the range.
979 */
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +0000980 fill_l1_tbl(l1_gpt_arr, cur_pa, last_gran_pa, gpi);
johpow019d134022021-06-16 17:57:28 -0500981
AlexeiFedorov7eaaac72024-03-13 15:18:02 +0000982 /* Advance cur_pa to first granule in next L0 region */
AlexeiFedoroveb6f6cd2024-03-13 13:59:09 +0000983 cur_pa = get_l1_end_pa(cur_pa, end_pa);
johpow019d134022021-06-16 17:57:28 -0500984 }
985}
986
987/*
988 * This function flushes a range of L0 descriptors used by a given PAS region
989 * array. There is a chance that some unmodified L0 descriptors would be flushed
990 * in the case that there are "holes" in an array of PAS regions but overall
991 * this should be faster than individually flushing each modified L0 descriptor
992 * as they are created.
993 *
994 * Parameters
995 * *pas Pointer to an array of PAS regions.
996 * pas_count Number of entries in the PAS array.
997 */
998static void flush_l0_for_pas_array(pas_region_t *pas, unsigned int pas_count)
999{
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001000 unsigned long idx;
1001 unsigned long start_idx;
1002 unsigned long end_idx;
johpow019d134022021-06-16 17:57:28 -05001003 uint64_t *l0 = (uint64_t *)gpt_config.plat_gpt_l0_base;
1004
1005 assert(pas != NULL);
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001006 assert(pas_count != 0U);
johpow019d134022021-06-16 17:57:28 -05001007
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001008 /* Initial start and end values */
johpow019d134022021-06-16 17:57:28 -05001009 start_idx = GPT_L0_IDX(pas[0].base_pa);
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001010 end_idx = GPT_L0_IDX(pas[0].base_pa + pas[0].size - 1UL);
johpow019d134022021-06-16 17:57:28 -05001011
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001012 /* Find lowest and highest L0 indices used in this PAS array */
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001013 for (idx = 1UL; idx < pas_count; idx++) {
johpow019d134022021-06-16 17:57:28 -05001014 if (GPT_L0_IDX(pas[idx].base_pa) < start_idx) {
1015 start_idx = GPT_L0_IDX(pas[idx].base_pa);
1016 }
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001017 if (GPT_L0_IDX(pas[idx].base_pa + pas[idx].size - 1UL) > end_idx) {
1018 end_idx = GPT_L0_IDX(pas[idx].base_pa + pas[idx].size - 1UL);
johpow019d134022021-06-16 17:57:28 -05001019 }
1020 }
1021
1022 /*
1023 * Flush all covered L0 descriptors, add 1 because we need to include
1024 * the end index value.
1025 */
1026 flush_dcache_range((uintptr_t)&l0[start_idx],
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001027 ((end_idx + 1UL) - start_idx) * sizeof(uint64_t));
johpow019d134022021-06-16 17:57:28 -05001028}
1029
1030/*
1031 * Public API to enable granule protection checks once the tables have all been
1032 * initialized. This function is called at first initialization and then again
1033 * later during warm boots of CPU cores.
1034 *
1035 * Return
1036 * Negative Linux error code in the event of a failure, 0 for success.
1037 */
1038int gpt_enable(void)
1039{
1040 u_register_t gpccr_el3;
1041
1042 /*
1043 * Granule tables must be initialised before enabling
1044 * granule protection.
1045 */
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001046 if (gpt_config.plat_gpt_l0_base == 0UL) {
1047 ERROR("GPT: Tables have not been initialized!\n");
johpow019d134022021-06-16 17:57:28 -05001048 return -EPERM;
1049 }
1050
johpow019d134022021-06-16 17:57:28 -05001051 /* Write the base address of the L0 tables into GPTBR */
1052 write_gptbr_el3(((gpt_config.plat_gpt_l0_base >> GPTBR_BADDR_VAL_SHIFT)
1053 >> GPTBR_BADDR_SHIFT) & GPTBR_BADDR_MASK);
1054
1055 /* GPCCR_EL3.PPS */
1056 gpccr_el3 = SET_GPCCR_PPS(gpt_config.pps);
1057
1058 /* GPCCR_EL3.PGS */
1059 gpccr_el3 |= SET_GPCCR_PGS(gpt_config.pgs);
1060
Soby Mathew521375d2021-10-11 14:38:46 +01001061 /*
1062 * Since EL3 maps the L1 region as Inner shareable, use the same
1063 * shareability attribute for GPC as well so that
1064 * GPC fetches are visible to PEs
1065 */
1066 gpccr_el3 |= SET_GPCCR_SH(GPCCR_SH_IS);
johpow019d134022021-06-16 17:57:28 -05001067
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001068 /* Outer and Inner cacheability set to Normal memory, WB, RA, WA */
johpow019d134022021-06-16 17:57:28 -05001069 gpccr_el3 |= SET_GPCCR_ORGN(GPCCR_ORGN_WB_RA_WA);
1070 gpccr_el3 |= SET_GPCCR_IRGN(GPCCR_IRGN_WB_RA_WA);
1071
Kathleen Capella221f7ce2022-07-22 16:26:36 -04001072 /* Prepopulate GPCCR_EL3 but don't enable GPC yet */
1073 write_gpccr_el3(gpccr_el3);
1074 isb();
1075
1076 /* Invalidate any stale TLB entries and any cached register fields */
1077 tlbipaallos();
1078 dsb();
1079 isb();
1080
johpow019d134022021-06-16 17:57:28 -05001081 /* Enable GPT */
1082 gpccr_el3 |= GPCCR_GPC_BIT;
1083
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001084 /* TODO: Configure GPCCR_EL3_GPCP for Fault control */
johpow019d134022021-06-16 17:57:28 -05001085 write_gpccr_el3(gpccr_el3);
Soby Mathew521375d2021-10-11 14:38:46 +01001086 isb();
johpow019d134022021-06-16 17:57:28 -05001087 tlbipaallos();
1088 dsb();
1089 isb();
1090
1091 return 0;
1092}
1093
1094/*
1095 * Public API to disable granule protection checks.
1096 */
1097void gpt_disable(void)
1098{
1099 u_register_t gpccr_el3 = read_gpccr_el3();
1100
1101 write_gpccr_el3(gpccr_el3 & ~GPCCR_GPC_BIT);
1102 dsbsy();
1103 isb();
1104}
1105
1106/*
1107 * Public API that initializes the entire protected space to GPT_GPI_ANY using
1108 * the L0 tables (block descriptors). Ideally, this function is invoked prior
1109 * to DDR discovery and initialization. The MMU must be initialized before
1110 * calling this function.
1111 *
1112 * Parameters
1113 * pps PPS value to use for table generation
1114 * l0_mem_base Base address of L0 tables in memory.
1115 * l0_mem_size Total size of memory available for L0 tables.
1116 *
1117 * Return
1118 * Negative Linux error code in the event of a failure, 0 for success.
1119 */
AlexeiFedorov86ffd7b2022-12-09 11:27:14 +00001120int gpt_init_l0_tables(gpccr_pps_e pps, uintptr_t l0_mem_base,
johpow019d134022021-06-16 17:57:28 -05001121 size_t l0_mem_size)
1122{
johpow019d134022021-06-16 17:57:28 -05001123 uint64_t gpt_desc;
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +01001124 size_t locks_size = 0;
1125 __unused bitlock_t *bit_locks;
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001126 int ret;
johpow019d134022021-06-16 17:57:28 -05001127
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001128 /* Ensure that MMU and Data caches are enabled */
johpow019d134022021-06-16 17:57:28 -05001129 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0U);
1130
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001131 /* Validate other parameters */
AlexeiFedoroveb6f6cd2024-03-13 13:59:09 +00001132 ret = validate_l0_params(pps, l0_mem_base, l0_mem_size);
Robert Wakim48e6b572021-10-21 15:39:56 +01001133 if (ret != 0) {
johpow019d134022021-06-16 17:57:28 -05001134 return ret;
1135 }
1136
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001137 /* Create the descriptor to initialize L0 entries with */
johpow019d134022021-06-16 17:57:28 -05001138 gpt_desc = GPT_L0_BLK_DESC(GPT_GPI_ANY);
1139
1140 /* Iterate through all L0 entries */
1141 for (unsigned int i = 0U; i < GPT_L0_REGION_COUNT(gpt_config.t); i++) {
1142 ((uint64_t *)l0_mem_base)[i] = gpt_desc;
1143 }
1144
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +01001145#if (RME_GPT_BITLOCK_BLOCK != 0)
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001146 /* Initialise bitlocks at the end of L0 table */
1147 bit_locks = (bitlock_t *)(l0_mem_base +
1148 GPT_L0_TABLE_SIZE(gpt_config.t));
1149
1150 /* Size of bitlocks in bytes */
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +01001151 locks_size = GPT_PPS_ACTUAL_SIZE(gpt_config.t) /
1152 (RME_GPT_BITLOCK_BLOCK * SZ_512M * 8U);
1153
1154 /*
1155 * If protected space size is less than the size covered
1156 * by 'bitlock' structure, initialise a single bitlock.
1157 */
1158 if (locks_size < LOCK_SIZE) {
1159 locks_size = LOCK_SIZE;
1160 }
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001161
1162 for (size_t i = 0UL; i < (locks_size/LOCK_SIZE); i++) {
1163 bit_locks[i].lock = 0U;
1164 }
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +01001165#endif
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001166
1167 /* Flush updated L0 tables and bitlocks to memory */
johpow019d134022021-06-16 17:57:28 -05001168 flush_dcache_range((uintptr_t)l0_mem_base,
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001169 GPT_L0_TABLE_SIZE(gpt_config.t) + locks_size);
johpow019d134022021-06-16 17:57:28 -05001170
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001171 /* Stash the L0 base address once initial setup is complete */
johpow019d134022021-06-16 17:57:28 -05001172 gpt_config.plat_gpt_l0_base = l0_mem_base;
1173
1174 return 0;
1175}
1176
1177/*
1178 * Public API that carves out PAS regions from the L0 tables and builds any L1
1179 * tables that are needed. This function ideally is run after DDR discovery and
1180 * initialization. The L0 tables must have already been initialized to GPI_ANY
1181 * when this function is called.
1182 *
1183 * This function can be called multiple times with different L1 memory ranges
1184 * and PAS regions if it is desirable to place L1 tables in different locations
1185 * in memory. (ex: you have multiple DDR banks and want to place the L1 tables
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001186 * in the DDR bank that they control).
johpow019d134022021-06-16 17:57:28 -05001187 *
1188 * Parameters
1189 * pgs PGS value to use for table generation.
1190 * l1_mem_base Base address of memory used for L1 tables.
1191 * l1_mem_size Total size of memory available for L1 tables.
1192 * *pas_regions Pointer to PAS regions structure array.
1193 * pas_count Total number of PAS regions.
1194 *
1195 * Return
1196 * Negative Linux error code in the event of a failure, 0 for success.
1197 */
1198int gpt_init_pas_l1_tables(gpccr_pgs_e pgs, uintptr_t l1_mem_base,
1199 size_t l1_mem_size, pas_region_t *pas_regions,
1200 unsigned int pas_count)
1201{
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001202 int l1_gpt_cnt, ret;
johpow019d134022021-06-16 17:57:28 -05001203
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001204 /* Ensure that MMU and Data caches are enabled */
johpow019d134022021-06-16 17:57:28 -05001205 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0U);
1206
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001207 /* PGS is needed for validate_pas_mappings so check it now */
johpow019d134022021-06-16 17:57:28 -05001208 if (pgs > GPT_PGS_MAX) {
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001209 ERROR("GPT: Invalid PGS: 0x%x\n", pgs);
johpow019d134022021-06-16 17:57:28 -05001210 return -EINVAL;
1211 }
1212 gpt_config.pgs = pgs;
1213 gpt_config.p = gpt_p_lookup[pgs];
1214
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001215 /* Make sure L0 tables have been initialized */
johpow019d134022021-06-16 17:57:28 -05001216 if (gpt_config.plat_gpt_l0_base == 0U) {
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001217 ERROR("GPT: L0 tables must be initialized first!\n");
johpow019d134022021-06-16 17:57:28 -05001218 return -EPERM;
1219 }
1220
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001221 /* Check if L1 GPTs are required and how many */
AlexeiFedoroveb6f6cd2024-03-13 13:59:09 +00001222 l1_gpt_cnt = validate_pas_mappings(pas_regions, pas_count);
johpow019d134022021-06-16 17:57:28 -05001223 if (l1_gpt_cnt < 0) {
1224 return l1_gpt_cnt;
1225 }
1226
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001227 VERBOSE("GPT: %i L1 GPTs requested\n", l1_gpt_cnt);
johpow019d134022021-06-16 17:57:28 -05001228
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001229 /* If L1 tables are needed then validate the L1 parameters */
johpow019d134022021-06-16 17:57:28 -05001230 if (l1_gpt_cnt > 0) {
AlexeiFedoroveb6f6cd2024-03-13 13:59:09 +00001231 ret = validate_l1_params(l1_mem_base, l1_mem_size,
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001232 (unsigned int)l1_gpt_cnt);
Robert Wakim48e6b572021-10-21 15:39:56 +01001233 if (ret != 0) {
johpow019d134022021-06-16 17:57:28 -05001234 return ret;
1235 }
1236
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001237 /* Set up parameters for L1 table generation */
johpow019d134022021-06-16 17:57:28 -05001238 gpt_l1_tbl = l1_mem_base;
johpow019d134022021-06-16 17:57:28 -05001239 }
1240
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001241 /* Number of L1 entries in 2MB depends on GPCCR_EL3.PGS value */
1242 gpt_l1_cnt_2mb = (unsigned int)GPT_L1_ENTRY_COUNT_2MB(gpt_config.p);
1243
1244 /* Mask for the L1 index field */
1245 gpt_l1_index_mask = GPT_L1_IDX_MASK(gpt_config.p);
1246
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001247 INFO("GPT: Boot Configuration\n");
johpow019d134022021-06-16 17:57:28 -05001248 INFO(" PPS/T: 0x%x/%u\n", gpt_config.pps, gpt_config.t);
1249 INFO(" PGS/P: 0x%x/%u\n", gpt_config.pgs, gpt_config.p);
1250 INFO(" L0GPTSZ/S: 0x%x/%u\n", GPT_L0GPTSZ, GPT_S_VAL);
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001251 INFO(" PAS count: %u\n", pas_count);
1252 INFO(" L0 base: 0x%"PRIxPTR"\n", gpt_config.plat_gpt_l0_base);
johpow019d134022021-06-16 17:57:28 -05001253
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001254 /* Generate the tables in memory */
johpow019d134022021-06-16 17:57:28 -05001255 for (unsigned int idx = 0U; idx < pas_count; idx++) {
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001256 VERBOSE("GPT: PAS[%u]: base 0x%"PRIxPTR"\tsize 0x%lx\tGPI 0x%x\ttype 0x%x\n",
1257 idx, pas_regions[idx].base_pa, pas_regions[idx].size,
1258 GPT_PAS_ATTR_GPI(pas_regions[idx].attrs),
1259 GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs));
johpow019d134022021-06-16 17:57:28 -05001260
1261 /* Check if a block or table descriptor is required */
1262 if (GPT_PAS_ATTR_MAP_TYPE(pas_regions[idx].attrs) ==
1263 GPT_PAS_ATTR_MAP_TYPE_BLOCK) {
AlexeiFedoroveb6f6cd2024-03-13 13:59:09 +00001264 generate_l0_blk_desc(&pas_regions[idx]);
johpow019d134022021-06-16 17:57:28 -05001265
1266 } else {
AlexeiFedoroveb6f6cd2024-03-13 13:59:09 +00001267 generate_l0_tbl_desc(&pas_regions[idx]);
johpow019d134022021-06-16 17:57:28 -05001268 }
1269 }
1270
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001271 /* Flush modified L0 tables */
johpow019d134022021-06-16 17:57:28 -05001272 flush_l0_for_pas_array(pas_regions, pas_count);
1273
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001274 /* Flush L1 tables if needed */
johpow019d134022021-06-16 17:57:28 -05001275 if (l1_gpt_cnt > 0) {
1276 flush_dcache_range(l1_mem_base,
1277 GPT_L1_TABLE_SIZE(gpt_config.p) *
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001278 (size_t)l1_gpt_cnt);
johpow019d134022021-06-16 17:57:28 -05001279 }
1280
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001281 /* Make sure that all the entries are written to the memory */
johpow019d134022021-06-16 17:57:28 -05001282 dsbishst();
Soby Mathew521375d2021-10-11 14:38:46 +01001283 tlbipaallos();
1284 dsb();
1285 isb();
johpow019d134022021-06-16 17:57:28 -05001286
1287 return 0;
1288}
1289
1290/*
1291 * Public API to initialize the runtime gpt_config structure based on the values
1292 * present in the GPTBR_EL3 and GPCCR_EL3 registers. GPT initialization
1293 * typically happens in a bootloader stage prior to setting up the EL3 runtime
1294 * environment for the granule transition service so this function detects the
1295 * initialization from a previous stage. Granule protection checks must be
1296 * enabled already or this function will return an error.
1297 *
1298 * Return
1299 * Negative Linux error code in the event of a failure, 0 for success.
1300 */
1301int gpt_runtime_init(void)
1302{
1303 u_register_t reg;
1304
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001305 /* Ensure that MMU and Data caches are enabled */
johpow019d134022021-06-16 17:57:28 -05001306 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0U);
1307
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001308 /* Ensure GPC are already enabled */
johpow019d134022021-06-16 17:57:28 -05001309 if ((read_gpccr_el3() & GPCCR_GPC_BIT) == 0U) {
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001310 ERROR("GPT: Granule protection checks are not enabled!\n");
johpow019d134022021-06-16 17:57:28 -05001311 return -EPERM;
1312 }
1313
1314 /*
1315 * Read the L0 table address from GPTBR, we don't need the L1 base
1316 * address since those are included in the L0 tables as needed.
1317 */
1318 reg = read_gptbr_el3();
1319 gpt_config.plat_gpt_l0_base = ((reg >> GPTBR_BADDR_SHIFT) &
1320 GPTBR_BADDR_MASK) <<
1321 GPTBR_BADDR_VAL_SHIFT;
1322
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001323 /* Read GPCCR to get PGS and PPS values */
johpow019d134022021-06-16 17:57:28 -05001324 reg = read_gpccr_el3();
1325 gpt_config.pps = (reg >> GPCCR_PPS_SHIFT) & GPCCR_PPS_MASK;
1326 gpt_config.t = gpt_t_lookup[gpt_config.pps];
1327 gpt_config.pgs = (reg >> GPCCR_PGS_SHIFT) & GPCCR_PGS_MASK;
1328 gpt_config.p = gpt_p_lookup[gpt_config.pgs];
1329
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001330 /* Number of L1 entries in 2MB depends on GPCCR_EL3.PGS value */
1331 gpt_l1_cnt_2mb = (unsigned int)GPT_L1_ENTRY_COUNT_2MB(gpt_config.p);
1332
1333 /* Mask for the L1 index field */
1334 gpt_l1_index_mask = GPT_L1_IDX_MASK(gpt_config.p);
1335
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +01001336#if (RME_GPT_BITLOCK_BLOCK != 0)
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001337 /* Bitlocks at the end of L0 table */
1338 gpt_bitlock_base = (bitlock_t *)(gpt_config.plat_gpt_l0_base +
1339 GPT_L0_TABLE_SIZE(gpt_config.t));
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +01001340#endif
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001341 VERBOSE("GPT: Runtime Configuration\n");
johpow019d134022021-06-16 17:57:28 -05001342 VERBOSE(" PPS/T: 0x%x/%u\n", gpt_config.pps, gpt_config.t);
1343 VERBOSE(" PGS/P: 0x%x/%u\n", gpt_config.pgs, gpt_config.p);
1344 VERBOSE(" L0GPTSZ/S: 0x%x/%u\n", GPT_L0GPTSZ, GPT_S_VAL);
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001345 VERBOSE(" L0 base: 0x%"PRIxPTR"\n", gpt_config.plat_gpt_l0_base);
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +01001346#if (RME_GPT_BITLOCK_BLOCK != 0)
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001347 VERBOSE(" Bitlocks: 0x%"PRIxPTR"\n", (uintptr_t)gpt_bitlock_base);
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +01001348#endif
johpow019d134022021-06-16 17:57:28 -05001349 return 0;
1350}
1351
1352/*
Robert Wakim48e6b572021-10-21 15:39:56 +01001353 * A helper to write the value (target_pas << gpi_shift) to the index of
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001354 * the gpt_l1_addr.
Robert Wakim48e6b572021-10-21 15:39:56 +01001355 */
1356static inline void write_gpt(uint64_t *gpt_l1_desc, uint64_t *gpt_l1_addr,
1357 unsigned int gpi_shift, unsigned int idx,
1358 unsigned int target_pas)
1359{
1360 *gpt_l1_desc &= ~(GPT_L1_GRAN_DESC_GPI_MASK << gpi_shift);
1361 *gpt_l1_desc |= ((uint64_t)target_pas << gpi_shift);
1362 gpt_l1_addr[idx] = *gpt_l1_desc;
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001363
1364 dsboshst();
Robert Wakim48e6b572021-10-21 15:39:56 +01001365}
1366
1367/*
1368 * Helper to retrieve the gpt_l1_* information from the base address
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001369 * returned in gpi_info.
Robert Wakim48e6b572021-10-21 15:39:56 +01001370 */
1371static int get_gpi_params(uint64_t base, gpi_info_t *gpi_info)
1372{
1373 uint64_t gpt_l0_desc, *gpt_l0_base;
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +01001374 __unused unsigned int block_idx;
Robert Wakim48e6b572021-10-21 15:39:56 +01001375
1376 gpt_l0_base = (uint64_t *)gpt_config.plat_gpt_l0_base;
1377 gpt_l0_desc = gpt_l0_base[GPT_L0_IDX(base)];
1378 if (GPT_L0_TYPE(gpt_l0_desc) != GPT_L0_TYPE_TBL_DESC) {
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001379 VERBOSE("GPT: Granule is not covered by a table descriptor!\n");
1380 VERBOSE(" Base=0x%"PRIx64"\n", base);
Robert Wakim48e6b572021-10-21 15:39:56 +01001381 return -EINVAL;
1382 }
1383
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001384 /* Get the table index and GPI shift from PA */
Robert Wakim48e6b572021-10-21 15:39:56 +01001385 gpi_info->gpt_l1_addr = GPT_L0_TBLD_ADDR(gpt_l0_desc);
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001386 gpi_info->idx = (unsigned int)GPT_L1_INDEX(base);
Robert Wakim48e6b572021-10-21 15:39:56 +01001387 gpi_info->gpi_shift = GPT_L1_GPI_IDX(gpt_config.p, base) << 2;
1388
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +01001389#if (RME_GPT_BITLOCK_BLOCK != 0)
1390 /* Block index */
1391 block_idx = (unsigned int)(base / (RME_GPT_BITLOCK_BLOCK * SZ_512M));
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001392
1393 /* Bitlock address and mask */
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +01001394 gpi_info->lock = &gpt_bitlock_base[block_idx / LOCK_BITS];
1395 gpi_info->mask = 1U << (block_idx & (LOCK_BITS - 1U));
1396#endif
Robert Wakim48e6b572021-10-21 15:39:56 +01001397 return 0;
1398}
1399
1400/*
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001401 * Helper to retrieve the gpt_l1_desc and GPI information from gpi_info.
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +01001402 * This function is called with bitlock or spinlock acquired.
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001403 */
1404static void read_gpi(gpi_info_t *gpi_info)
1405{
1406 gpi_info->gpt_l1_desc = (gpi_info->gpt_l1_addr)[gpi_info->idx];
1407
1408 if ((gpi_info->gpt_l1_desc & GPT_L1_TYPE_CONT_DESC_MASK) ==
1409 GPT_L1_TYPE_CONT_DESC) {
1410 /* Read GPI from Contiguous descriptor */
1411 gpi_info->gpi = (unsigned int)GPT_L1_CONT_GPI(gpi_info->gpt_l1_desc);
1412 } else {
1413 /* Read GPI from Granules descriptor */
1414 gpi_info->gpi = (unsigned int)((gpi_info->gpt_l1_desc >> gpi_info->gpi_shift) &
1415 GPT_L1_GRAN_DESC_GPI_MASK);
1416 }
1417}
1418
1419static void flush_page_to_popa(uintptr_t addr)
1420{
1421 size_t size = GPT_PGS_ACTUAL_SIZE(gpt_config.p);
1422
1423 if (is_feat_mte2_supported()) {
1424 flush_dcache_to_popa_range_mte2(addr, size);
1425 } else {
1426 flush_dcache_to_popa_range(addr, size);
1427 }
1428}
1429
1430/*
1431 * Helper function to check if all L1 entries in 2MB block have
1432 * the same Granules descriptor value.
1433 *
1434 * Parameters
1435 * base Base address of the region to be checked
1436 * gpi_info Pointer to 'gpt_config_t' structure
1437 * l1_desc GPT Granules descriptor with all entries
1438 * set to the same GPI.
1439 *
1440 * Return
1441 * true if L1 all entries have the same descriptor value, false otherwise.
1442 */
1443__unused static bool check_fuse_2mb(uint64_t base, const gpi_info_t *gpi_info,
1444 uint64_t l1_desc)
1445{
1446 /* Last L1 entry index in 2MB block */
1447 unsigned int long idx = GPT_L1_INDEX(ALIGN_2MB(base)) +
1448 gpt_l1_cnt_2mb - 1UL;
1449
1450 /* Number of L1 entries in 2MB block */
1451 unsigned int cnt = gpt_l1_cnt_2mb;
1452
1453 /*
1454 * Start check from the last L1 entry and continue until the first
1455 * non-matching to the passed Granules descriptor value is found.
1456 */
1457 while (cnt-- != 0U) {
1458 if (gpi_info->gpt_l1_addr[idx--] != l1_desc) {
1459 /* Non-matching L1 entry found */
1460 return false;
1461 }
1462 }
1463
1464 return true;
1465}
1466
1467__unused static void fuse_2mb(uint64_t base, const gpi_info_t *gpi_info,
1468 uint64_t l1_desc)
1469{
1470 /* L1 entry index of the start of 2MB block */
1471 unsigned long idx_2 = GPT_L1_INDEX(ALIGN_2MB(base));
1472
1473 /* 2MB Contiguous descriptor */
1474 uint64_t l1_cont_desc = GPT_L1_CONT_DESC(l1_desc, 2MB);
1475
1476 VERBOSE("GPT: %s(0x%"PRIxPTR" 0x%"PRIx64")\n", __func__, base, l1_desc);
1477
1478 fill_desc(&gpi_info->gpt_l1_addr[idx_2], l1_cont_desc, L1_QWORDS_2MB);
1479}
1480
1481/*
1482 * Helper function to check if all 1st L1 entries of 2MB blocks
1483 * in 32MB have the same 2MB Contiguous descriptor value.
1484 *
1485 * Parameters
1486 * base Base address of the region to be checked
1487 * gpi_info Pointer to 'gpt_config_t' structure
1488 * l1_desc GPT Granules descriptor.
1489 *
1490 * Return
1491 * true if all L1 entries have the same descriptor value, false otherwise.
1492 */
1493__unused static bool check_fuse_32mb(uint64_t base, const gpi_info_t *gpi_info,
1494 uint64_t l1_desc)
1495{
1496 /* The 1st L1 entry index of the last 2MB block in 32MB */
1497 unsigned long idx = GPT_L1_INDEX(ALIGN_32MB(base)) +
1498 (15UL * gpt_l1_cnt_2mb);
1499
1500 /* 2MB Contiguous descriptor */
1501 uint64_t l1_cont_desc = GPT_L1_CONT_DESC(l1_desc, 2MB);
1502
1503 /* Number of 2MB blocks in 32MB */
1504 unsigned int cnt = 16U;
1505
1506 /* Set the first L1 entry to 2MB Contiguous descriptor */
1507 gpi_info->gpt_l1_addr[GPT_L1_INDEX(ALIGN_2MB(base))] = l1_cont_desc;
1508
1509 /*
1510 * Start check from the 1st L1 entry of the last 2MB block and
1511 * continue until the first non-matching to 2MB Contiguous descriptor
1512 * value is found.
1513 */
1514 while (cnt-- != 0U) {
1515 if (gpi_info->gpt_l1_addr[idx] != l1_cont_desc) {
1516 /* Non-matching L1 entry found */
1517 return false;
1518 }
1519 idx -= gpt_l1_cnt_2mb;
1520 }
1521
1522 return true;
1523}
1524
1525__unused static void fuse_32mb(uint64_t base, const gpi_info_t *gpi_info,
1526 uint64_t l1_desc)
1527{
1528 /* L1 entry index of the start of 32MB block */
1529 unsigned long idx_32 = GPT_L1_INDEX(ALIGN_32MB(base));
1530
1531 /* 32MB Contiguous descriptor */
1532 uint64_t l1_cont_desc = GPT_L1_CONT_DESC(l1_desc, 32MB);
1533
1534 VERBOSE("GPT: %s(0x%"PRIxPTR" 0x%"PRIx64")\n", __func__, base, l1_desc);
1535
1536 fill_desc(&gpi_info->gpt_l1_addr[idx_32], l1_cont_desc, L1_QWORDS_32MB);
1537}
1538
1539/*
1540 * Helper function to check if all 1st L1 entries of 32MB blocks
1541 * in 512MB have the same 32MB Contiguous descriptor value.
1542 *
1543 * Parameters
1544 * base Base address of the region to be checked
1545 * gpi_info Pointer to 'gpt_config_t' structure
1546 * l1_desc GPT Granules descriptor.
1547 *
1548 * Return
1549 * true if all L1 entries have the same descriptor value, false otherwise.
1550 */
1551__unused static bool check_fuse_512mb(uint64_t base, const gpi_info_t *gpi_info,
1552 uint64_t l1_desc)
1553{
1554 /* The 1st L1 entry index of the last 32MB block in 512MB */
1555 unsigned long idx = GPT_L1_INDEX(ALIGN_512MB(base)) +
1556 (15UL * 16UL * gpt_l1_cnt_2mb);
1557
1558 /* 32MB Contiguous descriptor */
1559 uint64_t l1_cont_desc = GPT_L1_CONT_DESC(l1_desc, 32MB);
1560
1561 /* Number of 32MB blocks in 512MB */
1562 unsigned int cnt = 16U;
1563
1564 /* Set the first L1 entry to 2MB Contiguous descriptor */
1565 gpi_info->gpt_l1_addr[GPT_L1_INDEX(ALIGN_32MB(base))] = l1_cont_desc;
1566
1567 /*
1568 * Start check from the 1st L1 entry of the last 32MB block and
1569 * continue until the first non-matching to 32MB Contiguous descriptor
1570 * value is found.
1571 */
1572 while (cnt-- != 0U) {
1573 if (gpi_info->gpt_l1_addr[idx] != l1_cont_desc) {
1574 /* Non-matching L1 entry found */
1575 return false;
1576 }
1577 idx -= 16UL * gpt_l1_cnt_2mb;
1578 }
1579
1580 return true;
1581}
1582
1583__unused static void fuse_512mb(uint64_t base, const gpi_info_t *gpi_info,
1584 uint64_t l1_desc)
1585{
1586 /* L1 entry index of the start of 512MB block */
1587 unsigned long idx_512 = GPT_L1_INDEX(ALIGN_512MB(base));
1588
1589 /* 512MB Contiguous descriptor */
1590 uint64_t l1_cont_desc = GPT_L1_CONT_DESC(l1_desc, 512MB);
1591
1592 VERBOSE("GPT: %s(0x%"PRIxPTR" 0x%"PRIx64")\n", __func__, base, l1_desc);
1593
1594 fill_desc(&gpi_info->gpt_l1_addr[idx_512], l1_cont_desc, L1_QWORDS_512MB);
1595}
1596
1597/*
1598 * Helper function to convert GPI entries in a single L1 table
1599 * from Granules to Contiguous descriptor.
1600 *
1601 * Parameters
1602 * base Base address of the region to be written
1603 * gpi_info Pointer to 'gpt_config_t' structure
1604 * l1_desc GPT Granules descriptor with all entries
1605 * set to the same GPI.
1606 */
1607__unused static void fuse_block(uint64_t base, const gpi_info_t *gpi_info,
1608 uint64_t l1_desc)
1609{
1610 /* Start with check for 2MB block */
1611 if (!check_fuse_2mb(base, gpi_info, l1_desc)) {
1612 /* Check for 2MB fusing failed */
1613 return;
1614 }
1615
1616#if (RME_GPT_MAX_BLOCK == 2)
1617 fuse_2mb(base, gpi_info, l1_desc);
1618#else
1619 /* Check for 32MB block */
1620 if (!check_fuse_32mb(base, gpi_info, l1_desc)) {
1621 /* Check for 32MB fusing failed, fuse to 2MB */
1622 fuse_2mb(base, gpi_info, l1_desc);
1623 return;
1624 }
1625
1626#if (RME_GPT_MAX_BLOCK == 32)
1627 fuse_32mb(base, gpi_info, l1_desc);
1628#else
1629 /* Check for 512MB block */
1630 if (!check_fuse_512mb(base, gpi_info, l1_desc)) {
1631 /* Check for 512MB fusing failed, fuse to 32MB */
1632 fuse_32mb(base, gpi_info, l1_desc);
1633 return;
1634 }
1635
1636 /* Fuse to 512MB */
1637 fuse_512mb(base, gpi_info, l1_desc);
1638
1639#endif /* RME_GPT_MAX_BLOCK == 32 */
1640#endif /* RME_GPT_MAX_BLOCK == 2 */
1641}
1642
1643/*
1644 * Helper function to convert GPI entries in a single L1 table
1645 * from Contiguous to Granules descriptor. This function updates
1646 * descriptor to Granules in passed 'gpt_config_t' structure as
1647 * the result of shuttering.
1648 *
1649 * Parameters
1650 * base Base address of the region to be written
1651 * gpi_info Pointer to 'gpt_config_t' structure
1652 * l1_desc GPT Granules descriptor set this range to.
1653 */
1654__unused static void shatter_block(uint64_t base, gpi_info_t *gpi_info,
1655 uint64_t l1_desc)
1656{
1657 /* Look-up table for 2MB, 32MB and 512MB locks shattering */
1658 static const gpt_shatter_func gpt_shatter_lookup[] = {
1659 shatter_2mb,
1660 shatter_32mb,
1661 shatter_512mb
1662 };
1663
1664 /* Look-up table for invalidation TLBs for 2MB, 32MB and 512MB blocks */
1665 static const gpt_tlbi_lookup_t tlbi_lookup[] = {
1666 { tlbirpalos_2m, ~(SZ_2M - 1UL) },
1667 { tlbirpalos_32m, ~(SZ_32M - 1UL) },
1668 { tlbirpalos_512m, ~(SZ_512M - 1UL) }
1669 };
1670
1671 /* Get shattering level from Contig field of Contiguous descriptor */
1672 unsigned long level = GPT_L1_CONT_CONTIG(gpi_info->gpt_l1_desc) - 1UL;
1673
1674 /* Shatter contiguous block */
1675 gpt_shatter_lookup[level](base, gpi_info, l1_desc);
1676
1677 tlbi_lookup[level].function(base & tlbi_lookup[level].mask);
1678 dsbosh();
1679
1680 /*
1681 * Update 'gpt_config_t' structure's descriptor to Granules to reflect
1682 * the shattered GPI back to caller.
1683 */
1684 gpi_info->gpt_l1_desc = l1_desc;
1685}
1686
1687/*
Robert Wakim48e6b572021-10-21 15:39:56 +01001688 * This function is the granule transition delegate service. When a granule
1689 * transition request occurs it is routed to this function to have the request,
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001690 * if valid, fulfilled following A1.1.1 Delegate of RME supplement.
johpow019d134022021-06-16 17:57:28 -05001691 *
Robert Wakim48e6b572021-10-21 15:39:56 +01001692 * TODO: implement support for transitioning multiple granules at once.
johpow019d134022021-06-16 17:57:28 -05001693 *
1694 * Parameters
Robert Wakim48e6b572021-10-21 15:39:56 +01001695 * base Base address of the region to transition, must be
1696 * aligned to granule size.
1697 * size Size of region to transition, must be aligned to granule
1698 * size.
johpow019d134022021-06-16 17:57:28 -05001699 * src_sec_state Security state of the caller.
johpow019d134022021-06-16 17:57:28 -05001700 *
1701 * Return
1702 * Negative Linux error code in the event of a failure, 0 for success.
1703 */
Robert Wakim48e6b572021-10-21 15:39:56 +01001704int gpt_delegate_pas(uint64_t base, size_t size, unsigned int src_sec_state)
johpow019d134022021-06-16 17:57:28 -05001705{
Robert Wakim48e6b572021-10-21 15:39:56 +01001706 gpi_info_t gpi_info;
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001707 uint64_t nse, __unused l1_desc;
Robert Wakim48e6b572021-10-21 15:39:56 +01001708 unsigned int target_pas;
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001709 int res;
Robert Wakim48e6b572021-10-21 15:39:56 +01001710
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001711 /* Ensure that the tables have been set up before taking requests */
Robert Wakim48e6b572021-10-21 15:39:56 +01001712 assert(gpt_config.plat_gpt_l0_base != 0UL);
johpow019d134022021-06-16 17:57:28 -05001713
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001714 /* Ensure that caches are enabled */
Robert Wakim48e6b572021-10-21 15:39:56 +01001715 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0UL);
1716
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001717 /* See if this is a single or a range of granule transition */
Robert Wakim48e6b572021-10-21 15:39:56 +01001718 if (size != GPT_PGS_ACTUAL_SIZE(gpt_config.p)) {
johpow019d134022021-06-16 17:57:28 -05001719 return -EINVAL;
1720 }
1721
Robert Wakim48e6b572021-10-21 15:39:56 +01001722 /* Check that base and size are valid */
1723 if ((ULONG_MAX - base) < size) {
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001724 VERBOSE("GPT: Transition request address overflow!\n");
1725 VERBOSE(" Base=0x%"PRIx64"\n", base);
Robert Wakim48e6b572021-10-21 15:39:56 +01001726 VERBOSE(" Size=0x%lx\n", size);
johpow019d134022021-06-16 17:57:28 -05001727 return -EINVAL;
1728 }
1729
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001730 /* Make sure base and size are valid */
1731 if (((base & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1UL)) != 0UL) ||
1732 ((size & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1UL)) != 0UL) ||
Robert Wakim48e6b572021-10-21 15:39:56 +01001733 (size == 0UL) ||
1734 ((base + size) >= GPT_PPS_ACTUAL_SIZE(gpt_config.t))) {
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001735 VERBOSE("GPT: Invalid granule transition address range!\n");
1736 VERBOSE(" Base=0x%"PRIx64"\n", base);
Robert Wakim48e6b572021-10-21 15:39:56 +01001737 VERBOSE(" Size=0x%lx\n", size);
johpow019d134022021-06-16 17:57:28 -05001738 return -EINVAL;
1739 }
Robert Wakim48e6b572021-10-21 15:39:56 +01001740
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001741 /* Delegate request can only come from REALM or SECURE */
1742 if ((src_sec_state != SMC_FROM_REALM) &&
1743 (src_sec_state != SMC_FROM_SECURE)) {
1744 VERBOSE("GPT: Invalid caller security state 0x%x\n",
1745 src_sec_state);
1746 return -EINVAL;
1747 }
1748
1749 if (src_sec_state == SMC_FROM_REALM) {
1750 target_pas = GPT_GPI_REALM;
1751 nse = (uint64_t)GPT_NSE_REALM << GPT_NSE_SHIFT;
1752 l1_desc = GPT_L1_REALM_DESC;
1753 } else {
Robert Wakim48e6b572021-10-21 15:39:56 +01001754 target_pas = GPT_GPI_SECURE;
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001755 nse = (uint64_t)GPT_NSE_SECURE << GPT_NSE_SHIFT;
1756 l1_desc = GPT_L1_SECURE_DESC;
Robert Wakim48e6b572021-10-21 15:39:56 +01001757 }
1758
Robert Wakim48e6b572021-10-21 15:39:56 +01001759 res = get_gpi_params(base, &gpi_info);
1760 if (res != 0) {
Robert Wakim48e6b572021-10-21 15:39:56 +01001761 return res;
1762 }
1763
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001764 /*
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +01001765 * Access to GPT is controlled by a lock to ensure that no more
1766 * than one CPU is allowed to make changes at any given time.
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001767 */
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +01001768 GPT_LOCK;
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001769 read_gpi(&gpi_info);
1770
Robert Wakim48e6b572021-10-21 15:39:56 +01001771 /* Check that the current address is in NS state */
1772 if (gpi_info.gpi != GPT_GPI_NS) {
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001773 VERBOSE("GPT: Only Granule in NS state can be delegated.\n");
Robert Wakim48e6b572021-10-21 15:39:56 +01001774 VERBOSE(" Caller: %u, Current GPI: %u\n", src_sec_state,
1775 gpi_info.gpi);
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +01001776 GPT_UNLOCK;
Javier Almansa Sobrinof809b162022-07-04 17:06:36 +01001777 return -EPERM;
johpow019d134022021-06-16 17:57:28 -05001778 }
1779
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001780#if (RME_GPT_MAX_BLOCK != 0)
1781 /* Check for Contiguous descriptor */
1782 if ((gpi_info.gpt_l1_desc & GPT_L1_TYPE_CONT_DESC_MASK) ==
1783 GPT_L1_TYPE_CONT_DESC) {
1784 shatter_block(base, &gpi_info, GPT_L1_NS_DESC);
Robert Wakim48e6b572021-10-21 15:39:56 +01001785 }
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001786#endif
Robert Wakim48e6b572021-10-21 15:39:56 +01001787 /*
1788 * In order to maintain mutual distrust between Realm and Secure
1789 * states, remove any data speculatively fetched into the target
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001790 * physical address space.
1791 * Issue DC CIPAPA or DC_CIGDPAPA on implementations with FEAT_MTE2.
Robert Wakim48e6b572021-10-21 15:39:56 +01001792 */
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001793 flush_page_to_popa(base | nse);
Robert Wakim48e6b572021-10-21 15:39:56 +01001794
1795 write_gpt(&gpi_info.gpt_l1_desc, gpi_info.gpt_l1_addr,
1796 gpi_info.gpi_shift, gpi_info.idx, target_pas);
Robert Wakim48e6b572021-10-21 15:39:56 +01001797
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001798 /* Ensure that all agents observe the new configuration */
1799 tlbi_page_dsbosh(base);
Robert Wakim48e6b572021-10-21 15:39:56 +01001800
1801 nse = (uint64_t)GPT_NSE_NS << GPT_NSE_SHIFT;
1802
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001803 /* Ensure that the scrubbed data have made it past the PoPA */
1804 flush_page_to_popa(base | nse);
1805
1806#if (RME_GPT_MAX_BLOCK != 0)
1807 if (gpi_info.gpt_l1_desc == l1_desc) {
1808 /* Try to fuse */
1809 fuse_block(base, &gpi_info, l1_desc);
Olivier Deprezc80d0de2024-01-17 15:12:04 +01001810 }
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001811#endif
Robert Wakim48e6b572021-10-21 15:39:56 +01001812
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +01001813 /* Unlock the lock to GPT */
1814 GPT_UNLOCK;
Robert Wakim48e6b572021-10-21 15:39:56 +01001815
1816 /*
1817 * The isb() will be done as part of context
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001818 * synchronization when returning to lower EL.
Robert Wakim48e6b572021-10-21 15:39:56 +01001819 */
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001820 VERBOSE("GPT: Granule 0x%"PRIx64" GPI 0x%x->0x%x\n",
Robert Wakim48e6b572021-10-21 15:39:56 +01001821 base, gpi_info.gpi, target_pas);
1822
johpow019d134022021-06-16 17:57:28 -05001823 return 0;
1824}
1825
1826/*
Robert Wakim48e6b572021-10-21 15:39:56 +01001827 * This function is the granule transition undelegate service. When a granule
johpow019d134022021-06-16 17:57:28 -05001828 * transition request occurs it is routed to this function where the request is
1829 * validated then fulfilled if possible.
1830 *
1831 * TODO: implement support for transitioning multiple granules at once.
1832 *
1833 * Parameters
1834 * base Base address of the region to transition, must be
1835 * aligned to granule size.
1836 * size Size of region to transition, must be aligned to granule
1837 * size.
1838 * src_sec_state Security state of the caller.
johpow019d134022021-06-16 17:57:28 -05001839 *
1840 * Return
1841 * Negative Linux error code in the event of a failure, 0 for success.
1842 */
Robert Wakim48e6b572021-10-21 15:39:56 +01001843int gpt_undelegate_pas(uint64_t base, size_t size, unsigned int src_sec_state)
johpow019d134022021-06-16 17:57:28 -05001844{
Robert Wakim48e6b572021-10-21 15:39:56 +01001845 gpi_info_t gpi_info;
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001846 uint64_t nse, __unused l1_desc;
Robert Wakim48e6b572021-10-21 15:39:56 +01001847 int res;
johpow019d134022021-06-16 17:57:28 -05001848
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001849 /* Ensure that the tables have been set up before taking requests */
Robert Wakim48e6b572021-10-21 15:39:56 +01001850 assert(gpt_config.plat_gpt_l0_base != 0UL);
johpow019d134022021-06-16 17:57:28 -05001851
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001852 /* Ensure that MMU and caches are enabled */
Robert Wakim48e6b572021-10-21 15:39:56 +01001853 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0UL);
1854
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001855 /* See if this is a single or a range of granule transition */
Robert Wakim48e6b572021-10-21 15:39:56 +01001856 if (size != GPT_PGS_ACTUAL_SIZE(gpt_config.p)) {
1857 return -EINVAL;
1858 }
1859
1860 /* Check that base and size are valid */
johpow019d134022021-06-16 17:57:28 -05001861 if ((ULONG_MAX - base) < size) {
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001862 VERBOSE("GPT: Transition request address overflow!\n");
1863 VERBOSE(" Base=0x%"PRIx64"\n", base);
johpow019d134022021-06-16 17:57:28 -05001864 VERBOSE(" Size=0x%lx\n", size);
1865 return -EINVAL;
1866 }
1867
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001868 /* Make sure base and size are valid */
1869 if (((base & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1UL)) != 0UL) ||
1870 ((size & (GPT_PGS_ACTUAL_SIZE(gpt_config.p) - 1UL)) != 0UL) ||
Robert Wakim48e6b572021-10-21 15:39:56 +01001871 (size == 0UL) ||
johpow019d134022021-06-16 17:57:28 -05001872 ((base + size) >= GPT_PPS_ACTUAL_SIZE(gpt_config.t))) {
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001873 VERBOSE("GPT: Invalid granule transition address range!\n");
1874 VERBOSE(" Base=0x%"PRIx64"\n", base);
johpow019d134022021-06-16 17:57:28 -05001875 VERBOSE(" Size=0x%lx\n", size);
1876 return -EINVAL;
1877 }
1878
Robert Wakim48e6b572021-10-21 15:39:56 +01001879 res = get_gpi_params(base, &gpi_info);
1880 if (res != 0) {
Robert Wakim48e6b572021-10-21 15:39:56 +01001881 return res;
1882 }
1883
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001884 /*
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +01001885 * Access to GPT is controlled by a lock to ensure that no more
1886 * than one CPU is allowed to make changes at any given time.
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001887 */
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +01001888 GPT_LOCK;
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001889 read_gpi(&gpi_info);
1890
Robert Wakim48e6b572021-10-21 15:39:56 +01001891 /* Check that the current address is in the delegated state */
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001892 if ((src_sec_state == SMC_FROM_REALM) &&
1893 (gpi_info.gpi == GPT_GPI_REALM)) {
1894 l1_desc = GPT_L1_REALM_DESC;
1895 nse = (uint64_t)GPT_NSE_REALM << GPT_NSE_SHIFT;
1896 } else if ((src_sec_state == SMC_FROM_SECURE) &&
1897 (gpi_info.gpi == GPT_GPI_SECURE)) {
1898 l1_desc = GPT_L1_SECURE_DESC;
1899 nse = (uint64_t)GPT_NSE_SECURE << GPT_NSE_SHIFT;
1900 } else {
1901 VERBOSE("GPT: Only Granule in REALM or SECURE state can be undelegated\n");
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001902 VERBOSE(" Caller: %u Current GPI: %u\n", src_sec_state,
Robert Wakim48e6b572021-10-21 15:39:56 +01001903 gpi_info.gpi);
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +01001904 GPT_UNLOCK;
Javier Almansa Sobrinof809b162022-07-04 17:06:36 +01001905 return -EPERM;
johpow019d134022021-06-16 17:57:28 -05001906 }
1907
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001908#if (RME_GPT_MAX_BLOCK != 0)
1909 /* Check for Contiguous descriptor */
1910 if ((gpi_info.gpt_l1_desc & GPT_L1_TYPE_CONT_DESC_MASK) ==
1911 GPT_L1_TYPE_CONT_DESC) {
1912 shatter_block(base, &gpi_info, l1_desc);
1913 }
1914#endif
1915 /*
1916 * In order to maintain mutual distrust between Realm and Secure
Robert Wakim48e6b572021-10-21 15:39:56 +01001917 * states, remove access now, in order to guarantee that writes
1918 * to the currently-accessible physical address space will not
1919 * later become observable.
1920 */
1921 write_gpt(&gpi_info.gpt_l1_desc, gpi_info.gpt_l1_addr,
1922 gpi_info.gpi_shift, gpi_info.idx, GPT_GPI_NO_ACCESS);
Robert Wakim48e6b572021-10-21 15:39:56 +01001923
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001924 /* Ensure that all agents observe the new NO_ACCESS configuration */
1925 tlbi_page_dsbosh(base);
Robert Wakim48e6b572021-10-21 15:39:56 +01001926
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001927 /* Ensure that the scrubbed data have made it past the PoPA */
1928 flush_page_to_popa(base | nse);
Robert Wakim48e6b572021-10-21 15:39:56 +01001929
1930 /*
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001931 * Remove any data loaded speculatively in NS space from before
1932 * the scrubbing.
Robert Wakim48e6b572021-10-21 15:39:56 +01001933 */
1934 nse = (uint64_t)GPT_NSE_NS << GPT_NSE_SHIFT;
1935
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001936 flush_page_to_popa(base | nse);
Robert Wakim48e6b572021-10-21 15:39:56 +01001937
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001938 /* Clear existing GPI encoding and transition granule */
Robert Wakim48e6b572021-10-21 15:39:56 +01001939 write_gpt(&gpi_info.gpt_l1_desc, gpi_info.gpt_l1_addr,
1940 gpi_info.gpi_shift, gpi_info.idx, GPT_GPI_NS);
johpow019d134022021-06-16 17:57:28 -05001941
Robert Wakim48e6b572021-10-21 15:39:56 +01001942 /* Ensure that all agents observe the new NS configuration */
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001943 tlbi_page_dsbosh(base);
johpow019d134022021-06-16 17:57:28 -05001944
AlexeiFedorovbd8b1bb2024-03-13 17:07:03 +00001945#if (RME_GPT_MAX_BLOCK != 0)
1946 if (gpi_info.gpt_l1_desc == GPT_L1_NS_DESC) {
1947 /* Try to fuse */
1948 fuse_block(base, &gpi_info, GPT_L1_NS_DESC);
1949 }
1950#endif
AlexeiFedorovc0ca2d72024-05-13 15:35:54 +01001951 /* Unlock the lock to GPT */
1952 GPT_UNLOCK;
johpow019d134022021-06-16 17:57:28 -05001953
Soby Mathew521375d2021-10-11 14:38:46 +01001954 /*
1955 * The isb() will be done as part of context
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001956 * synchronization when returning to lower EL.
Soby Mathew521375d2021-10-11 14:38:46 +01001957 */
AlexeiFedorov7eaaac72024-03-13 15:18:02 +00001958 VERBOSE("GPT: Granule 0x%"PRIx64" GPI 0x%x->0x%x\n",
Robert Wakim48e6b572021-10-21 15:39:56 +01001959 base, gpi_info.gpi, GPT_GPI_NS);
johpow019d134022021-06-16 17:57:28 -05001960
1961 return 0;
1962}