blob: 8be6d942d46bfd7344d695a81191ae7ad72d9eaf [file] [log] [blame]
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001/*
Antonio Nino Diaz3f518922018-01-05 11:30:36 +00002 * Copyright (c) 2017-2018, ARM Limited and Contributors. All rights reserved.
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00003 *
dp-armfa3cf0b2017-05-03 09:38:09 +01004 * SPDX-License-Identifier: BSD-3-Clause
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00005 */
6
7#include <arch.h>
8#include <arch_helpers.h>
9#include <assert.h>
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +000010#include <common_def.h>
11#include <debug.h>
12#include <errno.h>
13#include <platform_def.h>
14#include <string.h>
15#include <types.h>
16#include <utils.h>
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +010017#include <xlat_tables_arch_private.h>
Sandrine Bailleux090c8492017-05-19 09:59:37 +010018#include <xlat_tables_defs.h>
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +000019#include <xlat_tables_v2.h>
Sandrine Bailleux090c8492017-05-19 09:59:37 +010020
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +000021#include "xlat_tables_private.h"
22
Sandrine Bailleux66342932017-07-18 13:26:36 +010023/*
24 * Each platform can define the size of its physical and virtual address spaces.
25 * If the platform hasn't defined one or both of them, default to
26 * ADDR_SPACE_SIZE. The latter is deprecated, though.
27 */
28#if ERROR_DEPRECATED
29# ifdef ADDR_SPACE_SIZE
30# error "ADDR_SPACE_SIZE is deprecated. Use PLAT_xxx_ADDR_SPACE_SIZE instead."
31# endif
32#elif defined(ADDR_SPACE_SIZE)
33# ifndef PLAT_PHY_ADDR_SPACE_SIZE
34# define PLAT_PHY_ADDR_SPACE_SIZE ADDR_SPACE_SIZE
35# endif
36# ifndef PLAT_VIRT_ADDR_SPACE_SIZE
37# define PLAT_VIRT_ADDR_SPACE_SIZE ADDR_SPACE_SIZE
38# endif
39#endif
40
41/*
42 * Allocate and initialise the default translation context for the BL image
43 * currently executing.
44 */
45REGISTER_XLAT_CONTEXT(tf, MAX_MMAP_REGIONS, MAX_XLAT_TABLES,
46 PLAT_VIRT_ADDR_SPACE_SIZE, PLAT_PHY_ADDR_SPACE_SIZE);
47
Antonio Nino Diazac998032017-02-27 17:23:54 +000048#if PLAT_XLAT_TABLES_DYNAMIC
49
50/*
51 * The following functions assume that they will be called using subtables only.
52 * The base table can't be unmapped, so it is not needed to do any special
53 * handling for it.
54 */
55
56/*
57 * Returns the index of the array corresponding to the specified translation
58 * table.
59 */
60static int xlat_table_get_index(xlat_ctx_t *ctx, const uint64_t *table)
61{
Varun Wadekar66231d12017-06-07 09:57:42 -070062 for (unsigned int i = 0; i < ctx->tables_num; i++)
Antonio Nino Diazac998032017-02-27 17:23:54 +000063 if (ctx->tables[i] == table)
64 return i;
65
66 /*
67 * Maybe we were asked to get the index of the base level table, which
68 * should never happen.
69 */
70 assert(0);
71
72 return -1;
73}
74
75/* Returns a pointer to an empty translation table. */
76static uint64_t *xlat_table_get_empty(xlat_ctx_t *ctx)
77{
Varun Wadekar66231d12017-06-07 09:57:42 -070078 for (unsigned int i = 0; i < ctx->tables_num; i++)
Antonio Nino Diazac998032017-02-27 17:23:54 +000079 if (ctx->tables_mapped_regions[i] == 0)
80 return ctx->tables[i];
81
82 return NULL;
83}
84
85/* Increments region count for a given table. */
86static void xlat_table_inc_regions_count(xlat_ctx_t *ctx, const uint64_t *table)
87{
88 ctx->tables_mapped_regions[xlat_table_get_index(ctx, table)]++;
89}
90
91/* Decrements region count for a given table. */
92static void xlat_table_dec_regions_count(xlat_ctx_t *ctx, const uint64_t *table)
93{
94 ctx->tables_mapped_regions[xlat_table_get_index(ctx, table)]--;
95}
96
97/* Returns 0 if the speficied table isn't empty, otherwise 1. */
98static int xlat_table_is_empty(xlat_ctx_t *ctx, const uint64_t *table)
99{
100 return !ctx->tables_mapped_regions[xlat_table_get_index(ctx, table)];
101}
102
103#else /* PLAT_XLAT_TABLES_DYNAMIC */
104
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000105/* Returns a pointer to the first empty translation table. */
106static uint64_t *xlat_table_get_empty(xlat_ctx_t *ctx)
107{
108 assert(ctx->next_table < ctx->tables_num);
109
110 return ctx->tables[ctx->next_table++];
111}
112
Antonio Nino Diazac998032017-02-27 17:23:54 +0000113#endif /* PLAT_XLAT_TABLES_DYNAMIC */
114
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +0100115/*
116 * Returns a block/page table descriptor for the given level and attributes.
117 */
Antonio Nino Diaze8811472018-04-17 15:10:18 +0100118static uint64_t xlat_desc(const xlat_ctx_t *ctx, uint32_t attr,
119 unsigned long long addr_pa, int level)
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000120{
121 uint64_t desc;
122 int mem_type;
123
124 /* Make sure that the granularity is fine enough to map this address. */
125 assert((addr_pa & XLAT_BLOCK_MASK(level)) == 0);
126
127 desc = addr_pa;
128 /*
129 * There are different translation table descriptors for level 3 and the
130 * rest.
131 */
132 desc |= (level == XLAT_TABLE_LEVEL_MAX) ? PAGE_DESC : BLOCK_DESC;
133 /*
134 * Always set the access flag, as TF doesn't manage access flag faults.
135 * Deduce other fields of the descriptor based on the MT_NS and MT_RW
136 * memory region attributes.
137 */
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +0100138 desc |= LOWER_ATTRS(ACCESS_FLAG);
139
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000140 desc |= (attr & MT_NS) ? LOWER_ATTRS(NS) : 0;
141 desc |= (attr & MT_RW) ? LOWER_ATTRS(AP_RW) : LOWER_ATTRS(AP_RO);
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000142
143 /*
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +0100144 * Do not allow unprivileged access when the mapping is for a privileged
145 * EL. For translation regimes that do not have mappings for access for
146 * lower exception levels, set AP[2] to AP_NO_ACCESS_UNPRIVILEGED.
147 */
148 if (ctx->xlat_regime == EL1_EL0_REGIME) {
149 if (attr & MT_USER) {
150 /* EL0 mapping requested, so we give User access */
151 desc |= LOWER_ATTRS(AP_ACCESS_UNPRIVILEGED);
152 } else {
153 /* EL1 mapping requested, no User access granted */
154 desc |= LOWER_ATTRS(AP_NO_ACCESS_UNPRIVILEGED);
155 }
156 } else {
157 assert(ctx->xlat_regime == EL3_REGIME);
Antonio Nino Diaz49074492018-04-26 12:59:08 +0100158 desc |= LOWER_ATTRS(AP_ONE_VA_RANGE_RES1);
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +0100159 }
160
161 /*
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000162 * Deduce shareability domain and executability of the memory region
163 * from the memory type of the attributes (MT_TYPE).
164 *
165 * Data accesses to device memory and non-cacheable normal memory are
166 * coherent for all observers in the system, and correspondingly are
167 * always treated as being Outer Shareable. Therefore, for these 2 types
168 * of memory, it is not strictly needed to set the shareability field
169 * in the translation tables.
170 */
171 mem_type = MT_TYPE(attr);
172 if (mem_type == MT_DEVICE) {
173 desc |= LOWER_ATTRS(ATTR_DEVICE_INDEX | OSH);
174 /*
175 * Always map device memory as execute-never.
176 * This is to avoid the possibility of a speculative instruction
177 * fetch, which could be an issue if this memory region
178 * corresponds to a read-sensitive peripheral.
179 */
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +0100180 desc |= xlat_arch_regime_get_xn_desc(ctx->xlat_regime);
Antonio Nino Diazefabaa92017-04-27 13:30:22 +0100181
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000182 } else { /* Normal memory */
183 /*
184 * Always map read-write normal memory as execute-never.
185 * (Trusted Firmware doesn't self-modify its code, therefore
186 * R/W memory is reserved for data storage, which must not be
187 * executable.)
188 * Note that setting the XN bit here is for consistency only.
Antonio Nino Diazefabaa92017-04-27 13:30:22 +0100189 * The function that enables the MMU sets the SCTLR_ELx.WXN bit,
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000190 * which makes any writable memory region to be treated as
191 * execute-never, regardless of the value of the XN bit in the
192 * translation table.
193 *
194 * For read-only memory, rely on the MT_EXECUTE/MT_EXECUTE_NEVER
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +0100195 * attribute to figure out the value of the XN bit. The actual
196 * XN bit(s) to set in the descriptor depends on the context's
197 * translation regime and the policy applied in
198 * xlat_arch_regime_get_xn_desc().
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000199 */
Antonio Nino Diazefabaa92017-04-27 13:30:22 +0100200 if ((attr & MT_RW) || (attr & MT_EXECUTE_NEVER)) {
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +0100201 desc |= xlat_arch_regime_get_xn_desc(ctx->xlat_regime);
Antonio Nino Diazefabaa92017-04-27 13:30:22 +0100202 }
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000203
204 if (mem_type == MT_MEMORY) {
205 desc |= LOWER_ATTRS(ATTR_IWBWA_OWBWA_NTR_INDEX | ISH);
206 } else {
207 assert(mem_type == MT_NON_CACHEABLE);
208 desc |= LOWER_ATTRS(ATTR_NON_CACHEABLE_INDEX | OSH);
209 }
210 }
211
212 return desc;
213}
214
215/*
216 * Enumeration of actions that can be made when mapping table entries depending
217 * on the previous value in that entry and information about the region being
218 * mapped.
219 */
220typedef enum {
221
222 /* Do nothing */
223 ACTION_NONE,
224
225 /* Write a block (or page, if in level 3) entry. */
226 ACTION_WRITE_BLOCK_ENTRY,
227
228 /*
229 * Create a new table and write a table entry pointing to it. Recurse
230 * into it for further processing.
231 */
232 ACTION_CREATE_NEW_TABLE,
233
234 /*
235 * There is a table descriptor in this entry, read it and recurse into
236 * that table for further processing.
237 */
238 ACTION_RECURSE_INTO_TABLE,
239
240} action_t;
241
Antonio Nino Diazac998032017-02-27 17:23:54 +0000242#if PLAT_XLAT_TABLES_DYNAMIC
243
244/*
245 * Recursive function that writes to the translation tables and unmaps the
246 * specified region.
247 */
248static void xlat_tables_unmap_region(xlat_ctx_t *ctx, mmap_region_t *mm,
249 const uintptr_t table_base_va,
250 uint64_t *const table_base,
251 const int table_entries,
Varun Wadekar66231d12017-06-07 09:57:42 -0700252 const unsigned int level)
Antonio Nino Diazac998032017-02-27 17:23:54 +0000253{
254 assert(level >= ctx->base_level && level <= XLAT_TABLE_LEVEL_MAX);
255
256 uint64_t *subtable;
257 uint64_t desc;
258
259 uintptr_t table_idx_va;
260 uintptr_t table_idx_end_va; /* End VA of this entry */
261
262 uintptr_t region_end_va = mm->base_va + mm->size - 1;
263
264 int table_idx;
265
266 if (mm->base_va > table_base_va) {
267 /* Find the first index of the table affected by the region. */
268 table_idx_va = mm->base_va & ~XLAT_BLOCK_MASK(level);
269
270 table_idx = (table_idx_va - table_base_va) >>
271 XLAT_ADDR_SHIFT(level);
272
273 assert(table_idx < table_entries);
274 } else {
275 /* Start from the beginning of the table. */
276 table_idx_va = table_base_va;
277 table_idx = 0;
278 }
279
280 while (table_idx < table_entries) {
281
282 table_idx_end_va = table_idx_va + XLAT_BLOCK_SIZE(level) - 1;
283
284 desc = table_base[table_idx];
285 uint64_t desc_type = desc & DESC_MASK;
286
287 action_t action = ACTION_NONE;
288
289 if ((mm->base_va <= table_idx_va) &&
290 (region_end_va >= table_idx_end_va)) {
291
292 /* Region covers all block */
293
294 if (level == 3) {
295 /*
296 * Last level, only page descriptors allowed,
297 * erase it.
298 */
299 assert(desc_type == PAGE_DESC);
300
301 action = ACTION_WRITE_BLOCK_ENTRY;
302 } else {
303 /*
304 * Other levels can have table descriptors. If
305 * so, recurse into it and erase descriptors
306 * inside it as needed. If there is a block
307 * descriptor, just erase it. If an invalid
308 * descriptor is found, this table isn't
309 * actually mapped, which shouldn't happen.
310 */
311 if (desc_type == TABLE_DESC) {
312 action = ACTION_RECURSE_INTO_TABLE;
313 } else {
314 assert(desc_type == BLOCK_DESC);
315 action = ACTION_WRITE_BLOCK_ENTRY;
316 }
317 }
318
319 } else if ((mm->base_va <= table_idx_end_va) ||
320 (region_end_va >= table_idx_va)) {
321
322 /*
323 * Region partially covers block.
324 *
325 * It can't happen in level 3.
326 *
327 * There must be a table descriptor here, if not there
328 * was a problem when mapping the region.
329 */
330
331 assert(level < 3);
332
333 assert(desc_type == TABLE_DESC);
334
335 action = ACTION_RECURSE_INTO_TABLE;
336 }
337
338 if (action == ACTION_WRITE_BLOCK_ENTRY) {
339
340 table_base[table_idx] = INVALID_DESC;
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +0100341 xlat_arch_tlbi_va_regime(table_idx_va, ctx->xlat_regime);
Antonio Nino Diazac998032017-02-27 17:23:54 +0000342
343 } else if (action == ACTION_RECURSE_INTO_TABLE) {
344
345 subtable = (uint64_t *)(uintptr_t)(desc & TABLE_ADDR_MASK);
346
347 /* Recurse to write into subtable */
348 xlat_tables_unmap_region(ctx, mm, table_idx_va,
349 subtable, XLAT_TABLE_ENTRIES,
350 level + 1);
351
352 /*
353 * If the subtable is now empty, remove its reference.
354 */
355 if (xlat_table_is_empty(ctx, subtable)) {
356 table_base[table_idx] = INVALID_DESC;
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +0100357 xlat_arch_tlbi_va_regime(table_idx_va,
358 ctx->xlat_regime);
Antonio Nino Diazac998032017-02-27 17:23:54 +0000359 }
360
361 } else {
362 assert(action == ACTION_NONE);
363 }
364
365 table_idx++;
366 table_idx_va += XLAT_BLOCK_SIZE(level);
367
368 /* If reached the end of the region, exit */
369 if (region_end_va <= table_idx_va)
370 break;
371 }
372
373 if (level > ctx->base_level)
374 xlat_table_dec_regions_count(ctx, table_base);
375}
376
377#endif /* PLAT_XLAT_TABLES_DYNAMIC */
378
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000379/*
380 * From the given arguments, it decides which action to take when mapping the
381 * specified region.
382 */
383static action_t xlat_tables_map_region_action(const mmap_region_t *mm,
384 const int desc_type, const unsigned long long dest_pa,
Sandrine Bailleux12e86442017-07-19 10:11:13 +0100385 const uintptr_t table_entry_base_va, const unsigned int level)
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000386{
387 uintptr_t mm_end_va = mm->base_va + mm->size - 1;
388 uintptr_t table_entry_end_va =
389 table_entry_base_va + XLAT_BLOCK_SIZE(level) - 1;
390
391 /*
392 * The descriptor types allowed depend on the current table level.
393 */
394
395 if ((mm->base_va <= table_entry_base_va) &&
396 (mm_end_va >= table_entry_end_va)) {
397
398 /*
399 * Table entry is covered by region
400 * --------------------------------
401 *
402 * This means that this table entry can describe the whole
403 * translation with this granularity in principle.
404 */
405
406 if (level == 3) {
407 /*
408 * Last level, only page descriptors are allowed.
409 */
410 if (desc_type == PAGE_DESC) {
411 /*
412 * There's another region mapped here, don't
413 * overwrite.
414 */
415 return ACTION_NONE;
416 } else {
417 assert(desc_type == INVALID_DESC);
418 return ACTION_WRITE_BLOCK_ENTRY;
419 }
420
421 } else {
422
423 /*
424 * Other levels. Table descriptors are allowed. Block
425 * descriptors too, but they have some limitations.
426 */
427
428 if (desc_type == TABLE_DESC) {
429 /* There's already a table, recurse into it. */
430 return ACTION_RECURSE_INTO_TABLE;
431
432 } else if (desc_type == INVALID_DESC) {
433 /*
434 * There's nothing mapped here, create a new
435 * entry.
436 *
437 * Check if the destination granularity allows
438 * us to use a block descriptor or we need a
439 * finer table for it.
440 *
441 * Also, check if the current level allows block
442 * descriptors. If not, create a table instead.
443 */
444 if ((dest_pa & XLAT_BLOCK_MASK(level)) ||
Sandrine Bailleux8f23fa82017-09-28 21:58:12 +0100445 (level < MIN_LVL_BLOCK_DESC) ||
446 (mm->granularity < XLAT_BLOCK_SIZE(level)))
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000447 return ACTION_CREATE_NEW_TABLE;
448 else
449 return ACTION_WRITE_BLOCK_ENTRY;
450
451 } else {
452 /*
453 * There's another region mapped here, don't
454 * overwrite.
455 */
456 assert(desc_type == BLOCK_DESC);
457
458 return ACTION_NONE;
459 }
460 }
461
462 } else if ((mm->base_va <= table_entry_end_va) ||
463 (mm_end_va >= table_entry_base_va)) {
464
465 /*
466 * Region partially covers table entry
467 * -----------------------------------
468 *
469 * This means that this table entry can't describe the whole
470 * translation, a finer table is needed.
471
472 * There cannot be partial block overlaps in level 3. If that
473 * happens, some of the preliminary checks when adding the
474 * mmap region failed to detect that PA and VA must at least be
475 * aligned to PAGE_SIZE.
476 */
477 assert(level < 3);
478
479 if (desc_type == INVALID_DESC) {
480 /*
481 * The block is not fully covered by the region. Create
482 * a new table, recurse into it and try to map the
483 * region with finer granularity.
484 */
485 return ACTION_CREATE_NEW_TABLE;
486
487 } else {
488 assert(desc_type == TABLE_DESC);
489 /*
490 * The block is not fully covered by the region, but
491 * there is already a table here. Recurse into it and
492 * try to map with finer granularity.
493 *
494 * PAGE_DESC for level 3 has the same value as
495 * TABLE_DESC, but this code can't run on a level 3
496 * table because there can't be overlaps in level 3.
497 */
498 return ACTION_RECURSE_INTO_TABLE;
499 }
500 }
501
502 /*
503 * This table entry is outside of the region specified in the arguments,
504 * don't write anything to it.
505 */
506 return ACTION_NONE;
507}
508
509/*
510 * Recursive function that writes to the translation tables and maps the
Antonio Nino Diazac998032017-02-27 17:23:54 +0000511 * specified region. On success, it returns the VA of the last byte that was
512 * succesfully mapped. On error, it returns the VA of the next entry that
513 * should have been mapped.
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000514 */
Antonio Nino Diazac998032017-02-27 17:23:54 +0000515static uintptr_t xlat_tables_map_region(xlat_ctx_t *ctx, mmap_region_t *mm,
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000516 const uintptr_t table_base_va,
517 uint64_t *const table_base,
518 const int table_entries,
Varun Wadekar66231d12017-06-07 09:57:42 -0700519 const unsigned int level)
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000520{
521 assert(level >= ctx->base_level && level <= XLAT_TABLE_LEVEL_MAX);
522
523 uintptr_t mm_end_va = mm->base_va + mm->size - 1;
524
525 uintptr_t table_idx_va;
526 unsigned long long table_idx_pa;
527
528 uint64_t *subtable;
529 uint64_t desc;
530
531 int table_idx;
532
533 if (mm->base_va > table_base_va) {
534 /* Find the first index of the table affected by the region. */
535 table_idx_va = mm->base_va & ~XLAT_BLOCK_MASK(level);
536
537 table_idx = (table_idx_va - table_base_va) >>
538 XLAT_ADDR_SHIFT(level);
539
540 assert(table_idx < table_entries);
541 } else {
542 /* Start from the beginning of the table. */
543 table_idx_va = table_base_va;
544 table_idx = 0;
545 }
546
Antonio Nino Diazac998032017-02-27 17:23:54 +0000547#if PLAT_XLAT_TABLES_DYNAMIC
548 if (level > ctx->base_level)
549 xlat_table_inc_regions_count(ctx, table_base);
550#endif
551
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000552 while (table_idx < table_entries) {
553
554 desc = table_base[table_idx];
555
556 table_idx_pa = mm->base_pa + table_idx_va - mm->base_va;
557
558 action_t action = xlat_tables_map_region_action(mm,
559 desc & DESC_MASK, table_idx_pa, table_idx_va, level);
560
561 if (action == ACTION_WRITE_BLOCK_ENTRY) {
562
563 table_base[table_idx] =
Antonio Nino Diaze8811472018-04-17 15:10:18 +0100564 xlat_desc(ctx, (uint32_t)mm->attr, table_idx_pa,
565 level);
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000566
567 } else if (action == ACTION_CREATE_NEW_TABLE) {
568
569 subtable = xlat_table_get_empty(ctx);
Antonio Nino Diazac998032017-02-27 17:23:54 +0000570 if (subtable == NULL) {
571 /* Not enough free tables to map this region */
572 return table_idx_va;
573 }
574
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000575 /* Point to new subtable from this one. */
Antonio Nino Diazac998032017-02-27 17:23:54 +0000576 table_base[table_idx] = TABLE_DESC | (unsigned long)subtable;
577
578 /* Recurse to write into subtable */
579 uintptr_t end_va = xlat_tables_map_region(ctx, mm, table_idx_va,
580 subtable, XLAT_TABLE_ENTRIES,
581 level + 1);
582 if (end_va != table_idx_va + XLAT_BLOCK_SIZE(level) - 1)
583 return end_va;
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000584
585 } else if (action == ACTION_RECURSE_INTO_TABLE) {
586
587 subtable = (uint64_t *)(uintptr_t)(desc & TABLE_ADDR_MASK);
588 /* Recurse to write into subtable */
Antonio Nino Diazac998032017-02-27 17:23:54 +0000589 uintptr_t end_va = xlat_tables_map_region(ctx, mm, table_idx_va,
590 subtable, XLAT_TABLE_ENTRIES,
591 level + 1);
592 if (end_va != table_idx_va + XLAT_BLOCK_SIZE(level) - 1)
593 return end_va;
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000594
595 } else {
596
597 assert(action == ACTION_NONE);
598
599 }
600
601 table_idx++;
602 table_idx_va += XLAT_BLOCK_SIZE(level);
603
604 /* If reached the end of the region, exit */
605 if (mm_end_va <= table_idx_va)
606 break;
607 }
Antonio Nino Diazac998032017-02-27 17:23:54 +0000608
609 return table_idx_va - 1;
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000610}
611
612void print_mmap(mmap_region_t *const mmap)
613{
614#if LOG_LEVEL >= LOG_LEVEL_VERBOSE
615 tf_printf("mmap:\n");
616 mmap_region_t *mm = mmap;
617
618 while (mm->size) {
Sandrine Bailleux8f23fa82017-09-28 21:58:12 +0100619 tf_printf(" VA:%p PA:0x%llx size:0x%zx attr:0x%x",
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000620 (void *)mm->base_va, mm->base_pa,
621 mm->size, mm->attr);
Sandrine Bailleux8f23fa82017-09-28 21:58:12 +0100622 tf_printf(" granularity:0x%zx\n", mm->granularity);
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000623 ++mm;
624 };
625 tf_printf("\n");
626#endif
627}
628
629/*
630 * Function that verifies that a region can be mapped.
631 * Returns:
632 * 0: Success, the mapping is allowed.
633 * EINVAL: Invalid values were used as arguments.
634 * ERANGE: The memory limits were surpassed.
635 * ENOMEM: There is not enough memory in the mmap array.
636 * EPERM: Region overlaps another one in an invalid way.
637 */
Douglas Raillard6a5f8f12017-09-21 08:42:21 +0100638static int mmap_add_region_check(xlat_ctx_t *ctx, const mmap_region_t *mm)
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000639{
Douglas Raillard6a5f8f12017-09-21 08:42:21 +0100640 unsigned long long base_pa = mm->base_pa;
641 uintptr_t base_va = mm->base_va;
642 size_t size = mm->size;
Sandrine Bailleux8f23fa82017-09-28 21:58:12 +0100643 size_t granularity = mm->granularity;
Douglas Raillard6a5f8f12017-09-21 08:42:21 +0100644
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000645 unsigned long long end_pa = base_pa + size - 1;
646 uintptr_t end_va = base_va + size - 1;
647
648 if (!IS_PAGE_ALIGNED(base_pa) || !IS_PAGE_ALIGNED(base_va) ||
649 !IS_PAGE_ALIGNED(size))
650 return -EINVAL;
651
Sandrine Bailleux8f23fa82017-09-28 21:58:12 +0100652 if ((granularity != XLAT_BLOCK_SIZE(1)) &&
653 (granularity != XLAT_BLOCK_SIZE(2)) &&
654 (granularity != XLAT_BLOCK_SIZE(3))) {
655 return -EINVAL;
656 }
657
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000658 /* Check for overflows */
659 if ((base_pa > end_pa) || (base_va > end_va))
660 return -ERANGE;
661
662 if ((base_va + (uintptr_t)size - (uintptr_t)1) > ctx->va_max_address)
663 return -ERANGE;
664
665 if ((base_pa + (unsigned long long)size - 1ULL) > ctx->pa_max_address)
666 return -ERANGE;
667
Douglas Raillard6a5f8f12017-09-21 08:42:21 +0100668 /* Check that there is space in the ctx->mmap array */
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000669 if (ctx->mmap[ctx->mmap_num - 1].size != 0)
670 return -ENOMEM;
671
672 /* Check for PAs and VAs overlaps with all other regions */
Douglas Raillard6a5f8f12017-09-21 08:42:21 +0100673 for (mmap_region_t *mm_cursor = ctx->mmap;
674 mm_cursor->size; ++mm_cursor) {
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000675
Douglas Raillard6a5f8f12017-09-21 08:42:21 +0100676 uintptr_t mm_cursor_end_va = mm_cursor->base_va
677 + mm_cursor->size - 1;
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000678
679 /*
680 * Check if one of the regions is completely inside the other
681 * one.
682 */
683 int fully_overlapped_va =
Douglas Raillard6a5f8f12017-09-21 08:42:21 +0100684 ((base_va >= mm_cursor->base_va) &&
685 (end_va <= mm_cursor_end_va)) ||
686
687 ((mm_cursor->base_va >= base_va) &&
688 (mm_cursor_end_va <= end_va));
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000689
690 /*
691 * Full VA overlaps are only allowed if both regions are
692 * identity mapped (zero offset) or have the same VA to PA
693 * offset. Also, make sure that it's not the exact same area.
Antonio Nino Diazac998032017-02-27 17:23:54 +0000694 * This can only be done with static regions.
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000695 */
696 if (fully_overlapped_va) {
697
Antonio Nino Diazac998032017-02-27 17:23:54 +0000698#if PLAT_XLAT_TABLES_DYNAMIC
Sandrine Bailleux8f23fa82017-09-28 21:58:12 +0100699 if ((mm->attr & MT_DYNAMIC) ||
Douglas Raillard6a5f8f12017-09-21 08:42:21 +0100700 (mm_cursor->attr & MT_DYNAMIC))
Antonio Nino Diazac998032017-02-27 17:23:54 +0000701 return -EPERM;
702#endif /* PLAT_XLAT_TABLES_DYNAMIC */
Douglas Raillard6a5f8f12017-09-21 08:42:21 +0100703 if ((mm_cursor->base_va - mm_cursor->base_pa) !=
704 (base_va - base_pa))
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000705 return -EPERM;
706
Douglas Raillard6a5f8f12017-09-21 08:42:21 +0100707 if ((base_va == mm_cursor->base_va) &&
708 (size == mm_cursor->size))
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000709 return -EPERM;
710
711 } else {
712 /*
713 * If the regions do not have fully overlapping VAs,
714 * then they must have fully separated VAs and PAs.
715 * Partial overlaps are not allowed
716 */
717
Douglas Raillard6a5f8f12017-09-21 08:42:21 +0100718 unsigned long long mm_cursor_end_pa =
719 mm_cursor->base_pa + mm_cursor->size - 1;
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000720
721 int separated_pa =
Douglas Raillard6a5f8f12017-09-21 08:42:21 +0100722 (end_pa < mm_cursor->base_pa) ||
723 (base_pa > mm_cursor_end_pa);
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000724 int separated_va =
Douglas Raillard6a5f8f12017-09-21 08:42:21 +0100725 (end_va < mm_cursor->base_va) ||
726 (base_va > mm_cursor_end_va);
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000727
728 if (!(separated_va && separated_pa))
729 return -EPERM;
730 }
731 }
732
733 return 0;
734}
735
Sandrine Bailleux66342932017-07-18 13:26:36 +0100736void mmap_add_region_ctx(xlat_ctx_t *ctx, const mmap_region_t *mm)
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000737{
738 mmap_region_t *mm_cursor = ctx->mmap;
Varun Wadekarccbd2e32018-04-03 10:44:41 -0700739 const mmap_region_t *mm_end = ctx->mmap + ctx->mmap_num;
740 mmap_region_t *mm_last;
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000741 unsigned long long end_pa = mm->base_pa + mm->size - 1;
742 uintptr_t end_va = mm->base_va + mm->size - 1;
743 int ret;
744
745 /* Ignore empty regions */
746 if (!mm->size)
747 return;
748
Antonio Nino Diazac998032017-02-27 17:23:54 +0000749 /* Static regions must be added before initializing the xlat tables. */
750 assert(!ctx->initialized);
751
Douglas Raillard6a5f8f12017-09-21 08:42:21 +0100752 ret = mmap_add_region_check(ctx, mm);
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000753 if (ret != 0) {
754 ERROR("mmap_add_region_check() failed. error %d\n", ret);
755 assert(0);
756 return;
757 }
758
759 /*
760 * Find correct place in mmap to insert new region.
761 *
762 * 1 - Lower region VA end first.
763 * 2 - Smaller region size first.
764 *
765 * VA 0 0xFF
766 *
767 * 1st |------|
768 * 2nd |------------|
769 * 3rd |------|
770 * 4th |---|
771 * 5th |---|
772 * 6th |----------|
773 * 7th |-------------------------------------|
774 *
775 * This is required for overlapping regions only. It simplifies adding
776 * regions with the loop in xlat_tables_init_internal because the outer
777 * ones won't overwrite block or page descriptors of regions added
778 * previously.
Antonio Nino Diazac998032017-02-27 17:23:54 +0000779 *
780 * Overlapping is only allowed for static regions.
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000781 */
782
783 while ((mm_cursor->base_va + mm_cursor->size - 1) < end_va
784 && mm_cursor->size)
785 ++mm_cursor;
786
787 while ((mm_cursor->base_va + mm_cursor->size - 1 == end_va)
788 && (mm_cursor->size < mm->size))
789 ++mm_cursor;
790
Varun Wadekarccbd2e32018-04-03 10:44:41 -0700791 /*
792 * Find the last entry marker in the mmap
793 */
794 mm_last = ctx->mmap;
795 while ((mm_last->size != 0U) && (mm_last < mm_end)) {
796 ++mm_last;
797 }
798
799 /*
800 * Check if we have enough space in the memory mapping table.
801 * This shouldn't happen as we have checked in mmap_add_region_check
802 * that there is free space.
803 */
804 assert(mm_last->size == 0U);
805
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000806 /* Make room for new region by moving other regions up by one place */
807 memmove(mm_cursor + 1, mm_cursor,
808 (uintptr_t)mm_last - (uintptr_t)mm_cursor);
809
810 /*
811 * Check we haven't lost the empty sentinel from the end of the array.
812 * This shouldn't happen as we have checked in mmap_add_region_check
813 * that there is free space.
814 */
Varun Wadekarccbd2e32018-04-03 10:44:41 -0700815 assert(mm_end->size == 0U);
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000816
Douglas Raillardf68d2ed2017-09-12 10:31:49 +0100817 *mm_cursor = *mm;
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000818
819 if (end_pa > ctx->max_pa)
820 ctx->max_pa = end_pa;
821 if (end_va > ctx->max_va)
822 ctx->max_va = end_va;
823}
824
Sandrine Bailleux66342932017-07-18 13:26:36 +0100825void mmap_add_region(unsigned long long base_pa,
826 uintptr_t base_va,
827 size_t size,
828 mmap_attr_t attr)
829{
Douglas Raillard35c09f12017-08-31 16:20:25 +0100830 mmap_region_t mm = MAP_REGION(base_pa, base_va, size, attr);
Sandrine Bailleux66342932017-07-18 13:26:36 +0100831 mmap_add_region_ctx(&tf_xlat_ctx, &mm);
832}
833
834
835void mmap_add_ctx(xlat_ctx_t *ctx, const mmap_region_t *mm)
836{
837 while (mm->size) {
838 mmap_add_region_ctx(ctx, mm);
839 mm++;
840 }
841}
842
843void mmap_add(const mmap_region_t *mm)
844{
845 mmap_add_ctx(&tf_xlat_ctx, mm);
846}
847
Antonio Nino Diazac998032017-02-27 17:23:54 +0000848#if PLAT_XLAT_TABLES_DYNAMIC
849
850int mmap_add_dynamic_region_ctx(xlat_ctx_t *ctx, mmap_region_t *mm)
851{
852 mmap_region_t *mm_cursor = ctx->mmap;
853 mmap_region_t *mm_last = mm_cursor + ctx->mmap_num;
854 unsigned long long end_pa = mm->base_pa + mm->size - 1;
855 uintptr_t end_va = mm->base_va + mm->size - 1;
856 int ret;
857
858 /* Nothing to do */
859 if (!mm->size)
860 return 0;
861
Douglas Raillard6a5f8f12017-09-21 08:42:21 +0100862 /* Now this region is a dynamic one */
863 mm->attr |= MT_DYNAMIC;
864
865 ret = mmap_add_region_check(ctx, mm);
Antonio Nino Diazac998032017-02-27 17:23:54 +0000866 if (ret != 0)
867 return ret;
868
869 /*
870 * Find the adequate entry in the mmap array in the same way done for
871 * static regions in mmap_add_region_ctx().
872 */
873
Douglas Raillard6a5f8f12017-09-21 08:42:21 +0100874 while ((mm_cursor->base_va + mm_cursor->size - 1)
875 < end_va && mm_cursor->size)
Antonio Nino Diazac998032017-02-27 17:23:54 +0000876 ++mm_cursor;
877
Douglas Raillard6a5f8f12017-09-21 08:42:21 +0100878 while ((mm_cursor->base_va + mm_cursor->size - 1 == end_va)
879 && (mm_cursor->size < mm->size))
Antonio Nino Diazac998032017-02-27 17:23:54 +0000880 ++mm_cursor;
881
882 /* Make room for new region by moving other regions up by one place */
Douglas Raillard6a5f8f12017-09-21 08:42:21 +0100883 memmove(mm_cursor + 1, mm_cursor,
884 (uintptr_t)mm_last - (uintptr_t)mm_cursor);
Antonio Nino Diazac998032017-02-27 17:23:54 +0000885
886 /*
887 * Check we haven't lost the empty sentinal from the end of the array.
888 * This shouldn't happen as we have checked in mmap_add_region_check
889 * that there is free space.
890 */
891 assert(mm_last->size == 0);
892
Douglas Raillardf68d2ed2017-09-12 10:31:49 +0100893 *mm_cursor = *mm;
Antonio Nino Diazac998032017-02-27 17:23:54 +0000894
895 /*
896 * Update the translation tables if the xlat tables are initialized. If
897 * not, this region will be mapped when they are initialized.
898 */
899 if (ctx->initialized) {
Douglas Raillard6a5f8f12017-09-21 08:42:21 +0100900 uintptr_t end_va = xlat_tables_map_region(ctx, mm_cursor,
901 0, ctx->base_table, ctx->base_table_entries,
902 ctx->base_level);
Antonio Nino Diazac998032017-02-27 17:23:54 +0000903
904 /* Failed to map, remove mmap entry, unmap and return error. */
905 if (end_va != mm_cursor->base_va + mm_cursor->size - 1) {
Douglas Raillard6a5f8f12017-09-21 08:42:21 +0100906 memmove(mm_cursor, mm_cursor + 1,
907 (uintptr_t)mm_last - (uintptr_t)mm_cursor);
Antonio Nino Diazac998032017-02-27 17:23:54 +0000908
909 /*
910 * Check if the mapping function actually managed to map
911 * anything. If not, just return now.
912 */
Antonio Nino Diaz3f518922018-01-05 11:30:36 +0000913 if (mm->base_va >= end_va)
Antonio Nino Diazac998032017-02-27 17:23:54 +0000914 return -ENOMEM;
915
916 /*
Douglas Raillard6a5f8f12017-09-21 08:42:21 +0100917 * Something went wrong after mapping some table
918 * entries, undo every change done up to this point.
Antonio Nino Diazac998032017-02-27 17:23:54 +0000919 */
920 mmap_region_t unmap_mm = {
921 .base_pa = 0,
922 .base_va = mm->base_va,
923 .size = end_va - mm->base_va,
924 .attr = 0
925 };
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +0100926 xlat_tables_unmap_region(ctx, &unmap_mm, 0, ctx->base_table,
927 ctx->base_table_entries, ctx->base_level);
Antonio Nino Diazac998032017-02-27 17:23:54 +0000928
929 return -ENOMEM;
930 }
931
932 /*
933 * Make sure that all entries are written to the memory. There
934 * is no need to invalidate entries when mapping dynamic regions
935 * because new table/block/page descriptors only replace old
936 * invalid descriptors, that aren't TLB cached.
937 */
938 dsbishst();
939 }
940
941 if (end_pa > ctx->max_pa)
942 ctx->max_pa = end_pa;
943 if (end_va > ctx->max_va)
944 ctx->max_va = end_va;
945
946 return 0;
947}
948
Sandrine Bailleux66342932017-07-18 13:26:36 +0100949int mmap_add_dynamic_region(unsigned long long base_pa,
950 uintptr_t base_va, size_t size, mmap_attr_t attr)
951{
Douglas Raillard35c09f12017-08-31 16:20:25 +0100952 mmap_region_t mm = MAP_REGION(base_pa, base_va, size, attr);
Sandrine Bailleux66342932017-07-18 13:26:36 +0100953 return mmap_add_dynamic_region_ctx(&tf_xlat_ctx, &mm);
954}
955
Antonio Nino Diazac998032017-02-27 17:23:54 +0000956/*
957 * Removes the region with given base Virtual Address and size from the given
958 * context.
959 *
960 * Returns:
961 * 0: Success.
962 * EINVAL: Invalid values were used as arguments (region not found).
963 * EPERM: Tried to remove a static region.
964 */
965int mmap_remove_dynamic_region_ctx(xlat_ctx_t *ctx, uintptr_t base_va,
966 size_t size)
967{
968 mmap_region_t *mm = ctx->mmap;
969 mmap_region_t *mm_last = mm + ctx->mmap_num;
970 int update_max_va_needed = 0;
971 int update_max_pa_needed = 0;
972
973 /* Check sanity of mmap array. */
974 assert(mm[ctx->mmap_num].size == 0);
975
976 while (mm->size) {
977 if ((mm->base_va == base_va) && (mm->size == size))
978 break;
979 ++mm;
980 }
981
982 /* Check that the region was found */
983 if (mm->size == 0)
984 return -EINVAL;
985
986 /* If the region is static it can't be removed */
987 if (!(mm->attr & MT_DYNAMIC))
988 return -EPERM;
989
990 /* Check if this region is using the top VAs or PAs. */
991 if ((mm->base_va + mm->size - 1) == ctx->max_va)
992 update_max_va_needed = 1;
993 if ((mm->base_pa + mm->size - 1) == ctx->max_pa)
994 update_max_pa_needed = 1;
995
996 /* Update the translation tables if needed */
997 if (ctx->initialized) {
998 xlat_tables_unmap_region(ctx, mm, 0, ctx->base_table,
999 ctx->base_table_entries,
1000 ctx->base_level);
1001 xlat_arch_tlbi_va_sync();
1002 }
1003
1004 /* Remove this region by moving the rest down by one place. */
1005 memmove(mm, mm + 1, (uintptr_t)mm_last - (uintptr_t)mm);
1006
1007 /* Check if we need to update the max VAs and PAs */
1008 if (update_max_va_needed) {
1009 ctx->max_va = 0;
1010 mm = ctx->mmap;
1011 while (mm->size) {
1012 if ((mm->base_va + mm->size - 1) > ctx->max_va)
1013 ctx->max_va = mm->base_va + mm->size - 1;
1014 ++mm;
1015 }
1016 }
1017
1018 if (update_max_pa_needed) {
1019 ctx->max_pa = 0;
1020 mm = ctx->mmap;
1021 while (mm->size) {
1022 if ((mm->base_pa + mm->size - 1) > ctx->max_pa)
1023 ctx->max_pa = mm->base_pa + mm->size - 1;
1024 ++mm;
1025 }
1026 }
1027
1028 return 0;
1029}
1030
Sandrine Bailleux66342932017-07-18 13:26:36 +01001031int mmap_remove_dynamic_region(uintptr_t base_va, size_t size)
1032{
1033 return mmap_remove_dynamic_region_ctx(&tf_xlat_ctx,
1034 base_va, size);
1035}
1036
Antonio Nino Diazac998032017-02-27 17:23:54 +00001037#endif /* PLAT_XLAT_TABLES_DYNAMIC */
1038
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001039#if LOG_LEVEL >= LOG_LEVEL_VERBOSE
1040
1041/* Print the attributes of the specified block descriptor. */
Sandrine Bailleuxc3708e22017-10-13 14:17:09 +01001042static void xlat_desc_print(const xlat_ctx_t *ctx, uint64_t desc)
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001043{
1044 int mem_type_index = ATTR_INDEX_GET(desc);
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +01001045 xlat_regime_t xlat_regime = ctx->xlat_regime;
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001046
1047 if (mem_type_index == ATTR_IWBWA_OWBWA_NTR_INDEX) {
1048 tf_printf("MEM");
1049 } else if (mem_type_index == ATTR_NON_CACHEABLE_INDEX) {
1050 tf_printf("NC");
1051 } else {
1052 assert(mem_type_index == ATTR_DEVICE_INDEX);
1053 tf_printf("DEV");
1054 }
1055
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +01001056 const char *priv_str = "(PRIV)";
1057 const char *user_str = "(USER)";
1058
1059 /*
1060 * Showing Privileged vs Unprivileged only makes sense for EL1&0
1061 * mappings
1062 */
1063 const char *ro_str = "-RO";
1064 const char *rw_str = "-RW";
1065 const char *no_access_str = "-NOACCESS";
1066
1067 if (xlat_regime == EL3_REGIME) {
1068 /* For EL3, the AP[2] bit is all what matters */
Roberto Vargasc8cc3512018-05-09 10:49:24 +01001069 tf_printf("%s", (desc & LOWER_ATTRS(AP_RO)) ? ro_str : rw_str);
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +01001070 } else {
1071 const char *ap_str = (desc & LOWER_ATTRS(AP_RO)) ? ro_str : rw_str;
Roberto Vargasc8cc3512018-05-09 10:49:24 +01001072 tf_printf("%s", ap_str);
1073 tf_printf("%s", priv_str);
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +01001074 /*
1075 * EL0 can only have the same permissions as EL1 or no
1076 * permissions at all.
1077 */
Roberto Vargasc8cc3512018-05-09 10:49:24 +01001078 tf_printf("%s",
1079 (desc & LOWER_ATTRS(AP_ACCESS_UNPRIVILEGED))
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +01001080 ? ap_str : no_access_str);
Roberto Vargasc8cc3512018-05-09 10:49:24 +01001081 tf_printf("%s", user_str);
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +01001082 }
1083
1084 const char *xn_str = "-XN";
1085 const char *exec_str = "-EXEC";
1086
1087 if (xlat_regime == EL3_REGIME) {
1088 /* For EL3, the XN bit is all what matters */
Roberto Vargasc8cc3512018-05-09 10:49:24 +01001089 tf_printf("%s", LOWER_ATTRS(XN) & desc ? xn_str : exec_str);
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +01001090 } else {
1091 /* For EL0 and EL1, we need to know who has which rights */
Roberto Vargasc8cc3512018-05-09 10:49:24 +01001092 tf_printf("%s", LOWER_ATTRS(PXN) & desc ? xn_str : exec_str);
1093 tf_printf("%s", priv_str);
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +01001094
Roberto Vargasc8cc3512018-05-09 10:49:24 +01001095 tf_printf("%s", LOWER_ATTRS(UXN) & desc ? xn_str : exec_str);
1096 tf_printf("%s", user_str);
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +01001097 }
1098
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001099 tf_printf(LOWER_ATTRS(NS) & desc ? "-NS" : "-S");
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001100}
1101
1102static const char * const level_spacers[] = {
Antonio Nino Diaz755e54f2017-02-13 11:35:49 +00001103 "[LV0] ",
1104 " [LV1] ",
1105 " [LV2] ",
1106 " [LV3] "
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001107};
1108
Antonio Nino Diaz755e54f2017-02-13 11:35:49 +00001109static const char *invalid_descriptors_ommited =
1110 "%s(%d invalid descriptors omitted)\n";
1111
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001112/*
1113 * Recursive function that reads the translation tables passed as an argument
1114 * and prints their status.
1115 */
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +01001116static void xlat_tables_print_internal(xlat_ctx_t *ctx,
1117 const uintptr_t table_base_va,
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001118 uint64_t *const table_base, const int table_entries,
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +01001119 const unsigned int level)
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001120{
1121 assert(level <= XLAT_TABLE_LEVEL_MAX);
1122
1123 uint64_t desc;
1124 uintptr_t table_idx_va = table_base_va;
1125 int table_idx = 0;
1126
1127 size_t level_size = XLAT_BLOCK_SIZE(level);
1128
Antonio Nino Diaz755e54f2017-02-13 11:35:49 +00001129 /*
1130 * Keep track of how many invalid descriptors are counted in a row.
1131 * Whenever multiple invalid descriptors are found, only the first one
1132 * is printed, and a line is added to inform about how many descriptors
1133 * have been omitted.
1134 */
1135 int invalid_row_count = 0;
1136
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001137 while (table_idx < table_entries) {
1138
1139 desc = table_base[table_idx];
1140
1141 if ((desc & DESC_MASK) == INVALID_DESC) {
1142
Antonio Nino Diaz755e54f2017-02-13 11:35:49 +00001143 if (invalid_row_count == 0) {
1144 tf_printf("%sVA:%p size:0x%zx\n",
1145 level_spacers[level],
1146 (void *)table_idx_va, level_size);
1147 }
1148 invalid_row_count++;
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001149
1150 } else {
1151
Antonio Nino Diaz755e54f2017-02-13 11:35:49 +00001152 if (invalid_row_count > 1) {
1153 tf_printf(invalid_descriptors_ommited,
1154 level_spacers[level],
1155 invalid_row_count - 1);
1156 }
1157 invalid_row_count = 0;
1158
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001159 /*
1160 * Check if this is a table or a block. Tables are only
1161 * allowed in levels other than 3, but DESC_PAGE has the
1162 * same value as DESC_TABLE, so we need to check.
1163 */
1164 if (((desc & DESC_MASK) == TABLE_DESC) &&
1165 (level < XLAT_TABLE_LEVEL_MAX)) {
1166 /*
1167 * Do not print any PA for a table descriptor,
1168 * as it doesn't directly map physical memory
1169 * but instead points to the next translation
1170 * table in the translation table walk.
1171 */
1172 tf_printf("%sVA:%p size:0x%zx\n",
1173 level_spacers[level],
1174 (void *)table_idx_va, level_size);
1175
1176 uintptr_t addr_inner = desc & TABLE_ADDR_MASK;
1177
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +01001178 xlat_tables_print_internal(ctx, table_idx_va,
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001179 (uint64_t *)addr_inner,
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +01001180 XLAT_TABLE_ENTRIES, level + 1);
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001181 } else {
1182 tf_printf("%sVA:%p PA:0x%llx size:0x%zx ",
1183 level_spacers[level],
1184 (void *)table_idx_va,
1185 (unsigned long long)(desc & TABLE_ADDR_MASK),
1186 level_size);
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +01001187 xlat_desc_print(ctx, desc);
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001188 tf_printf("\n");
1189 }
1190 }
1191
1192 table_idx++;
1193 table_idx_va += level_size;
1194 }
Antonio Nino Diaz755e54f2017-02-13 11:35:49 +00001195
1196 if (invalid_row_count > 1) {
1197 tf_printf(invalid_descriptors_ommited,
1198 level_spacers[level], invalid_row_count - 1);
1199 }
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001200}
1201
1202#endif /* LOG_LEVEL >= LOG_LEVEL_VERBOSE */
1203
1204void xlat_tables_print(xlat_ctx_t *ctx)
1205{
1206#if LOG_LEVEL >= LOG_LEVEL_VERBOSE
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +01001207 const char *xlat_regime_str;
1208 if (ctx->xlat_regime == EL1_EL0_REGIME) {
1209 xlat_regime_str = "1&0";
1210 } else {
1211 assert(ctx->xlat_regime == EL3_REGIME);
1212 xlat_regime_str = "3";
1213 }
Sandrine Bailleux4e8f1452017-05-26 15:47:08 +01001214 VERBOSE("Translation tables state:\n");
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +01001215 VERBOSE(" Xlat regime: EL%s\n", xlat_regime_str);
Sandrine Bailleux4e8f1452017-05-26 15:47:08 +01001216 VERBOSE(" Max allowed PA: 0x%llx\n", ctx->pa_max_address);
1217 VERBOSE(" Max allowed VA: %p\n", (void *) ctx->va_max_address);
1218 VERBOSE(" Max mapped PA: 0x%llx\n", ctx->max_pa);
1219 VERBOSE(" Max mapped VA: %p\n", (void *) ctx->max_va);
1220
1221 VERBOSE(" Initial lookup level: %i\n", ctx->base_level);
1222 VERBOSE(" Entries @initial lookup level: %i\n",
1223 ctx->base_table_entries);
1224
1225 int used_page_tables;
1226#if PLAT_XLAT_TABLES_DYNAMIC
1227 used_page_tables = 0;
Sandrine Bailleuxde6628d2017-08-01 09:16:38 +01001228 for (unsigned int i = 0; i < ctx->tables_num; ++i) {
Sandrine Bailleux4e8f1452017-05-26 15:47:08 +01001229 if (ctx->tables_mapped_regions[i] != 0)
1230 ++used_page_tables;
1231 }
1232#else
1233 used_page_tables = ctx->next_table;
1234#endif
1235 VERBOSE(" Used %i sub-tables out of %i (spare: %i)\n",
1236 used_page_tables, ctx->tables_num,
1237 ctx->tables_num - used_page_tables);
1238
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +01001239 xlat_tables_print_internal(ctx, 0, ctx->base_table,
1240 ctx->base_table_entries, ctx->base_level);
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001241#endif /* LOG_LEVEL >= LOG_LEVEL_VERBOSE */
1242}
1243
Sandrine Bailleux66342932017-07-18 13:26:36 +01001244void init_xlat_tables_ctx(xlat_ctx_t *ctx)
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001245{
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +01001246 assert(ctx != NULL);
Sandrine Bailleux66342932017-07-18 13:26:36 +01001247 assert(!ctx->initialized);
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +01001248 assert(ctx->xlat_regime == EL3_REGIME || ctx->xlat_regime == EL1_EL0_REGIME);
1249 assert(!is_mmu_enabled_ctx(ctx));
Sandrine Bailleux66342932017-07-18 13:26:36 +01001250
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +01001251 mmap_region_t *mm = ctx->mmap;
Sandrine Bailleux66342932017-07-18 13:26:36 +01001252
Antonio Nino Diazdcf9d922017-10-04 16:52:15 +01001253 print_mmap(mm);
Sandrine Bailleux66342932017-07-18 13:26:36 +01001254
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001255 /* All tables must be zeroed before mapping any region. */
1256
Varun Wadekar66231d12017-06-07 09:57:42 -07001257 for (unsigned int i = 0; i < ctx->base_table_entries; i++)
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001258 ctx->base_table[i] = INVALID_DESC;
1259
Varun Wadekar66231d12017-06-07 09:57:42 -07001260 for (unsigned int j = 0; j < ctx->tables_num; j++) {
Antonio Nino Diazac998032017-02-27 17:23:54 +00001261#if PLAT_XLAT_TABLES_DYNAMIC
1262 ctx->tables_mapped_regions[j] = 0;
1263#endif
Varun Wadekar66231d12017-06-07 09:57:42 -07001264 for (unsigned int i = 0; i < XLAT_TABLE_ENTRIES; i++)
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001265 ctx->tables[j][i] = INVALID_DESC;
1266 }
1267
Antonio Nino Diazac998032017-02-27 17:23:54 +00001268 while (mm->size) {
1269 uintptr_t end_va = xlat_tables_map_region(ctx, mm, 0, ctx->base_table,
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001270 ctx->base_table_entries, ctx->base_level);
1271
Antonio Nino Diazac998032017-02-27 17:23:54 +00001272 if (end_va != mm->base_va + mm->size - 1) {
1273 ERROR("Not enough memory to map region:\n"
1274 " VA:%p PA:0x%llx size:0x%zx attr:0x%x\n",
1275 (void *)mm->base_va, mm->base_pa, mm->size, mm->attr);
1276 panic();
1277 }
1278
1279 mm++;
1280 }
1281
Sandrine Bailleux46c53a22017-07-11 15:11:10 +01001282 assert(ctx->pa_max_address <= xlat_arch_get_max_supported_pa());
Sandrine Bailleux66342932017-07-18 13:26:36 +01001283 assert(ctx->max_va <= ctx->va_max_address);
1284 assert(ctx->max_pa <= ctx->pa_max_address);
1285
Sandrine Bailleuxc5b63772017-05-31 13:31:48 +01001286 ctx->initialized = 1;
1287
1288 xlat_tables_print(ctx);
Sandrine Bailleux66342932017-07-18 13:26:36 +01001289}
1290
1291void init_xlat_tables(void)
1292{
1293 init_xlat_tables_ctx(&tf_xlat_ctx);
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001294}
Sandrine Bailleux66342932017-07-18 13:26:36 +01001295
Sandrine Bailleuxc5b63772017-05-31 13:31:48 +01001296/*
1297 * If dynamic allocation of new regions is disabled then by the time we call the
1298 * function enabling the MMU, we'll have registered all the memory regions to
1299 * map for the system's lifetime. Therefore, at this point we know the maximum
1300 * physical address that will ever be mapped.
1301 *
1302 * If dynamic allocation is enabled then we can't make any such assumption
1303 * because the maximum physical address could get pushed while adding a new
1304 * region. Therefore, in this case we have to assume that the whole address
1305 * space size might be mapped.
1306 */
1307#ifdef PLAT_XLAT_TABLES_DYNAMIC
Sandrine Bailleux46c53a22017-07-11 15:11:10 +01001308#define MAX_PHYS_ADDR tf_xlat_ctx.pa_max_address
Sandrine Bailleuxc5b63772017-05-31 13:31:48 +01001309#else
1310#define MAX_PHYS_ADDR tf_xlat_ctx.max_pa
1311#endif
1312
Sandrine Bailleux66342932017-07-18 13:26:36 +01001313#ifdef AARCH32
1314
1315void enable_mmu_secure(unsigned int flags)
1316{
Sandrine Bailleux46c53a22017-07-11 15:11:10 +01001317 enable_mmu_arch(flags, tf_xlat_ctx.base_table, MAX_PHYS_ADDR,
1318 tf_xlat_ctx.va_max_address);
Sandrine Bailleux66342932017-07-18 13:26:36 +01001319}
1320
1321#else
1322
1323void enable_mmu_el1(unsigned int flags)
1324{
Sandrine Bailleux46c53a22017-07-11 15:11:10 +01001325 enable_mmu_arch(flags, tf_xlat_ctx.base_table, MAX_PHYS_ADDR,
1326 tf_xlat_ctx.va_max_address);
Sandrine Bailleux66342932017-07-18 13:26:36 +01001327}
1328
1329void enable_mmu_el3(unsigned int flags)
1330{
Sandrine Bailleux46c53a22017-07-11 15:11:10 +01001331 enable_mmu_arch(flags, tf_xlat_ctx.base_table, MAX_PHYS_ADDR,
1332 tf_xlat_ctx.va_max_address);
Sandrine Bailleux66342932017-07-18 13:26:36 +01001333}
1334
1335#endif /* AARCH32 */
Sandrine Bailleuxc3708e22017-10-13 14:17:09 +01001336
1337/*
1338 * Do a translation table walk to find the block or page descriptor that maps
1339 * virtual_addr.
1340 *
1341 * On success, return the address of the descriptor within the translation
1342 * table. Its lookup level is stored in '*out_level'.
1343 * On error, return NULL.
1344 *
1345 * xlat_table_base
1346 * Base address for the initial lookup level.
1347 * xlat_table_base_entries
1348 * Number of entries in the translation table for the initial lookup level.
1349 * virt_addr_space_size
1350 * Size in bytes of the virtual address space.
1351 */
1352static uint64_t *find_xlat_table_entry(uintptr_t virtual_addr,
1353 void *xlat_table_base,
1354 int xlat_table_base_entries,
1355 unsigned long long virt_addr_space_size,
1356 int *out_level)
1357{
1358 unsigned int start_level;
1359 uint64_t *table;
1360 int entries;
1361
1362 VERBOSE("%s(%p)\n", __func__, (void *)virtual_addr);
1363
1364 start_level = GET_XLAT_TABLE_LEVEL_BASE(virt_addr_space_size);
1365 VERBOSE("Starting translation table walk from level %i\n", start_level);
1366
1367 table = xlat_table_base;
1368 entries = xlat_table_base_entries;
1369
1370 for (unsigned int level = start_level;
1371 level <= XLAT_TABLE_LEVEL_MAX;
1372 ++level) {
1373 int idx;
1374 uint64_t desc;
1375 uint64_t desc_type;
1376
1377 VERBOSE("Table address: %p\n", (void *)table);
1378
1379 idx = XLAT_TABLE_IDX(virtual_addr, level);
1380 VERBOSE("Index into level %i table: %i\n", level, idx);
1381 if (idx >= entries) {
1382 VERBOSE("Invalid address\n");
1383 return NULL;
1384 }
1385
1386 desc = table[idx];
1387 desc_type = desc & DESC_MASK;
1388 VERBOSE("Descriptor at level %i: 0x%llx\n", level,
1389 (unsigned long long)desc);
1390
1391 if (desc_type == INVALID_DESC) {
1392 VERBOSE("Invalid entry (memory not mapped)\n");
1393 return NULL;
1394 }
1395
1396 if (level == XLAT_TABLE_LEVEL_MAX) {
1397 /*
1398 * There can't be table entries at the final lookup
1399 * level.
1400 */
1401 assert(desc_type == PAGE_DESC);
1402 VERBOSE("Descriptor mapping a memory page (size: 0x%llx)\n",
1403 (unsigned long long)XLAT_BLOCK_SIZE(XLAT_TABLE_LEVEL_MAX));
1404 *out_level = level;
1405 return &table[idx];
1406 }
1407
1408 if (desc_type == BLOCK_DESC) {
1409 VERBOSE("Descriptor mapping a memory block (size: 0x%llx)\n",
1410 (unsigned long long)XLAT_BLOCK_SIZE(level));
1411 *out_level = level;
1412 return &table[idx];
1413 }
1414
1415 assert(desc_type == TABLE_DESC);
1416 VERBOSE("Table descriptor, continuing xlat table walk...\n");
1417 table = (uint64_t *)(uintptr_t)(desc & TABLE_ADDR_MASK);
1418 entries = XLAT_TABLE_ENTRIES;
1419 }
1420
1421 /*
1422 * This shouldn't be reached, the translation table walk should end at
1423 * most at level XLAT_TABLE_LEVEL_MAX and return from inside the loop.
1424 */
1425 assert(0);
1426
1427 return NULL;
1428}
1429
1430
1431static int get_mem_attributes_internal(const xlat_ctx_t *ctx, uintptr_t base_va,
Antonio Nino Diaze8811472018-04-17 15:10:18 +01001432 uint32_t *attributes, uint64_t **table_entry,
Sandrine Bailleuxc3708e22017-10-13 14:17:09 +01001433 unsigned long long *addr_pa, int *table_level)
1434{
1435 uint64_t *entry;
1436 uint64_t desc;
1437 int level;
1438 unsigned long long virt_addr_space_size;
1439
1440 /*
1441 * Sanity-check arguments.
1442 */
1443 assert(ctx != NULL);
1444 assert(ctx->initialized);
1445 assert(ctx->xlat_regime == EL1_EL0_REGIME || ctx->xlat_regime == EL3_REGIME);
1446
1447 virt_addr_space_size = (unsigned long long)ctx->va_max_address + 1;
1448 assert(virt_addr_space_size > 0);
1449
1450 entry = find_xlat_table_entry(base_va,
1451 ctx->base_table,
1452 ctx->base_table_entries,
1453 virt_addr_space_size,
1454 &level);
1455 if (entry == NULL) {
1456 WARN("Address %p is not mapped.\n", (void *)base_va);
1457 return -EINVAL;
1458 }
1459
1460 if (addr_pa != NULL) {
1461 *addr_pa = *entry & TABLE_ADDR_MASK;
1462 }
1463
1464 if (table_entry != NULL) {
1465 *table_entry = entry;
1466 }
1467
1468 if (table_level != NULL) {
1469 *table_level = level;
1470 }
1471
1472 desc = *entry;
1473
1474#if LOG_LEVEL >= LOG_LEVEL_VERBOSE
1475 VERBOSE("Attributes: ");
1476 xlat_desc_print(ctx, desc);
1477 tf_printf("\n");
1478#endif /* LOG_LEVEL >= LOG_LEVEL_VERBOSE */
1479
1480 assert(attributes != NULL);
1481 *attributes = 0;
1482
1483 int attr_index = (desc >> ATTR_INDEX_SHIFT) & ATTR_INDEX_MASK;
1484
1485 if (attr_index == ATTR_IWBWA_OWBWA_NTR_INDEX) {
1486 *attributes |= MT_MEMORY;
1487 } else if (attr_index == ATTR_NON_CACHEABLE_INDEX) {
1488 *attributes |= MT_NON_CACHEABLE;
1489 } else {
1490 assert(attr_index == ATTR_DEVICE_INDEX);
1491 *attributes |= MT_DEVICE;
1492 }
1493
1494 int ap2_bit = (desc >> AP2_SHIFT) & 1;
1495
1496 if (ap2_bit == AP2_RW)
1497 *attributes |= MT_RW;
1498
1499 if (ctx->xlat_regime == EL1_EL0_REGIME) {
1500 int ap1_bit = (desc >> AP1_SHIFT) & 1;
1501 if (ap1_bit == AP1_ACCESS_UNPRIVILEGED)
1502 *attributes |= MT_USER;
1503 }
1504
1505 int ns_bit = (desc >> NS_SHIFT) & 1;
1506
1507 if (ns_bit == 1)
1508 *attributes |= MT_NS;
1509
1510 uint64_t xn_mask = xlat_arch_regime_get_xn_desc(ctx->xlat_regime);
1511
1512 if ((desc & xn_mask) == xn_mask) {
1513 *attributes |= MT_EXECUTE_NEVER;
1514 } else {
1515 assert((desc & xn_mask) == 0);
1516 }
1517
1518 return 0;
1519}
1520
1521
1522int get_mem_attributes(const xlat_ctx_t *ctx, uintptr_t base_va,
Antonio Nino Diaze8811472018-04-17 15:10:18 +01001523 uint32_t *attributes)
Sandrine Bailleuxc3708e22017-10-13 14:17:09 +01001524{
1525 return get_mem_attributes_internal(ctx, base_va, attributes,
1526 NULL, NULL, NULL);
1527}
Sandrine Bailleux439c9012017-10-17 12:02:03 +01001528
1529
1530int change_mem_attributes(xlat_ctx_t *ctx,
1531 uintptr_t base_va,
1532 size_t size,
Antonio Nino Diaze8811472018-04-17 15:10:18 +01001533 uint32_t attr)
Sandrine Bailleux439c9012017-10-17 12:02:03 +01001534{
1535 /* Note: This implementation isn't optimized. */
1536
1537 assert(ctx != NULL);
1538 assert(ctx->initialized);
1539
1540 unsigned long long virt_addr_space_size =
1541 (unsigned long long)ctx->va_max_address + 1;
1542 assert(virt_addr_space_size > 0);
1543
1544 if (!IS_PAGE_ALIGNED(base_va)) {
1545 WARN("%s: Address %p is not aligned on a page boundary.\n",
1546 __func__, (void *)base_va);
1547 return -EINVAL;
1548 }
1549
1550 if (size == 0) {
1551 WARN("%s: Size is 0.\n", __func__);
1552 return -EINVAL;
1553 }
1554
1555 if ((size % PAGE_SIZE) != 0) {
1556 WARN("%s: Size 0x%zx is not a multiple of a page size.\n",
1557 __func__, size);
1558 return -EINVAL;
1559 }
1560
1561 if (((attr & MT_EXECUTE_NEVER) == 0) && ((attr & MT_RW) != 0)) {
1562 WARN("%s() doesn't allow to remap memory as read-write and executable.\n",
1563 __func__);
1564 return -EINVAL;
1565 }
1566
1567 int pages_count = size / PAGE_SIZE;
1568
1569 VERBOSE("Changing memory attributes of %i pages starting from address %p...\n",
1570 pages_count, (void *)base_va);
1571
1572 uintptr_t base_va_original = base_va;
1573
1574 /*
1575 * Sanity checks.
1576 */
1577 for (int i = 0; i < pages_count; ++i) {
1578 uint64_t *entry;
1579 uint64_t desc;
1580 int level;
1581
1582 entry = find_xlat_table_entry(base_va,
1583 ctx->base_table,
1584 ctx->base_table_entries,
1585 virt_addr_space_size,
1586 &level);
1587 if (entry == NULL) {
1588 WARN("Address %p is not mapped.\n", (void *)base_va);
1589 return -EINVAL;
1590 }
1591
1592 desc = *entry;
1593
1594 /*
1595 * Check that all the required pages are mapped at page
1596 * granularity.
1597 */
1598 if (((desc & DESC_MASK) != PAGE_DESC) ||
1599 (level != XLAT_TABLE_LEVEL_MAX)) {
1600 WARN("Address %p is not mapped at the right granularity.\n",
1601 (void *)base_va);
1602 WARN("Granularity is 0x%llx, should be 0x%x.\n",
1603 (unsigned long long)XLAT_BLOCK_SIZE(level), PAGE_SIZE);
1604 return -EINVAL;
1605 }
1606
1607 /*
1608 * If the region type is device, it shouldn't be executable.
1609 */
1610 int attr_index = (desc >> ATTR_INDEX_SHIFT) & ATTR_INDEX_MASK;
1611 if (attr_index == ATTR_DEVICE_INDEX) {
1612 if ((attr & MT_EXECUTE_NEVER) == 0) {
1613 WARN("Setting device memory as executable at address %p.",
1614 (void *)base_va);
1615 return -EINVAL;
1616 }
1617 }
1618
1619 base_va += PAGE_SIZE;
1620 }
1621
1622 /* Restore original value. */
1623 base_va = base_va_original;
1624
1625 VERBOSE("%s: All pages are mapped, now changing their attributes...\n",
1626 __func__);
1627
1628 for (int i = 0; i < pages_count; ++i) {
1629
Antonio Nino Diaze8811472018-04-17 15:10:18 +01001630 uint32_t old_attr, new_attr;
Sandrine Bailleux439c9012017-10-17 12:02:03 +01001631 uint64_t *entry;
1632 int level;
1633 unsigned long long addr_pa;
1634
1635 get_mem_attributes_internal(ctx, base_va, &old_attr,
1636 &entry, &addr_pa, &level);
1637
1638 VERBOSE("Old attributes: 0x%x\n", old_attr);
1639
1640 /*
1641 * From attr, only MT_RO/MT_RW, MT_EXECUTE/MT_EXECUTE_NEVER and
1642 * MT_USER/MT_PRIVILEGED are taken into account. Any other
1643 * information is ignored.
1644 */
1645
1646 /* Clean the old attributes so that they can be rebuilt. */
1647 new_attr = old_attr & ~(MT_RW|MT_EXECUTE_NEVER|MT_USER);
1648
1649 /*
1650 * Update attributes, but filter out the ones this function
1651 * isn't allowed to change.
1652 */
1653 new_attr |= attr & (MT_RW|MT_EXECUTE_NEVER|MT_USER);
1654
1655 VERBOSE("New attributes: 0x%x\n", new_attr);
1656
1657 /*
1658 * The break-before-make sequence requires writing an invalid
1659 * descriptor and making sure that the system sees the change
1660 * before writing the new descriptor.
1661 */
1662 *entry = INVALID_DESC;
1663
1664 /* Invalidate any cached copy of this mapping in the TLBs. */
1665 xlat_arch_tlbi_va_regime(base_va, ctx->xlat_regime);
1666
1667 /* Ensure completion of the invalidation. */
1668 xlat_arch_tlbi_va_sync();
1669
1670 /* Write new descriptor */
1671 *entry = xlat_desc(ctx, new_attr, addr_pa, level);
1672
1673 base_va += PAGE_SIZE;
1674 }
1675
1676 /* Ensure that the last descriptor writen is seen by the system. */
1677 dsbish();
1678
1679 return 0;
1680}