blob: 581f77032418fff742e728ba197abae7fc568fc8 [file] [log] [blame]
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001/*
2 * Copyright (c) 2017, ARM Limited and Contributors. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions are met:
6 *
7 * Redistributions of source code must retain the above copyright notice, this
8 * list of conditions and the following disclaimer.
9 *
10 * Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 *
14 * Neither the name of ARM nor the names of its contributors may be used
15 * to endorse or promote products derived from this software without specific
16 * prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28 * POSSIBILITY OF SUCH DAMAGE.
29 */
30
31#include <arch.h>
32#include <arch_helpers.h>
33#include <assert.h>
34#include <cassert.h>
35#include <common_def.h>
36#include <debug.h>
37#include <errno.h>
38#include <platform_def.h>
39#include <string.h>
40#include <types.h>
41#include <utils.h>
42#include <xlat_tables_v2.h>
43#ifdef AARCH32
44# include "aarch32/xlat_tables_arch.h"
45#else
46# include "aarch64/xlat_tables_arch.h"
47#endif
48#include "xlat_tables_private.h"
49
Antonio Nino Diazac998032017-02-27 17:23:54 +000050#if PLAT_XLAT_TABLES_DYNAMIC
51
52/*
53 * The following functions assume that they will be called using subtables only.
54 * The base table can't be unmapped, so it is not needed to do any special
55 * handling for it.
56 */
57
58/*
59 * Returns the index of the array corresponding to the specified translation
60 * table.
61 */
62static int xlat_table_get_index(xlat_ctx_t *ctx, const uint64_t *table)
63{
64 for (int i = 0; i < ctx->tables_num; i++)
65 if (ctx->tables[i] == table)
66 return i;
67
68 /*
69 * Maybe we were asked to get the index of the base level table, which
70 * should never happen.
71 */
72 assert(0);
73
74 return -1;
75}
76
77/* Returns a pointer to an empty translation table. */
78static uint64_t *xlat_table_get_empty(xlat_ctx_t *ctx)
79{
80 for (int i = 0; i < ctx->tables_num; i++)
81 if (ctx->tables_mapped_regions[i] == 0)
82 return ctx->tables[i];
83
84 return NULL;
85}
86
87/* Increments region count for a given table. */
88static void xlat_table_inc_regions_count(xlat_ctx_t *ctx, const uint64_t *table)
89{
90 ctx->tables_mapped_regions[xlat_table_get_index(ctx, table)]++;
91}
92
93/* Decrements region count for a given table. */
94static void xlat_table_dec_regions_count(xlat_ctx_t *ctx, const uint64_t *table)
95{
96 ctx->tables_mapped_regions[xlat_table_get_index(ctx, table)]--;
97}
98
99/* Returns 0 if the speficied table isn't empty, otherwise 1. */
100static int xlat_table_is_empty(xlat_ctx_t *ctx, const uint64_t *table)
101{
102 return !ctx->tables_mapped_regions[xlat_table_get_index(ctx, table)];
103}
104
105#else /* PLAT_XLAT_TABLES_DYNAMIC */
106
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000107/* Returns a pointer to the first empty translation table. */
108static uint64_t *xlat_table_get_empty(xlat_ctx_t *ctx)
109{
110 assert(ctx->next_table < ctx->tables_num);
111
112 return ctx->tables[ctx->next_table++];
113}
114
Antonio Nino Diazac998032017-02-27 17:23:54 +0000115#endif /* PLAT_XLAT_TABLES_DYNAMIC */
116
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000117/* Returns a block/page table descriptor for the given level and attributes. */
Sandrine Bailleux04980a32017-04-19 14:02:23 +0100118static uint64_t xlat_desc(mmap_attr_t attr, unsigned long long addr_pa,
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000119 int level)
120{
121 uint64_t desc;
122 int mem_type;
123
124 /* Make sure that the granularity is fine enough to map this address. */
125 assert((addr_pa & XLAT_BLOCK_MASK(level)) == 0);
126
127 desc = addr_pa;
128 /*
129 * There are different translation table descriptors for level 3 and the
130 * rest.
131 */
132 desc |= (level == XLAT_TABLE_LEVEL_MAX) ? PAGE_DESC : BLOCK_DESC;
133 /*
134 * Always set the access flag, as TF doesn't manage access flag faults.
135 * Deduce other fields of the descriptor based on the MT_NS and MT_RW
136 * memory region attributes.
137 */
138 desc |= (attr & MT_NS) ? LOWER_ATTRS(NS) : 0;
139 desc |= (attr & MT_RW) ? LOWER_ATTRS(AP_RW) : LOWER_ATTRS(AP_RO);
140 desc |= LOWER_ATTRS(ACCESS_FLAG);
141
142 /*
143 * Deduce shareability domain and executability of the memory region
144 * from the memory type of the attributes (MT_TYPE).
145 *
146 * Data accesses to device memory and non-cacheable normal memory are
147 * coherent for all observers in the system, and correspondingly are
148 * always treated as being Outer Shareable. Therefore, for these 2 types
149 * of memory, it is not strictly needed to set the shareability field
150 * in the translation tables.
151 */
152 mem_type = MT_TYPE(attr);
153 if (mem_type == MT_DEVICE) {
154 desc |= LOWER_ATTRS(ATTR_DEVICE_INDEX | OSH);
155 /*
156 * Always map device memory as execute-never.
157 * This is to avoid the possibility of a speculative instruction
158 * fetch, which could be an issue if this memory region
159 * corresponds to a read-sensitive peripheral.
160 */
161 desc |= UPPER_ATTRS(XN);
162 } else { /* Normal memory */
163 /*
164 * Always map read-write normal memory as execute-never.
165 * (Trusted Firmware doesn't self-modify its code, therefore
166 * R/W memory is reserved for data storage, which must not be
167 * executable.)
168 * Note that setting the XN bit here is for consistency only.
169 * The enable_mmu_elx() function sets the SCTLR_EL3.WXN bit,
170 * which makes any writable memory region to be treated as
171 * execute-never, regardless of the value of the XN bit in the
172 * translation table.
173 *
174 * For read-only memory, rely on the MT_EXECUTE/MT_EXECUTE_NEVER
175 * attribute to figure out the value of the XN bit.
176 */
177 if ((attr & MT_RW) || (attr & MT_EXECUTE_NEVER))
178 desc |= UPPER_ATTRS(XN);
179
180 if (mem_type == MT_MEMORY) {
181 desc |= LOWER_ATTRS(ATTR_IWBWA_OWBWA_NTR_INDEX | ISH);
182 } else {
183 assert(mem_type == MT_NON_CACHEABLE);
184 desc |= LOWER_ATTRS(ATTR_NON_CACHEABLE_INDEX | OSH);
185 }
186 }
187
188 return desc;
189}
190
191/*
192 * Enumeration of actions that can be made when mapping table entries depending
193 * on the previous value in that entry and information about the region being
194 * mapped.
195 */
196typedef enum {
197
198 /* Do nothing */
199 ACTION_NONE,
200
201 /* Write a block (or page, if in level 3) entry. */
202 ACTION_WRITE_BLOCK_ENTRY,
203
204 /*
205 * Create a new table and write a table entry pointing to it. Recurse
206 * into it for further processing.
207 */
208 ACTION_CREATE_NEW_TABLE,
209
210 /*
211 * There is a table descriptor in this entry, read it and recurse into
212 * that table for further processing.
213 */
214 ACTION_RECURSE_INTO_TABLE,
215
216} action_t;
217
Antonio Nino Diazac998032017-02-27 17:23:54 +0000218#if PLAT_XLAT_TABLES_DYNAMIC
219
220/*
221 * Recursive function that writes to the translation tables and unmaps the
222 * specified region.
223 */
224static void xlat_tables_unmap_region(xlat_ctx_t *ctx, mmap_region_t *mm,
225 const uintptr_t table_base_va,
226 uint64_t *const table_base,
227 const int table_entries,
228 const int level)
229{
230 assert(level >= ctx->base_level && level <= XLAT_TABLE_LEVEL_MAX);
231
232 uint64_t *subtable;
233 uint64_t desc;
234
235 uintptr_t table_idx_va;
236 uintptr_t table_idx_end_va; /* End VA of this entry */
237
238 uintptr_t region_end_va = mm->base_va + mm->size - 1;
239
240 int table_idx;
241
242 if (mm->base_va > table_base_va) {
243 /* Find the first index of the table affected by the region. */
244 table_idx_va = mm->base_va & ~XLAT_BLOCK_MASK(level);
245
246 table_idx = (table_idx_va - table_base_va) >>
247 XLAT_ADDR_SHIFT(level);
248
249 assert(table_idx < table_entries);
250 } else {
251 /* Start from the beginning of the table. */
252 table_idx_va = table_base_va;
253 table_idx = 0;
254 }
255
256 while (table_idx < table_entries) {
257
258 table_idx_end_va = table_idx_va + XLAT_BLOCK_SIZE(level) - 1;
259
260 desc = table_base[table_idx];
261 uint64_t desc_type = desc & DESC_MASK;
262
263 action_t action = ACTION_NONE;
264
265 if ((mm->base_va <= table_idx_va) &&
266 (region_end_va >= table_idx_end_va)) {
267
268 /* Region covers all block */
269
270 if (level == 3) {
271 /*
272 * Last level, only page descriptors allowed,
273 * erase it.
274 */
275 assert(desc_type == PAGE_DESC);
276
277 action = ACTION_WRITE_BLOCK_ENTRY;
278 } else {
279 /*
280 * Other levels can have table descriptors. If
281 * so, recurse into it and erase descriptors
282 * inside it as needed. If there is a block
283 * descriptor, just erase it. If an invalid
284 * descriptor is found, this table isn't
285 * actually mapped, which shouldn't happen.
286 */
287 if (desc_type == TABLE_DESC) {
288 action = ACTION_RECURSE_INTO_TABLE;
289 } else {
290 assert(desc_type == BLOCK_DESC);
291 action = ACTION_WRITE_BLOCK_ENTRY;
292 }
293 }
294
295 } else if ((mm->base_va <= table_idx_end_va) ||
296 (region_end_va >= table_idx_va)) {
297
298 /*
299 * Region partially covers block.
300 *
301 * It can't happen in level 3.
302 *
303 * There must be a table descriptor here, if not there
304 * was a problem when mapping the region.
305 */
306
307 assert(level < 3);
308
309 assert(desc_type == TABLE_DESC);
310
311 action = ACTION_RECURSE_INTO_TABLE;
312 }
313
314 if (action == ACTION_WRITE_BLOCK_ENTRY) {
315
316 table_base[table_idx] = INVALID_DESC;
317 xlat_arch_tlbi_va(table_idx_va);
318
319 } else if (action == ACTION_RECURSE_INTO_TABLE) {
320
321 subtable = (uint64_t *)(uintptr_t)(desc & TABLE_ADDR_MASK);
322
323 /* Recurse to write into subtable */
324 xlat_tables_unmap_region(ctx, mm, table_idx_va,
325 subtable, XLAT_TABLE_ENTRIES,
326 level + 1);
327
328 /*
329 * If the subtable is now empty, remove its reference.
330 */
331 if (xlat_table_is_empty(ctx, subtable)) {
332 table_base[table_idx] = INVALID_DESC;
333 xlat_arch_tlbi_va(table_idx_va);
334 }
335
336 } else {
337 assert(action == ACTION_NONE);
338 }
339
340 table_idx++;
341 table_idx_va += XLAT_BLOCK_SIZE(level);
342
343 /* If reached the end of the region, exit */
344 if (region_end_va <= table_idx_va)
345 break;
346 }
347
348 if (level > ctx->base_level)
349 xlat_table_dec_regions_count(ctx, table_base);
350}
351
352#endif /* PLAT_XLAT_TABLES_DYNAMIC */
353
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000354/*
355 * From the given arguments, it decides which action to take when mapping the
356 * specified region.
357 */
358static action_t xlat_tables_map_region_action(const mmap_region_t *mm,
359 const int desc_type, const unsigned long long dest_pa,
360 const uintptr_t table_entry_base_va, const int level)
361{
362 uintptr_t mm_end_va = mm->base_va + mm->size - 1;
363 uintptr_t table_entry_end_va =
364 table_entry_base_va + XLAT_BLOCK_SIZE(level) - 1;
365
366 /*
367 * The descriptor types allowed depend on the current table level.
368 */
369
370 if ((mm->base_va <= table_entry_base_va) &&
371 (mm_end_va >= table_entry_end_va)) {
372
373 /*
374 * Table entry is covered by region
375 * --------------------------------
376 *
377 * This means that this table entry can describe the whole
378 * translation with this granularity in principle.
379 */
380
381 if (level == 3) {
382 /*
383 * Last level, only page descriptors are allowed.
384 */
385 if (desc_type == PAGE_DESC) {
386 /*
387 * There's another region mapped here, don't
388 * overwrite.
389 */
390 return ACTION_NONE;
391 } else {
392 assert(desc_type == INVALID_DESC);
393 return ACTION_WRITE_BLOCK_ENTRY;
394 }
395
396 } else {
397
398 /*
399 * Other levels. Table descriptors are allowed. Block
400 * descriptors too, but they have some limitations.
401 */
402
403 if (desc_type == TABLE_DESC) {
404 /* There's already a table, recurse into it. */
405 return ACTION_RECURSE_INTO_TABLE;
406
407 } else if (desc_type == INVALID_DESC) {
408 /*
409 * There's nothing mapped here, create a new
410 * entry.
411 *
412 * Check if the destination granularity allows
413 * us to use a block descriptor or we need a
414 * finer table for it.
415 *
416 * Also, check if the current level allows block
417 * descriptors. If not, create a table instead.
418 */
419 if ((dest_pa & XLAT_BLOCK_MASK(level)) ||
420 (level < MIN_LVL_BLOCK_DESC))
421 return ACTION_CREATE_NEW_TABLE;
422 else
423 return ACTION_WRITE_BLOCK_ENTRY;
424
425 } else {
426 /*
427 * There's another region mapped here, don't
428 * overwrite.
429 */
430 assert(desc_type == BLOCK_DESC);
431
432 return ACTION_NONE;
433 }
434 }
435
436 } else if ((mm->base_va <= table_entry_end_va) ||
437 (mm_end_va >= table_entry_base_va)) {
438
439 /*
440 * Region partially covers table entry
441 * -----------------------------------
442 *
443 * This means that this table entry can't describe the whole
444 * translation, a finer table is needed.
445
446 * There cannot be partial block overlaps in level 3. If that
447 * happens, some of the preliminary checks when adding the
448 * mmap region failed to detect that PA and VA must at least be
449 * aligned to PAGE_SIZE.
450 */
451 assert(level < 3);
452
453 if (desc_type == INVALID_DESC) {
454 /*
455 * The block is not fully covered by the region. Create
456 * a new table, recurse into it and try to map the
457 * region with finer granularity.
458 */
459 return ACTION_CREATE_NEW_TABLE;
460
461 } else {
462 assert(desc_type == TABLE_DESC);
463 /*
464 * The block is not fully covered by the region, but
465 * there is already a table here. Recurse into it and
466 * try to map with finer granularity.
467 *
468 * PAGE_DESC for level 3 has the same value as
469 * TABLE_DESC, but this code can't run on a level 3
470 * table because there can't be overlaps in level 3.
471 */
472 return ACTION_RECURSE_INTO_TABLE;
473 }
474 }
475
476 /*
477 * This table entry is outside of the region specified in the arguments,
478 * don't write anything to it.
479 */
480 return ACTION_NONE;
481}
482
483/*
484 * Recursive function that writes to the translation tables and maps the
Antonio Nino Diazac998032017-02-27 17:23:54 +0000485 * specified region. On success, it returns the VA of the last byte that was
486 * succesfully mapped. On error, it returns the VA of the next entry that
487 * should have been mapped.
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000488 */
Antonio Nino Diazac998032017-02-27 17:23:54 +0000489static uintptr_t xlat_tables_map_region(xlat_ctx_t *ctx, mmap_region_t *mm,
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000490 const uintptr_t table_base_va,
491 uint64_t *const table_base,
492 const int table_entries,
493 const int level)
494{
495 assert(level >= ctx->base_level && level <= XLAT_TABLE_LEVEL_MAX);
496
497 uintptr_t mm_end_va = mm->base_va + mm->size - 1;
498
499 uintptr_t table_idx_va;
500 unsigned long long table_idx_pa;
501
502 uint64_t *subtable;
503 uint64_t desc;
504
505 int table_idx;
506
507 if (mm->base_va > table_base_va) {
508 /* Find the first index of the table affected by the region. */
509 table_idx_va = mm->base_va & ~XLAT_BLOCK_MASK(level);
510
511 table_idx = (table_idx_va - table_base_va) >>
512 XLAT_ADDR_SHIFT(level);
513
514 assert(table_idx < table_entries);
515 } else {
516 /* Start from the beginning of the table. */
517 table_idx_va = table_base_va;
518 table_idx = 0;
519 }
520
Antonio Nino Diazac998032017-02-27 17:23:54 +0000521#if PLAT_XLAT_TABLES_DYNAMIC
522 if (level > ctx->base_level)
523 xlat_table_inc_regions_count(ctx, table_base);
524#endif
525
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000526 while (table_idx < table_entries) {
527
528 desc = table_base[table_idx];
529
530 table_idx_pa = mm->base_pa + table_idx_va - mm->base_va;
531
532 action_t action = xlat_tables_map_region_action(mm,
533 desc & DESC_MASK, table_idx_pa, table_idx_va, level);
534
535 if (action == ACTION_WRITE_BLOCK_ENTRY) {
536
537 table_base[table_idx] =
538 xlat_desc(mm->attr, table_idx_pa, level);
539
540 } else if (action == ACTION_CREATE_NEW_TABLE) {
541
542 subtable = xlat_table_get_empty(ctx);
Antonio Nino Diazac998032017-02-27 17:23:54 +0000543 if (subtable == NULL) {
544 /* Not enough free tables to map this region */
545 return table_idx_va;
546 }
547
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000548 /* Point to new subtable from this one. */
Antonio Nino Diazac998032017-02-27 17:23:54 +0000549 table_base[table_idx] = TABLE_DESC | (unsigned long)subtable;
550
551 /* Recurse to write into subtable */
552 uintptr_t end_va = xlat_tables_map_region(ctx, mm, table_idx_va,
553 subtable, XLAT_TABLE_ENTRIES,
554 level + 1);
555 if (end_va != table_idx_va + XLAT_BLOCK_SIZE(level) - 1)
556 return end_va;
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000557
558 } else if (action == ACTION_RECURSE_INTO_TABLE) {
559
560 subtable = (uint64_t *)(uintptr_t)(desc & TABLE_ADDR_MASK);
561 /* Recurse to write into subtable */
Antonio Nino Diazac998032017-02-27 17:23:54 +0000562 uintptr_t end_va = xlat_tables_map_region(ctx, mm, table_idx_va,
563 subtable, XLAT_TABLE_ENTRIES,
564 level + 1);
565 if (end_va != table_idx_va + XLAT_BLOCK_SIZE(level) - 1)
566 return end_va;
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000567
568 } else {
569
570 assert(action == ACTION_NONE);
571
572 }
573
574 table_idx++;
575 table_idx_va += XLAT_BLOCK_SIZE(level);
576
577 /* If reached the end of the region, exit */
578 if (mm_end_va <= table_idx_va)
579 break;
580 }
Antonio Nino Diazac998032017-02-27 17:23:54 +0000581
582 return table_idx_va - 1;
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000583}
584
585void print_mmap(mmap_region_t *const mmap)
586{
587#if LOG_LEVEL >= LOG_LEVEL_VERBOSE
588 tf_printf("mmap:\n");
589 mmap_region_t *mm = mmap;
590
591 while (mm->size) {
592 tf_printf(" VA:%p PA:0x%llx size:0x%zx attr:0x%x\n",
593 (void *)mm->base_va, mm->base_pa,
594 mm->size, mm->attr);
595 ++mm;
596 };
597 tf_printf("\n");
598#endif
599}
600
601/*
602 * Function that verifies that a region can be mapped.
603 * Returns:
604 * 0: Success, the mapping is allowed.
605 * EINVAL: Invalid values were used as arguments.
606 * ERANGE: The memory limits were surpassed.
607 * ENOMEM: There is not enough memory in the mmap array.
608 * EPERM: Region overlaps another one in an invalid way.
609 */
610static int mmap_add_region_check(xlat_ctx_t *ctx, unsigned long long base_pa,
611 uintptr_t base_va, size_t size,
Sandrine Bailleux04980a32017-04-19 14:02:23 +0100612 mmap_attr_t attr)
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000613{
614 mmap_region_t *mm = ctx->mmap;
615 unsigned long long end_pa = base_pa + size - 1;
616 uintptr_t end_va = base_va + size - 1;
617
618 if (!IS_PAGE_ALIGNED(base_pa) || !IS_PAGE_ALIGNED(base_va) ||
619 !IS_PAGE_ALIGNED(size))
620 return -EINVAL;
621
622 /* Check for overflows */
623 if ((base_pa > end_pa) || (base_va > end_va))
624 return -ERANGE;
625
626 if ((base_va + (uintptr_t)size - (uintptr_t)1) > ctx->va_max_address)
627 return -ERANGE;
628
629 if ((base_pa + (unsigned long long)size - 1ULL) > ctx->pa_max_address)
630 return -ERANGE;
631
632 /* Check that there is space in the mmap array */
633 if (ctx->mmap[ctx->mmap_num - 1].size != 0)
634 return -ENOMEM;
635
636 /* Check for PAs and VAs overlaps with all other regions */
637 for (mm = ctx->mmap; mm->size; ++mm) {
638
639 uintptr_t mm_end_va = mm->base_va + mm->size - 1;
640
641 /*
642 * Check if one of the regions is completely inside the other
643 * one.
644 */
645 int fully_overlapped_va =
646 ((base_va >= mm->base_va) && (end_va <= mm_end_va)) ||
647 ((mm->base_va >= base_va) && (mm_end_va <= end_va));
648
649 /*
650 * Full VA overlaps are only allowed if both regions are
651 * identity mapped (zero offset) or have the same VA to PA
652 * offset. Also, make sure that it's not the exact same area.
Antonio Nino Diazac998032017-02-27 17:23:54 +0000653 * This can only be done with static regions.
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000654 */
655 if (fully_overlapped_va) {
656
Antonio Nino Diazac998032017-02-27 17:23:54 +0000657#if PLAT_XLAT_TABLES_DYNAMIC
658 if ((attr & MT_DYNAMIC) || (mm->attr & MT_DYNAMIC))
659 return -EPERM;
660#endif /* PLAT_XLAT_TABLES_DYNAMIC */
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000661 if ((mm->base_va - mm->base_pa) != (base_va - base_pa))
662 return -EPERM;
663
664 if ((base_va == mm->base_va) && (size == mm->size))
665 return -EPERM;
666
667 } else {
668 /*
669 * If the regions do not have fully overlapping VAs,
670 * then they must have fully separated VAs and PAs.
671 * Partial overlaps are not allowed
672 */
673
674 unsigned long long mm_end_pa =
675 mm->base_pa + mm->size - 1;
676
677 int separated_pa =
678 (end_pa < mm->base_pa) || (base_pa > mm_end_pa);
679 int separated_va =
680 (end_va < mm->base_va) || (base_va > mm_end_va);
681
682 if (!(separated_va && separated_pa))
683 return -EPERM;
684 }
685 }
686
687 return 0;
688}
689
690void mmap_add_region_ctx(xlat_ctx_t *ctx, mmap_region_t *mm)
691{
692 mmap_region_t *mm_cursor = ctx->mmap;
693 mmap_region_t *mm_last = mm_cursor + ctx->mmap_num;
694 unsigned long long end_pa = mm->base_pa + mm->size - 1;
695 uintptr_t end_va = mm->base_va + mm->size - 1;
696 int ret;
697
698 /* Ignore empty regions */
699 if (!mm->size)
700 return;
701
Antonio Nino Diazac998032017-02-27 17:23:54 +0000702 /* Static regions must be added before initializing the xlat tables. */
703 assert(!ctx->initialized);
704
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000705 ret = mmap_add_region_check(ctx, mm->base_pa, mm->base_va, mm->size,
706 mm->attr);
707 if (ret != 0) {
708 ERROR("mmap_add_region_check() failed. error %d\n", ret);
709 assert(0);
710 return;
711 }
712
713 /*
714 * Find correct place in mmap to insert new region.
715 *
716 * 1 - Lower region VA end first.
717 * 2 - Smaller region size first.
718 *
719 * VA 0 0xFF
720 *
721 * 1st |------|
722 * 2nd |------------|
723 * 3rd |------|
724 * 4th |---|
725 * 5th |---|
726 * 6th |----------|
727 * 7th |-------------------------------------|
728 *
729 * This is required for overlapping regions only. It simplifies adding
730 * regions with the loop in xlat_tables_init_internal because the outer
731 * ones won't overwrite block or page descriptors of regions added
732 * previously.
Antonio Nino Diazac998032017-02-27 17:23:54 +0000733 *
734 * Overlapping is only allowed for static regions.
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000735 */
736
737 while ((mm_cursor->base_va + mm_cursor->size - 1) < end_va
738 && mm_cursor->size)
739 ++mm_cursor;
740
741 while ((mm_cursor->base_va + mm_cursor->size - 1 == end_va)
742 && (mm_cursor->size < mm->size))
743 ++mm_cursor;
744
745 /* Make room for new region by moving other regions up by one place */
746 memmove(mm_cursor + 1, mm_cursor,
747 (uintptr_t)mm_last - (uintptr_t)mm_cursor);
748
749 /*
750 * Check we haven't lost the empty sentinel from the end of the array.
751 * This shouldn't happen as we have checked in mmap_add_region_check
752 * that there is free space.
753 */
754 assert(mm_last->size == 0);
755
756 mm_cursor->base_pa = mm->base_pa;
757 mm_cursor->base_va = mm->base_va;
758 mm_cursor->size = mm->size;
759 mm_cursor->attr = mm->attr;
760
761 if (end_pa > ctx->max_pa)
762 ctx->max_pa = end_pa;
763 if (end_va > ctx->max_va)
764 ctx->max_va = end_va;
765}
766
Antonio Nino Diazac998032017-02-27 17:23:54 +0000767#if PLAT_XLAT_TABLES_DYNAMIC
768
769int mmap_add_dynamic_region_ctx(xlat_ctx_t *ctx, mmap_region_t *mm)
770{
771 mmap_region_t *mm_cursor = ctx->mmap;
772 mmap_region_t *mm_last = mm_cursor + ctx->mmap_num;
773 unsigned long long end_pa = mm->base_pa + mm->size - 1;
774 uintptr_t end_va = mm->base_va + mm->size - 1;
775 int ret;
776
777 /* Nothing to do */
778 if (!mm->size)
779 return 0;
780
781 ret = mmap_add_region_check(ctx, mm->base_pa, mm->base_va, mm->size, mm->attr | MT_DYNAMIC);
782 if (ret != 0)
783 return ret;
784
785 /*
786 * Find the adequate entry in the mmap array in the same way done for
787 * static regions in mmap_add_region_ctx().
788 */
789
790 while ((mm_cursor->base_va + mm_cursor->size - 1) < end_va && mm_cursor->size)
791 ++mm_cursor;
792
793 while ((mm_cursor->base_va + mm_cursor->size - 1 == end_va) && (mm_cursor->size < mm->size))
794 ++mm_cursor;
795
796 /* Make room for new region by moving other regions up by one place */
797 memmove(mm_cursor + 1, mm_cursor, (uintptr_t)mm_last - (uintptr_t)mm_cursor);
798
799 /*
800 * Check we haven't lost the empty sentinal from the end of the array.
801 * This shouldn't happen as we have checked in mmap_add_region_check
802 * that there is free space.
803 */
804 assert(mm_last->size == 0);
805
806 mm_cursor->base_pa = mm->base_pa;
807 mm_cursor->base_va = mm->base_va;
808 mm_cursor->size = mm->size;
809 mm_cursor->attr = mm->attr | MT_DYNAMIC;
810
811 /*
812 * Update the translation tables if the xlat tables are initialized. If
813 * not, this region will be mapped when they are initialized.
814 */
815 if (ctx->initialized) {
816 uintptr_t end_va = xlat_tables_map_region(ctx, mm_cursor, 0, ctx->base_table,
817 ctx->base_table_entries, ctx->base_level);
818
819 /* Failed to map, remove mmap entry, unmap and return error. */
820 if (end_va != mm_cursor->base_va + mm_cursor->size - 1) {
821 memmove(mm_cursor, mm_cursor + 1, (uintptr_t)mm_last - (uintptr_t)mm_cursor);
822
823 /*
824 * Check if the mapping function actually managed to map
825 * anything. If not, just return now.
826 */
827 if (mm_cursor->base_va >= end_va)
828 return -ENOMEM;
829
830 /*
831 * Something went wrong after mapping some table entries,
832 * undo every change done up to this point.
833 */
834 mmap_region_t unmap_mm = {
835 .base_pa = 0,
836 .base_va = mm->base_va,
837 .size = end_va - mm->base_va,
838 .attr = 0
839 };
840 xlat_tables_unmap_region(ctx, &unmap_mm, 0, ctx->base_table,
841 ctx->base_table_entries, ctx->base_level);
842
843 return -ENOMEM;
844 }
845
846 /*
847 * Make sure that all entries are written to the memory. There
848 * is no need to invalidate entries when mapping dynamic regions
849 * because new table/block/page descriptors only replace old
850 * invalid descriptors, that aren't TLB cached.
851 */
852 dsbishst();
853 }
854
855 if (end_pa > ctx->max_pa)
856 ctx->max_pa = end_pa;
857 if (end_va > ctx->max_va)
858 ctx->max_va = end_va;
859
860 return 0;
861}
862
863/*
864 * Removes the region with given base Virtual Address and size from the given
865 * context.
866 *
867 * Returns:
868 * 0: Success.
869 * EINVAL: Invalid values were used as arguments (region not found).
870 * EPERM: Tried to remove a static region.
871 */
872int mmap_remove_dynamic_region_ctx(xlat_ctx_t *ctx, uintptr_t base_va,
873 size_t size)
874{
875 mmap_region_t *mm = ctx->mmap;
876 mmap_region_t *mm_last = mm + ctx->mmap_num;
877 int update_max_va_needed = 0;
878 int update_max_pa_needed = 0;
879
880 /* Check sanity of mmap array. */
881 assert(mm[ctx->mmap_num].size == 0);
882
883 while (mm->size) {
884 if ((mm->base_va == base_va) && (mm->size == size))
885 break;
886 ++mm;
887 }
888
889 /* Check that the region was found */
890 if (mm->size == 0)
891 return -EINVAL;
892
893 /* If the region is static it can't be removed */
894 if (!(mm->attr & MT_DYNAMIC))
895 return -EPERM;
896
897 /* Check if this region is using the top VAs or PAs. */
898 if ((mm->base_va + mm->size - 1) == ctx->max_va)
899 update_max_va_needed = 1;
900 if ((mm->base_pa + mm->size - 1) == ctx->max_pa)
901 update_max_pa_needed = 1;
902
903 /* Update the translation tables if needed */
904 if (ctx->initialized) {
905 xlat_tables_unmap_region(ctx, mm, 0, ctx->base_table,
906 ctx->base_table_entries,
907 ctx->base_level);
908 xlat_arch_tlbi_va_sync();
909 }
910
911 /* Remove this region by moving the rest down by one place. */
912 memmove(mm, mm + 1, (uintptr_t)mm_last - (uintptr_t)mm);
913
914 /* Check if we need to update the max VAs and PAs */
915 if (update_max_va_needed) {
916 ctx->max_va = 0;
917 mm = ctx->mmap;
918 while (mm->size) {
919 if ((mm->base_va + mm->size - 1) > ctx->max_va)
920 ctx->max_va = mm->base_va + mm->size - 1;
921 ++mm;
922 }
923 }
924
925 if (update_max_pa_needed) {
926 ctx->max_pa = 0;
927 mm = ctx->mmap;
928 while (mm->size) {
929 if ((mm->base_pa + mm->size - 1) > ctx->max_pa)
930 ctx->max_pa = mm->base_pa + mm->size - 1;
931 ++mm;
932 }
933 }
934
935 return 0;
936}
937
938#endif /* PLAT_XLAT_TABLES_DYNAMIC */
939
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000940#if LOG_LEVEL >= LOG_LEVEL_VERBOSE
941
942/* Print the attributes of the specified block descriptor. */
943static void xlat_desc_print(uint64_t desc)
944{
945 int mem_type_index = ATTR_INDEX_GET(desc);
946
947 if (mem_type_index == ATTR_IWBWA_OWBWA_NTR_INDEX) {
948 tf_printf("MEM");
949 } else if (mem_type_index == ATTR_NON_CACHEABLE_INDEX) {
950 tf_printf("NC");
951 } else {
952 assert(mem_type_index == ATTR_DEVICE_INDEX);
953 tf_printf("DEV");
954 }
955
956 tf_printf(LOWER_ATTRS(AP_RO) & desc ? "-RO" : "-RW");
957 tf_printf(LOWER_ATTRS(NS) & desc ? "-NS" : "-S");
958 tf_printf(UPPER_ATTRS(XN) & desc ? "-XN" : "-EXEC");
959}
960
961static const char * const level_spacers[] = {
Antonio Nino Diaz755e54f2017-02-13 11:35:49 +0000962 "[LV0] ",
963 " [LV1] ",
964 " [LV2] ",
965 " [LV3] "
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000966};
967
Antonio Nino Diaz755e54f2017-02-13 11:35:49 +0000968static const char *invalid_descriptors_ommited =
969 "%s(%d invalid descriptors omitted)\n";
970
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000971/*
972 * Recursive function that reads the translation tables passed as an argument
973 * and prints their status.
974 */
975static void xlat_tables_print_internal(const uintptr_t table_base_va,
976 uint64_t *const table_base, const int table_entries,
977 const int level)
978{
979 assert(level <= XLAT_TABLE_LEVEL_MAX);
980
981 uint64_t desc;
982 uintptr_t table_idx_va = table_base_va;
983 int table_idx = 0;
984
985 size_t level_size = XLAT_BLOCK_SIZE(level);
986
Antonio Nino Diaz755e54f2017-02-13 11:35:49 +0000987 /*
988 * Keep track of how many invalid descriptors are counted in a row.
989 * Whenever multiple invalid descriptors are found, only the first one
990 * is printed, and a line is added to inform about how many descriptors
991 * have been omitted.
992 */
993 int invalid_row_count = 0;
994
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +0000995 while (table_idx < table_entries) {
996
997 desc = table_base[table_idx];
998
999 if ((desc & DESC_MASK) == INVALID_DESC) {
1000
Antonio Nino Diaz755e54f2017-02-13 11:35:49 +00001001 if (invalid_row_count == 0) {
1002 tf_printf("%sVA:%p size:0x%zx\n",
1003 level_spacers[level],
1004 (void *)table_idx_va, level_size);
1005 }
1006 invalid_row_count++;
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001007
1008 } else {
1009
Antonio Nino Diaz755e54f2017-02-13 11:35:49 +00001010 if (invalid_row_count > 1) {
1011 tf_printf(invalid_descriptors_ommited,
1012 level_spacers[level],
1013 invalid_row_count - 1);
1014 }
1015 invalid_row_count = 0;
1016
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001017 /*
1018 * Check if this is a table or a block. Tables are only
1019 * allowed in levels other than 3, but DESC_PAGE has the
1020 * same value as DESC_TABLE, so we need to check.
1021 */
1022 if (((desc & DESC_MASK) == TABLE_DESC) &&
1023 (level < XLAT_TABLE_LEVEL_MAX)) {
1024 /*
1025 * Do not print any PA for a table descriptor,
1026 * as it doesn't directly map physical memory
1027 * but instead points to the next translation
1028 * table in the translation table walk.
1029 */
1030 tf_printf("%sVA:%p size:0x%zx\n",
1031 level_spacers[level],
1032 (void *)table_idx_va, level_size);
1033
1034 uintptr_t addr_inner = desc & TABLE_ADDR_MASK;
1035
1036 xlat_tables_print_internal(table_idx_va,
1037 (uint64_t *)addr_inner,
1038 XLAT_TABLE_ENTRIES, level+1);
1039 } else {
1040 tf_printf("%sVA:%p PA:0x%llx size:0x%zx ",
1041 level_spacers[level],
1042 (void *)table_idx_va,
1043 (unsigned long long)(desc & TABLE_ADDR_MASK),
1044 level_size);
1045 xlat_desc_print(desc);
1046 tf_printf("\n");
1047 }
1048 }
1049
1050 table_idx++;
1051 table_idx_va += level_size;
1052 }
Antonio Nino Diaz755e54f2017-02-13 11:35:49 +00001053
1054 if (invalid_row_count > 1) {
1055 tf_printf(invalid_descriptors_ommited,
1056 level_spacers[level], invalid_row_count - 1);
1057 }
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001058}
1059
1060#endif /* LOG_LEVEL >= LOG_LEVEL_VERBOSE */
1061
1062void xlat_tables_print(xlat_ctx_t *ctx)
1063{
1064#if LOG_LEVEL >= LOG_LEVEL_VERBOSE
1065 xlat_tables_print_internal(0, ctx->base_table, ctx->base_table_entries,
1066 ctx->base_level);
1067#endif /* LOG_LEVEL >= LOG_LEVEL_VERBOSE */
1068}
1069
1070void init_xlation_table(xlat_ctx_t *ctx)
1071{
1072 mmap_region_t *mm = ctx->mmap;
1073
1074 /* All tables must be zeroed before mapping any region. */
1075
1076 for (int i = 0; i < ctx->base_table_entries; i++)
1077 ctx->base_table[i] = INVALID_DESC;
1078
1079 for (int j = 0; j < ctx->tables_num; j++) {
Antonio Nino Diazac998032017-02-27 17:23:54 +00001080#if PLAT_XLAT_TABLES_DYNAMIC
1081 ctx->tables_mapped_regions[j] = 0;
1082#endif
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001083 for (int i = 0; i < XLAT_TABLE_ENTRIES; i++)
1084 ctx->tables[j][i] = INVALID_DESC;
1085 }
1086
Antonio Nino Diazac998032017-02-27 17:23:54 +00001087 while (mm->size) {
1088 uintptr_t end_va = xlat_tables_map_region(ctx, mm, 0, ctx->base_table,
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001089 ctx->base_table_entries, ctx->base_level);
1090
Antonio Nino Diazac998032017-02-27 17:23:54 +00001091 if (end_va != mm->base_va + mm->size - 1) {
1092 ERROR("Not enough memory to map region:\n"
1093 " VA:%p PA:0x%llx size:0x%zx attr:0x%x\n",
1094 (void *)mm->base_va, mm->base_pa, mm->size, mm->attr);
1095 panic();
1096 }
1097
1098 mm++;
1099 }
1100
Antonio Nino Diaz233c7c12017-03-08 14:40:23 +00001101 ctx->initialized = 1;
1102}