| /* |
| * Copyright (c) 2013-2017, ARM Limited and Contributors. All rights reserved. |
| * |
| * Redistribution and use in source and binary forms, with or without |
| * modification, are permitted provided that the following conditions are met: |
| * |
| * Redistributions of source code must retain the above copyright notice, this |
| * list of conditions and the following disclaimer. |
| * |
| * Redistributions in binary form must reproduce the above copyright notice, |
| * this list of conditions and the following disclaimer in the documentation |
| * and/or other materials provided with the distribution. |
| * |
| * Neither the name of ARM nor the names of its contributors may be used |
| * to endorse or promote products derived from this software without specific |
| * prior written permission. |
| * |
| * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
| * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
| * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE |
| * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
| * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
| * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
| * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
| * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
| * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
| * POSSIBILITY OF SUCH DAMAGE. |
| */ |
| |
| #include <arch.h> |
| #include <asm_macros.S> |
| #include <assert_macros.S> |
| |
| .globl get_afflvl_shift |
| .globl mpidr_mask_lower_afflvls |
| .globl eret |
| .globl smc |
| |
| .globl zero_normalmem |
| .globl zeromem |
| .globl zeromem16 |
| .globl memcpy16 |
| |
| .globl disable_mmu_el3 |
| .globl disable_mmu_icache_el3 |
| |
| #if SUPPORT_VFP |
| .globl enable_vfp |
| #endif |
| |
| func get_afflvl_shift |
| cmp x0, #3 |
| cinc x0, x0, eq |
| mov x1, #MPIDR_AFFLVL_SHIFT |
| lsl x0, x0, x1 |
| ret |
| endfunc get_afflvl_shift |
| |
| func mpidr_mask_lower_afflvls |
| cmp x1, #3 |
| cinc x1, x1, eq |
| mov x2, #MPIDR_AFFLVL_SHIFT |
| lsl x2, x1, x2 |
| lsr x0, x0, x2 |
| lsl x0, x0, x2 |
| ret |
| endfunc mpidr_mask_lower_afflvls |
| |
| |
| func eret |
| eret |
| endfunc eret |
| |
| |
| func smc |
| smc #0 |
| endfunc smc |
| |
| /* ----------------------------------------------------------------------- |
| * void zeromem16(void *mem, unsigned int length); |
| * |
| * Initialise a memory region to 0. |
| * The memory address must be 16-byte aligned. |
| * NOTE: This function is deprecated and zeromem should be used instead. |
| * ----------------------------------------------------------------------- |
| */ |
| .equ zeromem16, zeromem |
| |
| /* ----------------------------------------------------------------------- |
| * void zero_normalmem(void *mem, unsigned int length); |
| * |
| * Initialise a region in normal memory to 0. This functions complies with the |
| * AAPCS and can be called from C code. |
| * |
| * NOTE: MMU must be enabled when using this function as it can only operate on |
| * normal memory. It is intended to be mainly used from C code when MMU |
| * is usually enabled. |
| * ----------------------------------------------------------------------- |
| */ |
| .equ zero_normalmem, zeromem_dczva |
| |
| /* ----------------------------------------------------------------------- |
| * void zeromem(void *mem, unsigned int length); |
| * |
| * Initialise a region of device memory to 0. This functions complies with the |
| * AAPCS and can be called from C code. |
| * |
| * NOTE: When data caches and MMU are enabled, zero_normalmem can usually be |
| * used instead for faster zeroing. |
| * |
| * ----------------------------------------------------------------------- |
| */ |
| func zeromem |
| /* x2 is the address past the last zeroed address */ |
| add x2, x0, x1 |
| /* |
| * Uses the fallback path that does not use DC ZVA instruction and |
| * therefore does not need enabled MMU |
| */ |
| b .Lzeromem_dczva_fallback_entry |
| endfunc zeromem |
| |
| /* ----------------------------------------------------------------------- |
| * void zeromem_dczva(void *mem, unsigned int length); |
| * |
| * Fill a region of normal memory of size "length" in bytes with null bytes. |
| * MMU must be enabled and the memory be of |
| * normal type. This is because this function internally uses the DC ZVA |
| * instruction, which generates an Alignment fault if used on any type of |
| * Device memory (see section D3.4.9 of the ARMv8 ARM, issue k). When the MMU |
| * is disabled, all memory behaves like Device-nGnRnE memory (see section |
| * D4.2.8), hence the requirement on the MMU being enabled. |
| * NOTE: The code assumes that the block size as defined in DCZID_EL0 |
| * register is at least 16 bytes. |
| * |
| * ----------------------------------------------------------------------- |
| */ |
| func zeromem_dczva |
| |
| /* |
| * The function consists of a series of loops that zero memory one byte |
| * at a time, 16 bytes at a time or using the DC ZVA instruction to |
| * zero aligned block of bytes, which is assumed to be more than 16. |
| * In the case where the DC ZVA instruction cannot be used or if the |
| * first 16 bytes loop would overflow, there is fallback path that does |
| * not use DC ZVA. |
| * Note: The fallback path is also used by the zeromem function that |
| * branches to it directly. |
| * |
| * +---------+ zeromem_dczva |
| * | entry | |
| * +----+----+ |
| * | |
| * v |
| * +---------+ |
| * | checks |>o-------+ (If any check fails, fallback) |
| * +----+----+ | |
| * | |---------------+ |
| * v | Fallback path | |
| * +------+------+ |---------------+ |
| * | 1 byte loop | | |
| * +------+------+ .Lzeromem_dczva_initial_1byte_aligned_end |
| * | | |
| * v | |
| * +-------+-------+ | |
| * | 16 bytes loop | | |
| * +-------+-------+ | |
| * | | |
| * v | |
| * +------+------+ .Lzeromem_dczva_blocksize_aligned |
| * | DC ZVA loop | | |
| * +------+------+ | |
| * +--------+ | | |
| * | | | | |
| * | v v | |
| * | +-------+-------+ .Lzeromem_dczva_final_16bytes_aligned |
| * | | 16 bytes loop | | |
| * | +-------+-------+ | |
| * | | | |
| * | v | |
| * | +------+------+ .Lzeromem_dczva_final_1byte_aligned |
| * | | 1 byte loop | | |
| * | +-------------+ | |
| * | | | |
| * | v | |
| * | +---+--+ | |
| * | | exit | | |
| * | +------+ | |
| * | | |
| * | +--------------+ +------------------+ zeromem |
| * | | +----------------| zeromem function | |
| * | | | +------------------+ |
| * | v v |
| * | +-------------+ .Lzeromem_dczva_fallback_entry |
| * | | 1 byte loop | |
| * | +------+------+ |
| * | | |
| * +-----------+ |
| */ |
| |
| /* |
| * Readable names for registers |
| * |
| * Registers x0, x1 and x2 are also set by zeromem which |
| * branches into the fallback path directly, so cursor, length and |
| * stop_address should not be retargeted to other registers. |
| */ |
| cursor .req x0 /* Start address and then current address */ |
| length .req x1 /* Length in bytes of the region to zero out */ |
| /* Reusing x1 as length is never used after block_mask is set */ |
| block_mask .req x1 /* Bitmask of the block size read in DCZID_EL0 */ |
| stop_address .req x2 /* Address past the last zeroed byte */ |
| block_size .req x3 /* Size of a block in bytes as read in DCZID_EL0 */ |
| tmp1 .req x4 |
| tmp2 .req x5 |
| |
| #if ENABLE_ASSERTIONS |
| /* |
| * Check for M bit (MMU enabled) of the current SCTLR_EL(1|3) |
| * register value and panic if the MMU is disabled. |
| */ |
| #if defined(IMAGE_BL1) || defined(IMAGE_BL31) |
| mrs tmp1, sctlr_el3 |
| #else |
| mrs tmp1, sctlr_el1 |
| #endif |
| |
| tst tmp1, #SCTLR_M_BIT |
| ASM_ASSERT(ne) |
| #endif /* ENABLE_ASSERTIONS */ |
| |
| /* stop_address is the address past the last to zero */ |
| add stop_address, cursor, length |
| |
| /* |
| * Get block_size = (log2(<block size>) >> 2) (see encoding of |
| * dczid_el0 reg) |
| */ |
| mrs block_size, dczid_el0 |
| |
| /* |
| * Select the 4 lowest bits and convert the extracted log2(<block size |
| * in words>) to <block size in bytes> |
| */ |
| ubfx block_size, block_size, #0, #4 |
| mov tmp2, #(1 << 2) |
| lsl block_size, tmp2, block_size |
| |
| #if ENABLE_ASSERTIONS |
| /* |
| * Assumes block size is at least 16 bytes to avoid manual realignment |
| * of the cursor at the end of the DCZVA loop. |
| */ |
| cmp block_size, #16 |
| ASM_ASSERT(hs) |
| #endif |
| /* |
| * Not worth doing all the setup for a region less than a block and |
| * protects against zeroing a whole block when the area to zero is |
| * smaller than that. Also, as it is assumed that the block size is at |
| * least 16 bytes, this also protects the initial aligning loops from |
| * trying to zero 16 bytes when length is less than 16. |
| */ |
| cmp length, block_size |
| b.lo .Lzeromem_dczva_fallback_entry |
| |
| /* |
| * Calculate the bitmask of the block alignment. It will never |
| * underflow as the block size is between 4 bytes and 2kB. |
| * block_mask = block_size - 1 |
| */ |
| sub block_mask, block_size, #1 |
| |
| /* |
| * length alias should not be used after this point unless it is |
| * defined as a register other than block_mask's. |
| */ |
| .unreq length |
| |
| /* |
| * If the start address is already aligned to zero block size, go |
| * straight to the cache zeroing loop. This is safe because at this |
| * point, the length cannot be smaller than a block size. |
| */ |
| tst cursor, block_mask |
| b.eq .Lzeromem_dczva_blocksize_aligned |
| |
| /* |
| * Calculate the first block-size-aligned address. It is assumed that |
| * the zero block size is at least 16 bytes. This address is the last |
| * address of this initial loop. |
| */ |
| orr tmp1, cursor, block_mask |
| add tmp1, tmp1, #1 |
| |
| /* |
| * If the addition overflows, skip the cache zeroing loops. This is |
| * quite unlikely however. |
| */ |
| cbz tmp1, .Lzeromem_dczva_fallback_entry |
| |
| /* |
| * If the first block-size-aligned address is past the last address, |
| * fallback to the simpler code. |
| */ |
| cmp tmp1, stop_address |
| b.hi .Lzeromem_dczva_fallback_entry |
| |
| /* |
| * If the start address is already aligned to 16 bytes, skip this loop. |
| * It is safe to do this because tmp1 (the stop address of the initial |
| * 16 bytes loop) will never be greater than the final stop address. |
| */ |
| tst cursor, #0xf |
| b.eq .Lzeromem_dczva_initial_1byte_aligned_end |
| |
| /* Calculate the next address aligned to 16 bytes */ |
| orr tmp2, cursor, #0xf |
| add tmp2, tmp2, #1 |
| /* If it overflows, fallback to the simple path (unlikely) */ |
| cbz tmp2, .Lzeromem_dczva_fallback_entry |
| /* |
| * Next aligned address cannot be after the stop address because the |
| * length cannot be smaller than 16 at this point. |
| */ |
| |
| /* First loop: zero byte per byte */ |
| 1: |
| strb wzr, [cursor], #1 |
| cmp cursor, tmp2 |
| b.ne 1b |
| .Lzeromem_dczva_initial_1byte_aligned_end: |
| |
| /* |
| * Second loop: we need to zero 16 bytes at a time from cursor to tmp1 |
| * before being able to use the code that deals with block-size-aligned |
| * addresses. |
| */ |
| cmp cursor, tmp1 |
| b.hs 2f |
| 1: |
| stp xzr, xzr, [cursor], #16 |
| cmp cursor, tmp1 |
| b.lo 1b |
| 2: |
| |
| /* |
| * Third loop: zero a block at a time using DC ZVA cache block zeroing |
| * instruction. |
| */ |
| .Lzeromem_dczva_blocksize_aligned: |
| /* |
| * Calculate the last block-size-aligned address. If the result equals |
| * to the start address, the loop will exit immediately. |
| */ |
| bic tmp1, stop_address, block_mask |
| |
| cmp cursor, tmp1 |
| b.hs 2f |
| 1: |
| /* Zero the block containing the cursor */ |
| dc zva, cursor |
| /* Increment the cursor by the size of a block */ |
| add cursor, cursor, block_size |
| cmp cursor, tmp1 |
| b.lo 1b |
| 2: |
| |
| /* |
| * Fourth loop: zero 16 bytes at a time and then byte per byte the |
| * remaining area |
| */ |
| .Lzeromem_dczva_final_16bytes_aligned: |
| /* |
| * Calculate the last 16 bytes aligned address. It is assumed that the |
| * block size will never be smaller than 16 bytes so that the current |
| * cursor is aligned to at least 16 bytes boundary. |
| */ |
| bic tmp1, stop_address, #15 |
| |
| cmp cursor, tmp1 |
| b.hs 2f |
| 1: |
| stp xzr, xzr, [cursor], #16 |
| cmp cursor, tmp1 |
| b.lo 1b |
| 2: |
| |
| /* Fifth and final loop: zero byte per byte */ |
| .Lzeromem_dczva_final_1byte_aligned: |
| cmp cursor, stop_address |
| b.eq 2f |
| 1: |
| strb wzr, [cursor], #1 |
| cmp cursor, stop_address |
| b.ne 1b |
| 2: |
| ret |
| |
| /* Fallback for unaligned start addresses */ |
| .Lzeromem_dczva_fallback_entry: |
| /* |
| * If the start address is already aligned to 16 bytes, skip this loop. |
| */ |
| tst cursor, #0xf |
| b.eq .Lzeromem_dczva_final_16bytes_aligned |
| |
| /* Calculate the next address aligned to 16 bytes */ |
| orr tmp1, cursor, #15 |
| add tmp1, tmp1, #1 |
| /* If it overflows, fallback to byte per byte zeroing */ |
| cbz tmp1, .Lzeromem_dczva_final_1byte_aligned |
| /* If the next aligned address is after the stop address, fall back */ |
| cmp tmp1, stop_address |
| b.hs .Lzeromem_dczva_final_1byte_aligned |
| |
| /* Fallback entry loop: zero byte per byte */ |
| 1: |
| strb wzr, [cursor], #1 |
| cmp cursor, tmp1 |
| b.ne 1b |
| |
| b .Lzeromem_dczva_final_16bytes_aligned |
| |
| .unreq cursor |
| /* |
| * length is already unreq'ed to reuse the register for another |
| * variable. |
| */ |
| .unreq stop_address |
| .unreq block_size |
| .unreq block_mask |
| .unreq tmp1 |
| .unreq tmp2 |
| endfunc zeromem_dczva |
| |
| /* -------------------------------------------------------------------------- |
| * void memcpy16(void *dest, const void *src, unsigned int length) |
| * |
| * Copy length bytes from memory area src to memory area dest. |
| * The memory areas should not overlap. |
| * Destination and source addresses must be 16-byte aligned. |
| * -------------------------------------------------------------------------- |
| */ |
| func memcpy16 |
| #if ENABLE_ASSERTIONS |
| orr x3, x0, x1 |
| tst x3, #0xf |
| ASM_ASSERT(eq) |
| #endif |
| /* copy 16 bytes at a time */ |
| m_loop16: |
| cmp x2, #16 |
| b.lo m_loop1 |
| ldp x3, x4, [x1], #16 |
| stp x3, x4, [x0], #16 |
| sub x2, x2, #16 |
| b m_loop16 |
| /* copy byte per byte */ |
| m_loop1: |
| cbz x2, m_end |
| ldrb w3, [x1], #1 |
| strb w3, [x0], #1 |
| subs x2, x2, #1 |
| b.ne m_loop1 |
| m_end: |
| ret |
| endfunc memcpy16 |
| |
| /* --------------------------------------------------------------------------- |
| * Disable the MMU at EL3 |
| * --------------------------------------------------------------------------- |
| */ |
| |
| func disable_mmu_el3 |
| mov x1, #(SCTLR_M_BIT | SCTLR_C_BIT) |
| do_disable_mmu: |
| mrs x0, sctlr_el3 |
| bic x0, x0, x1 |
| msr sctlr_el3, x0 |
| isb // ensure MMU is off |
| dsb sy |
| ret |
| endfunc disable_mmu_el3 |
| |
| |
| func disable_mmu_icache_el3 |
| mov x1, #(SCTLR_M_BIT | SCTLR_C_BIT | SCTLR_I_BIT) |
| b do_disable_mmu |
| endfunc disable_mmu_icache_el3 |
| |
| /* --------------------------------------------------------------------------- |
| * Enable the use of VFP at EL3 |
| * --------------------------------------------------------------------------- |
| */ |
| #if SUPPORT_VFP |
| func enable_vfp |
| mrs x0, cpacr_el1 |
| orr x0, x0, #CPACR_VFP_BITS |
| msr cpacr_el1, x0 |
| mrs x0, cptr_el3 |
| mov x1, #AARCH64_CPTR_TFP |
| bic x0, x0, x1 |
| msr cptr_el3, x0 |
| isb |
| ret |
| endfunc enable_vfp |
| #endif |