Mikael Olsson | 7da6619 | 2021-02-12 17:30:22 +0100 | [diff] [blame] | 1 | /* |
Joshua Pimm | 6bc8067 | 2022-10-19 15:46:27 +0100 | [diff] [blame] | 2 | * Copyright (c) 2021-2023, Arm Limited. All rights reserved. |
Mikael Olsson | 7da6619 | 2021-02-12 17:30:22 +0100 | [diff] [blame] | 3 | * |
| 4 | * SPDX-License-Identifier: BSD-3-Clause |
| 5 | */ |
| 6 | |
| 7 | #include <stdint.h> |
| 8 | #include <stdbool.h> |
| 9 | |
| 10 | #include <common/debug.h> |
| 11 | #include <common/runtime_svc.h> |
| 12 | #include <drivers/arm/ethosn.h> |
| 13 | #include <drivers/delay_timer.h> |
| 14 | #include <lib/mmio.h> |
Mikael Olsson | 3288b46 | 2022-08-15 17:12:58 +0200 | [diff] [blame] | 15 | #include <lib/utils_def.h> |
Mikael Olsson | 7da6619 | 2021-02-12 17:30:22 +0100 | [diff] [blame] | 16 | #include <plat/arm/common/fconf_ethosn_getter.h> |
| 17 | |
Rajasekaran Kalidoss | f8a18b8 | 2022-11-16 17:16:44 +0100 | [diff] [blame] | 18 | #include <platform_def.h> |
| 19 | |
Mikael Olsson | a7df0d6 | 2023-01-13 09:56:41 +0100 | [diff] [blame] | 20 | #if ARM_ETHOSN_NPU_TZMP1 |
| 21 | #include "ethosn_big_fw.h" |
| 22 | #endif |
| 23 | |
Laurent Carlier | 5205df2 | 2021-09-16 15:10:35 +0100 | [diff] [blame] | 24 | /* |
Mikael Olsson | 3288b46 | 2022-08-15 17:12:58 +0200 | [diff] [blame] | 25 | * Number of Arm(R) Ethos(TM)-N NPU (NPU) devices available |
Laurent Carlier | 5205df2 | 2021-09-16 15:10:35 +0100 | [diff] [blame] | 26 | */ |
Mikael Olsson | 3288b46 | 2022-08-15 17:12:58 +0200 | [diff] [blame] | 27 | #define ETHOSN_NUM_DEVICES \ |
| 28 | FCONF_GET_PROPERTY(hw_config, ethosn_config, num_devices) |
Mikael Olsson | 7da6619 | 2021-02-12 17:30:22 +0100 | [diff] [blame] | 29 | |
Mikael Olsson | 3288b46 | 2022-08-15 17:12:58 +0200 | [diff] [blame] | 30 | #define ETHOSN_GET_DEVICE(dev_idx) \ |
| 31 | FCONF_GET_PROPERTY(hw_config, ethosn_device, dev_idx) |
Mikael Olsson | 7da6619 | 2021-02-12 17:30:22 +0100 | [diff] [blame] | 32 | |
| 33 | /* NPU core sec registry address */ |
| 34 | #define ETHOSN_CORE_SEC_REG(core_addr, reg_offset) \ |
| 35 | (core_addr + reg_offset) |
| 36 | |
Mikael Olsson | d6cedcb | 2023-01-27 18:53:48 +0100 | [diff] [blame] | 37 | #define ETHOSN_FW_VA_BASE 0x20000000UL |
| 38 | #define ETHOSN_WORKING_DATA_VA_BASE 0x40000000UL |
| 39 | #define ETHOSN_COMMAND_STREAM_VA_BASE 0x60000000UL |
| 40 | |
Mikael Olsson | 7da6619 | 2021-02-12 17:30:22 +0100 | [diff] [blame] | 41 | /* Reset timeout in us */ |
| 42 | #define ETHOSN_RESET_TIMEOUT_US U(10 * 1000 * 1000) |
| 43 | #define ETHOSN_RESET_WAIT_US U(1) |
| 44 | |
Mikael Olsson | fe9677f | 2023-02-10 11:36:19 +0100 | [diff] [blame] | 45 | #define ETHOSN_AUX_FEAT_LEVEL_IRQ U(0x1) |
| 46 | #define ETHOSN_AUX_FEAT_STASHING U(0x2) |
| 47 | |
| 48 | #define SEC_AUXCTLR_REG U(0x0024) |
| 49 | #define SEC_AUXCTLR_VAL U(0x80) |
| 50 | #define SEC_AUXCTLR_LEVEL_IRQ_VAL U(0x04) |
| 51 | #define SEC_AUXCTLR_STASHING_VAL U(0xA5000000) |
| 52 | |
Mikael Olsson | 7da6619 | 2021-02-12 17:30:22 +0100 | [diff] [blame] | 53 | #define SEC_DEL_REG U(0x0004) |
Mikael Olsson | c7d3a27 | 2023-02-10 16:59:23 +0100 | [diff] [blame^] | 54 | #if ARM_ETHOSN_NPU_TZMP1 |
| 55 | #define SEC_DEL_VAL U(0x808) |
| 56 | #else |
Mikael Olsson | fe9677f | 2023-02-10 11:36:19 +0100 | [diff] [blame] | 57 | #define SEC_DEL_VAL U(0x80C) |
Mikael Olsson | c7d3a27 | 2023-02-10 16:59:23 +0100 | [diff] [blame^] | 58 | #endif |
Mikael Olsson | 7da6619 | 2021-02-12 17:30:22 +0100 | [diff] [blame] | 59 | #define SEC_DEL_EXCC_MASK U(0x20) |
| 60 | |
| 61 | #define SEC_SECCTLR_REG U(0x0010) |
Mikael Olsson | bfd9da7 | 2023-01-11 10:36:22 +0100 | [diff] [blame] | 62 | /* Set bit[10] = 1 to workaround erratum 2838783 */ |
| 63 | #define SEC_SECCTLR_VAL U(0x403) |
Mikael Olsson | 7da6619 | 2021-02-12 17:30:22 +0100 | [diff] [blame] | 64 | |
Mikael Olsson | df0bb04 | 2023-02-10 16:59:03 +0100 | [diff] [blame] | 65 | #define SEC_DEL_ADDR_EXT_REG U(0x201C) |
| 66 | #define SEC_DEL_ADDR_EXT_VAL U(0x1) |
Mikael Olsson | 7da6619 | 2021-02-12 17:30:22 +0100 | [diff] [blame] | 67 | |
| 68 | #define SEC_SYSCTRL0_REG U(0x0018) |
Mikael Olsson | c7d3a27 | 2023-02-10 16:59:23 +0100 | [diff] [blame^] | 69 | #define SEC_SYSCTRL0_CPU_WAIT U(1) |
Mikael Olsson | 47675f2 | 2022-11-04 15:01:02 +0100 | [diff] [blame] | 70 | #define SEC_SYSCTRL0_SLEEPING U(1U << 4) |
Mikael Olsson | c7d3a27 | 2023-02-10 16:59:23 +0100 | [diff] [blame^] | 71 | #define SEC_SYSCTRL0_INITVTOR_MASK U(0x1FFFFF80) |
Mikael Olsson | 7da6619 | 2021-02-12 17:30:22 +0100 | [diff] [blame] | 72 | #define SEC_SYSCTRL0_SOFT_RESET U(3U << 29) |
| 73 | #define SEC_SYSCTRL0_HARD_RESET U(1U << 31) |
| 74 | |
Mikael Olsson | fe9677f | 2023-02-10 11:36:19 +0100 | [diff] [blame] | 75 | #define SEC_SYSCTRL1_REG U(0x001C) |
| 76 | #define SEC_SYSCTRL1_VAL U(0x180110) |
| 77 | |
Rajasekaran Kalidoss | f8a18b8 | 2022-11-16 17:16:44 +0100 | [diff] [blame] | 78 | #define SEC_NSAID_REG_BASE U(0x3004) |
| 79 | #define SEC_NSAID_OFFSET U(0x1000) |
| 80 | |
Mikael Olsson | 3288b46 | 2022-08-15 17:12:58 +0200 | [diff] [blame] | 81 | #define SEC_MMUSID_REG_BASE U(0x3008) |
| 82 | #define SEC_MMUSID_OFFSET U(0x1000) |
| 83 | |
Mikael Olsson | df0bb04 | 2023-02-10 16:59:03 +0100 | [diff] [blame] | 84 | #define SEC_ADDR_EXT_REG_BASE U(0x3018) |
| 85 | #define SEC_ADDR_EXT_OFFSET U(0x1000) |
| 86 | #define SEC_ADDR_EXT_SHIFT U(0x14) |
| 87 | #define SEC_ADDR_EXT_MASK U(0x1FFFFE00) |
| 88 | |
| 89 | #define SEC_ATTR_CTLR_REG_BASE U(0x3010) |
| 90 | #define SEC_ATTR_CTLR_OFFSET U(0x1000) |
| 91 | #define SEC_ATTR_CTLR_NUM U(9) |
| 92 | #define SEC_ATTR_CTLR_VAL U(0x1) |
| 93 | |
Mikael Olsson | a7df0d6 | 2023-01-13 09:56:41 +0100 | [diff] [blame] | 94 | #define SEC_NPU_ID_REG U(0xF000) |
| 95 | #define SEC_NPU_ID_ARCH_VER_SHIFT U(0X10) |
| 96 | |
Mikael Olsson | c7d3a27 | 2023-02-10 16:59:23 +0100 | [diff] [blame^] | 97 | #define FIRMWARE_STREAM_INDEX U(0x0) |
| 98 | #define PLE_STREAM_INDEX U(0x4) |
Rajasekaran Kalidoss | f8a18b8 | 2022-11-16 17:16:44 +0100 | [diff] [blame] | 99 | #define INPUT_STREAM_INDEX U(0x6) |
| 100 | #define INTERMEDIATE_STREAM_INDEX U(0x7) |
| 101 | #define OUTPUT_STREAM_INDEX U(0x8) |
| 102 | |
Mikael Olsson | df0bb04 | 2023-02-10 16:59:03 +0100 | [diff] [blame] | 103 | #define TO_EXTEND_ADDR(addr) \ |
| 104 | ((addr >> SEC_ADDR_EXT_SHIFT) & SEC_ADDR_EXT_MASK) |
| 105 | |
Mikael Olsson | a7df0d6 | 2023-01-13 09:56:41 +0100 | [diff] [blame] | 106 | #if ARM_ETHOSN_NPU_TZMP1 |
| 107 | CASSERT(ARM_ETHOSN_NPU_FW_IMAGE_BASE > 0U, assert_ethosn_invalid_fw_image_base); |
| 108 | static const struct ethosn_big_fw *big_fw; |
Mikael Olsson | c7d3a27 | 2023-02-10 16:59:23 +0100 | [diff] [blame^] | 109 | |
| 110 | #define FW_INITVTOR_ADDR(big_fw) \ |
| 111 | ((ETHOSN_FW_VA_BASE + big_fw->vector_table_offset) & \ |
| 112 | SEC_SYSCTRL0_INITVTOR_MASK) |
| 113 | |
| 114 | #define SYSCTRL0_INITVTOR_ADDR(value) \ |
| 115 | (value & SEC_SYSCTRL0_INITVTOR_MASK) |
| 116 | |
Mikael Olsson | a7df0d6 | 2023-01-13 09:56:41 +0100 | [diff] [blame] | 117 | #endif |
| 118 | |
Mikael Olsson | 3288b46 | 2022-08-15 17:12:58 +0200 | [diff] [blame] | 119 | static bool ethosn_get_device_and_core(uintptr_t core_addr, |
| 120 | const struct ethosn_device_t **dev_match, |
| 121 | const struct ethosn_core_t **core_match) |
Laurent Carlier | 5205df2 | 2021-09-16 15:10:35 +0100 | [diff] [blame] | 122 | { |
Mikael Olsson | 3288b46 | 2022-08-15 17:12:58 +0200 | [diff] [blame] | 123 | uint32_t dev_idx; |
| 124 | uint32_t core_idx; |
| 125 | |
| 126 | for (dev_idx = 0U; dev_idx < ETHOSN_NUM_DEVICES; ++dev_idx) { |
| 127 | const struct ethosn_device_t *dev = ETHOSN_GET_DEVICE(dev_idx); |
| 128 | |
| 129 | for (core_idx = 0U; core_idx < dev->num_cores; ++core_idx) { |
| 130 | const struct ethosn_core_t *core = &(dev->cores[core_idx]); |
| 131 | |
| 132 | if (core->addr == core_addr) { |
| 133 | *dev_match = dev; |
| 134 | *core_match = core; |
| 135 | return true; |
| 136 | } |
Laurent Carlier | 5205df2 | 2021-09-16 15:10:35 +0100 | [diff] [blame] | 137 | } |
| 138 | } |
| 139 | |
Mikael Olsson | 3288b46 | 2022-08-15 17:12:58 +0200 | [diff] [blame] | 140 | WARN("ETHOSN: Unknown core address given to SMC call.\n"); |
Laurent Carlier | 5205df2 | 2021-09-16 15:10:35 +0100 | [diff] [blame] | 141 | return false; |
| 142 | } |
| 143 | |
Rajasekaran Kalidoss | f8a18b8 | 2022-11-16 17:16:44 +0100 | [diff] [blame] | 144 | #if ARM_ETHOSN_NPU_TZMP1 |
Mikael Olsson | a7df0d6 | 2023-01-13 09:56:41 +0100 | [diff] [blame] | 145 | static uint32_t ethosn_core_read_arch_version(uintptr_t core_addr) |
| 146 | { |
| 147 | uint32_t npu_id = mmio_read_32(ETHOSN_CORE_SEC_REG(core_addr, |
| 148 | SEC_NPU_ID_REG)); |
| 149 | |
| 150 | return (npu_id >> SEC_NPU_ID_ARCH_VER_SHIFT); |
| 151 | } |
| 152 | |
Rajasekaran Kalidoss | f8a18b8 | 2022-11-16 17:16:44 +0100 | [diff] [blame] | 153 | static void ethosn_configure_stream_nsaid(const struct ethosn_core_t *core, |
| 154 | bool is_protected) |
| 155 | { |
| 156 | size_t i; |
| 157 | uint32_t streams[9] = {0, 0, 0, 0, 0, 0, 0, 0, 0}; |
| 158 | |
Mikael Olsson | c7d3a27 | 2023-02-10 16:59:23 +0100 | [diff] [blame^] | 159 | streams[FIRMWARE_STREAM_INDEX] = ARM_ETHOSN_NPU_PROT_FW_NSAID; |
| 160 | streams[PLE_STREAM_INDEX] = ARM_ETHOSN_NPU_PROT_FW_NSAID; |
| 161 | |
Rajasekaran Kalidoss | f8a18b8 | 2022-11-16 17:16:44 +0100 | [diff] [blame] | 162 | if (is_protected) { |
| 163 | streams[INPUT_STREAM_INDEX] = ARM_ETHOSN_NPU_PROT_DATA_NSAID; |
| 164 | streams[INTERMEDIATE_STREAM_INDEX] = |
| 165 | ARM_ETHOSN_NPU_PROT_DATA_NSAID; |
| 166 | streams[OUTPUT_STREAM_INDEX] = ARM_ETHOSN_NPU_PROT_DATA_NSAID; |
| 167 | } |
| 168 | |
| 169 | for (i = 0U; i < ARRAY_SIZE(streams); ++i) { |
| 170 | const uintptr_t reg_addr = SEC_NSAID_REG_BASE + |
| 171 | (SEC_NSAID_OFFSET * i); |
| 172 | mmio_write_32(ETHOSN_CORE_SEC_REG(core->addr, reg_addr), |
| 173 | streams[i]); |
| 174 | } |
| 175 | } |
Mikael Olsson | c7d3a27 | 2023-02-10 16:59:23 +0100 | [diff] [blame^] | 176 | |
| 177 | static void ethosn_configure_vector_table(uintptr_t core_addr) |
| 178 | { |
| 179 | mmio_setbits_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_SYSCTRL0_REG), |
| 180 | FW_INITVTOR_ADDR(big_fw)); |
| 181 | } |
| 182 | |
Rajasekaran Kalidoss | f8a18b8 | 2022-11-16 17:16:44 +0100 | [diff] [blame] | 183 | #endif |
| 184 | |
Mikael Olsson | fe9677f | 2023-02-10 11:36:19 +0100 | [diff] [blame] | 185 | static void ethosn_configure_events(uintptr_t core_addr) |
| 186 | { |
| 187 | mmio_write_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_SYSCTRL1_REG), SEC_SYSCTRL1_VAL); |
| 188 | } |
| 189 | |
| 190 | static bool ethosn_configure_aux_features(const struct ethosn_device_t *device, |
| 191 | uintptr_t core_addr, |
| 192 | uint32_t features) |
| 193 | { |
| 194 | uint32_t val = SEC_AUXCTLR_VAL; |
| 195 | |
| 196 | if (features & ETHOSN_AUX_FEAT_LEVEL_IRQ) { |
| 197 | val |= SEC_AUXCTLR_LEVEL_IRQ_VAL; |
| 198 | } |
| 199 | |
| 200 | if (features & ETHOSN_AUX_FEAT_STASHING) { |
| 201 | /* Stashing can't be used with reserved memory */ |
| 202 | if (device->has_reserved_memory) { |
| 203 | return false; |
| 204 | } |
| 205 | |
| 206 | val |= SEC_AUXCTLR_STASHING_VAL; |
| 207 | } |
| 208 | |
| 209 | mmio_setbits_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_AUXCTLR_REG), val); |
| 210 | |
| 211 | return true; |
| 212 | } |
| 213 | |
Mikael Olsson | 3288b46 | 2022-08-15 17:12:58 +0200 | [diff] [blame] | 214 | static void ethosn_configure_smmu_streams(const struct ethosn_device_t *device, |
| 215 | const struct ethosn_core_t *core, |
| 216 | uint32_t asset_alloc_idx) |
| 217 | { |
| 218 | const struct ethosn_main_allocator_t *main_alloc = |
| 219 | &(core->main_allocator); |
| 220 | const struct ethosn_asset_allocator_t *asset_alloc = |
| 221 | &(device->asset_allocators[asset_alloc_idx]); |
| 222 | const uint32_t streams[9] = { |
| 223 | main_alloc->firmware.stream_id, |
| 224 | main_alloc->working_data.stream_id, |
| 225 | asset_alloc->command_stream.stream_id, |
| 226 | 0U, /* Not used*/ |
| 227 | main_alloc->firmware.stream_id, |
| 228 | asset_alloc->weight_data.stream_id, |
| 229 | asset_alloc->buffer_data.stream_id, |
| 230 | asset_alloc->intermediate_data.stream_id, |
| 231 | asset_alloc->buffer_data.stream_id |
| 232 | }; |
| 233 | size_t i; |
| 234 | |
| 235 | for (i = 0U; i < ARRAY_SIZE(streams); ++i) { |
| 236 | const uintptr_t reg_addr = SEC_MMUSID_REG_BASE + |
| 237 | (SEC_MMUSID_OFFSET * i); |
| 238 | mmio_write_32(ETHOSN_CORE_SEC_REG(core->addr, reg_addr), |
| 239 | streams[i]); |
| 240 | } |
| 241 | } |
| 242 | |
Mikael Olsson | df0bb04 | 2023-02-10 16:59:03 +0100 | [diff] [blame] | 243 | static void ethosn_configure_stream_addr_extends(const struct ethosn_device_t *device, |
| 244 | uintptr_t core_addr) |
| 245 | { |
| 246 | uint32_t addr_extends[3] = { 0 }; |
| 247 | size_t i; |
| 248 | |
| 249 | if (device->has_reserved_memory) { |
| 250 | const uint32_t addr = TO_EXTEND_ADDR(device->reserved_memory_addr); |
| 251 | |
| 252 | addr_extends[0] = addr; |
| 253 | addr_extends[1] = addr; |
| 254 | addr_extends[2] = addr; |
| 255 | } else { |
| 256 | addr_extends[0] = TO_EXTEND_ADDR(ETHOSN_FW_VA_BASE); |
| 257 | addr_extends[1] = TO_EXTEND_ADDR(ETHOSN_WORKING_DATA_VA_BASE); |
| 258 | addr_extends[2] = TO_EXTEND_ADDR(ETHOSN_COMMAND_STREAM_VA_BASE); |
| 259 | } |
| 260 | |
| 261 | for (i = 0U; i < ARRAY_SIZE(addr_extends); ++i) { |
| 262 | const uintptr_t reg_addr = SEC_ADDR_EXT_REG_BASE + |
| 263 | (SEC_ADDR_EXT_OFFSET * i); |
| 264 | mmio_write_32(ETHOSN_CORE_SEC_REG(core_addr, reg_addr), |
| 265 | addr_extends[i]); |
| 266 | } |
| 267 | } |
| 268 | |
| 269 | static void ethosn_configure_stream_attr_ctlr(uintptr_t core_addr) |
| 270 | { |
| 271 | size_t i; |
| 272 | |
| 273 | for (i = 0U; i < SEC_ATTR_CTLR_NUM; ++i) { |
| 274 | const uintptr_t reg_addr = SEC_ATTR_CTLR_REG_BASE + |
| 275 | (SEC_ATTR_CTLR_OFFSET * i); |
| 276 | mmio_write_32(ETHOSN_CORE_SEC_REG(core_addr, reg_addr), |
| 277 | SEC_ATTR_CTLR_VAL); |
| 278 | } |
| 279 | } |
| 280 | |
Mikael Olsson | 7da6619 | 2021-02-12 17:30:22 +0100 | [diff] [blame] | 281 | static void ethosn_delegate_to_ns(uintptr_t core_addr) |
| 282 | { |
| 283 | mmio_setbits_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_SECCTLR_REG), |
| 284 | SEC_SECCTLR_VAL); |
| 285 | |
| 286 | mmio_setbits_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_DEL_REG), |
| 287 | SEC_DEL_VAL); |
| 288 | |
Mikael Olsson | 7da6619 | 2021-02-12 17:30:22 +0100 | [diff] [blame] | 289 | mmio_setbits_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_DEL_ADDR_EXT_REG), |
| 290 | SEC_DEL_ADDR_EXT_VAL); |
| 291 | } |
| 292 | |
Laurent Carlier | 5205df2 | 2021-09-16 15:10:35 +0100 | [diff] [blame] | 293 | static int ethosn_is_sec(uintptr_t core_addr) |
Mikael Olsson | 7da6619 | 2021-02-12 17:30:22 +0100 | [diff] [blame] | 294 | { |
Laurent Carlier | 5205df2 | 2021-09-16 15:10:35 +0100 | [diff] [blame] | 295 | if ((mmio_read_32(ETHOSN_CORE_SEC_REG(core_addr, SEC_DEL_REG)) |
Mikael Olsson | 7da6619 | 2021-02-12 17:30:22 +0100 | [diff] [blame] | 296 | & SEC_DEL_EXCC_MASK) != 0U) { |
| 297 | return 0; |
| 298 | } |
| 299 | |
| 300 | return 1; |
| 301 | } |
| 302 | |
Mikael Olsson | 47675f2 | 2022-11-04 15:01:02 +0100 | [diff] [blame] | 303 | static int ethosn_core_is_sleeping(uintptr_t core_addr) |
| 304 | { |
| 305 | const uintptr_t sysctrl0_reg = |
| 306 | ETHOSN_CORE_SEC_REG(core_addr, SEC_SYSCTRL0_REG); |
| 307 | const uint32_t sleeping_mask = SEC_SYSCTRL0_SLEEPING; |
| 308 | |
| 309 | return ((mmio_read_32(sysctrl0_reg) & sleeping_mask) == sleeping_mask); |
| 310 | } |
| 311 | |
Mikael Olsson | f663803 | 2023-01-27 18:26:36 +0100 | [diff] [blame] | 312 | static bool ethosn_core_reset(uintptr_t core_addr, bool hard_reset) |
Mikael Olsson | 7da6619 | 2021-02-12 17:30:22 +0100 | [diff] [blame] | 313 | { |
| 314 | unsigned int timeout; |
| 315 | const uintptr_t sysctrl0_reg = |
| 316 | ETHOSN_CORE_SEC_REG(core_addr, SEC_SYSCTRL0_REG); |
Mikael Olsson | f663803 | 2023-01-27 18:26:36 +0100 | [diff] [blame] | 317 | const uint32_t reset_val = hard_reset ? SEC_SYSCTRL0_HARD_RESET : |
| 318 | SEC_SYSCTRL0_SOFT_RESET; |
Mikael Olsson | 7da6619 | 2021-02-12 17:30:22 +0100 | [diff] [blame] | 319 | |
| 320 | mmio_write_32(sysctrl0_reg, reset_val); |
| 321 | |
| 322 | /* Wait for reset to complete */ |
| 323 | for (timeout = 0U; timeout < ETHOSN_RESET_TIMEOUT_US; |
| 324 | timeout += ETHOSN_RESET_WAIT_US) { |
| 325 | |
| 326 | if ((mmio_read_32(sysctrl0_reg) & reset_val) == 0U) { |
| 327 | break; |
| 328 | } |
| 329 | |
| 330 | udelay(ETHOSN_RESET_WAIT_US); |
| 331 | } |
| 332 | |
| 333 | return timeout < ETHOSN_RESET_TIMEOUT_US; |
| 334 | } |
| 335 | |
Mikael Olsson | c7d3a27 | 2023-02-10 16:59:23 +0100 | [diff] [blame^] | 336 | static int ethosn_core_boot_fw(uintptr_t core_addr) |
| 337 | { |
| 338 | #if ARM_ETHOSN_NPU_TZMP1 |
| 339 | const uintptr_t sysctrl0_reg = ETHOSN_CORE_SEC_REG(core_addr, SEC_SYSCTRL0_REG); |
| 340 | const uint32_t sysctrl0_val = mmio_read_32(sysctrl0_reg); |
| 341 | const bool waiting = (sysctrl0_val & SEC_SYSCTRL0_CPU_WAIT); |
| 342 | |
| 343 | if (!waiting) { |
| 344 | WARN("ETHOSN: Firmware is already running.\n"); |
| 345 | return ETHOSN_INVALID_STATE; |
| 346 | } |
| 347 | |
| 348 | if (SYSCTRL0_INITVTOR_ADDR(sysctrl0_val) != FW_INITVTOR_ADDR(big_fw)) { |
| 349 | WARN("ETHOSN: Unknown vector table won't boot firmware.\n"); |
| 350 | return ETHOSN_INVALID_CONFIGURATION; |
| 351 | } |
| 352 | |
| 353 | mmio_clrbits_32(sysctrl0_reg, SEC_SYSCTRL0_CPU_WAIT); |
| 354 | |
| 355 | return ETHOSN_SUCCESS; |
| 356 | #else |
| 357 | return ETHOSN_NOT_SUPPORTED; |
| 358 | #endif |
| 359 | } |
| 360 | |
Mikael Olsson | f663803 | 2023-01-27 18:26:36 +0100 | [diff] [blame] | 361 | static int ethosn_core_full_reset(const struct ethosn_device_t *device, |
| 362 | const struct ethosn_core_t *core, |
| 363 | bool hard_reset, |
| 364 | u_register_t asset_alloc_idx, |
Mikael Olsson | fe9677f | 2023-02-10 11:36:19 +0100 | [diff] [blame] | 365 | u_register_t is_protected, |
| 366 | u_register_t aux_features) |
Mikael Olsson | 7da6619 | 2021-02-12 17:30:22 +0100 | [diff] [blame] | 367 | { |
Mikael Olsson | f663803 | 2023-01-27 18:26:36 +0100 | [diff] [blame] | 368 | if (!device->has_reserved_memory && |
| 369 | asset_alloc_idx >= device->num_allocators) { |
| 370 | WARN("ETHOSN: Unknown asset allocator index given to SMC call.\n"); |
| 371 | return ETHOSN_UNKNOWN_ALLOCATOR_IDX; |
Mikael Olsson | 7da6619 | 2021-02-12 17:30:22 +0100 | [diff] [blame] | 372 | } |
| 373 | |
Mikael Olsson | f663803 | 2023-01-27 18:26:36 +0100 | [diff] [blame] | 374 | if (!ethosn_core_reset(core->addr, hard_reset)) { |
| 375 | return ETHOSN_FAILURE; |
Mikael Olsson | 7da6619 | 2021-02-12 17:30:22 +0100 | [diff] [blame] | 376 | } |
| 377 | |
Mikael Olsson | fe9677f | 2023-02-10 11:36:19 +0100 | [diff] [blame] | 378 | if (!ethosn_configure_aux_features(device, core->addr, aux_features)) { |
| 379 | return ETHOSN_INVALID_CONFIGURATION; |
| 380 | } |
| 381 | |
| 382 | ethosn_configure_events(core->addr); |
| 383 | |
Mikael Olsson | f663803 | 2023-01-27 18:26:36 +0100 | [diff] [blame] | 384 | if (!device->has_reserved_memory) { |
| 385 | ethosn_configure_smmu_streams(device, core, asset_alloc_idx); |
| 386 | |
| 387 | #if ARM_ETHOSN_NPU_TZMP1 |
| 388 | ethosn_configure_stream_nsaid(core, is_protected); |
| 389 | #endif |
Mikael Olsson | 7da6619 | 2021-02-12 17:30:22 +0100 | [diff] [blame] | 390 | } |
| 391 | |
Mikael Olsson | df0bb04 | 2023-02-10 16:59:03 +0100 | [diff] [blame] | 392 | ethosn_configure_stream_addr_extends(device, core->addr); |
| 393 | ethosn_configure_stream_attr_ctlr(core->addr); |
| 394 | |
Mikael Olsson | c7d3a27 | 2023-02-10 16:59:23 +0100 | [diff] [blame^] | 395 | #if ARM_ETHOSN_NPU_TZMP1 |
| 396 | ethosn_configure_vector_table(core->addr); |
| 397 | #endif |
| 398 | |
Mikael Olsson | f663803 | 2023-01-27 18:26:36 +0100 | [diff] [blame] | 399 | ethosn_delegate_to_ns(core->addr); |
| 400 | |
| 401 | return ETHOSN_SUCCESS; |
| 402 | } |
| 403 | |
| 404 | static uintptr_t ethosn_smc_core_reset_handler(const struct ethosn_device_t *device, |
| 405 | const struct ethosn_core_t *core, |
| 406 | bool hard_reset, |
| 407 | u_register_t asset_alloc_idx, |
| 408 | u_register_t reset_type, |
| 409 | u_register_t is_protected, |
Mikael Olsson | fe9677f | 2023-02-10 11:36:19 +0100 | [diff] [blame] | 410 | u_register_t aux_features, |
Mikael Olsson | f663803 | 2023-01-27 18:26:36 +0100 | [diff] [blame] | 411 | void *handle) |
| 412 | { |
| 413 | int ret; |
| 414 | |
| 415 | switch (reset_type) { |
| 416 | case ETHOSN_RESET_TYPE_FULL: |
| 417 | ret = ethosn_core_full_reset(device, core, hard_reset, |
Mikael Olsson | fe9677f | 2023-02-10 11:36:19 +0100 | [diff] [blame] | 418 | asset_alloc_idx, is_protected, |
| 419 | aux_features); |
Mikael Olsson | f663803 | 2023-01-27 18:26:36 +0100 | [diff] [blame] | 420 | break; |
| 421 | case ETHOSN_RESET_TYPE_HALT: |
| 422 | ret = ethosn_core_reset(core->addr, hard_reset) ? ETHOSN_SUCCESS : ETHOSN_FAILURE; |
| 423 | break; |
| 424 | default: |
| 425 | WARN("ETHOSN: Invalid reset type given to SMC call.\n"); |
| 426 | ret = ETHOSN_INVALID_PARAMETER; |
| 427 | break; |
Laurent Carlier | 5205df2 | 2021-09-16 15:10:35 +0100 | [diff] [blame] | 428 | } |
| 429 | |
Mikael Olsson | f663803 | 2023-01-27 18:26:36 +0100 | [diff] [blame] | 430 | SMC_RET1(handle, ret); |
| 431 | } |
| 432 | |
| 433 | static uintptr_t ethosn_smc_core_handler(uint32_t fid, |
| 434 | u_register_t core_addr, |
| 435 | u_register_t asset_alloc_idx, |
| 436 | u_register_t reset_type, |
| 437 | u_register_t is_protected, |
Mikael Olsson | fe9677f | 2023-02-10 11:36:19 +0100 | [diff] [blame] | 438 | u_register_t aux_features, |
Mikael Olsson | f663803 | 2023-01-27 18:26:36 +0100 | [diff] [blame] | 439 | void *handle) |
| 440 | { |
| 441 | bool hard_reset = false; |
| 442 | const struct ethosn_device_t *device = NULL; |
| 443 | const struct ethosn_core_t *core = NULL; |
| 444 | |
Mikael Olsson | 3288b46 | 2022-08-15 17:12:58 +0200 | [diff] [blame] | 445 | if (!ethosn_get_device_and_core(core_addr, &device, &core)) { |
Laurent Carlier | 5205df2 | 2021-09-16 15:10:35 +0100 | [diff] [blame] | 446 | SMC_RET1(handle, ETHOSN_UNKNOWN_CORE_ADDRESS); |
| 447 | } |
| 448 | |
Laurent Carlier | 5205df2 | 2021-09-16 15:10:35 +0100 | [diff] [blame] | 449 | switch (fid) { |
Mikael Olsson | 7da6619 | 2021-02-12 17:30:22 +0100 | [diff] [blame] | 450 | case ETHOSN_FNUM_IS_SEC: |
Mikael Olsson | 3288b46 | 2022-08-15 17:12:58 +0200 | [diff] [blame] | 451 | SMC_RET1(handle, ethosn_is_sec(core->addr)); |
Mikael Olsson | 47675f2 | 2022-11-04 15:01:02 +0100 | [diff] [blame] | 452 | case ETHOSN_FNUM_IS_SLEEPING: |
| 453 | SMC_RET1(handle, ethosn_core_is_sleeping(core->addr)); |
Mikael Olsson | 7da6619 | 2021-02-12 17:30:22 +0100 | [diff] [blame] | 454 | case ETHOSN_FNUM_HARD_RESET: |
Mikael Olsson | f663803 | 2023-01-27 18:26:36 +0100 | [diff] [blame] | 455 | hard_reset = true; |
Mikael Olsson | 7da6619 | 2021-02-12 17:30:22 +0100 | [diff] [blame] | 456 | /* Fallthrough */ |
| 457 | case ETHOSN_FNUM_SOFT_RESET: |
Mikael Olsson | f663803 | 2023-01-27 18:26:36 +0100 | [diff] [blame] | 458 | return ethosn_smc_core_reset_handler(device, core, |
| 459 | hard_reset, |
| 460 | asset_alloc_idx, |
| 461 | reset_type, |
| 462 | is_protected, |
Mikael Olsson | fe9677f | 2023-02-10 11:36:19 +0100 | [diff] [blame] | 463 | aux_features, |
Mikael Olsson | f663803 | 2023-01-27 18:26:36 +0100 | [diff] [blame] | 464 | handle); |
Mikael Olsson | c7d3a27 | 2023-02-10 16:59:23 +0100 | [diff] [blame^] | 465 | case ETHOSN_FNUM_BOOT_FW: |
| 466 | SMC_RET1(handle, ethosn_core_boot_fw(core->addr)); |
Mikael Olsson | f663803 | 2023-01-27 18:26:36 +0100 | [diff] [blame] | 467 | default: |
| 468 | WARN("ETHOSN: Unimplemented SMC call: 0x%x\n", fid); |
| 469 | SMC_RET1(handle, SMC_UNK); |
| 470 | } |
| 471 | } |
| 472 | |
Mikael Olsson | d6cedcb | 2023-01-27 18:53:48 +0100 | [diff] [blame] | 473 | static uintptr_t ethosn_smc_fw_prop_handler(u_register_t fw_property, |
| 474 | void *handle) |
| 475 | { |
| 476 | #if ARM_ETHOSN_NPU_TZMP1 |
| 477 | switch (fw_property) { |
| 478 | case ETHOSN_FW_PROP_VERSION: |
| 479 | SMC_RET4(handle, ETHOSN_SUCCESS, |
| 480 | big_fw->fw_ver_major, |
| 481 | big_fw->fw_ver_minor, |
| 482 | big_fw->fw_ver_patch); |
| 483 | case ETHOSN_FW_PROP_MEM_INFO: |
| 484 | SMC_RET3(handle, ETHOSN_SUCCESS, |
| 485 | ((void *)big_fw) + big_fw->offset, |
| 486 | big_fw->size); |
| 487 | case ETHOSN_FW_PROP_OFFSETS: |
| 488 | SMC_RET3(handle, ETHOSN_SUCCESS, |
| 489 | big_fw->ple_offset, |
| 490 | big_fw->unpriv_stack_offset); |
| 491 | case ETHOSN_FW_PROP_VA_MAP: |
| 492 | SMC_RET4(handle, ETHOSN_SUCCESS, |
| 493 | ETHOSN_FW_VA_BASE, |
| 494 | ETHOSN_WORKING_DATA_VA_BASE, |
| 495 | ETHOSN_COMMAND_STREAM_VA_BASE); |
| 496 | default: |
| 497 | WARN("ETHOSN: Unknown firmware property\n"); |
| 498 | SMC_RET1(handle, ETHOSN_INVALID_PARAMETER); |
| 499 | } |
| 500 | #else |
| 501 | SMC_RET1(handle, ETHOSN_NOT_SUPPORTED); |
| 502 | #endif |
| 503 | } |
| 504 | |
Mikael Olsson | f663803 | 2023-01-27 18:26:36 +0100 | [diff] [blame] | 505 | uintptr_t ethosn_smc_handler(uint32_t smc_fid, |
| 506 | u_register_t x1, |
| 507 | u_register_t x2, |
| 508 | u_register_t x3, |
| 509 | u_register_t x4, |
| 510 | void *cookie, |
| 511 | void *handle, |
| 512 | u_register_t flags) |
| 513 | { |
| 514 | const uint32_t fid = smc_fid & FUNCID_NUM_MASK; |
Mikael Olsson | 3288b46 | 2022-08-15 17:12:58 +0200 | [diff] [blame] | 515 | |
Mikael Olsson | f663803 | 2023-01-27 18:26:36 +0100 | [diff] [blame] | 516 | /* Only SiP fast calls are expected */ |
| 517 | if ((GET_SMC_TYPE(smc_fid) != SMC_TYPE_FAST) || |
| 518 | (GET_SMC_OEN(smc_fid) != OEN_SIP_START)) { |
| 519 | SMC_RET1(handle, SMC_UNK); |
| 520 | } |
Rajasekaran Kalidoss | f8a18b8 | 2022-11-16 17:16:44 +0100 | [diff] [blame] | 521 | |
Mikael Olsson | f663803 | 2023-01-27 18:26:36 +0100 | [diff] [blame] | 522 | /* Truncate parameters to 32-bits for SMC32 */ |
| 523 | if (GET_SMC_CC(smc_fid) == SMC_32) { |
| 524 | x1 &= 0xFFFFFFFF; |
| 525 | x2 &= 0xFFFFFFFF; |
| 526 | x3 &= 0xFFFFFFFF; |
| 527 | x4 &= 0xFFFFFFFF; |
| 528 | } |
Mikael Olsson | 3288b46 | 2022-08-15 17:12:58 +0200 | [diff] [blame] | 529 | |
Mikael Olsson | c7d3a27 | 2023-02-10 16:59:23 +0100 | [diff] [blame^] | 530 | if (!is_ethosn_fid(smc_fid) || (fid > ETHOSN_FNUM_BOOT_FW)) { |
Mikael Olsson | f663803 | 2023-01-27 18:26:36 +0100 | [diff] [blame] | 531 | WARN("ETHOSN: Unknown SMC call: 0x%x\n", smc_fid); |
Mikael Olsson | 7da6619 | 2021-02-12 17:30:22 +0100 | [diff] [blame] | 532 | SMC_RET1(handle, SMC_UNK); |
| 533 | } |
Mikael Olsson | f663803 | 2023-01-27 18:26:36 +0100 | [diff] [blame] | 534 | |
Mikael Olsson | d6cedcb | 2023-01-27 18:53:48 +0100 | [diff] [blame] | 535 | switch (fid) { |
| 536 | case ETHOSN_FNUM_VERSION: |
Mikael Olsson | f663803 | 2023-01-27 18:26:36 +0100 | [diff] [blame] | 537 | SMC_RET2(handle, ETHOSN_VERSION_MAJOR, ETHOSN_VERSION_MINOR); |
Mikael Olsson | d6cedcb | 2023-01-27 18:53:48 +0100 | [diff] [blame] | 538 | case ETHOSN_FNUM_GET_FW_PROP: |
| 539 | return ethosn_smc_fw_prop_handler(x1, handle); |
Mikael Olsson | f663803 | 2023-01-27 18:26:36 +0100 | [diff] [blame] | 540 | } |
| 541 | |
Mikael Olsson | fe9677f | 2023-02-10 11:36:19 +0100 | [diff] [blame] | 542 | return ethosn_smc_core_handler(fid, x1, x2, x3, x4, |
| 543 | SMC_GET_GP(handle, CTX_GPREG_X5), |
| 544 | handle); |
Mikael Olsson | 7da6619 | 2021-02-12 17:30:22 +0100 | [diff] [blame] | 545 | } |
Mikael Olsson | 461bf7d | 2023-01-18 18:05:15 +0100 | [diff] [blame] | 546 | |
| 547 | int ethosn_smc_setup(void) |
| 548 | { |
Mikael Olsson | a7df0d6 | 2023-01-13 09:56:41 +0100 | [diff] [blame] | 549 | #if ARM_ETHOSN_NPU_TZMP1 |
| 550 | struct ethosn_device_t *dev; |
| 551 | uint32_t arch_ver; |
| 552 | #endif |
| 553 | |
Mikael Olsson | 461bf7d | 2023-01-18 18:05:15 +0100 | [diff] [blame] | 554 | if (ETHOSN_NUM_DEVICES == 0U) { |
| 555 | ERROR("ETHOSN: No NPU found\n"); |
| 556 | return ETHOSN_FAILURE; |
| 557 | } |
| 558 | |
Mikael Olsson | a7df0d6 | 2023-01-13 09:56:41 +0100 | [diff] [blame] | 559 | #if ARM_ETHOSN_NPU_TZMP1 |
| 560 | |
| 561 | /* Only one NPU core is supported in the TZMP1 setup */ |
| 562 | if ((ETHOSN_NUM_DEVICES != 1U) || |
| 563 | (ETHOSN_GET_DEVICE(0U)->num_cores != 1U)) { |
| 564 | ERROR("ETHOSN: TZMP1 doesn't support multiple NPU cores\n"); |
| 565 | return ETHOSN_FAILURE; |
| 566 | } |
| 567 | |
| 568 | dev = ETHOSN_GET_DEVICE(0U); |
Mikael Olsson | c7d3a27 | 2023-02-10 16:59:23 +0100 | [diff] [blame^] | 569 | if (dev->has_reserved_memory) { |
| 570 | ERROR("ETHOSN: TZMP1 doesn't support using reserved memory\n"); |
| 571 | return ETHOSN_FAILURE; |
| 572 | } |
| 573 | |
Mikael Olsson | a7df0d6 | 2023-01-13 09:56:41 +0100 | [diff] [blame] | 574 | arch_ver = ethosn_core_read_arch_version(dev->cores[0U].addr); |
| 575 | big_fw = (struct ethosn_big_fw *)ARM_ETHOSN_NPU_FW_IMAGE_BASE; |
| 576 | |
| 577 | if (!ethosn_big_fw_verify_header(big_fw, arch_ver)) { |
| 578 | return ETHOSN_FAILURE; |
| 579 | } |
| 580 | |
| 581 | NOTICE("ETHOSN: TZMP1 setup succeeded with firmware version %u.%u.%u\n", |
| 582 | big_fw->fw_ver_major, big_fw->fw_ver_minor, |
| 583 | big_fw->fw_ver_patch); |
| 584 | #else |
| 585 | NOTICE("ETHOSN: Setup succeeded\n"); |
| 586 | #endif |
| 587 | |
Mikael Olsson | 461bf7d | 2023-01-18 18:05:15 +0100 | [diff] [blame] | 588 | return 0; |
| 589 | } |