Tom Rini | 10e4779 | 2018-05-06 17:58:06 -0400 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0+ |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 2 | /* |
| 3 | * Copyright (C) 2015 Google, Inc |
| 4 | * |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 5 | * Based on code from the coreboot file of the same name |
| 6 | */ |
| 7 | |
| 8 | #include <common.h> |
| 9 | #include <cpu.h> |
| 10 | #include <dm.h> |
| 11 | #include <errno.h> |
Simon Glass | 0f2af88 | 2020-05-10 11:40:05 -0600 | [diff] [blame] | 12 | #include <log.h> |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 13 | #include <malloc.h> |
Miao Yan | 9210627 | 2016-05-22 19:37:17 -0700 | [diff] [blame] | 14 | #include <qfw.h> |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 15 | #include <asm/atomic.h> |
| 16 | #include <asm/cpu.h> |
| 17 | #include <asm/interrupt.h> |
Simon Glass | 4a30bbb | 2020-07-17 08:48:16 -0600 | [diff] [blame] | 18 | #include <asm/io.h> |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 19 | #include <asm/lapic.h> |
Simon Glass | c17d450 | 2016-03-11 22:07:09 -0700 | [diff] [blame] | 20 | #include <asm/microcode.h> |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 21 | #include <asm/mp.h> |
Bin Meng | e5d0500 | 2015-06-23 12:18:50 +0800 | [diff] [blame] | 22 | #include <asm/msr.h> |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 23 | #include <asm/mtrr.h> |
Bin Meng | e5d0500 | 2015-06-23 12:18:50 +0800 | [diff] [blame] | 24 | #include <asm/processor.h> |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 25 | #include <asm/sipi.h> |
| 26 | #include <dm/device-internal.h> |
| 27 | #include <dm/uclass-internal.h> |
Miao Yan | 35f54b2 | 2016-01-07 01:32:04 -0800 | [diff] [blame] | 28 | #include <dm/lists.h> |
| 29 | #include <dm/root.h> |
Simon Glass | dbd7954 | 2020-05-10 11:40:11 -0600 | [diff] [blame] | 30 | #include <linux/delay.h> |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 31 | #include <linux/linkage.h> |
| 32 | |
Simon Glass | daa93d9 | 2015-07-31 09:31:31 -0600 | [diff] [blame] | 33 | DECLARE_GLOBAL_DATA_PTR; |
| 34 | |
Simon Glass | 00906cb | 2020-07-17 08:48:30 -0600 | [diff] [blame] | 35 | /* |
| 36 | * Setting up multiprocessing |
| 37 | * |
| 38 | * See https://www.intel.com/content/www/us/en/intelligent-systems/intel-boot-loader-development-kit/minimal-intel-architecture-boot-loader-paper.html |
| 39 | * |
| 40 | * Note that this file refers to the boot CPU (the one U-Boot is running on) as |
| 41 | * the BSP (BootStrap Processor) and the others as APs (Application Processors). |
| 42 | * |
| 43 | * This module works by loading some setup code into RAM at AP_DEFAULT_BASE and |
| 44 | * telling each AP to execute it. The code that each AP runs is in |
| 45 | * sipi_vector.S (see ap_start16) which includes a struct sipi_params at the |
| 46 | * end of it. Those parameters are set up by the C code. |
| 47 | |
| 48 | * Setting up is handled by load_sipi_vector(). It inits the common block of |
| 49 | * parameters (sipi_params) which tell the APs what to do. This block includes |
| 50 | * microcode and the MTTRs (Memory-Type-Range Registers) from the main CPU. |
| 51 | * There is also an ap_count which each AP increments as it starts up, so the |
| 52 | * BSP can tell how many checked in. |
| 53 | * |
| 54 | * The APs are started with a SIPI (Startup Inter-Processor Interrupt) which |
| 55 | * tells an AP to start executing at a particular address, in this case |
| 56 | * AP_DEFAULT_BASE which contains the code copied from ap_start16. This protocol |
| 57 | * is handled by start_aps(). |
| 58 | * |
| 59 | * After being started, each AP runs the code in ap_start16, switches to 32-bit |
| 60 | * mode, runs the code at ap_start, then jumps to c_handler which is ap_init(). |
| 61 | * This runs a very simple 'flight plan' described in mp_steps(). This sets up |
| 62 | * the CPU and waits for further instructions by looking at its entry in |
| 63 | * ap_callbacks[]. Note that the flight plan is only actually run for each CPU |
| 64 | * in bsp_do_flight_plan(): once the BSP completes each flight record, it sets |
| 65 | * mp_flight_record->barrier to 1 to allow the APs to executed the record one |
| 66 | * by one. |
| 67 | * |
| 68 | * CPUS are numbered sequentially from 0 using the device tree: |
| 69 | * |
| 70 | * cpus { |
| 71 | * u-boot,dm-pre-reloc; |
| 72 | * #address-cells = <1>; |
| 73 | * #size-cells = <0>; |
| 74 | * |
| 75 | * cpu@0 { |
| 76 | * u-boot,dm-pre-reloc; |
| 77 | * device_type = "cpu"; |
| 78 | * compatible = "intel,apl-cpu"; |
| 79 | * reg = <0>; |
| 80 | * intel,apic-id = <0>; |
| 81 | * }; |
| 82 | * |
| 83 | * cpu@1 { |
| 84 | * device_type = "cpu"; |
| 85 | * compatible = "intel,apl-cpu"; |
| 86 | * reg = <1>; |
| 87 | * intel,apic-id = <2>; |
| 88 | * }; |
| 89 | * |
Simon Glass | 6feac81 | 2020-12-16 21:20:22 -0700 | [diff] [blame] | 90 | * Here the 'reg' property is the CPU number and then is placed in dev_seq(cpu) |
Simon Glass | 00906cb | 2020-07-17 08:48:30 -0600 | [diff] [blame] | 91 | * so that we can index into ap_callbacks[] using that. The APIC ID is different |
| 92 | * and may not be sequential (it typically is if hyperthreading is supported). |
| 93 | * |
| 94 | * Once APs are inited they wait in ap_wait_for_instruction() for instructions. |
| 95 | * Instructions come in the form of a function to run. This logic is in |
| 96 | * mp_run_on_cpus() which supports running on any one AP, all APs, just the BSP |
| 97 | * or all CPUs. The BSP logic is handled directly in mp_run_on_cpus(), by |
| 98 | * calling the function. For the APs, callback information is stored in a |
| 99 | * single, common struct mp_callback and a pointer to this is written to each |
| 100 | * AP's slot in ap_callbacks[] by run_ap_work(). All APs get the message even |
| 101 | * if it is only for one of them. When an AP notices a message it checks whether |
| 102 | * it should call the function (see check in ap_wait_for_instruction()) and then |
| 103 | * does so if needed. After that it sets its slot to NULL to indicate it is |
| 104 | * done. |
| 105 | * |
| 106 | * While U-Boot is running it can use mp_run_on_cpus() to run code on the APs. |
| 107 | * An example of this is the 'mtrr' command which allows reading and changing |
| 108 | * the MTRRs on all CPUs. |
| 109 | * |
| 110 | * Before U-Boot exits it calls mp_park_aps() which tells all CPUs to halt by |
| 111 | * executing a 'hlt' instruction. That allows them to be used by Linux when it |
| 112 | * starts up. |
| 113 | */ |
| 114 | |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 115 | /* This also needs to match the sipi.S assembly code for saved MSR encoding */ |
Simon Glass | 00906cb | 2020-07-17 08:48:30 -0600 | [diff] [blame] | 116 | struct __packed saved_msr { |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 117 | uint32_t index; |
| 118 | uint32_t lo; |
| 119 | uint32_t hi; |
Simon Glass | 00906cb | 2020-07-17 08:48:30 -0600 | [diff] [blame] | 120 | }; |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 121 | |
Simon Glass | 00906cb | 2020-07-17 08:48:30 -0600 | [diff] [blame] | 122 | /** |
| 123 | * struct mp_flight_plan - Holds the flight plan |
| 124 | * |
| 125 | * @num_records: Number of flight records |
| 126 | * @records: Pointer to each record |
| 127 | */ |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 128 | struct mp_flight_plan { |
| 129 | int num_records; |
| 130 | struct mp_flight_record *records; |
| 131 | }; |
| 132 | |
Simon Glass | 4a30bbb | 2020-07-17 08:48:16 -0600 | [diff] [blame] | 133 | /** |
| 134 | * struct mp_callback - Callback information for APs |
| 135 | * |
| 136 | * @func: Function to run |
| 137 | * @arg: Argument to pass to the function |
Simon Glass | 6feac81 | 2020-12-16 21:20:22 -0700 | [diff] [blame] | 138 | * @logical_cpu_number: Either a CPU number (i.e. dev_seq(cpu) or a special |
Simon Glass | 4a30bbb | 2020-07-17 08:48:16 -0600 | [diff] [blame] | 139 | * value like MP_SELECT_BSP. It tells the AP whether it should process this |
| 140 | * callback |
| 141 | */ |
| 142 | struct mp_callback { |
Simon Glass | 6871dff | 2020-07-17 08:48:19 -0600 | [diff] [blame] | 143 | mp_run_func func; |
Simon Glass | 4a30bbb | 2020-07-17 08:48:16 -0600 | [diff] [blame] | 144 | void *arg; |
| 145 | int logical_cpu_number; |
| 146 | }; |
| 147 | |
Simon Glass | 00906cb | 2020-07-17 08:48:30 -0600 | [diff] [blame] | 148 | /* Stores the flight plan so that APs can find it */ |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 149 | static struct mp_flight_plan mp_info; |
| 150 | |
Simon Glass | 4a30bbb | 2020-07-17 08:48:16 -0600 | [diff] [blame] | 151 | /* |
| 152 | * ap_callbacks - Callback mailbox array |
| 153 | * |
| 154 | * Array of callback, one entry for each available CPU, indexed by the CPU |
Simon Glass | 6feac81 | 2020-12-16 21:20:22 -0700 | [diff] [blame] | 155 | * number, which is dev_seq(cpu). The entry for the main CPU is never used. |
Simon Glass | 4a30bbb | 2020-07-17 08:48:16 -0600 | [diff] [blame] | 156 | * When this is NULL, there is no pending work for the CPU to run. When |
| 157 | * non-NULL it points to the mp_callback structure. This is shared between all |
| 158 | * CPUs, so should only be written by the main CPU. |
| 159 | */ |
| 160 | static struct mp_callback **ap_callbacks; |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 161 | |
| 162 | static inline void barrier_wait(atomic_t *b) |
| 163 | { |
| 164 | while (atomic_read(b) == 0) |
| 165 | asm("pause"); |
| 166 | mfence(); |
| 167 | } |
| 168 | |
| 169 | static inline void release_barrier(atomic_t *b) |
| 170 | { |
| 171 | mfence(); |
| 172 | atomic_set(b, 1); |
| 173 | } |
| 174 | |
Bin Meng | e5d0500 | 2015-06-23 12:18:50 +0800 | [diff] [blame] | 175 | static inline void stop_this_cpu(void) |
| 176 | { |
| 177 | /* Called by an AP when it is ready to halt and wait for a new task */ |
| 178 | for (;;) |
| 179 | cpu_hlt(); |
| 180 | } |
| 181 | |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 182 | /* Returns 1 if timeout waiting for APs. 0 if target APs found */ |
| 183 | static int wait_for_aps(atomic_t *val, int target, int total_delay, |
| 184 | int delay_step) |
| 185 | { |
| 186 | int timeout = 0; |
| 187 | int delayed = 0; |
| 188 | |
| 189 | while (atomic_read(val) != target) { |
| 190 | udelay(delay_step); |
| 191 | delayed += delay_step; |
| 192 | if (delayed >= total_delay) { |
| 193 | timeout = 1; |
| 194 | break; |
| 195 | } |
| 196 | } |
| 197 | |
| 198 | return timeout; |
| 199 | } |
| 200 | |
| 201 | static void ap_do_flight_plan(struct udevice *cpu) |
| 202 | { |
| 203 | int i; |
| 204 | |
| 205 | for (i = 0; i < mp_info.num_records; i++) { |
| 206 | struct mp_flight_record *rec = &mp_info.records[i]; |
| 207 | |
| 208 | atomic_inc(&rec->cpus_entered); |
| 209 | barrier_wait(&rec->barrier); |
| 210 | |
| 211 | if (rec->ap_call != NULL) |
| 212 | rec->ap_call(cpu, rec->ap_arg); |
| 213 | } |
| 214 | } |
| 215 | |
Miao Yan | 2ee1000 | 2016-01-07 01:32:02 -0800 | [diff] [blame] | 216 | static int find_cpu_by_apic_id(int apic_id, struct udevice **devp) |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 217 | { |
| 218 | struct udevice *dev; |
| 219 | |
| 220 | *devp = NULL; |
| 221 | for (uclass_find_first_device(UCLASS_CPU, &dev); |
| 222 | dev; |
| 223 | uclass_find_next_device(&dev)) { |
Simon Glass | b75b15b | 2020-12-03 16:55:23 -0700 | [diff] [blame] | 224 | struct cpu_plat *plat = dev_get_parent_plat(dev); |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 225 | |
| 226 | if (plat->cpu_id == apic_id) { |
| 227 | *devp = dev; |
| 228 | return 0; |
| 229 | } |
| 230 | } |
| 231 | |
| 232 | return -ENOENT; |
| 233 | } |
| 234 | |
| 235 | /* |
| 236 | * By the time APs call ap_init() caching has been setup, and microcode has |
| 237 | * been loaded |
| 238 | */ |
| 239 | static void ap_init(unsigned int cpu_index) |
| 240 | { |
| 241 | struct udevice *dev; |
| 242 | int apic_id; |
| 243 | int ret; |
| 244 | |
| 245 | /* Ensure the local apic is enabled */ |
| 246 | enable_lapic(); |
| 247 | |
| 248 | apic_id = lapicid(); |
Miao Yan | 2ee1000 | 2016-01-07 01:32:02 -0800 | [diff] [blame] | 249 | ret = find_cpu_by_apic_id(apic_id, &dev); |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 250 | if (ret) { |
| 251 | debug("Unknown CPU apic_id %x\n", apic_id); |
| 252 | goto done; |
| 253 | } |
| 254 | |
| 255 | debug("AP: slot %d apic_id %x, dev %s\n", cpu_index, apic_id, |
| 256 | dev ? dev->name : "(apic_id not found)"); |
| 257 | |
Simon Glass | 4a30bbb | 2020-07-17 08:48:16 -0600 | [diff] [blame] | 258 | /* |
| 259 | * Walk the flight plan, which only returns if CONFIG_SMP_AP_WORK is not |
| 260 | * enabled |
| 261 | */ |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 262 | ap_do_flight_plan(dev); |
| 263 | |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 264 | done: |
| 265 | stop_this_cpu(); |
| 266 | } |
| 267 | |
| 268 | static const unsigned int fixed_mtrrs[NUM_FIXED_MTRRS] = { |
| 269 | MTRR_FIX_64K_00000_MSR, MTRR_FIX_16K_80000_MSR, MTRR_FIX_16K_A0000_MSR, |
| 270 | MTRR_FIX_4K_C0000_MSR, MTRR_FIX_4K_C8000_MSR, MTRR_FIX_4K_D0000_MSR, |
| 271 | MTRR_FIX_4K_D8000_MSR, MTRR_FIX_4K_E0000_MSR, MTRR_FIX_4K_E8000_MSR, |
| 272 | MTRR_FIX_4K_F0000_MSR, MTRR_FIX_4K_F8000_MSR, |
| 273 | }; |
| 274 | |
| 275 | static inline struct saved_msr *save_msr(int index, struct saved_msr *entry) |
| 276 | { |
| 277 | msr_t msr; |
| 278 | |
| 279 | msr = msr_read(index); |
| 280 | entry->index = index; |
| 281 | entry->lo = msr.lo; |
| 282 | entry->hi = msr.hi; |
| 283 | |
| 284 | /* Return the next entry */ |
| 285 | entry++; |
| 286 | return entry; |
| 287 | } |
| 288 | |
| 289 | static int save_bsp_msrs(char *start, int size) |
| 290 | { |
| 291 | int msr_count; |
| 292 | int num_var_mtrrs; |
| 293 | struct saved_msr *msr_entry; |
| 294 | int i; |
| 295 | msr_t msr; |
| 296 | |
| 297 | /* Determine number of MTRRs need to be saved */ |
| 298 | msr = msr_read(MTRR_CAP_MSR); |
| 299 | num_var_mtrrs = msr.lo & 0xff; |
| 300 | |
| 301 | /* 2 * num_var_mtrrs for base and mask. +1 for IA32_MTRR_DEF_TYPE */ |
| 302 | msr_count = 2 * num_var_mtrrs + NUM_FIXED_MTRRS + 1; |
| 303 | |
| 304 | if ((msr_count * sizeof(struct saved_msr)) > size) { |
Simon Glass | 17dbe89 | 2016-03-06 19:28:22 -0700 | [diff] [blame] | 305 | printf("Cannot mirror all %d msrs\n", msr_count); |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 306 | return -ENOSPC; |
| 307 | } |
| 308 | |
| 309 | msr_entry = (void *)start; |
| 310 | for (i = 0; i < NUM_FIXED_MTRRS; i++) |
| 311 | msr_entry = save_msr(fixed_mtrrs[i], msr_entry); |
| 312 | |
| 313 | for (i = 0; i < num_var_mtrrs; i++) { |
| 314 | msr_entry = save_msr(MTRR_PHYS_BASE_MSR(i), msr_entry); |
| 315 | msr_entry = save_msr(MTRR_PHYS_MASK_MSR(i), msr_entry); |
| 316 | } |
| 317 | |
| 318 | msr_entry = save_msr(MTRR_DEF_TYPE_MSR, msr_entry); |
| 319 | |
| 320 | return msr_count; |
| 321 | } |
| 322 | |
Miao Yan | 6067762 | 2016-01-07 01:32:03 -0800 | [diff] [blame] | 323 | static int load_sipi_vector(atomic_t **ap_countp, int num_cpus) |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 324 | { |
| 325 | struct sipi_params_16bit *params16; |
| 326 | struct sipi_params *params; |
| 327 | static char msr_save[512]; |
| 328 | char *stack; |
| 329 | ulong addr; |
| 330 | int code_len; |
| 331 | int size; |
| 332 | int ret; |
| 333 | |
| 334 | /* Copy in the code */ |
| 335 | code_len = ap_start16_code_end - ap_start16; |
| 336 | debug("Copying SIPI code to %x: %d bytes\n", AP_DEFAULT_BASE, |
| 337 | code_len); |
| 338 | memcpy((void *)AP_DEFAULT_BASE, ap_start16, code_len); |
| 339 | |
| 340 | addr = AP_DEFAULT_BASE + (ulong)sipi_params_16bit - (ulong)ap_start16; |
| 341 | params16 = (struct sipi_params_16bit *)addr; |
| 342 | params16->ap_start = (uint32_t)ap_start; |
| 343 | params16->gdt = (uint32_t)gd->arch.gdt; |
| 344 | params16->gdt_limit = X86_GDT_SIZE - 1; |
| 345 | debug("gdt = %x, gdt_limit = %x\n", params16->gdt, params16->gdt_limit); |
| 346 | |
| 347 | params = (struct sipi_params *)sipi_params; |
| 348 | debug("SIPI 32-bit params at %p\n", params); |
| 349 | params->idt_ptr = (uint32_t)x86_get_idt(); |
| 350 | |
| 351 | params->stack_size = CONFIG_AP_STACK_SIZE; |
Miao Yan | 6067762 | 2016-01-07 01:32:03 -0800 | [diff] [blame] | 352 | size = params->stack_size * num_cpus; |
Stephen Warren | 5923b59 | 2016-02-12 14:27:56 -0700 | [diff] [blame] | 353 | stack = memalign(4096, size); |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 354 | if (!stack) |
| 355 | return -ENOMEM; |
| 356 | params->stack_top = (u32)(stack + size); |
Andy Shevchenko | 43b3ac5 | 2017-02-17 16:49:00 +0300 | [diff] [blame] | 357 | #if !defined(CONFIG_QEMU) && !defined(CONFIG_HAVE_FSP) && \ |
| 358 | !defined(CONFIG_INTEL_MID) |
Simon Glass | 8dda587 | 2016-03-11 22:07:11 -0700 | [diff] [blame] | 359 | params->microcode_ptr = ucode_base; |
| 360 | debug("Microcode at %x\n", params->microcode_ptr); |
| 361 | #endif |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 362 | params->msr_table_ptr = (u32)msr_save; |
| 363 | ret = save_bsp_msrs(msr_save, sizeof(msr_save)); |
| 364 | if (ret < 0) |
| 365 | return ret; |
| 366 | params->msr_count = ret; |
| 367 | |
| 368 | params->c_handler = (uint32_t)&ap_init; |
| 369 | |
| 370 | *ap_countp = ¶ms->ap_count; |
| 371 | atomic_set(*ap_countp, 0); |
| 372 | debug("SIPI vector is ready\n"); |
| 373 | |
| 374 | return 0; |
| 375 | } |
| 376 | |
| 377 | static int check_cpu_devices(int expected_cpus) |
| 378 | { |
| 379 | int i; |
| 380 | |
| 381 | for (i = 0; i < expected_cpus; i++) { |
| 382 | struct udevice *dev; |
| 383 | int ret; |
| 384 | |
| 385 | ret = uclass_find_device(UCLASS_CPU, i, &dev); |
| 386 | if (ret) { |
| 387 | debug("Cannot find CPU %d in device tree\n", i); |
| 388 | return ret; |
| 389 | } |
| 390 | } |
| 391 | |
| 392 | return 0; |
| 393 | } |
| 394 | |
| 395 | /* Returns 1 for timeout. 0 on success */ |
Simon Glass | 17dbe89 | 2016-03-06 19:28:22 -0700 | [diff] [blame] | 396 | static int apic_wait_timeout(int total_delay, const char *msg) |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 397 | { |
| 398 | int total = 0; |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 399 | |
Simon Glass | 17dbe89 | 2016-03-06 19:28:22 -0700 | [diff] [blame] | 400 | if (!(lapic_read(LAPIC_ICR) & LAPIC_ICR_BUSY)) |
| 401 | return 0; |
| 402 | |
| 403 | debug("Waiting for %s...", msg); |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 404 | while (lapic_read(LAPIC_ICR) & LAPIC_ICR_BUSY) { |
Simon Glass | 17dbe89 | 2016-03-06 19:28:22 -0700 | [diff] [blame] | 405 | udelay(50); |
| 406 | total += 50; |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 407 | if (total >= total_delay) { |
Simon Glass | 17dbe89 | 2016-03-06 19:28:22 -0700 | [diff] [blame] | 408 | debug("timed out: aborting\n"); |
| 409 | return -ETIMEDOUT; |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 410 | } |
| 411 | } |
Simon Glass | 17dbe89 | 2016-03-06 19:28:22 -0700 | [diff] [blame] | 412 | debug("done\n"); |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 413 | |
Simon Glass | 17dbe89 | 2016-03-06 19:28:22 -0700 | [diff] [blame] | 414 | return 0; |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 415 | } |
| 416 | |
Simon Glass | a3ee7b8 | 2020-07-17 08:48:10 -0600 | [diff] [blame] | 417 | /** |
| 418 | * start_aps() - Start up the APs and count how many we find |
| 419 | * |
| 420 | * This is called on the boot processor to start up all the other processors |
| 421 | * (here called APs). |
| 422 | * |
| 423 | * @num_aps: Number of APs we expect to find |
| 424 | * @ap_count: Initially zero. Incremented by this function for each AP found |
| 425 | * @return 0 if all APs were set up correctly or there are none to set up, |
| 426 | * -ENOSPC if the SIPI vector is too high in memory, |
| 427 | * -ETIMEDOUT if the ICR is busy or the second SIPI fails to complete |
| 428 | * -EIO if not all APs check in correctly |
| 429 | */ |
| 430 | static int start_aps(int num_aps, atomic_t *ap_count) |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 431 | { |
| 432 | int sipi_vector; |
| 433 | /* Max location is 4KiB below 1MiB */ |
| 434 | const int max_vector_loc = ((1 << 20) - (1 << 12)) >> 12; |
| 435 | |
Simon Glass | a3ee7b8 | 2020-07-17 08:48:10 -0600 | [diff] [blame] | 436 | if (num_aps == 0) |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 437 | return 0; |
| 438 | |
| 439 | /* The vector is sent as a 4k aligned address in one byte */ |
| 440 | sipi_vector = AP_DEFAULT_BASE >> 12; |
| 441 | |
| 442 | if (sipi_vector > max_vector_loc) { |
| 443 | printf("SIPI vector too large! 0x%08x\n", |
| 444 | sipi_vector); |
Simon Glass | f9b5800 | 2019-04-25 21:58:41 -0600 | [diff] [blame] | 445 | return -ENOSPC; |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 446 | } |
| 447 | |
Simon Glass | a3ee7b8 | 2020-07-17 08:48:10 -0600 | [diff] [blame] | 448 | debug("Attempting to start %d APs\n", num_aps); |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 449 | |
Simon Glass | 17dbe89 | 2016-03-06 19:28:22 -0700 | [diff] [blame] | 450 | if (apic_wait_timeout(1000, "ICR not to be busy")) |
| 451 | return -ETIMEDOUT; |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 452 | |
| 453 | /* Send INIT IPI to all but self */ |
Bin Meng | e5d0500 | 2015-06-23 12:18:50 +0800 | [diff] [blame] | 454 | lapic_write(LAPIC_ICR2, SET_LAPIC_DEST_FIELD(0)); |
| 455 | lapic_write(LAPIC_ICR, LAPIC_DEST_ALLBUT | LAPIC_INT_ASSERT | |
| 456 | LAPIC_DM_INIT); |
Simon Glass | 17dbe89 | 2016-03-06 19:28:22 -0700 | [diff] [blame] | 457 | debug("Waiting for 10ms after sending INIT\n"); |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 458 | mdelay(10); |
| 459 | |
| 460 | /* Send 1st SIPI */ |
Simon Glass | 17dbe89 | 2016-03-06 19:28:22 -0700 | [diff] [blame] | 461 | if (apic_wait_timeout(1000, "ICR not to be busy")) |
| 462 | return -ETIMEDOUT; |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 463 | |
Bin Meng | e5d0500 | 2015-06-23 12:18:50 +0800 | [diff] [blame] | 464 | lapic_write(LAPIC_ICR2, SET_LAPIC_DEST_FIELD(0)); |
| 465 | lapic_write(LAPIC_ICR, LAPIC_DEST_ALLBUT | LAPIC_INT_ASSERT | |
| 466 | LAPIC_DM_STARTUP | sipi_vector); |
Simon Glass | 17dbe89 | 2016-03-06 19:28:22 -0700 | [diff] [blame] | 467 | if (apic_wait_timeout(10000, "first SIPI to complete")) |
| 468 | return -ETIMEDOUT; |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 469 | |
| 470 | /* Wait for CPUs to check in up to 200 us */ |
Simon Glass | a3ee7b8 | 2020-07-17 08:48:10 -0600 | [diff] [blame] | 471 | wait_for_aps(ap_count, num_aps, 200, 15); |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 472 | |
| 473 | /* Send 2nd SIPI */ |
Simon Glass | 17dbe89 | 2016-03-06 19:28:22 -0700 | [diff] [blame] | 474 | if (apic_wait_timeout(1000, "ICR not to be busy")) |
| 475 | return -ETIMEDOUT; |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 476 | |
Bin Meng | e5d0500 | 2015-06-23 12:18:50 +0800 | [diff] [blame] | 477 | lapic_write(LAPIC_ICR2, SET_LAPIC_DEST_FIELD(0)); |
| 478 | lapic_write(LAPIC_ICR, LAPIC_DEST_ALLBUT | LAPIC_INT_ASSERT | |
| 479 | LAPIC_DM_STARTUP | sipi_vector); |
Simon Glass | 17dbe89 | 2016-03-06 19:28:22 -0700 | [diff] [blame] | 480 | if (apic_wait_timeout(10000, "second SIPI to complete")) |
| 481 | return -ETIMEDOUT; |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 482 | |
| 483 | /* Wait for CPUs to check in */ |
Simon Glass | a3ee7b8 | 2020-07-17 08:48:10 -0600 | [diff] [blame] | 484 | if (wait_for_aps(ap_count, num_aps, 10000, 50)) { |
Simon Glass | 17dbe89 | 2016-03-06 19:28:22 -0700 | [diff] [blame] | 485 | debug("Not all APs checked in: %d/%d\n", |
Simon Glass | a3ee7b8 | 2020-07-17 08:48:10 -0600 | [diff] [blame] | 486 | atomic_read(ap_count), num_aps); |
Simon Glass | f9b5800 | 2019-04-25 21:58:41 -0600 | [diff] [blame] | 487 | return -EIO; |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 488 | } |
| 489 | |
| 490 | return 0; |
| 491 | } |
| 492 | |
Simon Glass | 3307d0c | 2020-07-17 08:48:11 -0600 | [diff] [blame] | 493 | /** |
| 494 | * bsp_do_flight_plan() - Do the flight plan on the BSP |
| 495 | * |
| 496 | * This runs the flight plan on the main CPU used to boot U-Boot |
| 497 | * |
| 498 | * @cpu: Device for the main CPU |
| 499 | * @plan: Flight plan to run |
| 500 | * @num_aps: Number of APs (CPUs other than the BSP) |
| 501 | * @returns 0 on success, -ETIMEDOUT if an AP failed to come up |
| 502 | */ |
| 503 | static int bsp_do_flight_plan(struct udevice *cpu, struct mp_flight_plan *plan, |
| 504 | int num_aps) |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 505 | { |
| 506 | int i; |
| 507 | int ret = 0; |
| 508 | const int timeout_us = 100000; |
| 509 | const int step_us = 100; |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 510 | |
Simon Glass | e40633d | 2020-07-17 08:48:08 -0600 | [diff] [blame] | 511 | for (i = 0; i < plan->num_records; i++) { |
| 512 | struct mp_flight_record *rec = &plan->records[i]; |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 513 | |
| 514 | /* Wait for APs if the record is not released */ |
| 515 | if (atomic_read(&rec->barrier) == 0) { |
| 516 | /* Wait for the APs to check in */ |
| 517 | if (wait_for_aps(&rec->cpus_entered, num_aps, |
| 518 | timeout_us, step_us)) { |
Simon Glass | 17dbe89 | 2016-03-06 19:28:22 -0700 | [diff] [blame] | 519 | debug("MP record %d timeout\n", i); |
Simon Glass | f9b5800 | 2019-04-25 21:58:41 -0600 | [diff] [blame] | 520 | ret = -ETIMEDOUT; |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 521 | } |
| 522 | } |
| 523 | |
| 524 | if (rec->bsp_call != NULL) |
| 525 | rec->bsp_call(cpu, rec->bsp_arg); |
| 526 | |
| 527 | release_barrier(&rec->barrier); |
| 528 | } |
Simon Glass | 3307d0c | 2020-07-17 08:48:11 -0600 | [diff] [blame] | 529 | |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 530 | return ret; |
| 531 | } |
| 532 | |
Simon Glass | 653781b | 2020-07-17 08:48:14 -0600 | [diff] [blame] | 533 | /** |
| 534 | * get_bsp() - Get information about the bootstrap processor |
| 535 | * |
| 536 | * @devp: If non-NULL, returns CPU device corresponding to the BSP |
| 537 | * @cpu_countp: If non-NULL, returns the total number of CPUs |
| 538 | * @return CPU number of the BSP, or -ve on error. If multiprocessing is not |
| 539 | * enabled, returns 0 |
| 540 | */ |
| 541 | static int get_bsp(struct udevice **devp, int *cpu_countp) |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 542 | { |
| 543 | char processor_name[CPU_MAX_NAME_LEN]; |
Simon Glass | 653781b | 2020-07-17 08:48:14 -0600 | [diff] [blame] | 544 | struct udevice *dev; |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 545 | int apic_id; |
| 546 | int ret; |
| 547 | |
| 548 | cpu_get_name(processor_name); |
Simon Glass | 17dbe89 | 2016-03-06 19:28:22 -0700 | [diff] [blame] | 549 | debug("CPU: %s\n", processor_name); |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 550 | |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 551 | apic_id = lapicid(); |
Simon Glass | 653781b | 2020-07-17 08:48:14 -0600 | [diff] [blame] | 552 | ret = find_cpu_by_apic_id(apic_id, &dev); |
| 553 | if (ret < 0) { |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 554 | printf("Cannot find boot CPU, APIC ID %d\n", apic_id); |
| 555 | return ret; |
| 556 | } |
Simon Glass | 653781b | 2020-07-17 08:48:14 -0600 | [diff] [blame] | 557 | ret = cpu_get_count(dev); |
| 558 | if (ret < 0) |
| 559 | return log_msg_ret("count", ret); |
| 560 | if (devp) |
| 561 | *devp = dev; |
| 562 | if (cpu_countp) |
| 563 | *cpu_countp = ret; |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 564 | |
Simon Glass | 6feac81 | 2020-12-16 21:20:22 -0700 | [diff] [blame] | 565 | return dev_seq(dev) >= 0 ? dev_seq(dev) : 0; |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 566 | } |
| 567 | |
Simon Glass | 4a30bbb | 2020-07-17 08:48:16 -0600 | [diff] [blame] | 568 | /** |
| 569 | * read_callback() - Read the pointer in a callback slot |
| 570 | * |
| 571 | * This is called by APs to read their callback slot to see if there is a |
| 572 | * pointer to new instructions |
| 573 | * |
| 574 | * @slot: Pointer to the AP's callback slot |
| 575 | * @return value of that pointer |
| 576 | */ |
| 577 | static struct mp_callback *read_callback(struct mp_callback **slot) |
| 578 | { |
| 579 | dmb(); |
| 580 | |
| 581 | return *slot; |
| 582 | } |
| 583 | |
| 584 | /** |
| 585 | * store_callback() - Store a pointer to the callback slot |
| 586 | * |
| 587 | * This is called by APs to write NULL into the callback slot when they have |
| 588 | * finished the work requested by the BSP. |
| 589 | * |
| 590 | * @slot: Pointer to the AP's callback slot |
| 591 | * @val: Value to write (e.g. NULL) |
| 592 | */ |
| 593 | static void store_callback(struct mp_callback **slot, struct mp_callback *val) |
| 594 | { |
| 595 | *slot = val; |
| 596 | dmb(); |
| 597 | } |
| 598 | |
| 599 | /** |
Simon Glass | 6871dff | 2020-07-17 08:48:19 -0600 | [diff] [blame] | 600 | * run_ap_work() - Run a callback on selected APs |
| 601 | * |
| 602 | * This writes @callback to all APs and waits for them all to acknowledge it, |
| 603 | * Note that whether each AP actually calls the callback depends on the value |
| 604 | * of logical_cpu_number (see struct mp_callback). The logical CPU number is |
| 605 | * the CPU device's req->seq value. |
| 606 | * |
| 607 | * @callback: Callback information to pass to all APs |
| 608 | * @bsp: CPU device for the BSP |
| 609 | * @num_cpus: The number of CPUs in the system (= number of APs + 1) |
| 610 | * @expire_ms: Timeout to wait for all APs to finish, in milliseconds, or 0 for |
| 611 | * no timeout |
| 612 | * @return 0 if OK, -ETIMEDOUT if one or more APs failed to respond in time |
| 613 | */ |
| 614 | static int run_ap_work(struct mp_callback *callback, struct udevice *bsp, |
| 615 | int num_cpus, uint expire_ms) |
| 616 | { |
Simon Glass | 6feac81 | 2020-12-16 21:20:22 -0700 | [diff] [blame] | 617 | int cur_cpu = dev_seq(bsp); |
Simon Glass | 6871dff | 2020-07-17 08:48:19 -0600 | [diff] [blame] | 618 | int num_aps = num_cpus - 1; /* number of non-BSPs to get this message */ |
| 619 | int cpus_accepted; |
| 620 | ulong start; |
| 621 | int i; |
| 622 | |
| 623 | if (!IS_ENABLED(CONFIG_SMP_AP_WORK)) { |
| 624 | printf("APs already parked. CONFIG_SMP_AP_WORK not enabled\n"); |
| 625 | return -ENOTSUPP; |
| 626 | } |
| 627 | |
| 628 | /* Signal to all the APs to run the func. */ |
| 629 | for (i = 0; i < num_cpus; i++) { |
| 630 | if (cur_cpu != i) |
| 631 | store_callback(&ap_callbacks[i], callback); |
| 632 | } |
| 633 | mfence(); |
| 634 | |
| 635 | /* Wait for all the APs to signal back that call has been accepted. */ |
| 636 | start = get_timer(0); |
| 637 | |
| 638 | do { |
| 639 | mdelay(1); |
| 640 | cpus_accepted = 0; |
| 641 | |
| 642 | for (i = 0; i < num_cpus; i++) { |
| 643 | if (cur_cpu == i) |
| 644 | continue; |
| 645 | if (!read_callback(&ap_callbacks[i])) |
| 646 | cpus_accepted++; |
| 647 | } |
| 648 | |
| 649 | if (expire_ms && get_timer(start) >= expire_ms) { |
| 650 | log(UCLASS_CPU, LOGL_CRIT, |
| 651 | "AP call expired; %d/%d CPUs accepted\n", |
| 652 | cpus_accepted, num_aps); |
| 653 | return -ETIMEDOUT; |
| 654 | } |
| 655 | } while (cpus_accepted != num_aps); |
| 656 | |
| 657 | /* Make sure we can see any data written by the APs */ |
| 658 | mfence(); |
| 659 | |
| 660 | return 0; |
| 661 | } |
| 662 | |
| 663 | /** |
Simon Glass | 4a30bbb | 2020-07-17 08:48:16 -0600 | [diff] [blame] | 664 | * ap_wait_for_instruction() - Wait for and process requests from the main CPU |
| 665 | * |
| 666 | * This is called by APs (here, everything other than the main boot CPU) to |
| 667 | * await instructions. They arrive in the form of a function call and argument, |
| 668 | * which is then called. This uses a simple mailbox with atomic read/set |
| 669 | * |
| 670 | * @cpu: CPU that is waiting |
| 671 | * @unused: Optional argument provided by struct mp_flight_record, not used here |
| 672 | * @return Does not return |
| 673 | */ |
| 674 | static int ap_wait_for_instruction(struct udevice *cpu, void *unused) |
| 675 | { |
| 676 | struct mp_callback lcb; |
| 677 | struct mp_callback **per_cpu_slot; |
| 678 | |
| 679 | if (!IS_ENABLED(CONFIG_SMP_AP_WORK)) |
| 680 | return 0; |
| 681 | |
Simon Glass | 6feac81 | 2020-12-16 21:20:22 -0700 | [diff] [blame] | 682 | per_cpu_slot = &ap_callbacks[dev_seq(cpu)]; |
Simon Glass | 4a30bbb | 2020-07-17 08:48:16 -0600 | [diff] [blame] | 683 | |
| 684 | while (1) { |
| 685 | struct mp_callback *cb = read_callback(per_cpu_slot); |
| 686 | |
| 687 | if (!cb) { |
| 688 | asm ("pause"); |
| 689 | continue; |
| 690 | } |
| 691 | |
| 692 | /* Copy to local variable before using the value */ |
| 693 | memcpy(&lcb, cb, sizeof(lcb)); |
| 694 | mfence(); |
| 695 | if (lcb.logical_cpu_number == MP_SELECT_ALL || |
| 696 | lcb.logical_cpu_number == MP_SELECT_APS || |
Simon Glass | 6feac81 | 2020-12-16 21:20:22 -0700 | [diff] [blame] | 697 | dev_seq(cpu) == lcb.logical_cpu_number) |
Simon Glass | 4a30bbb | 2020-07-17 08:48:16 -0600 | [diff] [blame] | 698 | lcb.func(lcb.arg); |
| 699 | |
| 700 | /* Indicate we are finished */ |
| 701 | store_callback(per_cpu_slot, NULL); |
| 702 | } |
| 703 | |
| 704 | return 0; |
| 705 | } |
| 706 | |
Simon Glass | 35ee0de | 2020-07-17 08:48:09 -0600 | [diff] [blame] | 707 | static int mp_init_cpu(struct udevice *cpu, void *unused) |
| 708 | { |
Simon Glass | b75b15b | 2020-12-03 16:55:23 -0700 | [diff] [blame] | 709 | struct cpu_plat *plat = dev_get_parent_plat(cpu); |
Simon Glass | 35ee0de | 2020-07-17 08:48:09 -0600 | [diff] [blame] | 710 | |
Simon Glass | 35ee0de | 2020-07-17 08:48:09 -0600 | [diff] [blame] | 711 | plat->ucode_version = microcode_read_rev(); |
| 712 | plat->device_id = gd->arch.x86_device; |
| 713 | |
| 714 | return device_probe(cpu); |
| 715 | } |
| 716 | |
| 717 | static struct mp_flight_record mp_steps[] = { |
| 718 | MP_FR_BLOCK_APS(mp_init_cpu, NULL, mp_init_cpu, NULL), |
Simon Glass | 4a30bbb | 2020-07-17 08:48:16 -0600 | [diff] [blame] | 719 | MP_FR_BLOCK_APS(ap_wait_for_instruction, NULL, NULL, NULL), |
Simon Glass | 35ee0de | 2020-07-17 08:48:09 -0600 | [diff] [blame] | 720 | }; |
| 721 | |
Simon Glass | 6871dff | 2020-07-17 08:48:19 -0600 | [diff] [blame] | 722 | int mp_run_on_cpus(int cpu_select, mp_run_func func, void *arg) |
| 723 | { |
| 724 | struct mp_callback lcb = { |
| 725 | .func = func, |
| 726 | .arg = arg, |
| 727 | .logical_cpu_number = cpu_select, |
| 728 | }; |
| 729 | struct udevice *dev; |
| 730 | int num_cpus; |
| 731 | int ret; |
| 732 | |
| 733 | ret = get_bsp(&dev, &num_cpus); |
| 734 | if (ret < 0) |
| 735 | return log_msg_ret("bsp", ret); |
| 736 | if (cpu_select == MP_SELECT_ALL || cpu_select == MP_SELECT_BSP || |
| 737 | cpu_select == ret) { |
| 738 | /* Run on BSP first */ |
| 739 | func(arg); |
| 740 | } |
| 741 | |
| 742 | if (!IS_ENABLED(CONFIG_SMP_AP_WORK) || |
| 743 | !(gd->flags & GD_FLG_SMP_READY)) { |
| 744 | /* Allow use of this function on the BSP only */ |
| 745 | if (cpu_select == MP_SELECT_BSP || !cpu_select) |
| 746 | return 0; |
| 747 | return -ENOTSUPP; |
| 748 | } |
| 749 | |
| 750 | /* Allow up to 1 second for all APs to finish */ |
| 751 | ret = run_ap_work(&lcb, dev, num_cpus, 1000 /* ms */); |
| 752 | if (ret) |
| 753 | return log_msg_ret("aps", ret); |
| 754 | |
| 755 | return 0; |
| 756 | } |
| 757 | |
Simon Glass | 32d5695 | 2020-07-17 08:48:20 -0600 | [diff] [blame] | 758 | static void park_this_cpu(void *unused) |
| 759 | { |
| 760 | stop_this_cpu(); |
| 761 | } |
| 762 | |
| 763 | int mp_park_aps(void) |
| 764 | { |
| 765 | int ret; |
| 766 | |
| 767 | ret = mp_run_on_cpus(MP_SELECT_APS, park_this_cpu, NULL); |
| 768 | if (ret) |
| 769 | return log_ret(ret); |
| 770 | |
| 771 | return 0; |
| 772 | } |
| 773 | |
Simon Glass | 44344f5 | 2020-07-17 08:48:21 -0600 | [diff] [blame] | 774 | int mp_first_cpu(int cpu_select) |
| 775 | { |
| 776 | struct udevice *dev; |
| 777 | int num_cpus; |
| 778 | int ret; |
| 779 | |
| 780 | /* |
| 781 | * This assumes that CPUs are numbered from 0. This function tries to |
| 782 | * avoid assuming the CPU 0 is the boot CPU |
| 783 | */ |
| 784 | if (cpu_select == MP_SELECT_ALL) |
| 785 | return 0; /* start with the first one */ |
| 786 | |
| 787 | ret = get_bsp(&dev, &num_cpus); |
| 788 | if (ret < 0) |
| 789 | return log_msg_ret("bsp", ret); |
| 790 | |
| 791 | /* Return boot CPU if requested */ |
| 792 | if (cpu_select == MP_SELECT_BSP) |
| 793 | return ret; |
| 794 | |
| 795 | /* Return something other than the boot CPU, if APs requested */ |
| 796 | if (cpu_select == MP_SELECT_APS && num_cpus > 1) |
| 797 | return ret == 0 ? 1 : 0; |
| 798 | |
| 799 | /* Try to check for an invalid value */ |
| 800 | if (cpu_select < 0 || cpu_select >= num_cpus) |
| 801 | return -EINVAL; |
| 802 | |
| 803 | return cpu_select; /* return the only selected one */ |
| 804 | } |
| 805 | |
| 806 | int mp_next_cpu(int cpu_select, int prev_cpu) |
| 807 | { |
| 808 | struct udevice *dev; |
| 809 | int num_cpus; |
| 810 | int ret; |
| 811 | int bsp; |
| 812 | |
| 813 | /* If we selected the BSP or a particular single CPU, we are done */ |
| 814 | if (!IS_ENABLED(CONFIG_SMP_AP_WORK) || cpu_select == MP_SELECT_BSP || |
| 815 | cpu_select >= 0) |
| 816 | return -EFBIG; |
| 817 | |
| 818 | /* Must be doing MP_SELECT_ALL or MP_SELECT_APS; return the next CPU */ |
| 819 | ret = get_bsp(&dev, &num_cpus); |
| 820 | if (ret < 0) |
| 821 | return log_msg_ret("bsp", ret); |
| 822 | bsp = ret; |
| 823 | |
| 824 | /* Move to the next CPU */ |
| 825 | assert(prev_cpu >= 0); |
| 826 | ret = prev_cpu + 1; |
| 827 | |
| 828 | /* Skip the BSP if needed */ |
| 829 | if (cpu_select == MP_SELECT_APS && ret == bsp) |
| 830 | ret++; |
| 831 | if (ret >= num_cpus) |
| 832 | return -EFBIG; |
| 833 | |
| 834 | return ret; |
| 835 | } |
| 836 | |
Simon Glass | e40633d | 2020-07-17 08:48:08 -0600 | [diff] [blame] | 837 | int mp_init(void) |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 838 | { |
Simon Glass | 3307d0c | 2020-07-17 08:48:11 -0600 | [diff] [blame] | 839 | int num_aps, num_cpus; |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 840 | atomic_t *ap_count; |
| 841 | struct udevice *cpu; |
Simon Glass | ebb239a | 2020-07-17 08:48:13 -0600 | [diff] [blame] | 842 | int ret; |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 843 | |
Simon Glass | 4c8243d | 2019-12-06 21:42:55 -0700 | [diff] [blame] | 844 | if (IS_ENABLED(CONFIG_QFW)) { |
| 845 | ret = qemu_cpu_fixup(); |
| 846 | if (ret) |
| 847 | return ret; |
| 848 | } |
Miao Yan | 35f54b2 | 2016-01-07 01:32:04 -0800 | [diff] [blame] | 849 | |
Simon Glass | 653781b | 2020-07-17 08:48:14 -0600 | [diff] [blame] | 850 | ret = get_bsp(&cpu, &num_cpus); |
| 851 | if (ret < 0) { |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 852 | debug("Cannot init boot CPU: err=%d\n", ret); |
| 853 | return ret; |
| 854 | } |
| 855 | |
Bin Meng | f967f9a | 2015-06-17 11:15:36 +0800 | [diff] [blame] | 856 | if (num_cpus < 2) |
| 857 | debug("Warning: Only 1 CPU is detected\n"); |
| 858 | |
| 859 | ret = check_cpu_devices(num_cpus); |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 860 | if (ret) |
Simon Glass | 653781b | 2020-07-17 08:48:14 -0600 | [diff] [blame] | 861 | log_warning("Warning: Device tree does not describe all CPUs. Extra ones will not be started correctly\n"); |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 862 | |
Simon Glass | 4a30bbb | 2020-07-17 08:48:16 -0600 | [diff] [blame] | 863 | ap_callbacks = calloc(num_cpus, sizeof(struct mp_callback *)); |
| 864 | if (!ap_callbacks) |
| 865 | return -ENOMEM; |
| 866 | |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 867 | /* Copy needed parameters so that APs have a reference to the plan */ |
Simon Glass | e40633d | 2020-07-17 08:48:08 -0600 | [diff] [blame] | 868 | mp_info.num_records = ARRAY_SIZE(mp_steps); |
| 869 | mp_info.records = mp_steps; |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 870 | |
| 871 | /* Load the SIPI vector */ |
Miao Yan | 6067762 | 2016-01-07 01:32:03 -0800 | [diff] [blame] | 872 | ret = load_sipi_vector(&ap_count, num_cpus); |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 873 | if (ap_count == NULL) |
Simon Glass | f9b5800 | 2019-04-25 21:58:41 -0600 | [diff] [blame] | 874 | return -ENOENT; |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 875 | |
| 876 | /* |
| 877 | * Make sure SIPI data hits RAM so the APs that come up will see |
| 878 | * the startup code even if the caches are disabled |
| 879 | */ |
| 880 | wbinvd(); |
| 881 | |
| 882 | /* Start the APs providing number of APs and the cpus_entered field */ |
Bin Meng | f967f9a | 2015-06-17 11:15:36 +0800 | [diff] [blame] | 883 | num_aps = num_cpus - 1; |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 884 | ret = start_aps(num_aps, ap_count); |
| 885 | if (ret) { |
| 886 | mdelay(1000); |
| 887 | debug("%d/%d eventually checked in?\n", atomic_read(ap_count), |
| 888 | num_aps); |
| 889 | return ret; |
| 890 | } |
| 891 | |
| 892 | /* Walk the flight plan for the BSP */ |
Simon Glass | 3307d0c | 2020-07-17 08:48:11 -0600 | [diff] [blame] | 893 | ret = bsp_do_flight_plan(cpu, &mp_info, num_aps); |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 894 | if (ret) { |
| 895 | debug("CPU init failed: err=%d\n", ret); |
| 896 | return ret; |
| 897 | } |
Simon Glass | 4943de6 | 2020-07-17 08:48:18 -0600 | [diff] [blame] | 898 | gd->flags |= GD_FLG_SMP_READY; |
Simon Glass | a9a4426 | 2015-04-29 22:25:59 -0600 | [diff] [blame] | 899 | |
| 900 | return 0; |
| 901 | } |