1 /*
2 * Copyright (c) 2008-2015 Travis Geiselbrecht
3 *
4 * Use of this source code is governed by a MIT-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/MIT
7 */
8 #include <lk/debug.h>
9 #include <lk/trace.h>
10 #include <stdlib.h>
11 #include <lk/err.h>
12 #include <lk/trace.h>
13 #include <stdio.h>
14 #include <lk/reg.h>
15 #include <arch.h>
16 #include <arch/atomic.h>
17 #include <arch/ops.h>
18 #include <arch/mmu.h>
19 #include <arch/arm.h>
20 #include <arch/arm/mmu.h>
21 #include <arch/mp.h>
22 #include <kernel/spinlock.h>
23 #include <kernel/thread.h>
24 #include <lk/main.h>
25 #include <lk/init.h>
26 #include <platform.h>
27 #include <target.h>
28 #include <kernel/thread.h>
29 #if WITH_KERNEL_VM
30 #include <kernel/vm.h>
31 #endif
32
33 #define LOCAL_TRACE 0
34
35 #if WITH_DEV_TIMER_ARM_CORTEX_A9
36 #include <dev/timer/arm_cortex_a9.h>
37 #endif
38 #if WITH_DEV_INTERRUPT_ARM_GIC
39 #include <dev/interrupt/arm_gic.h>
40 #endif
41 #if WITH_DEV_CACHE_PL310
42 #include <dev/cache/pl310.h>
43 #endif
44
45 /* initial and abort stacks */
46 uint8_t abort_stack[ARCH_DEFAULT_STACK_SIZE *SMP_MAX_CPUS] __CPU_ALIGN;
47
48 static void arm_basic_setup(void);
49 static void spinlock_test(void);
50 static void spinlock_test_secondary(void);
51
52 #if WITH_SMP
53 /* smp boot lock */
54 spin_lock_t arm_boot_cpu_lock = 1;
55 volatile int secondaries_to_init = 0;
56 #endif
57
arch_early_init(void)58 void arch_early_init(void) {
59 /* turn off the cache */
60 arch_disable_cache(UCACHE);
61 #if WITH_DEV_CACHE_PL310
62 pl310_set_enable(false);
63 #endif
64
65 arm_basic_setup();
66
67 #if WITH_SMP && ARM_CPU_CORTEX_A9
68 /* enable snoop control */
69 addr_t scu_base = arm_read_cbar();
70 *REG32(scu_base) |= (1<<0); /* enable SCU */
71 #endif
72
73 #if ARCH_HAS_MMU
74 arm_mmu_early_init();
75
76 platform_init_mmu_mappings();
77 #endif
78
79 /* turn the cache back on */
80 #if WITH_DEV_CACHE_PL310
81 pl310_set_enable(true);
82 #endif
83 arch_enable_cache(UCACHE);
84 }
85
arch_init(void)86 void arch_init(void) {
87 #if WITH_SMP
88 arch_mp_init_percpu();
89
90 LTRACEF("midr 0x%x\n", arm_read_midr());
91 LTRACEF("sctlr 0x%x\n", arm_read_sctlr());
92 LTRACEF("actlr 0x%x\n", arm_read_actlr());
93 #if ARM_CPU_CORTEX_A9
94 LTRACEF("cbar 0x%x\n", arm_read_cbar());
95 #endif
96 LTRACEF("mpidr 0x%x\n", arm_read_mpidr());
97 LTRACEF("ttbcr 0x%x\n", arm_read_ttbcr());
98 LTRACEF("ttbr0 0x%x\n", arm_read_ttbr0());
99 LTRACEF("dacr 0x%x\n", arm_read_dacr());
100 #if ARM_CPU_CORTEX_A7
101 LTRACEF("l2ctlr 0x%x\n", arm_read_l2ctlr());
102 LTRACEF("l2ectlr 0x%x\n", arm_read_l2ectlr());
103 #endif
104
105 #if ARM_CPU_CORTEX_A9
106 addr_t scu_base = arm_read_cbar();
107 uint32_t scu_config = *REG32(scu_base + 4);
108 secondaries_to_init = scu_config & 0x3;
109 #elif ARM_CPU_CORTEX_A7 || ARM_CPU_CORTEX_A15
110 uint32_t l2ctlr = arm_read_l2ctlr();
111 secondaries_to_init = (l2ctlr >> 24);
112 #else
113 secondaries_to_init = SMP_MAX_CPUS - 1; /* TODO: get count from somewhere else, or add cpus as they boot */
114 #endif
115
116 lk_init_secondary_cpus(secondaries_to_init);
117
118 /* in platforms where the cpus have already been started, go ahead and wake up all the
119 * secondary cpus here.
120 */
121 dprintf(SPEW, "releasing %d secondary cpu%c\n", secondaries_to_init, secondaries_to_init != 1 ? 's' : ' ');
122
123 /* release the secondary cpus */
124 spin_unlock(&arm_boot_cpu_lock);
125
126 /* flush the release of the lock, since the secondary cpus are running without cache on */
127 arch_clean_cache_range((addr_t)&arm_boot_cpu_lock, sizeof(arm_boot_cpu_lock));
128
129 #if ARM_ARCH_WAIT_FOR_SECONDARIES
130 /* wait for secondary cpus to boot before arm_mmu_init below, which will remove
131 * temporary boot mappings
132 * TODO: find a cleaner way to do this than this #define
133 */
134 while (secondaries_to_init > 0) {
135 __asm__ volatile("wfe");
136 }
137 #endif
138 #endif // WITH_SMP
139
140 //spinlock_test();
141
142 #if ARCH_HAS_MMU
143 /* finish initializing the mmu */
144 arm_mmu_init();
145 #endif
146 }
147
148 #if WITH_SMP
149 void arm_secondary_entry(uint asm_cpu_num);
arm_secondary_entry(uint asm_cpu_num)150 void arm_secondary_entry(uint asm_cpu_num) {
151 uint cpu = arch_curr_cpu_num();
152 if (cpu != asm_cpu_num)
153 return;
154
155 arm_basic_setup();
156
157 /* enable the local L1 cache */
158 //arch_enable_cache(UCACHE);
159
160 // XXX may not be safe, but just hard enable i and d cache here
161 // at the moment cannot rely on arch_enable_cache not dumping the L2
162 uint32_t sctlr = arm_read_sctlr();
163 sctlr |= (1<<12) | (1<<2); // enable i and dcache
164 arm_write_sctlr(sctlr);
165
166 /* run early secondary cpu init routines up to the threading level */
167 lk_init_level(LK_INIT_FLAG_SECONDARY_CPUS, LK_INIT_LEVEL_EARLIEST, LK_INIT_LEVEL_THREADING - 1);
168
169 arch_mp_init_percpu();
170
171 LTRACEF("cpu num %d\n", cpu);
172 LTRACEF("sctlr 0x%x\n", arm_read_sctlr());
173 LTRACEF("actlr 0x%x\n", arm_read_actlr());
174
175 /* we're done, tell the main cpu we're up */
176 atomic_add(&secondaries_to_init, -1);
177 smp_mb();
178 __asm__ volatile("sev");
179
180 lk_secondary_cpu_entry();
181 }
182 #endif
183
arm_basic_setup(void)184 static void arm_basic_setup(void) {
185 uint32_t sctlr = arm_read_sctlr();
186
187 /* ARMV7 bits */
188 sctlr &= ~(1<<10); /* swp disable */
189 sctlr |= (1<<11); /* enable program flow prediction */
190 sctlr &= ~(1<<14); /* random cache/tlb replacement */
191 sctlr &= ~(1<<25); /* E bit set to 0 on exception */
192 sctlr &= ~(1<<30); /* no thumb exceptions */
193 sctlr |= (1<<22); /* enable unaligned access */
194 sctlr &= ~(1<<1); /* disable alignment abort */
195
196 arm_write_sctlr(sctlr);
197
198 uint32_t actlr = arm_read_actlr();
199 #if ARM_CPU_CORTEX_A9
200 actlr |= (1<<2); /* enable dcache prefetch */
201 #if WITH_DEV_CACHE_PL310
202 actlr |= (1<<7); /* L2 exclusive cache */
203 actlr |= (1<<3); /* L2 write full line of zeroes */
204 actlr |= (1<<1); /* L2 prefetch hint enable */
205 #endif
206 #if WITH_SMP
207 /* enable smp mode, cache and tlb broadcast */
208 actlr |= (1<<6) | (1<<0);
209 #endif
210 #endif // ARM_CPU_CORTEX_A9
211 #if ARM_CPU_CORTEX_A7
212 #if WITH_SMP
213 /* enable smp mode */
214 actlr |= (1<<6);
215 #endif
216 #endif // ARM_CPU_CORTEX_A7
217
218 arm_write_actlr(actlr);
219
220 #if ENABLE_CYCLE_COUNTER && ARM_ISA_ARMV7
221 /* enable the cycle count register */
222 uint32_t en;
223 __asm__ volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (en));
224 en &= ~(1<<3); /* cycle count every cycle */
225 en |= 1; /* enable all performance counters */
226 __asm__ volatile("mcr p15, 0, %0, c9, c12, 0" :: "r" (en));
227
228 /* enable cycle counter */
229 en = (1<<31);
230 __asm__ volatile("mcr p15, 0, %0, c9, c12, 1" :: "r" (en));
231 #endif
232
233 #if ARM_WITH_VFP
234 /* enable cp10 and cp11 */
235 uint32_t val = arm_read_cpacr();
236 val |= (3<<22)|(3<<20);
237 arm_write_cpacr(val);
238
239 /* set enable bit in fpexc */
240 __asm__ volatile("mrc p10, 7, %0, c8, c0, 0" : "=r" (val));
241 val |= (1<<30);
242 __asm__ volatile("mcr p10, 7, %0, c8, c0, 0" :: "r" (val));
243
244 /* make sure the fpu starts off disabled */
245 arm_fpu_set_enable(false);
246 #endif
247
248 /* set the vector base to our exception vectors so we don't need to double map at 0 */
249 #if ARM_ISA_ARMV7
250 arm_write_vbar(KERNEL_BASE + KERNEL_LOAD_OFFSET);
251 #endif
252 }
253
arch_quiesce(void)254 void arch_quiesce(void) {
255 #if ENABLE_CYCLE_COUNTER
256 #if ARM_ISA_ARMV7
257 /* disable the cycle count and performance counters */
258 uint32_t en;
259 __asm__ volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (en));
260 en &= ~1; /* disable all performance counters */
261 __asm__ volatile("mcr p15, 0, %0, c9, c12, 0" :: "r" (en));
262
263 /* disable cycle counter */
264 en = 0;
265 __asm__ volatile("mcr p15, 0, %0, c9, c12, 1" :: "r" (en));
266 #endif
267 #if ARM_CPU_ARM1136
268 /* disable the cycle count and performance counters */
269 uint32_t en;
270 __asm__ volatile("mrc p15, 0, %0, c15, c12, 0" : "=r" (en));
271 en &= ~1; /* disable all performance counters */
272 __asm__ volatile("mcr p15, 0, %0, c15, c12, 0" :: "r" (en));
273 #endif
274 #endif
275
276 uint32_t actlr = arm_read_actlr();
277 #if ARM_CPU_CORTEX_A9
278 actlr = 0; /* put the aux control register back to default */
279 #endif // ARM_CPU_CORTEX_A9
280 arm_write_actlr(actlr);
281 }
282
283 #if ARM_ISA_ARMV7
284 /* virtual to physical translation */
arm_vtop(addr_t va,addr_t * pa)285 status_t arm_vtop(addr_t va, addr_t *pa) {
286 spin_lock_saved_state_t irqstate;
287
288 arch_interrupt_save(&irqstate, SPIN_LOCK_FLAG_INTERRUPTS);
289
290 arm_write_ats1cpr(va & ~(PAGE_SIZE-1));
291 uint32_t par = arm_read_par();
292
293 arch_interrupt_restore(irqstate, SPIN_LOCK_FLAG_INTERRUPTS);
294
295 if (par & 1)
296 return ERR_NOT_FOUND;
297
298 if (pa) {
299 *pa = (par & 0xfffff000) | (va & 0xfff);
300 }
301
302 return NO_ERROR;
303 }
304 #endif
305
arch_chain_load(void * entry,ulong arg0,ulong arg1,ulong arg2,ulong arg3)306 void arch_chain_load(void *entry, ulong arg0, ulong arg1, ulong arg2, ulong arg3) {
307 LTRACEF("entry %p, args 0x%lx 0x%lx 0x%lx 0x%lx\n", entry, arg0, arg1, arg2, arg3);
308
309 /* we are going to shut down the system, start by disabling interrupts */
310 arch_disable_ints();
311
312 /* give target and platform a chance to put hardware into a suitable
313 * state for chain loading.
314 */
315 target_quiesce();
316 platform_quiesce();
317
318 paddr_t entry_pa;
319 paddr_t loader_pa;
320
321 #if WITH_KERNEL_VM
322 /* get the physical address of the entry point we're going to branch to */
323 if (arm_vtop((addr_t)entry, &entry_pa) < 0) {
324 panic("error translating entry physical address\n");
325 }
326
327 /* add the low bits of the virtual address back */
328 entry_pa |= ((addr_t)entry & 0xfff);
329
330 LTRACEF("entry pa 0x%lx\n", entry_pa);
331
332 /* figure out the mapping for the chain load routine */
333 if (arm_vtop((addr_t)&arm_chain_load, &loader_pa) < 0) {
334 panic("error translating loader physical address\n");
335 }
336
337 /* add the low bits of the virtual address back */
338 loader_pa |= ((addr_t)&arm_chain_load & 0xfff);
339
340 paddr_t loader_pa_section = ROUNDDOWN(loader_pa, SECTION_SIZE);
341
342 LTRACEF("loader address %p, phys 0x%lx, surrounding large page 0x%lx\n",
343 &arm_chain_load, loader_pa, loader_pa_section);
344
345 arch_aspace_t *aspace;
346 bool need_context_switch;
347 // if loader_pa is within the kernel aspace, we can simply use arch_mmu_map to identity map it
348 // if its outside, we need to create a new aspace and context switch to it
349 if (arch_mmu_is_valid_vaddr(&vmm_get_kernel_aspace()->arch_aspace, loader_pa)) {
350 aspace = &vmm_get_kernel_aspace()->arch_aspace;
351 need_context_switch = false;
352 } else {
353 aspace = malloc(sizeof(*aspace));
354 arch_mmu_init_aspace(aspace, loader_pa_section, SECTION_SIZE, 0);
355 need_context_switch = true;
356 }
357
358 /* using large pages, map around the target location */
359 arch_mmu_map(aspace, loader_pa_section, loader_pa_section, (2 * SECTION_SIZE / PAGE_SIZE), 0);
360 if (need_context_switch) arch_mmu_context_switch(aspace);
361 #else
362 /* for non vm case, just branch directly into it */
363 entry_pa = (paddr_t)entry;
364 loader_pa = (paddr_t)&arm_chain_load;
365 #endif
366
367 LTRACEF("disabling instruction/data cache\n");
368 arch_disable_cache(UCACHE);
369 #if WITH_DEV_CACHE_PL310
370 pl310_set_enable(false);
371 #endif
372
373 /* put the booting cpu back into close to a default state */
374 arch_quiesce();
375
376 // linux wont re-enable the FPU during boot, so it must be enabled when chainloading
377 arm_fpu_set_enable(true);
378
379 LTRACEF("branching to physical address of loader\n");
380
381 /* branch to the physical address version of the chain loader routine */
382 void (*loader)(paddr_t entry, ulong, ulong, ulong, ulong) __NO_RETURN = (void *)loader_pa;
383 loader(entry_pa, arg0, arg1, arg2, arg3);
384 }
385
386 static spin_lock_t lock = 0;
387
spinlock_test(void)388 static void spinlock_test(void) {
389 TRACE_ENTRY;
390
391 spin_lock_saved_state_t state;
392 spin_lock_irqsave(&lock, state);
393
394 TRACEF("cpu0: i have the lock\n");
395 spin(1000000);
396 TRACEF("cpu0: releasing it\n");
397
398 spin_unlock_irqrestore(&lock, state);
399
400 spin(1000000);
401 }
402
spinlock_test_secondary(void)403 static void spinlock_test_secondary(void) {
404 TRACE_ENTRY;
405
406 spin(500000);
407 spin_lock_saved_state_t state;
408 spin_lock_irqsave(&lock, state);
409
410 TRACEF("cpu1: i have the lock\n");
411 spin(250000);
412 TRACEF("cpu1: releasing it\n");
413
414 spin_unlock_irqrestore(&lock, state);
415 }
416
417 /* switch to user mode, set the user stack pointer to user_stack_top, put the svc stack pointer to the top of the kernel stack */
arch_enter_uspace(vaddr_t entry_point,vaddr_t user_stack_top)418 void arch_enter_uspace(vaddr_t entry_point, vaddr_t user_stack_top) {
419 DEBUG_ASSERT(IS_ALIGNED(user_stack_top, 8));
420
421 thread_t *ct = get_current_thread();
422
423 vaddr_t kernel_stack_top = (uintptr_t)ct->stack + ct->stack_size;
424 kernel_stack_top = ROUNDDOWN(kernel_stack_top, 8);
425
426 uint32_t spsr = CPSR_MODE_USR;
427 spsr |= (entry_point & 1) ? CPSR_THUMB : 0;
428
429 arch_disable_ints();
430
431 asm volatile(
432 "ldmia %[ustack], { sp }^;"
433 "msr spsr, %[spsr];"
434 "mov sp, %[kstack];"
435 "movs pc, %[entry];"
436 :
437 : [ustack]"r"(&user_stack_top),
438 [kstack]"r"(kernel_stack_top),
439 [entry]"r"(entry_point),
440 [spsr]"r"(spsr)
441 : "memory");
442 __UNREACHABLE;
443 }
444