1#include <lk/asm.h>
2#include <arch/arm64/mmu.h>
3#include <arch/asm_macros.h>
4#include <kernel/vm.h>
5
6/*
7 * Register use:
8 *  x0-x3   Arguments
9 *  x9-x15  Scratch
10 *  x19-x28 Globals
11 */
12tmp                     .req x9
13tmp2                    .req x10
14wtmp2                   .req w10
15idx                     .req x11
16idx_shift               .req x12
17page_table              .req x13
18new_page_table          .req x14
19phys_offset             .req x15
20
21cpuid                   .req x19
22page_table0             .req x20
23page_table1             .req x21
24mmu_initial_mapping     .req x22
25vaddr                   .req x23
26paddr                   .req x24
27mapping_size            .req x25
28size                    .req x26
29attr                    .req x27
30
31.section .text.boot
32FUNCTION(_start)
33.globl arm_reset
34arm_reset:
35    bl      arm64_elX_to_el1
36
37#if WITH_KERNEL_VM
38    /* enable caches so atomics and spinlocks work */
39    mrs     tmp, sctlr_el1
40    orr     tmp, tmp, #(1<<12) /* Enable icache */
41    orr     tmp, tmp, #(1<<2)  /* Enable dcache/ucache */
42    orr     tmp, tmp, #(1<<3)  /* Enable Stack Alignment Check EL1 */
43    orr     tmp, tmp, #(1<<4)  /* Enable Stack Alignment Check EL0 */
44    bic     tmp, tmp, #(1<<1)  /* Disable Alignment Checking for EL1 EL0 */
45    msr     sctlr_el1, tmp
46
47    /* set up the mmu according to mmu_initial_mappings */
48
49    /* load the base of the translation table and clear the table */
50    adrp    page_table1, arm64_kernel_translation_table
51    add     page_table1, page_table1, #:lo12:arm64_kernel_translation_table
52
53    /* Prepare tt_trampoline page table */
54    /* Calculate pagetable physical addresses */
55    adrp    page_table0, tt_trampoline
56    add     page_table0, page_table0, #:lo12:tt_trampoline
57
58#if WITH_SMP
59    mrs     cpuid, mpidr_el1
60    ubfx    cpuid, cpuid, #0, #SMP_CPU_ID_BITS
61    cbnz    cpuid, .Lmmu_enable_secondary
62#endif
63
64    mov     tmp, #0
65
66    /* walk through all the entries in the translation table, setting them up */
67.Lclear_top_page_table_loop:
68    str     xzr, [page_table1, tmp, lsl #3]
69    add     tmp, tmp, #1
70    cmp     tmp, #MMU_KERNEL_PAGE_TABLE_ENTRIES_TOP
71    bne     .Lclear_top_page_table_loop
72
73    /* load the address of the mmu_initial_mappings table and start processing */
74    adrp    mmu_initial_mapping, mmu_initial_mappings
75    add     mmu_initial_mapping, mmu_initial_mapping, #:lo12:mmu_initial_mappings
76
77.Linitial_mapping_loop:
78/* Read entry of mmu_initial_mappings (likely defined in platform.c) */
79    ldp     paddr, vaddr, [mmu_initial_mapping, #__MMU_INITIAL_MAPPING_PHYS_OFFSET]
80    ldp     size, tmp, [mmu_initial_mapping, #__MMU_INITIAL_MAPPING_SIZE_OFFSET]
81
82    tbzmask tmp, MMU_INITIAL_MAPPING_FLAG_DYNAMIC, .Lnot_dynamic
83    adr     paddr, _start
84    mov     size, x0
85    str     paddr, [mmu_initial_mapping, #__MMU_INITIAL_MAPPING_PHYS_OFFSET]
86    str     size, [mmu_initial_mapping, #__MMU_INITIAL_MAPPING_SIZE_OFFSET]
87
88.Lnot_dynamic:
89    /* if size == 0, end of list, done with initial mapping */
90    cbz     size, .Linitial_mapping_done
91    mov     mapping_size, size
92
93    /* set up the flags */
94    tbzmask tmp, MMU_INITIAL_MAPPING_FLAG_UNCACHED, .Lnot_uncached
95    ldr     attr, =MMU_INITIAL_MAP_STRONGLY_ORDERED
96    b       .Lmem_type_done
97
98.Lnot_uncached:
99    /* is this memory mapped to device/peripherals? */
100    tbzmask tmp, MMU_INITIAL_MAPPING_FLAG_DEVICE, .Lnot_device
101    ldr     attr, =MMU_INITIAL_MAP_DEVICE
102    b       .Lmem_type_done
103.Lnot_device:
104
105/* Determine the segment in which the memory resides and set appropriate
106 *  attributes.  In order to handle offset kernels, the following rules are
107 *  implemented below:
108 *      KERNEL_BASE    to __code_start             -read/write (see note below)
109 *      __code_start   to __rodata_start (.text)   -read only
110 *      __rodata_start to __data_start   (.rodata) -read only, execute never
111 *      __data_start   to .....          (.data)   -read/write
112 *
113 *  The space below __code_start is presently left as read/write (same as .data)
114 *   mainly as a workaround for the raspberry pi boot process.  Boot vectors for
115 *   secondary CPUs are in this area and need to be updated by cpu0 once the system
116 *   is ready to boot the secondary processors.
117 *   TODO: handle this via mmu_initial_mapping entries, which may need to be
118 *         extended with additional flag types
119 */
120.Lmapping_size_loop:
121    ldr     attr, =MMU_PTE_KERNEL_DATA_FLAGS
122    ldr     tmp, =__code_start
123    subs    size, tmp, vaddr
124    /* If page is below  the entry point (_start) mark as kernel data */
125    b.hi    .Lmem_type_done
126
127    ldr     attr, =MMU_PTE_KERNEL_RO_FLAGS
128    ldr     tmp, =__rodata_start
129    subs    size, tmp, vaddr
130    b.hi    .Lmem_type_done
131    orr     attr, attr, #MMU_PTE_ATTR_PXN
132    ldr     tmp, =__data_start
133    subs    size, tmp, vaddr
134    b.hi    .Lmem_type_done
135    ldr     attr, =MMU_PTE_KERNEL_DATA_FLAGS
136    ldr     tmp, =_end
137    subs    size, tmp, vaddr
138    b.lo    . /* Error: _end < vaddr */
139    cmp     mapping_size, size
140    b.lo    . /* Error: mapping_size < size => RAM size too small for data/bss */
141    mov     size, mapping_size
142
143.Lmem_type_done:
144    subs    mapping_size, mapping_size, size
145    b.lo    . /* Error: mapping_size < size (RAM size too small for code/rodata?) */
146
147    /* Check that paddr, vaddr and size are page aligned */
148    orr     tmp, vaddr, paddr
149    orr     tmp, tmp, size
150    tst     tmp, #(1 << MMU_KERNEL_PAGE_SIZE_SHIFT) - 1
151    bne     . /* Error: not page aligned */
152
153    /* Clear top bits of virtual address (should be all set) */
154    eor     vaddr, vaddr, #(~0 << MMU_KERNEL_SIZE_SHIFT)
155
156    /* Check that top bits were all set */
157    tst     vaddr, #(~0 << MMU_KERNEL_SIZE_SHIFT)
158    bne     . /* Error: vaddr out of range */
159
160.Lmap_range_top_loop:
161    /* Select top level page table */
162    mov     page_table, page_table1
163    mov     idx_shift, #MMU_KERNEL_TOP_SHIFT
164
165    lsr     idx, vaddr, idx_shift
166
167
168/* determine the type of page table entry to use given alignment and size
169 *  of the chunk of memory we are mapping
170 */
171.Lmap_range_one_table_loop:
172    /* Check if current level allow block descriptors */
173    cmp     idx_shift, #MMU_PTE_DESCRIPTOR_BLOCK_MAX_SHIFT
174    b.hi    .Lmap_range_need_page_table
175
176    /* Check if paddr and vaddr alignment allows a block descriptor */
177    orr     tmp2, vaddr, paddr
178    lsr     tmp, tmp2, idx_shift
179    lsl     tmp, tmp, idx_shift
180    cmp     tmp, tmp2
181    b.ne    .Lmap_range_need_page_table
182
183    /* Check if size is large enough for a block mapping */
184    lsr     tmp, size, idx_shift
185    cbz     tmp, .Lmap_range_need_page_table
186
187    /* Select descriptor type, page for level 3, block for level 0-2 */
188    orr     tmp, attr, #MMU_PTE_L3_DESCRIPTOR_PAGE
189    cmp     idx_shift, MMU_KERNEL_PAGE_SIZE_SHIFT
190    beq     .Lmap_range_l3
191    orr     tmp, attr, #MMU_PTE_L012_DESCRIPTOR_BLOCK
192.Lmap_range_l3:
193
194    /* Write page table entry */
195    orr     tmp, tmp, paddr
196    str     tmp, [page_table, idx, lsl #3]
197
198    /* Move to next page table entry */
199    mov     tmp, #1
200    lsl     tmp, tmp, idx_shift
201    add     vaddr, vaddr, tmp
202    add     paddr, paddr, tmp
203    subs    size, size, tmp
204    /* TODO: add local loop if next entry is in the same page table */
205    b.ne    .Lmap_range_top_loop /* size != 0 */
206
207    /* Restore top bits of virtual address (should be all set) */
208    eor     vaddr, vaddr, #(~0 << MMU_KERNEL_SIZE_SHIFT)
209    /* Move to next subtype of ram mmu_initial_mappings entry */
210    cbnz     mapping_size, .Lmapping_size_loop
211
212    /* Move to next mmu_initial_mappings entry */
213    add     mmu_initial_mapping, mmu_initial_mapping, __MMU_INITIAL_MAPPING_SIZE
214    b       .Linitial_mapping_loop
215
216.Lmap_range_need_page_table:
217    /* Check if page table entry is unused */
218    ldr     new_page_table, [page_table, idx, lsl #3]
219    cbnz    new_page_table, .Lmap_range_has_page_table
220
221    /* Calculate phys offset (needed for memory allocation) */
222.Lphys_offset:
223    adr     phys_offset, .Lphys_offset /* phys */
224    ldr     tmp, =.Lphys_offset /* virt */
225    sub     phys_offset, tmp, phys_offset
226
227    /* Allocate new page table */
228    calloc_bootmem_aligned new_page_table, tmp, tmp2, MMU_KERNEL_PAGE_SIZE_SHIFT, phys_offset
229
230    /* Write page table entry (with allocated page table) */
231    orr     new_page_table, new_page_table, #MMU_PTE_L012_DESCRIPTOR_TABLE
232    str     new_page_table, [page_table, idx, lsl #3]
233
234.Lmap_range_has_page_table:
235    /* Check descriptor type */
236    and     tmp, new_page_table, #MMU_PTE_DESCRIPTOR_MASK
237    cmp     tmp, #MMU_PTE_L012_DESCRIPTOR_TABLE
238    b.ne    . /* Error: entry already in use (as a block entry) */
239
240    /* switch to next page table level */
241    bic     page_table, new_page_table, #MMU_PTE_DESCRIPTOR_MASK
242    mov     tmp, #~0
243    lsl     tmp, tmp, idx_shift
244    bic     tmp, vaddr, tmp
245    sub     idx_shift, idx_shift, #(MMU_KERNEL_PAGE_SIZE_SHIFT - 3)
246    lsr     idx, tmp, idx_shift
247
248    b       .Lmap_range_one_table_loop
249
250.Linitial_mapping_done:
251
252    /* Prepare tt_trampoline page table */
253
254    /* Zero tt_trampoline translation tables */
255    mov     tmp, #0
256.Lclear_tt_trampoline:
257    str     xzr, [page_table0, tmp, lsl#3]
258    add     tmp, tmp, #1
259    cmp     tmp, #MMU_PAGE_TABLE_ENTRIES_IDENT
260    blt     .Lclear_tt_trampoline
261
262    /* Setup mapping at phys -> phys */
263    adr     tmp, .Lmmu_on_pc
264    lsr     tmp, tmp, #MMU_IDENT_TOP_SHIFT    /* tmp = paddr idx */
265    ldr     tmp2, =MMU_PTE_IDENT_FLAGS
266    add     tmp2, tmp2, tmp, lsl #MMU_IDENT_TOP_SHIFT  /* tmp2 = pt entry */
267
268    str     tmp2, [page_table0, tmp, lsl #3]     /* tt_trampoline[paddr idx] = pt entry */
269
270#if WITH_SMP
271    adrp    tmp, page_tables_not_ready
272    add     tmp, tmp, #:lo12:page_tables_not_ready
273    str     wzr, [tmp]
274    b       .Lpage_tables_ready
275
276.Lmmu_enable_secondary:
277    adrp    tmp, page_tables_not_ready
278    add     tmp, tmp, #:lo12:page_tables_not_ready
279.Lpage_tables_not_ready:
280    ldr     wtmp2, [tmp]
281    cbnz    wtmp2, .Lpage_tables_not_ready
282.Lpage_tables_ready:
283#endif
284
285    /* set up the mmu */
286
287    /* Invalidate TLB */
288    tlbi    vmalle1is
289    dsb     sy
290    isb
291
292    /* Initialize Memory Attribute Indirection Register */
293    ldr     tmp, =MMU_MAIR_VAL
294    msr     mair_el1, tmp
295
296    /* Initialize TCR_EL1 */
297    /* set cacheable attributes on translation walk */
298    /* (SMP extensions) non-shareable, inner write-back write-allocate */
299    ldr     tmp, =MMU_TCR_FLAGS_IDENT
300    msr     tcr_el1, tmp
301
302    isb
303
304    /* Write ttbr with phys addr of the translation table */
305    msr     ttbr0_el1, page_table0
306    msr     ttbr1_el1, page_table1
307    isb
308
309    /* Read SCTLR */
310    mrs     tmp, sctlr_el1
311
312    /* Turn on the MMU */
313    orr     tmp, tmp, #0x1
314
315    /* Write back SCTLR */
316    msr     sctlr_el1, tmp
317.Lmmu_on_pc:
318    isb
319
320    /* Jump to virtual code address */
321    ldr     tmp, =.Lmmu_on_vaddr
322    br      tmp
323
324.Lmmu_on_vaddr:
325
326    /* Disable trampoline page-table in ttbr0 */
327    ldr     tmp, =MMU_TCR_FLAGS_KERNEL
328    msr     tcr_el1, tmp
329    isb
330
331
332    /* Invalidate TLB */
333    tlbi    vmalle1
334    dsb     sy
335    isb
336
337#if WITH_SMP
338    cbnz    cpuid, .Lsecondary_boot
339#endif
340#endif /* WITH_KERNEL_VM */
341
342    ldr tmp, =__stack_end
343    mov sp, tmp
344
345    /* clear bss */
346.L__do_bss:
347    /* clear out the bss excluding the stack and kernel translation table  */
348    /* NOTE: relies on __post_prebss_bss_start and __bss_end being 8 byte aligned */
349    ldr     tmp, =__post_prebss_bss_start
350    ldr     tmp2, =__bss_end
351    sub     tmp2, tmp2, tmp
352    cbz     tmp2, .L__bss_loop_done
353.L__bss_loop:
354    sub     tmp2, tmp2, #8
355    str     xzr, [tmp], #8
356    cbnz    tmp2, .L__bss_loop
357.L__bss_loop_done:
358
359    bl  lk_main
360    b   .
361
362#if WITH_SMP
363.Lsecondary_boot:
364    and     tmp, cpuid, #0xff
365    cmp     tmp, #(1 << SMP_CPU_CLUSTER_SHIFT)
366    bge     .Lunsupported_cpu_trap
367    bic     cpuid, cpuid, #0xff
368    orr     cpuid, tmp, cpuid, LSR #(8 - SMP_CPU_CLUSTER_SHIFT)
369
370    cmp     cpuid, #SMP_MAX_CPUS
371    bge     .Lunsupported_cpu_trap
372
373    /* Set up the stack */
374    ldr     tmp, =__stack_end
375    mov     tmp2, #ARCH_DEFAULT_STACK_SIZE
376    mul     tmp2, tmp2, cpuid
377    sub     sp, tmp, tmp2
378
379    mov     x0, cpuid
380    bl      arm64_secondary_entry
381
382.Lunsupported_cpu_trap:
383    wfe
384    b       .Lunsupported_cpu_trap
385#endif
386
387.ltorg
388
389#if WITH_SMP
390.data
391DATA(page_tables_not_ready)
392    .long       1
393#endif
394
395.section .bss.prebss.stack
396    .align 4
397DATA(__stack)
398    .skip ARCH_DEFAULT_STACK_SIZE * SMP_MAX_CPUS
399DATA(__stack_end)
400
401#if WITH_KERNEL_VM
402.section ".bss.prebss.translation_table"
403.align 3 + MMU_PAGE_TABLE_ENTRIES_IDENT_SHIFT
404DATA(tt_trampoline)
405    .skip 8 * MMU_PAGE_TABLE_ENTRIES_IDENT
406#endif
407