1/*
2 * Copyright 2020, Data61, CSIRO (ABN 41 687 119 230)
3 *
4 * SPDX-License-Identifier: GPL-2.0-only
5 */
6
7/* The kernel expects to be booted by a Multiboot compliant bootloader.
8 * See Multiboot specifications:
9 * www.gnu.org/software/grub/manual/multiboot
10 * www.gnu.org/software/grub/manual/multiboot2
11 *
12 * The multiboot header's flags field is set to 3, indicating that we want
13 * modules loaded on page boundaries, access to memory map information, and
14 * information about the video mode table. Bit 16 of the multiboot header is
15 * not set, indicating that the structure of the image should be taken from its
16 * ELF headers.
17 *
18 * When the bootloader jumps to the entry point it is not in long mode and
19 * 64-bit instructions are not accessible (Multiboot 1 does not have support
20 * for this). While in protected mode, setup including the initialisation of
21 * 64-bit paging structures is done before manually enabling long mode and
22 * continuing. */
23
24#include <config.h>
25#include <machine/assembler.h>
26
27#define IA32_EFER_MSR 0xC0000080
28#define IA32_APIC_BASE_MSR 0x01B
29#define APIC_ID_OFFSET 0x020
30
31.section .phys.text
32
33.code32
34
35BEGIN_FUNC(print_string)
36    movw $0x3f8, %dx
371:
38    movb (%ebx), %al
39    outb %al, %dx
40    incl %ebx
41    decl %ecx
42    jnz  1b
43    ret
44END_FUNC(print_string)
45
46BEGIN_FUNC(hang)
471:
48    hlt
49    jmp  1b
50END_FUNC(hang)
51
52#ifdef CONFIG_HUGE_PAGE
53BEGIN_FUNC(huge_page_check)
54    movl $0x80000001, %eax
55    cpuid
56    andl $0x04000000, %edx
57    jz   1f
58    ret
591:
60    movl $huge_page_error_string, %ebx
61    movl $huge_page_error_size, %ecx
62    call print_string
63    call hang
64
65huge_page_error_string:
66    .string "Huge page not supported by the processor"
67    .set huge_page_error_size, . - huge_page_error_string
68END_FUNC(huge_page_check)
69#endif /* CONFIG_HUGE_PAGE */
70
71BEGIN_FUNC(setup_pml4)
72#ifdef CONFIG_HUGE_PAGE
73    call huge_page_check
74#endif /* CONFIG_HUGE_PAGE */
75    movl %cr0, %eax
76    andl $0x7fffffff, %eax
77    movl %eax, %cr0
78
79    movl $boot_pml4, %edi
80    movl $0x0, %edx
81    movl $1024, %ecx
821:
83    movl %edx, (%edi)
84    addl $4, %edi
85    loop 1b
86
87    movl $boot_pdpt, %edi
88    movl $1024, %ecx
891:
90    movl %edx, (%edi)
91    addl $4, %edi
92    loop 1b
93
94    movl $boot_pml4, %edi
95    movl $boot_pdpt, %ecx
96    orl  $0x7, %ecx
97    movl %ecx, (%edi)
98    movl %ecx, 0x800(%edi)
99    movl %ecx, 4088(%edi)
100
101    movl $_boot_pd, %ecx
102    orl  $0x7, %ecx
103    movl $boot_pdpt, %edi
104    movl %ecx, (%edi)
105    movl %ecx, 4080(%edi)
106    addl $0x1000, %ecx
107    movl %ecx, 8(%edi)
108    addl $0x1000, %ecx
109    movl %ecx, 16(%edi)
110    addl $0x1000, %ecx
111    movl %ecx, 24(%edi)
112
113    /* Map first 4GiB into the _boot_pd. */
114    movl $_boot_pd, %edi
115    movl $2048, %ecx
116    movl $0x87, %edx
1172:
118    movl %edx, (%edi)
119    addl $0x200000, %edx
120    addl $8, %edi
121    loop 2b
122    ret
123END_FUNC(setup_pml4)
124
125#ifdef CONFIG_SUPPORT_PCID
126BEGIN_FUNC(pcid_check)
127    movl $0x1, %eax
128    xorl %ecx, %ecx
129    cpuid
130    andl $0x20000, %ecx
131    jz   1f
132    ret
1331:
134    movl $pcid_error_string, %ebx
135    movl $pcid_error_size, %ecx
136    call print_string
137    call hang
138
139pcid_error_string:
140    .string "PCIDs not supported by the processor"
141    .set pcid_error_size, . - pcid_error_string
142END_FUNC(pcid_check)
143
144BEGIN_FUNC(invpcid_check)
145    movl $0x7, %eax
146    xorl %ecx, %ecx
147    cpuid
148    andl $0x400, %ebx
149    jz   1f
150    ret
1511:
152    movl  $invpcid_error_string, %ebx
153    movl  $invpcid_error_size, %ecx
154    call  print_string
155    call  hang
156
157invpcid_error_string:
158    .string "INVPCID instruction not supported by the processor"
159    .set invpcid_error_size, . - invpcid_error_string
160END_FUNC(invpcid_check)
161#endif /* CONFIG_SUPPORT_PCID */
162
163BEGIN_FUNC(syscall_check)
164    movl $0x80000001, %eax
165    xorl %ecx, %ecx
166    cpuid
167    andl $0x20000000, %edx
168    jz   1f
169    ret
1701:
171    movl  $syscall_error_string, %ebx
172    movl  $syscall_error_size, %ecx
173    call  print_string
174    call  hang
175
176syscall_error_string:
177    .string "SYSCALL/SYSRET instruction not supported by the processor"
178    .set syscall_error_size, . - syscall_error_string
179END_FUNC(syscall_check)
180
181#ifdef CONFIG_FSGSBASE_INST
182BEGIN_FUNC(fsgsbase_enable)
183    movl $0x7, %eax
184    xorl %ecx, %ecx
185    cpuid
186    andl $1, %ebx
187    jz   1f
188    movl %cr4, %eax
189    /* Enable the bit in cr4. */
190    orl  $0x10000, %eax
191    movl %eax, %cr4
192    ret
1931:
194    movl $fsgsbase_error_string, %ebx
195    movl $fsgsbase_error_size, %ecx
196    call print_string
197    call hang
198
199fsgsbase_error_string:
200    .string "fsgsbase instructions not supported by the processor"
201    .set fsgsbase_error_size, . - fsgsbase_error_string
202END_FUNC(fsgsbase_enable)
203#endif /* CONFIG_FSGSBASE_INST */
204
205#ifdef CONFIG_SYSCALL
206BEGIN_FUNC(syscall_enable)
207    call syscall_check
208    /* Set SCE (bit 0) in the extended feature MSR. */
209    movl $IA32_EFER_MSR, %ecx
210    rdmsr
211    orl $0x1, %eax
212    wrmsr
213    ret
214END_FUNC(syscall_enable)
215#endif /* CONFIG_SYSCALL */
216
217BEGIN_FUNC(enable_x64_mode)
218#ifdef CONFIG_SUPPORT_PCID
219    call pcid_check
220    call invpcid_check
221#endif
222    /* Put base pointer in cr3. */
223    movl $boot_pml4, %eax
224    movl %eax, %cr3
225    /* Set PAE (bit 5), as this is required before switching to long mode. */
226    movl %cr4, %eax
227    orl $0x20, %eax
228    movl %eax, %cr4
229    /* Set LME (bit 8) in the extended feature MSR. */
230    movl $IA32_EFER_MSR, %ecx
231    rdmsr
232    orl $0x100, %eax
233    wrmsr
234    /* Set PG (bit 31) of cr0 to enable paging. */
235    movl %cr0, %eax
236    orl $0x80000000, %eax
237    movl %eax, %cr0
238#ifdef CONFIG_SUPPORT_PCID
239    /* Enable PCID (bit 17), must be done in long mode. */
240    movl %cr4, %eax
241    orl  $0x20000, %eax
242    movl %eax, %cr4
243#endif
244    ret
245END_FUNC(enable_x64_mode)
246
247BEGIN_FUNC(common_init)
248    /* Disable paging. */
249    movl %cr0, %eax
250    andl $0x7fffffff, %eax
251    movl %eax, %cr0
252
253#ifdef CONFIG_FSGSBASE_INST
254    call fsgsbase_enable
255#endif /* CONFIG_FSGSBASE_INST */
256
257    /* Initialize boot PML4 and switch to long mode. */
258    call setup_pml4
259    call enable_x64_mode
260    lgdt _gdt64_ptr
261
262#ifdef CONFIG_SYSCALL
263    call syscall_enable
264#endif
265
266    ret
267END_FUNC(common_init)
268
269BEGIN_FUNC(_start)
270    /* Assume we are MultiBooted, e.g. by GRUB.
271     * While not immediately checked, the magic number is checked prior to
272     * Multiboot dependent operations. */
273    movl %eax, %edi /* multiboot_magic    */
274    movl %ebx, %esi /* multiboot_info_ptr */
275
276    /* Load kernel boot stack pointer. */
277    leal boot_stack_top, %esp
278
279    /* Reset EFLAGS register (also disables interrupts etc.). */
280    pushl $0
281    popf
282
283    /* Already push parameters for calling boot_sys later. Push
284     * them as 8 byte values so we can easily pop later. */
285    pushl $0
286    pushl %esi /* 2nd parameter: multiboot_info_ptr */
287    pushl $0
288    pushl %edi /* 1st parameter: multiboot_magic    */
289
290    call common_init
291
292    /* Reload CS with long bit to enable long mode. */
293    ljmp $8, $_start64
294END_FUNC(_start)
295
296.code64
297BEGIN_FUNC(_start64)
298    /* Leave phys code behind and jump to the high kernel virtual address. */
299    movabs $_entry_64, %rax
300    jmp *%rax
301END_FUNC(_start64)
302
303
304.section .phys.data
305.align 16
306_gdt64:
307    .quad 0x0000000000000000
308    .word   0
309    .word   0
310    .byte   0
311    .byte   0x98
312    .byte   0x20
313    .byte   0
314    .word   0
315    .word   0
316    .byte   0
317    .byte   0x90
318    .byte   0
319    .byte   0
320
321
322_gdt64_ptr:
323    .word (3 * 8) - 1
324    .long _gdt64
325
326.section .phys.bss
327.align 4096
328_boot_pd:
329    .fill 16384
330
331.section .boot.text
332
333BEGIN_FUNC(_entry_64)
334    /* Update our stack pointer. */
335    movq $0xffffffff80000000, %rax
336    addq %rax, %rsp
337    addq %rax, %rbp
338
339    /* Pop the multiboot parameters off. */
340    pop %rdi
341    pop %rsi
342
343    /* Load our real kernel stack. */
344    leaq kernel_stack_alloc + (1 << CONFIG_KERNEL_STACK_BITS), %rsp
345
346    movabs $restore_user_context, %rax
347    push %rax
348    jmp boot_sys
349END_FUNC(_entry_64)
350
351.section .phys.text
352
353#ifdef ENABLE_SMP_SUPPORT
354
355BEGIN_FUNC(boot_cpu_start)
356.code16
357    /* Set DS equal to CS and load GDTR register with GDT pointer. */
358    movw %cs, %ax
359    movw %ax, %ds
360    lgdt _boot_gdt_ptr - boot_cpu_start
361
362    /* Enable protected mode. */
363    movl %cr0, %eax
364    orl  $1,   %eax
365    movl %eax, %cr0
366
367    /* Reload CS with a far jump. */
368    ljmpl $0x08, $1f
369
370.code32
3711:
372    /* Load DS/ES/SS with kernel data segment selector. */
373    movw $0x10, %ax
374    movw %ax,   %ds
375    movw %ax,   %es
376    movw %ax,   %ss
377
378    /* Use temporary kernel boot stack pointer. */
379    leal boot_stack_top, %esp
380
381    /* Reset EFLAGS register (also disables interrupts etc.). */
382    pushl $0
383    popf
384
385    call common_init
386
387    /* Reload CS with long bit to enable long mode. */
388    ljmp $8, $_start_ap64
389    jmp 1b
390END_FUNC(boot_cpu_start)
391
392.code64
393BEGIN_FUNC(_start_ap64)
394    /* Leave phys code behind and jump to the high kernel virtual address. */
395    movabs $_entry_ap64, %rax
396    jmp *%rax
397END_FUNC(_start_ap64)
398
399_boot_gdt_ptr:
400    .word   (3 * 8) - 1 /* Limit: 3 segments * 8 bytes - 1 byte */
401    .long   _boot_gdt   /* Address of boot GDT */
402
403/* GDT for getting us through 32-bit protected mode. */
404    .align 16
405_boot_gdt:
406    .quad 0x0000000000000000 /* Null segment */
407    .quad 0x00cf9b000000ffff /* 4GB kernel code segment */
408    .quad 0x00cf93000000ffff /* 4GB kernel data segment */
409
410.global boot_cpu_end
411boot_cpu_end:
412
413.section .boot.text
414
415BEGIN_FUNC(_entry_ap64)
416    /* Get the index of this cpu. */
417    movq smp_aps_index, %rcx
418
419    /* Switch to a real kernel stack. */
420    leaq kernel_stack_alloc, %rsp
421    inc %rcx
422    shlq $CONFIG_KERNEL_STACK_BITS, %rcx
423    addq %rcx, %rsp
424
425    movabs $restore_user_context, %rax
426    push %rax
427    jmp boot_node
428END_FUNC(_entry_ap64)
429
430#endif /* ENABLE_SMP_SUPPORT */
431