1 /*
2 * Copyright 2020, Data61, CSIRO (ABN 41 687 119 230)
3 *
4 * SPDX-License-Identifier: GPL-2.0-only
5 */
6
7 #pragma once
8
9 #include <config.h>
10 #include <api/failures.h>
11
12 #define VCPU_VMCS_SIZE 4096
13 #define VCPU_IOBITMAP_SIZE 8192
14
15 #define VMX_CONTROL_VPID 0x00000000
16
17 #define VMX_GUEST_ES_SELECTOR 0x00000800
18 #define VMX_GUEST_CS_SELECTOR 0x00000802
19 #define VMX_GUEST_SS_SELECTOR 0x00000804
20 #define VMX_GUEST_DS_SELECTOR 0x00000806
21 #define VMX_GUEST_FS_SELECTOR 0x00000808
22 #define VMX_GUEST_GS_SELECTOR 0x0000080A
23 #define VMX_GUEST_LDTR_SELECTOR 0x0000080C
24 #define VMX_GUEST_TR_SELECTOR 0x0000080E
25
26 #define VMX_HOST_ES_SELECTOR 0x00000C00
27 #define VMX_HOST_CS_SELECTOR 0x00000C02
28 #define VMX_HOST_SS_SELECTOR 0x00000C04
29 #define VMX_HOST_DS_SELECTOR 0x00000C06
30 #define VMX_HOST_FS_SELECTOR 0x00000C08
31 #define VMX_HOST_GS_SELECTOR 0x00000C0A
32 #define VMX_HOST_TR_SELECTOR 0x00000C0C
33
34 #define VMX_CONTROL_IOA_ADDRESS 0x00002000
35 #define VMX_CONTROL_IOB_ADDRESS 0x00002002
36 #define VMX_CONTROL_MSR_ADDRESS 0x00002004
37 #define VMX_CONTROL_EXIT_MSR_STORE_ADDRESS 0x00002006
38 #define VMX_CONTROL_EXIT_MSR_LOAD_ADDRESS 0x00002008
39 #define VMX_CONTROL_ENTRY_MSR_LOAD_ADDRESS 0x0000200A
40 #define VMX_CONTROL_EXECUTIVE_VMCS_POINTER 0x0000200C
41 #define VMX_CONTROL_TSC_OFFSET 0x00002010
42 #define VMX_CONTROL_VIRTUAL_APIC_ADDRESS 0x00002012
43 #define VMX_CONTROL_APIC_ACCESS_ADDRESS 0x00002014
44 #define VMX_CONTROL_EPT_POINTER 0x0000201A
45
46 #define VMX_DATA_GUEST_PHYSICAL 0x00002400
47
48 #define VMX_GUEST_VMCS_LINK_POINTER 0x00002800
49 #define VMX_GUEST_VMCS_LINK_POINTER_HIGH 0x00002801
50 #define VMX_GUEST_DEBUGCTRL 0x00002802
51 #define VMX_GUEST_PAT 0x00002804
52 #define VMX_GUEST_EFER 0x00002806
53 #define VMX_GUEST_PERF_GLOBAL_CTRL 0x00002808
54 #define VMX_GUEST_PDPTE0 0x0000280A
55 #define VMX_GUEST_PDPTE1 0x0000280C
56 #define VMX_GUEST_PDPTE2 0x0000280E
57 #define VMX_GUEST_PDPTE3 0x00002810
58
59 #define VMX_HOST_PAT 0x00002C00
60 #define VMX_HOST_EFER 0x00002C02
61 #define VMX_HOST_PERF_GLOBAL_CTRL 0x00002C04
62
63 #define VMX_CONTROL_PIN_EXECUTION_CONTROLS 0x00004000
64 #define VMX_CONTROL_PRIMARY_PROCESSOR_CONTROLS 0x00004002
65 #define VMX_CONTROL_EXCEPTION_BITMAP 0x00004004
66 #define VMX_CONTROL_PAGE_FAULT_MASK 0x00004006
67 #define VMX_CONTROL_PAGE_FAULT_MATCH 0x00004008
68 #define VMX_CONTROL_CR3_TARGET_COUNT 0x0000400A
69 #define VMX_CONTROL_EXIT_CONTROLS 0x0000400C
70 #define VMX_CONTROL_EXIT_MSR_STORE_COUNT 0x0000400E
71 #define VMX_CONTROL_EXIT_MSR_LOAD_COUNT 0x00004010
72 #define VMX_CONTROL_ENTRY_CONTROLS 0x00004012
73 #define VMX_CONTROL_ENTRY_MSR_LOAD_COUNT 0x00004014
74 #define VMX_CONTROL_ENTRY_INTERRUPTION_INFO 0x00004016
75 #define VMX_CONTROL_ENTRY_EXCEPTION_ERROR_CODE 0x00004018
76 #define VMX_CONTROL_ENTRY_INSTRUCTION_LENGTH 0x0000401A
77 #define VMX_CONTROL_TPR_THRESHOLD 0x0000401C
78 #define VMX_CONTROL_SECONDARY_PROCESSOR_CONTROLS 0x0000401E
79 #define VMX_CONTROL_PLE_GAP 0x00004020
80 #define VMX_CONTROL_PLE_WINDOW 0x00004022
81
82 #define VMX_DATA_INSTRUCTION_ERROR 0x00004400
83 #define VMX_DATA_EXIT_REASON 0x00004402
84 #define VMX_DATA_EXIT_INTERRUPT_INFO 0x00004404
85 #define VMX_DATA_EXIT_INTERRUPT_ERROR 0x00004406
86 #define VMX_DATA_IDT_VECTOR_INFO 0x00004408
87 #define VMX_DATA_IDT_VECTOR_ERROR 0x0000440A
88 #define VMX_DATA_EXIT_INSTRUCTION_LENGTH 0x0000440C
89 #define VMX_DATA_EXIT_INSTRUCTION_INFO 0x0000440E
90
91 #define VMX_GUEST_ES_LIMIT 0x00004800
92 #define VMX_GUEST_CS_LIMIT 0x00004802
93 #define VMX_GUEST_SS_LIMIT 0x00004804
94 #define VMX_GUEST_DS_LIMIT 0x00004806
95 #define VMX_GUEST_FS_LIMIT 0x00004808
96 #define VMX_GUEST_GS_LIMIT 0x0000480A
97 #define VMX_GUEST_LDTR_LIMIT 0x0000480C
98 #define VMX_GUEST_TR_LIMIT 0x0000480E
99 #define VMX_GUEST_GDTR_LIMIT 0x00004810
100 #define VMX_GUEST_IDTR_LIMIT 0x00004812
101 #define VMX_GUEST_ES_ACCESS_RIGHTS 0x00004814
102 #define VMX_GUEST_CS_ACCESS_RIGHTS 0x00004816
103 #define VMX_GUEST_SS_ACCESS_RIGHTS 0x00004818
104 #define VMX_GUEST_DS_ACCESS_RIGHTS 0x0000481A
105 #define VMX_GUEST_FS_ACCESS_RIGHTS 0x0000481C
106 #define VMX_GUEST_GS_ACCESS_RIGHTS 0x0000481E
107 #define VMX_GUEST_LDTR_ACCESS_RIGHTS 0x00004820
108 #define VMX_GUEST_TR_ACCESS_RIGHTS 0x00004822
109 #define VMX_GUEST_INTERRUPTABILITY 0x00004824
110 #define VMX_GUEST_ACTIVITY 0x00004826
111 #define VMX_GUEST_SMBASE 0x00004828
112 #define VMX_GUEST_SYSENTER_CS 0x0000482A
113 #define VMX_GUEST_PREEMPTION_TIMER_VALUE 0x0000482E
114
115 #define VMX_HOST_SYSENTER_CS 0x00004C00
116
117 #define VMX_CONTROL_CR0_MASK 0x00006000
118 #define VMX_CONTROL_CR4_MASK 0x00006002
119 #define VMX_CONTROL_CR0_READ_SHADOW 0x00006004
120 #define VMX_CONTROL_CR4_READ_SHADOW 0x00006006
121 #define VMX_CONTROL_CR3_TARGET0 0x00006008
122 #define VMX_CONTROL_CR3_TARGET1 0x0000600A
123 #define VMX_CONTROL_CR3_TARGET2 0x0000600C
124 #define VMX_CONTROL_CR3_TARGET3 0x0000600E
125
126 #define VMX_DATA_EXIT_QUALIFICATION 0x00006400
127 #define VMX_DATA_IO_RCX 0x00006402
128 #define VMX_DATA_IO_RSI 0x00006404
129 #define VMX_DATA_IO_RDI 0x00006406
130 #define VMX_DATA_IO_RIP 0x00006408
131 #define VMX_DATA_GUEST_LINEAR_ADDRESS 0x0000640A
132
133 #define VMX_GUEST_CR0 0x00006800
134 #define VMX_GUEST_CR3 0x00006802
135 #define VMX_GUEST_CR4 0x00006804
136 #define VMX_GUEST_ES_BASE 0x00006806
137 #define VMX_GUEST_CS_BASE 0x00006808
138 #define VMX_GUEST_SS_BASE 0x0000680A
139 #define VMX_GUEST_DS_BASE 0x0000680C
140 #define VMX_GUEST_FS_BASE 0x0000680E
141 #define VMX_GUEST_GS_BASE 0x00006810
142 #define VMX_GUEST_LDTR_BASE 0x00006812
143 #define VMX_GUEST_TR_BASE 0x00006814
144 #define VMX_GUEST_GDTR_BASE 0x00006816
145 #define VMX_GUEST_IDTR_BASE 0x00006818
146 #define VMX_GUEST_DR7 0x0000681A
147 #define VMX_GUEST_RSP 0x0000681C
148 #define VMX_GUEST_RIP 0x0000681E
149 #define VMX_GUEST_RFLAGS 0x00006820
150 #define VMX_GUEST_PENDING_DEBUG_EXCEPTIONS 0x00006822
151 #define VMX_GUEST_SYSENTER_ESP 0x00006824
152 #define VMX_GUEST_SYSENTER_EIP 0x00006826
153
154 #define VMX_HOST_CR0 0x00006C00
155 #define VMX_HOST_CR3 0x00006C02
156 #define VMX_HOST_CR4 0x00006C04
157 #define VMX_HOST_FS_BASE 0x00006C06
158 #define VMX_HOST_GS_BASE 0x00006C08
159 #define VMX_HOST_TR_BASE 0x00006C0A
160 #define VMX_HOST_GDTR_BASE 0x00006C0C
161 #define VMX_HOST_IDTR_BASE 0x00006C0E
162 #define VMX_HOST_SYSENTER_ESP 0x00006C10
163 #define VMX_HOST_SYSENTER_EIP 0x00006C12
164 #define VMX_HOST_RSP 0x00006C14
165 #define VMX_HOST_RIP 0x00006C16
166
167 /* Exit reasons. */
168 enum exit_reasons {
169 EXCEPTION_OR_NMI = 0x00,
170 EXTERNAL_INTERRUPT = 0x01,
171 TRIPLE_FAULT = 0x02,
172 INIT_SIGNAL = 0x03,
173 SIPI = 0x04,
174 /*IO_SMI = 0x05,
175 * OTHER_SMI = 0x06,*/
176 INTERRUPT_WINDOW = 0x07,
177 NMI_WINDOW = 0x08,
178 TASK_SWITCH = 0x09,
179 CPUID = 0x0A,
180 GETSEC = 0x0B,
181 HLT = 0x0C,
182 INVD = 0x0D,
183 INVLPG = 0x0E,
184 RDPMC = 0x0F,
185 RDTSC = 0x10,
186 RSM = 0x11,
187 VMCALL = 0x12,
188 VMCLEAR = 0x13,
189 VMLAUNCH = 0x14,
190 VMPTRLD = 0x15,
191 VMPTRST = 0x16,
192 VMREAD = 0x17,
193 VMRESUME = 0x18,
194 VMWRITE = 0x19,
195 VMXOFF = 0x1A,
196 VMXON = 0x1B,
197 CONTROL_REGISTER = 0x1C,
198 MOV_DR = 0x1D,
199 IO = 0x1E,
200 RDMSR = 0x1F,
201 WRMSR = 0x20,
202 INVALID_GUEST_STATE = 0x21,
203 MSR_LOAD_FAIL = 0x22,
204 /* 0x23 */
205 MWAIT = 0x24,
206 MONITOR_TRAP_FLAG = 0x25,
207 /* 0x26 */
208 MONITOR = 0x27,
209 PAUSE = 0x28,
210 MACHINE_CHECK = 0x29,
211 /* 0x2A */
212 TPR_BELOW_THRESHOLD = 0x2B,
213 APIC_ACCESS = 0x2C,
214 GDTR_OR_IDTR = 0x2E,
215 LDTR_OR_TR = 0x2F,
216 EPT_VIOLATION = 0x30,
217 EPT_MISCONFIGURATION = 0x31,
218 INVEPT = 0x32,
219 RDTSCP = 0x33,
220 VMX_PREEMPTION_TIMER = 0x34,
221 INVVPID = 0x35,
222 WBINVD = 0x36,
223 XSETBV = 0x37
224 };
225
226 #define VPID_INVALID 0
227 #define VPID_FIRST 1
228 #define VPID_LAST (CONFIG_MAX_VPIDS - 1)
229
230 typedef uint16_t vpid_t;
231
232 #ifdef CONFIG_VTX
233 #define VTX_TERNARY(vtx, nonvtx) vtx
234
235 enum vcpu_gp_register {
236 VCPU_EAX = 0,
237 VCPU_EBX,
238 VCPU_ECX,
239 VCPU_EDX,
240 VCPU_ESI,
241 VCPU_EDI,
242 VCPU_EBP,
243 n_vcpu_gp_register,
244 /* We need to define a sentinal value to detect ESP that is strictly distinct
245 * from any of our other GP register indexes, so put that here */
246 VCPU_ESP,
247 };
248
249 typedef enum vcpu_gp_register vcpu_gp_register_t;;
250
251 const vcpu_gp_register_t crExitRegs[];
252
253 struct vcpu {
254 /* Storage for VMCS region. First field of vcpu_t so they share address.
255 * Will use at most 4KiB of memory. Statically reserve 4KiB for convenience. */
256 char vmcs[VCPU_VMCS_SIZE];
257 word_t io[VCPU_IOBITMAP_SIZE / sizeof(word_t)];
258
259 /* Place the fpu state here so that it is aligned */
260 user_fpu_state_t fpuState;
261
262 /* General purpose registers that we have to save and restore as they
263 * are not part of the vmcs */
264 word_t gp_registers[n_vcpu_gp_register];
265 #if defined(ENABLE_SMP_SUPPORT) && defined(CONFIG_ARCH_IA32)
266 word_t kernelSP;
267 #endif
268
269 /* TCB associated with this VCPU. */
270 struct tcb *vcpuTCB;
271 bool_t launched;
272
273 /* Currently assigned VPID */
274 vpid_t vpid;
275
276 /* This is the cr0 value has requested by the VCPU owner. The actual cr0 value set at
277 * any particular time may be different to this for lazy fpu management, but we will
278 * still respect whatever semantics the user has put into cr0. The cr0 value may
279 * additionally be modified by the guest depending on how the VCPU owner has configured
280 * the cr0 shadow/mask */
281 word_t cr0;
282
283 /* These are the values for the shadow and mask that the VCPU owner has requested.
284 * Like cr0, they actual values set may be slightly different but semantics will
285 * be preserved */
286 word_t cr0_shadow;
287 word_t cr0_mask;
288 word_t exception_bitmap;
289
290 /* These values serve as a cache of what is presently in the VMCS allowing for
291 * optimizing away unnecessary calls to vmwrite/vmread */
292 word_t cached_exception_bitmap;
293 word_t cached_cr0_shadow;
294 word_t cached_cr0_mask;
295 word_t cached_cr0;
296
297 /* Last used EPT root */
298 word_t last_ept_root;
299
300 #ifndef CONFIG_KERNEL_SKIM_WINDOW
301 /* Last set host cr3 */
302 word_t last_host_cr3;
303 #endif
304
305 #ifdef ENABLE_SMP_SUPPORT
306 /* Core this VCPU was last loaded on, or is currently loaded on */
307 word_t last_cpu;
308 #endif /* ENABLE_SMP_SUPPORT */
309 };
310 typedef struct vcpu vcpu_t;
311
312 compile_assert(vcpu_size_sane, sizeof(vcpu_t) <= BIT(seL4_X86_VCPUBits))
313 unverified_compile_assert(vcpu_fpu_state_alignment_valid,
314 OFFSETOF(vcpu_t, fpuState) % MIN_FPU_ALIGNMENT == 0)
315
316 /* Initializes a VCPU object with default values. A VCPU object that is not inititlized
317 * must not be run/loaded with vmptrld */
318 void vcpu_init(vcpu_t *vcpu);
319
320 /* Cleans up the VCPU object such that its memory can be freed */
321 void vcpu_finalise(vcpu_t *vcpu);
322
323 exception_t decodeX86VCPUInvocation(
324 word_t invLabel,
325 word_t length,
326 cptr_t cptr,
327 cte_t *slot,
328 cap_t cap,
329 word_t *buffer
330 );
331
332 /* Updates the state of the provided VCPU for a SysVMEnter syscall. The state
333 * is pulled from the current threads messages registers */
334 void vcpu_update_state_sysvmenter(vcpu_t *vcpu);
335
336 void vcpu_sysvmenter_reply_to_user(tcb_t *tcb);
337
338 bool_t vtx_init(void);
339 exception_t handleVmexit(void);
340 exception_t handleVmEntryFail(void);
341 void restoreVMCS(void);
342 void clearCurrentVCPU(void);
343
344 #ifdef ENABLE_SMP_SUPPORT
345 void VMCheckBoundNotification(tcb_t *tcb);
346 #endif /* ENABLE_SMP_SUPPORT */
347
348 void invept(ept_pml4e_t *ept_pml4);
349
350 /* Removes any IO port mappings that have been cached for the given VPID */
351 void clearVPIDIOPortMappings(vpid_t vpid, uint16_t first, uint16_t last);
352
vmread(word_t field)353 static inline word_t vmread(word_t field)
354 {
355 word_t value;
356 asm volatile(
357 "vmread %1, %0"
358 : "=r"(value)
359 : "r"(field)
360 : "cc"
361 );
362 return value;
363 }
364
365 #include <machine/io.h>
366
vmwrite(word_t field,word_t value)367 static inline void vmwrite(word_t field, word_t value)
368 {
369 asm volatile(
370 "vmwrite %0, %1"
371 :
372 : "r"(value), "r"(field)
373 : "cc"
374 );
375 }
376
377 #else /* CONFIG_VTX */
378 #define VTX_TERNARY(vtx, nonvtx) nonvtx
379 #endif /* CONFIG_VTX */
380