1 /****************************************************************************** 2 * include/asm-x86/spec_ctrl.h 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; If not, see <http://www.gnu.org/licenses/>. 16 * 17 * Copyright (c) 2017-2018 Citrix Systems Ltd. 18 */ 19 20 #ifndef __X86_SPEC_CTRL_ASM_H__ 21 #define __X86_SPEC_CTRL_ASM_H__ 22 23 #ifdef __ASSEMBLY__ 24 #include <asm/msr-index.h> 25 #include <asm/spec_ctrl.h> 26 27 /* 28 * Saving and restoring MSR_SPEC_CTRL state is a little tricky. 29 * 30 * We want the guests choice of SPEC_CTRL while in guest context, and Xen's 31 * choice (set or clear, depending on the hardware) while running in Xen 32 * context. Therefore, a simplistic algorithm is: 33 * 34 * - Set/clear IBRS on entry to Xen 35 * - Set the guests' choice on exit to guest 36 * - Leave SPEC_CTRL unchanged on exit to xen 37 * 38 * There are two complicating factors: 39 * 1) HVM guests can have direct access to the MSR, so it can change 40 * behind Xen's back. 41 * 2) An NMI or MCE can interrupt at any point, including early in the entry 42 * path, or late in the exit path after restoring the guest value. This 43 * will corrupt the guest value. 44 * 45 * Factor 1 is dealt with by relying on NMIs/MCEs being blocked immediately 46 * after VMEXIT. The VMEXIT-specific code reads MSR_SPEC_CTRL and updates 47 * current before loading Xen's MSR_SPEC_CTRL setting. 48 * 49 * Factor 2 is harder. We maintain a shadow_spec_ctrl value, and a use_shadow 50 * boolean in the per cpu spec_ctrl_flags. The synchronous use is: 51 * 52 * 1) Store guest value in shadow_spec_ctrl 53 * 2) Set the use_shadow boolean 54 * 3) Load guest value into MSR_SPEC_CTRL 55 * 4) Exit to guest 56 * 5) Entry from guest 57 * 6) Clear the use_shadow boolean 58 * 7) Load Xen's value into MSR_SPEC_CTRL 59 * 60 * The asynchronous use for interrupts/exceptions is: 61 * - Set/clear IBRS on entry to Xen 62 * - On exit to Xen, check use_shadow 63 * - If set, load shadow_spec_ctrl 64 * 65 * Therefore, an interrupt/exception which hits the synchronous path between 66 * steps 2 and 6 will restore the shadow value rather than leaving Xen's value 67 * loaded and corrupting the value used in guest context. 68 * 69 * The following ASM fragments implement this algorithm. See their local 70 * comments for further details. 71 * - SPEC_CTRL_ENTRY_FROM_HVM 72 * - SPEC_CTRL_ENTRY_FROM_PV 73 * - SPEC_CTRL_ENTRY_FROM_INTR 74 * - SPEC_CTRL_ENTRY_FROM_INTR_IST 75 * - SPEC_CTRL_EXIT_TO_XEN_IST 76 * - SPEC_CTRL_EXIT_TO_XEN 77 * - SPEC_CTRL_EXIT_TO_PV 78 * - SPEC_CTRL_EXIT_TO_HVM 79 */ 80 81 .macro DO_OVERWRITE_RSB tmp=rax 82 /* 83 * Requires nothing 84 * Clobbers \tmp (%rax by default), %rcx 85 * 86 * Requires 256 bytes of {,shadow}stack space, but %rsp/SSP has no net 87 * change. Based on Google's performance numbers, the loop is unrolled to 16 88 * iterations and two calls per iteration. 89 * 90 * The call filling the RSB needs a nonzero displacement. A nop would do, but 91 * we use "1: pause; lfence; jmp 1b" to safely contains any ret-based 92 * speculation, even if the loop is speculatively executed prematurely. 93 * 94 * %rsp is preserved by using an extra GPR because a) we've got plenty spare, 95 * b) the two movs are shorter to encode than `add $32*8, %rsp`, and c) can be 96 * optimised with mov-elimination in modern cores. 97 */ 98 mov $16, %ecx /* 16 iterations, two calls per loop */ 99 mov %rsp, %\tmp /* Store the current %rsp */ 100 101 .L\@_fill_rsb_loop: 102 103 .irp n, 1, 2 /* Unrolled twice. */ 104 call .L\@_insert_rsb_entry_\n /* Create an RSB entry. */ 105 106 .L\@_capture_speculation_\n: 107 pause 108 lfence 109 jmp .L\@_capture_speculation_\n /* Capture rogue speculation. */ 110 111 .L\@_insert_rsb_entry_\n: 112 .endr 113 114 sub $1, %ecx 115 jnz .L\@_fill_rsb_loop 116 mov %\tmp, %rsp /* Restore old %rsp */ 117 118 #ifdef CONFIG_XEN_SHSTK 119 mov $1, %ecx 120 rdsspd %ecx 121 cmp $1, %ecx 122 je .L\@_shstk_done 123 mov $64, %ecx /* 64 * 4 bytes, given incsspd */ 124 incsspd %ecx /* Restore old SSP */ 125 .L\@_shstk_done: 126 #endif 127 .endm 128 129 .macro DO_SPEC_CTRL_ENTRY_FROM_HVM 130 /* 131 * Requires %rbx=current, %rsp=regs/cpuinfo 132 * Clobbers %rax, %rcx, %rdx 133 * 134 * The common case is that a guest has direct access to MSR_SPEC_CTRL, at 135 * which point we need to save the guest value before setting IBRS for Xen. 136 * Unilaterally saving the guest value is shorter and faster than checking. 137 */ 138 mov $MSR_SPEC_CTRL, %ecx 139 rdmsr 140 141 /* Stash the value from hardware. */ 142 mov VCPU_arch_msrs(%rbx), %rdx 143 mov %eax, VCPUMSR_spec_ctrl_raw(%rdx) 144 xor %edx, %edx 145 146 /* Clear SPEC_CTRL shadowing *before* loading Xen's value. */ 147 andb $~SCF_use_shadow, CPUINFO_spec_ctrl_flags(%rsp) 148 149 /* Load Xen's intended value. */ 150 movzbl CPUINFO_xen_spec_ctrl(%rsp), %eax 151 wrmsr 152 .endm 153 154 .macro DO_SPEC_CTRL_ENTRY maybexen:req 155 /* 156 * Requires %rsp=regs (also cpuinfo if !maybexen) 157 * Requires %r14=stack_end (if maybexen) 158 * Clobbers %rax, %rcx, %rdx 159 * 160 * PV guests can't update MSR_SPEC_CTRL behind Xen's back, so no need to read 161 * it back. Entries from guest context need to clear SPEC_CTRL shadowing, 162 * while entries from Xen must leave shadowing in its current state. 163 */ 164 mov $MSR_SPEC_CTRL, %ecx 165 xor %edx, %edx 166 167 /* 168 * Clear SPEC_CTRL shadowing *before* loading Xen's value. If entering 169 * from a possibly-xen context, %rsp doesn't necessarily alias the cpuinfo 170 * block so calculate the position directly. 171 */ 172 .if \maybexen 173 xor %eax, %eax 174 /* Branchless `if ( !xen ) clear_shadowing` */ 175 testb $3, UREGS_cs(%rsp) 176 setnz %al 177 not %eax 178 and %al, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%r14) 179 movzbl STACK_CPUINFO_FIELD(xen_spec_ctrl)(%r14), %eax 180 .else 181 andb $~SCF_use_shadow, CPUINFO_spec_ctrl_flags(%rsp) 182 movzbl CPUINFO_xen_spec_ctrl(%rsp), %eax 183 .endif 184 185 wrmsr 186 .endm 187 188 .macro DO_SPEC_CTRL_EXIT_TO_XEN 189 /* 190 * Requires %rbx=stack_end 191 * Clobbers %rax, %rcx, %rdx 192 * 193 * When returning to Xen context, look to see whether SPEC_CTRL shadowing is 194 * in effect, and reload the shadow value. This covers race conditions which 195 * exist with an NMI/MCE/etc hitting late in the return-to-guest path. 196 */ 197 xor %edx, %edx 198 199 testb $SCF_use_shadow, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%rbx) 200 jz .L\@_skip 201 202 mov STACK_CPUINFO_FIELD(shadow_spec_ctrl)(%rbx), %eax 203 mov $MSR_SPEC_CTRL, %ecx 204 wrmsr 205 206 .L\@_skip: 207 .endm 208 209 .macro DO_SPEC_CTRL_EXIT_TO_GUEST 210 /* 211 * Requires %eax=spec_ctrl, %rsp=regs/cpuinfo 212 * Clobbers %rcx, %rdx 213 * 214 * When returning to guest context, set up SPEC_CTRL shadowing and load the 215 * guest value. 216 */ 217 /* Set up shadow value *before* enabling shadowing. */ 218 mov %eax, CPUINFO_shadow_spec_ctrl(%rsp) 219 220 /* Set SPEC_CTRL shadowing *before* loading the guest value. */ 221 orb $SCF_use_shadow, CPUINFO_spec_ctrl_flags(%rsp) 222 223 mov $MSR_SPEC_CTRL, %ecx 224 xor %edx, %edx 225 wrmsr 226 .endm 227 228 /* Use after a VMEXIT from an HVM guest. */ 229 #define SPEC_CTRL_ENTRY_FROM_HVM \ 230 ALTERNATIVE "", DO_OVERWRITE_RSB, X86_FEATURE_SC_RSB_HVM; \ 231 ALTERNATIVE "", DO_SPEC_CTRL_ENTRY_FROM_HVM, \ 232 X86_FEATURE_SC_MSR_HVM 233 234 /* Use after an entry from PV context (syscall/sysenter/int80/int82/etc). */ 235 #define SPEC_CTRL_ENTRY_FROM_PV \ 236 ALTERNATIVE "", DO_OVERWRITE_RSB, X86_FEATURE_SC_RSB_PV; \ 237 ALTERNATIVE "", __stringify(DO_SPEC_CTRL_ENTRY maybexen=0), \ 238 X86_FEATURE_SC_MSR_PV 239 240 /* Use in interrupt/exception context. May interrupt Xen or PV context. */ 241 #define SPEC_CTRL_ENTRY_FROM_INTR \ 242 ALTERNATIVE "", DO_OVERWRITE_RSB, X86_FEATURE_SC_RSB_PV; \ 243 ALTERNATIVE "", __stringify(DO_SPEC_CTRL_ENTRY maybexen=1), \ 244 X86_FEATURE_SC_MSR_PV 245 246 /* Use when exiting to Xen context. */ 247 #define SPEC_CTRL_EXIT_TO_XEN \ 248 ALTERNATIVE "", \ 249 DO_SPEC_CTRL_EXIT_TO_XEN, X86_FEATURE_SC_MSR_PV 250 251 /* Use when exiting to PV guest context. */ 252 #define SPEC_CTRL_EXIT_TO_PV \ 253 ALTERNATIVE "", \ 254 DO_SPEC_CTRL_EXIT_TO_GUEST, X86_FEATURE_SC_MSR_PV; \ 255 ALTERNATIVE "", __stringify(verw CPUINFO_verw_sel(%rsp)), \ 256 X86_FEATURE_SC_VERW_PV 257 258 /* Use when exiting to HVM guest context. */ 259 #define SPEC_CTRL_EXIT_TO_HVM \ 260 ALTERNATIVE "", \ 261 DO_SPEC_CTRL_EXIT_TO_GUEST, X86_FEATURE_SC_MSR_HVM; \ 262 ALTERNATIVE "", __stringify(verw CPUINFO_verw_sel(%rsp)), \ 263 X86_FEATURE_SC_VERW_HVM 264 265 /* 266 * Use in IST interrupt/exception context. May interrupt Xen or PV context. 267 * Fine grain control of SCF_ist_wrmsr is needed for safety in the S3 resume 268 * path to avoid using MSR_SPEC_CTRL before the microcode introducing it has 269 * been reloaded. 270 */ 271 .macro SPEC_CTRL_ENTRY_FROM_INTR_IST 272 /* 273 * Requires %rsp=regs, %r14=stack_end 274 * Clobbers %rax, %rcx, %rdx 275 * 276 * This is logical merge of DO_OVERWRITE_RSB and DO_SPEC_CTRL_ENTRY 277 * maybexen=1, but with conditionals rather than alternatives. 278 */ 279 movzbl STACK_CPUINFO_FIELD(spec_ctrl_flags)(%r14), %eax 280 281 test $SCF_ist_rsb, %al 282 jz .L\@_skip_rsb 283 284 DO_OVERWRITE_RSB tmp=rdx /* Clobbers %rcx/%rdx */ 285 286 .L\@_skip_rsb: 287 288 test $SCF_ist_wrmsr, %al 289 jz .L\@_skip_wrmsr 290 291 xor %edx, %edx 292 testb $3, UREGS_cs(%rsp) 293 setnz %dl 294 not %edx 295 and %dl, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%r14) 296 297 /* Load Xen's intended value. */ 298 mov $MSR_SPEC_CTRL, %ecx 299 movzbl STACK_CPUINFO_FIELD(xen_spec_ctrl)(%r14), %eax 300 xor %edx, %edx 301 wrmsr 302 303 /* Opencoded UNLIKELY_START() with no condition. */ 304 UNLIKELY_DISPATCH_LABEL(\@_serialise): 305 .subsection 1 306 /* 307 * In the case that we might need to set SPEC_CTRL.IBRS for safety, we 308 * need to ensure that an attacker can't poison the `jz .L\@_skip_wrmsr` 309 * to speculate around the WRMSR. As a result, we need a dispatch 310 * serialising instruction in the else clause. 311 */ 312 .L\@_skip_wrmsr: 313 lfence 314 UNLIKELY_END(\@_serialise) 315 .endm 316 317 /* Use when exiting to Xen in IST context. */ 318 .macro SPEC_CTRL_EXIT_TO_XEN_IST 319 /* 320 * Requires %rbx=stack_end 321 * Clobbers %rax, %rcx, %rdx 322 */ 323 testb $SCF_ist_wrmsr, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%rbx) 324 jz .L\@_skip 325 326 DO_SPEC_CTRL_EXIT_TO_XEN 327 328 .L\@_skip: 329 .endm 330 331 #endif /* __ASSEMBLY__ */ 332 #endif /* !__X86_SPEC_CTRL_ASM_H__ */ 333 334 /* 335 * Local variables: 336 * mode: C 337 * c-file-style: "BSD" 338 * c-basic-offset: 4 339 * tab-width: 4 340 * indent-tabs-mode: nil 341 * End: 342 */ 343