1 /******************************************************************************
2 * alternative.c
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; If not, see <http://www.gnu.org/licenses/>.
16 */
17
18 #include <xen/types.h>
19 #include <asm/processor.h>
20 #include <asm/alternative.h>
21 #include <xen/init.h>
22 #include <asm/system.h>
23 #include <asm/traps.h>
24 #include <asm/nmi.h>
25 #include <xen/livepatch.h>
26
27 #define MAX_PATCH_LEN (255-1)
28
29 extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
30
31 #ifdef K8_NOP1
32 static const unsigned char k8nops[] init_or_livepatch_const = {
33 K8_NOP1,
34 K8_NOP2,
35 K8_NOP3,
36 K8_NOP4,
37 K8_NOP5,
38 K8_NOP6,
39 K8_NOP7,
40 K8_NOP8
41 };
42 static const unsigned char * const k8_nops[ASM_NOP_MAX+1] init_or_livepatch_constrel = {
43 NULL,
44 k8nops,
45 k8nops + 1,
46 k8nops + 1 + 2,
47 k8nops + 1 + 2 + 3,
48 k8nops + 1 + 2 + 3 + 4,
49 k8nops + 1 + 2 + 3 + 4 + 5,
50 k8nops + 1 + 2 + 3 + 4 + 5 + 6,
51 k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7
52 };
53 #endif
54
55 #ifdef P6_NOP1
56 static const unsigned char p6nops[] init_or_livepatch_const = {
57 P6_NOP1,
58 P6_NOP2,
59 P6_NOP3,
60 P6_NOP4,
61 P6_NOP5,
62 P6_NOP6,
63 P6_NOP7,
64 P6_NOP8
65 };
66 static const unsigned char * const p6_nops[ASM_NOP_MAX+1] init_or_livepatch_constrel = {
67 NULL,
68 p6nops,
69 p6nops + 1,
70 p6nops + 1 + 2,
71 p6nops + 1 + 2 + 3,
72 p6nops + 1 + 2 + 3 + 4,
73 p6nops + 1 + 2 + 3 + 4 + 5,
74 p6nops + 1 + 2 + 3 + 4 + 5 + 6,
75 p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7
76 };
77 #endif
78
79 static const unsigned char * const *ideal_nops init_or_livepatch_data = k8_nops;
80
mask_nmi_callback(const struct cpu_user_regs * regs,int cpu)81 static int __init mask_nmi_callback(const struct cpu_user_regs *regs, int cpu)
82 {
83 return 1;
84 }
85
arch_init_ideal_nops(void)86 static void __init arch_init_ideal_nops(void)
87 {
88 /*
89 * Due to a decoder implementation quirk, some
90 * specific Intel CPUs actually perform better with
91 * the "k8_nops" than with the SDM-recommended NOPs.
92 */
93 if ( (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
94 !(boot_cpu_data.x86 == 6 &&
95 boot_cpu_data.x86_model >= 0x0f &&
96 boot_cpu_data.x86_model != 0x1c &&
97 boot_cpu_data.x86_model != 0x26 &&
98 boot_cpu_data.x86_model != 0x27 &&
99 boot_cpu_data.x86_model < 0x30) )
100 ideal_nops = p6_nops;
101 }
102
103 /* Use this to add nops to a buffer, then text_poke the whole buffer. */
add_nops(void * insns,unsigned int len)104 void init_or_livepatch add_nops(void *insns, unsigned int len)
105 {
106 while ( len > 0 )
107 {
108 unsigned int noplen = len;
109 if ( noplen > ASM_NOP_MAX )
110 noplen = ASM_NOP_MAX;
111 memcpy(insns, ideal_nops[noplen], noplen);
112 insns += noplen;
113 len -= noplen;
114 }
115 }
116
117 /*
118 * text_poke - Update instructions on a live kernel or non-executed code.
119 * @addr: address to modify
120 * @opcode: source of the copy
121 * @len: length to copy
122 *
123 * When you use this code to patch more than one byte of an instruction
124 * you need to make sure that other CPUs cannot execute this code in parallel.
125 * Also no thread must be currently preempted in the middle of these
126 * instructions. And on the local CPU you need to be protected again NMI or MCE
127 * handlers seeing an inconsistent instruction while you patch.
128 *
129 * You should run this with interrupts disabled or on code that is not
130 * executing.
131 *
132 * "noinline" to cause control flow change and thus invalidate I$ and
133 * cause refetch after modification.
134 */
135 static void *init_or_livepatch noinline
text_poke(void * addr,const void * opcode,size_t len)136 text_poke(void *addr, const void *opcode, size_t len)
137 {
138 return memcpy(addr, opcode, len);
139 }
140
141 /*
142 * Replace instructions with better alternatives for this CPU type.
143 * This runs before SMP is initialized to avoid SMP problems with
144 * self modifying code. This implies that asymmetric systems where
145 * APs have less capabilities than the boot processor are not handled.
146 * Tough. Make sure you disable such features by hand.
147 */
apply_alternatives(const struct alt_instr * start,const struct alt_instr * end)148 void init_or_livepatch apply_alternatives(const struct alt_instr *start,
149 const struct alt_instr *end)
150 {
151 const struct alt_instr *a;
152 u8 *instr, *replacement;
153 u8 insnbuf[MAX_PATCH_LEN];
154
155 printk(KERN_INFO "alt table %p -> %p\n", start, end);
156
157 /*
158 * The scan order should be from start to end. A later scanned
159 * alternative code can overwrite a previous scanned alternative code.
160 * Some kernel functions (e.g. memcpy, memset, etc) use this order to
161 * patch code.
162 *
163 * So be careful if you want to change the scan order to any other
164 * order.
165 */
166 for ( a = start; a < end; a++ )
167 {
168 instr = (u8 *)&a->instr_offset + a->instr_offset;
169 replacement = (u8 *)&a->repl_offset + a->repl_offset;
170 BUG_ON(a->replacementlen > a->instrlen);
171 BUG_ON(a->instrlen > sizeof(insnbuf));
172 BUG_ON(a->cpuid >= NCAPINTS * 32);
173 if ( !boot_cpu_has(a->cpuid) )
174 continue;
175
176 memcpy(insnbuf, replacement, a->replacementlen);
177
178 /* 0xe8/0xe9 are relative branches; fix the offset. */
179 if ( a->replacementlen >= 5 && (*insnbuf & 0xfe) == 0xe8 )
180 *(s32 *)(insnbuf + 1) += replacement - instr;
181
182 add_nops(insnbuf + a->replacementlen,
183 a->instrlen - a->replacementlen);
184 text_poke(instr, insnbuf, a->instrlen);
185 }
186 }
187
188 /*
189 * This routine is called with local interrupt disabled and used during
190 * bootup.
191 */
alternative_instructions(void)192 void __init alternative_instructions(void)
193 {
194 nmi_callback_t *saved_nmi_callback;
195 unsigned long cr0 = read_cr0();
196
197 arch_init_ideal_nops();
198
199 /*
200 * The patching is not fully atomic, so try to avoid local interruptions
201 * that might execute the to be patched code.
202 * Other CPUs are not running.
203 */
204 saved_nmi_callback = set_nmi_callback(mask_nmi_callback);
205
206 /*
207 * Don't stop machine check exceptions while patching.
208 * MCEs only happen when something got corrupted and in this
209 * case we must do something about the corruption.
210 * Ignoring it is worse than a unlikely patching race.
211 * Also machine checks tend to be broadcast and if one CPU
212 * goes into machine check the others follow quickly, so we don't
213 * expect a machine check to cause undue problems during to code
214 * patching.
215 */
216 ASSERT(!local_irq_is_enabled());
217
218 /* Disable WP to allow application of alternatives to read-only pages. */
219 write_cr0(cr0 & ~X86_CR0_WP);
220
221 apply_alternatives(__alt_instructions, __alt_instructions_end);
222
223 /* Reinstate WP. */
224 write_cr0(cr0);
225
226 set_nmi_callback(saved_nmi_callback);
227 }
228