1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kasan.h>
3 #include <linux/sched/task.h>
4 #include <linux/pgtable.h>
5 #include <asm/pgalloc.h>
6 #include <asm/kasan.h>
7 #include <asm/mem_detect.h>
8 #include <asm/processor.h>
9 #include <asm/sclp.h>
10 #include <asm/facility.h>
11 #include <asm/sections.h>
12 #include <asm/setup.h>
13 #include <asm/uv.h>
14
15 static unsigned long segment_pos __initdata;
16 static unsigned long segment_low __initdata;
17 static bool has_edat __initdata;
18 static bool has_nx __initdata;
19
20 #define __sha(x) ((unsigned long)kasan_mem_to_shadow((void *)x))
21
kasan_early_panic(const char * reason)22 static void __init kasan_early_panic(const char *reason)
23 {
24 sclp_early_printk("The Linux kernel failed to boot with the KernelAddressSanitizer:\n");
25 sclp_early_printk(reason);
26 disabled_wait();
27 }
28
kasan_early_alloc_segment(void)29 static void * __init kasan_early_alloc_segment(void)
30 {
31 segment_pos -= _SEGMENT_SIZE;
32
33 if (segment_pos < segment_low)
34 kasan_early_panic("out of memory during initialisation\n");
35
36 return __va(segment_pos);
37 }
38
kasan_early_alloc_pages(unsigned int order)39 static void * __init kasan_early_alloc_pages(unsigned int order)
40 {
41 pgalloc_pos -= (PAGE_SIZE << order);
42
43 if (pgalloc_pos < pgalloc_low)
44 kasan_early_panic("out of memory during initialisation\n");
45
46 return __va(pgalloc_pos);
47 }
48
kasan_early_crst_alloc(unsigned long val)49 static void * __init kasan_early_crst_alloc(unsigned long val)
50 {
51 unsigned long *table;
52
53 table = kasan_early_alloc_pages(CRST_ALLOC_ORDER);
54 if (table)
55 crst_table_init(table, val);
56 return table;
57 }
58
kasan_early_pte_alloc(void)59 static pte_t * __init kasan_early_pte_alloc(void)
60 {
61 static void *pte_leftover;
62 pte_t *pte;
63
64 BUILD_BUG_ON(_PAGE_TABLE_SIZE * 2 != PAGE_SIZE);
65
66 if (!pte_leftover) {
67 pte_leftover = kasan_early_alloc_pages(0);
68 pte = pte_leftover + _PAGE_TABLE_SIZE;
69 } else {
70 pte = pte_leftover;
71 pte_leftover = NULL;
72 }
73 memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE);
74 return pte;
75 }
76
77 enum populate_mode {
78 POPULATE_MAP,
79 POPULATE_ZERO_SHADOW,
80 POPULATE_SHALLOW
81 };
82
pgprot_clear_bit(pgprot_t pgprot,unsigned long bit)83 static inline pgprot_t pgprot_clear_bit(pgprot_t pgprot, unsigned long bit)
84 {
85 return __pgprot(pgprot_val(pgprot) & ~bit);
86 }
87
kasan_early_pgtable_populate(unsigned long address,unsigned long end,enum populate_mode mode)88 static void __init kasan_early_pgtable_populate(unsigned long address,
89 unsigned long end,
90 enum populate_mode mode)
91 {
92 pgprot_t pgt_prot_zero = PAGE_KERNEL_RO;
93 pgprot_t pgt_prot = PAGE_KERNEL;
94 pgprot_t sgt_prot = SEGMENT_KERNEL;
95 pgd_t *pg_dir;
96 p4d_t *p4_dir;
97 pud_t *pu_dir;
98 pmd_t *pm_dir;
99 pte_t *pt_dir;
100 pmd_t pmd;
101 pte_t pte;
102
103 if (!has_nx) {
104 pgt_prot_zero = pgprot_clear_bit(pgt_prot_zero, _PAGE_NOEXEC);
105 pgt_prot = pgprot_clear_bit(pgt_prot, _PAGE_NOEXEC);
106 sgt_prot = pgprot_clear_bit(sgt_prot, _SEGMENT_ENTRY_NOEXEC);
107 }
108
109 while (address < end) {
110 pg_dir = pgd_offset_k(address);
111 if (pgd_none(*pg_dir)) {
112 if (mode == POPULATE_ZERO_SHADOW &&
113 IS_ALIGNED(address, PGDIR_SIZE) &&
114 end - address >= PGDIR_SIZE) {
115 pgd_populate(&init_mm, pg_dir,
116 kasan_early_shadow_p4d);
117 address = (address + PGDIR_SIZE) & PGDIR_MASK;
118 continue;
119 }
120 p4_dir = kasan_early_crst_alloc(_REGION2_ENTRY_EMPTY);
121 pgd_populate(&init_mm, pg_dir, p4_dir);
122 }
123
124 if (mode == POPULATE_SHALLOW) {
125 address = (address + P4D_SIZE) & P4D_MASK;
126 continue;
127 }
128
129 p4_dir = p4d_offset(pg_dir, address);
130 if (p4d_none(*p4_dir)) {
131 if (mode == POPULATE_ZERO_SHADOW &&
132 IS_ALIGNED(address, P4D_SIZE) &&
133 end - address >= P4D_SIZE) {
134 p4d_populate(&init_mm, p4_dir,
135 kasan_early_shadow_pud);
136 address = (address + P4D_SIZE) & P4D_MASK;
137 continue;
138 }
139 pu_dir = kasan_early_crst_alloc(_REGION3_ENTRY_EMPTY);
140 p4d_populate(&init_mm, p4_dir, pu_dir);
141 }
142
143 pu_dir = pud_offset(p4_dir, address);
144 if (pud_none(*pu_dir)) {
145 if (mode == POPULATE_ZERO_SHADOW &&
146 IS_ALIGNED(address, PUD_SIZE) &&
147 end - address >= PUD_SIZE) {
148 pud_populate(&init_mm, pu_dir,
149 kasan_early_shadow_pmd);
150 address = (address + PUD_SIZE) & PUD_MASK;
151 continue;
152 }
153 pm_dir = kasan_early_crst_alloc(_SEGMENT_ENTRY_EMPTY);
154 pud_populate(&init_mm, pu_dir, pm_dir);
155 }
156
157 pm_dir = pmd_offset(pu_dir, address);
158 if (pmd_none(*pm_dir)) {
159 if (IS_ALIGNED(address, PMD_SIZE) &&
160 end - address >= PMD_SIZE) {
161 if (mode == POPULATE_ZERO_SHADOW) {
162 pmd_populate(&init_mm, pm_dir, kasan_early_shadow_pte);
163 address = (address + PMD_SIZE) & PMD_MASK;
164 continue;
165 } else if (has_edat) {
166 void *page = kasan_early_alloc_segment();
167
168 memset(page, 0, _SEGMENT_SIZE);
169 pmd = __pmd(__pa(page));
170 pmd = set_pmd_bit(pmd, sgt_prot);
171 set_pmd(pm_dir, pmd);
172 address = (address + PMD_SIZE) & PMD_MASK;
173 continue;
174 }
175 }
176 pt_dir = kasan_early_pte_alloc();
177 pmd_populate(&init_mm, pm_dir, pt_dir);
178 } else if (pmd_large(*pm_dir)) {
179 address = (address + PMD_SIZE) & PMD_MASK;
180 continue;
181 }
182
183 pt_dir = pte_offset_kernel(pm_dir, address);
184 if (pte_none(*pt_dir)) {
185 void *page;
186
187 switch (mode) {
188 case POPULATE_MAP:
189 page = kasan_early_alloc_pages(0);
190 memset(page, 0, PAGE_SIZE);
191 pte = __pte(__pa(page));
192 pte = set_pte_bit(pte, pgt_prot);
193 set_pte(pt_dir, pte);
194 break;
195 case POPULATE_ZERO_SHADOW:
196 page = kasan_early_shadow_page;
197 pte = __pte(__pa(page));
198 pte = set_pte_bit(pte, pgt_prot_zero);
199 set_pte(pt_dir, pte);
200 break;
201 case POPULATE_SHALLOW:
202 /* should never happen */
203 break;
204 }
205 }
206 address += PAGE_SIZE;
207 }
208 }
209
kasan_early_detect_facilities(void)210 static void __init kasan_early_detect_facilities(void)
211 {
212 if (test_facility(8)) {
213 has_edat = true;
214 __ctl_set_bit(0, 23);
215 }
216 if (!noexec_disabled && test_facility(130)) {
217 has_nx = true;
218 __ctl_set_bit(0, 20);
219 }
220 }
221
kasan_early_init(void)222 void __init kasan_early_init(void)
223 {
224 pte_t pte_z = __pte(__pa(kasan_early_shadow_page) | pgprot_val(PAGE_KERNEL_RO));
225 pmd_t pmd_z = __pmd(__pa(kasan_early_shadow_pte) | _SEGMENT_ENTRY);
226 pud_t pud_z = __pud(__pa(kasan_early_shadow_pmd) | _REGION3_ENTRY);
227 p4d_t p4d_z = __p4d(__pa(kasan_early_shadow_pud) | _REGION2_ENTRY);
228 unsigned long untracked_end = MODULES_VADDR;
229 unsigned long shadow_alloc_size;
230 unsigned long start, end;
231 int i;
232
233 kasan_early_detect_facilities();
234 if (!has_nx)
235 pte_z = clear_pte_bit(pte_z, __pgprot(_PAGE_NOEXEC));
236
237 BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, P4D_SIZE));
238 BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, P4D_SIZE));
239
240 /* init kasan zero shadow */
241 crst_table_init((unsigned long *)kasan_early_shadow_p4d, p4d_val(p4d_z));
242 crst_table_init((unsigned long *)kasan_early_shadow_pud, pud_val(pud_z));
243 crst_table_init((unsigned long *)kasan_early_shadow_pmd, pmd_val(pmd_z));
244 memset64((u64 *)kasan_early_shadow_pte, pte_val(pte_z), PTRS_PER_PTE);
245
246 if (has_edat) {
247 shadow_alloc_size = get_mem_detect_usable_total() >> KASAN_SHADOW_SCALE_SHIFT;
248 segment_pos = round_down(pgalloc_pos, _SEGMENT_SIZE);
249 segment_low = segment_pos - shadow_alloc_size;
250 segment_low = round_down(segment_low, _SEGMENT_SIZE);
251 pgalloc_pos = segment_low;
252 }
253 /*
254 * Current memory layout:
255 * +- 0 -------------+ +- shadow start -+
256 * |1:1 ident mapping| /|1/8 of ident map|
257 * | | / | |
258 * +-end of ident map+ / +----------------+
259 * | ... gap ... | / | kasan |
260 * | | / | zero page |
261 * +- vmalloc area -+ / | mapping |
262 * | vmalloc_size | / | (untracked) |
263 * +- modules vaddr -+ / +----------------+
264 * | 2Gb |/ | unmapped | allocated per module
265 * +- shadow start -+ +----------------+
266 * | 1/8 addr space | | zero pg mapping| (untracked)
267 * +- shadow end ----+---------+- shadow end ---+
268 *
269 * Current memory layout (KASAN_VMALLOC):
270 * +- 0 -------------+ +- shadow start -+
271 * |1:1 ident mapping| /|1/8 of ident map|
272 * | | / | |
273 * +-end of ident map+ / +----------------+
274 * | ... gap ... | / | kasan zero page| (untracked)
275 * | | / | mapping |
276 * +- vmalloc area -+ / +----------------+
277 * | vmalloc_size | / |shallow populate|
278 * +- modules vaddr -+ / +----------------+
279 * | 2Gb |/ |shallow populate|
280 * +- shadow start -+ +----------------+
281 * | 1/8 addr space | | zero pg mapping| (untracked)
282 * +- shadow end ----+---------+- shadow end ---+
283 */
284 /* populate kasan shadow (for identity mapping and zero page mapping) */
285 for_each_mem_detect_usable_block(i, &start, &end)
286 kasan_early_pgtable_populate(__sha(start), __sha(end), POPULATE_MAP);
287 if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) {
288 untracked_end = VMALLOC_START;
289 /* shallowly populate kasan shadow for vmalloc and modules */
290 kasan_early_pgtable_populate(__sha(VMALLOC_START), __sha(MODULES_END),
291 POPULATE_SHALLOW);
292 }
293 /* populate kasan shadow for untracked memory */
294 kasan_early_pgtable_populate(__sha(ident_map_size), __sha(untracked_end),
295 POPULATE_ZERO_SHADOW);
296 kasan_early_pgtable_populate(__sha(MODULES_END), __sha(_REGION1_SIZE),
297 POPULATE_ZERO_SHADOW);
298 /* enable kasan */
299 init_task.kasan_depth = 0;
300 sclp_early_printk("KernelAddressSanitizer initialized\n");
301 }
302