1 /*
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * SPDX-FileCopyrightText: Copyright TF-RMM Contributors.
5 */
6
7 #include <arch.h>
8 #include <arch_features.h>
9 #include <attestation.h>
10 #include <buffer.h>
11 #include <cpuid.h>
12 #include <exit.h>
13 #include <fpu_helpers.h>
14 #include <rec.h>
15 #include <run.h>
16 #include <smc-rmi.h>
17 #include <sve.h>
18 #include <timers.h>
19
20 static struct ns_state g_ns_data[MAX_CPUS];
21 static uint8_t g_sve_data[MAX_CPUS][sizeof(struct sve_state)]
22 __attribute__((aligned(sizeof(__uint128_t))));
23
24 /*
25 * Initialize the aux data and any buffer pointers to the aux granule memory for
26 * use by REC when it is entered.
27 */
init_aux_data(struct rec_aux_data * aux_data,void * rec_aux,unsigned int num_rec_aux)28 static void init_aux_data(struct rec_aux_data *aux_data,
29 void *rec_aux,
30 unsigned int num_rec_aux)
31 {
32 aux_data->attest_heap_buf = (uint8_t *)rec_aux;
33
34 /* Ensure we have enough aux granules for use by REC */
35 assert(num_rec_aux >= REC_HEAP_PAGES);
36 }
37
38 /*
39 * The parent REC granules lock is expected to be acquired
40 * before functions map_rec_aux() and unmap_rec_aux() are called.
41 */
map_rec_aux(struct granule * rec_aux_pages[],unsigned long num_aux)42 static void *map_rec_aux(struct granule *rec_aux_pages[], unsigned long num_aux)
43 {
44 void *rec_aux = NULL;
45
46 for (unsigned long i = 0UL; i < num_aux; i++) {
47 void *aux = granule_map(rec_aux_pages[i], SLOT_REC_AUX0 + i);
48
49 if (i == 0UL) {
50 rec_aux = aux;
51 }
52 }
53 return rec_aux;
54 }
55
unmap_rec_aux(void * rec_aux,unsigned long num_aux)56 static void unmap_rec_aux(void *rec_aux, unsigned long num_aux)
57 {
58 unsigned char *rec_aux_vaddr = (unsigned char *)rec_aux;
59
60 for (unsigned long i = 0UL; i < num_aux; i++) {
61 buffer_unmap(rec_aux_vaddr + i * GRANULE_SIZE);
62 }
63 }
64
save_sysreg_state(struct sysreg_state * sysregs)65 static void save_sysreg_state(struct sysreg_state *sysregs)
66 {
67 sysregs->sp_el0 = read_sp_el0();
68 sysregs->sp_el1 = read_sp_el1();
69 sysregs->elr_el1 = read_elr_el12();
70 sysregs->spsr_el1 = read_spsr_el12();
71 sysregs->pmcr_el0 = read_pmcr_el0();
72 sysregs->pmuserenr_el0 = read_pmuserenr_el0();
73 sysregs->tpidrro_el0 = read_tpidrro_el0();
74 sysregs->tpidr_el0 = read_tpidr_el0();
75 sysregs->csselr_el1 = read_csselr_el1();
76 sysregs->sctlr_el1 = read_sctlr_el12();
77 sysregs->actlr_el1 = read_actlr_el1();
78 sysregs->cpacr_el1 = read_cpacr_el12();
79 sysregs->ttbr0_el1 = read_ttbr0_el12();
80 sysregs->ttbr1_el1 = read_ttbr1_el12();
81 sysregs->tcr_el1 = read_tcr_el12();
82 sysregs->esr_el1 = read_esr_el12();
83 sysregs->afsr0_el1 = read_afsr0_el12();
84 sysregs->afsr1_el1 = read_afsr1_el12();
85 sysregs->far_el1 = read_far_el12();
86 sysregs->mair_el1 = read_mair_el12();
87 sysregs->vbar_el1 = read_vbar_el12();
88
89 sysregs->contextidr_el1 = read_contextidr_el12();
90 sysregs->tpidr_el1 = read_tpidr_el1();
91 sysregs->amair_el1 = read_amair_el12();
92 sysregs->cntkctl_el1 = read_cntkctl_el12();
93 sysregs->par_el1 = read_par_el1();
94 sysregs->mdscr_el1 = read_mdscr_el1();
95 sysregs->mdccint_el1 = read_mdccint_el1();
96 sysregs->disr_el1 = read_disr_el1();
97 MPAM(sysregs->mpam0_el1 = read_mpam0_el1();)
98
99 /* Timer registers */
100 sysregs->cntpoff_el2 = read_cntpoff_el2();
101 sysregs->cntvoff_el2 = read_cntvoff_el2();
102 sysregs->cntp_ctl_el0 = read_cntp_ctl_el02();
103 sysregs->cntp_cval_el0 = read_cntp_cval_el02();
104 sysregs->cntv_ctl_el0 = read_cntv_ctl_el02();
105 sysregs->cntv_cval_el0 = read_cntv_cval_el02();
106 }
107
save_realm_state(struct rec * rec)108 static void save_realm_state(struct rec *rec)
109 {
110 save_sysreg_state(&rec->sysregs);
111
112 rec->pc = read_elr_el2();
113 rec->pstate = read_spsr_el2();
114
115 gic_save_state(&rec->sysregs.gicstate);
116 }
117
restore_sysreg_state(struct sysreg_state * sysregs)118 static void restore_sysreg_state(struct sysreg_state *sysregs)
119 {
120 write_sp_el0(sysregs->sp_el0);
121 write_sp_el1(sysregs->sp_el1);
122 write_elr_el12(sysregs->elr_el1);
123 write_spsr_el12(sysregs->spsr_el1);
124 write_pmcr_el0(sysregs->pmcr_el0);
125 write_pmuserenr_el0(sysregs->pmuserenr_el0);
126 write_tpidrro_el0(sysregs->tpidrro_el0);
127 write_tpidr_el0(sysregs->tpidr_el0);
128 write_csselr_el1(sysregs->csselr_el1);
129 write_sctlr_el12(sysregs->sctlr_el1);
130 write_actlr_el1(sysregs->actlr_el1);
131 write_cpacr_el12(sysregs->cpacr_el1);
132 write_ttbr0_el12(sysregs->ttbr0_el1);
133 write_ttbr1_el12(sysregs->ttbr1_el1);
134 write_tcr_el12(sysregs->tcr_el1);
135 write_esr_el12(sysregs->esr_el1);
136 write_afsr0_el12(sysregs->afsr0_el1);
137 write_afsr1_el12(sysregs->afsr1_el1);
138 write_far_el12(sysregs->far_el1);
139 write_mair_el12(sysregs->mair_el1);
140 write_vbar_el12(sysregs->vbar_el1);
141
142 write_contextidr_el12(sysregs->contextidr_el1);
143 write_tpidr_el1(sysregs->tpidr_el1);
144 write_amair_el12(sysregs->amair_el1);
145 write_cntkctl_el12(sysregs->cntkctl_el1);
146 write_par_el1(sysregs->par_el1);
147 write_mdscr_el1(sysregs->mdscr_el1);
148 write_mdccint_el1(sysregs->mdccint_el1);
149 write_disr_el1(sysregs->disr_el1);
150 MPAM(write_mpam0_el1(sysregs->mpam0_el1);)
151 write_vmpidr_el2(sysregs->vmpidr_el2);
152
153 /* Timer registers */
154 write_cntpoff_el2(sysregs->cntpoff_el2);
155 write_cntvoff_el2(sysregs->cntvoff_el2);
156
157 /*
158 * Restore CNTx_CVAL registers before CNTx_CTL to avoid
159 * raising the interrupt signal briefly before lowering
160 * it again due to some expired CVAL left in the timer
161 * register.
162 */
163 write_cntp_cval_el02(sysregs->cntp_cval_el0);
164 write_cntp_ctl_el02(sysregs->cntp_ctl_el0);
165 write_cntv_cval_el02(sysregs->cntv_cval_el0);
166 write_cntv_ctl_el02(sysregs->cntv_ctl_el0);
167 }
168
restore_realm_state(struct rec * rec)169 static void restore_realm_state(struct rec *rec)
170 {
171 /*
172 * Restore this early to give time to the timer mask to propagate to
173 * the GIC. Issue an ISB to ensure the register write is actually
174 * performed before doing the remaining work.
175 */
176 write_cnthctl_el2(rec->sysregs.cnthctl_el2);
177 isb();
178
179 restore_sysreg_state(&rec->sysregs);
180 write_elr_el2(rec->pc);
181 write_spsr_el2(rec->pstate);
182 write_hcr_el2(rec->sysregs.hcr_el2);
183
184 gic_restore_state(&rec->sysregs.gicstate);
185 }
186
configure_realm_stage2(struct rec * rec)187 static void configure_realm_stage2(struct rec *rec)
188 {
189 write_vtcr_el2(rec->common_sysregs.vtcr_el2);
190 write_vttbr_el2(rec->common_sysregs.vttbr_el2);
191 }
192
save_ns_state(struct ns_state * ns_state)193 static void save_ns_state(struct ns_state *ns_state)
194 {
195 save_sysreg_state(&ns_state->sysregs);
196
197 /*
198 * CNTHCTL_EL2 is saved/restored separately from the main system
199 * registers, because the Realm configuration is written on every
200 * entry to the Realm, see `check_pending_timers`.
201 */
202 ns_state->sysregs.cnthctl_el2 = read_cnthctl_el2();
203
204 ns_state->icc_sre_el2 = read_icc_sre_el2();
205 }
206
restore_ns_state(struct ns_state * ns_state)207 static void restore_ns_state(struct ns_state *ns_state)
208 {
209 restore_sysreg_state(&ns_state->sysregs);
210
211 /*
212 * CNTHCTL_EL2 is saved/restored separately from the main system
213 * registers, because the Realm configuration is written on every
214 * entry to the Realm, see `check_pending_timers`.
215 */
216 write_cnthctl_el2(ns_state->sysregs.cnthctl_el2);
217
218 write_icc_sre_el2(ns_state->icc_sre_el2);
219 }
220
activate_events(struct rec * rec)221 static void activate_events(struct rec *rec)
222 {
223 /*
224 * The only event that may be activated at the Realm is the SError.
225 */
226 if (rec->serror_info.inject) {
227 write_vsesr_el2(rec->serror_info.vsesr_el2);
228 write_hcr_el2(rec->sysregs.hcr_el2 | HCR_VSE);
229 rec->serror_info.inject = false;
230 }
231 }
232
inject_serror(struct rec * rec,unsigned long vsesr)233 void inject_serror(struct rec *rec, unsigned long vsesr)
234 {
235 rec->serror_info.vsesr_el2 = vsesr;
236 rec->serror_info.inject = true;
237 }
238
rec_run_loop(struct rec * rec,struct rmi_rec_exit * rec_exit)239 void rec_run_loop(struct rec *rec, struct rmi_rec_exit *rec_exit)
240 {
241 struct ns_state *ns_state;
242 int realm_exception_code;
243 void *rec_aux;
244 unsigned int cpuid = my_cpuid();
245
246 assert(rec->ns == NULL);
247
248 assert(cpuid < MAX_CPUS);
249 ns_state = &g_ns_data[cpuid];
250
251 /* ensure SVE/FPU context is cleared */
252 assert(ns_state->sve == NULL);
253 assert(ns_state->fpu == NULL);
254
255 /* Map auxiliary granules */
256 rec_aux = map_rec_aux(rec->g_aux, rec->num_rec_aux);
257
258 init_aux_data(&(rec->aux_data), rec_aux, rec->num_rec_aux);
259
260 /*
261 * The attset heap on the REC aux pages is mapped now. It is time to
262 * associate it with the current CPU.
263 * This heap will be used for attestation RSI calls when the
264 * REC is running.
265 */
266 attestation_heap_ctx_assign_pe(&rec->alloc_info.ctx);
267
268 /*
269 * Initialise the heap for attestation if necessary.
270 */
271 if (!rec->alloc_info.ctx_initialised) {
272 (void)attestation_heap_ctx_init(rec->aux_data.attest_heap_buf,
273 REC_HEAP_PAGES * SZ_4K);
274 rec->alloc_info.ctx_initialised = true;
275 }
276
277 if (is_feat_sve_present()) {
278 ns_state->sve = (struct sve_state *)&g_sve_data[cpuid];
279 } else {
280 ns_state->fpu = (struct fpu_state *)&g_sve_data[cpuid];
281 }
282
283 save_ns_state(ns_state);
284 restore_realm_state(rec);
285
286 /* Prepare for lazy save/restore of FPU/SIMD registers. */
287 rec->ns = ns_state;
288 assert(rec->fpu_ctx.used == false);
289
290 configure_realm_stage2(rec);
291
292 do {
293 /*
294 * We must check the status of the arch timers in every
295 * iteration of the loop to ensure we update the timer
296 * mask on each entry to the realm and that we report any
297 * change in output level to the NS caller.
298 */
299 if (check_pending_timers(rec)) {
300 rec_exit->exit_reason = RMI_EXIT_IRQ;
301 break;
302 }
303
304 activate_events(rec);
305 realm_exception_code = run_realm(&rec->regs[0]);
306 } while (handle_realm_exit(rec, rec_exit, realm_exception_code));
307
308 /*
309 * Check if FPU/SIMD was used, and if it was, save the realm state,
310 * restore the NS state, and reenable traps in CPTR_EL2.
311 */
312 if (rec->fpu_ctx.used) {
313 unsigned long cptr;
314
315 cptr = read_cptr_el2();
316 cptr &= ~(CPTR_EL2_ZEN_MASK << CPTR_EL2_ZEN_SHIFT);
317 cptr |= (CPTR_EL2_ZEN_NO_TRAP_11 << CPTR_EL2_ZEN_SHIFT);
318 write_cptr_el2(cptr);
319
320 fpu_save_state(&rec->fpu_ctx.fpu);
321 if (ns_state->sve != NULL) {
322 restore_sve_state(ns_state->sve);
323 } else {
324 assert(ns_state->fpu != NULL);
325 fpu_restore_state(ns_state->fpu);
326 }
327
328 cptr = read_cptr_el2();
329 cptr &= ~(CPTR_EL2_FPEN_MASK << CPTR_EL2_FPEN_SHIFT);
330 cptr |= (CPTR_EL2_FPEN_TRAP_ALL_00 << CPTR_EL2_FPEN_SHIFT);
331 cptr &= ~(CPTR_EL2_ZEN_MASK << CPTR_EL2_ZEN_SHIFT);
332 cptr |= (CPTR_EL2_ZEN_TRAP_ALL_00 << CPTR_EL2_ZEN_SHIFT);
333 write_cptr_el2(cptr);
334 rec->fpu_ctx.used = false;
335 }
336
337 /*
338 * Clear FPU/SVE context while exiting
339 */
340 ns_state->sve = NULL;
341 ns_state->fpu = NULL;
342
343 /*
344 * Clear NS pointer since that struct is local to this function.
345 */
346 rec->ns = NULL;
347
348 report_timer_state_to_ns(rec_exit);
349
350 save_realm_state(rec);
351 restore_ns_state(ns_state);
352
353 /* Undo the heap association */
354 attestation_heap_ctx_unassign_pe(&rec->alloc_info.ctx);
355 /* Unmap auxiliary granules */
356 unmap_rec_aux(rec_aux, rec->num_rec_aux);
357 }
358