1 /*
2 * Copyright (c) 2013, Google Inc. All rights reserved.
3 *
4 * Use of this source code is governed by a MIT-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/MIT
7 */
8 #include <dev/timer/arm_generic.h>
9
10 #include <arch/ops.h>
11 #include <assert.h>
12 #include <lk/init.h>
13 #include <platform.h>
14 #include <platform/interrupts.h>
15 #include <platform/timer.h>
16 #include <lk/trace.h>
17
18 #define LOCAL_TRACE 0
19
20 #include <lib/fixed_point.h>
21
22 #if ARCH_ARM64
23
24 /* CNTFRQ AArch64 register */
25 #define TIMER_REG_CNTFRQ cntfrq_el0
26
27 /* CNTP AArch64 registers */
28 #define TIMER_REG_CNTP_CTL cntp_ctl_el0
29 #define TIMER_REG_CNTP_CVAL cntp_cval_el0
30 #define TIMER_REG_CNTP_TVAL cntp_tval_el0
31 #define TIMER_REG_CNTPCT cntpct_el0
32
33 /* CNTPS AArch64 registers */
34 #define TIMER_REG_CNTPS_CTL cntps_ctl_el1
35 #define TIMER_REG_CNTPS_CVAL cntps_cval_el1
36 #define TIMER_REG_CNTPS_TVAL cntps_tval_el1
37 #define TIMER_REG_CNTPSCT cntpct_el0
38
39 /* CNTV AArch64 registers */
40 #define TIMER_REG_CNTV_CTL cntv_ctl_el0
41 #define TIMER_REG_CNTV_CVAL cntv_cval_el0
42 #define TIMER_REG_CNTV_TVAL cntv_tval_el0
43 #define TIMER_REG_CNTVCT cntvct_el0
44
45 #define READ_TIMER_REG32(reg) ARM64_READ_SYSREG(reg)
46 #define READ_TIMER_REG64(reg) ARM64_READ_SYSREG(reg)
47 #define WRITE_TIMER_REG32(reg, val) ARM64_WRITE_SYSREG(reg, val)
48 #define WRITE_TIMER_REG64(reg, val) ARM64_WRITE_SYSREG(reg, val)
49
50 #else
51
52 /* CNTFRQ AArch32 register */
53 #define TIMER_REG_CNTFRQ "c0, 0"
54
55 /* CNTP AArch32 registers */
56 #define TIMER_REG_CNTP_CTL "c2, 1"
57 #define TIMER_REG_CNTP_CVAL "2"
58 #define TIMER_REG_CNTP_TVAL "c2, 0"
59 #define TIMER_REG_CNTPCT "0"
60
61 /* CNTPS AArch32 registers are banked and accessed though CNTP */
62 #define CNTPS CNTP
63
64 /* CNTV AArch32 registers */
65 #define TIMER_REG_CNTV_CTL "c3, 1"
66 #define TIMER_REG_CNTV_CVAL "3"
67 #define TIMER_REG_CNTV_TVAL "c3, 0"
68 #define TIMER_REG_CNTVCT "1"
69
70 #define READ_TIMER_REG32(reg) \
71 ({ \
72 uint32_t _val; \
73 __asm__ volatile("mrc p15, 0, %0, c14, " reg : "=r" (_val)); \
74 _val; \
75 })
76
77 #define READ_TIMER_REG64(reg) \
78 ({ \
79 uint64_t _val; \
80 __asm__ volatile("mrrc p15, " reg ", %0, %H0, c14" : "=r" (_val)); \
81 _val; \
82 })
83
84 #define WRITE_TIMER_REG32(reg, val) \
85 ({ \
86 __asm__ volatile("mcr p15, 0, %0, c14, " reg :: "r" (val)); \
87 ISB; \
88 })
89
90 #define WRITE_TIMER_REG64(reg, val) \
91 ({ \
92 __asm__ volatile("mcrr p15, " reg ", %0, %H0, c14" :: "r" (val)); \
93 ISB; \
94 })
95
96 #endif
97
98 #ifndef TIMER_ARM_GENERIC_SELECTED
99 #define TIMER_ARM_GENERIC_SELECTED CNTP
100 #endif
101
102 #define COMBINE3(a,b,c) a ## b ## c
103 #define XCOMBINE3(a,b,c) COMBINE3(a, b, c)
104
105 #define SELECTED_TIMER_REG(reg) XCOMBINE3(TIMER_REG_, TIMER_ARM_GENERIC_SELECTED, reg)
106 #define TIMER_REG_CTL SELECTED_TIMER_REG(_CTL)
107 #define TIMER_REG_CVAL SELECTED_TIMER_REG(_CVAL)
108 #define TIMER_REG_TVAL SELECTED_TIMER_REG(_TVAL)
109 #define TIMER_REG_CT SELECTED_TIMER_REG(CT)
110
111
112 static platform_timer_callback t_callback;
113 static int timer_irq;
114
115 struct fp_32_64 cntpct_per_ms;
116 struct fp_32_64 ms_per_cntpct;
117 struct fp_32_64 us_per_cntpct;
118
lk_time_to_cntpct(lk_time_t lk_time)119 static uint64_t lk_time_to_cntpct(lk_time_t lk_time) {
120 return u64_mul_u32_fp32_64(lk_time, cntpct_per_ms);
121 }
122
cntpct_to_lk_time(uint64_t cntpct)123 static lk_time_t cntpct_to_lk_time(uint64_t cntpct) {
124 return u32_mul_u64_fp32_64(cntpct, ms_per_cntpct);
125 }
126
cntpct_to_lk_bigtime(uint64_t cntpct)127 static lk_bigtime_t cntpct_to_lk_bigtime(uint64_t cntpct) {
128 return u64_mul_u64_fp32_64(cntpct, us_per_cntpct);
129 }
130
read_cntfrq(void)131 static uint32_t read_cntfrq(void) {
132 uint32_t cntfrq;
133
134 cntfrq = READ_TIMER_REG32(TIMER_REG_CNTFRQ);
135 LTRACEF("cntfrq: 0x%08x, %u\n", cntfrq, cntfrq);
136 return cntfrq;
137 }
138
read_cntp_ctl(void)139 static uint32_t read_cntp_ctl(void) {
140 uint32_t cntp_ctl;
141
142 cntp_ctl = READ_TIMER_REG32(TIMER_REG_CTL);
143 return cntp_ctl;
144 }
145
write_cntp_ctl(uint32_t cntp_ctl)146 static void write_cntp_ctl(uint32_t cntp_ctl) {
147 LTRACEF_LEVEL(3, "cntp_ctl: 0x%x %x\n", cntp_ctl, read_cntp_ctl());
148 WRITE_TIMER_REG32(TIMER_REG_CTL, cntp_ctl);
149 }
150
write_cntp_cval(uint64_t cntp_cval)151 static void write_cntp_cval(uint64_t cntp_cval) {
152 LTRACEF_LEVEL(3, "cntp_cval: 0x%016llx, %llu\n", cntp_cval, cntp_cval);
153 WRITE_TIMER_REG64(TIMER_REG_CVAL, cntp_cval);
154 }
155
write_cntp_tval(int32_t cntp_tval)156 static void write_cntp_tval(int32_t cntp_tval) {
157 LTRACEF_LEVEL(3, "cntp_tval: 0x%08x, %d\n", cntp_tval, cntp_tval);
158 WRITE_TIMER_REG32(TIMER_REG_TVAL, cntp_tval);
159 }
160
read_cntpct(void)161 static uint64_t read_cntpct(void) {
162 uint64_t cntpct;
163
164 cntpct = READ_TIMER_REG64(TIMER_REG_CT);
165 LTRACEF_LEVEL(3, "cntpct: 0x%016llx, %llu\n", cntpct, cntpct);
166 return cntpct;
167 }
168
platform_tick(void * arg)169 static enum handler_return platform_tick(void *arg) {
170 write_cntp_ctl(0);
171 if (t_callback) {
172 return t_callback(arg, current_time());
173 } else {
174 return INT_NO_RESCHEDULE;
175 }
176 }
177
platform_set_oneshot_timer(platform_timer_callback callback,void * arg,lk_time_t interval)178 status_t platform_set_oneshot_timer(platform_timer_callback callback, void *arg, lk_time_t interval) {
179 uint64_t cntpct_interval = lk_time_to_cntpct(interval);
180
181 ASSERT(arg == NULL);
182
183 t_callback = callback;
184 if (cntpct_interval <= INT_MAX)
185 write_cntp_tval(cntpct_interval);
186 else
187 write_cntp_cval(read_cntpct() + cntpct_interval);
188 write_cntp_ctl(1);
189
190 return 0;
191 }
192
platform_stop_timer(void)193 void platform_stop_timer(void) {
194 write_cntp_ctl(0);
195 }
196
current_time_hires(void)197 lk_bigtime_t current_time_hires(void) {
198 return cntpct_to_lk_bigtime(read_cntpct());
199 }
200
current_time(void)201 lk_time_t current_time(void) {
202 return cntpct_to_lk_time(read_cntpct());
203 }
204
abs_int32(int32_t a)205 static uint32_t abs_int32(int32_t a) {
206 return (a > 0) ? a : -a;
207 }
208
abs_int64(int64_t a)209 static uint64_t abs_int64(int64_t a) {
210 return (a > 0) ? a : -a;
211 }
212
test_time_conversion_check_result(uint64_t a,uint64_t b,uint64_t limit,bool is32)213 static void test_time_conversion_check_result(uint64_t a, uint64_t b, uint64_t limit, bool is32) {
214 if (a != b) {
215 uint64_t diff = is32 ? abs_int32(a - b) : abs_int64(a - b);
216 if (diff <= limit)
217 LTRACEF("ROUNDED by %llu (up to %llu allowed)\n", diff, limit);
218 else
219 TRACEF("FAIL, off by %llu\n", diff);
220 }
221 }
222
test_lk_time_to_cntpct(uint32_t cntfrq,lk_time_t lk_time)223 static void test_lk_time_to_cntpct(uint32_t cntfrq, lk_time_t lk_time) {
224 uint64_t cntpct = lk_time_to_cntpct(lk_time);
225 uint64_t expected_cntpct = ((uint64_t)cntfrq * lk_time + 500) / 1000;
226
227 test_time_conversion_check_result(cntpct, expected_cntpct, 1, false);
228 LTRACEF_LEVEL(2, "lk_time_to_cntpct(%u): got %llu, expect %llu\n", lk_time, cntpct, expected_cntpct);
229 }
230
test_cntpct_to_lk_time(uint32_t cntfrq,lk_time_t expected_lk_time,uint32_t wrap_count)231 static void test_cntpct_to_lk_time(uint32_t cntfrq, lk_time_t expected_lk_time, uint32_t wrap_count) {
232 lk_time_t lk_time;
233 uint64_t cntpct;
234
235 cntpct = (uint64_t)cntfrq * expected_lk_time / 1000;
236 if ((uint64_t)cntfrq * wrap_count > UINT_MAX)
237 cntpct += (((uint64_t)cntfrq << 32) / 1000) * wrap_count;
238 else
239 cntpct += (((uint64_t)(cntfrq * wrap_count) << 32) / 1000);
240 lk_time = cntpct_to_lk_time(cntpct);
241
242 test_time_conversion_check_result(lk_time, expected_lk_time, (1000 + cntfrq - 1) / cntfrq, true);
243 LTRACEF_LEVEL(2, "cntpct_to_lk_time(%llu): got %u, expect %u\n", cntpct, lk_time, expected_lk_time);
244 }
245
test_cntpct_to_lk_bigtime(uint32_t cntfrq,uint64_t expected_s)246 static void test_cntpct_to_lk_bigtime(uint32_t cntfrq, uint64_t expected_s) {
247 lk_bigtime_t expected_lk_bigtime = expected_s * 1000 * 1000;
248 uint64_t cntpct = (uint64_t)cntfrq * expected_s;
249 lk_bigtime_t lk_bigtime = cntpct_to_lk_bigtime(cntpct);
250
251 test_time_conversion_check_result(lk_bigtime, expected_lk_bigtime, (1000 * 1000 + cntfrq - 1) / cntfrq, false);
252 LTRACEF_LEVEL(2, "cntpct_to_lk_bigtime(%llu): got %llu, expect %llu\n", cntpct, lk_bigtime, expected_lk_bigtime);
253 }
254
test_time_conversions(uint32_t cntfrq)255 static void test_time_conversions(uint32_t cntfrq) {
256 test_lk_time_to_cntpct(cntfrq, 0);
257 test_lk_time_to_cntpct(cntfrq, 1);
258 test_lk_time_to_cntpct(cntfrq, INT_MAX);
259 test_lk_time_to_cntpct(cntfrq, INT_MAX + 1U);
260 test_lk_time_to_cntpct(cntfrq, ~0);
261 test_cntpct_to_lk_time(cntfrq, 0, 0);
262 test_cntpct_to_lk_time(cntfrq, INT_MAX, 0);
263 test_cntpct_to_lk_time(cntfrq, INT_MAX + 1U, 0);
264 test_cntpct_to_lk_time(cntfrq, ~0, 0);
265 test_cntpct_to_lk_time(cntfrq, 0, 1);
266 test_cntpct_to_lk_time(cntfrq, 0, 7);
267 test_cntpct_to_lk_time(cntfrq, 0, 70);
268 test_cntpct_to_lk_time(cntfrq, 0, 700);
269 test_cntpct_to_lk_bigtime(cntfrq, 0);
270 test_cntpct_to_lk_bigtime(cntfrq, 1);
271 test_cntpct_to_lk_bigtime(cntfrq, 60 * 60 * 24);
272 test_cntpct_to_lk_bigtime(cntfrq, 60 * 60 * 24 * 365);
273 test_cntpct_to_lk_bigtime(cntfrq, 60 * 60 * 24 * (365 * 10 + 2));
274 test_cntpct_to_lk_bigtime(cntfrq, 60ULL * 60 * 24 * (365 * 100 + 2));
275 }
276
arm_generic_timer_init_conversion_factors(uint32_t cntfrq)277 static void arm_generic_timer_init_conversion_factors(uint32_t cntfrq) {
278 fp_32_64_div_32_32(&cntpct_per_ms, cntfrq, 1000);
279 fp_32_64_div_32_32(&ms_per_cntpct, 1000, cntfrq);
280 fp_32_64_div_32_32(&us_per_cntpct, 1000 * 1000, cntfrq);
281 LTRACEF("cntpct_per_ms: %08x.%08x%08x\n", cntpct_per_ms.l0, cntpct_per_ms.l32, cntpct_per_ms.l64);
282 LTRACEF("ms_per_cntpct: %08x.%08x%08x\n", ms_per_cntpct.l0, ms_per_cntpct.l32, ms_per_cntpct.l64);
283 LTRACEF("us_per_cntpct: %08x.%08x%08x\n", us_per_cntpct.l0, us_per_cntpct.l32, us_per_cntpct.l64);
284 }
285
arm_generic_timer_init(int irq,uint32_t freq_override)286 void arm_generic_timer_init(int irq, uint32_t freq_override) {
287 uint32_t cntfrq;
288
289 if (freq_override == 0) {
290 cntfrq = read_cntfrq();
291
292 if (!cntfrq) {
293 TRACEF("Failed to initialize timer, frequency is 0\n");
294 return;
295 }
296 } else {
297 cntfrq = freq_override;
298 }
299
300 #if LOCAL_TRACE
301 LTRACEF("Test min cntfrq\n");
302 arm_generic_timer_init_conversion_factors(1);
303 test_time_conversions(1);
304 LTRACEF("Test max cntfrq\n");
305 arm_generic_timer_init_conversion_factors(~0);
306 test_time_conversions(~0);
307 LTRACEF("Set actual cntfrq\n");
308 #endif
309 arm_generic_timer_init_conversion_factors(cntfrq);
310 test_time_conversions(cntfrq);
311
312 LTRACEF("register irq %d on cpu %d\n", irq, arch_curr_cpu_num());
313 register_int_handler(irq, &platform_tick, NULL);
314 unmask_interrupt(irq);
315
316 timer_irq = irq;
317 }
318
arm_generic_timer_init_secondary_cpu(uint level)319 static void arm_generic_timer_init_secondary_cpu(uint level) {
320 LTRACEF("register irq %d on cpu %d\n", timer_irq, arch_curr_cpu_num());
321 register_int_handler(timer_irq, &platform_tick, NULL);
322 unmask_interrupt(timer_irq);
323 }
324
325 /* secondary cpu initialize the timer just before the kernel starts with interrupts enabled */
326 LK_INIT_HOOK_FLAGS(arm_generic_timer_init_secondary_cpu,
327 arm_generic_timer_init_secondary_cpu,
328 LK_INIT_LEVEL_THREADING - 1, LK_INIT_FLAG_SECONDARY_CPUS);
329
arm_generic_timer_resume_cpu(uint level)330 static void arm_generic_timer_resume_cpu(uint level) {
331 /* Always trigger a timer interrupt on each cpu for now */
332 write_cntp_tval(0);
333 write_cntp_ctl(1);
334 }
335
336 LK_INIT_HOOK_FLAGS(arm_generic_timer_resume_cpu, arm_generic_timer_resume_cpu,
337 LK_INIT_LEVEL_PLATFORM, LK_INIT_FLAG_CPU_RESUME);
338