1 // SPDX-License-Identifier: GPL-2.0-only OR MIT
2 /*
3 * Apple RTKit IPC library
4 * Copyright (C) The Asahi Linux Contributors
5 */
6
7 #include "rtkit-internal.h"
8
9 enum {
10 APPLE_RTKIT_PWR_STATE_OFF = 0x00, /* power off, cannot be restarted */
11 APPLE_RTKIT_PWR_STATE_SLEEP = 0x01, /* sleeping, can be restarted */
12 APPLE_RTKIT_PWR_STATE_IDLE = 0x201, /* sleeping, retain state */
13 APPLE_RTKIT_PWR_STATE_QUIESCED = 0x10, /* running but no communication */
14 APPLE_RTKIT_PWR_STATE_ON = 0x20, /* normal operating state */
15 };
16
17 enum {
18 APPLE_RTKIT_EP_MGMT = 0,
19 APPLE_RTKIT_EP_CRASHLOG = 1,
20 APPLE_RTKIT_EP_SYSLOG = 2,
21 APPLE_RTKIT_EP_DEBUG = 3,
22 APPLE_RTKIT_EP_IOREPORT = 4,
23 APPLE_RTKIT_EP_OSLOG = 8,
24 };
25
26 #define APPLE_RTKIT_MGMT_TYPE GENMASK_ULL(59, 52)
27
28 enum {
29 APPLE_RTKIT_MGMT_HELLO = 1,
30 APPLE_RTKIT_MGMT_HELLO_REPLY = 2,
31 APPLE_RTKIT_MGMT_STARTEP = 5,
32 APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE = 6,
33 APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE_ACK = 7,
34 APPLE_RTKIT_MGMT_EPMAP = 8,
35 APPLE_RTKIT_MGMT_EPMAP_REPLY = 8,
36 APPLE_RTKIT_MGMT_SET_AP_PWR_STATE = 0xb,
37 APPLE_RTKIT_MGMT_SET_AP_PWR_STATE_ACK = 0xb,
38 };
39
40 #define APPLE_RTKIT_MGMT_HELLO_MINVER GENMASK_ULL(15, 0)
41 #define APPLE_RTKIT_MGMT_HELLO_MAXVER GENMASK_ULL(31, 16)
42
43 #define APPLE_RTKIT_MGMT_EPMAP_LAST BIT_ULL(51)
44 #define APPLE_RTKIT_MGMT_EPMAP_BASE GENMASK_ULL(34, 32)
45 #define APPLE_RTKIT_MGMT_EPMAP_BITMAP GENMASK_ULL(31, 0)
46
47 #define APPLE_RTKIT_MGMT_EPMAP_REPLY_MORE BIT_ULL(0)
48
49 #define APPLE_RTKIT_MGMT_STARTEP_EP GENMASK_ULL(39, 32)
50 #define APPLE_RTKIT_MGMT_STARTEP_FLAG BIT_ULL(1)
51
52 #define APPLE_RTKIT_MGMT_PWR_STATE GENMASK_ULL(15, 0)
53
54 #define APPLE_RTKIT_CRASHLOG_CRASH 1
55
56 #define APPLE_RTKIT_BUFFER_REQUEST 1
57 #define APPLE_RTKIT_BUFFER_REQUEST_SIZE GENMASK_ULL(51, 44)
58 #define APPLE_RTKIT_BUFFER_REQUEST_IOVA GENMASK_ULL(41, 0)
59
60 #define APPLE_RTKIT_SYSLOG_TYPE GENMASK_ULL(59, 52)
61
62 #define APPLE_RTKIT_SYSLOG_LOG 5
63
64 #define APPLE_RTKIT_SYSLOG_INIT 8
65 #define APPLE_RTKIT_SYSLOG_N_ENTRIES GENMASK_ULL(7, 0)
66 #define APPLE_RTKIT_SYSLOG_MSG_SIZE GENMASK_ULL(31, 24)
67
68 #define APPLE_RTKIT_OSLOG_TYPE GENMASK_ULL(63, 56)
69 #define APPLE_RTKIT_OSLOG_INIT 1
70 #define APPLE_RTKIT_OSLOG_ACK 3
71
72 #define APPLE_RTKIT_MIN_SUPPORTED_VERSION 11
73 #define APPLE_RTKIT_MAX_SUPPORTED_VERSION 12
74
75 struct apple_rtkit_msg {
76 struct completion *completion;
77 struct apple_mbox_msg mbox_msg;
78 };
79
80 struct apple_rtkit_rx_work {
81 struct apple_rtkit *rtk;
82 u8 ep;
83 u64 msg;
84 struct work_struct work;
85 };
86
apple_rtkit_is_running(struct apple_rtkit * rtk)87 bool apple_rtkit_is_running(struct apple_rtkit *rtk)
88 {
89 if (rtk->crashed)
90 return false;
91 if ((rtk->iop_power_state & 0xff) != APPLE_RTKIT_PWR_STATE_ON)
92 return false;
93 if ((rtk->ap_power_state & 0xff) != APPLE_RTKIT_PWR_STATE_ON)
94 return false;
95 return true;
96 }
97 EXPORT_SYMBOL_GPL(apple_rtkit_is_running);
98
apple_rtkit_is_crashed(struct apple_rtkit * rtk)99 bool apple_rtkit_is_crashed(struct apple_rtkit *rtk)
100 {
101 return rtk->crashed;
102 }
103 EXPORT_SYMBOL_GPL(apple_rtkit_is_crashed);
104
apple_rtkit_management_send(struct apple_rtkit * rtk,u8 type,u64 msg)105 static void apple_rtkit_management_send(struct apple_rtkit *rtk, u8 type,
106 u64 msg)
107 {
108 msg &= ~APPLE_RTKIT_MGMT_TYPE;
109 msg |= FIELD_PREP(APPLE_RTKIT_MGMT_TYPE, type);
110 apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_MGMT, msg, NULL, false);
111 }
112
apple_rtkit_management_rx_hello(struct apple_rtkit * rtk,u64 msg)113 static void apple_rtkit_management_rx_hello(struct apple_rtkit *rtk, u64 msg)
114 {
115 u64 reply;
116
117 int min_ver = FIELD_GET(APPLE_RTKIT_MGMT_HELLO_MINVER, msg);
118 int max_ver = FIELD_GET(APPLE_RTKIT_MGMT_HELLO_MAXVER, msg);
119 int want_ver = min(APPLE_RTKIT_MAX_SUPPORTED_VERSION, max_ver);
120
121 dev_dbg(rtk->dev, "RTKit: Min ver %d, max ver %d\n", min_ver, max_ver);
122
123 if (min_ver > APPLE_RTKIT_MAX_SUPPORTED_VERSION) {
124 dev_err(rtk->dev, "RTKit: Firmware min version %d is too new\n",
125 min_ver);
126 goto abort_boot;
127 }
128
129 if (max_ver < APPLE_RTKIT_MIN_SUPPORTED_VERSION) {
130 dev_err(rtk->dev, "RTKit: Firmware max version %d is too old\n",
131 max_ver);
132 goto abort_boot;
133 }
134
135 dev_info(rtk->dev, "RTKit: Initializing (protocol version %d)\n",
136 want_ver);
137 rtk->version = want_ver;
138
139 reply = FIELD_PREP(APPLE_RTKIT_MGMT_HELLO_MINVER, want_ver);
140 reply |= FIELD_PREP(APPLE_RTKIT_MGMT_HELLO_MAXVER, want_ver);
141 apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_HELLO_REPLY, reply);
142
143 return;
144
145 abort_boot:
146 rtk->boot_result = -EINVAL;
147 complete_all(&rtk->epmap_completion);
148 }
149
apple_rtkit_management_rx_epmap(struct apple_rtkit * rtk,u64 msg)150 static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg)
151 {
152 int i, ep;
153 u64 reply;
154 unsigned long bitmap = FIELD_GET(APPLE_RTKIT_MGMT_EPMAP_BITMAP, msg);
155 u32 base = FIELD_GET(APPLE_RTKIT_MGMT_EPMAP_BASE, msg);
156
157 dev_dbg(rtk->dev,
158 "RTKit: received endpoint bitmap 0x%lx with base 0x%x\n",
159 bitmap, base);
160
161 for_each_set_bit(i, &bitmap, 32) {
162 ep = 32 * base + i;
163 dev_dbg(rtk->dev, "RTKit: Discovered endpoint 0x%02x\n", ep);
164 set_bit(ep, rtk->endpoints);
165 }
166
167 reply = FIELD_PREP(APPLE_RTKIT_MGMT_EPMAP_BASE, base);
168 if (msg & APPLE_RTKIT_MGMT_EPMAP_LAST)
169 reply |= APPLE_RTKIT_MGMT_EPMAP_LAST;
170 else
171 reply |= APPLE_RTKIT_MGMT_EPMAP_REPLY_MORE;
172
173 apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_EPMAP_REPLY, reply);
174
175 if (!(msg & APPLE_RTKIT_MGMT_EPMAP_LAST))
176 return;
177
178 for_each_set_bit(ep, rtk->endpoints, APPLE_RTKIT_APP_ENDPOINT_START) {
179 switch (ep) {
180 /* the management endpoint is started by default */
181 case APPLE_RTKIT_EP_MGMT:
182 break;
183
184 /* without starting these RTKit refuses to boot */
185 case APPLE_RTKIT_EP_SYSLOG:
186 case APPLE_RTKIT_EP_CRASHLOG:
187 case APPLE_RTKIT_EP_DEBUG:
188 case APPLE_RTKIT_EP_IOREPORT:
189 case APPLE_RTKIT_EP_OSLOG:
190 dev_dbg(rtk->dev,
191 "RTKit: Starting system endpoint 0x%02x\n", ep);
192 apple_rtkit_start_ep(rtk, ep);
193 break;
194
195 default:
196 dev_warn(rtk->dev,
197 "RTKit: Unknown system endpoint: 0x%02x\n",
198 ep);
199 }
200 }
201
202 rtk->boot_result = 0;
203 complete_all(&rtk->epmap_completion);
204 }
205
apple_rtkit_management_rx_iop_pwr_ack(struct apple_rtkit * rtk,u64 msg)206 static void apple_rtkit_management_rx_iop_pwr_ack(struct apple_rtkit *rtk,
207 u64 msg)
208 {
209 unsigned int new_state = FIELD_GET(APPLE_RTKIT_MGMT_PWR_STATE, msg);
210
211 dev_dbg(rtk->dev, "RTKit: IOP power state transition: 0x%x -> 0x%x\n",
212 rtk->iop_power_state, new_state);
213 rtk->iop_power_state = new_state;
214
215 complete_all(&rtk->iop_pwr_ack_completion);
216 }
217
apple_rtkit_management_rx_ap_pwr_ack(struct apple_rtkit * rtk,u64 msg)218 static void apple_rtkit_management_rx_ap_pwr_ack(struct apple_rtkit *rtk,
219 u64 msg)
220 {
221 unsigned int new_state = FIELD_GET(APPLE_RTKIT_MGMT_PWR_STATE, msg);
222
223 dev_dbg(rtk->dev, "RTKit: AP power state transition: 0x%x -> 0x%x\n",
224 rtk->ap_power_state, new_state);
225 rtk->ap_power_state = new_state;
226
227 complete_all(&rtk->ap_pwr_ack_completion);
228 }
229
apple_rtkit_management_rx(struct apple_rtkit * rtk,u64 msg)230 static void apple_rtkit_management_rx(struct apple_rtkit *rtk, u64 msg)
231 {
232 u8 type = FIELD_GET(APPLE_RTKIT_MGMT_TYPE, msg);
233
234 switch (type) {
235 case APPLE_RTKIT_MGMT_HELLO:
236 apple_rtkit_management_rx_hello(rtk, msg);
237 break;
238 case APPLE_RTKIT_MGMT_EPMAP:
239 apple_rtkit_management_rx_epmap(rtk, msg);
240 break;
241 case APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE_ACK:
242 apple_rtkit_management_rx_iop_pwr_ack(rtk, msg);
243 break;
244 case APPLE_RTKIT_MGMT_SET_AP_PWR_STATE_ACK:
245 apple_rtkit_management_rx_ap_pwr_ack(rtk, msg);
246 break;
247 default:
248 dev_warn(
249 rtk->dev,
250 "RTKit: unknown management message: 0x%llx (type: 0x%02x)\n",
251 msg, type);
252 }
253 }
254
apple_rtkit_common_rx_get_buffer(struct apple_rtkit * rtk,struct apple_rtkit_shmem * buffer,u8 ep,u64 msg)255 static int apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk,
256 struct apple_rtkit_shmem *buffer,
257 u8 ep, u64 msg)
258 {
259 size_t n_4kpages = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_SIZE, msg);
260 u64 reply;
261 int err;
262
263 buffer->buffer = NULL;
264 buffer->iomem = NULL;
265 buffer->is_mapped = false;
266 buffer->iova = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg);
267 buffer->size = n_4kpages << 12;
268
269 dev_dbg(rtk->dev, "RTKit: buffer request for 0x%zx bytes at %pad\n",
270 buffer->size, &buffer->iova);
271
272 if (buffer->iova &&
273 (!rtk->ops->shmem_setup || !rtk->ops->shmem_destroy)) {
274 err = -EINVAL;
275 goto error;
276 }
277
278 if (rtk->ops->shmem_setup) {
279 err = rtk->ops->shmem_setup(rtk->cookie, buffer);
280 if (err)
281 goto error;
282 } else {
283 buffer->buffer = dma_alloc_coherent(rtk->dev, buffer->size,
284 &buffer->iova, GFP_KERNEL);
285 if (!buffer->buffer) {
286 err = -ENOMEM;
287 goto error;
288 }
289 }
290
291 if (!buffer->is_mapped) {
292 reply = FIELD_PREP(APPLE_RTKIT_SYSLOG_TYPE,
293 APPLE_RTKIT_BUFFER_REQUEST);
294 reply |= FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_SIZE, n_4kpages);
295 reply |= FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_IOVA,
296 buffer->iova);
297 apple_rtkit_send_message(rtk, ep, reply, NULL, false);
298 }
299
300 return 0;
301
302 error:
303 buffer->buffer = NULL;
304 buffer->iomem = NULL;
305 buffer->iova = 0;
306 buffer->size = 0;
307 buffer->is_mapped = false;
308 return err;
309 }
310
apple_rtkit_free_buffer(struct apple_rtkit * rtk,struct apple_rtkit_shmem * bfr)311 static void apple_rtkit_free_buffer(struct apple_rtkit *rtk,
312 struct apple_rtkit_shmem *bfr)
313 {
314 if (bfr->size == 0)
315 return;
316
317 if (rtk->ops->shmem_destroy)
318 rtk->ops->shmem_destroy(rtk->cookie, bfr);
319 else if (bfr->buffer)
320 dma_free_coherent(rtk->dev, bfr->size, bfr->buffer, bfr->iova);
321
322 bfr->buffer = NULL;
323 bfr->iomem = NULL;
324 bfr->iova = 0;
325 bfr->size = 0;
326 bfr->is_mapped = false;
327 }
328
apple_rtkit_memcpy(struct apple_rtkit * rtk,void * dst,struct apple_rtkit_shmem * bfr,size_t offset,size_t len)329 static void apple_rtkit_memcpy(struct apple_rtkit *rtk, void *dst,
330 struct apple_rtkit_shmem *bfr, size_t offset,
331 size_t len)
332 {
333 if (bfr->iomem)
334 memcpy_fromio(dst, bfr->iomem + offset, len);
335 else
336 memcpy(dst, bfr->buffer + offset, len);
337 }
338
apple_rtkit_crashlog_rx(struct apple_rtkit * rtk,u64 msg)339 static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg)
340 {
341 u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg);
342 u8 *bfr;
343
344 if (type != APPLE_RTKIT_CRASHLOG_CRASH) {
345 dev_warn(rtk->dev, "RTKit: Unknown crashlog message: %llx\n",
346 msg);
347 return;
348 }
349
350 if (!rtk->crashlog_buffer.size) {
351 apple_rtkit_common_rx_get_buffer(rtk, &rtk->crashlog_buffer,
352 APPLE_RTKIT_EP_CRASHLOG, msg);
353 return;
354 }
355
356 dev_err(rtk->dev, "RTKit: co-processor has crashed\n");
357
358 /*
359 * create a shadow copy here to make sure the co-processor isn't able
360 * to change the log while we're dumping it. this also ensures
361 * the buffer is in normal memory and not iomem for e.g. the SMC
362 */
363 bfr = kzalloc(rtk->crashlog_buffer.size, GFP_KERNEL);
364 if (bfr) {
365 apple_rtkit_memcpy(rtk, bfr, &rtk->crashlog_buffer, 0,
366 rtk->crashlog_buffer.size);
367 apple_rtkit_crashlog_dump(rtk, bfr, rtk->crashlog_buffer.size);
368 kfree(bfr);
369 } else {
370 dev_err(rtk->dev,
371 "RTKit: Couldn't allocate crashlog shadow buffer\n");
372 }
373
374 rtk->crashed = true;
375 if (rtk->ops->crashed)
376 rtk->ops->crashed(rtk->cookie);
377 }
378
apple_rtkit_ioreport_rx(struct apple_rtkit * rtk,u64 msg)379 static void apple_rtkit_ioreport_rx(struct apple_rtkit *rtk, u64 msg)
380 {
381 u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg);
382
383 switch (type) {
384 case APPLE_RTKIT_BUFFER_REQUEST:
385 apple_rtkit_common_rx_get_buffer(rtk, &rtk->ioreport_buffer,
386 APPLE_RTKIT_EP_IOREPORT, msg);
387 break;
388 /* unknown, must be ACKed or the co-processor will hang */
389 case 0x8:
390 case 0xc:
391 apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_IOREPORT, msg,
392 NULL, false);
393 break;
394 default:
395 dev_warn(rtk->dev, "RTKit: Unknown ioreport message: %llx\n",
396 msg);
397 }
398 }
399
apple_rtkit_syslog_rx_init(struct apple_rtkit * rtk,u64 msg)400 static void apple_rtkit_syslog_rx_init(struct apple_rtkit *rtk, u64 msg)
401 {
402 rtk->syslog_n_entries = FIELD_GET(APPLE_RTKIT_SYSLOG_N_ENTRIES, msg);
403 rtk->syslog_msg_size = FIELD_GET(APPLE_RTKIT_SYSLOG_MSG_SIZE, msg);
404
405 rtk->syslog_msg_buffer = kzalloc(rtk->syslog_msg_size, GFP_KERNEL);
406
407 dev_dbg(rtk->dev,
408 "RTKit: syslog initialized: entries: %zd, msg_size: %zd\n",
409 rtk->syslog_n_entries, rtk->syslog_msg_size);
410 }
411
apple_rtkit_syslog_rx_log(struct apple_rtkit * rtk,u64 msg)412 static void apple_rtkit_syslog_rx_log(struct apple_rtkit *rtk, u64 msg)
413 {
414 u8 idx = msg & 0xff;
415 char log_context[24];
416 size_t entry_size = 0x20 + rtk->syslog_msg_size;
417
418 if (!rtk->syslog_msg_buffer) {
419 dev_warn(
420 rtk->dev,
421 "RTKit: received syslog message but no syslog_msg_buffer\n");
422 goto done;
423 }
424 if (!rtk->syslog_buffer.size) {
425 dev_warn(
426 rtk->dev,
427 "RTKit: received syslog message but syslog_buffer.size is zero\n");
428 goto done;
429 }
430 if (!rtk->syslog_buffer.buffer && !rtk->syslog_buffer.iomem) {
431 dev_warn(
432 rtk->dev,
433 "RTKit: received syslog message but no syslog_buffer.buffer or syslog_buffer.iomem\n");
434 goto done;
435 }
436 if (idx > rtk->syslog_n_entries) {
437 dev_warn(rtk->dev, "RTKit: syslog index %d out of range\n",
438 idx);
439 goto done;
440 }
441
442 apple_rtkit_memcpy(rtk, log_context, &rtk->syslog_buffer,
443 idx * entry_size + 8, sizeof(log_context));
444 apple_rtkit_memcpy(rtk, rtk->syslog_msg_buffer, &rtk->syslog_buffer,
445 idx * entry_size + 8 + sizeof(log_context),
446 rtk->syslog_msg_size);
447
448 log_context[sizeof(log_context) - 1] = 0;
449 rtk->syslog_msg_buffer[rtk->syslog_msg_size - 1] = 0;
450 dev_info(rtk->dev, "RTKit: syslog message: %s: %s\n", log_context,
451 rtk->syslog_msg_buffer);
452
453 done:
454 apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_SYSLOG, msg, NULL, false);
455 }
456
apple_rtkit_syslog_rx(struct apple_rtkit * rtk,u64 msg)457 static void apple_rtkit_syslog_rx(struct apple_rtkit *rtk, u64 msg)
458 {
459 u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg);
460
461 switch (type) {
462 case APPLE_RTKIT_BUFFER_REQUEST:
463 apple_rtkit_common_rx_get_buffer(rtk, &rtk->syslog_buffer,
464 APPLE_RTKIT_EP_SYSLOG, msg);
465 break;
466 case APPLE_RTKIT_SYSLOG_INIT:
467 apple_rtkit_syslog_rx_init(rtk, msg);
468 break;
469 case APPLE_RTKIT_SYSLOG_LOG:
470 apple_rtkit_syslog_rx_log(rtk, msg);
471 break;
472 default:
473 dev_warn(rtk->dev, "RTKit: Unknown syslog message: %llx\n",
474 msg);
475 }
476 }
477
apple_rtkit_oslog_rx_init(struct apple_rtkit * rtk,u64 msg)478 static void apple_rtkit_oslog_rx_init(struct apple_rtkit *rtk, u64 msg)
479 {
480 u64 ack;
481
482 dev_dbg(rtk->dev, "RTKit: oslog init: msg: 0x%llx\n", msg);
483 ack = FIELD_PREP(APPLE_RTKIT_OSLOG_TYPE, APPLE_RTKIT_OSLOG_ACK);
484 apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_OSLOG, ack, NULL, false);
485 }
486
apple_rtkit_oslog_rx(struct apple_rtkit * rtk,u64 msg)487 static void apple_rtkit_oslog_rx(struct apple_rtkit *rtk, u64 msg)
488 {
489 u8 type = FIELD_GET(APPLE_RTKIT_OSLOG_TYPE, msg);
490
491 switch (type) {
492 case APPLE_RTKIT_OSLOG_INIT:
493 apple_rtkit_oslog_rx_init(rtk, msg);
494 break;
495 default:
496 dev_warn(rtk->dev, "RTKit: Unknown oslog message: %llx\n", msg);
497 }
498 }
499
apple_rtkit_rx_work(struct work_struct * work)500 static void apple_rtkit_rx_work(struct work_struct *work)
501 {
502 struct apple_rtkit_rx_work *rtk_work =
503 container_of(work, struct apple_rtkit_rx_work, work);
504 struct apple_rtkit *rtk = rtk_work->rtk;
505
506 switch (rtk_work->ep) {
507 case APPLE_RTKIT_EP_MGMT:
508 apple_rtkit_management_rx(rtk, rtk_work->msg);
509 break;
510 case APPLE_RTKIT_EP_CRASHLOG:
511 apple_rtkit_crashlog_rx(rtk, rtk_work->msg);
512 break;
513 case APPLE_RTKIT_EP_SYSLOG:
514 apple_rtkit_syslog_rx(rtk, rtk_work->msg);
515 break;
516 case APPLE_RTKIT_EP_IOREPORT:
517 apple_rtkit_ioreport_rx(rtk, rtk_work->msg);
518 break;
519 case APPLE_RTKIT_EP_OSLOG:
520 apple_rtkit_oslog_rx(rtk, rtk_work->msg);
521 break;
522 case APPLE_RTKIT_APP_ENDPOINT_START ... 0xff:
523 if (rtk->ops->recv_message)
524 rtk->ops->recv_message(rtk->cookie, rtk_work->ep,
525 rtk_work->msg);
526 else
527 dev_warn(
528 rtk->dev,
529 "Received unexpected message to EP%02d: %llx\n",
530 rtk_work->ep, rtk_work->msg);
531 break;
532 default:
533 dev_warn(rtk->dev,
534 "RTKit: message to unknown endpoint %02x: %llx\n",
535 rtk_work->ep, rtk_work->msg);
536 }
537
538 kfree(rtk_work);
539 }
540
apple_rtkit_rx(struct mbox_client * cl,void * mssg)541 static void apple_rtkit_rx(struct mbox_client *cl, void *mssg)
542 {
543 struct apple_rtkit *rtk = container_of(cl, struct apple_rtkit, mbox_cl);
544 struct apple_mbox_msg *msg = mssg;
545 struct apple_rtkit_rx_work *work;
546 u8 ep = msg->msg1;
547
548 /*
549 * The message was read from a MMIO FIFO and we have to make
550 * sure all reads from buffers sent with that message happen
551 * afterwards.
552 */
553 dma_rmb();
554
555 if (!test_bit(ep, rtk->endpoints))
556 dev_warn(rtk->dev,
557 "RTKit: Message to undiscovered endpoint 0x%02x\n",
558 ep);
559
560 if (ep >= APPLE_RTKIT_APP_ENDPOINT_START &&
561 rtk->ops->recv_message_early &&
562 rtk->ops->recv_message_early(rtk->cookie, ep, msg->msg0))
563 return;
564
565 work = kzalloc(sizeof(*work), GFP_ATOMIC);
566 if (!work)
567 return;
568
569 work->rtk = rtk;
570 work->ep = ep;
571 work->msg = msg->msg0;
572 INIT_WORK(&work->work, apple_rtkit_rx_work);
573 queue_work(rtk->wq, &work->work);
574 }
575
apple_rtkit_tx_done(struct mbox_client * cl,void * mssg,int r)576 static void apple_rtkit_tx_done(struct mbox_client *cl, void *mssg, int r)
577 {
578 struct apple_rtkit_msg *msg =
579 container_of(mssg, struct apple_rtkit_msg, mbox_msg);
580
581 if (r == -ETIME)
582 return;
583
584 if (msg->completion)
585 complete(msg->completion);
586 kfree(msg);
587 }
588
apple_rtkit_send_message(struct apple_rtkit * rtk,u8 ep,u64 message,struct completion * completion,bool atomic)589 int apple_rtkit_send_message(struct apple_rtkit *rtk, u8 ep, u64 message,
590 struct completion *completion, bool atomic)
591 {
592 struct apple_rtkit_msg *msg;
593 int ret;
594 gfp_t flags;
595
596 if (rtk->crashed)
597 return -EINVAL;
598 if (ep >= APPLE_RTKIT_APP_ENDPOINT_START &&
599 !apple_rtkit_is_running(rtk))
600 return -EINVAL;
601
602 if (atomic)
603 flags = GFP_ATOMIC;
604 else
605 flags = GFP_KERNEL;
606
607 msg = kzalloc(sizeof(*msg), flags);
608 if (!msg)
609 return -ENOMEM;
610
611 msg->mbox_msg.msg0 = message;
612 msg->mbox_msg.msg1 = ep;
613 msg->completion = completion;
614
615 /*
616 * The message will be sent with a MMIO write. We need the barrier
617 * here to ensure any previous writes to buffers are visible to the
618 * device before that MMIO write happens.
619 */
620 dma_wmb();
621
622 ret = mbox_send_message(rtk->mbox_chan, &msg->mbox_msg);
623 if (ret < 0) {
624 kfree(msg);
625 return ret;
626 }
627
628 return 0;
629 }
630 EXPORT_SYMBOL_GPL(apple_rtkit_send_message);
631
apple_rtkit_send_message_wait(struct apple_rtkit * rtk,u8 ep,u64 message,unsigned long timeout,bool atomic)632 int apple_rtkit_send_message_wait(struct apple_rtkit *rtk, u8 ep, u64 message,
633 unsigned long timeout, bool atomic)
634 {
635 DECLARE_COMPLETION_ONSTACK(completion);
636 int ret;
637 long t;
638
639 ret = apple_rtkit_send_message(rtk, ep, message, &completion, atomic);
640 if (ret < 0)
641 return ret;
642
643 if (atomic) {
644 ret = mbox_flush(rtk->mbox_chan, timeout);
645 if (ret < 0)
646 return ret;
647
648 if (try_wait_for_completion(&completion))
649 return 0;
650
651 return -ETIME;
652 } else {
653 t = wait_for_completion_interruptible_timeout(
654 &completion, msecs_to_jiffies(timeout));
655 if (t < 0)
656 return t;
657 else if (t == 0)
658 return -ETIME;
659 return 0;
660 }
661 }
662 EXPORT_SYMBOL_GPL(apple_rtkit_send_message_wait);
663
apple_rtkit_poll(struct apple_rtkit * rtk)664 int apple_rtkit_poll(struct apple_rtkit *rtk)
665 {
666 return mbox_client_peek_data(rtk->mbox_chan);
667 }
668 EXPORT_SYMBOL_GPL(apple_rtkit_poll);
669
apple_rtkit_start_ep(struct apple_rtkit * rtk,u8 endpoint)670 int apple_rtkit_start_ep(struct apple_rtkit *rtk, u8 endpoint)
671 {
672 u64 msg;
673
674 if (!test_bit(endpoint, rtk->endpoints))
675 return -EINVAL;
676 if (endpoint >= APPLE_RTKIT_APP_ENDPOINT_START &&
677 !apple_rtkit_is_running(rtk))
678 return -EINVAL;
679
680 msg = FIELD_PREP(APPLE_RTKIT_MGMT_STARTEP_EP, endpoint);
681 msg |= APPLE_RTKIT_MGMT_STARTEP_FLAG;
682 apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_STARTEP, msg);
683
684 return 0;
685 }
686 EXPORT_SYMBOL_GPL(apple_rtkit_start_ep);
687
apple_rtkit_request_mbox_chan(struct apple_rtkit * rtk)688 static int apple_rtkit_request_mbox_chan(struct apple_rtkit *rtk)
689 {
690 if (rtk->mbox_name)
691 rtk->mbox_chan = mbox_request_channel_byname(&rtk->mbox_cl,
692 rtk->mbox_name);
693 else
694 rtk->mbox_chan =
695 mbox_request_channel(&rtk->mbox_cl, rtk->mbox_idx);
696
697 if (IS_ERR(rtk->mbox_chan))
698 return PTR_ERR(rtk->mbox_chan);
699 return 0;
700 }
701
apple_rtkit_init(struct device * dev,void * cookie,const char * mbox_name,int mbox_idx,const struct apple_rtkit_ops * ops)702 struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie,
703 const char *mbox_name, int mbox_idx,
704 const struct apple_rtkit_ops *ops)
705 {
706 struct apple_rtkit *rtk;
707 int ret;
708
709 if (!ops)
710 return ERR_PTR(-EINVAL);
711
712 rtk = kzalloc(sizeof(*rtk), GFP_KERNEL);
713 if (!rtk)
714 return ERR_PTR(-ENOMEM);
715
716 rtk->dev = dev;
717 rtk->cookie = cookie;
718 rtk->ops = ops;
719
720 init_completion(&rtk->epmap_completion);
721 init_completion(&rtk->iop_pwr_ack_completion);
722 init_completion(&rtk->ap_pwr_ack_completion);
723
724 bitmap_zero(rtk->endpoints, APPLE_RTKIT_MAX_ENDPOINTS);
725 set_bit(APPLE_RTKIT_EP_MGMT, rtk->endpoints);
726
727 rtk->mbox_name = mbox_name;
728 rtk->mbox_idx = mbox_idx;
729 rtk->mbox_cl.dev = dev;
730 rtk->mbox_cl.tx_block = false;
731 rtk->mbox_cl.knows_txdone = false;
732 rtk->mbox_cl.rx_callback = &apple_rtkit_rx;
733 rtk->mbox_cl.tx_done = &apple_rtkit_tx_done;
734
735 rtk->wq = alloc_ordered_workqueue("rtkit-%s", WQ_MEM_RECLAIM,
736 dev_name(rtk->dev));
737 if (!rtk->wq) {
738 ret = -ENOMEM;
739 goto free_rtk;
740 }
741
742 ret = apple_rtkit_request_mbox_chan(rtk);
743 if (ret)
744 goto destroy_wq;
745
746 return rtk;
747
748 destroy_wq:
749 destroy_workqueue(rtk->wq);
750 free_rtk:
751 kfree(rtk);
752 return ERR_PTR(ret);
753 }
754 EXPORT_SYMBOL_GPL(apple_rtkit_init);
755
apple_rtkit_wait_for_completion(struct completion * c)756 static int apple_rtkit_wait_for_completion(struct completion *c)
757 {
758 long t;
759
760 t = wait_for_completion_interruptible_timeout(c,
761 msecs_to_jiffies(1000));
762 if (t < 0)
763 return t;
764 else if (t == 0)
765 return -ETIME;
766 else
767 return 0;
768 }
769
apple_rtkit_reinit(struct apple_rtkit * rtk)770 int apple_rtkit_reinit(struct apple_rtkit *rtk)
771 {
772 /* make sure we don't handle any messages while reinitializing */
773 mbox_free_channel(rtk->mbox_chan);
774 flush_workqueue(rtk->wq);
775
776 apple_rtkit_free_buffer(rtk, &rtk->ioreport_buffer);
777 apple_rtkit_free_buffer(rtk, &rtk->crashlog_buffer);
778 apple_rtkit_free_buffer(rtk, &rtk->syslog_buffer);
779
780 kfree(rtk->syslog_msg_buffer);
781
782 rtk->syslog_msg_buffer = NULL;
783 rtk->syslog_n_entries = 0;
784 rtk->syslog_msg_size = 0;
785
786 bitmap_zero(rtk->endpoints, APPLE_RTKIT_MAX_ENDPOINTS);
787 set_bit(APPLE_RTKIT_EP_MGMT, rtk->endpoints);
788
789 reinit_completion(&rtk->epmap_completion);
790 reinit_completion(&rtk->iop_pwr_ack_completion);
791 reinit_completion(&rtk->ap_pwr_ack_completion);
792
793 rtk->crashed = false;
794 rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_OFF;
795 rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_OFF;
796
797 return apple_rtkit_request_mbox_chan(rtk);
798 }
799 EXPORT_SYMBOL_GPL(apple_rtkit_reinit);
800
apple_rtkit_set_ap_power_state(struct apple_rtkit * rtk,unsigned int state)801 static int apple_rtkit_set_ap_power_state(struct apple_rtkit *rtk,
802 unsigned int state)
803 {
804 u64 msg;
805 int ret;
806
807 reinit_completion(&rtk->ap_pwr_ack_completion);
808
809 msg = FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, state);
810 apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE,
811 msg);
812
813 ret = apple_rtkit_wait_for_completion(&rtk->ap_pwr_ack_completion);
814 if (ret)
815 return ret;
816
817 if (rtk->ap_power_state != state)
818 return -EINVAL;
819 return 0;
820 }
821
apple_rtkit_set_iop_power_state(struct apple_rtkit * rtk,unsigned int state)822 static int apple_rtkit_set_iop_power_state(struct apple_rtkit *rtk,
823 unsigned int state)
824 {
825 u64 msg;
826 int ret;
827
828 reinit_completion(&rtk->iop_pwr_ack_completion);
829
830 msg = FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, state);
831 apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE,
832 msg);
833
834 ret = apple_rtkit_wait_for_completion(&rtk->iop_pwr_ack_completion);
835 if (ret)
836 return ret;
837
838 if (rtk->iop_power_state != state)
839 return -EINVAL;
840 return 0;
841 }
842
apple_rtkit_boot(struct apple_rtkit * rtk)843 int apple_rtkit_boot(struct apple_rtkit *rtk)
844 {
845 int ret;
846
847 if (apple_rtkit_is_running(rtk))
848 return 0;
849 if (rtk->crashed)
850 return -EINVAL;
851
852 dev_dbg(rtk->dev, "RTKit: waiting for boot to finish\n");
853 ret = apple_rtkit_wait_for_completion(&rtk->epmap_completion);
854 if (ret)
855 return ret;
856 if (rtk->boot_result)
857 return rtk->boot_result;
858
859 dev_dbg(rtk->dev, "RTKit: waiting for IOP power state ACK\n");
860 ret = apple_rtkit_wait_for_completion(&rtk->iop_pwr_ack_completion);
861 if (ret)
862 return ret;
863
864 return apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON);
865 }
866 EXPORT_SYMBOL_GPL(apple_rtkit_boot);
867
apple_rtkit_shutdown(struct apple_rtkit * rtk)868 int apple_rtkit_shutdown(struct apple_rtkit *rtk)
869 {
870 int ret;
871
872 /* if OFF is used here the co-processor will not wake up again */
873 ret = apple_rtkit_set_ap_power_state(rtk,
874 APPLE_RTKIT_PWR_STATE_QUIESCED);
875 if (ret)
876 return ret;
877
878 ret = apple_rtkit_set_iop_power_state(rtk, APPLE_RTKIT_PWR_STATE_SLEEP);
879 if (ret)
880 return ret;
881
882 return apple_rtkit_reinit(rtk);
883 }
884 EXPORT_SYMBOL_GPL(apple_rtkit_shutdown);
885
apple_rtkit_idle(struct apple_rtkit * rtk)886 int apple_rtkit_idle(struct apple_rtkit *rtk)
887 {
888 int ret;
889
890 /* if OFF is used here the co-processor will not wake up again */
891 ret = apple_rtkit_set_ap_power_state(rtk,
892 APPLE_RTKIT_PWR_STATE_IDLE);
893 if (ret)
894 return ret;
895
896 ret = apple_rtkit_set_iop_power_state(rtk, APPLE_RTKIT_PWR_STATE_IDLE);
897 if (ret)
898 return ret;
899
900 rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_IDLE;
901 rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_IDLE;
902 return 0;
903 }
904 EXPORT_SYMBOL_GPL(apple_rtkit_idle);
905
apple_rtkit_quiesce(struct apple_rtkit * rtk)906 int apple_rtkit_quiesce(struct apple_rtkit *rtk)
907 {
908 int ret;
909
910 ret = apple_rtkit_set_ap_power_state(rtk,
911 APPLE_RTKIT_PWR_STATE_QUIESCED);
912 if (ret)
913 return ret;
914
915 ret = apple_rtkit_set_iop_power_state(rtk,
916 APPLE_RTKIT_PWR_STATE_QUIESCED);
917 if (ret)
918 return ret;
919
920 ret = apple_rtkit_reinit(rtk);
921 if (ret)
922 return ret;
923
924 rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_QUIESCED;
925 rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_QUIESCED;
926 return 0;
927 }
928 EXPORT_SYMBOL_GPL(apple_rtkit_quiesce);
929
apple_rtkit_wake(struct apple_rtkit * rtk)930 int apple_rtkit_wake(struct apple_rtkit *rtk)
931 {
932 u64 msg;
933
934 if (apple_rtkit_is_running(rtk))
935 return -EINVAL;
936
937 reinit_completion(&rtk->iop_pwr_ack_completion);
938
939 /*
940 * Use open-coded apple_rtkit_set_iop_power_state since apple_rtkit_boot
941 * will wait for the completion anyway.
942 */
943 msg = FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, APPLE_RTKIT_PWR_STATE_ON);
944 apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE,
945 msg);
946
947 return apple_rtkit_boot(rtk);
948 }
949 EXPORT_SYMBOL_GPL(apple_rtkit_wake);
950
apple_rtkit_free(struct apple_rtkit * rtk)951 void apple_rtkit_free(struct apple_rtkit *rtk)
952 {
953 mbox_free_channel(rtk->mbox_chan);
954 destroy_workqueue(rtk->wq);
955
956 apple_rtkit_free_buffer(rtk, &rtk->ioreport_buffer);
957 apple_rtkit_free_buffer(rtk, &rtk->crashlog_buffer);
958 apple_rtkit_free_buffer(rtk, &rtk->syslog_buffer);
959
960 kfree(rtk->syslog_msg_buffer);
961 kfree(rtk);
962 }
963 EXPORT_SYMBOL_GPL(apple_rtkit_free);
964
apple_rtkit_free_wrapper(void * data)965 static void apple_rtkit_free_wrapper(void *data)
966 {
967 apple_rtkit_free(data);
968 }
969
devm_apple_rtkit_init(struct device * dev,void * cookie,const char * mbox_name,int mbox_idx,const struct apple_rtkit_ops * ops)970 struct apple_rtkit *devm_apple_rtkit_init(struct device *dev, void *cookie,
971 const char *mbox_name, int mbox_idx,
972 const struct apple_rtkit_ops *ops)
973 {
974 struct apple_rtkit *rtk;
975 int ret;
976
977 rtk = apple_rtkit_init(dev, cookie, mbox_name, mbox_idx, ops);
978 if (IS_ERR(rtk))
979 return rtk;
980
981 ret = devm_add_action_or_reset(dev, apple_rtkit_free_wrapper, rtk);
982 if (ret)
983 return ERR_PTR(ret);
984
985 return rtk;
986 }
987 EXPORT_SYMBOL_GPL(devm_apple_rtkit_init);
988
989 MODULE_LICENSE("Dual MIT/GPL");
990 MODULE_AUTHOR("Sven Peter <sven@svenpeter.dev>");
991 MODULE_DESCRIPTION("Apple RTKit driver");
992