1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * VFIO based Physical Subchannel device driver
4 *
5 * Copyright IBM Corp. 2017
6 * Copyright Red Hat, Inc. 2019
7 *
8 * Author(s): Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com>
9 * Xiao Feng Ren <renxiaof@linux.vnet.ibm.com>
10 * Cornelia Huck <cohuck@redhat.com>
11 */
12
13 #include <linux/module.h>
14 #include <linux/init.h>
15 #include <linux/device.h>
16 #include <linux/slab.h>
17 #include <linux/uuid.h>
18 #include <linux/mdev.h>
19
20 #include <asm/isc.h>
21
22 #include "chp.h"
23 #include "ioasm.h"
24 #include "css.h"
25 #include "vfio_ccw_private.h"
26
27 struct workqueue_struct *vfio_ccw_work_q;
28 static struct kmem_cache *vfio_ccw_io_region;
29 static struct kmem_cache *vfio_ccw_cmd_region;
30 static struct kmem_cache *vfio_ccw_schib_region;
31 static struct kmem_cache *vfio_ccw_crw_region;
32
33 debug_info_t *vfio_ccw_debug_msg_id;
34 debug_info_t *vfio_ccw_debug_trace_id;
35
36 /*
37 * Helpers
38 */
vfio_ccw_sch_quiesce(struct subchannel * sch)39 int vfio_ccw_sch_quiesce(struct subchannel *sch)
40 {
41 struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev);
42 DECLARE_COMPLETION_ONSTACK(completion);
43 int iretry, ret = 0;
44
45 spin_lock_irq(sch->lock);
46 if (!sch->schib.pmcw.ena)
47 goto out_unlock;
48 ret = cio_disable_subchannel(sch);
49 if (ret != -EBUSY)
50 goto out_unlock;
51
52 iretry = 255;
53 do {
54
55 ret = cio_cancel_halt_clear(sch, &iretry);
56
57 if (ret == -EIO) {
58 pr_err("vfio_ccw: could not quiesce subchannel 0.%x.%04x!\n",
59 sch->schid.ssid, sch->schid.sch_no);
60 break;
61 }
62
63 /*
64 * Flush all I/O and wait for
65 * cancel/halt/clear completion.
66 */
67 private->completion = &completion;
68 spin_unlock_irq(sch->lock);
69
70 if (ret == -EBUSY)
71 wait_for_completion_timeout(&completion, 3*HZ);
72
73 private->completion = NULL;
74 flush_workqueue(vfio_ccw_work_q);
75 spin_lock_irq(sch->lock);
76 ret = cio_disable_subchannel(sch);
77 } while (ret == -EBUSY);
78 out_unlock:
79 private->state = VFIO_CCW_STATE_NOT_OPER;
80 spin_unlock_irq(sch->lock);
81 return ret;
82 }
83
vfio_ccw_sch_io_todo(struct work_struct * work)84 static void vfio_ccw_sch_io_todo(struct work_struct *work)
85 {
86 struct vfio_ccw_private *private;
87 struct irb *irb;
88 bool is_final;
89 bool cp_is_finished = false;
90
91 private = container_of(work, struct vfio_ccw_private, io_work);
92 irb = &private->irb;
93
94 is_final = !(scsw_actl(&irb->scsw) &
95 (SCSW_ACTL_DEVACT | SCSW_ACTL_SCHACT));
96 if (scsw_is_solicited(&irb->scsw)) {
97 cp_update_scsw(&private->cp, &irb->scsw);
98 if (is_final && private->state == VFIO_CCW_STATE_CP_PENDING) {
99 cp_free(&private->cp);
100 cp_is_finished = true;
101 }
102 }
103 mutex_lock(&private->io_mutex);
104 memcpy(private->io_region->irb_area, irb, sizeof(*irb));
105 mutex_unlock(&private->io_mutex);
106
107 /*
108 * Reset to IDLE only if processing of a channel program
109 * has finished. Do not overwrite a possible processing
110 * state if the final interrupt was for HSCH or CSCH.
111 */
112 if (private->mdev && cp_is_finished)
113 private->state = VFIO_CCW_STATE_IDLE;
114
115 if (private->io_trigger)
116 eventfd_signal(private->io_trigger, 1);
117 }
118
vfio_ccw_crw_todo(struct work_struct * work)119 static void vfio_ccw_crw_todo(struct work_struct *work)
120 {
121 struct vfio_ccw_private *private;
122
123 private = container_of(work, struct vfio_ccw_private, crw_work);
124
125 if (!list_empty(&private->crw) && private->crw_trigger)
126 eventfd_signal(private->crw_trigger, 1);
127 }
128
129 /*
130 * Css driver callbacks
131 */
vfio_ccw_sch_irq(struct subchannel * sch)132 static void vfio_ccw_sch_irq(struct subchannel *sch)
133 {
134 struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev);
135
136 inc_irq_stat(IRQIO_CIO);
137 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_INTERRUPT);
138 }
139
vfio_ccw_alloc_private(struct subchannel * sch)140 static struct vfio_ccw_private *vfio_ccw_alloc_private(struct subchannel *sch)
141 {
142 struct vfio_ccw_private *private;
143
144 private = kzalloc(sizeof(*private), GFP_KERNEL);
145 if (!private)
146 return ERR_PTR(-ENOMEM);
147
148 private->sch = sch;
149 mutex_init(&private->io_mutex);
150 private->state = VFIO_CCW_STATE_NOT_OPER;
151 INIT_LIST_HEAD(&private->crw);
152 INIT_WORK(&private->io_work, vfio_ccw_sch_io_todo);
153 INIT_WORK(&private->crw_work, vfio_ccw_crw_todo);
154 atomic_set(&private->avail, 1);
155
156 private->cp.guest_cp = kcalloc(CCWCHAIN_LEN_MAX, sizeof(struct ccw1),
157 GFP_KERNEL);
158 if (!private->cp.guest_cp)
159 goto out_free_private;
160
161 private->io_region = kmem_cache_zalloc(vfio_ccw_io_region,
162 GFP_KERNEL | GFP_DMA);
163 if (!private->io_region)
164 goto out_free_cp;
165
166 private->cmd_region = kmem_cache_zalloc(vfio_ccw_cmd_region,
167 GFP_KERNEL | GFP_DMA);
168 if (!private->cmd_region)
169 goto out_free_io;
170
171 private->schib_region = kmem_cache_zalloc(vfio_ccw_schib_region,
172 GFP_KERNEL | GFP_DMA);
173
174 if (!private->schib_region)
175 goto out_free_cmd;
176
177 private->crw_region = kmem_cache_zalloc(vfio_ccw_crw_region,
178 GFP_KERNEL | GFP_DMA);
179
180 if (!private->crw_region)
181 goto out_free_schib;
182 return private;
183
184 out_free_schib:
185 kmem_cache_free(vfio_ccw_schib_region, private->schib_region);
186 out_free_cmd:
187 kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region);
188 out_free_io:
189 kmem_cache_free(vfio_ccw_io_region, private->io_region);
190 out_free_cp:
191 kfree(private->cp.guest_cp);
192 out_free_private:
193 mutex_destroy(&private->io_mutex);
194 kfree(private);
195 return ERR_PTR(-ENOMEM);
196 }
197
vfio_ccw_free_private(struct vfio_ccw_private * private)198 static void vfio_ccw_free_private(struct vfio_ccw_private *private)
199 {
200 struct vfio_ccw_crw *crw, *temp;
201
202 list_for_each_entry_safe(crw, temp, &private->crw, next) {
203 list_del(&crw->next);
204 kfree(crw);
205 }
206
207 kmem_cache_free(vfio_ccw_crw_region, private->crw_region);
208 kmem_cache_free(vfio_ccw_schib_region, private->schib_region);
209 kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region);
210 kmem_cache_free(vfio_ccw_io_region, private->io_region);
211 kfree(private->cp.guest_cp);
212 mutex_destroy(&private->io_mutex);
213 kfree(private);
214 }
215
vfio_ccw_sch_probe(struct subchannel * sch)216 static int vfio_ccw_sch_probe(struct subchannel *sch)
217 {
218 struct pmcw *pmcw = &sch->schib.pmcw;
219 struct vfio_ccw_private *private;
220 int ret = -ENOMEM;
221
222 if (pmcw->qf) {
223 dev_warn(&sch->dev, "vfio: ccw: does not support QDIO: %s\n",
224 dev_name(&sch->dev));
225 return -ENODEV;
226 }
227
228 private = vfio_ccw_alloc_private(sch);
229 if (IS_ERR(private))
230 return PTR_ERR(private);
231
232 dev_set_drvdata(&sch->dev, private);
233
234 spin_lock_irq(sch->lock);
235 sch->isc = VFIO_CCW_ISC;
236 ret = cio_enable_subchannel(sch, (u32)(unsigned long)sch);
237 spin_unlock_irq(sch->lock);
238 if (ret)
239 goto out_free;
240
241 private->state = VFIO_CCW_STATE_STANDBY;
242
243 ret = vfio_ccw_mdev_reg(sch);
244 if (ret)
245 goto out_disable;
246
247 if (dev_get_uevent_suppress(&sch->dev)) {
248 dev_set_uevent_suppress(&sch->dev, 0);
249 kobject_uevent(&sch->dev.kobj, KOBJ_ADD);
250 }
251
252 VFIO_CCW_MSG_EVENT(4, "bound to subchannel %x.%x.%04x\n",
253 sch->schid.cssid, sch->schid.ssid,
254 sch->schid.sch_no);
255 return 0;
256
257 out_disable:
258 cio_disable_subchannel(sch);
259 out_free:
260 dev_set_drvdata(&sch->dev, NULL);
261 vfio_ccw_free_private(private);
262 return ret;
263 }
264
vfio_ccw_sch_remove(struct subchannel * sch)265 static void vfio_ccw_sch_remove(struct subchannel *sch)
266 {
267 struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev);
268
269 vfio_ccw_sch_quiesce(sch);
270 vfio_ccw_mdev_unreg(sch);
271
272 dev_set_drvdata(&sch->dev, NULL);
273
274 vfio_ccw_free_private(private);
275
276 VFIO_CCW_MSG_EVENT(4, "unbound from subchannel %x.%x.%04x\n",
277 sch->schid.cssid, sch->schid.ssid,
278 sch->schid.sch_no);
279 }
280
vfio_ccw_sch_shutdown(struct subchannel * sch)281 static void vfio_ccw_sch_shutdown(struct subchannel *sch)
282 {
283 vfio_ccw_sch_quiesce(sch);
284 }
285
286 /**
287 * vfio_ccw_sch_event - process subchannel event
288 * @sch: subchannel
289 * @process: non-zero if function is called in process context
290 *
291 * An unspecified event occurred for this subchannel. Adjust data according
292 * to the current operational state of the subchannel. Return zero when the
293 * event has been handled sufficiently or -EAGAIN when this function should
294 * be called again in process context.
295 */
vfio_ccw_sch_event(struct subchannel * sch,int process)296 static int vfio_ccw_sch_event(struct subchannel *sch, int process)
297 {
298 struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev);
299 unsigned long flags;
300 int rc = -EAGAIN;
301
302 spin_lock_irqsave(sch->lock, flags);
303 if (!device_is_registered(&sch->dev))
304 goto out_unlock;
305
306 if (work_pending(&sch->todo_work))
307 goto out_unlock;
308
309 if (cio_update_schib(sch)) {
310 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_NOT_OPER);
311 rc = 0;
312 goto out_unlock;
313 }
314
315 private = dev_get_drvdata(&sch->dev);
316 if (private->state == VFIO_CCW_STATE_NOT_OPER) {
317 private->state = private->mdev ? VFIO_CCW_STATE_IDLE :
318 VFIO_CCW_STATE_STANDBY;
319 }
320 rc = 0;
321
322 out_unlock:
323 spin_unlock_irqrestore(sch->lock, flags);
324
325 return rc;
326 }
327
vfio_ccw_queue_crw(struct vfio_ccw_private * private,unsigned int rsc,unsigned int erc,unsigned int rsid)328 static void vfio_ccw_queue_crw(struct vfio_ccw_private *private,
329 unsigned int rsc,
330 unsigned int erc,
331 unsigned int rsid)
332 {
333 struct vfio_ccw_crw *crw;
334
335 /*
336 * If unable to allocate a CRW, just drop the event and
337 * carry on. The guest will either see a later one or
338 * learn when it issues its own store subchannel.
339 */
340 crw = kzalloc(sizeof(*crw), GFP_ATOMIC);
341 if (!crw)
342 return;
343
344 /*
345 * Build the CRW based on the inputs given to us.
346 */
347 crw->crw.rsc = rsc;
348 crw->crw.erc = erc;
349 crw->crw.rsid = rsid;
350
351 list_add_tail(&crw->next, &private->crw);
352 queue_work(vfio_ccw_work_q, &private->crw_work);
353 }
354
vfio_ccw_chp_event(struct subchannel * sch,struct chp_link * link,int event)355 static int vfio_ccw_chp_event(struct subchannel *sch,
356 struct chp_link *link, int event)
357 {
358 struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev);
359 int mask = chp_ssd_get_mask(&sch->ssd_info, link);
360 int retry = 255;
361
362 if (!private || !mask)
363 return 0;
364
365 trace_vfio_ccw_chp_event(private->sch->schid, mask, event);
366 VFIO_CCW_MSG_EVENT(2, "%pUl (%x.%x.%04x): mask=0x%x event=%d\n",
367 mdev_uuid(private->mdev), sch->schid.cssid,
368 sch->schid.ssid, sch->schid.sch_no,
369 mask, event);
370
371 if (cio_update_schib(sch))
372 return -ENODEV;
373
374 switch (event) {
375 case CHP_VARY_OFF:
376 /* Path logically turned off */
377 sch->opm &= ~mask;
378 sch->lpm &= ~mask;
379 if (sch->schib.pmcw.lpum & mask)
380 cio_cancel_halt_clear(sch, &retry);
381 break;
382 case CHP_OFFLINE:
383 /* Path is gone */
384 if (sch->schib.pmcw.lpum & mask)
385 cio_cancel_halt_clear(sch, &retry);
386 vfio_ccw_queue_crw(private, CRW_RSC_CPATH, CRW_ERC_PERRN,
387 link->chpid.id);
388 break;
389 case CHP_VARY_ON:
390 /* Path logically turned on */
391 sch->opm |= mask;
392 sch->lpm |= mask;
393 break;
394 case CHP_ONLINE:
395 /* Path became available */
396 sch->lpm |= mask & sch->opm;
397 vfio_ccw_queue_crw(private, CRW_RSC_CPATH, CRW_ERC_INIT,
398 link->chpid.id);
399 break;
400 }
401
402 return 0;
403 }
404
405 static struct css_device_id vfio_ccw_sch_ids[] = {
406 { .match_flags = 0x1, .type = SUBCHANNEL_TYPE_IO, },
407 { /* end of list */ },
408 };
409 MODULE_DEVICE_TABLE(css, vfio_ccw_sch_ids);
410
411 static struct css_driver vfio_ccw_sch_driver = {
412 .drv = {
413 .name = "vfio_ccw",
414 .owner = THIS_MODULE,
415 },
416 .subchannel_type = vfio_ccw_sch_ids,
417 .irq = vfio_ccw_sch_irq,
418 .probe = vfio_ccw_sch_probe,
419 .remove = vfio_ccw_sch_remove,
420 .shutdown = vfio_ccw_sch_shutdown,
421 .sch_event = vfio_ccw_sch_event,
422 .chp_event = vfio_ccw_chp_event,
423 };
424
vfio_ccw_debug_init(void)425 static int __init vfio_ccw_debug_init(void)
426 {
427 vfio_ccw_debug_msg_id = debug_register("vfio_ccw_msg", 16, 1,
428 11 * sizeof(long));
429 if (!vfio_ccw_debug_msg_id)
430 goto out_unregister;
431 debug_register_view(vfio_ccw_debug_msg_id, &debug_sprintf_view);
432 debug_set_level(vfio_ccw_debug_msg_id, 2);
433 vfio_ccw_debug_trace_id = debug_register("vfio_ccw_trace", 16, 1, 16);
434 if (!vfio_ccw_debug_trace_id)
435 goto out_unregister;
436 debug_register_view(vfio_ccw_debug_trace_id, &debug_hex_ascii_view);
437 debug_set_level(vfio_ccw_debug_trace_id, 2);
438 return 0;
439
440 out_unregister:
441 debug_unregister(vfio_ccw_debug_msg_id);
442 debug_unregister(vfio_ccw_debug_trace_id);
443 return -1;
444 }
445
vfio_ccw_debug_exit(void)446 static void vfio_ccw_debug_exit(void)
447 {
448 debug_unregister(vfio_ccw_debug_msg_id);
449 debug_unregister(vfio_ccw_debug_trace_id);
450 }
451
vfio_ccw_destroy_regions(void)452 static void vfio_ccw_destroy_regions(void)
453 {
454 kmem_cache_destroy(vfio_ccw_crw_region);
455 kmem_cache_destroy(vfio_ccw_schib_region);
456 kmem_cache_destroy(vfio_ccw_cmd_region);
457 kmem_cache_destroy(vfio_ccw_io_region);
458 }
459
vfio_ccw_sch_init(void)460 static int __init vfio_ccw_sch_init(void)
461 {
462 int ret;
463
464 ret = vfio_ccw_debug_init();
465 if (ret)
466 return ret;
467
468 vfio_ccw_work_q = create_singlethread_workqueue("vfio-ccw");
469 if (!vfio_ccw_work_q) {
470 ret = -ENOMEM;
471 goto out_regions;
472 }
473
474 vfio_ccw_io_region = kmem_cache_create_usercopy("vfio_ccw_io_region",
475 sizeof(struct ccw_io_region), 0,
476 SLAB_ACCOUNT, 0,
477 sizeof(struct ccw_io_region), NULL);
478 if (!vfio_ccw_io_region) {
479 ret = -ENOMEM;
480 goto out_regions;
481 }
482
483 vfio_ccw_cmd_region = kmem_cache_create_usercopy("vfio_ccw_cmd_region",
484 sizeof(struct ccw_cmd_region), 0,
485 SLAB_ACCOUNT, 0,
486 sizeof(struct ccw_cmd_region), NULL);
487 if (!vfio_ccw_cmd_region) {
488 ret = -ENOMEM;
489 goto out_regions;
490 }
491
492 vfio_ccw_schib_region = kmem_cache_create_usercopy("vfio_ccw_schib_region",
493 sizeof(struct ccw_schib_region), 0,
494 SLAB_ACCOUNT, 0,
495 sizeof(struct ccw_schib_region), NULL);
496
497 if (!vfio_ccw_schib_region) {
498 ret = -ENOMEM;
499 goto out_regions;
500 }
501
502 vfio_ccw_crw_region = kmem_cache_create_usercopy("vfio_ccw_crw_region",
503 sizeof(struct ccw_crw_region), 0,
504 SLAB_ACCOUNT, 0,
505 sizeof(struct ccw_crw_region), NULL);
506
507 if (!vfio_ccw_crw_region) {
508 ret = -ENOMEM;
509 goto out_regions;
510 }
511
512 ret = mdev_register_driver(&vfio_ccw_mdev_driver);
513 if (ret)
514 goto out_regions;
515
516 isc_register(VFIO_CCW_ISC);
517 ret = css_driver_register(&vfio_ccw_sch_driver);
518 if (ret) {
519 isc_unregister(VFIO_CCW_ISC);
520 goto out_driver;
521 }
522
523 return ret;
524
525 out_driver:
526 mdev_unregister_driver(&vfio_ccw_mdev_driver);
527 out_regions:
528 vfio_ccw_destroy_regions();
529 destroy_workqueue(vfio_ccw_work_q);
530 vfio_ccw_debug_exit();
531 return ret;
532 }
533
vfio_ccw_sch_exit(void)534 static void __exit vfio_ccw_sch_exit(void)
535 {
536 css_driver_unregister(&vfio_ccw_sch_driver);
537 mdev_unregister_driver(&vfio_ccw_mdev_driver);
538 isc_unregister(VFIO_CCW_ISC);
539 vfio_ccw_destroy_regions();
540 destroy_workqueue(vfio_ccw_work_q);
541 vfio_ccw_debug_exit();
542 }
543 module_init(vfio_ccw_sch_init);
544 module_exit(vfio_ccw_sch_exit);
545
546 MODULE_LICENSE("GPL v2");
547