1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2020 Intel Corporation
4 * Author: Johannes Berg <johannes@sipsolutions.net>
5 */
6 #include <linux/module.h>
7 #include <linux/pci.h>
8 #include <linux/virtio.h>
9 #include <linux/virtio_config.h>
10 #include <linux/logic_iomem.h>
11 #include <linux/irqdomain.h>
12 #include <linux/virtio_pcidev.h>
13 #include <linux/virtio-uml.h>
14 #include <linux/delay.h>
15 #include <linux/msi.h>
16 #include <asm/unaligned.h>
17 #include <irq_kern.h>
18
19 #define MAX_DEVICES 8
20 #define MAX_MSI_VECTORS 32
21 #define CFG_SPACE_SIZE 4096
22
23 /* for MSI-X we have a 32-bit payload */
24 #define MAX_IRQ_MSG_SIZE (sizeof(struct virtio_pcidev_msg) + sizeof(u32))
25 #define NUM_IRQ_MSGS 10
26
27 #define HANDLE_NO_FREE(ptr) ((void *)((unsigned long)(ptr) | 1))
28 #define HANDLE_IS_NO_FREE(ptr) ((unsigned long)(ptr) & 1)
29
30 struct um_pci_device {
31 struct virtio_device *vdev;
32
33 /* for now just standard BARs */
34 u8 resptr[PCI_STD_NUM_BARS];
35
36 struct virtqueue *cmd_vq, *irq_vq;
37
38 #define UM_PCI_STAT_WAITING 0
39 unsigned long status;
40
41 int irq;
42 };
43
44 struct um_pci_device_reg {
45 struct um_pci_device *dev;
46 void __iomem *iomem;
47 };
48
49 static struct pci_host_bridge *bridge;
50 static DEFINE_MUTEX(um_pci_mtx);
51 static struct um_pci_device_reg um_pci_devices[MAX_DEVICES];
52 static struct fwnode_handle *um_pci_fwnode;
53 static struct irq_domain *um_pci_inner_domain;
54 static struct irq_domain *um_pci_msi_domain;
55 static unsigned long um_pci_msi_used[BITS_TO_LONGS(MAX_MSI_VECTORS)];
56
57 #define UM_VIRT_PCI_MAXDELAY 40000
58
59 struct um_pci_message_buffer {
60 struct virtio_pcidev_msg hdr;
61 u8 data[8];
62 };
63
64 static struct um_pci_message_buffer __percpu *um_pci_msg_bufs;
65
um_pci_send_cmd(struct um_pci_device * dev,struct virtio_pcidev_msg * cmd,unsigned int cmd_size,const void * extra,unsigned int extra_size,void * out,unsigned int out_size)66 static int um_pci_send_cmd(struct um_pci_device *dev,
67 struct virtio_pcidev_msg *cmd,
68 unsigned int cmd_size,
69 const void *extra, unsigned int extra_size,
70 void *out, unsigned int out_size)
71 {
72 struct scatterlist out_sg, extra_sg, in_sg;
73 struct scatterlist *sgs_list[] = {
74 [0] = &out_sg,
75 [1] = extra ? &extra_sg : &in_sg,
76 [2] = extra ? &in_sg : NULL,
77 };
78 struct um_pci_message_buffer *buf;
79 int delay_count = 0;
80 int ret, len;
81 bool posted;
82
83 if (WARN_ON(cmd_size < sizeof(*cmd) || cmd_size > sizeof(*buf)))
84 return -EINVAL;
85
86 switch (cmd->op) {
87 case VIRTIO_PCIDEV_OP_CFG_WRITE:
88 case VIRTIO_PCIDEV_OP_MMIO_WRITE:
89 case VIRTIO_PCIDEV_OP_MMIO_MEMSET:
90 /* in PCI, writes are posted, so don't wait */
91 posted = !out;
92 WARN_ON(!posted);
93 break;
94 default:
95 posted = false;
96 break;
97 }
98
99 buf = get_cpu_var(um_pci_msg_bufs);
100 memcpy(buf, cmd, cmd_size);
101
102 if (posted) {
103 u8 *ncmd = kmalloc(cmd_size + extra_size, GFP_ATOMIC);
104
105 if (ncmd) {
106 memcpy(ncmd, cmd, cmd_size);
107 if (extra)
108 memcpy(ncmd + cmd_size, extra, extra_size);
109 cmd = (void *)ncmd;
110 cmd_size += extra_size;
111 extra = NULL;
112 extra_size = 0;
113 } else {
114 /* try without allocating memory */
115 posted = false;
116 cmd = (void *)buf;
117 }
118 } else {
119 cmd = (void *)buf;
120 }
121
122 sg_init_one(&out_sg, cmd, cmd_size);
123 if (extra)
124 sg_init_one(&extra_sg, extra, extra_size);
125 if (out)
126 sg_init_one(&in_sg, out, out_size);
127
128 /* add to internal virtio queue */
129 ret = virtqueue_add_sgs(dev->cmd_vq, sgs_list,
130 extra ? 2 : 1,
131 out ? 1 : 0,
132 posted ? cmd : HANDLE_NO_FREE(cmd),
133 GFP_ATOMIC);
134 if (ret)
135 goto out;
136
137 if (posted) {
138 virtqueue_kick(dev->cmd_vq);
139 ret = 0;
140 goto out;
141 }
142
143 /* kick and poll for getting a response on the queue */
144 set_bit(UM_PCI_STAT_WAITING, &dev->status);
145 virtqueue_kick(dev->cmd_vq);
146
147 while (1) {
148 void *completed = virtqueue_get_buf(dev->cmd_vq, &len);
149
150 if (completed == HANDLE_NO_FREE(cmd))
151 break;
152
153 if (completed && !HANDLE_IS_NO_FREE(completed))
154 kfree(completed);
155
156 if (WARN_ONCE(virtqueue_is_broken(dev->cmd_vq) ||
157 ++delay_count > UM_VIRT_PCI_MAXDELAY,
158 "um virt-pci delay: %d", delay_count)) {
159 ret = -EIO;
160 break;
161 }
162 udelay(1);
163 }
164 clear_bit(UM_PCI_STAT_WAITING, &dev->status);
165
166 out:
167 put_cpu_var(um_pci_msg_bufs);
168 return ret;
169 }
170
um_pci_cfgspace_read(void * priv,unsigned int offset,int size)171 static unsigned long um_pci_cfgspace_read(void *priv, unsigned int offset,
172 int size)
173 {
174 struct um_pci_device_reg *reg = priv;
175 struct um_pci_device *dev = reg->dev;
176 struct virtio_pcidev_msg hdr = {
177 .op = VIRTIO_PCIDEV_OP_CFG_READ,
178 .size = size,
179 .addr = offset,
180 };
181 /* buf->data is maximum size - we may only use parts of it */
182 struct um_pci_message_buffer *buf;
183 u8 *data;
184 unsigned long ret = ~0ULL;
185
186 if (!dev)
187 return ~0ULL;
188
189 buf = get_cpu_var(um_pci_msg_bufs);
190 data = buf->data;
191
192 memset(data, 0xff, sizeof(data));
193
194 switch (size) {
195 case 1:
196 case 2:
197 case 4:
198 #ifdef CONFIG_64BIT
199 case 8:
200 #endif
201 break;
202 default:
203 WARN(1, "invalid config space read size %d\n", size);
204 goto out;
205 }
206
207 if (um_pci_send_cmd(dev, &hdr, sizeof(hdr), NULL, 0, data, 8))
208 goto out;
209
210 switch (size) {
211 case 1:
212 ret = data[0];
213 break;
214 case 2:
215 ret = le16_to_cpup((void *)data);
216 break;
217 case 4:
218 ret = le32_to_cpup((void *)data);
219 break;
220 #ifdef CONFIG_64BIT
221 case 8:
222 ret = le64_to_cpup((void *)data);
223 break;
224 #endif
225 default:
226 break;
227 }
228
229 out:
230 put_cpu_var(um_pci_msg_bufs);
231 return ret;
232 }
233
um_pci_cfgspace_write(void * priv,unsigned int offset,int size,unsigned long val)234 static void um_pci_cfgspace_write(void *priv, unsigned int offset, int size,
235 unsigned long val)
236 {
237 struct um_pci_device_reg *reg = priv;
238 struct um_pci_device *dev = reg->dev;
239 struct {
240 struct virtio_pcidev_msg hdr;
241 /* maximum size - we may only use parts of it */
242 u8 data[8];
243 } msg = {
244 .hdr = {
245 .op = VIRTIO_PCIDEV_OP_CFG_WRITE,
246 .size = size,
247 .addr = offset,
248 },
249 };
250
251 if (!dev)
252 return;
253
254 switch (size) {
255 case 1:
256 msg.data[0] = (u8)val;
257 break;
258 case 2:
259 put_unaligned_le16(val, (void *)msg.data);
260 break;
261 case 4:
262 put_unaligned_le32(val, (void *)msg.data);
263 break;
264 #ifdef CONFIG_64BIT
265 case 8:
266 put_unaligned_le64(val, (void *)msg.data);
267 break;
268 #endif
269 default:
270 WARN(1, "invalid config space write size %d\n", size);
271 return;
272 }
273
274 WARN_ON(um_pci_send_cmd(dev, &msg.hdr, sizeof(msg), NULL, 0, NULL, 0));
275 }
276
277 static const struct logic_iomem_ops um_pci_device_cfgspace_ops = {
278 .read = um_pci_cfgspace_read,
279 .write = um_pci_cfgspace_write,
280 };
281
um_pci_bar_copy_from(void * priv,void * buffer,unsigned int offset,int size)282 static void um_pci_bar_copy_from(void *priv, void *buffer,
283 unsigned int offset, int size)
284 {
285 u8 *resptr = priv;
286 struct um_pci_device *dev = container_of(resptr - *resptr,
287 struct um_pci_device,
288 resptr[0]);
289 struct virtio_pcidev_msg hdr = {
290 .op = VIRTIO_PCIDEV_OP_MMIO_READ,
291 .bar = *resptr,
292 .size = size,
293 .addr = offset,
294 };
295
296 memset(buffer, 0xff, size);
297
298 um_pci_send_cmd(dev, &hdr, sizeof(hdr), NULL, 0, buffer, size);
299 }
300
um_pci_bar_read(void * priv,unsigned int offset,int size)301 static unsigned long um_pci_bar_read(void *priv, unsigned int offset,
302 int size)
303 {
304 /* buf->data is maximum size - we may only use parts of it */
305 struct um_pci_message_buffer *buf;
306 u8 *data;
307 unsigned long ret = ~0ULL;
308
309 buf = get_cpu_var(um_pci_msg_bufs);
310 data = buf->data;
311
312 switch (size) {
313 case 1:
314 case 2:
315 case 4:
316 #ifdef CONFIG_64BIT
317 case 8:
318 #endif
319 break;
320 default:
321 WARN(1, "invalid config space read size %d\n", size);
322 goto out;
323 }
324
325 um_pci_bar_copy_from(priv, data, offset, size);
326
327 switch (size) {
328 case 1:
329 ret = data[0];
330 break;
331 case 2:
332 ret = le16_to_cpup((void *)data);
333 break;
334 case 4:
335 ret = le32_to_cpup((void *)data);
336 break;
337 #ifdef CONFIG_64BIT
338 case 8:
339 ret = le64_to_cpup((void *)data);
340 break;
341 #endif
342 default:
343 break;
344 }
345
346 out:
347 put_cpu_var(um_pci_msg_bufs);
348 return ret;
349 }
350
um_pci_bar_copy_to(void * priv,unsigned int offset,const void * buffer,int size)351 static void um_pci_bar_copy_to(void *priv, unsigned int offset,
352 const void *buffer, int size)
353 {
354 u8 *resptr = priv;
355 struct um_pci_device *dev = container_of(resptr - *resptr,
356 struct um_pci_device,
357 resptr[0]);
358 struct virtio_pcidev_msg hdr = {
359 .op = VIRTIO_PCIDEV_OP_MMIO_WRITE,
360 .bar = *resptr,
361 .size = size,
362 .addr = offset,
363 };
364
365 um_pci_send_cmd(dev, &hdr, sizeof(hdr), buffer, size, NULL, 0);
366 }
367
um_pci_bar_write(void * priv,unsigned int offset,int size,unsigned long val)368 static void um_pci_bar_write(void *priv, unsigned int offset, int size,
369 unsigned long val)
370 {
371 /* maximum size - we may only use parts of it */
372 u8 data[8];
373
374 switch (size) {
375 case 1:
376 data[0] = (u8)val;
377 break;
378 case 2:
379 put_unaligned_le16(val, (void *)data);
380 break;
381 case 4:
382 put_unaligned_le32(val, (void *)data);
383 break;
384 #ifdef CONFIG_64BIT
385 case 8:
386 put_unaligned_le64(val, (void *)data);
387 break;
388 #endif
389 default:
390 WARN(1, "invalid config space write size %d\n", size);
391 return;
392 }
393
394 um_pci_bar_copy_to(priv, offset, data, size);
395 }
396
um_pci_bar_set(void * priv,unsigned int offset,u8 value,int size)397 static void um_pci_bar_set(void *priv, unsigned int offset, u8 value, int size)
398 {
399 u8 *resptr = priv;
400 struct um_pci_device *dev = container_of(resptr - *resptr,
401 struct um_pci_device,
402 resptr[0]);
403 struct {
404 struct virtio_pcidev_msg hdr;
405 u8 data;
406 } msg = {
407 .hdr = {
408 .op = VIRTIO_PCIDEV_OP_CFG_WRITE,
409 .bar = *resptr,
410 .size = size,
411 .addr = offset,
412 },
413 .data = value,
414 };
415
416 um_pci_send_cmd(dev, &msg.hdr, sizeof(msg), NULL, 0, NULL, 0);
417 }
418
419 static const struct logic_iomem_ops um_pci_device_bar_ops = {
420 .read = um_pci_bar_read,
421 .write = um_pci_bar_write,
422 .set = um_pci_bar_set,
423 .copy_from = um_pci_bar_copy_from,
424 .copy_to = um_pci_bar_copy_to,
425 };
426
um_pci_map_bus(struct pci_bus * bus,unsigned int devfn,int where)427 static void __iomem *um_pci_map_bus(struct pci_bus *bus, unsigned int devfn,
428 int where)
429 {
430 struct um_pci_device_reg *dev;
431 unsigned int busn = bus->number;
432
433 if (busn > 0)
434 return NULL;
435
436 /* not allowing functions for now ... */
437 if (devfn % 8)
438 return NULL;
439
440 if (devfn / 8 >= ARRAY_SIZE(um_pci_devices))
441 return NULL;
442
443 dev = &um_pci_devices[devfn / 8];
444 if (!dev)
445 return NULL;
446
447 return (void __iomem *)((unsigned long)dev->iomem + where);
448 }
449
450 static struct pci_ops um_pci_ops = {
451 .map_bus = um_pci_map_bus,
452 .read = pci_generic_config_read,
453 .write = pci_generic_config_write,
454 };
455
um_pci_rescan(void)456 static void um_pci_rescan(void)
457 {
458 pci_lock_rescan_remove();
459 pci_rescan_bus(bridge->bus);
460 pci_unlock_rescan_remove();
461 }
462
um_pci_irq_vq_addbuf(struct virtqueue * vq,void * buf,bool kick)463 static void um_pci_irq_vq_addbuf(struct virtqueue *vq, void *buf, bool kick)
464 {
465 struct scatterlist sg[1];
466
467 sg_init_one(sg, buf, MAX_IRQ_MSG_SIZE);
468 if (virtqueue_add_inbuf(vq, sg, 1, buf, GFP_ATOMIC))
469 kfree(buf);
470 else if (kick)
471 virtqueue_kick(vq);
472 }
473
um_pci_handle_irq_message(struct virtqueue * vq,struct virtio_pcidev_msg * msg)474 static void um_pci_handle_irq_message(struct virtqueue *vq,
475 struct virtio_pcidev_msg *msg)
476 {
477 struct virtio_device *vdev = vq->vdev;
478 struct um_pci_device *dev = vdev->priv;
479
480 /* we should properly chain interrupts, but on ARCH=um we don't care */
481
482 switch (msg->op) {
483 case VIRTIO_PCIDEV_OP_INT:
484 generic_handle_irq(dev->irq);
485 break;
486 case VIRTIO_PCIDEV_OP_MSI:
487 /* our MSI message is just the interrupt number */
488 if (msg->size == sizeof(u32))
489 generic_handle_irq(le32_to_cpup((void *)msg->data));
490 else
491 generic_handle_irq(le16_to_cpup((void *)msg->data));
492 break;
493 case VIRTIO_PCIDEV_OP_PME:
494 /* nothing to do - we already woke up due to the message */
495 break;
496 default:
497 dev_err(&vdev->dev, "unexpected virt-pci message %d\n", msg->op);
498 break;
499 }
500 }
501
um_pci_cmd_vq_cb(struct virtqueue * vq)502 static void um_pci_cmd_vq_cb(struct virtqueue *vq)
503 {
504 struct virtio_device *vdev = vq->vdev;
505 struct um_pci_device *dev = vdev->priv;
506 void *cmd;
507 int len;
508
509 if (test_bit(UM_PCI_STAT_WAITING, &dev->status))
510 return;
511
512 while ((cmd = virtqueue_get_buf(vq, &len))) {
513 if (WARN_ON(HANDLE_IS_NO_FREE(cmd)))
514 continue;
515 kfree(cmd);
516 }
517 }
518
um_pci_irq_vq_cb(struct virtqueue * vq)519 static void um_pci_irq_vq_cb(struct virtqueue *vq)
520 {
521 struct virtio_pcidev_msg *msg;
522 int len;
523
524 while ((msg = virtqueue_get_buf(vq, &len))) {
525 if (len >= sizeof(*msg))
526 um_pci_handle_irq_message(vq, msg);
527
528 /* recycle the message buffer */
529 um_pci_irq_vq_addbuf(vq, msg, true);
530 }
531 }
532
um_pci_init_vqs(struct um_pci_device * dev)533 static int um_pci_init_vqs(struct um_pci_device *dev)
534 {
535 struct virtqueue *vqs[2];
536 static const char *const names[2] = { "cmd", "irq" };
537 vq_callback_t *cbs[2] = { um_pci_cmd_vq_cb, um_pci_irq_vq_cb };
538 int err, i;
539
540 err = virtio_find_vqs(dev->vdev, 2, vqs, cbs, names, NULL);
541 if (err)
542 return err;
543
544 dev->cmd_vq = vqs[0];
545 dev->irq_vq = vqs[1];
546
547 for (i = 0; i < NUM_IRQ_MSGS; i++) {
548 void *msg = kzalloc(MAX_IRQ_MSG_SIZE, GFP_KERNEL);
549
550 if (msg)
551 um_pci_irq_vq_addbuf(dev->irq_vq, msg, false);
552 }
553
554 virtqueue_kick(dev->irq_vq);
555
556 return 0;
557 }
558
um_pci_virtio_probe(struct virtio_device * vdev)559 static int um_pci_virtio_probe(struct virtio_device *vdev)
560 {
561 struct um_pci_device *dev;
562 int i, free = -1;
563 int err = -ENOSPC;
564
565 dev = kzalloc(sizeof(*dev), GFP_KERNEL);
566 if (!dev)
567 return -ENOMEM;
568
569 dev->vdev = vdev;
570 vdev->priv = dev;
571
572 mutex_lock(&um_pci_mtx);
573 for (i = 0; i < MAX_DEVICES; i++) {
574 if (um_pci_devices[i].dev)
575 continue;
576 free = i;
577 break;
578 }
579
580 if (free < 0)
581 goto error;
582
583 err = um_pci_init_vqs(dev);
584 if (err)
585 goto error;
586
587 dev->irq = irq_alloc_desc(numa_node_id());
588 if (dev->irq < 0) {
589 err = dev->irq;
590 goto error;
591 }
592 um_pci_devices[free].dev = dev;
593 vdev->priv = dev;
594
595 mutex_unlock(&um_pci_mtx);
596
597 device_set_wakeup_enable(&vdev->dev, true);
598
599 /*
600 * In order to do suspend-resume properly, don't allow VQs
601 * to be suspended.
602 */
603 virtio_uml_set_no_vq_suspend(vdev, true);
604
605 um_pci_rescan();
606 return 0;
607 error:
608 mutex_unlock(&um_pci_mtx);
609 kfree(dev);
610 return err;
611 }
612
um_pci_virtio_remove(struct virtio_device * vdev)613 static void um_pci_virtio_remove(struct virtio_device *vdev)
614 {
615 struct um_pci_device *dev = vdev->priv;
616 int i;
617
618 /* Stop all virtqueues */
619 vdev->config->reset(vdev);
620 vdev->config->del_vqs(vdev);
621
622 device_set_wakeup_enable(&vdev->dev, false);
623
624 mutex_lock(&um_pci_mtx);
625 for (i = 0; i < MAX_DEVICES; i++) {
626 if (um_pci_devices[i].dev != dev)
627 continue;
628 um_pci_devices[i].dev = NULL;
629 irq_free_desc(dev->irq);
630 }
631 mutex_unlock(&um_pci_mtx);
632
633 um_pci_rescan();
634
635 kfree(dev);
636 }
637
638 static struct virtio_device_id id_table[] = {
639 { CONFIG_UML_PCI_OVER_VIRTIO_DEVICE_ID, VIRTIO_DEV_ANY_ID },
640 { 0 },
641 };
642 MODULE_DEVICE_TABLE(virtio, id_table);
643
644 static struct virtio_driver um_pci_virtio_driver = {
645 .driver.name = "virtio-pci",
646 .driver.owner = THIS_MODULE,
647 .id_table = id_table,
648 .probe = um_pci_virtio_probe,
649 .remove = um_pci_virtio_remove,
650 };
651
652 static struct resource virt_cfgspace_resource = {
653 .name = "PCI config space",
654 .start = 0xf0000000 - MAX_DEVICES * CFG_SPACE_SIZE,
655 .end = 0xf0000000 - 1,
656 .flags = IORESOURCE_MEM,
657 };
658
um_pci_map_cfgspace(unsigned long offset,size_t size,const struct logic_iomem_ops ** ops,void ** priv)659 static long um_pci_map_cfgspace(unsigned long offset, size_t size,
660 const struct logic_iomem_ops **ops,
661 void **priv)
662 {
663 if (WARN_ON(size > CFG_SPACE_SIZE || offset % CFG_SPACE_SIZE))
664 return -EINVAL;
665
666 if (offset / CFG_SPACE_SIZE < MAX_DEVICES) {
667 *ops = &um_pci_device_cfgspace_ops;
668 *priv = &um_pci_devices[offset / CFG_SPACE_SIZE];
669 return 0;
670 }
671
672 WARN(1, "cannot map offset 0x%lx/0x%zx\n", offset, size);
673 return -ENOENT;
674 }
675
676 static const struct logic_iomem_region_ops um_pci_cfgspace_ops = {
677 .map = um_pci_map_cfgspace,
678 };
679
680 static struct resource virt_iomem_resource = {
681 .name = "PCI iomem",
682 .start = 0xf0000000,
683 .end = 0xffffffff,
684 .flags = IORESOURCE_MEM,
685 };
686
687 struct um_pci_map_iomem_data {
688 unsigned long offset;
689 size_t size;
690 const struct logic_iomem_ops **ops;
691 void **priv;
692 long ret;
693 };
694
um_pci_map_iomem_walk(struct pci_dev * pdev,void * _data)695 static int um_pci_map_iomem_walk(struct pci_dev *pdev, void *_data)
696 {
697 struct um_pci_map_iomem_data *data = _data;
698 struct um_pci_device_reg *reg = &um_pci_devices[pdev->devfn / 8];
699 struct um_pci_device *dev;
700 int i;
701
702 if (!reg->dev)
703 return 0;
704
705 for (i = 0; i < ARRAY_SIZE(dev->resptr); i++) {
706 struct resource *r = &pdev->resource[i];
707
708 if ((r->flags & IORESOURCE_TYPE_BITS) != IORESOURCE_MEM)
709 continue;
710
711 /*
712 * must be the whole or part of the resource,
713 * not allowed to only overlap
714 */
715 if (data->offset < r->start || data->offset > r->end)
716 continue;
717 if (data->offset + data->size - 1 > r->end)
718 continue;
719
720 dev = reg->dev;
721 *data->ops = &um_pci_device_bar_ops;
722 dev->resptr[i] = i;
723 *data->priv = &dev->resptr[i];
724 data->ret = data->offset - r->start;
725
726 /* no need to continue */
727 return 1;
728 }
729
730 return 0;
731 }
732
um_pci_map_iomem(unsigned long offset,size_t size,const struct logic_iomem_ops ** ops,void ** priv)733 static long um_pci_map_iomem(unsigned long offset, size_t size,
734 const struct logic_iomem_ops **ops,
735 void **priv)
736 {
737 struct um_pci_map_iomem_data data = {
738 /* we want the full address here */
739 .offset = offset + virt_iomem_resource.start,
740 .size = size,
741 .ops = ops,
742 .priv = priv,
743 .ret = -ENOENT,
744 };
745
746 pci_walk_bus(bridge->bus, um_pci_map_iomem_walk, &data);
747 return data.ret;
748 }
749
750 static const struct logic_iomem_region_ops um_pci_iomem_ops = {
751 .map = um_pci_map_iomem,
752 };
753
um_pci_compose_msi_msg(struct irq_data * data,struct msi_msg * msg)754 static void um_pci_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
755 {
756 /*
757 * This is a very low address and not actually valid 'physical' memory
758 * in UML, so we can simply map MSI(-X) vectors to there, it cannot be
759 * legitimately written to by the device in any other way.
760 * We use the (virtual) IRQ number here as the message to simplify the
761 * code that receives the message, where for now we simply trust the
762 * device to send the correct message.
763 */
764 msg->address_hi = 0;
765 msg->address_lo = 0xa0000;
766 msg->data = data->irq;
767 }
768
769 static struct irq_chip um_pci_msi_bottom_irq_chip = {
770 .name = "UM virtio MSI",
771 .irq_compose_msi_msg = um_pci_compose_msi_msg,
772 };
773
um_pci_inner_domain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * args)774 static int um_pci_inner_domain_alloc(struct irq_domain *domain,
775 unsigned int virq, unsigned int nr_irqs,
776 void *args)
777 {
778 unsigned long bit;
779
780 WARN_ON(nr_irqs != 1);
781
782 mutex_lock(&um_pci_mtx);
783 bit = find_first_zero_bit(um_pci_msi_used, MAX_MSI_VECTORS);
784 if (bit >= MAX_MSI_VECTORS) {
785 mutex_unlock(&um_pci_mtx);
786 return -ENOSPC;
787 }
788
789 set_bit(bit, um_pci_msi_used);
790 mutex_unlock(&um_pci_mtx);
791
792 irq_domain_set_info(domain, virq, bit, &um_pci_msi_bottom_irq_chip,
793 domain->host_data, handle_simple_irq,
794 NULL, NULL);
795
796 return 0;
797 }
798
um_pci_inner_domain_free(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs)799 static void um_pci_inner_domain_free(struct irq_domain *domain,
800 unsigned int virq, unsigned int nr_irqs)
801 {
802 struct irq_data *d = irq_domain_get_irq_data(domain, virq);
803
804 mutex_lock(&um_pci_mtx);
805
806 if (!test_bit(d->hwirq, um_pci_msi_used))
807 pr_err("trying to free unused MSI#%lu\n", d->hwirq);
808 else
809 __clear_bit(d->hwirq, um_pci_msi_used);
810
811 mutex_unlock(&um_pci_mtx);
812 }
813
814 static const struct irq_domain_ops um_pci_inner_domain_ops = {
815 .alloc = um_pci_inner_domain_alloc,
816 .free = um_pci_inner_domain_free,
817 };
818
819 static struct irq_chip um_pci_msi_irq_chip = {
820 .name = "UM virtio PCIe MSI",
821 .irq_mask = pci_msi_mask_irq,
822 .irq_unmask = pci_msi_unmask_irq,
823 };
824
825 static struct msi_domain_info um_pci_msi_domain_info = {
826 .flags = MSI_FLAG_USE_DEF_DOM_OPS |
827 MSI_FLAG_USE_DEF_CHIP_OPS |
828 MSI_FLAG_PCI_MSIX,
829 .chip = &um_pci_msi_irq_chip,
830 };
831
832 static struct resource busn_resource = {
833 .name = "PCI busn",
834 .start = 0,
835 .end = 0,
836 .flags = IORESOURCE_BUS,
837 };
838
um_pci_map_irq(const struct pci_dev * pdev,u8 slot,u8 pin)839 static int um_pci_map_irq(const struct pci_dev *pdev, u8 slot, u8 pin)
840 {
841 struct um_pci_device_reg *reg = &um_pci_devices[pdev->devfn / 8];
842
843 if (WARN_ON(!reg->dev))
844 return -EINVAL;
845
846 /* Yes, we map all pins to the same IRQ ... doesn't matter for now. */
847 return reg->dev->irq;
848 }
849
pci_root_bus_fwnode(struct pci_bus * bus)850 void *pci_root_bus_fwnode(struct pci_bus *bus)
851 {
852 return um_pci_fwnode;
853 }
854
um_pci_init(void)855 static int um_pci_init(void)
856 {
857 int err, i;
858
859 WARN_ON(logic_iomem_add_region(&virt_cfgspace_resource,
860 &um_pci_cfgspace_ops));
861 WARN_ON(logic_iomem_add_region(&virt_iomem_resource,
862 &um_pci_iomem_ops));
863
864 if (WARN(CONFIG_UML_PCI_OVER_VIRTIO_DEVICE_ID < 0,
865 "No virtio device ID configured for PCI - no PCI support\n"))
866 return 0;
867
868 um_pci_msg_bufs = alloc_percpu(struct um_pci_message_buffer);
869 if (!um_pci_msg_bufs)
870 return -ENOMEM;
871
872 bridge = pci_alloc_host_bridge(0);
873 if (!bridge) {
874 err = -ENOMEM;
875 goto free;
876 }
877
878 um_pci_fwnode = irq_domain_alloc_named_fwnode("um-pci");
879 if (!um_pci_fwnode) {
880 err = -ENOMEM;
881 goto free;
882 }
883
884 um_pci_inner_domain = __irq_domain_add(um_pci_fwnode, MAX_MSI_VECTORS,
885 MAX_MSI_VECTORS, 0,
886 &um_pci_inner_domain_ops, NULL);
887 if (!um_pci_inner_domain) {
888 err = -ENOMEM;
889 goto free;
890 }
891
892 um_pci_msi_domain = pci_msi_create_irq_domain(um_pci_fwnode,
893 &um_pci_msi_domain_info,
894 um_pci_inner_domain);
895 if (!um_pci_msi_domain) {
896 err = -ENOMEM;
897 goto free;
898 }
899
900 pci_add_resource(&bridge->windows, &virt_iomem_resource);
901 pci_add_resource(&bridge->windows, &busn_resource);
902 bridge->ops = &um_pci_ops;
903 bridge->map_irq = um_pci_map_irq;
904
905 for (i = 0; i < MAX_DEVICES; i++) {
906 resource_size_t start;
907
908 start = virt_cfgspace_resource.start + i * CFG_SPACE_SIZE;
909 um_pci_devices[i].iomem = ioremap(start, CFG_SPACE_SIZE);
910 if (WARN(!um_pci_devices[i].iomem, "failed to map %d\n", i)) {
911 err = -ENOMEM;
912 goto free;
913 }
914 }
915
916 err = pci_host_probe(bridge);
917 if (err)
918 goto free;
919
920 err = register_virtio_driver(&um_pci_virtio_driver);
921 if (err)
922 goto free;
923 return 0;
924 free:
925 if (um_pci_inner_domain)
926 irq_domain_remove(um_pci_inner_domain);
927 if (um_pci_fwnode)
928 irq_domain_free_fwnode(um_pci_fwnode);
929 if (bridge) {
930 pci_free_resource_list(&bridge->windows);
931 pci_free_host_bridge(bridge);
932 }
933 free_percpu(um_pci_msg_bufs);
934 return err;
935 }
936 module_init(um_pci_init);
937
um_pci_exit(void)938 static void um_pci_exit(void)
939 {
940 unregister_virtio_driver(&um_pci_virtio_driver);
941 irq_domain_remove(um_pci_msi_domain);
942 irq_domain_remove(um_pci_inner_domain);
943 pci_free_resource_list(&bridge->windows);
944 pci_free_host_bridge(bridge);
945 free_percpu(um_pci_msg_bufs);
946 }
947 module_exit(um_pci_exit);
948