1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2 /* Copyright(c) 2014 - 2020 Intel Corporation */
3 #include <linux/kernel.h>
4 #include <linux/init.h>
5 #include <linux/types.h>
6 #include <linux/pci.h>
7 #include <linux/slab.h>
8 #include <linux/errno.h>
9 #include <linux/interrupt.h>
10 #include <linux/workqueue.h>
11 #include "adf_accel_devices.h"
12 #include "adf_common_drv.h"
13 #include "adf_cfg.h"
14 #include "adf_cfg_strings.h"
15 #include "adf_cfg_common.h"
16 #include "adf_transport_access_macros.h"
17 #include "adf_transport_internal.h"
18 #include "adf_pf2vf_msg.h"
19
20 #define ADF_VINTSOU_OFFSET 0x204
21 #define ADF_VINTMSK_OFFSET 0x208
22 #define ADF_VINTSOU_BUN BIT(0)
23 #define ADF_VINTSOU_PF2VF BIT(1)
24
25 static struct workqueue_struct *adf_vf_stop_wq;
26
27 struct adf_vf_stop_data {
28 struct adf_accel_dev *accel_dev;
29 struct work_struct work;
30 };
31
adf_enable_pf2vf_interrupts(struct adf_accel_dev * accel_dev)32 void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev)
33 {
34 struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev;
35 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
36 void __iomem *pmisc_bar_addr =
37 pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)].virt_addr;
38
39 ADF_CSR_WR(pmisc_bar_addr, ADF_VINTMSK_OFFSET, 0x0);
40 }
41
adf_disable_pf2vf_interrupts(struct adf_accel_dev * accel_dev)42 void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev)
43 {
44 struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev;
45 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
46 void __iomem *pmisc_bar_addr =
47 pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)].virt_addr;
48
49 ADF_CSR_WR(pmisc_bar_addr, ADF_VINTMSK_OFFSET, 0x2);
50 }
51 EXPORT_SYMBOL_GPL(adf_disable_pf2vf_interrupts);
52
adf_enable_msi(struct adf_accel_dev * accel_dev)53 static int adf_enable_msi(struct adf_accel_dev *accel_dev)
54 {
55 struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;
56 int stat = pci_alloc_irq_vectors(pci_dev_info->pci_dev, 1, 1,
57 PCI_IRQ_MSI);
58 if (unlikely(stat < 0)) {
59 dev_err(&GET_DEV(accel_dev),
60 "Failed to enable MSI interrupt: %d\n", stat);
61 return stat;
62 }
63
64 return 0;
65 }
66
adf_disable_msi(struct adf_accel_dev * accel_dev)67 static void adf_disable_msi(struct adf_accel_dev *accel_dev)
68 {
69 struct pci_dev *pdev = accel_to_pci_dev(accel_dev);
70
71 pci_free_irq_vectors(pdev);
72 }
73
adf_dev_stop_async(struct work_struct * work)74 static void adf_dev_stop_async(struct work_struct *work)
75 {
76 struct adf_vf_stop_data *stop_data =
77 container_of(work, struct adf_vf_stop_data, work);
78 struct adf_accel_dev *accel_dev = stop_data->accel_dev;
79
80 adf_dev_stop(accel_dev);
81 adf_dev_shutdown(accel_dev);
82
83 /* Re-enable PF2VF interrupts */
84 adf_enable_pf2vf_interrupts(accel_dev);
85 kfree(stop_data);
86 }
87
adf_pf2vf_bh_handler(void * data)88 static void adf_pf2vf_bh_handler(void *data)
89 {
90 struct adf_accel_dev *accel_dev = data;
91 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
92 struct adf_bar *pmisc =
93 &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)];
94 void __iomem *pmisc_bar_addr = pmisc->virt_addr;
95 u32 msg;
96
97 /* Read the message from PF */
98 msg = ADF_CSR_RD(pmisc_bar_addr, hw_data->get_pf2vf_offset(0));
99 if (!(msg & ADF_PF2VF_INT)) {
100 dev_info(&GET_DEV(accel_dev),
101 "Spurious PF2VF interrupt, msg %X. Ignored\n", msg);
102 goto out;
103 }
104
105 if (!(msg & ADF_PF2VF_MSGORIGIN_SYSTEM))
106 /* Ignore legacy non-system (non-kernel) PF2VF messages */
107 goto err;
108
109 switch ((msg & ADF_PF2VF_MSGTYPE_MASK) >> ADF_PF2VF_MSGTYPE_SHIFT) {
110 case ADF_PF2VF_MSGTYPE_RESTARTING: {
111 struct adf_vf_stop_data *stop_data;
112
113 dev_dbg(&GET_DEV(accel_dev),
114 "Restarting msg received from PF 0x%x\n", msg);
115
116 clear_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status);
117
118 stop_data = kzalloc(sizeof(*stop_data), GFP_ATOMIC);
119 if (!stop_data) {
120 dev_err(&GET_DEV(accel_dev),
121 "Couldn't schedule stop for vf_%d\n",
122 accel_dev->accel_id);
123 return;
124 }
125 stop_data->accel_dev = accel_dev;
126 INIT_WORK(&stop_data->work, adf_dev_stop_async);
127 queue_work(adf_vf_stop_wq, &stop_data->work);
128 /* To ack, clear the PF2VFINT bit */
129 msg &= ~ADF_PF2VF_INT;
130 ADF_CSR_WR(pmisc_bar_addr, hw_data->get_pf2vf_offset(0), msg);
131 return;
132 }
133 case ADF_PF2VF_MSGTYPE_VERSION_RESP:
134 dev_dbg(&GET_DEV(accel_dev),
135 "Version resp received from PF 0x%x\n", msg);
136 accel_dev->vf.pf_version =
137 (msg & ADF_PF2VF_VERSION_RESP_VERS_MASK) >>
138 ADF_PF2VF_VERSION_RESP_VERS_SHIFT;
139 accel_dev->vf.compatible =
140 (msg & ADF_PF2VF_VERSION_RESP_RESULT_MASK) >>
141 ADF_PF2VF_VERSION_RESP_RESULT_SHIFT;
142 complete(&accel_dev->vf.iov_msg_completion);
143 break;
144 default:
145 goto err;
146 }
147
148 /* To ack, clear the PF2VFINT bit */
149 msg &= ~ADF_PF2VF_INT;
150 ADF_CSR_WR(pmisc_bar_addr, hw_data->get_pf2vf_offset(0), msg);
151
152 out:
153 /* Re-enable PF2VF interrupts */
154 adf_enable_pf2vf_interrupts(accel_dev);
155 return;
156 err:
157 dev_err(&GET_DEV(accel_dev),
158 "Unknown message from PF (0x%x); leaving PF2VF ints disabled\n",
159 msg);
160 }
161
adf_setup_pf2vf_bh(struct adf_accel_dev * accel_dev)162 static int adf_setup_pf2vf_bh(struct adf_accel_dev *accel_dev)
163 {
164 tasklet_init(&accel_dev->vf.pf2vf_bh_tasklet,
165 (void *)adf_pf2vf_bh_handler, (unsigned long)accel_dev);
166
167 mutex_init(&accel_dev->vf.vf2pf_lock);
168 return 0;
169 }
170
adf_cleanup_pf2vf_bh(struct adf_accel_dev * accel_dev)171 static void adf_cleanup_pf2vf_bh(struct adf_accel_dev *accel_dev)
172 {
173 tasklet_disable(&accel_dev->vf.pf2vf_bh_tasklet);
174 tasklet_kill(&accel_dev->vf.pf2vf_bh_tasklet);
175 mutex_destroy(&accel_dev->vf.vf2pf_lock);
176 }
177
adf_isr(int irq,void * privdata)178 static irqreturn_t adf_isr(int irq, void *privdata)
179 {
180 struct adf_accel_dev *accel_dev = privdata;
181 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
182 struct adf_hw_csr_ops *csr_ops = &hw_data->csr_ops;
183 struct adf_bar *pmisc =
184 &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)];
185 void __iomem *pmisc_bar_addr = pmisc->virt_addr;
186 bool handled = false;
187 u32 v_int, v_mask;
188
189 /* Read VF INT source CSR to determine the source of VF interrupt */
190 v_int = ADF_CSR_RD(pmisc_bar_addr, ADF_VINTSOU_OFFSET);
191
192 /* Read VF INT mask CSR to determine which sources are masked */
193 v_mask = ADF_CSR_RD(pmisc_bar_addr, ADF_VINTMSK_OFFSET);
194
195 /*
196 * Recompute v_int ignoring sources that are masked. This is to
197 * avoid rescheduling the tasklet for interrupts already handled
198 */
199 v_int &= ~v_mask;
200
201 /* Check for PF2VF interrupt */
202 if (v_int & ADF_VINTSOU_PF2VF) {
203 /* Disable PF to VF interrupt */
204 adf_disable_pf2vf_interrupts(accel_dev);
205
206 /* Schedule tasklet to handle interrupt BH */
207 tasklet_hi_schedule(&accel_dev->vf.pf2vf_bh_tasklet);
208 handled = true;
209 }
210
211 /* Check bundle interrupt */
212 if (v_int & ADF_VINTSOU_BUN) {
213 struct adf_etr_data *etr_data = accel_dev->transport;
214 struct adf_etr_bank_data *bank = &etr_data->banks[0];
215
216 /* Disable Flag and Coalesce Ring Interrupts */
217 csr_ops->write_csr_int_flag_and_col(bank->csr_addr,
218 bank->bank_number, 0);
219 tasklet_hi_schedule(&bank->resp_handler);
220 handled = true;
221 }
222
223 return handled ? IRQ_HANDLED : IRQ_NONE;
224 }
225
adf_request_msi_irq(struct adf_accel_dev * accel_dev)226 static int adf_request_msi_irq(struct adf_accel_dev *accel_dev)
227 {
228 struct pci_dev *pdev = accel_to_pci_dev(accel_dev);
229 unsigned int cpu;
230 int ret;
231
232 snprintf(accel_dev->vf.irq_name, ADF_MAX_MSIX_VECTOR_NAME,
233 "qat_%02x:%02d.%02d", pdev->bus->number, PCI_SLOT(pdev->devfn),
234 PCI_FUNC(pdev->devfn));
235 ret = request_irq(pdev->irq, adf_isr, 0, accel_dev->vf.irq_name,
236 (void *)accel_dev);
237 if (ret) {
238 dev_err(&GET_DEV(accel_dev), "failed to enable irq for %s\n",
239 accel_dev->vf.irq_name);
240 return ret;
241 }
242 cpu = accel_dev->accel_id % num_online_cpus();
243 irq_set_affinity_hint(pdev->irq, get_cpu_mask(cpu));
244 accel_dev->vf.irq_enabled = true;
245
246 return ret;
247 }
248
adf_setup_bh(struct adf_accel_dev * accel_dev)249 static int adf_setup_bh(struct adf_accel_dev *accel_dev)
250 {
251 struct adf_etr_data *priv_data = accel_dev->transport;
252
253 tasklet_init(&priv_data->banks[0].resp_handler, adf_response_handler,
254 (unsigned long)priv_data->banks);
255 return 0;
256 }
257
adf_cleanup_bh(struct adf_accel_dev * accel_dev)258 static void adf_cleanup_bh(struct adf_accel_dev *accel_dev)
259 {
260 struct adf_etr_data *priv_data = accel_dev->transport;
261
262 tasklet_disable(&priv_data->banks[0].resp_handler);
263 tasklet_kill(&priv_data->banks[0].resp_handler);
264 }
265
266 /**
267 * adf_vf_isr_resource_free() - Free IRQ for acceleration device
268 * @accel_dev: Pointer to acceleration device.
269 *
270 * Function frees interrupts for acceleration device virtual function.
271 */
adf_vf_isr_resource_free(struct adf_accel_dev * accel_dev)272 void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev)
273 {
274 struct pci_dev *pdev = accel_to_pci_dev(accel_dev);
275
276 if (accel_dev->vf.irq_enabled) {
277 irq_set_affinity_hint(pdev->irq, NULL);
278 free_irq(pdev->irq, accel_dev);
279 }
280 adf_cleanup_bh(accel_dev);
281 adf_cleanup_pf2vf_bh(accel_dev);
282 adf_disable_msi(accel_dev);
283 }
284 EXPORT_SYMBOL_GPL(adf_vf_isr_resource_free);
285
286 /**
287 * adf_vf_isr_resource_alloc() - Allocate IRQ for acceleration device
288 * @accel_dev: Pointer to acceleration device.
289 *
290 * Function allocates interrupts for acceleration device virtual function.
291 *
292 * Return: 0 on success, error code otherwise.
293 */
adf_vf_isr_resource_alloc(struct adf_accel_dev * accel_dev)294 int adf_vf_isr_resource_alloc(struct adf_accel_dev *accel_dev)
295 {
296 if (adf_enable_msi(accel_dev))
297 goto err_out;
298
299 if (adf_setup_pf2vf_bh(accel_dev))
300 goto err_disable_msi;
301
302 if (adf_setup_bh(accel_dev))
303 goto err_cleanup_pf2vf_bh;
304
305 if (adf_request_msi_irq(accel_dev))
306 goto err_cleanup_bh;
307
308 return 0;
309
310 err_cleanup_bh:
311 adf_cleanup_bh(accel_dev);
312
313 err_cleanup_pf2vf_bh:
314 adf_cleanup_pf2vf_bh(accel_dev);
315
316 err_disable_msi:
317 adf_disable_msi(accel_dev);
318
319 err_out:
320 return -EFAULT;
321 }
322 EXPORT_SYMBOL_GPL(adf_vf_isr_resource_alloc);
323
324 /**
325 * adf_flush_vf_wq() - Flush workqueue for VF
326 * @accel_dev: Pointer to acceleration device.
327 *
328 * Function disables the PF/VF interrupts on the VF so that no new messages
329 * are received and flushes the workqueue 'adf_vf_stop_wq'.
330 *
331 * Return: void.
332 */
adf_flush_vf_wq(struct adf_accel_dev * accel_dev)333 void adf_flush_vf_wq(struct adf_accel_dev *accel_dev)
334 {
335 adf_disable_pf2vf_interrupts(accel_dev);
336
337 flush_workqueue(adf_vf_stop_wq);
338 }
339 EXPORT_SYMBOL_GPL(adf_flush_vf_wq);
340
341 /**
342 * adf_init_vf_wq() - Init workqueue for VF
343 *
344 * Function init workqueue 'adf_vf_stop_wq' for VF.
345 *
346 * Return: 0 on success, error code otherwise.
347 */
adf_init_vf_wq(void)348 int __init adf_init_vf_wq(void)
349 {
350 adf_vf_stop_wq = alloc_workqueue("adf_vf_stop_wq", WQ_MEM_RECLAIM, 0);
351
352 return !adf_vf_stop_wq ? -EFAULT : 0;
353 }
354
adf_exit_vf_wq(void)355 void adf_exit_vf_wq(void)
356 {
357 if (adf_vf_stop_wq)
358 destroy_workqueue(adf_vf_stop_wq);
359
360 adf_vf_stop_wq = NULL;
361 }
362