1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2 /* Copyright(c) 2014 - 2020 Intel Corporation */
3 #include <linux/kernel.h>
4 #include <linux/init.h>
5 #include <linux/types.h>
6 #include <linux/pci.h>
7 #include <linux/slab.h>
8 #include <linux/errno.h>
9 #include <linux/interrupt.h>
10 #include "adf_accel_devices.h"
11 #include "adf_common_drv.h"
12 #include "adf_cfg.h"
13 #include "adf_cfg_strings.h"
14 #include "adf_cfg_common.h"
15 #include "adf_transport_access_macros.h"
16 #include "adf_transport_internal.h"
17
18 #define ADF_MAX_NUM_VFS 32
19
adf_enable_msix(struct adf_accel_dev * accel_dev)20 static int adf_enable_msix(struct adf_accel_dev *accel_dev)
21 {
22 struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;
23 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
24 u32 msix_num_entries = hw_data->num_banks + 1;
25 int ret;
26
27 if (hw_data->set_msix_rttable)
28 hw_data->set_msix_rttable(accel_dev);
29
30 ret = pci_alloc_irq_vectors(pci_dev_info->pci_dev, msix_num_entries,
31 msix_num_entries, PCI_IRQ_MSIX);
32 if (unlikely(ret < 0)) {
33 dev_err(&GET_DEV(accel_dev),
34 "Failed to allocate %d MSI-X vectors\n",
35 msix_num_entries);
36 return ret;
37 }
38 return 0;
39 }
40
adf_disable_msix(struct adf_accel_pci * pci_dev_info)41 static void adf_disable_msix(struct adf_accel_pci *pci_dev_info)
42 {
43 pci_free_irq_vectors(pci_dev_info->pci_dev);
44 }
45
adf_msix_isr_bundle(int irq,void * bank_ptr)46 static irqreturn_t adf_msix_isr_bundle(int irq, void *bank_ptr)
47 {
48 struct adf_etr_bank_data *bank = bank_ptr;
49 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev);
50
51 csr_ops->write_csr_int_flag_and_col(bank->csr_addr, bank->bank_number,
52 0);
53 tasklet_hi_schedule(&bank->resp_handler);
54 return IRQ_HANDLED;
55 }
56
adf_msix_isr_ae(int irq,void * dev_ptr)57 static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr)
58 {
59 struct adf_accel_dev *accel_dev = dev_ptr;
60
61 #ifdef CONFIG_PCI_IOV
62 /* If SR-IOV is enabled (vf_info is non-NULL), check for VF->PF ints */
63 if (accel_dev->pf.vf_info) {
64 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
65 struct adf_bar *pmisc =
66 &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)];
67 void __iomem *pmisc_addr = pmisc->virt_addr;
68 unsigned long vf_mask;
69
70 /* Get the interrupt sources triggered by VFs */
71 vf_mask = hw_data->get_vf2pf_sources(pmisc_addr);
72
73 if (vf_mask) {
74 struct adf_accel_vf_info *vf_info;
75 bool irq_handled = false;
76 int i;
77
78 /* Disable VF2PF interrupts for VFs with pending ints */
79 adf_disable_vf2pf_interrupts_irq(accel_dev, vf_mask);
80
81 /*
82 * Handle VF2PF interrupt unless the VF is malicious and
83 * is attempting to flood the host OS with VF2PF interrupts.
84 */
85 for_each_set_bit(i, &vf_mask, ADF_MAX_NUM_VFS) {
86 vf_info = accel_dev->pf.vf_info + i;
87
88 if (!__ratelimit(&vf_info->vf2pf_ratelimit)) {
89 dev_info(&GET_DEV(accel_dev),
90 "Too many ints from VF%d\n",
91 vf_info->vf_nr + 1);
92 continue;
93 }
94
95 adf_schedule_vf2pf_handler(vf_info);
96 irq_handled = true;
97 }
98
99 if (irq_handled)
100 return IRQ_HANDLED;
101 }
102 }
103 #endif /* CONFIG_PCI_IOV */
104
105 dev_dbg(&GET_DEV(accel_dev), "qat_dev%d spurious AE interrupt\n",
106 accel_dev->accel_id);
107
108 return IRQ_NONE;
109 }
110
adf_free_irqs(struct adf_accel_dev * accel_dev)111 static void adf_free_irqs(struct adf_accel_dev *accel_dev)
112 {
113 struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;
114 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
115 struct adf_irq *irqs = pci_dev_info->msix_entries.irqs;
116 struct adf_etr_data *etr_data = accel_dev->transport;
117 int clust_irq = hw_data->num_banks;
118 int irq, i = 0;
119
120 if (pci_dev_info->msix_entries.num_entries > 1) {
121 for (i = 0; i < hw_data->num_banks; i++) {
122 if (irqs[i].enabled) {
123 irq = pci_irq_vector(pci_dev_info->pci_dev, i);
124 irq_set_affinity_hint(irq, NULL);
125 free_irq(irq, &etr_data->banks[i]);
126 }
127 }
128 }
129
130 if (irqs[i].enabled) {
131 irq = pci_irq_vector(pci_dev_info->pci_dev, clust_irq);
132 free_irq(irq, accel_dev);
133 }
134 }
135
adf_request_irqs(struct adf_accel_dev * accel_dev)136 static int adf_request_irqs(struct adf_accel_dev *accel_dev)
137 {
138 struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;
139 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
140 struct adf_irq *irqs = pci_dev_info->msix_entries.irqs;
141 struct adf_etr_data *etr_data = accel_dev->transport;
142 int clust_irq = hw_data->num_banks;
143 int ret, irq, i = 0;
144 char *name;
145
146 /* Request msix irq for all banks unless SR-IOV enabled */
147 if (!accel_dev->pf.vf_info) {
148 for (i = 0; i < hw_data->num_banks; i++) {
149 struct adf_etr_bank_data *bank = &etr_data->banks[i];
150 unsigned int cpu, cpus = num_online_cpus();
151
152 name = irqs[i].name;
153 snprintf(name, ADF_MAX_MSIX_VECTOR_NAME,
154 "qat%d-bundle%d", accel_dev->accel_id, i);
155 irq = pci_irq_vector(pci_dev_info->pci_dev, i);
156 if (unlikely(irq < 0)) {
157 dev_err(&GET_DEV(accel_dev),
158 "Failed to get IRQ number of device vector %d - %s\n",
159 i, name);
160 ret = irq;
161 goto err;
162 }
163 ret = request_irq(irq, adf_msix_isr_bundle, 0,
164 &name[0], bank);
165 if (ret) {
166 dev_err(&GET_DEV(accel_dev),
167 "Failed to allocate IRQ %d for %s\n",
168 irq, name);
169 goto err;
170 }
171
172 cpu = ((accel_dev->accel_id * hw_data->num_banks) +
173 i) % cpus;
174 irq_set_affinity_hint(irq, get_cpu_mask(cpu));
175 irqs[i].enabled = true;
176 }
177 }
178
179 /* Request msix irq for AE */
180 name = irqs[i].name;
181 snprintf(name, ADF_MAX_MSIX_VECTOR_NAME,
182 "qat%d-ae-cluster", accel_dev->accel_id);
183 irq = pci_irq_vector(pci_dev_info->pci_dev, clust_irq);
184 if (unlikely(irq < 0)) {
185 dev_err(&GET_DEV(accel_dev),
186 "Failed to get IRQ number of device vector %d - %s\n",
187 i, name);
188 ret = irq;
189 goto err;
190 }
191 ret = request_irq(irq, adf_msix_isr_ae, 0, &name[0], accel_dev);
192 if (ret) {
193 dev_err(&GET_DEV(accel_dev),
194 "Failed to allocate IRQ %d for %s\n", irq, name);
195 goto err;
196 }
197 irqs[i].enabled = true;
198 return ret;
199 err:
200 adf_free_irqs(accel_dev);
201 return ret;
202 }
203
adf_isr_alloc_msix_vectors_data(struct adf_accel_dev * accel_dev)204 static int adf_isr_alloc_msix_vectors_data(struct adf_accel_dev *accel_dev)
205 {
206 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
207 u32 msix_num_entries = 1;
208 struct adf_irq *irqs;
209
210 /* If SR-IOV is disabled (vf_info is NULL), add entries for each bank */
211 if (!accel_dev->pf.vf_info)
212 msix_num_entries += hw_data->num_banks;
213
214 irqs = kzalloc_node(msix_num_entries * sizeof(*irqs),
215 GFP_KERNEL, dev_to_node(&GET_DEV(accel_dev)));
216 if (!irqs)
217 return -ENOMEM;
218
219 accel_dev->accel_pci_dev.msix_entries.num_entries = msix_num_entries;
220 accel_dev->accel_pci_dev.msix_entries.irqs = irqs;
221 return 0;
222 }
223
adf_isr_free_msix_vectors_data(struct adf_accel_dev * accel_dev)224 static void adf_isr_free_msix_vectors_data(struct adf_accel_dev *accel_dev)
225 {
226 kfree(accel_dev->accel_pci_dev.msix_entries.irqs);
227 accel_dev->accel_pci_dev.msix_entries.irqs = NULL;
228 }
229
adf_setup_bh(struct adf_accel_dev * accel_dev)230 static int adf_setup_bh(struct adf_accel_dev *accel_dev)
231 {
232 struct adf_etr_data *priv_data = accel_dev->transport;
233 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
234 int i;
235
236 for (i = 0; i < hw_data->num_banks; i++)
237 tasklet_init(&priv_data->banks[i].resp_handler,
238 adf_response_handler,
239 (unsigned long)&priv_data->banks[i]);
240 return 0;
241 }
242
adf_cleanup_bh(struct adf_accel_dev * accel_dev)243 static void adf_cleanup_bh(struct adf_accel_dev *accel_dev)
244 {
245 struct adf_etr_data *priv_data = accel_dev->transport;
246 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
247 int i;
248
249 for (i = 0; i < hw_data->num_banks; i++) {
250 tasklet_disable(&priv_data->banks[i].resp_handler);
251 tasklet_kill(&priv_data->banks[i].resp_handler);
252 }
253 }
254
255 /**
256 * adf_isr_resource_free() - Free IRQ for acceleration device
257 * @accel_dev: Pointer to acceleration device.
258 *
259 * Function frees interrupts for acceleration device.
260 */
adf_isr_resource_free(struct adf_accel_dev * accel_dev)261 void adf_isr_resource_free(struct adf_accel_dev *accel_dev)
262 {
263 adf_free_irqs(accel_dev);
264 adf_cleanup_bh(accel_dev);
265 adf_disable_msix(&accel_dev->accel_pci_dev);
266 adf_isr_free_msix_vectors_data(accel_dev);
267 }
268 EXPORT_SYMBOL_GPL(adf_isr_resource_free);
269
270 /**
271 * adf_isr_resource_alloc() - Allocate IRQ for acceleration device
272 * @accel_dev: Pointer to acceleration device.
273 *
274 * Function allocates interrupts for acceleration device.
275 *
276 * Return: 0 on success, error code otherwise.
277 */
adf_isr_resource_alloc(struct adf_accel_dev * accel_dev)278 int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev)
279 {
280 int ret;
281
282 ret = adf_isr_alloc_msix_vectors_data(accel_dev);
283 if (ret)
284 goto err_out;
285
286 ret = adf_enable_msix(accel_dev);
287 if (ret)
288 goto err_free_msix_table;
289
290 ret = adf_setup_bh(accel_dev);
291 if (ret)
292 goto err_disable_msix;
293
294 ret = adf_request_irqs(accel_dev);
295 if (ret)
296 goto err_cleanup_bh;
297
298 return 0;
299
300 err_cleanup_bh:
301 adf_cleanup_bh(accel_dev);
302
303 err_disable_msix:
304 adf_disable_msix(&accel_dev->accel_pci_dev);
305
306 err_free_msix_table:
307 adf_isr_free_msix_vectors_data(accel_dev);
308
309 err_out:
310 return ret;
311 }
312 EXPORT_SYMBOL_GPL(adf_isr_resource_alloc);
313