1 // SPDX-License-Identifier: GPL-2.0
2
3 /*
4 * Copyright 2016-2019 HabanaLabs, Ltd.
5 * All Rights Reserved.
6 */
7
8 #include "habanalabs.h"
9
10 #include <linux/slab.h>
11
12 /**
13 * struct hl_eqe_work - This structure is used to schedule work of EQ
14 * entry and cpucp_reset event
15 *
16 * @eq_work: workqueue object to run when EQ entry is received
17 * @hdev: pointer to device structure
18 * @eq_entry: copy of the EQ entry
19 */
20 struct hl_eqe_work {
21 struct work_struct eq_work;
22 struct hl_device *hdev;
23 struct hl_eq_entry eq_entry;
24 };
25
26 /**
27 * hl_cq_inc_ptr - increment ci or pi of cq
28 *
29 * @ptr: the current ci or pi value of the completion queue
30 *
31 * Increment ptr by 1. If it reaches the number of completion queue
32 * entries, set it to 0
33 */
hl_cq_inc_ptr(u32 ptr)34 inline u32 hl_cq_inc_ptr(u32 ptr)
35 {
36 ptr++;
37 if (unlikely(ptr == HL_CQ_LENGTH))
38 ptr = 0;
39 return ptr;
40 }
41
42 /**
43 * hl_eq_inc_ptr - increment ci of eq
44 *
45 * @ptr: the current ci value of the event queue
46 *
47 * Increment ptr by 1. If it reaches the number of event queue
48 * entries, set it to 0
49 */
hl_eq_inc_ptr(u32 ptr)50 static inline u32 hl_eq_inc_ptr(u32 ptr)
51 {
52 ptr++;
53 if (unlikely(ptr == HL_EQ_LENGTH))
54 ptr = 0;
55 return ptr;
56 }
57
irq_handle_eqe(struct work_struct * work)58 static void irq_handle_eqe(struct work_struct *work)
59 {
60 struct hl_eqe_work *eqe_work = container_of(work, struct hl_eqe_work,
61 eq_work);
62 struct hl_device *hdev = eqe_work->hdev;
63
64 hdev->asic_funcs->handle_eqe(hdev, &eqe_work->eq_entry);
65
66 kfree(eqe_work);
67 }
68
69 /**
70 * hl_irq_handler_cq - irq handler for completion queue
71 *
72 * @irq: irq number
73 * @arg: pointer to completion queue structure
74 *
75 */
hl_irq_handler_cq(int irq,void * arg)76 irqreturn_t hl_irq_handler_cq(int irq, void *arg)
77 {
78 struct hl_cq *cq = arg;
79 struct hl_device *hdev = cq->hdev;
80 struct hl_hw_queue *queue;
81 struct hl_cs_job *job;
82 bool shadow_index_valid;
83 u16 shadow_index;
84 struct hl_cq_entry *cq_entry, *cq_base;
85
86 if (hdev->disabled) {
87 dev_dbg(hdev->dev,
88 "Device disabled but received IRQ %d for CQ %d\n",
89 irq, cq->hw_queue_id);
90 return IRQ_HANDLED;
91 }
92
93 cq_base = cq->kernel_address;
94
95 while (1) {
96 bool entry_ready = ((le32_to_cpu(cq_base[cq->ci].data) &
97 CQ_ENTRY_READY_MASK)
98 >> CQ_ENTRY_READY_SHIFT);
99
100 if (!entry_ready)
101 break;
102
103 cq_entry = (struct hl_cq_entry *) &cq_base[cq->ci];
104
105 /* Make sure we read CQ entry contents after we've
106 * checked the ownership bit.
107 */
108 dma_rmb();
109
110 shadow_index_valid = ((le32_to_cpu(cq_entry->data) &
111 CQ_ENTRY_SHADOW_INDEX_VALID_MASK)
112 >> CQ_ENTRY_SHADOW_INDEX_VALID_SHIFT);
113
114 shadow_index = (u16) ((le32_to_cpu(cq_entry->data) &
115 CQ_ENTRY_SHADOW_INDEX_MASK)
116 >> CQ_ENTRY_SHADOW_INDEX_SHIFT);
117
118 queue = &hdev->kernel_queues[cq->hw_queue_id];
119
120 if ((shadow_index_valid) && (!hdev->disabled)) {
121 job = queue->shadow_queue[hl_pi_2_offset(shadow_index)];
122 queue_work(hdev->cq_wq[cq->cq_idx], &job->finish_work);
123 }
124
125 atomic_inc(&queue->ci);
126
127 /* Clear CQ entry ready bit */
128 cq_entry->data = cpu_to_le32(le32_to_cpu(cq_entry->data) &
129 ~CQ_ENTRY_READY_MASK);
130
131 cq->ci = hl_cq_inc_ptr(cq->ci);
132
133 /* Increment free slots */
134 atomic_inc(&cq->free_slots_cnt);
135 }
136
137 return IRQ_HANDLED;
138 }
139
handle_user_cq(struct hl_device * hdev,struct hl_user_interrupt * user_cq)140 static void handle_user_cq(struct hl_device *hdev,
141 struct hl_user_interrupt *user_cq)
142 {
143 struct hl_user_pending_interrupt *pend;
144 ktime_t now = ktime_get();
145
146 spin_lock(&user_cq->wait_list_lock);
147 list_for_each_entry(pend, &user_cq->wait_list_head, wait_list_node) {
148 pend->fence.timestamp = now;
149 complete_all(&pend->fence.completion);
150 }
151 spin_unlock(&user_cq->wait_list_lock);
152 }
153
154 /**
155 * hl_irq_handler_user_cq - irq handler for user completion queues
156 *
157 * @irq: irq number
158 * @arg: pointer to user interrupt structure
159 *
160 */
hl_irq_handler_user_cq(int irq,void * arg)161 irqreturn_t hl_irq_handler_user_cq(int irq, void *arg)
162 {
163 struct hl_user_interrupt *user_cq = arg;
164 struct hl_device *hdev = user_cq->hdev;
165
166 dev_dbg(hdev->dev,
167 "got user completion interrupt id %u",
168 user_cq->interrupt_id);
169
170 /* Handle user cq interrupts registered on all interrupts */
171 handle_user_cq(hdev, &hdev->common_user_interrupt);
172
173 /* Handle user cq interrupts registered on this specific interrupt */
174 handle_user_cq(hdev, user_cq);
175
176 return IRQ_HANDLED;
177 }
178
179 /**
180 * hl_irq_handler_default - default irq handler
181 *
182 * @irq: irq number
183 * @arg: pointer to user interrupt structure
184 *
185 */
hl_irq_handler_default(int irq,void * arg)186 irqreturn_t hl_irq_handler_default(int irq, void *arg)
187 {
188 struct hl_user_interrupt *user_interrupt = arg;
189 struct hl_device *hdev = user_interrupt->hdev;
190 u32 interrupt_id = user_interrupt->interrupt_id;
191
192 dev_err(hdev->dev,
193 "got invalid user interrupt %u",
194 interrupt_id);
195
196 return IRQ_HANDLED;
197 }
198
199 /**
200 * hl_irq_handler_eq - irq handler for event queue
201 *
202 * @irq: irq number
203 * @arg: pointer to event queue structure
204 *
205 */
hl_irq_handler_eq(int irq,void * arg)206 irqreturn_t hl_irq_handler_eq(int irq, void *arg)
207 {
208 struct hl_eq *eq = arg;
209 struct hl_device *hdev = eq->hdev;
210 struct hl_eq_entry *eq_entry;
211 struct hl_eq_entry *eq_base;
212 struct hl_eqe_work *handle_eqe_work;
213 bool entry_ready;
214 u32 cur_eqe;
215 u16 cur_eqe_index;
216
217 eq_base = eq->kernel_address;
218
219 while (1) {
220 cur_eqe = le32_to_cpu(eq_base[eq->ci].hdr.ctl);
221 entry_ready = !!FIELD_GET(EQ_CTL_READY_MASK, cur_eqe);
222
223 if (!entry_ready)
224 break;
225
226 cur_eqe_index = FIELD_GET(EQ_CTL_INDEX_MASK, cur_eqe);
227 if ((hdev->event_queue.check_eqe_index) &&
228 (((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK)
229 != cur_eqe_index)) {
230 dev_dbg(hdev->dev,
231 "EQE 0x%x in queue is ready but index does not match %d!=%d",
232 eq_base[eq->ci].hdr.ctl,
233 ((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK),
234 cur_eqe_index);
235 break;
236 }
237
238 eq->prev_eqe_index++;
239
240 eq_entry = &eq_base[eq->ci];
241
242 /*
243 * Make sure we read EQ entry contents after we've
244 * checked the ownership bit.
245 */
246 dma_rmb();
247
248 if (hdev->disabled) {
249 dev_warn(hdev->dev,
250 "Device disabled but received IRQ %d for EQ\n",
251 irq);
252 goto skip_irq;
253 }
254
255 handle_eqe_work = kmalloc(sizeof(*handle_eqe_work), GFP_ATOMIC);
256 if (handle_eqe_work) {
257 INIT_WORK(&handle_eqe_work->eq_work, irq_handle_eqe);
258 handle_eqe_work->hdev = hdev;
259
260 memcpy(&handle_eqe_work->eq_entry, eq_entry,
261 sizeof(*eq_entry));
262
263 queue_work(hdev->eq_wq, &handle_eqe_work->eq_work);
264 }
265 skip_irq:
266 /* Clear EQ entry ready bit */
267 eq_entry->hdr.ctl =
268 cpu_to_le32(le32_to_cpu(eq_entry->hdr.ctl) &
269 ~EQ_CTL_READY_MASK);
270
271 eq->ci = hl_eq_inc_ptr(eq->ci);
272
273 hdev->asic_funcs->update_eq_ci(hdev, eq->ci);
274 }
275
276 return IRQ_HANDLED;
277 }
278
279 /**
280 * hl_cq_init - main initialization function for an cq object
281 *
282 * @hdev: pointer to device structure
283 * @q: pointer to cq structure
284 * @hw_queue_id: The H/W queue ID this completion queue belongs to
285 *
286 * Allocate dma-able memory for the completion queue and initialize fields
287 * Returns 0 on success
288 */
hl_cq_init(struct hl_device * hdev,struct hl_cq * q,u32 hw_queue_id)289 int hl_cq_init(struct hl_device *hdev, struct hl_cq *q, u32 hw_queue_id)
290 {
291 void *p;
292
293 p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, HL_CQ_SIZE_IN_BYTES,
294 &q->bus_address, GFP_KERNEL | __GFP_ZERO);
295 if (!p)
296 return -ENOMEM;
297
298 q->hdev = hdev;
299 q->kernel_address = p;
300 q->hw_queue_id = hw_queue_id;
301 q->ci = 0;
302 q->pi = 0;
303
304 atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH);
305
306 return 0;
307 }
308
309 /**
310 * hl_cq_fini - destroy completion queue
311 *
312 * @hdev: pointer to device structure
313 * @q: pointer to cq structure
314 *
315 * Free the completion queue memory
316 */
hl_cq_fini(struct hl_device * hdev,struct hl_cq * q)317 void hl_cq_fini(struct hl_device *hdev, struct hl_cq *q)
318 {
319 hdev->asic_funcs->asic_dma_free_coherent(hdev, HL_CQ_SIZE_IN_BYTES,
320 q->kernel_address,
321 q->bus_address);
322 }
323
hl_cq_reset(struct hl_device * hdev,struct hl_cq * q)324 void hl_cq_reset(struct hl_device *hdev, struct hl_cq *q)
325 {
326 q->ci = 0;
327 q->pi = 0;
328
329 atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH);
330
331 /*
332 * It's not enough to just reset the PI/CI because the H/W may have
333 * written valid completion entries before it was halted and therefore
334 * we need to clean the actual queues so we won't process old entries
335 * when the device is operational again
336 */
337
338 memset(q->kernel_address, 0, HL_CQ_SIZE_IN_BYTES);
339 }
340
341 /**
342 * hl_eq_init - main initialization function for an event queue object
343 *
344 * @hdev: pointer to device structure
345 * @q: pointer to eq structure
346 *
347 * Allocate dma-able memory for the event queue and initialize fields
348 * Returns 0 on success
349 */
hl_eq_init(struct hl_device * hdev,struct hl_eq * q)350 int hl_eq_init(struct hl_device *hdev, struct hl_eq *q)
351 {
352 void *p;
353
354 p = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
355 HL_EQ_SIZE_IN_BYTES,
356 &q->bus_address);
357 if (!p)
358 return -ENOMEM;
359
360 q->hdev = hdev;
361 q->kernel_address = p;
362 q->ci = 0;
363 q->prev_eqe_index = 0;
364
365 return 0;
366 }
367
368 /**
369 * hl_eq_fini - destroy event queue
370 *
371 * @hdev: pointer to device structure
372 * @q: pointer to eq structure
373 *
374 * Free the event queue memory
375 */
hl_eq_fini(struct hl_device * hdev,struct hl_eq * q)376 void hl_eq_fini(struct hl_device *hdev, struct hl_eq *q)
377 {
378 flush_workqueue(hdev->eq_wq);
379
380 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
381 HL_EQ_SIZE_IN_BYTES,
382 q->kernel_address);
383 }
384
hl_eq_reset(struct hl_device * hdev,struct hl_eq * q)385 void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q)
386 {
387 q->ci = 0;
388 q->prev_eqe_index = 0;
389
390 /*
391 * It's not enough to just reset the PI/CI because the H/W may have
392 * written valid completion entries before it was halted and therefore
393 * we need to clean the actual queues so we won't process old entries
394 * when the device is operational again
395 */
396
397 memset(q->kernel_address, 0, HL_EQ_SIZE_IN_BYTES);
398 }
399