1 // SPDX-License-Identifier: ISC
2 /*
3 * Copyright (c) 2005-2011 Atheros Communications Inc.
4 * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
5 * Copyright (c) 2018 The Linux Foundation. All rights reserved.
6 */
7
8 #include "hif.h"
9 #include "ce.h"
10 #include "debug.h"
11
12 /*
13 * Support for Copy Engine hardware, which is mainly used for
14 * communication between Host and Target over a PCIe interconnect.
15 */
16
17 /*
18 * A single CopyEngine (CE) comprises two "rings":
19 * a source ring
20 * a destination ring
21 *
22 * Each ring consists of a number of descriptors which specify
23 * an address, length, and meta-data.
24 *
25 * Typically, one side of the PCIe/AHB/SNOC interconnect (Host or Target)
26 * controls one ring and the other side controls the other ring.
27 * The source side chooses when to initiate a transfer and it
28 * chooses what to send (buffer address, length). The destination
29 * side keeps a supply of "anonymous receive buffers" available and
30 * it handles incoming data as it arrives (when the destination
31 * receives an interrupt).
32 *
33 * The sender may send a simple buffer (address/length) or it may
34 * send a small list of buffers. When a small list is sent, hardware
35 * "gathers" these and they end up in a single destination buffer
36 * with a single interrupt.
37 *
38 * There are several "contexts" managed by this layer -- more, it
39 * may seem -- than should be needed. These are provided mainly for
40 * maximum flexibility and especially to facilitate a simpler HIF
41 * implementation. There are per-CopyEngine recv, send, and watermark
42 * contexts. These are supplied by the caller when a recv, send,
43 * or watermark handler is established and they are echoed back to
44 * the caller when the respective callbacks are invoked. There is
45 * also a per-transfer context supplied by the caller when a buffer
46 * (or sendlist) is sent and when a buffer is enqueued for recv.
47 * These per-transfer contexts are echoed back to the caller when
48 * the buffer is sent/received.
49 */
50
shadow_sr_wr_ind_addr(struct ath10k * ar,struct ath10k_ce_pipe * ce_state)51 static inline u32 shadow_sr_wr_ind_addr(struct ath10k *ar,
52 struct ath10k_ce_pipe *ce_state)
53 {
54 u32 ce_id = ce_state->id;
55 u32 addr = 0;
56
57 switch (ce_id) {
58 case 0:
59 addr = 0x00032000;
60 break;
61 case 3:
62 addr = 0x0003200C;
63 break;
64 case 4:
65 addr = 0x00032010;
66 break;
67 case 5:
68 addr = 0x00032014;
69 break;
70 case 7:
71 addr = 0x0003201C;
72 break;
73 default:
74 ath10k_warn(ar, "invalid CE id: %d", ce_id);
75 break;
76 }
77 return addr;
78 }
79
shadow_dst_wr_ind_addr(struct ath10k * ar,struct ath10k_ce_pipe * ce_state)80 static inline u32 shadow_dst_wr_ind_addr(struct ath10k *ar,
81 struct ath10k_ce_pipe *ce_state)
82 {
83 u32 ce_id = ce_state->id;
84 u32 addr = 0;
85
86 switch (ce_id) {
87 case 1:
88 addr = 0x00032034;
89 break;
90 case 2:
91 addr = 0x00032038;
92 break;
93 case 5:
94 addr = 0x00032044;
95 break;
96 case 7:
97 addr = 0x0003204C;
98 break;
99 case 8:
100 addr = 0x00032050;
101 break;
102 case 9:
103 addr = 0x00032054;
104 break;
105 case 10:
106 addr = 0x00032058;
107 break;
108 case 11:
109 addr = 0x0003205C;
110 break;
111 default:
112 ath10k_warn(ar, "invalid CE id: %d", ce_id);
113 break;
114 }
115
116 return addr;
117 }
118
119 static inline unsigned int
ath10k_set_ring_byte(unsigned int offset,struct ath10k_hw_ce_regs_addr_map * addr_map)120 ath10k_set_ring_byte(unsigned int offset,
121 struct ath10k_hw_ce_regs_addr_map *addr_map)
122 {
123 return ((offset << addr_map->lsb) & addr_map->mask);
124 }
125
126 static inline unsigned int
ath10k_get_ring_byte(unsigned int offset,struct ath10k_hw_ce_regs_addr_map * addr_map)127 ath10k_get_ring_byte(unsigned int offset,
128 struct ath10k_hw_ce_regs_addr_map *addr_map)
129 {
130 return ((offset & addr_map->mask) >> (addr_map->lsb));
131 }
132
ath10k_ce_read32(struct ath10k * ar,u32 offset)133 static inline u32 ath10k_ce_read32(struct ath10k *ar, u32 offset)
134 {
135 struct ath10k_ce *ce = ath10k_ce_priv(ar);
136
137 return ce->bus_ops->read32(ar, offset);
138 }
139
ath10k_ce_write32(struct ath10k * ar,u32 offset,u32 value)140 static inline void ath10k_ce_write32(struct ath10k *ar, u32 offset, u32 value)
141 {
142 struct ath10k_ce *ce = ath10k_ce_priv(ar);
143
144 ce->bus_ops->write32(ar, offset, value);
145 }
146
ath10k_ce_dest_ring_write_index_set(struct ath10k * ar,u32 ce_ctrl_addr,unsigned int n)147 static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar,
148 u32 ce_ctrl_addr,
149 unsigned int n)
150 {
151 ath10k_ce_write32(ar, ce_ctrl_addr +
152 ar->hw_ce_regs->dst_wr_index_addr, n);
153 }
154
ath10k_ce_dest_ring_write_index_get(struct ath10k * ar,u32 ce_ctrl_addr)155 static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar,
156 u32 ce_ctrl_addr)
157 {
158 return ath10k_ce_read32(ar, ce_ctrl_addr +
159 ar->hw_ce_regs->dst_wr_index_addr);
160 }
161
ath10k_ce_src_ring_write_index_set(struct ath10k * ar,u32 ce_ctrl_addr,unsigned int n)162 static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar,
163 u32 ce_ctrl_addr,
164 unsigned int n)
165 {
166 ath10k_ce_write32(ar, ce_ctrl_addr +
167 ar->hw_ce_regs->sr_wr_index_addr, n);
168 }
169
ath10k_ce_src_ring_write_index_get(struct ath10k * ar,u32 ce_ctrl_addr)170 static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar,
171 u32 ce_ctrl_addr)
172 {
173 return ath10k_ce_read32(ar, ce_ctrl_addr +
174 ar->hw_ce_regs->sr_wr_index_addr);
175 }
176
ath10k_ce_src_ring_read_index_from_ddr(struct ath10k * ar,u32 ce_id)177 static inline u32 ath10k_ce_src_ring_read_index_from_ddr(struct ath10k *ar,
178 u32 ce_id)
179 {
180 struct ath10k_ce *ce = ath10k_ce_priv(ar);
181
182 return ce->vaddr_rri[ce_id] & CE_DDR_RRI_MASK;
183 }
184
ath10k_ce_src_ring_read_index_get(struct ath10k * ar,u32 ce_ctrl_addr)185 static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar,
186 u32 ce_ctrl_addr)
187 {
188 struct ath10k_ce *ce = ath10k_ce_priv(ar);
189 u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
190 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
191 u32 index;
192
193 if (ar->hw_params.rri_on_ddr &&
194 (ce_state->attr_flags & CE_ATTR_DIS_INTR))
195 index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_id);
196 else
197 index = ath10k_ce_read32(ar, ce_ctrl_addr +
198 ar->hw_ce_regs->current_srri_addr);
199
200 return index;
201 }
202
203 static inline void
ath10k_ce_shadow_src_ring_write_index_set(struct ath10k * ar,struct ath10k_ce_pipe * ce_state,unsigned int value)204 ath10k_ce_shadow_src_ring_write_index_set(struct ath10k *ar,
205 struct ath10k_ce_pipe *ce_state,
206 unsigned int value)
207 {
208 ath10k_ce_write32(ar, shadow_sr_wr_ind_addr(ar, ce_state), value);
209 }
210
ath10k_ce_src_ring_base_addr_set(struct ath10k * ar,u32 ce_id,u64 addr)211 static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar,
212 u32 ce_id,
213 u64 addr)
214 {
215 struct ath10k_ce *ce = ath10k_ce_priv(ar);
216 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
217 u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id);
218 u32 addr_lo = lower_32_bits(addr);
219
220 ath10k_ce_write32(ar, ce_ctrl_addr +
221 ar->hw_ce_regs->sr_base_addr_lo, addr_lo);
222
223 if (ce_state->ops->ce_set_src_ring_base_addr_hi) {
224 ce_state->ops->ce_set_src_ring_base_addr_hi(ar, ce_ctrl_addr,
225 addr);
226 }
227 }
228
ath10k_ce_set_src_ring_base_addr_hi(struct ath10k * ar,u32 ce_ctrl_addr,u64 addr)229 static void ath10k_ce_set_src_ring_base_addr_hi(struct ath10k *ar,
230 u32 ce_ctrl_addr,
231 u64 addr)
232 {
233 u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK;
234
235 ath10k_ce_write32(ar, ce_ctrl_addr +
236 ar->hw_ce_regs->sr_base_addr_hi, addr_hi);
237 }
238
ath10k_ce_src_ring_size_set(struct ath10k * ar,u32 ce_ctrl_addr,unsigned int n)239 static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar,
240 u32 ce_ctrl_addr,
241 unsigned int n)
242 {
243 ath10k_ce_write32(ar, ce_ctrl_addr +
244 ar->hw_ce_regs->sr_size_addr, n);
245 }
246
ath10k_ce_src_ring_dmax_set(struct ath10k * ar,u32 ce_ctrl_addr,unsigned int n)247 static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar,
248 u32 ce_ctrl_addr,
249 unsigned int n)
250 {
251 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
252
253 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
254 ctrl_regs->addr);
255
256 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
257 (ctrl1_addr & ~(ctrl_regs->dmax->mask)) |
258 ath10k_set_ring_byte(n, ctrl_regs->dmax));
259 }
260
ath10k_ce_src_ring_byte_swap_set(struct ath10k * ar,u32 ce_ctrl_addr,unsigned int n)261 static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar,
262 u32 ce_ctrl_addr,
263 unsigned int n)
264 {
265 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
266
267 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
268 ctrl_regs->addr);
269
270 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
271 (ctrl1_addr & ~(ctrl_regs->src_ring->mask)) |
272 ath10k_set_ring_byte(n, ctrl_regs->src_ring));
273 }
274
ath10k_ce_dest_ring_byte_swap_set(struct ath10k * ar,u32 ce_ctrl_addr,unsigned int n)275 static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar,
276 u32 ce_ctrl_addr,
277 unsigned int n)
278 {
279 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
280
281 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
282 ctrl_regs->addr);
283
284 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
285 (ctrl1_addr & ~(ctrl_regs->dst_ring->mask)) |
286 ath10k_set_ring_byte(n, ctrl_regs->dst_ring));
287 }
288
289 static inline
ath10k_ce_dest_ring_read_index_from_ddr(struct ath10k * ar,u32 ce_id)290 u32 ath10k_ce_dest_ring_read_index_from_ddr(struct ath10k *ar, u32 ce_id)
291 {
292 struct ath10k_ce *ce = ath10k_ce_priv(ar);
293
294 return (ce->vaddr_rri[ce_id] >> CE_DDR_DRRI_SHIFT) &
295 CE_DDR_RRI_MASK;
296 }
297
ath10k_ce_dest_ring_read_index_get(struct ath10k * ar,u32 ce_ctrl_addr)298 static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar,
299 u32 ce_ctrl_addr)
300 {
301 struct ath10k_ce *ce = ath10k_ce_priv(ar);
302 u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
303 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
304 u32 index;
305
306 if (ar->hw_params.rri_on_ddr &&
307 (ce_state->attr_flags & CE_ATTR_DIS_INTR))
308 index = ath10k_ce_dest_ring_read_index_from_ddr(ar, ce_id);
309 else
310 index = ath10k_ce_read32(ar, ce_ctrl_addr +
311 ar->hw_ce_regs->current_drri_addr);
312
313 return index;
314 }
315
ath10k_ce_dest_ring_base_addr_set(struct ath10k * ar,u32 ce_id,u64 addr)316 static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar,
317 u32 ce_id,
318 u64 addr)
319 {
320 struct ath10k_ce *ce = ath10k_ce_priv(ar);
321 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
322 u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id);
323 u32 addr_lo = lower_32_bits(addr);
324
325 ath10k_ce_write32(ar, ce_ctrl_addr +
326 ar->hw_ce_regs->dr_base_addr_lo, addr_lo);
327
328 if (ce_state->ops->ce_set_dest_ring_base_addr_hi) {
329 ce_state->ops->ce_set_dest_ring_base_addr_hi(ar, ce_ctrl_addr,
330 addr);
331 }
332 }
333
ath10k_ce_set_dest_ring_base_addr_hi(struct ath10k * ar,u32 ce_ctrl_addr,u64 addr)334 static void ath10k_ce_set_dest_ring_base_addr_hi(struct ath10k *ar,
335 u32 ce_ctrl_addr,
336 u64 addr)
337 {
338 u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK;
339 u32 reg_value;
340
341 reg_value = ath10k_ce_read32(ar, ce_ctrl_addr +
342 ar->hw_ce_regs->dr_base_addr_hi);
343 reg_value &= ~CE_DESC_ADDR_HI_MASK;
344 reg_value |= addr_hi;
345 ath10k_ce_write32(ar, ce_ctrl_addr +
346 ar->hw_ce_regs->dr_base_addr_hi, reg_value);
347 }
348
ath10k_ce_dest_ring_size_set(struct ath10k * ar,u32 ce_ctrl_addr,unsigned int n)349 static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar,
350 u32 ce_ctrl_addr,
351 unsigned int n)
352 {
353 ath10k_ce_write32(ar, ce_ctrl_addr +
354 ar->hw_ce_regs->dr_size_addr, n);
355 }
356
ath10k_ce_src_ring_highmark_set(struct ath10k * ar,u32 ce_ctrl_addr,unsigned int n)357 static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar,
358 u32 ce_ctrl_addr,
359 unsigned int n)
360 {
361 struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
362 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);
363
364 ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,
365 (addr & ~(srcr_wm->wm_high->mask)) |
366 (ath10k_set_ring_byte(n, srcr_wm->wm_high)));
367 }
368
ath10k_ce_src_ring_lowmark_set(struct ath10k * ar,u32 ce_ctrl_addr,unsigned int n)369 static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar,
370 u32 ce_ctrl_addr,
371 unsigned int n)
372 {
373 struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
374 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);
375
376 ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,
377 (addr & ~(srcr_wm->wm_low->mask)) |
378 (ath10k_set_ring_byte(n, srcr_wm->wm_low)));
379 }
380
ath10k_ce_dest_ring_highmark_set(struct ath10k * ar,u32 ce_ctrl_addr,unsigned int n)381 static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar,
382 u32 ce_ctrl_addr,
383 unsigned int n)
384 {
385 struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
386 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);
387
388 ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,
389 (addr & ~(dstr_wm->wm_high->mask)) |
390 (ath10k_set_ring_byte(n, dstr_wm->wm_high)));
391 }
392
ath10k_ce_dest_ring_lowmark_set(struct ath10k * ar,u32 ce_ctrl_addr,unsigned int n)393 static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar,
394 u32 ce_ctrl_addr,
395 unsigned int n)
396 {
397 struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
398 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);
399
400 ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,
401 (addr & ~(dstr_wm->wm_low->mask)) |
402 (ath10k_set_ring_byte(n, dstr_wm->wm_low)));
403 }
404
ath10k_ce_copy_complete_inter_enable(struct ath10k * ar,u32 ce_ctrl_addr)405 static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar,
406 u32 ce_ctrl_addr)
407 {
408 struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
409
410 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
411 ar->hw_ce_regs->host_ie_addr);
412
413 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
414 host_ie_addr | host_ie->copy_complete->mask);
415 }
416
ath10k_ce_copy_complete_intr_disable(struct ath10k * ar,u32 ce_ctrl_addr)417 static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar,
418 u32 ce_ctrl_addr)
419 {
420 struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
421
422 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
423 ar->hw_ce_regs->host_ie_addr);
424
425 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
426 host_ie_addr & ~(host_ie->copy_complete->mask));
427 }
428
ath10k_ce_watermark_intr_disable(struct ath10k * ar,u32 ce_ctrl_addr)429 static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar,
430 u32 ce_ctrl_addr)
431 {
432 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
433
434 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
435 ar->hw_ce_regs->host_ie_addr);
436
437 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
438 host_ie_addr & ~(wm_regs->wm_mask));
439 }
440
ath10k_ce_error_intr_enable(struct ath10k * ar,u32 ce_ctrl_addr)441 static inline void ath10k_ce_error_intr_enable(struct ath10k *ar,
442 u32 ce_ctrl_addr)
443 {
444 struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs;
445
446 u32 misc_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
447 ar->hw_ce_regs->misc_ie_addr);
448
449 ath10k_ce_write32(ar,
450 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
451 misc_ie_addr | misc_regs->err_mask);
452 }
453
ath10k_ce_error_intr_disable(struct ath10k * ar,u32 ce_ctrl_addr)454 static inline void ath10k_ce_error_intr_disable(struct ath10k *ar,
455 u32 ce_ctrl_addr)
456 {
457 struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs;
458
459 u32 misc_ie_addr = ath10k_ce_read32(ar,
460 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr);
461
462 ath10k_ce_write32(ar,
463 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
464 misc_ie_addr & ~(misc_regs->err_mask));
465 }
466
ath10k_ce_engine_int_status_clear(struct ath10k * ar,u32 ce_ctrl_addr,unsigned int mask)467 static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar,
468 u32 ce_ctrl_addr,
469 unsigned int mask)
470 {
471 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
472
473 ath10k_ce_write32(ar, ce_ctrl_addr + wm_regs->addr, mask);
474 }
475
476 /*
477 * Guts of ath10k_ce_send.
478 * The caller takes responsibility for any needed locking.
479 */
_ath10k_ce_send_nolock(struct ath10k_ce_pipe * ce_state,void * per_transfer_context,dma_addr_t buffer,unsigned int nbytes,unsigned int transfer_id,unsigned int flags)480 static int _ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
481 void *per_transfer_context,
482 dma_addr_t buffer,
483 unsigned int nbytes,
484 unsigned int transfer_id,
485 unsigned int flags)
486 {
487 struct ath10k *ar = ce_state->ar;
488 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
489 struct ce_desc *desc, sdesc;
490 unsigned int nentries_mask = src_ring->nentries_mask;
491 unsigned int sw_index = src_ring->sw_index;
492 unsigned int write_index = src_ring->write_index;
493 u32 ctrl_addr = ce_state->ctrl_addr;
494 u32 desc_flags = 0;
495 int ret = 0;
496
497 if (nbytes > ce_state->src_sz_max)
498 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
499 __func__, nbytes, ce_state->src_sz_max);
500
501 if (unlikely(CE_RING_DELTA(nentries_mask,
502 write_index, sw_index - 1) <= 0)) {
503 ret = -ENOSR;
504 goto exit;
505 }
506
507 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
508 write_index);
509
510 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
511
512 if (flags & CE_SEND_FLAG_GATHER)
513 desc_flags |= CE_DESC_FLAGS_GATHER;
514 if (flags & CE_SEND_FLAG_BYTE_SWAP)
515 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
516
517 sdesc.addr = __cpu_to_le32(buffer);
518 sdesc.nbytes = __cpu_to_le16(nbytes);
519 sdesc.flags = __cpu_to_le16(desc_flags);
520
521 *desc = sdesc;
522
523 src_ring->per_transfer_context[write_index] = per_transfer_context;
524
525 /* Update Source Ring Write Index */
526 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
527
528 /* WORKAROUND */
529 if (!(flags & CE_SEND_FLAG_GATHER))
530 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index);
531
532 src_ring->write_index = write_index;
533 exit:
534 return ret;
535 }
536
_ath10k_ce_send_nolock_64(struct ath10k_ce_pipe * ce_state,void * per_transfer_context,dma_addr_t buffer,unsigned int nbytes,unsigned int transfer_id,unsigned int flags)537 static int _ath10k_ce_send_nolock_64(struct ath10k_ce_pipe *ce_state,
538 void *per_transfer_context,
539 dma_addr_t buffer,
540 unsigned int nbytes,
541 unsigned int transfer_id,
542 unsigned int flags)
543 {
544 struct ath10k *ar = ce_state->ar;
545 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
546 struct ce_desc_64 *desc, sdesc;
547 unsigned int nentries_mask = src_ring->nentries_mask;
548 unsigned int sw_index;
549 unsigned int write_index = src_ring->write_index;
550 u32 ctrl_addr = ce_state->ctrl_addr;
551 __le32 *addr;
552 u32 desc_flags = 0;
553 int ret = 0;
554
555 if (test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags))
556 return -ESHUTDOWN;
557
558 if (nbytes > ce_state->src_sz_max)
559 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
560 __func__, nbytes, ce_state->src_sz_max);
561
562 if (ar->hw_params.rri_on_ddr)
563 sw_index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_state->id);
564 else
565 sw_index = src_ring->sw_index;
566
567 if (unlikely(CE_RING_DELTA(nentries_mask,
568 write_index, sw_index - 1) <= 0)) {
569 ret = -ENOSR;
570 goto exit;
571 }
572
573 desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,
574 write_index);
575
576 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
577
578 if (flags & CE_SEND_FLAG_GATHER)
579 desc_flags |= CE_DESC_FLAGS_GATHER;
580
581 if (flags & CE_SEND_FLAG_BYTE_SWAP)
582 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
583
584 addr = (__le32 *)&sdesc.addr;
585
586 flags |= upper_32_bits(buffer) & CE_DESC_ADDR_HI_MASK;
587 addr[0] = __cpu_to_le32(buffer);
588 addr[1] = __cpu_to_le32(flags);
589 if (flags & CE_SEND_FLAG_GATHER)
590 addr[1] |= __cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER);
591 else
592 addr[1] &= ~(__cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER));
593
594 sdesc.nbytes = __cpu_to_le16(nbytes);
595 sdesc.flags = __cpu_to_le16(desc_flags);
596
597 *desc = sdesc;
598
599 src_ring->per_transfer_context[write_index] = per_transfer_context;
600
601 /* Update Source Ring Write Index */
602 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
603
604 if (!(flags & CE_SEND_FLAG_GATHER)) {
605 if (ar->hw_params.shadow_reg_support)
606 ath10k_ce_shadow_src_ring_write_index_set(ar, ce_state,
607 write_index);
608 else
609 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr,
610 write_index);
611 }
612
613 src_ring->write_index = write_index;
614 exit:
615 return ret;
616 }
617
ath10k_ce_send_nolock(struct ath10k_ce_pipe * ce_state,void * per_transfer_context,dma_addr_t buffer,unsigned int nbytes,unsigned int transfer_id,unsigned int flags)618 int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
619 void *per_transfer_context,
620 dma_addr_t buffer,
621 unsigned int nbytes,
622 unsigned int transfer_id,
623 unsigned int flags)
624 {
625 return ce_state->ops->ce_send_nolock(ce_state, per_transfer_context,
626 buffer, nbytes, transfer_id, flags);
627 }
628 EXPORT_SYMBOL(ath10k_ce_send_nolock);
629
__ath10k_ce_send_revert(struct ath10k_ce_pipe * pipe)630 void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe)
631 {
632 struct ath10k *ar = pipe->ar;
633 struct ath10k_ce *ce = ath10k_ce_priv(ar);
634 struct ath10k_ce_ring *src_ring = pipe->src_ring;
635 u32 ctrl_addr = pipe->ctrl_addr;
636
637 lockdep_assert_held(&ce->ce_lock);
638
639 /*
640 * This function must be called only if there is an incomplete
641 * scatter-gather transfer (before index register is updated)
642 * that needs to be cleaned up.
643 */
644 if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index))
645 return;
646
647 if (WARN_ON_ONCE(src_ring->write_index ==
648 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr)))
649 return;
650
651 src_ring->write_index--;
652 src_ring->write_index &= src_ring->nentries_mask;
653
654 src_ring->per_transfer_context[src_ring->write_index] = NULL;
655 }
656 EXPORT_SYMBOL(__ath10k_ce_send_revert);
657
ath10k_ce_send(struct ath10k_ce_pipe * ce_state,void * per_transfer_context,dma_addr_t buffer,unsigned int nbytes,unsigned int transfer_id,unsigned int flags)658 int ath10k_ce_send(struct ath10k_ce_pipe *ce_state,
659 void *per_transfer_context,
660 dma_addr_t buffer,
661 unsigned int nbytes,
662 unsigned int transfer_id,
663 unsigned int flags)
664 {
665 struct ath10k *ar = ce_state->ar;
666 struct ath10k_ce *ce = ath10k_ce_priv(ar);
667 int ret;
668
669 spin_lock_bh(&ce->ce_lock);
670 ret = ath10k_ce_send_nolock(ce_state, per_transfer_context,
671 buffer, nbytes, transfer_id, flags);
672 spin_unlock_bh(&ce->ce_lock);
673
674 return ret;
675 }
676 EXPORT_SYMBOL(ath10k_ce_send);
677
ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe * pipe)678 int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe)
679 {
680 struct ath10k *ar = pipe->ar;
681 struct ath10k_ce *ce = ath10k_ce_priv(ar);
682 int delta;
683
684 spin_lock_bh(&ce->ce_lock);
685 delta = CE_RING_DELTA(pipe->src_ring->nentries_mask,
686 pipe->src_ring->write_index,
687 pipe->src_ring->sw_index - 1);
688 spin_unlock_bh(&ce->ce_lock);
689
690 return delta;
691 }
692 EXPORT_SYMBOL(ath10k_ce_num_free_src_entries);
693
__ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe * pipe)694 int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe)
695 {
696 struct ath10k *ar = pipe->ar;
697 struct ath10k_ce *ce = ath10k_ce_priv(ar);
698 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
699 unsigned int nentries_mask = dest_ring->nentries_mask;
700 unsigned int write_index = dest_ring->write_index;
701 unsigned int sw_index = dest_ring->sw_index;
702
703 lockdep_assert_held(&ce->ce_lock);
704
705 return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1);
706 }
707 EXPORT_SYMBOL(__ath10k_ce_rx_num_free_bufs);
708
__ath10k_ce_rx_post_buf(struct ath10k_ce_pipe * pipe,void * ctx,dma_addr_t paddr)709 static int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
710 dma_addr_t paddr)
711 {
712 struct ath10k *ar = pipe->ar;
713 struct ath10k_ce *ce = ath10k_ce_priv(ar);
714 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
715 unsigned int nentries_mask = dest_ring->nentries_mask;
716 unsigned int write_index = dest_ring->write_index;
717 unsigned int sw_index = dest_ring->sw_index;
718 struct ce_desc *base = dest_ring->base_addr_owner_space;
719 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index);
720 u32 ctrl_addr = pipe->ctrl_addr;
721
722 lockdep_assert_held(&ce->ce_lock);
723
724 if ((pipe->id != 5) &&
725 CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
726 return -ENOSPC;
727
728 desc->addr = __cpu_to_le32(paddr);
729 desc->nbytes = 0;
730
731 dest_ring->per_transfer_context[write_index] = ctx;
732 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
733 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
734 dest_ring->write_index = write_index;
735
736 return 0;
737 }
738
__ath10k_ce_rx_post_buf_64(struct ath10k_ce_pipe * pipe,void * ctx,dma_addr_t paddr)739 static int __ath10k_ce_rx_post_buf_64(struct ath10k_ce_pipe *pipe,
740 void *ctx,
741 dma_addr_t paddr)
742 {
743 struct ath10k *ar = pipe->ar;
744 struct ath10k_ce *ce = ath10k_ce_priv(ar);
745 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
746 unsigned int nentries_mask = dest_ring->nentries_mask;
747 unsigned int write_index = dest_ring->write_index;
748 unsigned int sw_index = dest_ring->sw_index;
749 struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
750 struct ce_desc_64 *desc =
751 CE_DEST_RING_TO_DESC_64(base, write_index);
752 u32 ctrl_addr = pipe->ctrl_addr;
753
754 lockdep_assert_held(&ce->ce_lock);
755
756 if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
757 return -ENOSPC;
758
759 desc->addr = __cpu_to_le64(paddr);
760 desc->addr &= __cpu_to_le64(CE_DESC_ADDR_MASK);
761
762 desc->nbytes = 0;
763
764 dest_ring->per_transfer_context[write_index] = ctx;
765 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
766 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
767 dest_ring->write_index = write_index;
768
769 return 0;
770 }
771
ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe * pipe,u32 nentries)772 void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries)
773 {
774 struct ath10k *ar = pipe->ar;
775 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
776 unsigned int nentries_mask = dest_ring->nentries_mask;
777 unsigned int write_index = dest_ring->write_index;
778 u32 ctrl_addr = pipe->ctrl_addr;
779 u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
780
781 /* Prevent CE ring stuck issue that will occur when ring is full.
782 * Make sure that write index is 1 less than read index.
783 */
784 if (((cur_write_idx + nentries) & nentries_mask) == dest_ring->sw_index)
785 nentries -= 1;
786
787 write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries);
788 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
789 dest_ring->write_index = write_index;
790 }
791 EXPORT_SYMBOL(ath10k_ce_rx_update_write_idx);
792
ath10k_ce_rx_post_buf(struct ath10k_ce_pipe * pipe,void * ctx,dma_addr_t paddr)793 int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
794 dma_addr_t paddr)
795 {
796 struct ath10k *ar = pipe->ar;
797 struct ath10k_ce *ce = ath10k_ce_priv(ar);
798 int ret;
799
800 spin_lock_bh(&ce->ce_lock);
801 ret = pipe->ops->ce_rx_post_buf(pipe, ctx, paddr);
802 spin_unlock_bh(&ce->ce_lock);
803
804 return ret;
805 }
806 EXPORT_SYMBOL(ath10k_ce_rx_post_buf);
807
808 /*
809 * Guts of ath10k_ce_completed_recv_next.
810 * The caller takes responsibility for any necessary locking.
811 */
812 static int
_ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe * ce_state,void ** per_transfer_contextp,unsigned int * nbytesp)813 _ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
814 void **per_transfer_contextp,
815 unsigned int *nbytesp)
816 {
817 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
818 unsigned int nentries_mask = dest_ring->nentries_mask;
819 unsigned int sw_index = dest_ring->sw_index;
820
821 struct ce_desc *base = dest_ring->base_addr_owner_space;
822 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
823 struct ce_desc sdesc;
824 u16 nbytes;
825
826 /* Copy in one go for performance reasons */
827 sdesc = *desc;
828
829 nbytes = __le16_to_cpu(sdesc.nbytes);
830 if (nbytes == 0) {
831 /*
832 * This closes a relatively unusual race where the Host
833 * sees the updated DRRI before the update to the
834 * corresponding descriptor has completed. We treat this
835 * as a descriptor that is not yet done.
836 */
837 return -EIO;
838 }
839
840 desc->nbytes = 0;
841
842 /* Return data from completed destination descriptor */
843 *nbytesp = nbytes;
844
845 if (per_transfer_contextp)
846 *per_transfer_contextp =
847 dest_ring->per_transfer_context[sw_index];
848
849 /* Copy engine 5 (HTT Rx) will reuse the same transfer context.
850 * So update transfer context all CEs except CE5.
851 */
852 if (ce_state->id != 5)
853 dest_ring->per_transfer_context[sw_index] = NULL;
854
855 /* Update sw_index */
856 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
857 dest_ring->sw_index = sw_index;
858
859 return 0;
860 }
861
862 static int
_ath10k_ce_completed_recv_next_nolock_64(struct ath10k_ce_pipe * ce_state,void ** per_transfer_contextp,unsigned int * nbytesp)863 _ath10k_ce_completed_recv_next_nolock_64(struct ath10k_ce_pipe *ce_state,
864 void **per_transfer_contextp,
865 unsigned int *nbytesp)
866 {
867 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
868 unsigned int nentries_mask = dest_ring->nentries_mask;
869 unsigned int sw_index = dest_ring->sw_index;
870 struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
871 struct ce_desc_64 *desc =
872 CE_DEST_RING_TO_DESC_64(base, sw_index);
873 struct ce_desc_64 sdesc;
874 u16 nbytes;
875
876 /* Copy in one go for performance reasons */
877 sdesc = *desc;
878
879 nbytes = __le16_to_cpu(sdesc.nbytes);
880 if (nbytes == 0) {
881 /* This closes a relatively unusual race where the Host
882 * sees the updated DRRI before the update to the
883 * corresponding descriptor has completed. We treat this
884 * as a descriptor that is not yet done.
885 */
886 return -EIO;
887 }
888
889 desc->nbytes = 0;
890
891 /* Return data from completed destination descriptor */
892 *nbytesp = nbytes;
893
894 if (per_transfer_contextp)
895 *per_transfer_contextp =
896 dest_ring->per_transfer_context[sw_index];
897
898 /* Copy engine 5 (HTT Rx) will reuse the same transfer context.
899 * So update transfer context all CEs except CE5.
900 */
901 if (ce_state->id != 5)
902 dest_ring->per_transfer_context[sw_index] = NULL;
903
904 /* Update sw_index */
905 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
906 dest_ring->sw_index = sw_index;
907
908 return 0;
909 }
910
ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe * ce_state,void ** per_transfer_ctx,unsigned int * nbytesp)911 int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
912 void **per_transfer_ctx,
913 unsigned int *nbytesp)
914 {
915 return ce_state->ops->ce_completed_recv_next_nolock(ce_state,
916 per_transfer_ctx,
917 nbytesp);
918 }
919 EXPORT_SYMBOL(ath10k_ce_completed_recv_next_nolock);
920
ath10k_ce_completed_recv_next(struct ath10k_ce_pipe * ce_state,void ** per_transfer_contextp,unsigned int * nbytesp)921 int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state,
922 void **per_transfer_contextp,
923 unsigned int *nbytesp)
924 {
925 struct ath10k *ar = ce_state->ar;
926 struct ath10k_ce *ce = ath10k_ce_priv(ar);
927 int ret;
928
929 spin_lock_bh(&ce->ce_lock);
930 ret = ce_state->ops->ce_completed_recv_next_nolock(ce_state,
931 per_transfer_contextp,
932 nbytesp);
933
934 spin_unlock_bh(&ce->ce_lock);
935
936 return ret;
937 }
938 EXPORT_SYMBOL(ath10k_ce_completed_recv_next);
939
_ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe * ce_state,void ** per_transfer_contextp,dma_addr_t * bufferp)940 static int _ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
941 void **per_transfer_contextp,
942 dma_addr_t *bufferp)
943 {
944 struct ath10k_ce_ring *dest_ring;
945 unsigned int nentries_mask;
946 unsigned int sw_index;
947 unsigned int write_index;
948 int ret;
949 struct ath10k *ar;
950 struct ath10k_ce *ce;
951
952 dest_ring = ce_state->dest_ring;
953
954 if (!dest_ring)
955 return -EIO;
956
957 ar = ce_state->ar;
958 ce = ath10k_ce_priv(ar);
959
960 spin_lock_bh(&ce->ce_lock);
961
962 nentries_mask = dest_ring->nentries_mask;
963 sw_index = dest_ring->sw_index;
964 write_index = dest_ring->write_index;
965 if (write_index != sw_index) {
966 struct ce_desc *base = dest_ring->base_addr_owner_space;
967 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
968
969 /* Return data from completed destination descriptor */
970 *bufferp = __le32_to_cpu(desc->addr);
971
972 if (per_transfer_contextp)
973 *per_transfer_contextp =
974 dest_ring->per_transfer_context[sw_index];
975
976 /* sanity */
977 dest_ring->per_transfer_context[sw_index] = NULL;
978 desc->nbytes = 0;
979
980 /* Update sw_index */
981 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
982 dest_ring->sw_index = sw_index;
983 ret = 0;
984 } else {
985 ret = -EIO;
986 }
987
988 spin_unlock_bh(&ce->ce_lock);
989
990 return ret;
991 }
992
_ath10k_ce_revoke_recv_next_64(struct ath10k_ce_pipe * ce_state,void ** per_transfer_contextp,dma_addr_t * bufferp)993 static int _ath10k_ce_revoke_recv_next_64(struct ath10k_ce_pipe *ce_state,
994 void **per_transfer_contextp,
995 dma_addr_t *bufferp)
996 {
997 struct ath10k_ce_ring *dest_ring;
998 unsigned int nentries_mask;
999 unsigned int sw_index;
1000 unsigned int write_index;
1001 int ret;
1002 struct ath10k *ar;
1003 struct ath10k_ce *ce;
1004
1005 dest_ring = ce_state->dest_ring;
1006
1007 if (!dest_ring)
1008 return -EIO;
1009
1010 ar = ce_state->ar;
1011 ce = ath10k_ce_priv(ar);
1012
1013 spin_lock_bh(&ce->ce_lock);
1014
1015 nentries_mask = dest_ring->nentries_mask;
1016 sw_index = dest_ring->sw_index;
1017 write_index = dest_ring->write_index;
1018 if (write_index != sw_index) {
1019 struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
1020 struct ce_desc_64 *desc =
1021 CE_DEST_RING_TO_DESC_64(base, sw_index);
1022
1023 /* Return data from completed destination descriptor */
1024 *bufferp = __le64_to_cpu(desc->addr);
1025
1026 if (per_transfer_contextp)
1027 *per_transfer_contextp =
1028 dest_ring->per_transfer_context[sw_index];
1029
1030 /* sanity */
1031 dest_ring->per_transfer_context[sw_index] = NULL;
1032 desc->nbytes = 0;
1033
1034 /* Update sw_index */
1035 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1036 dest_ring->sw_index = sw_index;
1037 ret = 0;
1038 } else {
1039 ret = -EIO;
1040 }
1041
1042 spin_unlock_bh(&ce->ce_lock);
1043
1044 return ret;
1045 }
1046
ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe * ce_state,void ** per_transfer_contextp,dma_addr_t * bufferp)1047 int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
1048 void **per_transfer_contextp,
1049 dma_addr_t *bufferp)
1050 {
1051 return ce_state->ops->ce_revoke_recv_next(ce_state,
1052 per_transfer_contextp,
1053 bufferp);
1054 }
1055 EXPORT_SYMBOL(ath10k_ce_revoke_recv_next);
1056
1057 /*
1058 * Guts of ath10k_ce_completed_send_next.
1059 * The caller takes responsibility for any necessary locking.
1060 */
_ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe * ce_state,void ** per_transfer_contextp)1061 static int _ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
1062 void **per_transfer_contextp)
1063 {
1064 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1065 u32 ctrl_addr = ce_state->ctrl_addr;
1066 struct ath10k *ar = ce_state->ar;
1067 unsigned int nentries_mask = src_ring->nentries_mask;
1068 unsigned int sw_index = src_ring->sw_index;
1069 unsigned int read_index;
1070 struct ce_desc *desc;
1071
1072 if (src_ring->hw_index == sw_index) {
1073 /*
1074 * The SW completion index has caught up with the cached
1075 * version of the HW completion index.
1076 * Update the cached HW completion index to see whether
1077 * the SW has really caught up to the HW, or if the cached
1078 * value of the HW index has become stale.
1079 */
1080
1081 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1082 if (read_index == 0xffffffff)
1083 return -ENODEV;
1084
1085 read_index &= nentries_mask;
1086 src_ring->hw_index = read_index;
1087 }
1088
1089 if (ar->hw_params.rri_on_ddr)
1090 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1091 else
1092 read_index = src_ring->hw_index;
1093
1094 if (read_index == sw_index)
1095 return -EIO;
1096
1097 if (per_transfer_contextp)
1098 *per_transfer_contextp =
1099 src_ring->per_transfer_context[sw_index];
1100
1101 /* sanity */
1102 src_ring->per_transfer_context[sw_index] = NULL;
1103 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
1104 sw_index);
1105 desc->nbytes = 0;
1106
1107 /* Update sw_index */
1108 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1109 src_ring->sw_index = sw_index;
1110
1111 return 0;
1112 }
1113
_ath10k_ce_completed_send_next_nolock_64(struct ath10k_ce_pipe * ce_state,void ** per_transfer_contextp)1114 static int _ath10k_ce_completed_send_next_nolock_64(struct ath10k_ce_pipe *ce_state,
1115 void **per_transfer_contextp)
1116 {
1117 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1118 u32 ctrl_addr = ce_state->ctrl_addr;
1119 struct ath10k *ar = ce_state->ar;
1120 unsigned int nentries_mask = src_ring->nentries_mask;
1121 unsigned int sw_index = src_ring->sw_index;
1122 unsigned int read_index;
1123 struct ce_desc_64 *desc;
1124
1125 if (src_ring->hw_index == sw_index) {
1126 /*
1127 * The SW completion index has caught up with the cached
1128 * version of the HW completion index.
1129 * Update the cached HW completion index to see whether
1130 * the SW has really caught up to the HW, or if the cached
1131 * value of the HW index has become stale.
1132 */
1133
1134 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1135 if (read_index == 0xffffffff)
1136 return -ENODEV;
1137
1138 read_index &= nentries_mask;
1139 src_ring->hw_index = read_index;
1140 }
1141
1142 if (ar->hw_params.rri_on_ddr)
1143 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1144 else
1145 read_index = src_ring->hw_index;
1146
1147 if (read_index == sw_index)
1148 return -EIO;
1149
1150 if (per_transfer_contextp)
1151 *per_transfer_contextp =
1152 src_ring->per_transfer_context[sw_index];
1153
1154 /* sanity */
1155 src_ring->per_transfer_context[sw_index] = NULL;
1156 desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,
1157 sw_index);
1158 desc->nbytes = 0;
1159
1160 /* Update sw_index */
1161 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1162 src_ring->sw_index = sw_index;
1163
1164 return 0;
1165 }
1166
ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe * ce_state,void ** per_transfer_contextp)1167 int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
1168 void **per_transfer_contextp)
1169 {
1170 return ce_state->ops->ce_completed_send_next_nolock(ce_state,
1171 per_transfer_contextp);
1172 }
1173 EXPORT_SYMBOL(ath10k_ce_completed_send_next_nolock);
1174
ath10k_ce_extract_desc_data(struct ath10k * ar,struct ath10k_ce_ring * src_ring,u32 sw_index,dma_addr_t * bufferp,u32 * nbytesp,u32 * transfer_idp)1175 static void ath10k_ce_extract_desc_data(struct ath10k *ar,
1176 struct ath10k_ce_ring *src_ring,
1177 u32 sw_index,
1178 dma_addr_t *bufferp,
1179 u32 *nbytesp,
1180 u32 *transfer_idp)
1181 {
1182 struct ce_desc *base = src_ring->base_addr_owner_space;
1183 struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index);
1184
1185 /* Return data from completed source descriptor */
1186 *bufferp = __le32_to_cpu(desc->addr);
1187 *nbytesp = __le16_to_cpu(desc->nbytes);
1188 *transfer_idp = MS(__le16_to_cpu(desc->flags),
1189 CE_DESC_FLAGS_META_DATA);
1190 }
1191
ath10k_ce_extract_desc_data_64(struct ath10k * ar,struct ath10k_ce_ring * src_ring,u32 sw_index,dma_addr_t * bufferp,u32 * nbytesp,u32 * transfer_idp)1192 static void ath10k_ce_extract_desc_data_64(struct ath10k *ar,
1193 struct ath10k_ce_ring *src_ring,
1194 u32 sw_index,
1195 dma_addr_t *bufferp,
1196 u32 *nbytesp,
1197 u32 *transfer_idp)
1198 {
1199 struct ce_desc_64 *base = src_ring->base_addr_owner_space;
1200 struct ce_desc_64 *desc =
1201 CE_SRC_RING_TO_DESC_64(base, sw_index);
1202
1203 /* Return data from completed source descriptor */
1204 *bufferp = __le64_to_cpu(desc->addr);
1205 *nbytesp = __le16_to_cpu(desc->nbytes);
1206 *transfer_idp = MS(__le16_to_cpu(desc->flags),
1207 CE_DESC_FLAGS_META_DATA);
1208 }
1209
1210 /* NB: Modeled after ath10k_ce_completed_send_next */
ath10k_ce_cancel_send_next(struct ath10k_ce_pipe * ce_state,void ** per_transfer_contextp,dma_addr_t * bufferp,unsigned int * nbytesp,unsigned int * transfer_idp)1211 int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state,
1212 void **per_transfer_contextp,
1213 dma_addr_t *bufferp,
1214 unsigned int *nbytesp,
1215 unsigned int *transfer_idp)
1216 {
1217 struct ath10k_ce_ring *src_ring;
1218 unsigned int nentries_mask;
1219 unsigned int sw_index;
1220 unsigned int write_index;
1221 int ret;
1222 struct ath10k *ar;
1223 struct ath10k_ce *ce;
1224
1225 src_ring = ce_state->src_ring;
1226
1227 if (!src_ring)
1228 return -EIO;
1229
1230 ar = ce_state->ar;
1231 ce = ath10k_ce_priv(ar);
1232
1233 spin_lock_bh(&ce->ce_lock);
1234
1235 nentries_mask = src_ring->nentries_mask;
1236 sw_index = src_ring->sw_index;
1237 write_index = src_ring->write_index;
1238
1239 if (write_index != sw_index) {
1240 ce_state->ops->ce_extract_desc_data(ar, src_ring, sw_index,
1241 bufferp, nbytesp,
1242 transfer_idp);
1243
1244 if (per_transfer_contextp)
1245 *per_transfer_contextp =
1246 src_ring->per_transfer_context[sw_index];
1247
1248 /* sanity */
1249 src_ring->per_transfer_context[sw_index] = NULL;
1250
1251 /* Update sw_index */
1252 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1253 src_ring->sw_index = sw_index;
1254 ret = 0;
1255 } else {
1256 ret = -EIO;
1257 }
1258
1259 spin_unlock_bh(&ce->ce_lock);
1260
1261 return ret;
1262 }
1263 EXPORT_SYMBOL(ath10k_ce_cancel_send_next);
1264
ath10k_ce_completed_send_next(struct ath10k_ce_pipe * ce_state,void ** per_transfer_contextp)1265 int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state,
1266 void **per_transfer_contextp)
1267 {
1268 struct ath10k *ar = ce_state->ar;
1269 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1270 int ret;
1271
1272 spin_lock_bh(&ce->ce_lock);
1273 ret = ath10k_ce_completed_send_next_nolock(ce_state,
1274 per_transfer_contextp);
1275 spin_unlock_bh(&ce->ce_lock);
1276
1277 return ret;
1278 }
1279 EXPORT_SYMBOL(ath10k_ce_completed_send_next);
1280
1281 /*
1282 * Guts of interrupt handler for per-engine interrupts on a particular CE.
1283 *
1284 * Invokes registered callbacks for recv_complete,
1285 * send_complete, and watermarks.
1286 */
ath10k_ce_per_engine_service(struct ath10k * ar,unsigned int ce_id)1287 void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id)
1288 {
1289 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1290 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1291 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
1292 u32 ctrl_addr = ce_state->ctrl_addr;
1293
1294 /*
1295 * Clear before handling
1296 *
1297 * Misc CE interrupts are not being handled, but still need
1298 * to be cleared.
1299 *
1300 * NOTE: When the last copy engine interrupt is cleared the
1301 * hardware will go to sleep. Once this happens any access to
1302 * the CE registers can cause a hardware fault.
1303 */
1304 ath10k_ce_engine_int_status_clear(ar, ctrl_addr,
1305 wm_regs->cc_mask | wm_regs->wm_mask);
1306
1307 if (ce_state->recv_cb)
1308 ce_state->recv_cb(ce_state);
1309
1310 if (ce_state->send_cb)
1311 ce_state->send_cb(ce_state);
1312 }
1313 EXPORT_SYMBOL(ath10k_ce_per_engine_service);
1314
1315 /*
1316 * Handler for per-engine interrupts on ALL active CEs.
1317 * This is used in cases where the system is sharing a
1318 * single interrupt for all CEs
1319 */
1320
ath10k_ce_per_engine_service_any(struct ath10k * ar)1321 void ath10k_ce_per_engine_service_any(struct ath10k *ar)
1322 {
1323 int ce_id;
1324 u32 intr_summary;
1325
1326 intr_summary = ath10k_ce_interrupt_summary(ar);
1327
1328 for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) {
1329 if (intr_summary & (1 << ce_id))
1330 intr_summary &= ~(1 << ce_id);
1331 else
1332 /* no intr pending on this CE */
1333 continue;
1334
1335 ath10k_ce_per_engine_service(ar, ce_id);
1336 }
1337 }
1338 EXPORT_SYMBOL(ath10k_ce_per_engine_service_any);
1339
1340 /*
1341 * Adjust interrupts for the copy complete handler.
1342 * If it's needed for either send or recv, then unmask
1343 * this interrupt; otherwise, mask it.
1344 *
1345 * Called with ce_lock held.
1346 */
ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe * ce_state)1347 static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state)
1348 {
1349 u32 ctrl_addr = ce_state->ctrl_addr;
1350 struct ath10k *ar = ce_state->ar;
1351 bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR;
1352
1353 if ((!disable_copy_compl_intr) &&
1354 (ce_state->send_cb || ce_state->recv_cb))
1355 ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr);
1356 else
1357 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
1358
1359 ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
1360 }
1361
ath10k_ce_disable_interrupt(struct ath10k * ar,int ce_id)1362 void ath10k_ce_disable_interrupt(struct ath10k *ar, int ce_id)
1363 {
1364 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1365 struct ath10k_ce_pipe *ce_state;
1366 u32 ctrl_addr;
1367
1368 ce_state = &ce->ce_states[ce_id];
1369 if (ce_state->attr_flags & CE_ATTR_POLL)
1370 return;
1371
1372 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1373
1374 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
1375 ath10k_ce_error_intr_disable(ar, ctrl_addr);
1376 ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
1377 }
1378 EXPORT_SYMBOL(ath10k_ce_disable_interrupt);
1379
ath10k_ce_disable_interrupts(struct ath10k * ar)1380 void ath10k_ce_disable_interrupts(struct ath10k *ar)
1381 {
1382 int ce_id;
1383
1384 for (ce_id = 0; ce_id < CE_COUNT; ce_id++)
1385 ath10k_ce_disable_interrupt(ar, ce_id);
1386 }
1387 EXPORT_SYMBOL(ath10k_ce_disable_interrupts);
1388
ath10k_ce_enable_interrupt(struct ath10k * ar,int ce_id)1389 void ath10k_ce_enable_interrupt(struct ath10k *ar, int ce_id)
1390 {
1391 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1392 struct ath10k_ce_pipe *ce_state;
1393
1394 ce_state = &ce->ce_states[ce_id];
1395 if (ce_state->attr_flags & CE_ATTR_POLL)
1396 return;
1397
1398 ath10k_ce_per_engine_handler_adjust(ce_state);
1399 }
1400 EXPORT_SYMBOL(ath10k_ce_enable_interrupt);
1401
ath10k_ce_enable_interrupts(struct ath10k * ar)1402 void ath10k_ce_enable_interrupts(struct ath10k *ar)
1403 {
1404 int ce_id;
1405
1406 /* Enable interrupts for copy engine that
1407 * are not using polling mode.
1408 */
1409 for (ce_id = 0; ce_id < CE_COUNT; ce_id++)
1410 ath10k_ce_enable_interrupt(ar, ce_id);
1411 }
1412 EXPORT_SYMBOL(ath10k_ce_enable_interrupts);
1413
ath10k_ce_init_src_ring(struct ath10k * ar,unsigned int ce_id,const struct ce_attr * attr)1414 static int ath10k_ce_init_src_ring(struct ath10k *ar,
1415 unsigned int ce_id,
1416 const struct ce_attr *attr)
1417 {
1418 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1419 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1420 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1421 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1422
1423 nentries = roundup_pow_of_two(attr->src_nentries);
1424
1425 if (ar->hw_params.target_64bit)
1426 memset(src_ring->base_addr_owner_space, 0,
1427 nentries * sizeof(struct ce_desc_64));
1428 else
1429 memset(src_ring->base_addr_owner_space, 0,
1430 nentries * sizeof(struct ce_desc));
1431
1432 src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1433 src_ring->sw_index &= src_ring->nentries_mask;
1434 src_ring->hw_index = src_ring->sw_index;
1435
1436 src_ring->write_index =
1437 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr);
1438 src_ring->write_index &= src_ring->nentries_mask;
1439
1440 ath10k_ce_src_ring_base_addr_set(ar, ce_id,
1441 src_ring->base_addr_ce_space);
1442 ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries);
1443 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max);
1444 ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0);
1445 ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0);
1446 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries);
1447
1448 ath10k_dbg(ar, ATH10K_DBG_BOOT,
1449 "boot init ce src ring id %d entries %d base_addr %pK\n",
1450 ce_id, nentries, src_ring->base_addr_owner_space);
1451
1452 return 0;
1453 }
1454
ath10k_ce_init_dest_ring(struct ath10k * ar,unsigned int ce_id,const struct ce_attr * attr)1455 static int ath10k_ce_init_dest_ring(struct ath10k *ar,
1456 unsigned int ce_id,
1457 const struct ce_attr *attr)
1458 {
1459 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1460 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1461 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
1462 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1463
1464 nentries = roundup_pow_of_two(attr->dest_nentries);
1465
1466 if (ar->hw_params.target_64bit)
1467 memset(dest_ring->base_addr_owner_space, 0,
1468 nentries * sizeof(struct ce_desc_64));
1469 else
1470 memset(dest_ring->base_addr_owner_space, 0,
1471 nentries * sizeof(struct ce_desc));
1472
1473 dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr);
1474 dest_ring->sw_index &= dest_ring->nentries_mask;
1475 dest_ring->write_index =
1476 ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
1477 dest_ring->write_index &= dest_ring->nentries_mask;
1478
1479 ath10k_ce_dest_ring_base_addr_set(ar, ce_id,
1480 dest_ring->base_addr_ce_space);
1481 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries);
1482 ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0);
1483 ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0);
1484 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries);
1485
1486 ath10k_dbg(ar, ATH10K_DBG_BOOT,
1487 "boot ce dest ring id %d entries %d base_addr %pK\n",
1488 ce_id, nentries, dest_ring->base_addr_owner_space);
1489
1490 return 0;
1491 }
1492
ath10k_ce_alloc_shadow_base(struct ath10k * ar,struct ath10k_ce_ring * src_ring,u32 nentries)1493 static int ath10k_ce_alloc_shadow_base(struct ath10k *ar,
1494 struct ath10k_ce_ring *src_ring,
1495 u32 nentries)
1496 {
1497 src_ring->shadow_base_unaligned = kcalloc(nentries,
1498 sizeof(struct ce_desc_64),
1499 GFP_KERNEL);
1500 if (!src_ring->shadow_base_unaligned)
1501 return -ENOMEM;
1502
1503 src_ring->shadow_base = (struct ce_desc_64 *)
1504 PTR_ALIGN(src_ring->shadow_base_unaligned,
1505 CE_DESC_RING_ALIGN);
1506 return 0;
1507 }
1508
1509 static struct ath10k_ce_ring *
ath10k_ce_alloc_src_ring(struct ath10k * ar,unsigned int ce_id,const struct ce_attr * attr)1510 ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id,
1511 const struct ce_attr *attr)
1512 {
1513 struct ath10k_ce_ring *src_ring;
1514 u32 nentries = attr->src_nentries;
1515 dma_addr_t base_addr;
1516 int ret;
1517
1518 nentries = roundup_pow_of_two(nentries);
1519
1520 src_ring = kzalloc(struct_size(src_ring, per_transfer_context,
1521 nentries), GFP_KERNEL);
1522 if (src_ring == NULL)
1523 return ERR_PTR(-ENOMEM);
1524
1525 src_ring->nentries = nentries;
1526 src_ring->nentries_mask = nentries - 1;
1527
1528 /*
1529 * Legacy platforms that do not support cache
1530 * coherent DMA are unsupported
1531 */
1532 src_ring->base_addr_owner_space_unaligned =
1533 dma_alloc_coherent(ar->dev,
1534 (nentries * sizeof(struct ce_desc) +
1535 CE_DESC_RING_ALIGN),
1536 &base_addr, GFP_KERNEL);
1537 if (!src_ring->base_addr_owner_space_unaligned) {
1538 kfree(src_ring);
1539 return ERR_PTR(-ENOMEM);
1540 }
1541
1542 src_ring->base_addr_ce_space_unaligned = base_addr;
1543
1544 src_ring->base_addr_owner_space =
1545 PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
1546 CE_DESC_RING_ALIGN);
1547 src_ring->base_addr_ce_space =
1548 ALIGN(src_ring->base_addr_ce_space_unaligned,
1549 CE_DESC_RING_ALIGN);
1550
1551 if (ar->hw_params.shadow_reg_support) {
1552 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
1553 if (ret) {
1554 dma_free_coherent(ar->dev,
1555 (nentries * sizeof(struct ce_desc) +
1556 CE_DESC_RING_ALIGN),
1557 src_ring->base_addr_owner_space_unaligned,
1558 base_addr);
1559 kfree(src_ring);
1560 return ERR_PTR(ret);
1561 }
1562 }
1563
1564 return src_ring;
1565 }
1566
1567 static struct ath10k_ce_ring *
ath10k_ce_alloc_src_ring_64(struct ath10k * ar,unsigned int ce_id,const struct ce_attr * attr)1568 ath10k_ce_alloc_src_ring_64(struct ath10k *ar, unsigned int ce_id,
1569 const struct ce_attr *attr)
1570 {
1571 struct ath10k_ce_ring *src_ring;
1572 u32 nentries = attr->src_nentries;
1573 dma_addr_t base_addr;
1574 int ret;
1575
1576 nentries = roundup_pow_of_two(nentries);
1577
1578 src_ring = kzalloc(struct_size(src_ring, per_transfer_context,
1579 nentries), GFP_KERNEL);
1580 if (!src_ring)
1581 return ERR_PTR(-ENOMEM);
1582
1583 src_ring->nentries = nentries;
1584 src_ring->nentries_mask = nentries - 1;
1585
1586 /* Legacy platforms that do not support cache
1587 * coherent DMA are unsupported
1588 */
1589 src_ring->base_addr_owner_space_unaligned =
1590 dma_alloc_coherent(ar->dev,
1591 (nentries * sizeof(struct ce_desc_64) +
1592 CE_DESC_RING_ALIGN),
1593 &base_addr, GFP_KERNEL);
1594 if (!src_ring->base_addr_owner_space_unaligned) {
1595 kfree(src_ring);
1596 return ERR_PTR(-ENOMEM);
1597 }
1598
1599 src_ring->base_addr_ce_space_unaligned = base_addr;
1600
1601 src_ring->base_addr_owner_space =
1602 PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
1603 CE_DESC_RING_ALIGN);
1604 src_ring->base_addr_ce_space =
1605 ALIGN(src_ring->base_addr_ce_space_unaligned,
1606 CE_DESC_RING_ALIGN);
1607
1608 if (ar->hw_params.shadow_reg_support) {
1609 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
1610 if (ret) {
1611 dma_free_coherent(ar->dev,
1612 (nentries * sizeof(struct ce_desc_64) +
1613 CE_DESC_RING_ALIGN),
1614 src_ring->base_addr_owner_space_unaligned,
1615 base_addr);
1616 kfree(src_ring);
1617 return ERR_PTR(ret);
1618 }
1619 }
1620
1621 return src_ring;
1622 }
1623
1624 static struct ath10k_ce_ring *
ath10k_ce_alloc_dest_ring(struct ath10k * ar,unsigned int ce_id,const struct ce_attr * attr)1625 ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id,
1626 const struct ce_attr *attr)
1627 {
1628 struct ath10k_ce_ring *dest_ring;
1629 u32 nentries;
1630 dma_addr_t base_addr;
1631
1632 nentries = roundup_pow_of_two(attr->dest_nentries);
1633
1634 dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,
1635 nentries), GFP_KERNEL);
1636 if (dest_ring == NULL)
1637 return ERR_PTR(-ENOMEM);
1638
1639 dest_ring->nentries = nentries;
1640 dest_ring->nentries_mask = nentries - 1;
1641
1642 /*
1643 * Legacy platforms that do not support cache
1644 * coherent DMA are unsupported
1645 */
1646 dest_ring->base_addr_owner_space_unaligned =
1647 dma_alloc_coherent(ar->dev,
1648 (nentries * sizeof(struct ce_desc) +
1649 CE_DESC_RING_ALIGN),
1650 &base_addr, GFP_KERNEL);
1651 if (!dest_ring->base_addr_owner_space_unaligned) {
1652 kfree(dest_ring);
1653 return ERR_PTR(-ENOMEM);
1654 }
1655
1656 dest_ring->base_addr_ce_space_unaligned = base_addr;
1657
1658 dest_ring->base_addr_owner_space =
1659 PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
1660 CE_DESC_RING_ALIGN);
1661 dest_ring->base_addr_ce_space =
1662 ALIGN(dest_ring->base_addr_ce_space_unaligned,
1663 CE_DESC_RING_ALIGN);
1664
1665 return dest_ring;
1666 }
1667
1668 static struct ath10k_ce_ring *
ath10k_ce_alloc_dest_ring_64(struct ath10k * ar,unsigned int ce_id,const struct ce_attr * attr)1669 ath10k_ce_alloc_dest_ring_64(struct ath10k *ar, unsigned int ce_id,
1670 const struct ce_attr *attr)
1671 {
1672 struct ath10k_ce_ring *dest_ring;
1673 u32 nentries;
1674 dma_addr_t base_addr;
1675
1676 nentries = roundup_pow_of_two(attr->dest_nentries);
1677
1678 dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,
1679 nentries), GFP_KERNEL);
1680 if (!dest_ring)
1681 return ERR_PTR(-ENOMEM);
1682
1683 dest_ring->nentries = nentries;
1684 dest_ring->nentries_mask = nentries - 1;
1685
1686 /* Legacy platforms that do not support cache
1687 * coherent DMA are unsupported
1688 */
1689 dest_ring->base_addr_owner_space_unaligned =
1690 dma_alloc_coherent(ar->dev,
1691 (nentries * sizeof(struct ce_desc_64) +
1692 CE_DESC_RING_ALIGN),
1693 &base_addr, GFP_KERNEL);
1694 if (!dest_ring->base_addr_owner_space_unaligned) {
1695 kfree(dest_ring);
1696 return ERR_PTR(-ENOMEM);
1697 }
1698
1699 dest_ring->base_addr_ce_space_unaligned = base_addr;
1700
1701 /* Correctly initialize memory to 0 to prevent garbage
1702 * data crashing system when download firmware
1703 */
1704 dest_ring->base_addr_owner_space =
1705 PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
1706 CE_DESC_RING_ALIGN);
1707 dest_ring->base_addr_ce_space =
1708 ALIGN(dest_ring->base_addr_ce_space_unaligned,
1709 CE_DESC_RING_ALIGN);
1710
1711 return dest_ring;
1712 }
1713
1714 /*
1715 * Initialize a Copy Engine based on caller-supplied attributes.
1716 * This may be called once to initialize both source and destination
1717 * rings or it may be called twice for separate source and destination
1718 * initialization. It may be that only one side or the other is
1719 * initialized by software/firmware.
1720 */
ath10k_ce_init_pipe(struct ath10k * ar,unsigned int ce_id,const struct ce_attr * attr)1721 int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id,
1722 const struct ce_attr *attr)
1723 {
1724 int ret;
1725
1726 if (attr->src_nentries) {
1727 ret = ath10k_ce_init_src_ring(ar, ce_id, attr);
1728 if (ret) {
1729 ath10k_err(ar, "Failed to initialize CE src ring for ID: %d (%d)\n",
1730 ce_id, ret);
1731 return ret;
1732 }
1733 }
1734
1735 if (attr->dest_nentries) {
1736 ret = ath10k_ce_init_dest_ring(ar, ce_id, attr);
1737 if (ret) {
1738 ath10k_err(ar, "Failed to initialize CE dest ring for ID: %d (%d)\n",
1739 ce_id, ret);
1740 return ret;
1741 }
1742 }
1743
1744 return 0;
1745 }
1746 EXPORT_SYMBOL(ath10k_ce_init_pipe);
1747
ath10k_ce_deinit_src_ring(struct ath10k * ar,unsigned int ce_id)1748 static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id)
1749 {
1750 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1751
1752 ath10k_ce_src_ring_base_addr_set(ar, ce_id, 0);
1753 ath10k_ce_src_ring_size_set(ar, ctrl_addr, 0);
1754 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, 0);
1755 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, 0);
1756 }
1757
ath10k_ce_deinit_dest_ring(struct ath10k * ar,unsigned int ce_id)1758 static void ath10k_ce_deinit_dest_ring(struct ath10k *ar, unsigned int ce_id)
1759 {
1760 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1761
1762 ath10k_ce_dest_ring_base_addr_set(ar, ce_id, 0);
1763 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, 0);
1764 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, 0);
1765 }
1766
ath10k_ce_deinit_pipe(struct ath10k * ar,unsigned int ce_id)1767 void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id)
1768 {
1769 ath10k_ce_deinit_src_ring(ar, ce_id);
1770 ath10k_ce_deinit_dest_ring(ar, ce_id);
1771 }
1772 EXPORT_SYMBOL(ath10k_ce_deinit_pipe);
1773
_ath10k_ce_free_pipe(struct ath10k * ar,int ce_id)1774 static void _ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1775 {
1776 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1777 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1778
1779 if (ce_state->src_ring) {
1780 if (ar->hw_params.shadow_reg_support)
1781 kfree(ce_state->src_ring->shadow_base_unaligned);
1782 dma_free_coherent(ar->dev,
1783 (ce_state->src_ring->nentries *
1784 sizeof(struct ce_desc) +
1785 CE_DESC_RING_ALIGN),
1786 ce_state->src_ring->base_addr_owner_space,
1787 ce_state->src_ring->base_addr_ce_space);
1788 kfree(ce_state->src_ring);
1789 }
1790
1791 if (ce_state->dest_ring) {
1792 dma_free_coherent(ar->dev,
1793 (ce_state->dest_ring->nentries *
1794 sizeof(struct ce_desc) +
1795 CE_DESC_RING_ALIGN),
1796 ce_state->dest_ring->base_addr_owner_space,
1797 ce_state->dest_ring->base_addr_ce_space);
1798 kfree(ce_state->dest_ring);
1799 }
1800
1801 ce_state->src_ring = NULL;
1802 ce_state->dest_ring = NULL;
1803 }
1804
_ath10k_ce_free_pipe_64(struct ath10k * ar,int ce_id)1805 static void _ath10k_ce_free_pipe_64(struct ath10k *ar, int ce_id)
1806 {
1807 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1808 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1809
1810 if (ce_state->src_ring) {
1811 if (ar->hw_params.shadow_reg_support)
1812 kfree(ce_state->src_ring->shadow_base_unaligned);
1813 dma_free_coherent(ar->dev,
1814 (ce_state->src_ring->nentries *
1815 sizeof(struct ce_desc_64) +
1816 CE_DESC_RING_ALIGN),
1817 ce_state->src_ring->base_addr_owner_space,
1818 ce_state->src_ring->base_addr_ce_space);
1819 kfree(ce_state->src_ring);
1820 }
1821
1822 if (ce_state->dest_ring) {
1823 dma_free_coherent(ar->dev,
1824 (ce_state->dest_ring->nentries *
1825 sizeof(struct ce_desc_64) +
1826 CE_DESC_RING_ALIGN),
1827 ce_state->dest_ring->base_addr_owner_space,
1828 ce_state->dest_ring->base_addr_ce_space);
1829 kfree(ce_state->dest_ring);
1830 }
1831
1832 ce_state->src_ring = NULL;
1833 ce_state->dest_ring = NULL;
1834 }
1835
ath10k_ce_free_pipe(struct ath10k * ar,int ce_id)1836 void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1837 {
1838 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1839 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1840
1841 ce_state->ops->ce_free_pipe(ar, ce_id);
1842 }
1843 EXPORT_SYMBOL(ath10k_ce_free_pipe);
1844
ath10k_ce_dump_registers(struct ath10k * ar,struct ath10k_fw_crash_data * crash_data)1845 void ath10k_ce_dump_registers(struct ath10k *ar,
1846 struct ath10k_fw_crash_data *crash_data)
1847 {
1848 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1849 struct ath10k_ce_crash_data ce_data;
1850 u32 addr, id;
1851
1852 lockdep_assert_held(&ar->dump_mutex);
1853
1854 ath10k_err(ar, "Copy Engine register dump:\n");
1855
1856 spin_lock_bh(&ce->ce_lock);
1857 for (id = 0; id < CE_COUNT; id++) {
1858 addr = ath10k_ce_base_address(ar, id);
1859 ce_data.base_addr = cpu_to_le32(addr);
1860
1861 ce_data.src_wr_idx =
1862 cpu_to_le32(ath10k_ce_src_ring_write_index_get(ar, addr));
1863 ce_data.src_r_idx =
1864 cpu_to_le32(ath10k_ce_src_ring_read_index_get(ar, addr));
1865 ce_data.dst_wr_idx =
1866 cpu_to_le32(ath10k_ce_dest_ring_write_index_get(ar, addr));
1867 ce_data.dst_r_idx =
1868 cpu_to_le32(ath10k_ce_dest_ring_read_index_get(ar, addr));
1869
1870 if (crash_data)
1871 crash_data->ce_crash_data[id] = ce_data;
1872
1873 ath10k_err(ar, "[%02d]: 0x%08x %3u %3u %3u %3u", id,
1874 le32_to_cpu(ce_data.base_addr),
1875 le32_to_cpu(ce_data.src_wr_idx),
1876 le32_to_cpu(ce_data.src_r_idx),
1877 le32_to_cpu(ce_data.dst_wr_idx),
1878 le32_to_cpu(ce_data.dst_r_idx));
1879 }
1880
1881 spin_unlock_bh(&ce->ce_lock);
1882 }
1883 EXPORT_SYMBOL(ath10k_ce_dump_registers);
1884
1885 static const struct ath10k_ce_ops ce_ops = {
1886 .ce_alloc_src_ring = ath10k_ce_alloc_src_ring,
1887 .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring,
1888 .ce_rx_post_buf = __ath10k_ce_rx_post_buf,
1889 .ce_completed_recv_next_nolock = _ath10k_ce_completed_recv_next_nolock,
1890 .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next,
1891 .ce_extract_desc_data = ath10k_ce_extract_desc_data,
1892 .ce_free_pipe = _ath10k_ce_free_pipe,
1893 .ce_send_nolock = _ath10k_ce_send_nolock,
1894 .ce_set_src_ring_base_addr_hi = NULL,
1895 .ce_set_dest_ring_base_addr_hi = NULL,
1896 .ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock,
1897 };
1898
1899 static const struct ath10k_ce_ops ce_64_ops = {
1900 .ce_alloc_src_ring = ath10k_ce_alloc_src_ring_64,
1901 .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring_64,
1902 .ce_rx_post_buf = __ath10k_ce_rx_post_buf_64,
1903 .ce_completed_recv_next_nolock =
1904 _ath10k_ce_completed_recv_next_nolock_64,
1905 .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next_64,
1906 .ce_extract_desc_data = ath10k_ce_extract_desc_data_64,
1907 .ce_free_pipe = _ath10k_ce_free_pipe_64,
1908 .ce_send_nolock = _ath10k_ce_send_nolock_64,
1909 .ce_set_src_ring_base_addr_hi = ath10k_ce_set_src_ring_base_addr_hi,
1910 .ce_set_dest_ring_base_addr_hi = ath10k_ce_set_dest_ring_base_addr_hi,
1911 .ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock_64,
1912 };
1913
ath10k_ce_set_ops(struct ath10k * ar,struct ath10k_ce_pipe * ce_state)1914 static void ath10k_ce_set_ops(struct ath10k *ar,
1915 struct ath10k_ce_pipe *ce_state)
1916 {
1917 switch (ar->hw_rev) {
1918 case ATH10K_HW_WCN3990:
1919 ce_state->ops = &ce_64_ops;
1920 break;
1921 default:
1922 ce_state->ops = &ce_ops;
1923 break;
1924 }
1925 }
1926
ath10k_ce_alloc_pipe(struct ath10k * ar,int ce_id,const struct ce_attr * attr)1927 int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id,
1928 const struct ce_attr *attr)
1929 {
1930 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1931 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1932 int ret;
1933
1934 ath10k_ce_set_ops(ar, ce_state);
1935 /* Make sure there's enough CE ringbuffer entries for HTT TX to avoid
1936 * additional TX locking checks.
1937 *
1938 * For the lack of a better place do the check here.
1939 */
1940 BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC >
1941 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1942 BUILD_BUG_ON(2 * TARGET_10_4_NUM_MSDU_DESC_PFC >
1943 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1944 BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC >
1945 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1946
1947 ce_state->ar = ar;
1948 ce_state->id = ce_id;
1949 ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1950 ce_state->attr_flags = attr->flags;
1951 ce_state->src_sz_max = attr->src_sz_max;
1952
1953 if (attr->src_nentries)
1954 ce_state->send_cb = attr->send_cb;
1955
1956 if (attr->dest_nentries)
1957 ce_state->recv_cb = attr->recv_cb;
1958
1959 if (attr->src_nentries) {
1960 ce_state->src_ring =
1961 ce_state->ops->ce_alloc_src_ring(ar, ce_id, attr);
1962 if (IS_ERR(ce_state->src_ring)) {
1963 ret = PTR_ERR(ce_state->src_ring);
1964 ath10k_err(ar, "failed to alloc CE src ring %d: %d\n",
1965 ce_id, ret);
1966 ce_state->src_ring = NULL;
1967 return ret;
1968 }
1969 }
1970
1971 if (attr->dest_nentries) {
1972 ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar,
1973 ce_id,
1974 attr);
1975 if (IS_ERR(ce_state->dest_ring)) {
1976 ret = PTR_ERR(ce_state->dest_ring);
1977 ath10k_err(ar, "failed to alloc CE dest ring %d: %d\n",
1978 ce_id, ret);
1979 ce_state->dest_ring = NULL;
1980 return ret;
1981 }
1982 }
1983
1984 return 0;
1985 }
1986 EXPORT_SYMBOL(ath10k_ce_alloc_pipe);
1987
ath10k_ce_alloc_rri(struct ath10k * ar)1988 void ath10k_ce_alloc_rri(struct ath10k *ar)
1989 {
1990 int i;
1991 u32 value;
1992 u32 ctrl1_regs;
1993 u32 ce_base_addr;
1994 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1995
1996 ce->vaddr_rri = dma_alloc_coherent(ar->dev,
1997 (CE_COUNT * sizeof(u32)),
1998 &ce->paddr_rri, GFP_KERNEL);
1999
2000 if (!ce->vaddr_rri)
2001 return;
2002
2003 ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_low,
2004 lower_32_bits(ce->paddr_rri));
2005 ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_high,
2006 (upper_32_bits(ce->paddr_rri) &
2007 CE_DESC_ADDR_HI_MASK));
2008
2009 for (i = 0; i < CE_COUNT; i++) {
2010 ctrl1_regs = ar->hw_ce_regs->ctrl1_regs->addr;
2011 ce_base_addr = ath10k_ce_base_address(ar, i);
2012 value = ath10k_ce_read32(ar, ce_base_addr + ctrl1_regs);
2013 value |= ar->hw_ce_regs->upd->mask;
2014 ath10k_ce_write32(ar, ce_base_addr + ctrl1_regs, value);
2015 }
2016 }
2017 EXPORT_SYMBOL(ath10k_ce_alloc_rri);
2018
ath10k_ce_free_rri(struct ath10k * ar)2019 void ath10k_ce_free_rri(struct ath10k *ar)
2020 {
2021 struct ath10k_ce *ce = ath10k_ce_priv(ar);
2022
2023 dma_free_coherent(ar->dev, (CE_COUNT * sizeof(u32)),
2024 ce->vaddr_rri,
2025 ce->paddr_rri);
2026 }
2027 EXPORT_SYMBOL(ath10k_ce_free_rri);
2028