1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
3
4 #include <linux/regulator/consumer.h>
5 #include <linux/reset.h>
6 #include <linux/clk.h>
7 #include <linux/slab.h>
8 #include <linux/dma-mapping.h>
9 #include <linux/platform_device.h>
10
11 #include "lima_device.h"
12 #include "lima_gp.h"
13 #include "lima_pp.h"
14 #include "lima_mmu.h"
15 #include "lima_pmu.h"
16 #include "lima_l2_cache.h"
17 #include "lima_dlbu.h"
18 #include "lima_bcast.h"
19 #include "lima_vm.h"
20
21 struct lima_ip_desc {
22 char *name;
23 char *irq_name;
24 bool must_have[lima_gpu_num];
25 int offset[lima_gpu_num];
26
27 int (*init)(struct lima_ip *ip);
28 void (*fini)(struct lima_ip *ip);
29 int (*resume)(struct lima_ip *ip);
30 void (*suspend)(struct lima_ip *ip);
31 };
32
33 #define LIMA_IP_DESC(ipname, mst0, mst1, off0, off1, func, irq) \
34 [lima_ip_##ipname] = { \
35 .name = #ipname, \
36 .irq_name = irq, \
37 .must_have = { \
38 [lima_gpu_mali400] = mst0, \
39 [lima_gpu_mali450] = mst1, \
40 }, \
41 .offset = { \
42 [lima_gpu_mali400] = off0, \
43 [lima_gpu_mali450] = off1, \
44 }, \
45 .init = lima_##func##_init, \
46 .fini = lima_##func##_fini, \
47 .resume = lima_##func##_resume, \
48 .suspend = lima_##func##_suspend, \
49 }
50
51 static struct lima_ip_desc lima_ip_desc[lima_ip_num] = {
52 LIMA_IP_DESC(pmu, false, false, 0x02000, 0x02000, pmu, "pmu"),
53 LIMA_IP_DESC(l2_cache0, true, true, 0x01000, 0x10000, l2_cache, NULL),
54 LIMA_IP_DESC(l2_cache1, false, true, -1, 0x01000, l2_cache, NULL),
55 LIMA_IP_DESC(l2_cache2, false, false, -1, 0x11000, l2_cache, NULL),
56 LIMA_IP_DESC(gp, true, true, 0x00000, 0x00000, gp, "gp"),
57 LIMA_IP_DESC(pp0, true, true, 0x08000, 0x08000, pp, "pp0"),
58 LIMA_IP_DESC(pp1, false, false, 0x0A000, 0x0A000, pp, "pp1"),
59 LIMA_IP_DESC(pp2, false, false, 0x0C000, 0x0C000, pp, "pp2"),
60 LIMA_IP_DESC(pp3, false, false, 0x0E000, 0x0E000, pp, "pp3"),
61 LIMA_IP_DESC(pp4, false, false, -1, 0x28000, pp, "pp4"),
62 LIMA_IP_DESC(pp5, false, false, -1, 0x2A000, pp, "pp5"),
63 LIMA_IP_DESC(pp6, false, false, -1, 0x2C000, pp, "pp6"),
64 LIMA_IP_DESC(pp7, false, false, -1, 0x2E000, pp, "pp7"),
65 LIMA_IP_DESC(gpmmu, true, true, 0x03000, 0x03000, mmu, "gpmmu"),
66 LIMA_IP_DESC(ppmmu0, true, true, 0x04000, 0x04000, mmu, "ppmmu0"),
67 LIMA_IP_DESC(ppmmu1, false, false, 0x05000, 0x05000, mmu, "ppmmu1"),
68 LIMA_IP_DESC(ppmmu2, false, false, 0x06000, 0x06000, mmu, "ppmmu2"),
69 LIMA_IP_DESC(ppmmu3, false, false, 0x07000, 0x07000, mmu, "ppmmu3"),
70 LIMA_IP_DESC(ppmmu4, false, false, -1, 0x1C000, mmu, "ppmmu4"),
71 LIMA_IP_DESC(ppmmu5, false, false, -1, 0x1D000, mmu, "ppmmu5"),
72 LIMA_IP_DESC(ppmmu6, false, false, -1, 0x1E000, mmu, "ppmmu6"),
73 LIMA_IP_DESC(ppmmu7, false, false, -1, 0x1F000, mmu, "ppmmu7"),
74 LIMA_IP_DESC(dlbu, false, true, -1, 0x14000, dlbu, NULL),
75 LIMA_IP_DESC(bcast, false, true, -1, 0x13000, bcast, NULL),
76 LIMA_IP_DESC(pp_bcast, false, true, -1, 0x16000, pp_bcast, "pp"),
77 LIMA_IP_DESC(ppmmu_bcast, false, true, -1, 0x15000, mmu, NULL),
78 };
79
lima_ip_name(struct lima_ip * ip)80 const char *lima_ip_name(struct lima_ip *ip)
81 {
82 return lima_ip_desc[ip->id].name;
83 }
84
lima_clk_enable(struct lima_device * dev)85 static int lima_clk_enable(struct lima_device *dev)
86 {
87 int err;
88
89 err = clk_prepare_enable(dev->clk_bus);
90 if (err)
91 return err;
92
93 err = clk_prepare_enable(dev->clk_gpu);
94 if (err)
95 goto error_out0;
96
97 if (dev->reset) {
98 err = reset_control_deassert(dev->reset);
99 if (err) {
100 dev_err(dev->dev,
101 "reset controller deassert failed %d\n", err);
102 goto error_out1;
103 }
104 }
105
106 return 0;
107
108 error_out1:
109 clk_disable_unprepare(dev->clk_gpu);
110 error_out0:
111 clk_disable_unprepare(dev->clk_bus);
112 return err;
113 }
114
lima_clk_disable(struct lima_device * dev)115 static void lima_clk_disable(struct lima_device *dev)
116 {
117 if (dev->reset)
118 reset_control_assert(dev->reset);
119 clk_disable_unprepare(dev->clk_gpu);
120 clk_disable_unprepare(dev->clk_bus);
121 }
122
lima_clk_init(struct lima_device * dev)123 static int lima_clk_init(struct lima_device *dev)
124 {
125 int err;
126
127 dev->clk_bus = devm_clk_get(dev->dev, "bus");
128 if (IS_ERR(dev->clk_bus)) {
129 err = PTR_ERR(dev->clk_bus);
130 if (err != -EPROBE_DEFER)
131 dev_err(dev->dev, "get bus clk failed %d\n", err);
132 dev->clk_bus = NULL;
133 return err;
134 }
135
136 dev->clk_gpu = devm_clk_get(dev->dev, "core");
137 if (IS_ERR(dev->clk_gpu)) {
138 err = PTR_ERR(dev->clk_gpu);
139 if (err != -EPROBE_DEFER)
140 dev_err(dev->dev, "get core clk failed %d\n", err);
141 dev->clk_gpu = NULL;
142 return err;
143 }
144
145 dev->reset = devm_reset_control_array_get_optional_shared(dev->dev);
146 if (IS_ERR(dev->reset)) {
147 err = PTR_ERR(dev->reset);
148 if (err != -EPROBE_DEFER)
149 dev_err(dev->dev, "get reset controller failed %d\n",
150 err);
151 dev->reset = NULL;
152 return err;
153 }
154
155 return lima_clk_enable(dev);
156 }
157
lima_clk_fini(struct lima_device * dev)158 static void lima_clk_fini(struct lima_device *dev)
159 {
160 lima_clk_disable(dev);
161 }
162
lima_regulator_enable(struct lima_device * dev)163 static int lima_regulator_enable(struct lima_device *dev)
164 {
165 int ret;
166
167 if (!dev->regulator)
168 return 0;
169
170 ret = regulator_enable(dev->regulator);
171 if (ret < 0) {
172 dev_err(dev->dev, "failed to enable regulator: %d\n", ret);
173 return ret;
174 }
175
176 return 0;
177 }
178
lima_regulator_disable(struct lima_device * dev)179 static void lima_regulator_disable(struct lima_device *dev)
180 {
181 if (dev->regulator)
182 regulator_disable(dev->regulator);
183 }
184
lima_regulator_init(struct lima_device * dev)185 static int lima_regulator_init(struct lima_device *dev)
186 {
187 int ret;
188
189 dev->regulator = devm_regulator_get_optional(dev->dev, "mali");
190 if (IS_ERR(dev->regulator)) {
191 ret = PTR_ERR(dev->regulator);
192 dev->regulator = NULL;
193 if (ret == -ENODEV)
194 return 0;
195 if (ret != -EPROBE_DEFER)
196 dev_err(dev->dev, "failed to get regulator: %d\n", ret);
197 return ret;
198 }
199
200 return lima_regulator_enable(dev);
201 }
202
lima_regulator_fini(struct lima_device * dev)203 static void lima_regulator_fini(struct lima_device *dev)
204 {
205 lima_regulator_disable(dev);
206 }
207
lima_init_ip(struct lima_device * dev,int index)208 static int lima_init_ip(struct lima_device *dev, int index)
209 {
210 struct platform_device *pdev = to_platform_device(dev->dev);
211 struct lima_ip_desc *desc = lima_ip_desc + index;
212 struct lima_ip *ip = dev->ip + index;
213 const char *irq_name = desc->irq_name;
214 int offset = desc->offset[dev->id];
215 bool must = desc->must_have[dev->id];
216 int err;
217
218 if (offset < 0)
219 return 0;
220
221 ip->dev = dev;
222 ip->id = index;
223 ip->iomem = dev->iomem + offset;
224 if (irq_name) {
225 err = must ? platform_get_irq_byname(pdev, irq_name) :
226 platform_get_irq_byname_optional(pdev, irq_name);
227 if (err < 0)
228 goto out;
229 ip->irq = err;
230 }
231
232 err = desc->init(ip);
233 if (!err) {
234 ip->present = true;
235 return 0;
236 }
237
238 out:
239 return must ? err : 0;
240 }
241
lima_fini_ip(struct lima_device * ldev,int index)242 static void lima_fini_ip(struct lima_device *ldev, int index)
243 {
244 struct lima_ip_desc *desc = lima_ip_desc + index;
245 struct lima_ip *ip = ldev->ip + index;
246
247 if (ip->present)
248 desc->fini(ip);
249 }
250
lima_resume_ip(struct lima_device * ldev,int index)251 static int lima_resume_ip(struct lima_device *ldev, int index)
252 {
253 struct lima_ip_desc *desc = lima_ip_desc + index;
254 struct lima_ip *ip = ldev->ip + index;
255 int ret = 0;
256
257 if (ip->present)
258 ret = desc->resume(ip);
259
260 return ret;
261 }
262
lima_suspend_ip(struct lima_device * ldev,int index)263 static void lima_suspend_ip(struct lima_device *ldev, int index)
264 {
265 struct lima_ip_desc *desc = lima_ip_desc + index;
266 struct lima_ip *ip = ldev->ip + index;
267
268 if (ip->present)
269 desc->suspend(ip);
270 }
271
lima_init_gp_pipe(struct lima_device * dev)272 static int lima_init_gp_pipe(struct lima_device *dev)
273 {
274 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp;
275 int err;
276
277 pipe->ldev = dev;
278
279 err = lima_sched_pipe_init(pipe, "gp");
280 if (err)
281 return err;
282
283 pipe->l2_cache[pipe->num_l2_cache++] = dev->ip + lima_ip_l2_cache0;
284 pipe->mmu[pipe->num_mmu++] = dev->ip + lima_ip_gpmmu;
285 pipe->processor[pipe->num_processor++] = dev->ip + lima_ip_gp;
286
287 err = lima_gp_pipe_init(dev);
288 if (err) {
289 lima_sched_pipe_fini(pipe);
290 return err;
291 }
292
293 return 0;
294 }
295
lima_fini_gp_pipe(struct lima_device * dev)296 static void lima_fini_gp_pipe(struct lima_device *dev)
297 {
298 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp;
299
300 lima_gp_pipe_fini(dev);
301 lima_sched_pipe_fini(pipe);
302 }
303
lima_init_pp_pipe(struct lima_device * dev)304 static int lima_init_pp_pipe(struct lima_device *dev)
305 {
306 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;
307 int err, i;
308
309 pipe->ldev = dev;
310
311 err = lima_sched_pipe_init(pipe, "pp");
312 if (err)
313 return err;
314
315 for (i = 0; i < LIMA_SCHED_PIPE_MAX_PROCESSOR; i++) {
316 struct lima_ip *pp = dev->ip + lima_ip_pp0 + i;
317 struct lima_ip *ppmmu = dev->ip + lima_ip_ppmmu0 + i;
318 struct lima_ip *l2_cache;
319
320 if (dev->id == lima_gpu_mali400)
321 l2_cache = dev->ip + lima_ip_l2_cache0;
322 else
323 l2_cache = dev->ip + lima_ip_l2_cache1 + (i >> 2);
324
325 if (pp->present && ppmmu->present && l2_cache->present) {
326 pipe->mmu[pipe->num_mmu++] = ppmmu;
327 pipe->processor[pipe->num_processor++] = pp;
328 if (!pipe->l2_cache[i >> 2])
329 pipe->l2_cache[pipe->num_l2_cache++] = l2_cache;
330 }
331 }
332
333 if (dev->ip[lima_ip_bcast].present) {
334 pipe->bcast_processor = dev->ip + lima_ip_pp_bcast;
335 pipe->bcast_mmu = dev->ip + lima_ip_ppmmu_bcast;
336 }
337
338 err = lima_pp_pipe_init(dev);
339 if (err) {
340 lima_sched_pipe_fini(pipe);
341 return err;
342 }
343
344 return 0;
345 }
346
lima_fini_pp_pipe(struct lima_device * dev)347 static void lima_fini_pp_pipe(struct lima_device *dev)
348 {
349 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;
350
351 lima_pp_pipe_fini(dev);
352 lima_sched_pipe_fini(pipe);
353 }
354
lima_device_init(struct lima_device * ldev)355 int lima_device_init(struct lima_device *ldev)
356 {
357 struct platform_device *pdev = to_platform_device(ldev->dev);
358 int err, i;
359
360 dma_set_coherent_mask(ldev->dev, DMA_BIT_MASK(32));
361
362 err = lima_clk_init(ldev);
363 if (err)
364 return err;
365
366 err = lima_regulator_init(ldev);
367 if (err)
368 goto err_out0;
369
370 ldev->empty_vm = lima_vm_create(ldev);
371 if (!ldev->empty_vm) {
372 err = -ENOMEM;
373 goto err_out1;
374 }
375
376 ldev->va_start = 0;
377 if (ldev->id == lima_gpu_mali450) {
378 ldev->va_end = LIMA_VA_RESERVE_START;
379 ldev->dlbu_cpu = dma_alloc_wc(
380 ldev->dev, LIMA_PAGE_SIZE,
381 &ldev->dlbu_dma, GFP_KERNEL | __GFP_NOWARN);
382 if (!ldev->dlbu_cpu) {
383 err = -ENOMEM;
384 goto err_out2;
385 }
386 } else
387 ldev->va_end = LIMA_VA_RESERVE_END;
388
389 ldev->iomem = devm_platform_ioremap_resource(pdev, 0);
390 if (IS_ERR(ldev->iomem)) {
391 dev_err(ldev->dev, "fail to ioremap iomem\n");
392 err = PTR_ERR(ldev->iomem);
393 goto err_out3;
394 }
395
396 for (i = 0; i < lima_ip_num; i++) {
397 err = lima_init_ip(ldev, i);
398 if (err)
399 goto err_out4;
400 }
401
402 err = lima_init_gp_pipe(ldev);
403 if (err)
404 goto err_out4;
405
406 err = lima_init_pp_pipe(ldev);
407 if (err)
408 goto err_out5;
409
410 ldev->dump.magic = LIMA_DUMP_MAGIC;
411 ldev->dump.version_major = LIMA_DUMP_MAJOR;
412 ldev->dump.version_minor = LIMA_DUMP_MINOR;
413 INIT_LIST_HEAD(&ldev->error_task_list);
414 mutex_init(&ldev->error_task_list_lock);
415
416 dev_info(ldev->dev, "bus rate = %lu\n", clk_get_rate(ldev->clk_bus));
417 dev_info(ldev->dev, "mod rate = %lu", clk_get_rate(ldev->clk_gpu));
418
419 return 0;
420
421 err_out5:
422 lima_fini_gp_pipe(ldev);
423 err_out4:
424 while (--i >= 0)
425 lima_fini_ip(ldev, i);
426 err_out3:
427 if (ldev->dlbu_cpu)
428 dma_free_wc(ldev->dev, LIMA_PAGE_SIZE,
429 ldev->dlbu_cpu, ldev->dlbu_dma);
430 err_out2:
431 lima_vm_put(ldev->empty_vm);
432 err_out1:
433 lima_regulator_fini(ldev);
434 err_out0:
435 lima_clk_fini(ldev);
436 return err;
437 }
438
lima_device_fini(struct lima_device * ldev)439 void lima_device_fini(struct lima_device *ldev)
440 {
441 int i;
442 struct lima_sched_error_task *et, *tmp;
443
444 list_for_each_entry_safe(et, tmp, &ldev->error_task_list, list) {
445 list_del(&et->list);
446 kvfree(et);
447 }
448 mutex_destroy(&ldev->error_task_list_lock);
449
450 lima_fini_pp_pipe(ldev);
451 lima_fini_gp_pipe(ldev);
452
453 for (i = lima_ip_num - 1; i >= 0; i--)
454 lima_fini_ip(ldev, i);
455
456 if (ldev->dlbu_cpu)
457 dma_free_wc(ldev->dev, LIMA_PAGE_SIZE,
458 ldev->dlbu_cpu, ldev->dlbu_dma);
459
460 lima_vm_put(ldev->empty_vm);
461
462 lima_regulator_fini(ldev);
463
464 lima_clk_fini(ldev);
465 }
466
lima_device_resume(struct device * dev)467 int lima_device_resume(struct device *dev)
468 {
469 struct lima_device *ldev = dev_get_drvdata(dev);
470 int i, err;
471
472 err = lima_clk_enable(ldev);
473 if (err) {
474 dev_err(dev, "resume clk fail %d\n", err);
475 return err;
476 }
477
478 err = lima_regulator_enable(ldev);
479 if (err) {
480 dev_err(dev, "resume regulator fail %d\n", err);
481 goto err_out0;
482 }
483
484 for (i = 0; i < lima_ip_num; i++) {
485 err = lima_resume_ip(ldev, i);
486 if (err) {
487 dev_err(dev, "resume ip %d fail\n", i);
488 goto err_out1;
489 }
490 }
491
492 err = lima_devfreq_resume(&ldev->devfreq);
493 if (err) {
494 dev_err(dev, "devfreq resume fail\n");
495 goto err_out1;
496 }
497
498 return 0;
499
500 err_out1:
501 while (--i >= 0)
502 lima_suspend_ip(ldev, i);
503 lima_regulator_disable(ldev);
504 err_out0:
505 lima_clk_disable(ldev);
506 return err;
507 }
508
lima_device_suspend(struct device * dev)509 int lima_device_suspend(struct device *dev)
510 {
511 struct lima_device *ldev = dev_get_drvdata(dev);
512 int i, err;
513
514 /* check any task running */
515 for (i = 0; i < lima_pipe_num; i++) {
516 if (atomic_read(&ldev->pipe[i].base.hw_rq_count))
517 return -EBUSY;
518 }
519
520 err = lima_devfreq_suspend(&ldev->devfreq);
521 if (err) {
522 dev_err(dev, "devfreq suspend fail\n");
523 return err;
524 }
525
526 for (i = lima_ip_num - 1; i >= 0; i--)
527 lima_suspend_ip(ldev, i);
528
529 lima_regulator_disable(ldev);
530
531 lima_clk_disable(ldev);
532
533 return 0;
534 }
535