1 /*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23 #include <linux/device.h>
24 #include <linux/export.h>
25 #include <linux/err.h>
26 #include <linux/fs.h>
27 #include <linux/file.h>
28 #include <linux/sched.h>
29 #include <linux/slab.h>
30 #include <linux/uaccess.h>
31 #include <linux/compat.h>
32 #include <uapi/linux/kfd_ioctl.h>
33 #include <linux/time.h>
34 #include <linux/mm.h>
35 #include <linux/mman.h>
36 #include <linux/dma-buf.h>
37 #include <asm/processor.h>
38 #include "kfd_priv.h"
39 #include "kfd_device_queue_manager.h"
40 #include "kfd_dbgmgr.h"
41 #include "kfd_svm.h"
42 #include "amdgpu_amdkfd.h"
43 #include "kfd_smi_events.h"
44
45 static long kfd_ioctl(struct file *, unsigned int, unsigned long);
46 static int kfd_open(struct inode *, struct file *);
47 static int kfd_release(struct inode *, struct file *);
48 static int kfd_mmap(struct file *, struct vm_area_struct *);
49
50 static const char kfd_dev_name[] = "kfd";
51
52 static const struct file_operations kfd_fops = {
53 .owner = THIS_MODULE,
54 .unlocked_ioctl = kfd_ioctl,
55 .compat_ioctl = compat_ptr_ioctl,
56 .open = kfd_open,
57 .release = kfd_release,
58 .mmap = kfd_mmap,
59 };
60
61 static int kfd_char_dev_major = -1;
62 static struct class *kfd_class;
63 struct device *kfd_device;
64
kfd_chardev_init(void)65 int kfd_chardev_init(void)
66 {
67 int err = 0;
68
69 kfd_char_dev_major = register_chrdev(0, kfd_dev_name, &kfd_fops);
70 err = kfd_char_dev_major;
71 if (err < 0)
72 goto err_register_chrdev;
73
74 kfd_class = class_create(THIS_MODULE, kfd_dev_name);
75 err = PTR_ERR(kfd_class);
76 if (IS_ERR(kfd_class))
77 goto err_class_create;
78
79 kfd_device = device_create(kfd_class, NULL,
80 MKDEV(kfd_char_dev_major, 0),
81 NULL, kfd_dev_name);
82 err = PTR_ERR(kfd_device);
83 if (IS_ERR(kfd_device))
84 goto err_device_create;
85
86 return 0;
87
88 err_device_create:
89 class_destroy(kfd_class);
90 err_class_create:
91 unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
92 err_register_chrdev:
93 return err;
94 }
95
kfd_chardev_exit(void)96 void kfd_chardev_exit(void)
97 {
98 device_destroy(kfd_class, MKDEV(kfd_char_dev_major, 0));
99 class_destroy(kfd_class);
100 unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
101 kfd_device = NULL;
102 }
103
kfd_chardev(void)104 struct device *kfd_chardev(void)
105 {
106 return kfd_device;
107 }
108
109
kfd_open(struct inode * inode,struct file * filep)110 static int kfd_open(struct inode *inode, struct file *filep)
111 {
112 struct kfd_process *process;
113 bool is_32bit_user_mode;
114
115 if (iminor(inode) != 0)
116 return -ENODEV;
117
118 is_32bit_user_mode = in_compat_syscall();
119
120 if (is_32bit_user_mode) {
121 dev_warn(kfd_device,
122 "Process %d (32-bit) failed to open /dev/kfd\n"
123 "32-bit processes are not supported by amdkfd\n",
124 current->pid);
125 return -EPERM;
126 }
127
128 process = kfd_create_process(filep);
129 if (IS_ERR(process))
130 return PTR_ERR(process);
131
132 if (kfd_is_locked()) {
133 dev_dbg(kfd_device, "kfd is locked!\n"
134 "process %d unreferenced", process->pasid);
135 kfd_unref_process(process);
136 return -EAGAIN;
137 }
138
139 /* filep now owns the reference returned by kfd_create_process */
140 filep->private_data = process;
141
142 dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n",
143 process->pasid, process->is_32bit_user_mode);
144
145 return 0;
146 }
147
kfd_release(struct inode * inode,struct file * filep)148 static int kfd_release(struct inode *inode, struct file *filep)
149 {
150 struct kfd_process *process = filep->private_data;
151
152 if (process)
153 kfd_unref_process(process);
154
155 return 0;
156 }
157
kfd_ioctl_get_version(struct file * filep,struct kfd_process * p,void * data)158 static int kfd_ioctl_get_version(struct file *filep, struct kfd_process *p,
159 void *data)
160 {
161 struct kfd_ioctl_get_version_args *args = data;
162
163 args->major_version = KFD_IOCTL_MAJOR_VERSION;
164 args->minor_version = KFD_IOCTL_MINOR_VERSION;
165
166 return 0;
167 }
168
set_queue_properties_from_user(struct queue_properties * q_properties,struct kfd_ioctl_create_queue_args * args)169 static int set_queue_properties_from_user(struct queue_properties *q_properties,
170 struct kfd_ioctl_create_queue_args *args)
171 {
172 if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) {
173 pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
174 return -EINVAL;
175 }
176
177 if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
178 pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
179 return -EINVAL;
180 }
181
182 if ((args->ring_base_address) &&
183 (!access_ok((const void __user *) args->ring_base_address,
184 sizeof(uint64_t)))) {
185 pr_err("Can't access ring base address\n");
186 return -EFAULT;
187 }
188
189 if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
190 pr_err("Ring size must be a power of 2 or 0\n");
191 return -EINVAL;
192 }
193
194 if (!access_ok((const void __user *) args->read_pointer_address,
195 sizeof(uint32_t))) {
196 pr_err("Can't access read pointer\n");
197 return -EFAULT;
198 }
199
200 if (!access_ok((const void __user *) args->write_pointer_address,
201 sizeof(uint32_t))) {
202 pr_err("Can't access write pointer\n");
203 return -EFAULT;
204 }
205
206 if (args->eop_buffer_address &&
207 !access_ok((const void __user *) args->eop_buffer_address,
208 sizeof(uint32_t))) {
209 pr_debug("Can't access eop buffer");
210 return -EFAULT;
211 }
212
213 if (args->ctx_save_restore_address &&
214 !access_ok((const void __user *) args->ctx_save_restore_address,
215 sizeof(uint32_t))) {
216 pr_debug("Can't access ctx save restore buffer");
217 return -EFAULT;
218 }
219
220 q_properties->is_interop = false;
221 q_properties->is_gws = false;
222 q_properties->queue_percent = args->queue_percentage;
223 q_properties->priority = args->queue_priority;
224 q_properties->queue_address = args->ring_base_address;
225 q_properties->queue_size = args->ring_size;
226 q_properties->read_ptr = (uint32_t *) args->read_pointer_address;
227 q_properties->write_ptr = (uint32_t *) args->write_pointer_address;
228 q_properties->eop_ring_buffer_address = args->eop_buffer_address;
229 q_properties->eop_ring_buffer_size = args->eop_buffer_size;
230 q_properties->ctx_save_restore_area_address =
231 args->ctx_save_restore_address;
232 q_properties->ctx_save_restore_area_size = args->ctx_save_restore_size;
233 q_properties->ctl_stack_size = args->ctl_stack_size;
234 if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE ||
235 args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
236 q_properties->type = KFD_QUEUE_TYPE_COMPUTE;
237 else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA)
238 q_properties->type = KFD_QUEUE_TYPE_SDMA;
239 else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA_XGMI)
240 q_properties->type = KFD_QUEUE_TYPE_SDMA_XGMI;
241 else
242 return -ENOTSUPP;
243
244 if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
245 q_properties->format = KFD_QUEUE_FORMAT_AQL;
246 else
247 q_properties->format = KFD_QUEUE_FORMAT_PM4;
248
249 pr_debug("Queue Percentage: %d, %d\n",
250 q_properties->queue_percent, args->queue_percentage);
251
252 pr_debug("Queue Priority: %d, %d\n",
253 q_properties->priority, args->queue_priority);
254
255 pr_debug("Queue Address: 0x%llX, 0x%llX\n",
256 q_properties->queue_address, args->ring_base_address);
257
258 pr_debug("Queue Size: 0x%llX, %u\n",
259 q_properties->queue_size, args->ring_size);
260
261 pr_debug("Queue r/w Pointers: %px, %px\n",
262 q_properties->read_ptr,
263 q_properties->write_ptr);
264
265 pr_debug("Queue Format: %d\n", q_properties->format);
266
267 pr_debug("Queue EOP: 0x%llX\n", q_properties->eop_ring_buffer_address);
268
269 pr_debug("Queue CTX save area: 0x%llX\n",
270 q_properties->ctx_save_restore_area_address);
271
272 return 0;
273 }
274
kfd_ioctl_create_queue(struct file * filep,struct kfd_process * p,void * data)275 static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
276 void *data)
277 {
278 struct kfd_ioctl_create_queue_args *args = data;
279 struct kfd_dev *dev;
280 int err = 0;
281 unsigned int queue_id;
282 struct kfd_process_device *pdd;
283 struct queue_properties q_properties;
284 uint32_t doorbell_offset_in_process = 0;
285
286 memset(&q_properties, 0, sizeof(struct queue_properties));
287
288 pr_debug("Creating queue ioctl\n");
289
290 err = set_queue_properties_from_user(&q_properties, args);
291 if (err)
292 return err;
293
294 pr_debug("Looking for gpu id 0x%x\n", args->gpu_id);
295 dev = kfd_device_by_id(args->gpu_id);
296 if (!dev) {
297 pr_debug("Could not find gpu id 0x%x\n", args->gpu_id);
298 return -EINVAL;
299 }
300
301 mutex_lock(&p->mutex);
302
303 pdd = kfd_bind_process_to_device(dev, p);
304 if (IS_ERR(pdd)) {
305 err = -ESRCH;
306 goto err_bind_process;
307 }
308
309 pr_debug("Creating queue for PASID 0x%x on gpu 0x%x\n",
310 p->pasid,
311 dev->id);
312
313 err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, &queue_id,
314 &doorbell_offset_in_process);
315 if (err != 0)
316 goto err_create_queue;
317
318 args->queue_id = queue_id;
319
320
321 /* Return gpu_id as doorbell offset for mmap usage */
322 args->doorbell_offset = KFD_MMAP_TYPE_DOORBELL;
323 args->doorbell_offset |= KFD_MMAP_GPU_ID(args->gpu_id);
324 if (KFD_IS_SOC15(dev->device_info->asic_family))
325 /* On SOC15 ASICs, include the doorbell offset within the
326 * process doorbell frame, which is 2 pages.
327 */
328 args->doorbell_offset |= doorbell_offset_in_process;
329
330 mutex_unlock(&p->mutex);
331
332 pr_debug("Queue id %d was created successfully\n", args->queue_id);
333
334 pr_debug("Ring buffer address == 0x%016llX\n",
335 args->ring_base_address);
336
337 pr_debug("Read ptr address == 0x%016llX\n",
338 args->read_pointer_address);
339
340 pr_debug("Write ptr address == 0x%016llX\n",
341 args->write_pointer_address);
342
343 return 0;
344
345 err_create_queue:
346 err_bind_process:
347 mutex_unlock(&p->mutex);
348 return err;
349 }
350
kfd_ioctl_destroy_queue(struct file * filp,struct kfd_process * p,void * data)351 static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p,
352 void *data)
353 {
354 int retval;
355 struct kfd_ioctl_destroy_queue_args *args = data;
356
357 pr_debug("Destroying queue id %d for pasid 0x%x\n",
358 args->queue_id,
359 p->pasid);
360
361 mutex_lock(&p->mutex);
362
363 retval = pqm_destroy_queue(&p->pqm, args->queue_id);
364
365 mutex_unlock(&p->mutex);
366 return retval;
367 }
368
kfd_ioctl_update_queue(struct file * filp,struct kfd_process * p,void * data)369 static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p,
370 void *data)
371 {
372 int retval;
373 struct kfd_ioctl_update_queue_args *args = data;
374 struct queue_properties properties;
375
376 if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) {
377 pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
378 return -EINVAL;
379 }
380
381 if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
382 pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
383 return -EINVAL;
384 }
385
386 if ((args->ring_base_address) &&
387 (!access_ok((const void __user *) args->ring_base_address,
388 sizeof(uint64_t)))) {
389 pr_err("Can't access ring base address\n");
390 return -EFAULT;
391 }
392
393 if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
394 pr_err("Ring size must be a power of 2 or 0\n");
395 return -EINVAL;
396 }
397
398 properties.queue_address = args->ring_base_address;
399 properties.queue_size = args->ring_size;
400 properties.queue_percent = args->queue_percentage;
401 properties.priority = args->queue_priority;
402
403 pr_debug("Updating queue id %d for pasid 0x%x\n",
404 args->queue_id, p->pasid);
405
406 mutex_lock(&p->mutex);
407
408 retval = pqm_update_queue_properties(&p->pqm, args->queue_id, &properties);
409
410 mutex_unlock(&p->mutex);
411
412 return retval;
413 }
414
kfd_ioctl_set_cu_mask(struct file * filp,struct kfd_process * p,void * data)415 static int kfd_ioctl_set_cu_mask(struct file *filp, struct kfd_process *p,
416 void *data)
417 {
418 int retval;
419 const int max_num_cus = 1024;
420 struct kfd_ioctl_set_cu_mask_args *args = data;
421 struct mqd_update_info minfo = {0};
422 uint32_t __user *cu_mask_ptr = (uint32_t __user *)args->cu_mask_ptr;
423 size_t cu_mask_size = sizeof(uint32_t) * (args->num_cu_mask / 32);
424
425 if ((args->num_cu_mask % 32) != 0) {
426 pr_debug("num_cu_mask 0x%x must be a multiple of 32",
427 args->num_cu_mask);
428 return -EINVAL;
429 }
430
431 minfo.cu_mask.count = args->num_cu_mask;
432 if (minfo.cu_mask.count == 0) {
433 pr_debug("CU mask cannot be 0");
434 return -EINVAL;
435 }
436
437 /* To prevent an unreasonably large CU mask size, set an arbitrary
438 * limit of max_num_cus bits. We can then just drop any CU mask bits
439 * past max_num_cus bits and just use the first max_num_cus bits.
440 */
441 if (minfo.cu_mask.count > max_num_cus) {
442 pr_debug("CU mask cannot be greater than 1024 bits");
443 minfo.cu_mask.count = max_num_cus;
444 cu_mask_size = sizeof(uint32_t) * (max_num_cus/32);
445 }
446
447 minfo.cu_mask.ptr = kzalloc(cu_mask_size, GFP_KERNEL);
448 if (!minfo.cu_mask.ptr)
449 return -ENOMEM;
450
451 retval = copy_from_user(minfo.cu_mask.ptr, cu_mask_ptr, cu_mask_size);
452 if (retval) {
453 pr_debug("Could not copy CU mask from userspace");
454 retval = -EFAULT;
455 goto out;
456 }
457
458 minfo.update_flag = UPDATE_FLAG_CU_MASK;
459
460 mutex_lock(&p->mutex);
461
462 retval = pqm_update_mqd(&p->pqm, args->queue_id, &minfo);
463
464 mutex_unlock(&p->mutex);
465
466 out:
467 kfree(minfo.cu_mask.ptr);
468 return retval;
469 }
470
kfd_ioctl_get_queue_wave_state(struct file * filep,struct kfd_process * p,void * data)471 static int kfd_ioctl_get_queue_wave_state(struct file *filep,
472 struct kfd_process *p, void *data)
473 {
474 struct kfd_ioctl_get_queue_wave_state_args *args = data;
475 int r;
476
477 mutex_lock(&p->mutex);
478
479 r = pqm_get_wave_state(&p->pqm, args->queue_id,
480 (void __user *)args->ctl_stack_address,
481 &args->ctl_stack_used_size,
482 &args->save_area_used_size);
483
484 mutex_unlock(&p->mutex);
485
486 return r;
487 }
488
kfd_ioctl_set_memory_policy(struct file * filep,struct kfd_process * p,void * data)489 static int kfd_ioctl_set_memory_policy(struct file *filep,
490 struct kfd_process *p, void *data)
491 {
492 struct kfd_ioctl_set_memory_policy_args *args = data;
493 struct kfd_dev *dev;
494 int err = 0;
495 struct kfd_process_device *pdd;
496 enum cache_policy default_policy, alternate_policy;
497
498 if (args->default_policy != KFD_IOC_CACHE_POLICY_COHERENT
499 && args->default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
500 return -EINVAL;
501 }
502
503 if (args->alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT
504 && args->alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
505 return -EINVAL;
506 }
507
508 dev = kfd_device_by_id(args->gpu_id);
509 if (!dev)
510 return -EINVAL;
511
512 mutex_lock(&p->mutex);
513
514 pdd = kfd_bind_process_to_device(dev, p);
515 if (IS_ERR(pdd)) {
516 err = -ESRCH;
517 goto out;
518 }
519
520 default_policy = (args->default_policy == KFD_IOC_CACHE_POLICY_COHERENT)
521 ? cache_policy_coherent : cache_policy_noncoherent;
522
523 alternate_policy =
524 (args->alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT)
525 ? cache_policy_coherent : cache_policy_noncoherent;
526
527 if (!dev->dqm->ops.set_cache_memory_policy(dev->dqm,
528 &pdd->qpd,
529 default_policy,
530 alternate_policy,
531 (void __user *)args->alternate_aperture_base,
532 args->alternate_aperture_size))
533 err = -EINVAL;
534
535 out:
536 mutex_unlock(&p->mutex);
537
538 return err;
539 }
540
kfd_ioctl_set_trap_handler(struct file * filep,struct kfd_process * p,void * data)541 static int kfd_ioctl_set_trap_handler(struct file *filep,
542 struct kfd_process *p, void *data)
543 {
544 struct kfd_ioctl_set_trap_handler_args *args = data;
545 struct kfd_dev *dev;
546 int err = 0;
547 struct kfd_process_device *pdd;
548
549 dev = kfd_device_by_id(args->gpu_id);
550 if (!dev)
551 return -EINVAL;
552
553 mutex_lock(&p->mutex);
554
555 pdd = kfd_bind_process_to_device(dev, p);
556 if (IS_ERR(pdd)) {
557 err = -ESRCH;
558 goto out;
559 }
560
561 kfd_process_set_trap_handler(&pdd->qpd, args->tba_addr, args->tma_addr);
562
563 out:
564 mutex_unlock(&p->mutex);
565
566 return err;
567 }
568
kfd_ioctl_dbg_register(struct file * filep,struct kfd_process * p,void * data)569 static int kfd_ioctl_dbg_register(struct file *filep,
570 struct kfd_process *p, void *data)
571 {
572 struct kfd_ioctl_dbg_register_args *args = data;
573 struct kfd_dev *dev;
574 struct kfd_dbgmgr *dbgmgr_ptr;
575 struct kfd_process_device *pdd;
576 bool create_ok;
577 long status = 0;
578
579 dev = kfd_device_by_id(args->gpu_id);
580 if (!dev)
581 return -EINVAL;
582
583 if (dev->device_info->asic_family == CHIP_CARRIZO) {
584 pr_debug("kfd_ioctl_dbg_register not supported on CZ\n");
585 return -EINVAL;
586 }
587
588 mutex_lock(&p->mutex);
589 mutex_lock(kfd_get_dbgmgr_mutex());
590
591 /*
592 * make sure that we have pdd, if this the first queue created for
593 * this process
594 */
595 pdd = kfd_bind_process_to_device(dev, p);
596 if (IS_ERR(pdd)) {
597 status = PTR_ERR(pdd);
598 goto out;
599 }
600
601 if (!dev->dbgmgr) {
602 /* In case of a legal call, we have no dbgmgr yet */
603 create_ok = kfd_dbgmgr_create(&dbgmgr_ptr, dev);
604 if (create_ok) {
605 status = kfd_dbgmgr_register(dbgmgr_ptr, p);
606 if (status != 0)
607 kfd_dbgmgr_destroy(dbgmgr_ptr);
608 else
609 dev->dbgmgr = dbgmgr_ptr;
610 }
611 } else {
612 pr_debug("debugger already registered\n");
613 status = -EINVAL;
614 }
615
616 out:
617 mutex_unlock(kfd_get_dbgmgr_mutex());
618 mutex_unlock(&p->mutex);
619
620 return status;
621 }
622
kfd_ioctl_dbg_unregister(struct file * filep,struct kfd_process * p,void * data)623 static int kfd_ioctl_dbg_unregister(struct file *filep,
624 struct kfd_process *p, void *data)
625 {
626 struct kfd_ioctl_dbg_unregister_args *args = data;
627 struct kfd_dev *dev;
628 long status;
629
630 dev = kfd_device_by_id(args->gpu_id);
631 if (!dev || !dev->dbgmgr)
632 return -EINVAL;
633
634 if (dev->device_info->asic_family == CHIP_CARRIZO) {
635 pr_debug("kfd_ioctl_dbg_unregister not supported on CZ\n");
636 return -EINVAL;
637 }
638
639 mutex_lock(kfd_get_dbgmgr_mutex());
640
641 status = kfd_dbgmgr_unregister(dev->dbgmgr, p);
642 if (!status) {
643 kfd_dbgmgr_destroy(dev->dbgmgr);
644 dev->dbgmgr = NULL;
645 }
646
647 mutex_unlock(kfd_get_dbgmgr_mutex());
648
649 return status;
650 }
651
652 /*
653 * Parse and generate variable size data structure for address watch.
654 * Total size of the buffer and # watch points is limited in order
655 * to prevent kernel abuse. (no bearing to the much smaller HW limitation
656 * which is enforced by dbgdev module)
657 * please also note that the watch address itself are not "copied from user",
658 * since it be set into the HW in user mode values.
659 *
660 */
kfd_ioctl_dbg_address_watch(struct file * filep,struct kfd_process * p,void * data)661 static int kfd_ioctl_dbg_address_watch(struct file *filep,
662 struct kfd_process *p, void *data)
663 {
664 struct kfd_ioctl_dbg_address_watch_args *args = data;
665 struct kfd_dev *dev;
666 struct dbg_address_watch_info aw_info;
667 unsigned char *args_buff;
668 long status;
669 void __user *cmd_from_user;
670 uint64_t watch_mask_value = 0;
671 unsigned int args_idx = 0;
672
673 memset((void *) &aw_info, 0, sizeof(struct dbg_address_watch_info));
674
675 dev = kfd_device_by_id(args->gpu_id);
676 if (!dev)
677 return -EINVAL;
678
679 if (dev->device_info->asic_family == CHIP_CARRIZO) {
680 pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n");
681 return -EINVAL;
682 }
683
684 cmd_from_user = (void __user *) args->content_ptr;
685
686 /* Validate arguments */
687
688 if ((args->buf_size_in_bytes > MAX_ALLOWED_AW_BUFF_SIZE) ||
689 (args->buf_size_in_bytes <= sizeof(*args) + sizeof(int) * 2) ||
690 (cmd_from_user == NULL))
691 return -EINVAL;
692
693 /* this is the actual buffer to work with */
694 args_buff = memdup_user(cmd_from_user,
695 args->buf_size_in_bytes - sizeof(*args));
696 if (IS_ERR(args_buff))
697 return PTR_ERR(args_buff);
698
699 aw_info.process = p;
700
701 aw_info.num_watch_points = *((uint32_t *)(&args_buff[args_idx]));
702 args_idx += sizeof(aw_info.num_watch_points);
703
704 aw_info.watch_mode = (enum HSA_DBG_WATCH_MODE *) &args_buff[args_idx];
705 args_idx += sizeof(enum HSA_DBG_WATCH_MODE) * aw_info.num_watch_points;
706
707 /*
708 * set watch address base pointer to point on the array base
709 * within args_buff
710 */
711 aw_info.watch_address = (uint64_t *) &args_buff[args_idx];
712
713 /* skip over the addresses buffer */
714 args_idx += sizeof(aw_info.watch_address) * aw_info.num_watch_points;
715
716 if (args_idx >= args->buf_size_in_bytes - sizeof(*args)) {
717 status = -EINVAL;
718 goto out;
719 }
720
721 watch_mask_value = (uint64_t) args_buff[args_idx];
722
723 if (watch_mask_value > 0) {
724 /*
725 * There is an array of masks.
726 * set watch mask base pointer to point on the array base
727 * within args_buff
728 */
729 aw_info.watch_mask = (uint64_t *) &args_buff[args_idx];
730
731 /* skip over the masks buffer */
732 args_idx += sizeof(aw_info.watch_mask) *
733 aw_info.num_watch_points;
734 } else {
735 /* just the NULL mask, set to NULL and skip over it */
736 aw_info.watch_mask = NULL;
737 args_idx += sizeof(aw_info.watch_mask);
738 }
739
740 if (args_idx >= args->buf_size_in_bytes - sizeof(args)) {
741 status = -EINVAL;
742 goto out;
743 }
744
745 /* Currently HSA Event is not supported for DBG */
746 aw_info.watch_event = NULL;
747
748 mutex_lock(kfd_get_dbgmgr_mutex());
749
750 status = kfd_dbgmgr_address_watch(dev->dbgmgr, &aw_info);
751
752 mutex_unlock(kfd_get_dbgmgr_mutex());
753
754 out:
755 kfree(args_buff);
756
757 return status;
758 }
759
760 /* Parse and generate fixed size data structure for wave control */
kfd_ioctl_dbg_wave_control(struct file * filep,struct kfd_process * p,void * data)761 static int kfd_ioctl_dbg_wave_control(struct file *filep,
762 struct kfd_process *p, void *data)
763 {
764 struct kfd_ioctl_dbg_wave_control_args *args = data;
765 struct kfd_dev *dev;
766 struct dbg_wave_control_info wac_info;
767 unsigned char *args_buff;
768 uint32_t computed_buff_size;
769 long status;
770 void __user *cmd_from_user;
771 unsigned int args_idx = 0;
772
773 memset((void *) &wac_info, 0, sizeof(struct dbg_wave_control_info));
774
775 /* we use compact form, independent of the packing attribute value */
776 computed_buff_size = sizeof(*args) +
777 sizeof(wac_info.mode) +
778 sizeof(wac_info.operand) +
779 sizeof(wac_info.dbgWave_msg.DbgWaveMsg) +
780 sizeof(wac_info.dbgWave_msg.MemoryVA) +
781 sizeof(wac_info.trapId);
782
783 dev = kfd_device_by_id(args->gpu_id);
784 if (!dev)
785 return -EINVAL;
786
787 if (dev->device_info->asic_family == CHIP_CARRIZO) {
788 pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n");
789 return -EINVAL;
790 }
791
792 /* input size must match the computed "compact" size */
793 if (args->buf_size_in_bytes != computed_buff_size) {
794 pr_debug("size mismatch, computed : actual %u : %u\n",
795 args->buf_size_in_bytes, computed_buff_size);
796 return -EINVAL;
797 }
798
799 cmd_from_user = (void __user *) args->content_ptr;
800
801 if (cmd_from_user == NULL)
802 return -EINVAL;
803
804 /* copy the entire buffer from user */
805
806 args_buff = memdup_user(cmd_from_user,
807 args->buf_size_in_bytes - sizeof(*args));
808 if (IS_ERR(args_buff))
809 return PTR_ERR(args_buff);
810
811 /* move ptr to the start of the "pay-load" area */
812 wac_info.process = p;
813
814 wac_info.operand = *((enum HSA_DBG_WAVEOP *)(&args_buff[args_idx]));
815 args_idx += sizeof(wac_info.operand);
816
817 wac_info.mode = *((enum HSA_DBG_WAVEMODE *)(&args_buff[args_idx]));
818 args_idx += sizeof(wac_info.mode);
819
820 wac_info.trapId = *((uint32_t *)(&args_buff[args_idx]));
821 args_idx += sizeof(wac_info.trapId);
822
823 wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value =
824 *((uint32_t *)(&args_buff[args_idx]));
825 wac_info.dbgWave_msg.MemoryVA = NULL;
826
827 mutex_lock(kfd_get_dbgmgr_mutex());
828
829 pr_debug("Calling dbg manager process %p, operand %u, mode %u, trapId %u, message %u\n",
830 wac_info.process, wac_info.operand,
831 wac_info.mode, wac_info.trapId,
832 wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value);
833
834 status = kfd_dbgmgr_wave_control(dev->dbgmgr, &wac_info);
835
836 pr_debug("Returned status of dbg manager is %ld\n", status);
837
838 mutex_unlock(kfd_get_dbgmgr_mutex());
839
840 kfree(args_buff);
841
842 return status;
843 }
844
kfd_ioctl_get_clock_counters(struct file * filep,struct kfd_process * p,void * data)845 static int kfd_ioctl_get_clock_counters(struct file *filep,
846 struct kfd_process *p, void *data)
847 {
848 struct kfd_ioctl_get_clock_counters_args *args = data;
849 struct kfd_dev *dev;
850
851 dev = kfd_device_by_id(args->gpu_id);
852 if (dev)
853 /* Reading GPU clock counter from KGD */
854 args->gpu_clock_counter = amdgpu_amdkfd_get_gpu_clock_counter(dev->kgd);
855 else
856 /* Node without GPU resource */
857 args->gpu_clock_counter = 0;
858
859 /* No access to rdtsc. Using raw monotonic time */
860 args->cpu_clock_counter = ktime_get_raw_ns();
861 args->system_clock_counter = ktime_get_boottime_ns();
862
863 /* Since the counter is in nano-seconds we use 1GHz frequency */
864 args->system_clock_freq = 1000000000;
865
866 return 0;
867 }
868
869
kfd_ioctl_get_process_apertures(struct file * filp,struct kfd_process * p,void * data)870 static int kfd_ioctl_get_process_apertures(struct file *filp,
871 struct kfd_process *p, void *data)
872 {
873 struct kfd_ioctl_get_process_apertures_args *args = data;
874 struct kfd_process_device_apertures *pAperture;
875 int i;
876
877 dev_dbg(kfd_device, "get apertures for PASID 0x%x", p->pasid);
878
879 args->num_of_nodes = 0;
880
881 mutex_lock(&p->mutex);
882 /* Run over all pdd of the process */
883 for (i = 0; i < p->n_pdds; i++) {
884 struct kfd_process_device *pdd = p->pdds[i];
885
886 pAperture =
887 &args->process_apertures[args->num_of_nodes];
888 pAperture->gpu_id = pdd->dev->id;
889 pAperture->lds_base = pdd->lds_base;
890 pAperture->lds_limit = pdd->lds_limit;
891 pAperture->gpuvm_base = pdd->gpuvm_base;
892 pAperture->gpuvm_limit = pdd->gpuvm_limit;
893 pAperture->scratch_base = pdd->scratch_base;
894 pAperture->scratch_limit = pdd->scratch_limit;
895
896 dev_dbg(kfd_device,
897 "node id %u\n", args->num_of_nodes);
898 dev_dbg(kfd_device,
899 "gpu id %u\n", pdd->dev->id);
900 dev_dbg(kfd_device,
901 "lds_base %llX\n", pdd->lds_base);
902 dev_dbg(kfd_device,
903 "lds_limit %llX\n", pdd->lds_limit);
904 dev_dbg(kfd_device,
905 "gpuvm_base %llX\n", pdd->gpuvm_base);
906 dev_dbg(kfd_device,
907 "gpuvm_limit %llX\n", pdd->gpuvm_limit);
908 dev_dbg(kfd_device,
909 "scratch_base %llX\n", pdd->scratch_base);
910 dev_dbg(kfd_device,
911 "scratch_limit %llX\n", pdd->scratch_limit);
912
913 if (++args->num_of_nodes >= NUM_OF_SUPPORTED_GPUS)
914 break;
915 }
916 mutex_unlock(&p->mutex);
917
918 return 0;
919 }
920
kfd_ioctl_get_process_apertures_new(struct file * filp,struct kfd_process * p,void * data)921 static int kfd_ioctl_get_process_apertures_new(struct file *filp,
922 struct kfd_process *p, void *data)
923 {
924 struct kfd_ioctl_get_process_apertures_new_args *args = data;
925 struct kfd_process_device_apertures *pa;
926 int ret;
927 int i;
928
929 dev_dbg(kfd_device, "get apertures for PASID 0x%x", p->pasid);
930
931 if (args->num_of_nodes == 0) {
932 /* Return number of nodes, so that user space can alloacate
933 * sufficient memory
934 */
935 mutex_lock(&p->mutex);
936 args->num_of_nodes = p->n_pdds;
937 goto out_unlock;
938 }
939
940 /* Fill in process-aperture information for all available
941 * nodes, but not more than args->num_of_nodes as that is
942 * the amount of memory allocated by user
943 */
944 pa = kzalloc((sizeof(struct kfd_process_device_apertures) *
945 args->num_of_nodes), GFP_KERNEL);
946 if (!pa)
947 return -ENOMEM;
948
949 mutex_lock(&p->mutex);
950
951 if (!p->n_pdds) {
952 args->num_of_nodes = 0;
953 kfree(pa);
954 goto out_unlock;
955 }
956
957 /* Run over all pdd of the process */
958 for (i = 0; i < min(p->n_pdds, args->num_of_nodes); i++) {
959 struct kfd_process_device *pdd = p->pdds[i];
960
961 pa[i].gpu_id = pdd->dev->id;
962 pa[i].lds_base = pdd->lds_base;
963 pa[i].lds_limit = pdd->lds_limit;
964 pa[i].gpuvm_base = pdd->gpuvm_base;
965 pa[i].gpuvm_limit = pdd->gpuvm_limit;
966 pa[i].scratch_base = pdd->scratch_base;
967 pa[i].scratch_limit = pdd->scratch_limit;
968
969 dev_dbg(kfd_device,
970 "gpu id %u\n", pdd->dev->id);
971 dev_dbg(kfd_device,
972 "lds_base %llX\n", pdd->lds_base);
973 dev_dbg(kfd_device,
974 "lds_limit %llX\n", pdd->lds_limit);
975 dev_dbg(kfd_device,
976 "gpuvm_base %llX\n", pdd->gpuvm_base);
977 dev_dbg(kfd_device,
978 "gpuvm_limit %llX\n", pdd->gpuvm_limit);
979 dev_dbg(kfd_device,
980 "scratch_base %llX\n", pdd->scratch_base);
981 dev_dbg(kfd_device,
982 "scratch_limit %llX\n", pdd->scratch_limit);
983 }
984 mutex_unlock(&p->mutex);
985
986 args->num_of_nodes = i;
987 ret = copy_to_user(
988 (void __user *)args->kfd_process_device_apertures_ptr,
989 pa,
990 (i * sizeof(struct kfd_process_device_apertures)));
991 kfree(pa);
992 return ret ? -EFAULT : 0;
993
994 out_unlock:
995 mutex_unlock(&p->mutex);
996 return 0;
997 }
998
kfd_ioctl_create_event(struct file * filp,struct kfd_process * p,void * data)999 static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p,
1000 void *data)
1001 {
1002 struct kfd_ioctl_create_event_args *args = data;
1003 int err;
1004
1005 /* For dGPUs the event page is allocated in user mode. The
1006 * handle is passed to KFD with the first call to this IOCTL
1007 * through the event_page_offset field.
1008 */
1009 if (args->event_page_offset) {
1010 struct kfd_dev *kfd;
1011 struct kfd_process_device *pdd;
1012 void *mem, *kern_addr;
1013 uint64_t size;
1014
1015 kfd = kfd_device_by_id(GET_GPU_ID(args->event_page_offset));
1016 if (!kfd) {
1017 pr_err("Getting device by id failed in %s\n", __func__);
1018 return -EINVAL;
1019 }
1020
1021 mutex_lock(&p->mutex);
1022
1023 if (p->signal_page) {
1024 pr_err("Event page is already set\n");
1025 err = -EINVAL;
1026 goto out_unlock;
1027 }
1028
1029 pdd = kfd_bind_process_to_device(kfd, p);
1030 if (IS_ERR(pdd)) {
1031 err = PTR_ERR(pdd);
1032 goto out_unlock;
1033 }
1034
1035 mem = kfd_process_device_translate_handle(pdd,
1036 GET_IDR_HANDLE(args->event_page_offset));
1037 if (!mem) {
1038 pr_err("Can't find BO, offset is 0x%llx\n",
1039 args->event_page_offset);
1040 err = -EINVAL;
1041 goto out_unlock;
1042 }
1043
1044 err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kfd->kgd,
1045 mem, &kern_addr, &size);
1046 if (err) {
1047 pr_err("Failed to map event page to kernel\n");
1048 goto out_unlock;
1049 }
1050
1051 err = kfd_event_page_set(p, kern_addr, size);
1052 if (err) {
1053 pr_err("Failed to set event page\n");
1054 amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(kfd->kgd, mem);
1055 goto out_unlock;
1056 }
1057
1058 p->signal_handle = args->event_page_offset;
1059
1060 mutex_unlock(&p->mutex);
1061 }
1062
1063 err = kfd_event_create(filp, p, args->event_type,
1064 args->auto_reset != 0, args->node_id,
1065 &args->event_id, &args->event_trigger_data,
1066 &args->event_page_offset,
1067 &args->event_slot_index);
1068
1069 return err;
1070
1071 out_unlock:
1072 mutex_unlock(&p->mutex);
1073 return err;
1074 }
1075
kfd_ioctl_destroy_event(struct file * filp,struct kfd_process * p,void * data)1076 static int kfd_ioctl_destroy_event(struct file *filp, struct kfd_process *p,
1077 void *data)
1078 {
1079 struct kfd_ioctl_destroy_event_args *args = data;
1080
1081 return kfd_event_destroy(p, args->event_id);
1082 }
1083
kfd_ioctl_set_event(struct file * filp,struct kfd_process * p,void * data)1084 static int kfd_ioctl_set_event(struct file *filp, struct kfd_process *p,
1085 void *data)
1086 {
1087 struct kfd_ioctl_set_event_args *args = data;
1088
1089 return kfd_set_event(p, args->event_id);
1090 }
1091
kfd_ioctl_reset_event(struct file * filp,struct kfd_process * p,void * data)1092 static int kfd_ioctl_reset_event(struct file *filp, struct kfd_process *p,
1093 void *data)
1094 {
1095 struct kfd_ioctl_reset_event_args *args = data;
1096
1097 return kfd_reset_event(p, args->event_id);
1098 }
1099
kfd_ioctl_wait_events(struct file * filp,struct kfd_process * p,void * data)1100 static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p,
1101 void *data)
1102 {
1103 struct kfd_ioctl_wait_events_args *args = data;
1104 int err;
1105
1106 err = kfd_wait_on_events(p, args->num_events,
1107 (void __user *)args->events_ptr,
1108 (args->wait_for_all != 0),
1109 args->timeout, &args->wait_result);
1110
1111 return err;
1112 }
kfd_ioctl_set_scratch_backing_va(struct file * filep,struct kfd_process * p,void * data)1113 static int kfd_ioctl_set_scratch_backing_va(struct file *filep,
1114 struct kfd_process *p, void *data)
1115 {
1116 struct kfd_ioctl_set_scratch_backing_va_args *args = data;
1117 struct kfd_process_device *pdd;
1118 struct kfd_dev *dev;
1119 long err;
1120
1121 dev = kfd_device_by_id(args->gpu_id);
1122 if (!dev)
1123 return -EINVAL;
1124
1125 mutex_lock(&p->mutex);
1126
1127 pdd = kfd_bind_process_to_device(dev, p);
1128 if (IS_ERR(pdd)) {
1129 err = PTR_ERR(pdd);
1130 goto bind_process_to_device_fail;
1131 }
1132
1133 pdd->qpd.sh_hidden_private_base = args->va_addr;
1134
1135 mutex_unlock(&p->mutex);
1136
1137 if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS &&
1138 pdd->qpd.vmid != 0 && dev->kfd2kgd->set_scratch_backing_va)
1139 dev->kfd2kgd->set_scratch_backing_va(
1140 dev->kgd, args->va_addr, pdd->qpd.vmid);
1141
1142 return 0;
1143
1144 bind_process_to_device_fail:
1145 mutex_unlock(&p->mutex);
1146 return err;
1147 }
1148
kfd_ioctl_get_tile_config(struct file * filep,struct kfd_process * p,void * data)1149 static int kfd_ioctl_get_tile_config(struct file *filep,
1150 struct kfd_process *p, void *data)
1151 {
1152 struct kfd_ioctl_get_tile_config_args *args = data;
1153 struct kfd_dev *dev;
1154 struct tile_config config;
1155 int err = 0;
1156
1157 dev = kfd_device_by_id(args->gpu_id);
1158 if (!dev)
1159 return -EINVAL;
1160
1161 amdgpu_amdkfd_get_tile_config(dev->kgd, &config);
1162
1163 args->gb_addr_config = config.gb_addr_config;
1164 args->num_banks = config.num_banks;
1165 args->num_ranks = config.num_ranks;
1166
1167 if (args->num_tile_configs > config.num_tile_configs)
1168 args->num_tile_configs = config.num_tile_configs;
1169 err = copy_to_user((void __user *)args->tile_config_ptr,
1170 config.tile_config_ptr,
1171 args->num_tile_configs * sizeof(uint32_t));
1172 if (err) {
1173 args->num_tile_configs = 0;
1174 return -EFAULT;
1175 }
1176
1177 if (args->num_macro_tile_configs > config.num_macro_tile_configs)
1178 args->num_macro_tile_configs =
1179 config.num_macro_tile_configs;
1180 err = copy_to_user((void __user *)args->macro_tile_config_ptr,
1181 config.macro_tile_config_ptr,
1182 args->num_macro_tile_configs * sizeof(uint32_t));
1183 if (err) {
1184 args->num_macro_tile_configs = 0;
1185 return -EFAULT;
1186 }
1187
1188 return 0;
1189 }
1190
kfd_ioctl_acquire_vm(struct file * filep,struct kfd_process * p,void * data)1191 static int kfd_ioctl_acquire_vm(struct file *filep, struct kfd_process *p,
1192 void *data)
1193 {
1194 struct kfd_ioctl_acquire_vm_args *args = data;
1195 struct kfd_process_device *pdd;
1196 struct kfd_dev *dev;
1197 struct file *drm_file;
1198 int ret;
1199
1200 dev = kfd_device_by_id(args->gpu_id);
1201 if (!dev)
1202 return -EINVAL;
1203
1204 drm_file = fget(args->drm_fd);
1205 if (!drm_file)
1206 return -EINVAL;
1207
1208 mutex_lock(&p->mutex);
1209
1210 pdd = kfd_get_process_device_data(dev, p);
1211 if (!pdd) {
1212 ret = -EINVAL;
1213 goto err_unlock;
1214 }
1215
1216 if (pdd->drm_file) {
1217 ret = pdd->drm_file == drm_file ? 0 : -EBUSY;
1218 goto err_unlock;
1219 }
1220
1221 ret = kfd_process_device_init_vm(pdd, drm_file);
1222 if (ret)
1223 goto err_unlock;
1224 /* On success, the PDD keeps the drm_file reference */
1225 mutex_unlock(&p->mutex);
1226
1227 return 0;
1228
1229 err_unlock:
1230 mutex_unlock(&p->mutex);
1231 fput(drm_file);
1232 return ret;
1233 }
1234
kfd_dev_is_large_bar(struct kfd_dev * dev)1235 bool kfd_dev_is_large_bar(struct kfd_dev *dev)
1236 {
1237 struct kfd_local_mem_info mem_info;
1238
1239 if (debug_largebar) {
1240 pr_debug("Simulate large-bar allocation on non large-bar machine\n");
1241 return true;
1242 }
1243
1244 if (dev->use_iommu_v2)
1245 return false;
1246
1247 amdgpu_amdkfd_get_local_mem_info(dev->kgd, &mem_info);
1248 if (mem_info.local_mem_size_private == 0 &&
1249 mem_info.local_mem_size_public > 0)
1250 return true;
1251 return false;
1252 }
1253
kfd_ioctl_alloc_memory_of_gpu(struct file * filep,struct kfd_process * p,void * data)1254 static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep,
1255 struct kfd_process *p, void *data)
1256 {
1257 struct kfd_ioctl_alloc_memory_of_gpu_args *args = data;
1258 struct kfd_process_device *pdd;
1259 void *mem;
1260 struct kfd_dev *dev;
1261 int idr_handle;
1262 long err;
1263 uint64_t offset = args->mmap_offset;
1264 uint32_t flags = args->flags;
1265
1266 if (args->size == 0)
1267 return -EINVAL;
1268
1269 #if IS_ENABLED(CONFIG_HSA_AMD_SVM)
1270 /* Flush pending deferred work to avoid racing with deferred actions
1271 * from previous memory map changes (e.g. munmap).
1272 */
1273 svm_range_list_lock_and_flush_work(&p->svms, current->mm);
1274 mutex_lock(&p->svms.lock);
1275 mmap_write_unlock(current->mm);
1276 if (interval_tree_iter_first(&p->svms.objects,
1277 args->va_addr >> PAGE_SHIFT,
1278 (args->va_addr + args->size - 1) >> PAGE_SHIFT)) {
1279 pr_err("Address: 0x%llx already allocated by SVM\n",
1280 args->va_addr);
1281 mutex_unlock(&p->svms.lock);
1282 return -EADDRINUSE;
1283 }
1284 mutex_unlock(&p->svms.lock);
1285 #endif
1286 dev = kfd_device_by_id(args->gpu_id);
1287 if (!dev)
1288 return -EINVAL;
1289
1290 if ((flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) &&
1291 (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) &&
1292 !kfd_dev_is_large_bar(dev)) {
1293 pr_err("Alloc host visible vram on small bar is not allowed\n");
1294 return -EINVAL;
1295 }
1296
1297 mutex_lock(&p->mutex);
1298
1299 pdd = kfd_bind_process_to_device(dev, p);
1300 if (IS_ERR(pdd)) {
1301 err = PTR_ERR(pdd);
1302 goto err_unlock;
1303 }
1304
1305 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) {
1306 if (args->size != kfd_doorbell_process_slice(dev)) {
1307 err = -EINVAL;
1308 goto err_unlock;
1309 }
1310 offset = kfd_get_process_doorbells(pdd);
1311 } else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
1312 if (args->size != PAGE_SIZE) {
1313 err = -EINVAL;
1314 goto err_unlock;
1315 }
1316 offset = amdgpu_amdkfd_get_mmio_remap_phys_addr(dev->kgd);
1317 if (!offset) {
1318 err = -ENOMEM;
1319 goto err_unlock;
1320 }
1321 }
1322
1323 err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
1324 dev->kgd, args->va_addr, args->size,
1325 pdd->drm_priv, (struct kgd_mem **) &mem, &offset,
1326 flags);
1327
1328 if (err)
1329 goto err_unlock;
1330
1331 idr_handle = kfd_process_device_create_obj_handle(pdd, mem);
1332 if (idr_handle < 0) {
1333 err = -EFAULT;
1334 goto err_free;
1335 }
1336
1337 /* Update the VRAM usage count */
1338 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)
1339 WRITE_ONCE(pdd->vram_usage, pdd->vram_usage + args->size);
1340
1341 mutex_unlock(&p->mutex);
1342
1343 args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
1344 args->mmap_offset = offset;
1345
1346 /* MMIO is mapped through kfd device
1347 * Generate a kfd mmap offset
1348 */
1349 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)
1350 args->mmap_offset = KFD_MMAP_TYPE_MMIO
1351 | KFD_MMAP_GPU_ID(args->gpu_id);
1352
1353 return 0;
1354
1355 err_free:
1356 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, (struct kgd_mem *)mem,
1357 pdd->drm_priv, NULL);
1358 err_unlock:
1359 mutex_unlock(&p->mutex);
1360 return err;
1361 }
1362
kfd_ioctl_free_memory_of_gpu(struct file * filep,struct kfd_process * p,void * data)1363 static int kfd_ioctl_free_memory_of_gpu(struct file *filep,
1364 struct kfd_process *p, void *data)
1365 {
1366 struct kfd_ioctl_free_memory_of_gpu_args *args = data;
1367 struct kfd_process_device *pdd;
1368 void *mem;
1369 struct kfd_dev *dev;
1370 int ret;
1371 uint64_t size = 0;
1372
1373 dev = kfd_device_by_id(GET_GPU_ID(args->handle));
1374 if (!dev)
1375 return -EINVAL;
1376
1377 mutex_lock(&p->mutex);
1378 /*
1379 * Safeguard to prevent user space from freeing signal BO.
1380 * It will be freed at process termination.
1381 */
1382 if (p->signal_handle && (p->signal_handle == args->handle)) {
1383 pr_err("Free signal BO is not allowed\n");
1384 ret = -EPERM;
1385 goto err_unlock;
1386 }
1387
1388 pdd = kfd_get_process_device_data(dev, p);
1389 if (!pdd) {
1390 pr_err("Process device data doesn't exist\n");
1391 ret = -EINVAL;
1392 goto err_unlock;
1393 }
1394
1395 mem = kfd_process_device_translate_handle(
1396 pdd, GET_IDR_HANDLE(args->handle));
1397 if (!mem) {
1398 ret = -EINVAL;
1399 goto err_unlock;
1400 }
1401
1402 ret = amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd,
1403 (struct kgd_mem *)mem, pdd->drm_priv, &size);
1404
1405 /* If freeing the buffer failed, leave the handle in place for
1406 * clean-up during process tear-down.
1407 */
1408 if (!ret)
1409 kfd_process_device_remove_obj_handle(
1410 pdd, GET_IDR_HANDLE(args->handle));
1411
1412 WRITE_ONCE(pdd->vram_usage, pdd->vram_usage - size);
1413
1414 err_unlock:
1415 mutex_unlock(&p->mutex);
1416 return ret;
1417 }
1418
kfd_ioctl_map_memory_to_gpu(struct file * filep,struct kfd_process * p,void * data)1419 static int kfd_ioctl_map_memory_to_gpu(struct file *filep,
1420 struct kfd_process *p, void *data)
1421 {
1422 struct kfd_ioctl_map_memory_to_gpu_args *args = data;
1423 struct kfd_process_device *pdd, *peer_pdd;
1424 void *mem;
1425 struct kfd_dev *dev, *peer;
1426 long err = 0;
1427 int i;
1428 uint32_t *devices_arr = NULL;
1429 bool table_freed = false;
1430
1431 dev = kfd_device_by_id(GET_GPU_ID(args->handle));
1432 if (!dev)
1433 return -EINVAL;
1434
1435 if (!args->n_devices) {
1436 pr_debug("Device IDs array empty\n");
1437 return -EINVAL;
1438 }
1439 if (args->n_success > args->n_devices) {
1440 pr_debug("n_success exceeds n_devices\n");
1441 return -EINVAL;
1442 }
1443
1444 devices_arr = kmalloc_array(args->n_devices, sizeof(*devices_arr),
1445 GFP_KERNEL);
1446 if (!devices_arr)
1447 return -ENOMEM;
1448
1449 err = copy_from_user(devices_arr,
1450 (void __user *)args->device_ids_array_ptr,
1451 args->n_devices * sizeof(*devices_arr));
1452 if (err != 0) {
1453 err = -EFAULT;
1454 goto copy_from_user_failed;
1455 }
1456
1457 mutex_lock(&p->mutex);
1458
1459 pdd = kfd_bind_process_to_device(dev, p);
1460 if (IS_ERR(pdd)) {
1461 err = PTR_ERR(pdd);
1462 goto bind_process_to_device_failed;
1463 }
1464
1465 mem = kfd_process_device_translate_handle(pdd,
1466 GET_IDR_HANDLE(args->handle));
1467 if (!mem) {
1468 err = -ENOMEM;
1469 goto get_mem_obj_from_handle_failed;
1470 }
1471
1472 for (i = args->n_success; i < args->n_devices; i++) {
1473 peer = kfd_device_by_id(devices_arr[i]);
1474 if (!peer) {
1475 pr_debug("Getting device by id failed for 0x%x\n",
1476 devices_arr[i]);
1477 err = -EINVAL;
1478 goto get_mem_obj_from_handle_failed;
1479 }
1480
1481 peer_pdd = kfd_bind_process_to_device(peer, p);
1482 if (IS_ERR(peer_pdd)) {
1483 err = PTR_ERR(peer_pdd);
1484 goto get_mem_obj_from_handle_failed;
1485 }
1486 err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
1487 peer->kgd, (struct kgd_mem *)mem,
1488 peer_pdd->drm_priv, &table_freed);
1489 if (err) {
1490 pr_err("Failed to map to gpu %d/%d\n",
1491 i, args->n_devices);
1492 goto map_memory_to_gpu_failed;
1493 }
1494 args->n_success = i+1;
1495 }
1496
1497 mutex_unlock(&p->mutex);
1498
1499 err = amdgpu_amdkfd_gpuvm_sync_memory(dev->kgd, (struct kgd_mem *) mem, true);
1500 if (err) {
1501 pr_debug("Sync memory failed, wait interrupted by user signal\n");
1502 goto sync_memory_failed;
1503 }
1504
1505 /* Flush TLBs after waiting for the page table updates to complete */
1506 if (table_freed) {
1507 for (i = 0; i < args->n_devices; i++) {
1508 peer = kfd_device_by_id(devices_arr[i]);
1509 if (WARN_ON_ONCE(!peer))
1510 continue;
1511 peer_pdd = kfd_get_process_device_data(peer, p);
1512 if (WARN_ON_ONCE(!peer_pdd))
1513 continue;
1514 kfd_flush_tlb(peer_pdd, TLB_FLUSH_LEGACY);
1515 }
1516 }
1517 kfree(devices_arr);
1518
1519 return err;
1520
1521 bind_process_to_device_failed:
1522 get_mem_obj_from_handle_failed:
1523 map_memory_to_gpu_failed:
1524 mutex_unlock(&p->mutex);
1525 copy_from_user_failed:
1526 sync_memory_failed:
1527 kfree(devices_arr);
1528
1529 return err;
1530 }
1531
kfd_ioctl_unmap_memory_from_gpu(struct file * filep,struct kfd_process * p,void * data)1532 static int kfd_ioctl_unmap_memory_from_gpu(struct file *filep,
1533 struct kfd_process *p, void *data)
1534 {
1535 struct kfd_ioctl_unmap_memory_from_gpu_args *args = data;
1536 struct kfd_process_device *pdd, *peer_pdd;
1537 void *mem;
1538 struct kfd_dev *dev, *peer;
1539 long err = 0;
1540 uint32_t *devices_arr = NULL, i;
1541
1542 dev = kfd_device_by_id(GET_GPU_ID(args->handle));
1543 if (!dev)
1544 return -EINVAL;
1545
1546 if (!args->n_devices) {
1547 pr_debug("Device IDs array empty\n");
1548 return -EINVAL;
1549 }
1550 if (args->n_success > args->n_devices) {
1551 pr_debug("n_success exceeds n_devices\n");
1552 return -EINVAL;
1553 }
1554
1555 devices_arr = kmalloc_array(args->n_devices, sizeof(*devices_arr),
1556 GFP_KERNEL);
1557 if (!devices_arr)
1558 return -ENOMEM;
1559
1560 err = copy_from_user(devices_arr,
1561 (void __user *)args->device_ids_array_ptr,
1562 args->n_devices * sizeof(*devices_arr));
1563 if (err != 0) {
1564 err = -EFAULT;
1565 goto copy_from_user_failed;
1566 }
1567
1568 mutex_lock(&p->mutex);
1569
1570 pdd = kfd_get_process_device_data(dev, p);
1571 if (!pdd) {
1572 err = -EINVAL;
1573 goto bind_process_to_device_failed;
1574 }
1575
1576 mem = kfd_process_device_translate_handle(pdd,
1577 GET_IDR_HANDLE(args->handle));
1578 if (!mem) {
1579 err = -ENOMEM;
1580 goto get_mem_obj_from_handle_failed;
1581 }
1582
1583 for (i = args->n_success; i < args->n_devices; i++) {
1584 peer = kfd_device_by_id(devices_arr[i]);
1585 if (!peer) {
1586 err = -EINVAL;
1587 goto get_mem_obj_from_handle_failed;
1588 }
1589
1590 peer_pdd = kfd_get_process_device_data(peer, p);
1591 if (!peer_pdd) {
1592 err = -ENODEV;
1593 goto get_mem_obj_from_handle_failed;
1594 }
1595 err = amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
1596 peer->kgd, (struct kgd_mem *)mem, peer_pdd->drm_priv);
1597 if (err) {
1598 pr_err("Failed to unmap from gpu %d/%d\n",
1599 i, args->n_devices);
1600 goto unmap_memory_from_gpu_failed;
1601 }
1602 args->n_success = i+1;
1603 }
1604 mutex_unlock(&p->mutex);
1605
1606 if (dev->device_info->asic_family == CHIP_ALDEBARAN) {
1607 err = amdgpu_amdkfd_gpuvm_sync_memory(dev->kgd,
1608 (struct kgd_mem *) mem, true);
1609 if (err) {
1610 pr_debug("Sync memory failed, wait interrupted by user signal\n");
1611 goto sync_memory_failed;
1612 }
1613
1614 /* Flush TLBs after waiting for the page table updates to complete */
1615 for (i = 0; i < args->n_devices; i++) {
1616 peer = kfd_device_by_id(devices_arr[i]);
1617 if (WARN_ON_ONCE(!peer))
1618 continue;
1619 peer_pdd = kfd_get_process_device_data(peer, p);
1620 if (WARN_ON_ONCE(!peer_pdd))
1621 continue;
1622 kfd_flush_tlb(peer_pdd, TLB_FLUSH_HEAVYWEIGHT);
1623 }
1624 }
1625 kfree(devices_arr);
1626
1627 return 0;
1628
1629 bind_process_to_device_failed:
1630 get_mem_obj_from_handle_failed:
1631 unmap_memory_from_gpu_failed:
1632 mutex_unlock(&p->mutex);
1633 copy_from_user_failed:
1634 sync_memory_failed:
1635 kfree(devices_arr);
1636 return err;
1637 }
1638
kfd_ioctl_alloc_queue_gws(struct file * filep,struct kfd_process * p,void * data)1639 static int kfd_ioctl_alloc_queue_gws(struct file *filep,
1640 struct kfd_process *p, void *data)
1641 {
1642 int retval;
1643 struct kfd_ioctl_alloc_queue_gws_args *args = data;
1644 struct queue *q;
1645 struct kfd_dev *dev;
1646
1647 mutex_lock(&p->mutex);
1648 q = pqm_get_user_queue(&p->pqm, args->queue_id);
1649
1650 if (q) {
1651 dev = q->device;
1652 } else {
1653 retval = -EINVAL;
1654 goto out_unlock;
1655 }
1656
1657 if (!dev->gws) {
1658 retval = -ENODEV;
1659 goto out_unlock;
1660 }
1661
1662 if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
1663 retval = -ENODEV;
1664 goto out_unlock;
1665 }
1666
1667 retval = pqm_set_gws(&p->pqm, args->queue_id, args->num_gws ? dev->gws : NULL);
1668 mutex_unlock(&p->mutex);
1669
1670 args->first_gws = 0;
1671 return retval;
1672
1673 out_unlock:
1674 mutex_unlock(&p->mutex);
1675 return retval;
1676 }
1677
kfd_ioctl_get_dmabuf_info(struct file * filep,struct kfd_process * p,void * data)1678 static int kfd_ioctl_get_dmabuf_info(struct file *filep,
1679 struct kfd_process *p, void *data)
1680 {
1681 struct kfd_ioctl_get_dmabuf_info_args *args = data;
1682 struct kfd_dev *dev = NULL;
1683 struct kgd_dev *dma_buf_kgd;
1684 void *metadata_buffer = NULL;
1685 uint32_t flags;
1686 unsigned int i;
1687 int r;
1688
1689 /* Find a KFD GPU device that supports the get_dmabuf_info query */
1690 for (i = 0; kfd_topology_enum_kfd_devices(i, &dev) == 0; i++)
1691 if (dev)
1692 break;
1693 if (!dev)
1694 return -EINVAL;
1695
1696 if (args->metadata_ptr) {
1697 metadata_buffer = kzalloc(args->metadata_size, GFP_KERNEL);
1698 if (!metadata_buffer)
1699 return -ENOMEM;
1700 }
1701
1702 /* Get dmabuf info from KGD */
1703 r = amdgpu_amdkfd_get_dmabuf_info(dev->kgd, args->dmabuf_fd,
1704 &dma_buf_kgd, &args->size,
1705 metadata_buffer, args->metadata_size,
1706 &args->metadata_size, &flags);
1707 if (r)
1708 goto exit;
1709
1710 /* Reverse-lookup gpu_id from kgd pointer */
1711 dev = kfd_device_by_kgd(dma_buf_kgd);
1712 if (!dev) {
1713 r = -EINVAL;
1714 goto exit;
1715 }
1716 args->gpu_id = dev->id;
1717 args->flags = flags;
1718
1719 /* Copy metadata buffer to user mode */
1720 if (metadata_buffer) {
1721 r = copy_to_user((void __user *)args->metadata_ptr,
1722 metadata_buffer, args->metadata_size);
1723 if (r != 0)
1724 r = -EFAULT;
1725 }
1726
1727 exit:
1728 kfree(metadata_buffer);
1729
1730 return r;
1731 }
1732
kfd_ioctl_import_dmabuf(struct file * filep,struct kfd_process * p,void * data)1733 static int kfd_ioctl_import_dmabuf(struct file *filep,
1734 struct kfd_process *p, void *data)
1735 {
1736 struct kfd_ioctl_import_dmabuf_args *args = data;
1737 struct kfd_process_device *pdd;
1738 struct dma_buf *dmabuf;
1739 struct kfd_dev *dev;
1740 int idr_handle;
1741 uint64_t size;
1742 void *mem;
1743 int r;
1744
1745 dev = kfd_device_by_id(args->gpu_id);
1746 if (!dev)
1747 return -EINVAL;
1748
1749 dmabuf = dma_buf_get(args->dmabuf_fd);
1750 if (IS_ERR(dmabuf))
1751 return PTR_ERR(dmabuf);
1752
1753 mutex_lock(&p->mutex);
1754
1755 pdd = kfd_bind_process_to_device(dev, p);
1756 if (IS_ERR(pdd)) {
1757 r = PTR_ERR(pdd);
1758 goto err_unlock;
1759 }
1760
1761 r = amdgpu_amdkfd_gpuvm_import_dmabuf(dev->kgd, dmabuf,
1762 args->va_addr, pdd->drm_priv,
1763 (struct kgd_mem **)&mem, &size,
1764 NULL);
1765 if (r)
1766 goto err_unlock;
1767
1768 idr_handle = kfd_process_device_create_obj_handle(pdd, mem);
1769 if (idr_handle < 0) {
1770 r = -EFAULT;
1771 goto err_free;
1772 }
1773
1774 mutex_unlock(&p->mutex);
1775 dma_buf_put(dmabuf);
1776
1777 args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
1778
1779 return 0;
1780
1781 err_free:
1782 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, (struct kgd_mem *)mem,
1783 pdd->drm_priv, NULL);
1784 err_unlock:
1785 mutex_unlock(&p->mutex);
1786 dma_buf_put(dmabuf);
1787 return r;
1788 }
1789
1790 /* Handle requests for watching SMI events */
kfd_ioctl_smi_events(struct file * filep,struct kfd_process * p,void * data)1791 static int kfd_ioctl_smi_events(struct file *filep,
1792 struct kfd_process *p, void *data)
1793 {
1794 struct kfd_ioctl_smi_events_args *args = data;
1795 struct kfd_dev *dev;
1796
1797 dev = kfd_device_by_id(args->gpuid);
1798 if (!dev)
1799 return -EINVAL;
1800
1801 return kfd_smi_event_open(dev, &args->anon_fd);
1802 }
1803
kfd_ioctl_set_xnack_mode(struct file * filep,struct kfd_process * p,void * data)1804 static int kfd_ioctl_set_xnack_mode(struct file *filep,
1805 struct kfd_process *p, void *data)
1806 {
1807 struct kfd_ioctl_set_xnack_mode_args *args = data;
1808 int r = 0;
1809
1810 mutex_lock(&p->mutex);
1811 if (args->xnack_enabled >= 0) {
1812 if (!list_empty(&p->pqm.queues)) {
1813 pr_debug("Process has user queues running\n");
1814 mutex_unlock(&p->mutex);
1815 return -EBUSY;
1816 }
1817 if (args->xnack_enabled && !kfd_process_xnack_mode(p, true))
1818 r = -EPERM;
1819 else
1820 p->xnack_enabled = args->xnack_enabled;
1821 } else {
1822 args->xnack_enabled = p->xnack_enabled;
1823 }
1824 mutex_unlock(&p->mutex);
1825
1826 return r;
1827 }
1828
1829 #if IS_ENABLED(CONFIG_HSA_AMD_SVM)
kfd_ioctl_svm(struct file * filep,struct kfd_process * p,void * data)1830 static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
1831 {
1832 struct kfd_ioctl_svm_args *args = data;
1833 int r = 0;
1834
1835 pr_debug("start 0x%llx size 0x%llx op 0x%x nattr 0x%x\n",
1836 args->start_addr, args->size, args->op, args->nattr);
1837
1838 if ((args->start_addr & ~PAGE_MASK) || (args->size & ~PAGE_MASK))
1839 return -EINVAL;
1840 if (!args->start_addr || !args->size)
1841 return -EINVAL;
1842
1843 mutex_lock(&p->mutex);
1844
1845 r = svm_ioctl(p, args->op, args->start_addr, args->size, args->nattr,
1846 args->attrs);
1847
1848 mutex_unlock(&p->mutex);
1849
1850 return r;
1851 }
1852 #else
kfd_ioctl_svm(struct file * filep,struct kfd_process * p,void * data)1853 static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
1854 {
1855 return -EPERM;
1856 }
1857 #endif
1858
1859 #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
1860 [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \
1861 .cmd_drv = 0, .name = #ioctl}
1862
1863 /** Ioctl table */
1864 static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
1865 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_VERSION,
1866 kfd_ioctl_get_version, 0),
1867
1868 AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_QUEUE,
1869 kfd_ioctl_create_queue, 0),
1870
1871 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_QUEUE,
1872 kfd_ioctl_destroy_queue, 0),
1873
1874 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_MEMORY_POLICY,
1875 kfd_ioctl_set_memory_policy, 0),
1876
1877 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_CLOCK_COUNTERS,
1878 kfd_ioctl_get_clock_counters, 0),
1879
1880 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES,
1881 kfd_ioctl_get_process_apertures, 0),
1882
1883 AMDKFD_IOCTL_DEF(AMDKFD_IOC_UPDATE_QUEUE,
1884 kfd_ioctl_update_queue, 0),
1885
1886 AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_EVENT,
1887 kfd_ioctl_create_event, 0),
1888
1889 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_EVENT,
1890 kfd_ioctl_destroy_event, 0),
1891
1892 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_EVENT,
1893 kfd_ioctl_set_event, 0),
1894
1895 AMDKFD_IOCTL_DEF(AMDKFD_IOC_RESET_EVENT,
1896 kfd_ioctl_reset_event, 0),
1897
1898 AMDKFD_IOCTL_DEF(AMDKFD_IOC_WAIT_EVENTS,
1899 kfd_ioctl_wait_events, 0),
1900
1901 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_REGISTER,
1902 kfd_ioctl_dbg_register, 0),
1903
1904 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_UNREGISTER,
1905 kfd_ioctl_dbg_unregister, 0),
1906
1907 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_ADDRESS_WATCH,
1908 kfd_ioctl_dbg_address_watch, 0),
1909
1910 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_WAVE_CONTROL,
1911 kfd_ioctl_dbg_wave_control, 0),
1912
1913 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_SCRATCH_BACKING_VA,
1914 kfd_ioctl_set_scratch_backing_va, 0),
1915
1916 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_TILE_CONFIG,
1917 kfd_ioctl_get_tile_config, 0),
1918
1919 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_TRAP_HANDLER,
1920 kfd_ioctl_set_trap_handler, 0),
1921
1922 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES_NEW,
1923 kfd_ioctl_get_process_apertures_new, 0),
1924
1925 AMDKFD_IOCTL_DEF(AMDKFD_IOC_ACQUIRE_VM,
1926 kfd_ioctl_acquire_vm, 0),
1927
1928 AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_MEMORY_OF_GPU,
1929 kfd_ioctl_alloc_memory_of_gpu, 0),
1930
1931 AMDKFD_IOCTL_DEF(AMDKFD_IOC_FREE_MEMORY_OF_GPU,
1932 kfd_ioctl_free_memory_of_gpu, 0),
1933
1934 AMDKFD_IOCTL_DEF(AMDKFD_IOC_MAP_MEMORY_TO_GPU,
1935 kfd_ioctl_map_memory_to_gpu, 0),
1936
1937 AMDKFD_IOCTL_DEF(AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU,
1938 kfd_ioctl_unmap_memory_from_gpu, 0),
1939
1940 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_CU_MASK,
1941 kfd_ioctl_set_cu_mask, 0),
1942
1943 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_QUEUE_WAVE_STATE,
1944 kfd_ioctl_get_queue_wave_state, 0),
1945
1946 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_DMABUF_INFO,
1947 kfd_ioctl_get_dmabuf_info, 0),
1948
1949 AMDKFD_IOCTL_DEF(AMDKFD_IOC_IMPORT_DMABUF,
1950 kfd_ioctl_import_dmabuf, 0),
1951
1952 AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_QUEUE_GWS,
1953 kfd_ioctl_alloc_queue_gws, 0),
1954
1955 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SMI_EVENTS,
1956 kfd_ioctl_smi_events, 0),
1957
1958 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SVM, kfd_ioctl_svm, 0),
1959
1960 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_XNACK_MODE,
1961 kfd_ioctl_set_xnack_mode, 0),
1962 };
1963
1964 #define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls)
1965
kfd_ioctl(struct file * filep,unsigned int cmd,unsigned long arg)1966 static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
1967 {
1968 struct kfd_process *process;
1969 amdkfd_ioctl_t *func;
1970 const struct amdkfd_ioctl_desc *ioctl = NULL;
1971 unsigned int nr = _IOC_NR(cmd);
1972 char stack_kdata[128];
1973 char *kdata = NULL;
1974 unsigned int usize, asize;
1975 int retcode = -EINVAL;
1976
1977 if (nr >= AMDKFD_CORE_IOCTL_COUNT)
1978 goto err_i1;
1979
1980 if ((nr >= AMDKFD_COMMAND_START) && (nr < AMDKFD_COMMAND_END)) {
1981 u32 amdkfd_size;
1982
1983 ioctl = &amdkfd_ioctls[nr];
1984
1985 amdkfd_size = _IOC_SIZE(ioctl->cmd);
1986 usize = asize = _IOC_SIZE(cmd);
1987 if (amdkfd_size > asize)
1988 asize = amdkfd_size;
1989
1990 cmd = ioctl->cmd;
1991 } else
1992 goto err_i1;
1993
1994 dev_dbg(kfd_device, "ioctl cmd 0x%x (#0x%x), arg 0x%lx\n", cmd, nr, arg);
1995
1996 /* Get the process struct from the filep. Only the process
1997 * that opened /dev/kfd can use the file descriptor. Child
1998 * processes need to create their own KFD device context.
1999 */
2000 process = filep->private_data;
2001 if (process->lead_thread != current->group_leader) {
2002 dev_dbg(kfd_device, "Using KFD FD in wrong process\n");
2003 retcode = -EBADF;
2004 goto err_i1;
2005 }
2006
2007 /* Do not trust userspace, use our own definition */
2008 func = ioctl->func;
2009
2010 if (unlikely(!func)) {
2011 dev_dbg(kfd_device, "no function\n");
2012 retcode = -EINVAL;
2013 goto err_i1;
2014 }
2015
2016 if (cmd & (IOC_IN | IOC_OUT)) {
2017 if (asize <= sizeof(stack_kdata)) {
2018 kdata = stack_kdata;
2019 } else {
2020 kdata = kmalloc(asize, GFP_KERNEL);
2021 if (!kdata) {
2022 retcode = -ENOMEM;
2023 goto err_i1;
2024 }
2025 }
2026 if (asize > usize)
2027 memset(kdata + usize, 0, asize - usize);
2028 }
2029
2030 if (cmd & IOC_IN) {
2031 if (copy_from_user(kdata, (void __user *)arg, usize) != 0) {
2032 retcode = -EFAULT;
2033 goto err_i1;
2034 }
2035 } else if (cmd & IOC_OUT) {
2036 memset(kdata, 0, usize);
2037 }
2038
2039 retcode = func(filep, process, kdata);
2040
2041 if (cmd & IOC_OUT)
2042 if (copy_to_user((void __user *)arg, kdata, usize) != 0)
2043 retcode = -EFAULT;
2044
2045 err_i1:
2046 if (!ioctl)
2047 dev_dbg(kfd_device, "invalid ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n",
2048 task_pid_nr(current), cmd, nr);
2049
2050 if (kdata != stack_kdata)
2051 kfree(kdata);
2052
2053 if (retcode)
2054 dev_dbg(kfd_device, "ioctl cmd (#0x%x), arg 0x%lx, ret = %d\n",
2055 nr, arg, retcode);
2056
2057 return retcode;
2058 }
2059
kfd_mmio_mmap(struct kfd_dev * dev,struct kfd_process * process,struct vm_area_struct * vma)2060 static int kfd_mmio_mmap(struct kfd_dev *dev, struct kfd_process *process,
2061 struct vm_area_struct *vma)
2062 {
2063 phys_addr_t address;
2064 int ret;
2065
2066 if (vma->vm_end - vma->vm_start != PAGE_SIZE)
2067 return -EINVAL;
2068
2069 address = amdgpu_amdkfd_get_mmio_remap_phys_addr(dev->kgd);
2070
2071 vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE |
2072 VM_DONTDUMP | VM_PFNMAP;
2073
2074 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
2075
2076 pr_debug("pasid 0x%x mapping mmio page\n"
2077 " target user address == 0x%08llX\n"
2078 " physical address == 0x%08llX\n"
2079 " vm_flags == 0x%04lX\n"
2080 " size == 0x%04lX\n",
2081 process->pasid, (unsigned long long) vma->vm_start,
2082 address, vma->vm_flags, PAGE_SIZE);
2083
2084 ret = io_remap_pfn_range(vma,
2085 vma->vm_start,
2086 address >> PAGE_SHIFT,
2087 PAGE_SIZE,
2088 vma->vm_page_prot);
2089 return ret;
2090 }
2091
2092
kfd_mmap(struct file * filp,struct vm_area_struct * vma)2093 static int kfd_mmap(struct file *filp, struct vm_area_struct *vma)
2094 {
2095 struct kfd_process *process;
2096 struct kfd_dev *dev = NULL;
2097 unsigned long mmap_offset;
2098 unsigned int gpu_id;
2099
2100 process = kfd_get_process(current);
2101 if (IS_ERR(process))
2102 return PTR_ERR(process);
2103
2104 mmap_offset = vma->vm_pgoff << PAGE_SHIFT;
2105 gpu_id = KFD_MMAP_GET_GPU_ID(mmap_offset);
2106 if (gpu_id)
2107 dev = kfd_device_by_id(gpu_id);
2108
2109 switch (mmap_offset & KFD_MMAP_TYPE_MASK) {
2110 case KFD_MMAP_TYPE_DOORBELL:
2111 if (!dev)
2112 return -ENODEV;
2113 return kfd_doorbell_mmap(dev, process, vma);
2114
2115 case KFD_MMAP_TYPE_EVENTS:
2116 return kfd_event_mmap(process, vma);
2117
2118 case KFD_MMAP_TYPE_RESERVED_MEM:
2119 if (!dev)
2120 return -ENODEV;
2121 return kfd_reserved_mem_mmap(dev, process, vma);
2122 case KFD_MMAP_TYPE_MMIO:
2123 if (!dev)
2124 return -ENODEV;
2125 return kfd_mmio_mmap(dev, process, vma);
2126 }
2127
2128 return -EFAULT;
2129 }
2130