Lines Matching refs:eb
312 static int eb_parse(struct i915_execbuffer *eb);
313 static int eb_pin_engine(struct i915_execbuffer *eb, bool throttle);
314 static void eb_unpin_engine(struct i915_execbuffer *eb);
316 static inline bool eb_use_cmdparser(const struct i915_execbuffer *eb) in eb_use_cmdparser() argument
318 return intel_engine_requires_cmd_parser(eb->context->engine) || in eb_use_cmdparser()
319 (intel_engine_using_cmd_parser(eb->context->engine) && in eb_use_cmdparser()
320 eb->args->batch_len); in eb_use_cmdparser()
323 static int eb_create(struct i915_execbuffer *eb) in eb_create() argument
325 if (!(eb->args->flags & I915_EXEC_HANDLE_LUT)) { in eb_create()
326 unsigned int size = 1 + ilog2(eb->buffer_count); in eb_create()
352 eb->buckets = kzalloc(sizeof(struct hlist_head) << size, in eb_create()
354 if (eb->buckets) in eb_create()
361 eb->lut_size = size; in eb_create()
363 eb->lut_size = -eb->buffer_count; in eb_create()
426 eb_pin_vma(struct i915_execbuffer *eb, in eb_pin_vma() argument
444 err = i915_vma_pin_ww(vma, &eb->ww, 0, 0, pin_flags); in eb_pin_vma()
453 err = i915_vma_pin_ww(vma, &eb->ww, in eb_pin_vma()
494 eb_validate_vma(struct i915_execbuffer *eb, in eb_validate_vma() argument
502 GRAPHICS_VER(eb->i915) >= 12 && !IS_TIGERLAKE(eb->i915)) in eb_validate_vma()
505 if (unlikely(entry->flags & eb->invalid_flags)) in eb_validate_vma()
534 if (!eb->reloc_cache.has_fence) { in eb_validate_vma()
538 eb->reloc_cache.needs_unfenced) && in eb_validate_vma()
547 is_batch_buffer(struct i915_execbuffer *eb, unsigned int buffer_idx) in is_batch_buffer() argument
549 return eb->args->flags & I915_EXEC_BATCH_FIRST ? in is_batch_buffer()
550 buffer_idx < eb->num_batches : in is_batch_buffer()
551 buffer_idx >= eb->args->buffer_count - eb->num_batches; in is_batch_buffer()
555 eb_add_vma(struct i915_execbuffer *eb, in eb_add_vma() argument
560 struct drm_i915_private *i915 = eb->i915; in eb_add_vma()
561 struct drm_i915_gem_exec_object2 *entry = &eb->exec[i]; in eb_add_vma()
562 struct eb_vma *ev = &eb->vma[i]; in eb_add_vma()
568 if (eb->lut_size > 0) { in eb_add_vma()
571 &eb->buckets[hash_32(entry->handle, in eb_add_vma()
572 eb->lut_size)]); in eb_add_vma()
576 list_add_tail(&ev->reloc_link, &eb->relocs); in eb_add_vma()
587 if (is_batch_buffer(eb, i)) { in eb_add_vma()
591 if (eb->reloc_cache.has_fence) in eb_add_vma()
594 eb->batches[*current_batch] = ev; in eb_add_vma()
603 eb->batch_start_offset, in eb_add_vma()
604 eb->args->batch_len, in eb_add_vma()
610 if (eb->args->batch_len == 0) in eb_add_vma()
611 eb->batch_len[*current_batch] = ev->vma->size - in eb_add_vma()
612 eb->batch_start_offset; in eb_add_vma()
614 eb->batch_len[*current_batch] = eb->args->batch_len; in eb_add_vma()
615 if (unlikely(eb->batch_len[*current_batch] == 0)) { /* impossible! */ in eb_add_vma()
643 static int eb_reserve_vma(struct i915_execbuffer *eb, in eb_reserve_vma() argument
658 err = i915_vma_pin_ww(vma, &eb->ww, in eb_reserve_vma()
666 eb->args->flags |= __EXEC_HAS_RELOC; in eb_reserve_vma()
686 static int eb_reserve(struct i915_execbuffer *eb) in eb_reserve() argument
688 const unsigned int count = eb->buffer_count; in eb_reserve()
710 list_for_each_entry(ev, &eb->unbound, bind_link) { in eb_reserve()
711 err = eb_reserve_vma(eb, ev, pin_flags); in eb_reserve()
719 INIT_LIST_HEAD(&eb->unbound); in eb_reserve()
724 ev = &eb->vma[i]; in eb_reserve()
734 list_add(&ev->bind_link, &eb->unbound); in eb_reserve()
737 list_add_tail(&ev->bind_link, &eb->unbound); in eb_reserve()
744 list_splice_tail(&last, &eb->unbound); in eb_reserve()
752 mutex_lock(&eb->context->vm->mutex); in eb_reserve()
753 err = i915_gem_evict_vm(eb->context->vm); in eb_reserve()
754 mutex_unlock(&eb->context->vm->mutex); in eb_reserve()
767 static int eb_select_context(struct i915_execbuffer *eb) in eb_select_context() argument
771 ctx = i915_gem_context_lookup(eb->file->driver_priv, eb->args->rsvd1); in eb_select_context()
775 eb->gem_context = ctx; in eb_select_context()
777 eb->invalid_flags |= EXEC_OBJECT_NEEDS_GTT; in eb_select_context()
782 static int __eb_add_lut(struct i915_execbuffer *eb, in __eb_add_lut() argument
785 struct i915_gem_context *ctx = eb->gem_context; in __eb_add_lut()
810 if (idr_find(&eb->file->object_idr, handle) == obj) { in __eb_add_lut()
832 static struct i915_vma *eb_lookup_vma(struct i915_execbuffer *eb, u32 handle) in eb_lookup_vma() argument
834 struct i915_address_space *vm = eb->context->vm; in eb_lookup_vma()
842 vma = radix_tree_lookup(&eb->gem_context->handles_vma, handle); in eb_lookup_vma()
849 obj = i915_gem_object_lookup(eb->file, handle); in eb_lookup_vma()
860 if (i915_gem_context_uses_protected_content(eb->gem_context) && in eb_lookup_vma()
875 err = __eb_add_lut(eb, handle, vma); in eb_lookup_vma()
885 static int eb_lookup_vmas(struct i915_execbuffer *eb) in eb_lookup_vmas() argument
890 INIT_LIST_HEAD(&eb->relocs); in eb_lookup_vmas()
892 for (i = 0; i < eb->buffer_count; i++) { in eb_lookup_vmas()
895 vma = eb_lookup_vma(eb, eb->exec[i].handle); in eb_lookup_vmas()
901 err = eb_validate_vma(eb, &eb->exec[i], vma); in eb_lookup_vmas()
907 err = eb_add_vma(eb, ¤t_batch, i, vma); in eb_lookup_vmas()
914 if (i + 1 < eb->buffer_count) { in eb_lookup_vmas()
921 eb->vma[i + 1].vma = NULL; in eb_lookup_vmas()
927 eb->vma[i].flags |= __EXEC_OBJECT_USERPTR_INIT; in eb_lookup_vmas()
928 eb->args->flags |= __EXEC_USERPTR_USED; in eb_lookup_vmas()
935 eb->vma[i].vma = NULL; in eb_lookup_vmas()
939 static int eb_lock_vmas(struct i915_execbuffer *eb) in eb_lock_vmas() argument
944 for (i = 0; i < eb->buffer_count; i++) { in eb_lock_vmas()
945 struct eb_vma *ev = &eb->vma[i]; in eb_lock_vmas()
948 err = i915_gem_object_lock(vma->obj, &eb->ww); in eb_lock_vmas()
956 static int eb_validate_vmas(struct i915_execbuffer *eb) in eb_validate_vmas() argument
961 INIT_LIST_HEAD(&eb->unbound); in eb_validate_vmas()
963 err = eb_lock_vmas(eb); in eb_validate_vmas()
967 for (i = 0; i < eb->buffer_count; i++) { in eb_validate_vmas()
968 struct drm_i915_gem_exec_object2 *entry = &eb->exec[i]; in eb_validate_vmas()
969 struct eb_vma *ev = &eb->vma[i]; in eb_validate_vmas()
972 err = eb_pin_vma(eb, entry, ev); in eb_validate_vmas()
979 eb->args->flags |= __EXEC_HAS_RELOC; in eb_validate_vmas()
984 list_add_tail(&ev->bind_link, &eb->unbound); in eb_validate_vmas()
999 eb_vma_misplaced(&eb->exec[i], vma, ev->flags)); in eb_validate_vmas()
1002 if (!list_empty(&eb->unbound)) in eb_validate_vmas()
1003 return eb_reserve(eb); in eb_validate_vmas()
1009 eb_get_vma(const struct i915_execbuffer *eb, unsigned long handle) in eb_get_vma() argument
1011 if (eb->lut_size < 0) { in eb_get_vma()
1012 if (handle >= -eb->lut_size) in eb_get_vma()
1014 return &eb->vma[handle]; in eb_get_vma()
1019 head = &eb->buckets[hash_32(handle, eb->lut_size)]; in eb_get_vma()
1028 static void eb_release_vmas(struct i915_execbuffer *eb, bool final) in eb_release_vmas() argument
1030 const unsigned int count = eb->buffer_count; in eb_release_vmas()
1034 struct eb_vma *ev = &eb->vma[i]; in eb_release_vmas()
1046 eb_unpin_engine(eb); in eb_release_vmas()
1049 static void eb_destroy(const struct i915_execbuffer *eb) in eb_destroy() argument
1051 if (eb->lut_size > 0) in eb_destroy()
1052 kfree(eb->buckets); in eb_destroy()
1095 static void reloc_cache_reset(struct reloc_cache *cache, struct i915_execbuffer *eb) in reloc_cache_reset() argument
1171 struct i915_execbuffer *eb, in reloc_iomap() argument
1174 struct reloc_cache *cache = &eb->reloc_cache; in reloc_iomap()
1196 vma = i915_gem_object_ggtt_pin_ww(obj, &eb->ww, NULL, 0, 0, in reloc_iomap()
1238 struct i915_execbuffer *eb, in reloc_vaddr() argument
1241 struct reloc_cache *cache = &eb->reloc_cache; in reloc_vaddr()
1249 vaddr = reloc_iomap(obj, eb, page); in reloc_vaddr()
1283 struct i915_execbuffer *eb, in relocate_entry() argument
1288 bool wide = eb->reloc_cache.use_64bit_reloc; in relocate_entry()
1292 vaddr = reloc_vaddr(vma->obj, eb, in relocate_entry()
1300 eb->reloc_cache.vaddr); in relocate_entry()
1313 eb_relocate_entry(struct i915_execbuffer *eb, in eb_relocate_entry() argument
1317 struct drm_i915_private *i915 = eb->i915; in eb_relocate_entry()
1322 target = eb_get_vma(eb, reloc->target_handle); in eb_relocate_entry()
1359 GRAPHICS_VER(eb->i915) == 6) { in eb_relocate_entry()
1378 ev->vma->size - (eb->reloc_cache.use_64bit_reloc ? 8 : 4))) { in eb_relocate_entry()
1405 return relocate_entry(ev->vma, reloc, eb, target->vma); in eb_relocate_entry()
1408 static int eb_relocate_vma(struct i915_execbuffer *eb, struct eb_vma *ev) in eb_relocate_vma() argument
1452 u64 offset = eb_relocate_entry(eb, ev, r); in eb_relocate_vma()
1488 reloc_cache_reset(&eb->reloc_cache, eb); in eb_relocate_vma()
1493 eb_relocate_vma_slow(struct i915_execbuffer *eb, struct eb_vma *ev) in eb_relocate_vma_slow() argument
1502 u64 offset = eb_relocate_entry(eb, ev, &relocs[i]); in eb_relocate_vma_slow()
1511 reloc_cache_reset(&eb->reloc_cache, eb); in eb_relocate_vma_slow()
1542 static int eb_copy_relocations(const struct i915_execbuffer *eb) in eb_copy_relocations() argument
1545 const unsigned int count = eb->buffer_count; in eb_copy_relocations()
1550 const unsigned int nreloc = eb->exec[i].relocation_count; in eb_copy_relocations()
1558 err = check_relocations(&eb->exec[i]); in eb_copy_relocations()
1562 urelocs = u64_to_user_ptr(eb->exec[i].relocs_ptr); in eb_copy_relocations()
1604 eb->exec[i].relocs_ptr = (uintptr_t)relocs; in eb_copy_relocations()
1616 relocs = u64_to_ptr(typeof(*relocs), eb->exec[i].relocs_ptr); in eb_copy_relocations()
1617 if (eb->exec[i].relocation_count) in eb_copy_relocations()
1623 static int eb_prefault_relocations(const struct i915_execbuffer *eb) in eb_prefault_relocations() argument
1625 const unsigned int count = eb->buffer_count; in eb_prefault_relocations()
1631 err = check_relocations(&eb->exec[i]); in eb_prefault_relocations()
1639 static int eb_reinit_userptr(struct i915_execbuffer *eb) in eb_reinit_userptr() argument
1641 const unsigned int count = eb->buffer_count; in eb_reinit_userptr()
1645 if (likely(!(eb->args->flags & __EXEC_USERPTR_USED))) in eb_reinit_userptr()
1649 struct eb_vma *ev = &eb->vma[i]; in eb_reinit_userptr()
1664 static noinline int eb_relocate_parse_slow(struct i915_execbuffer *eb) in eb_relocate_parse_slow() argument
1677 eb_release_vmas(eb, false); in eb_relocate_parse_slow()
1678 i915_gem_ww_ctx_fini(&eb->ww); in eb_relocate_parse_slow()
1694 err = eb_prefault_relocations(eb); in eb_relocate_parse_slow()
1696 err = eb_copy_relocations(eb); in eb_relocate_parse_slow()
1704 err = eb_reinit_userptr(eb); in eb_relocate_parse_slow()
1706 i915_gem_ww_ctx_init(&eb->ww, true); in eb_relocate_parse_slow()
1712 err = eb_pin_engine(eb, false); in eb_relocate_parse_slow()
1716 err = eb_validate_vmas(eb); in eb_relocate_parse_slow()
1720 GEM_BUG_ON(!eb->batches[0]); in eb_relocate_parse_slow()
1722 list_for_each_entry(ev, &eb->relocs, reloc_link) { in eb_relocate_parse_slow()
1724 err = eb_relocate_vma(eb, ev); in eb_relocate_parse_slow()
1728 err = eb_relocate_vma_slow(eb, ev); in eb_relocate_parse_slow()
1744 err = eb_parse(eb); in eb_relocate_parse_slow()
1757 eb_release_vmas(eb, false); in eb_relocate_parse_slow()
1758 err = i915_gem_ww_ctx_backoff(&eb->ww); in eb_relocate_parse_slow()
1768 const unsigned int count = eb->buffer_count; in eb_relocate_parse_slow()
1773 &eb->exec[i]; in eb_relocate_parse_slow()
1787 static int eb_relocate_parse(struct i915_execbuffer *eb) in eb_relocate_parse() argument
1793 err = eb_pin_engine(eb, throttle); in eb_relocate_parse()
1804 err = eb_validate_vmas(eb); in eb_relocate_parse()
1811 if (eb->args->flags & __EXEC_HAS_RELOC) { in eb_relocate_parse()
1814 list_for_each_entry(ev, &eb->relocs, reloc_link) { in eb_relocate_parse()
1815 err = eb_relocate_vma(eb, ev); in eb_relocate_parse()
1827 err = eb_parse(eb); in eb_relocate_parse()
1831 eb_release_vmas(eb, false); in eb_relocate_parse()
1832 err = i915_gem_ww_ctx_backoff(&eb->ww); in eb_relocate_parse()
1840 err = eb_relocate_parse_slow(eb); in eb_relocate_parse()
1849 eb->args->flags &= ~__EXEC_HAS_RELOC; in eb_relocate_parse()
1870 eb_find_first_request_added(struct i915_execbuffer *eb) in eb_find_first_request_added() argument
1874 for_each_batch_add_order(eb, i) in eb_find_first_request_added()
1875 if (eb->requests[i]) in eb_find_first_request_added()
1876 return eb->requests[i]; in eb_find_first_request_added()
1883 static int eb_move_to_gpu(struct i915_execbuffer *eb) in eb_move_to_gpu() argument
1885 const unsigned int count = eb->buffer_count; in eb_move_to_gpu()
1890 struct eb_vma *ev = &eb->vma[i]; in eb_move_to_gpu()
1900 for_each_batch_create_order(eb, j) { in eb_move_to_gpu()
1901 if (!eb->requests[j]) in eb_move_to_gpu()
1907 eb->requests[j]->capture_list; in eb_move_to_gpu()
1909 eb->requests[j]->capture_list = capture; in eb_move_to_gpu()
1945 (eb_find_first_request_added(eb), obj, in eb_move_to_gpu()
1949 for_each_batch_add_order(eb, j) { in eb_move_to_gpu()
1952 if (!eb->requests[j]) in eb_move_to_gpu()
1955 err = _i915_vma_move_to_active(vma, eb->requests[j], in eb_move_to_gpu()
1957 eb->composite_fence ? in eb_move_to_gpu()
1958 eb->composite_fence : in eb_move_to_gpu()
1959 &eb->requests[j]->fence, in eb_move_to_gpu()
1965 if (!err && (eb->args->flags & __EXEC_USERPTR_USED)) { in eb_move_to_gpu()
1966 read_lock(&eb->i915->mm.notifier_lock); in eb_move_to_gpu()
1973 struct eb_vma *ev = &eb->vma[i]; in eb_move_to_gpu()
1984 read_unlock(&eb->i915->mm.notifier_lock); in eb_move_to_gpu()
1992 intel_gt_chipset_flush(eb->gt); in eb_move_to_gpu()
1996 for_each_batch_create_order(eb, j) { in eb_move_to_gpu()
1997 if (!eb->requests[j]) in eb_move_to_gpu()
2000 i915_request_set_error_once(eb->requests[j], err); in eb_move_to_gpu()
2056 shadow_batch_pin(struct i915_execbuffer *eb, in shadow_batch_pin() argument
2068 err = i915_vma_pin_ww(vma, &eb->ww, 0, 0, flags); in shadow_batch_pin()
2075 static struct i915_vma *eb_dispatch_secure(struct i915_execbuffer *eb, struct i915_vma *vma) in eb_dispatch_secure() argument
2081 if (eb->batch_flags & I915_DISPATCH_SECURE) in eb_dispatch_secure()
2082 return i915_gem_object_ggtt_pin_ww(vma->obj, &eb->ww, NULL, 0, 0, 0); in eb_dispatch_secure()
2087 static int eb_parse(struct i915_execbuffer *eb) in eb_parse() argument
2089 struct drm_i915_private *i915 = eb->i915; in eb_parse()
2090 struct intel_gt_buffer_pool_node *pool = eb->batch_pool; in eb_parse()
2095 if (!eb_use_cmdparser(eb)) { in eb_parse()
2096 batch = eb_dispatch_secure(eb, eb->batches[0]->vma); in eb_parse()
2103 if (intel_context_is_parallel(eb->context)) in eb_parse()
2106 len = eb->batch_len[0]; in eb_parse()
2107 if (!CMDPARSER_USES_GGTT(eb->i915)) { in eb_parse()
2112 if (!eb->context->vm->has_read_only) { in eb_parse()
2120 if (unlikely(len < eb->batch_len[0])) /* last paranoid check of overflow */ in eb_parse()
2124 pool = intel_gt_get_buffer_pool(eb->gt, len, in eb_parse()
2128 eb->batch_pool = pool; in eb_parse()
2131 err = i915_gem_object_lock(pool->obj, &eb->ww); in eb_parse()
2135 shadow = shadow_batch_pin(eb, pool->obj, eb->context->vm, PIN_USER); in eb_parse()
2145 if (CMDPARSER_USES_GGTT(eb->i915)) { in eb_parse()
2148 shadow = shadow_batch_pin(eb, pool->obj, in eb_parse()
2149 &eb->gt->ggtt->vm, in eb_parse()
2158 eb->batch_flags |= I915_DISPATCH_SECURE; in eb_parse()
2161 batch = eb_dispatch_secure(eb, shadow); in eb_parse()
2171 err = intel_engine_cmd_parser(eb->context->engine, in eb_parse()
2172 eb->batches[0]->vma, in eb_parse()
2173 eb->batch_start_offset, in eb_parse()
2174 eb->batch_len[0], in eb_parse()
2179 eb->batches[0] = &eb->vma[eb->buffer_count++]; in eb_parse()
2180 eb->batches[0]->vma = i915_vma_get(shadow); in eb_parse()
2181 eb->batches[0]->flags = __EXEC_OBJECT_HAS_PIN; in eb_parse()
2183 eb->trampoline = trampoline; in eb_parse()
2184 eb->batch_start_offset = 0; in eb_parse()
2188 if (intel_context_is_parallel(eb->context)) in eb_parse()
2191 eb->batches[0] = &eb->vma[eb->buffer_count++]; in eb_parse()
2192 eb->batches[0]->flags = __EXEC_OBJECT_HAS_PIN; in eb_parse()
2193 eb->batches[0]->vma = i915_vma_get(batch); in eb_parse()
2209 static int eb_request_submit(struct i915_execbuffer *eb, in eb_request_submit() argument
2219 if (eb->args->flags & I915_EXEC_GEN7_SOL_RESET) { in eb_request_submit()
2239 eb->batch_start_offset, in eb_request_submit()
2241 eb->batch_flags); in eb_request_submit()
2245 if (eb->trampoline) { in eb_request_submit()
2247 GEM_BUG_ON(eb->batch_start_offset); in eb_request_submit()
2249 eb->trampoline->node.start + in eb_request_submit()
2258 static int eb_submit(struct i915_execbuffer *eb) in eb_submit() argument
2263 err = eb_move_to_gpu(eb); in eb_submit()
2265 for_each_batch_create_order(eb, i) { in eb_submit()
2266 if (!eb->requests[i]) in eb_submit()
2269 trace_i915_request_queue(eb->requests[i], eb->batch_flags); in eb_submit()
2271 err = eb_request_submit(eb, eb->requests[i], in eb_submit()
2272 eb->batches[i]->vma, in eb_submit()
2273 eb->batch_len[i]); in eb_submit()
2310 static struct i915_request *eb_throttle(struct i915_execbuffer *eb, struct intel_context *ce) in eb_throttle() argument
2344 static int eb_pin_timeline(struct i915_execbuffer *eb, struct intel_context *ce, in eb_pin_timeline() argument
2364 rq = eb_throttle(eb, ce); in eb_pin_timeline()
2368 bool nonblock = eb->file->filp->f_flags & O_NONBLOCK; in eb_pin_timeline()
2390 static int eb_pin_engine(struct i915_execbuffer *eb, bool throttle) in eb_pin_engine() argument
2392 struct intel_context *ce = eb->context, *child; in eb_pin_engine()
2396 GEM_BUG_ON(eb->args->flags & __EXEC_ENGINE_PINNED); in eb_pin_engine()
2406 err = intel_context_pin_ww(ce, &eb->ww); in eb_pin_engine()
2410 err = intel_context_pin_ww(child, &eb->ww); in eb_pin_engine()
2415 err = eb_pin_timeline(eb, child, throttle); in eb_pin_engine()
2420 err = eb_pin_timeline(eb, ce, throttle); in eb_pin_engine()
2424 eb->args->flags |= __EXEC_ENGINE_PINNED; in eb_pin_engine()
2441 static void eb_unpin_engine(struct i915_execbuffer *eb) in eb_unpin_engine() argument
2443 struct intel_context *ce = eb->context, *child; in eb_unpin_engine()
2445 if (!(eb->args->flags & __EXEC_ENGINE_PINNED)) in eb_unpin_engine()
2448 eb->args->flags &= ~__EXEC_ENGINE_PINNED; in eb_unpin_engine()
2466 eb_select_legacy_ring(struct i915_execbuffer *eb) in eb_select_legacy_ring() argument
2468 struct drm_i915_private *i915 = eb->i915; in eb_select_legacy_ring()
2469 struct drm_i915_gem_execbuffer2 *args = eb->args; in eb_select_legacy_ring()
2484 bsd_idx = gen8_dispatch_bsd_engine(i915, eb->file); in eb_select_legacy_ring()
2509 eb_select_engine(struct i915_execbuffer *eb) in eb_select_engine() argument
2515 if (i915_gem_context_user_engines(eb->gem_context)) in eb_select_engine()
2516 idx = eb->args->flags & I915_EXEC_RING_MASK; in eb_select_engine()
2518 idx = eb_select_legacy_ring(eb); in eb_select_engine()
2520 ce = i915_gem_context_get_engine(eb->gem_context, idx); in eb_select_engine()
2525 if (eb->buffer_count < ce->parallel.number_children + 1) { in eb_select_engine()
2529 if (eb->batch_start_offset || eb->args->batch_len) { in eb_select_engine()
2534 eb->num_batches = ce->parallel.number_children + 1; in eb_select_engine()
2561 eb->context = ce; in eb_select_engine()
2562 eb->gt = ce->engine->gt; in eb_select_engine()
2580 eb_put_engine(struct i915_execbuffer *eb) in eb_put_engine() argument
2584 intel_gt_pm_put(eb->gt); in eb_put_engine()
2585 for_each_child(eb->context, child) in eb_put_engine()
2587 intel_context_put(eb->context); in eb_put_engine()
2602 add_timeline_fence_array(struct i915_execbuffer *eb, in add_timeline_fence_array() argument
2619 SIZE_MAX / sizeof(*f)) - eb->num_fences) in add_timeline_fence_array()
2630 f = krealloc(eb->fences, in add_timeline_fence_array()
2631 (eb->num_fences + nfences) * sizeof(*f), in add_timeline_fence_array()
2636 eb->fences = f; in add_timeline_fence_array()
2637 f += eb->num_fences; in add_timeline_fence_array()
2659 syncobj = drm_syncobj_find(eb->file, user_fence.handle); in add_timeline_fence_array()
2724 eb->num_fences++; in add_timeline_fence_array()
2730 static int add_fence_array(struct i915_execbuffer *eb) in add_fence_array() argument
2732 struct drm_i915_gem_execbuffer2 *args = eb->args; in add_fence_array()
2747 SIZE_MAX / sizeof(*f) - eb->num_fences)) in add_fence_array()
2754 f = krealloc(eb->fences, in add_fence_array()
2755 (eb->num_fences + num_fences) * sizeof(*f), in add_fence_array()
2760 eb->fences = f; in add_fence_array()
2761 f += eb->num_fences; in add_fence_array()
2773 syncobj = drm_syncobj_find(eb->file, user_fence.handle); in add_fence_array()
2796 eb->num_fences++; in add_fence_array()
2809 await_fence_array(struct i915_execbuffer *eb, in await_fence_array() argument
2815 for (n = 0; n < eb->num_fences; n++) { in await_fence_array()
2819 syncobj = ptr_unpack_bits(eb->fences[n].syncobj, &flags, 2); in await_fence_array()
2821 if (!eb->fences[n].dma_fence) in await_fence_array()
2824 err = i915_request_await_dma_fence(rq, eb->fences[n].dma_fence); in await_fence_array()
2832 static void signal_fence_array(const struct i915_execbuffer *eb, in signal_fence_array() argument
2837 for (n = 0; n < eb->num_fences; n++) { in signal_fence_array()
2841 syncobj = ptr_unpack_bits(eb->fences[n].syncobj, &flags, 2); in signal_fence_array()
2845 if (eb->fences[n].chain_fence) { in signal_fence_array()
2847 eb->fences[n].chain_fence, in signal_fence_array()
2849 eb->fences[n].value); in signal_fence_array()
2854 eb->fences[n].chain_fence = NULL; in signal_fence_array()
2864 struct i915_execbuffer *eb = data; in parse_timeline_fences() local
2870 return add_timeline_fence_array(eb, &timeline_fences); in parse_timeline_fences()
2882 static int eb_request_add(struct i915_execbuffer *eb, struct i915_request *rq, in eb_request_add() argument
2897 if (likely(!intel_context_is_closed(eb->context))) { in eb_request_add()
2898 attr = eb->gem_context->sched; in eb_request_add()
2906 if (intel_context_is_parallel(eb->context)) { in eb_request_add()
2928 static int eb_requests_add(struct i915_execbuffer *eb, int err) in eb_requests_add() argument
2936 for_each_batch_add_order(eb, i) { in eb_requests_add()
2937 struct i915_request *rq = eb->requests[i]; in eb_requests_add()
2941 err |= eb_request_add(eb, rq, err, i == 0); in eb_requests_add()
2953 struct i915_execbuffer *eb) in parse_execbuf2_extensions() argument
2961 if (eb->args->flags & I915_EXEC_FENCE_ARRAY) in parse_execbuf2_extensions()
2970 eb); in parse_execbuf2_extensions()
2973 static void eb_requests_get(struct i915_execbuffer *eb) in eb_requests_get() argument
2977 for_each_batch_create_order(eb, i) { in eb_requests_get()
2978 if (!eb->requests[i]) in eb_requests_get()
2981 i915_request_get(eb->requests[i]); in eb_requests_get()
2985 static void eb_requests_put(struct i915_execbuffer *eb) in eb_requests_put() argument
2989 for_each_batch_create_order(eb, i) { in eb_requests_put()
2990 if (!eb->requests[i]) in eb_requests_put()
2993 i915_request_put(eb->requests[i]); in eb_requests_put()
2998 eb_composite_fence_create(struct i915_execbuffer *eb, int out_fence_fd) in eb_composite_fence_create() argument
3005 GEM_BUG_ON(!intel_context_is_parent(eb->context)); in eb_composite_fence_create()
3007 fences = kmalloc_array(eb->num_batches, sizeof(*fences), GFP_KERNEL); in eb_composite_fence_create()
3011 for_each_batch_create_order(eb, i) { in eb_composite_fence_create()
3012 fences[i] = &eb->requests[i]->fence; in eb_composite_fence_create()
3014 &eb->requests[i]->fence.flags); in eb_composite_fence_create()
3017 fence_array = dma_fence_array_create(eb->num_batches, in eb_composite_fence_create()
3019 eb->context->parallel.fence_context, in eb_composite_fence_create()
3020 eb->context->parallel.seqno++, in eb_composite_fence_create()
3028 for_each_batch_create_order(eb, i) in eb_composite_fence_create()
3039 eb->composite_fence = &fence_array->base; in eb_composite_fence_create()
3045 eb_fences_add(struct i915_execbuffer *eb, struct i915_request *rq, in eb_fences_add() argument
3051 if (unlikely(eb->gem_context->syncobj)) { in eb_fences_add()
3054 fence = drm_syncobj_fence_get(eb->gem_context->syncobj); in eb_fences_add()
3062 if (eb->args->flags & I915_EXEC_FENCE_SUBMIT) in eb_fences_add()
3070 if (eb->fences) { in eb_fences_add()
3071 err = await_fence_array(eb, rq); in eb_fences_add()
3076 if (intel_context_is_parallel(eb->context)) { in eb_fences_add()
3077 out_fence = eb_composite_fence_create(eb, out_fence_fd); in eb_fences_add()
3090 eb_find_context(struct i915_execbuffer *eb, unsigned int context_number) in eb_find_context() argument
3095 return eb->context; in eb_find_context()
3097 for_each_child(eb->context, child) in eb_find_context()
3107 eb_requests_create(struct i915_execbuffer *eb, struct dma_fence *in_fence, in eb_requests_create() argument
3113 for_each_batch_create_order(eb, i) { in eb_requests_create()
3115 eb->requests[i] = i915_request_create(eb_find_context(eb, i)); in eb_requests_create()
3116 if (IS_ERR(eb->requests[i])) { in eb_requests_create()
3117 out_fence = ERR_PTR(PTR_ERR(eb->requests[i])); in eb_requests_create()
3118 eb->requests[i] = NULL; in eb_requests_create()
3127 if (i + 1 == eb->num_batches) { in eb_requests_create()
3128 out_fence = eb_fences_add(eb, eb->requests[i], in eb_requests_create()
3141 eb->requests[i]->batch = eb->batches[i]->vma; in eb_requests_create()
3142 if (eb->batch_pool) { in eb_requests_create()
3143 GEM_BUG_ON(intel_context_is_parallel(eb->context)); in eb_requests_create()
3144 intel_gt_buffer_pool_mark_active(eb->batch_pool, in eb_requests_create()
3145 eb->requests[i]); in eb_requests_create()
3159 struct i915_execbuffer eb; in i915_gem_do_execbuffer() local
3169 eb.i915 = i915; in i915_gem_do_execbuffer()
3170 eb.file = file; in i915_gem_do_execbuffer()
3171 eb.args = args; in i915_gem_do_execbuffer()
3175 eb.exec = exec; in i915_gem_do_execbuffer()
3176 eb.vma = (struct eb_vma *)(exec + args->buffer_count + 1); in i915_gem_do_execbuffer()
3177 eb.vma[0].vma = NULL; in i915_gem_do_execbuffer()
3178 eb.batch_pool = NULL; in i915_gem_do_execbuffer()
3180 eb.invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS; in i915_gem_do_execbuffer()
3181 reloc_cache_init(&eb.reloc_cache, eb.i915); in i915_gem_do_execbuffer()
3183 eb.buffer_count = args->buffer_count; in i915_gem_do_execbuffer()
3184 eb.batch_start_offset = args->batch_start_offset; in i915_gem_do_execbuffer()
3185 eb.trampoline = NULL; in i915_gem_do_execbuffer()
3187 eb.fences = NULL; in i915_gem_do_execbuffer()
3188 eb.num_fences = 0; in i915_gem_do_execbuffer()
3190 memset(eb.requests, 0, sizeof(struct i915_request *) * in i915_gem_do_execbuffer()
3191 ARRAY_SIZE(eb.requests)); in i915_gem_do_execbuffer()
3192 eb.composite_fence = NULL; in i915_gem_do_execbuffer()
3194 eb.batch_flags = 0; in i915_gem_do_execbuffer()
3206 eb.batch_flags |= I915_DISPATCH_SECURE; in i915_gem_do_execbuffer()
3209 eb.batch_flags |= I915_DISPATCH_PINNED; in i915_gem_do_execbuffer()
3211 err = parse_execbuf2_extensions(args, &eb); in i915_gem_do_execbuffer()
3215 err = add_fence_array(&eb); in i915_gem_do_execbuffer()
3240 err = eb_create(&eb); in i915_gem_do_execbuffer()
3244 GEM_BUG_ON(!eb.lut_size); in i915_gem_do_execbuffer()
3246 err = eb_select_context(&eb); in i915_gem_do_execbuffer()
3250 err = eb_select_engine(&eb); in i915_gem_do_execbuffer()
3254 err = eb_lookup_vmas(&eb); in i915_gem_do_execbuffer()
3256 eb_release_vmas(&eb, true); in i915_gem_do_execbuffer()
3260 i915_gem_ww_ctx_init(&eb.ww, true); in i915_gem_do_execbuffer()
3262 err = eb_relocate_parse(&eb); in i915_gem_do_execbuffer()
3275 ww_acquire_done(&eb.ww.ctx); in i915_gem_do_execbuffer()
3277 out_fence = eb_requests_create(&eb, in_fence, out_fence_fd); in i915_gem_do_execbuffer()
3281 if (eb.requests[0]) in i915_gem_do_execbuffer()
3287 err = eb_submit(&eb); in i915_gem_do_execbuffer()
3290 eb_requests_get(&eb); in i915_gem_do_execbuffer()
3291 err = eb_requests_add(&eb, err); in i915_gem_do_execbuffer()
3293 if (eb.fences) in i915_gem_do_execbuffer()
3294 signal_fence_array(&eb, eb.composite_fence ? in i915_gem_do_execbuffer()
3295 eb.composite_fence : in i915_gem_do_execbuffer()
3296 &eb.requests[0]->fence); in i915_gem_do_execbuffer()
3309 if (unlikely(eb.gem_context->syncobj)) { in i915_gem_do_execbuffer()
3310 drm_syncobj_replace_fence(eb.gem_context->syncobj, in i915_gem_do_execbuffer()
3311 eb.composite_fence ? in i915_gem_do_execbuffer()
3312 eb.composite_fence : in i915_gem_do_execbuffer()
3313 &eb.requests[0]->fence); in i915_gem_do_execbuffer()
3316 if (!out_fence && eb.composite_fence) in i915_gem_do_execbuffer()
3317 dma_fence_put(eb.composite_fence); in i915_gem_do_execbuffer()
3319 eb_requests_put(&eb); in i915_gem_do_execbuffer()
3322 eb_release_vmas(&eb, true); in i915_gem_do_execbuffer()
3323 if (eb.trampoline) in i915_gem_do_execbuffer()
3324 i915_vma_unpin(eb.trampoline); in i915_gem_do_execbuffer()
3326 i915_gem_ww_ctx_fini(&eb.ww); in i915_gem_do_execbuffer()
3328 if (eb.batch_pool) in i915_gem_do_execbuffer()
3329 intel_gt_buffer_pool_put(eb.batch_pool); in i915_gem_do_execbuffer()
3331 eb_put_engine(&eb); in i915_gem_do_execbuffer()
3333 i915_gem_context_put(eb.gem_context); in i915_gem_do_execbuffer()
3335 eb_destroy(&eb); in i915_gem_do_execbuffer()
3342 put_fence_array(eb.fences, eb.num_fences); in i915_gem_do_execbuffer()