Lines Matching refs:iovad
18 static bool iova_rcache_insert(struct iova_domain *iovad,
21 static unsigned long iova_rcache_get(struct iova_domain *iovad,
24 static void init_iova_rcaches(struct iova_domain *iovad);
25 static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad);
26 static void free_iova_rcaches(struct iova_domain *iovad);
27 static void fq_destroy_all_entries(struct iova_domain *iovad);
32 struct iova_domain *iovad; in iova_cpuhp_dead() local
34 iovad = hlist_entry_safe(node, struct iova_domain, cpuhp_dead); in iova_cpuhp_dead()
36 free_cpu_cached_iovas(cpu, iovad); in iova_cpuhp_dead()
40 static void free_global_cached_iovas(struct iova_domain *iovad);
48 init_iova_domain(struct iova_domain *iovad, unsigned long granule, in init_iova_domain() argument
58 spin_lock_init(&iovad->iova_rbtree_lock); in init_iova_domain()
59 iovad->rbroot = RB_ROOT; in init_iova_domain()
60 iovad->cached_node = &iovad->anchor.node; in init_iova_domain()
61 iovad->cached32_node = &iovad->anchor.node; in init_iova_domain()
62 iovad->granule = granule; in init_iova_domain()
63 iovad->start_pfn = start_pfn; in init_iova_domain()
64 iovad->dma_32bit_pfn = 1UL << (32 - iova_shift(iovad)); in init_iova_domain()
65 iovad->max32_alloc_size = iovad->dma_32bit_pfn; in init_iova_domain()
66 iovad->flush_cb = NULL; in init_iova_domain()
67 iovad->fq = NULL; in init_iova_domain()
68 iovad->anchor.pfn_lo = iovad->anchor.pfn_hi = IOVA_ANCHOR; in init_iova_domain()
69 rb_link_node(&iovad->anchor.node, NULL, &iovad->rbroot.rb_node); in init_iova_domain()
70 rb_insert_color(&iovad->anchor.node, &iovad->rbroot); in init_iova_domain()
71 cpuhp_state_add_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD, &iovad->cpuhp_dead); in init_iova_domain()
72 init_iova_rcaches(iovad); in init_iova_domain()
76 static bool has_iova_flush_queue(struct iova_domain *iovad) in has_iova_flush_queue() argument
78 return !!iovad->fq; in has_iova_flush_queue()
81 static void free_iova_flush_queue(struct iova_domain *iovad) in free_iova_flush_queue() argument
83 if (!has_iova_flush_queue(iovad)) in free_iova_flush_queue()
86 if (timer_pending(&iovad->fq_timer)) in free_iova_flush_queue()
87 del_timer(&iovad->fq_timer); in free_iova_flush_queue()
89 fq_destroy_all_entries(iovad); in free_iova_flush_queue()
91 free_percpu(iovad->fq); in free_iova_flush_queue()
93 iovad->fq = NULL; in free_iova_flush_queue()
94 iovad->flush_cb = NULL; in free_iova_flush_queue()
95 iovad->entry_dtor = NULL; in free_iova_flush_queue()
98 int init_iova_flush_queue(struct iova_domain *iovad, in init_iova_flush_queue() argument
104 atomic64_set(&iovad->fq_flush_start_cnt, 0); in init_iova_flush_queue()
105 atomic64_set(&iovad->fq_flush_finish_cnt, 0); in init_iova_flush_queue()
111 iovad->flush_cb = flush_cb; in init_iova_flush_queue()
112 iovad->entry_dtor = entry_dtor; in init_iova_flush_queue()
124 iovad->fq = queue; in init_iova_flush_queue()
126 timer_setup(&iovad->fq_timer, fq_flush_timeout, 0); in init_iova_flush_queue()
127 atomic_set(&iovad->fq_timer_on, 0); in init_iova_flush_queue()
133 __get_cached_rbnode(struct iova_domain *iovad, unsigned long limit_pfn) in __get_cached_rbnode() argument
135 if (limit_pfn <= iovad->dma_32bit_pfn) in __get_cached_rbnode()
136 return iovad->cached32_node; in __get_cached_rbnode()
138 return iovad->cached_node; in __get_cached_rbnode()
142 __cached_rbnode_insert_update(struct iova_domain *iovad, struct iova *new) in __cached_rbnode_insert_update() argument
144 if (new->pfn_hi < iovad->dma_32bit_pfn) in __cached_rbnode_insert_update()
145 iovad->cached32_node = &new->node; in __cached_rbnode_insert_update()
147 iovad->cached_node = &new->node; in __cached_rbnode_insert_update()
151 __cached_rbnode_delete_update(struct iova_domain *iovad, struct iova *free) in __cached_rbnode_delete_update() argument
155 cached_iova = to_iova(iovad->cached32_node); in __cached_rbnode_delete_update()
157 (free->pfn_hi < iovad->dma_32bit_pfn && in __cached_rbnode_delete_update()
159 iovad->cached32_node = rb_next(&free->node); in __cached_rbnode_delete_update()
160 iovad->max32_alloc_size = iovad->dma_32bit_pfn; in __cached_rbnode_delete_update()
163 cached_iova = to_iova(iovad->cached_node); in __cached_rbnode_delete_update()
165 iovad->cached_node = rb_next(&free->node); in __cached_rbnode_delete_update()
168 static struct rb_node *iova_find_limit(struct iova_domain *iovad, unsigned long limit_pfn) in iova_find_limit() argument
179 if (limit_pfn > iovad->dma_32bit_pfn) in iova_find_limit()
180 return &iovad->anchor.node; in iova_find_limit()
182 node = iovad->rbroot.rb_node; in iova_find_limit()
233 static int __alloc_and_insert_iova_range(struct iova_domain *iovad, in __alloc_and_insert_iova_range() argument
242 unsigned long high_pfn = limit_pfn, low_pfn = iovad->start_pfn; in __alloc_and_insert_iova_range()
248 spin_lock_irqsave(&iovad->iova_rbtree_lock, flags); in __alloc_and_insert_iova_range()
249 if (limit_pfn <= iovad->dma_32bit_pfn && in __alloc_and_insert_iova_range()
250 size >= iovad->max32_alloc_size) in __alloc_and_insert_iova_range()
253 curr = __get_cached_rbnode(iovad, limit_pfn); in __alloc_and_insert_iova_range()
267 if (low_pfn == iovad->start_pfn && retry_pfn < limit_pfn) { in __alloc_and_insert_iova_range()
270 curr = iova_find_limit(iovad, limit_pfn); in __alloc_and_insert_iova_range()
274 iovad->max32_alloc_size = size; in __alloc_and_insert_iova_range()
283 iova_insert_rbtree(&iovad->rbroot, new, prev); in __alloc_and_insert_iova_range()
284 __cached_rbnode_insert_update(iovad, new); in __alloc_and_insert_iova_range()
286 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); in __alloc_and_insert_iova_range()
290 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); in __alloc_and_insert_iova_range()
369 alloc_iova(struct iova_domain *iovad, unsigned long size, in alloc_iova() argument
380 ret = __alloc_and_insert_iova_range(iovad, size, limit_pfn + 1, in alloc_iova()
393 private_find_iova(struct iova_domain *iovad, unsigned long pfn) in private_find_iova() argument
395 struct rb_node *node = iovad->rbroot.rb_node; in private_find_iova()
397 assert_spin_locked(&iovad->iova_rbtree_lock); in private_find_iova()
413 static void remove_iova(struct iova_domain *iovad, struct iova *iova) in remove_iova() argument
415 assert_spin_locked(&iovad->iova_rbtree_lock); in remove_iova()
416 __cached_rbnode_delete_update(iovad, iova); in remove_iova()
417 rb_erase(&iova->node, &iovad->rbroot); in remove_iova()
427 struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn) in find_iova() argument
433 spin_lock_irqsave(&iovad->iova_rbtree_lock, flags); in find_iova()
434 iova = private_find_iova(iovad, pfn); in find_iova()
435 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); in find_iova()
447 __free_iova(struct iova_domain *iovad, struct iova *iova) in __free_iova() argument
451 spin_lock_irqsave(&iovad->iova_rbtree_lock, flags); in __free_iova()
452 remove_iova(iovad, iova); in __free_iova()
453 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); in __free_iova()
466 free_iova(struct iova_domain *iovad, unsigned long pfn) in free_iova() argument
471 spin_lock_irqsave(&iovad->iova_rbtree_lock, flags); in free_iova()
472 iova = private_find_iova(iovad, pfn); in free_iova()
474 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); in free_iova()
477 remove_iova(iovad, iova); in free_iova()
478 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); in free_iova()
494 alloc_iova_fast(struct iova_domain *iovad, unsigned long size, in alloc_iova_fast() argument
500 iova_pfn = iova_rcache_get(iovad, size, limit_pfn + 1); in alloc_iova_fast()
505 new_iova = alloc_iova(iovad, size, limit_pfn, true); in alloc_iova_fast()
515 free_cpu_cached_iovas(cpu, iovad); in alloc_iova_fast()
516 free_global_cached_iovas(iovad); in alloc_iova_fast()
533 free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size) in free_iova_fast() argument
535 if (iova_rcache_insert(iovad, pfn, size)) in free_iova_fast()
538 free_iova(iovad, pfn); in free_iova_fast()
562 static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq) in fq_ring_free() argument
564 u64 counter = atomic64_read(&iovad->fq_flush_finish_cnt); in fq_ring_free()
574 if (iovad->entry_dtor) in fq_ring_free()
575 iovad->entry_dtor(fq->entries[idx].data); in fq_ring_free()
577 free_iova_fast(iovad, in fq_ring_free()
585 static void iova_domain_flush(struct iova_domain *iovad) in iova_domain_flush() argument
587 atomic64_inc(&iovad->fq_flush_start_cnt); in iova_domain_flush()
588 iovad->flush_cb(iovad); in iova_domain_flush()
589 atomic64_inc(&iovad->fq_flush_finish_cnt); in iova_domain_flush()
592 static void fq_destroy_all_entries(struct iova_domain *iovad) in fq_destroy_all_entries() argument
601 if (!iovad->entry_dtor) in fq_destroy_all_entries()
605 struct iova_fq *fq = per_cpu_ptr(iovad->fq, cpu); in fq_destroy_all_entries()
609 iovad->entry_dtor(fq->entries[idx].data); in fq_destroy_all_entries()
615 struct iova_domain *iovad = from_timer(iovad, t, fq_timer); in fq_flush_timeout() local
618 atomic_set(&iovad->fq_timer_on, 0); in fq_flush_timeout()
619 iova_domain_flush(iovad); in fq_flush_timeout()
625 fq = per_cpu_ptr(iovad->fq, cpu); in fq_flush_timeout()
627 fq_ring_free(iovad, fq); in fq_flush_timeout()
632 void queue_iova(struct iova_domain *iovad, in queue_iova() argument
649 fq = raw_cpu_ptr(iovad->fq); in queue_iova()
657 fq_ring_free(iovad, fq); in queue_iova()
660 iova_domain_flush(iovad); in queue_iova()
661 fq_ring_free(iovad, fq); in queue_iova()
669 fq->entries[idx].counter = atomic64_read(&iovad->fq_flush_start_cnt); in queue_iova()
674 if (!atomic_read(&iovad->fq_timer_on) && in queue_iova()
675 !atomic_xchg(&iovad->fq_timer_on, 1)) in queue_iova()
676 mod_timer(&iovad->fq_timer, in queue_iova()
685 void put_iova_domain(struct iova_domain *iovad) in put_iova_domain() argument
690 &iovad->cpuhp_dead); in put_iova_domain()
692 free_iova_flush_queue(iovad); in put_iova_domain()
693 free_iova_rcaches(iovad); in put_iova_domain()
694 rbtree_postorder_for_each_entry_safe(iova, tmp, &iovad->rbroot, node) in put_iova_domain()
725 __insert_new_range(struct iova_domain *iovad, in __insert_new_range() argument
732 iova_insert_rbtree(&iovad->rbroot, iova, NULL); in __insert_new_range()
756 reserve_iova(struct iova_domain *iovad, in reserve_iova() argument
765 if (WARN_ON((pfn_hi | pfn_lo) > (ULLONG_MAX >> iova_shift(iovad)))) in reserve_iova()
768 spin_lock_irqsave(&iovad->iova_rbtree_lock, flags); in reserve_iova()
769 for (node = rb_first(&iovad->rbroot); node; node = rb_next(node)) { in reserve_iova()
785 iova = __insert_new_range(iovad, pfn_lo, pfn_hi); in reserve_iova()
788 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); in reserve_iova()
825 iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad) in iova_magazine_free_pfns() argument
833 spin_lock_irqsave(&iovad->iova_rbtree_lock, flags); in iova_magazine_free_pfns()
836 struct iova *iova = private_find_iova(iovad, mag->pfns[i]); in iova_magazine_free_pfns()
841 remove_iova(iovad, iova); in iova_magazine_free_pfns()
845 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); in iova_magazine_free_pfns()
887 static void init_iova_rcaches(struct iova_domain *iovad) in init_iova_rcaches() argument
895 rcache = &iovad->rcaches[i]; in init_iova_rcaches()
916 static bool __iova_rcache_insert(struct iova_domain *iovad, in __iova_rcache_insert() argument
957 iova_magazine_free_pfns(mag_to_free, iovad); in __iova_rcache_insert()
964 static bool iova_rcache_insert(struct iova_domain *iovad, unsigned long pfn, in iova_rcache_insert() argument
972 return __iova_rcache_insert(iovad, &iovad->rcaches[log_size], pfn); in iova_rcache_insert()
1019 static unsigned long iova_rcache_get(struct iova_domain *iovad, in iova_rcache_get() argument
1028 return __iova_rcache_get(&iovad->rcaches[log_size], limit_pfn - size); in iova_rcache_get()
1034 static void free_iova_rcaches(struct iova_domain *iovad) in free_iova_rcaches() argument
1042 rcache = &iovad->rcaches[i]; in free_iova_rcaches()
1057 static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad) in free_cpu_cached_iovas() argument
1065 rcache = &iovad->rcaches[i]; in free_cpu_cached_iovas()
1068 iova_magazine_free_pfns(cpu_rcache->loaded, iovad); in free_cpu_cached_iovas()
1069 iova_magazine_free_pfns(cpu_rcache->prev, iovad); in free_cpu_cached_iovas()
1077 static void free_global_cached_iovas(struct iova_domain *iovad) in free_global_cached_iovas() argument
1084 rcache = &iovad->rcaches[i]; in free_global_cached_iovas()
1087 iova_magazine_free_pfns(rcache->depot[j], iovad); in free_global_cached_iovas()