Lines Matching refs:subscriptions
87 mn_itree_is_invalidating(struct mmu_notifier_subscriptions *subscriptions) in mn_itree_is_invalidating() argument
89 lockdep_assert_held(&subscriptions->lock); in mn_itree_is_invalidating()
90 return subscriptions->invalidate_seq & 1; in mn_itree_is_invalidating()
94 mn_itree_inv_start_range(struct mmu_notifier_subscriptions *subscriptions, in mn_itree_inv_start_range() argument
101 spin_lock(&subscriptions->lock); in mn_itree_inv_start_range()
102 subscriptions->active_invalidate_ranges++; in mn_itree_inv_start_range()
103 node = interval_tree_iter_first(&subscriptions->itree, range->start, in mn_itree_inv_start_range()
106 subscriptions->invalidate_seq |= 1; in mn_itree_inv_start_range()
111 *seq = subscriptions->invalidate_seq; in mn_itree_inv_start_range()
112 spin_unlock(&subscriptions->lock); in mn_itree_inv_start_range()
129 static void mn_itree_inv_end(struct mmu_notifier_subscriptions *subscriptions) in mn_itree_inv_end() argument
134 spin_lock(&subscriptions->lock); in mn_itree_inv_end()
135 if (--subscriptions->active_invalidate_ranges || in mn_itree_inv_end()
136 !mn_itree_is_invalidating(subscriptions)) { in mn_itree_inv_end()
137 spin_unlock(&subscriptions->lock); in mn_itree_inv_end()
142 subscriptions->invalidate_seq++; in mn_itree_inv_end()
151 &subscriptions->deferred_list, in mn_itree_inv_end()
155 &subscriptions->itree); in mn_itree_inv_end()
158 &subscriptions->itree); in mn_itree_inv_end()
161 spin_unlock(&subscriptions->lock); in mn_itree_inv_end()
163 wake_up_all(&subscriptions->wq); in mn_itree_inv_end()
188 struct mmu_notifier_subscriptions *subscriptions = in mmu_interval_read_begin() local
232 spin_lock(&subscriptions->lock); in mmu_interval_read_begin()
235 is_invalidating = seq == subscriptions->invalidate_seq; in mmu_interval_read_begin()
236 spin_unlock(&subscriptions->lock); in mmu_interval_read_begin()
248 wait_event(subscriptions->wq, in mmu_interval_read_begin()
249 READ_ONCE(subscriptions->invalidate_seq) != seq); in mmu_interval_read_begin()
261 static void mn_itree_release(struct mmu_notifier_subscriptions *subscriptions, in mn_itree_release() argument
276 mn_itree_inv_start_range(subscriptions, &range, &cur_seq); in mn_itree_release()
284 mn_itree_inv_end(subscriptions); in mn_itree_release()
299 static void mn_hlist_release(struct mmu_notifier_subscriptions *subscriptions, in mn_hlist_release() argument
310 hlist_for_each_entry_rcu(subscription, &subscriptions->list, hlist, in mn_hlist_release()
321 spin_lock(&subscriptions->lock); in mn_hlist_release()
322 while (unlikely(!hlist_empty(&subscriptions->list))) { in mn_hlist_release()
323 subscription = hlist_entry(subscriptions->list.first, in mn_hlist_release()
333 spin_unlock(&subscriptions->lock); in mn_hlist_release()
350 struct mmu_notifier_subscriptions *subscriptions = in __mmu_notifier_release() local
353 if (subscriptions->has_itree) in __mmu_notifier_release()
354 mn_itree_release(subscriptions, mm); in __mmu_notifier_release()
356 if (!hlist_empty(&subscriptions->list)) in __mmu_notifier_release()
357 mn_hlist_release(subscriptions, mm); in __mmu_notifier_release()
444 static int mn_itree_invalidate(struct mmu_notifier_subscriptions *subscriptions, in mn_itree_invalidate() argument
451 mn_itree_inv_start_range(subscriptions, range, &cur_seq); in mn_itree_invalidate()
471 mn_itree_inv_end(subscriptions); in mn_itree_invalidate()
476 struct mmu_notifier_subscriptions *subscriptions, in mn_hlist_invalidate_range_start() argument
484 hlist_for_each_entry_rcu(subscription, &subscriptions->list, hlist, in mn_hlist_invalidate_range_start()
522 hlist_for_each_entry_rcu(subscription, &subscriptions->list, in mn_hlist_invalidate_range_start()
538 struct mmu_notifier_subscriptions *subscriptions = in __mmu_notifier_invalidate_range_start() local
542 if (subscriptions->has_itree) { in __mmu_notifier_invalidate_range_start()
543 ret = mn_itree_invalidate(subscriptions, range); in __mmu_notifier_invalidate_range_start()
547 if (!hlist_empty(&subscriptions->list)) in __mmu_notifier_invalidate_range_start()
548 return mn_hlist_invalidate_range_start(subscriptions, range); in __mmu_notifier_invalidate_range_start()
553 mn_hlist_invalidate_end(struct mmu_notifier_subscriptions *subscriptions, in mn_hlist_invalidate_end() argument
560 hlist_for_each_entry_rcu(subscription, &subscriptions->list, hlist, in mn_hlist_invalidate_end()
595 struct mmu_notifier_subscriptions *subscriptions = in __mmu_notifier_invalidate_range_end() local
599 if (subscriptions->has_itree) in __mmu_notifier_invalidate_range_end()
600 mn_itree_inv_end(subscriptions); in __mmu_notifier_invalidate_range_end()
602 if (!hlist_empty(&subscriptions->list)) in __mmu_notifier_invalidate_range_end()
603 mn_hlist_invalidate_end(subscriptions, range, only_end); in __mmu_notifier_invalidate_range_end()
632 struct mmu_notifier_subscriptions *subscriptions = NULL; in __mmu_notifier_register() local
644 subscriptions = kzalloc( in __mmu_notifier_register()
646 if (!subscriptions) in __mmu_notifier_register()
649 INIT_HLIST_HEAD(&subscriptions->list); in __mmu_notifier_register()
650 spin_lock_init(&subscriptions->lock); in __mmu_notifier_register()
651 subscriptions->invalidate_seq = 2; in __mmu_notifier_register()
652 subscriptions->itree = RB_ROOT_CACHED; in __mmu_notifier_register()
653 init_waitqueue_head(&subscriptions->wq); in __mmu_notifier_register()
654 INIT_HLIST_HEAD(&subscriptions->deferred_list); in __mmu_notifier_register()
677 if (subscriptions) in __mmu_notifier_register()
678 smp_store_release(&mm->notifier_subscriptions, subscriptions); in __mmu_notifier_register()
698 kfree(subscriptions); in __mmu_notifier_register()
915 struct mmu_notifier_subscriptions *subscriptions, unsigned long start, in __mmu_interval_notifier_insert() argument
951 spin_lock(&subscriptions->lock); in __mmu_interval_notifier_insert()
952 if (subscriptions->active_invalidate_ranges) { in __mmu_interval_notifier_insert()
953 if (mn_itree_is_invalidating(subscriptions)) in __mmu_interval_notifier_insert()
955 &subscriptions->deferred_list); in __mmu_interval_notifier_insert()
957 subscriptions->invalidate_seq |= 1; in __mmu_interval_notifier_insert()
959 &subscriptions->itree); in __mmu_interval_notifier_insert()
961 interval_sub->invalidate_seq = subscriptions->invalidate_seq; in __mmu_interval_notifier_insert()
963 WARN_ON(mn_itree_is_invalidating(subscriptions)); in __mmu_interval_notifier_insert()
971 subscriptions->invalidate_seq - 1; in __mmu_interval_notifier_insert()
973 &subscriptions->itree); in __mmu_interval_notifier_insert()
975 spin_unlock(&subscriptions->lock); in __mmu_interval_notifier_insert()
1000 struct mmu_notifier_subscriptions *subscriptions; in mmu_interval_notifier_insert() local
1005 subscriptions = smp_load_acquire(&mm->notifier_subscriptions); in mmu_interval_notifier_insert()
1006 if (!subscriptions || !subscriptions->has_itree) { in mmu_interval_notifier_insert()
1010 subscriptions = mm->notifier_subscriptions; in mmu_interval_notifier_insert()
1012 return __mmu_interval_notifier_insert(interval_sub, mm, subscriptions, in mmu_interval_notifier_insert()
1022 struct mmu_notifier_subscriptions *subscriptions = in mmu_interval_notifier_insert_locked() local
1028 if (!subscriptions || !subscriptions->has_itree) { in mmu_interval_notifier_insert_locked()
1032 subscriptions = mm->notifier_subscriptions; in mmu_interval_notifier_insert_locked()
1034 return __mmu_interval_notifier_insert(interval_sub, mm, subscriptions, in mmu_interval_notifier_insert_locked()
1052 struct mmu_notifier_subscriptions *subscriptions = in mmu_interval_notifier_remove() local
1058 spin_lock(&subscriptions->lock); in mmu_interval_notifier_remove()
1059 if (mn_itree_is_invalidating(subscriptions)) { in mmu_interval_notifier_remove()
1068 &subscriptions->deferred_list); in mmu_interval_notifier_remove()
1069 seq = subscriptions->invalidate_seq; in mmu_interval_notifier_remove()
1074 &subscriptions->itree); in mmu_interval_notifier_remove()
1076 spin_unlock(&subscriptions->lock); in mmu_interval_notifier_remove()
1085 wait_event(subscriptions->wq, in mmu_interval_notifier_remove()
1086 READ_ONCE(subscriptions->invalidate_seq) != seq); in mmu_interval_notifier_remove()