1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _FUTEX_H
3 #define _FUTEX_H
4
5 #include <linux/futex.h>
6 #include <linux/sched/wake_q.h>
7
8 #ifdef CONFIG_PREEMPT_RT
9 #include <linux/rcuwait.h>
10 #endif
11
12 #include <asm/futex.h>
13
14 /*
15 * Futex flags used to encode options to functions and preserve them across
16 * restarts.
17 */
18 #ifdef CONFIG_MMU
19 # define FLAGS_SHARED 0x01
20 #else
21 /*
22 * NOMMU does not have per process address space. Let the compiler optimize
23 * code away.
24 */
25 # define FLAGS_SHARED 0x00
26 #endif
27 #define FLAGS_CLOCKRT 0x02
28 #define FLAGS_HAS_TIMEOUT 0x04
29
30 #ifdef CONFIG_HAVE_FUTEX_CMPXCHG
31 #define futex_cmpxchg_enabled 1
32 #else
33 extern int __read_mostly futex_cmpxchg_enabled;
34 #endif
35
36 #ifdef CONFIG_FAIL_FUTEX
37 extern bool should_fail_futex(bool fshared);
38 #else
should_fail_futex(bool fshared)39 static inline bool should_fail_futex(bool fshared)
40 {
41 return false;
42 }
43 #endif
44
45 /*
46 * Hash buckets are shared by all the futex_keys that hash to the same
47 * location. Each key may have multiple futex_q structures, one for each task
48 * waiting on a futex.
49 */
50 struct futex_hash_bucket {
51 atomic_t waiters;
52 spinlock_t lock;
53 struct plist_head chain;
54 } ____cacheline_aligned_in_smp;
55
56 /*
57 * Priority Inheritance state:
58 */
59 struct futex_pi_state {
60 /*
61 * list of 'owned' pi_state instances - these have to be
62 * cleaned up in do_exit() if the task exits prematurely:
63 */
64 struct list_head list;
65
66 /*
67 * The PI object:
68 */
69 struct rt_mutex_base pi_mutex;
70
71 struct task_struct *owner;
72 refcount_t refcount;
73
74 union futex_key key;
75 } __randomize_layout;
76
77 /**
78 * struct futex_q - The hashed futex queue entry, one per waiting task
79 * @list: priority-sorted list of tasks waiting on this futex
80 * @task: the task waiting on the futex
81 * @lock_ptr: the hash bucket lock
82 * @key: the key the futex is hashed on
83 * @pi_state: optional priority inheritance state
84 * @rt_waiter: rt_waiter storage for use with requeue_pi
85 * @requeue_pi_key: the requeue_pi target futex key
86 * @bitset: bitset for the optional bitmasked wakeup
87 * @requeue_state: State field for futex_requeue_pi()
88 * @requeue_wait: RCU wait for futex_requeue_pi() (RT only)
89 *
90 * We use this hashed waitqueue, instead of a normal wait_queue_entry_t, so
91 * we can wake only the relevant ones (hashed queues may be shared).
92 *
93 * A futex_q has a woken state, just like tasks have TASK_RUNNING.
94 * It is considered woken when plist_node_empty(&q->list) || q->lock_ptr == 0.
95 * The order of wakeup is always to make the first condition true, then
96 * the second.
97 *
98 * PI futexes are typically woken before they are removed from the hash list via
99 * the rt_mutex code. See futex_unqueue_pi().
100 */
101 struct futex_q {
102 struct plist_node list;
103
104 struct task_struct *task;
105 spinlock_t *lock_ptr;
106 union futex_key key;
107 struct futex_pi_state *pi_state;
108 struct rt_mutex_waiter *rt_waiter;
109 union futex_key *requeue_pi_key;
110 u32 bitset;
111 atomic_t requeue_state;
112 #ifdef CONFIG_PREEMPT_RT
113 struct rcuwait requeue_wait;
114 #endif
115 } __randomize_layout;
116
117 extern const struct futex_q futex_q_init;
118
119 enum futex_access {
120 FUTEX_READ,
121 FUTEX_WRITE
122 };
123
124 extern int get_futex_key(u32 __user *uaddr, bool fshared, union futex_key *key,
125 enum futex_access rw);
126
127 extern struct hrtimer_sleeper *
128 futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout,
129 int flags, u64 range_ns);
130
131 extern struct futex_hash_bucket *futex_hash(union futex_key *key);
132
133 /**
134 * futex_match - Check whether two futex keys are equal
135 * @key1: Pointer to key1
136 * @key2: Pointer to key2
137 *
138 * Return 1 if two futex_keys are equal, 0 otherwise.
139 */
futex_match(union futex_key * key1,union futex_key * key2)140 static inline int futex_match(union futex_key *key1, union futex_key *key2)
141 {
142 return (key1 && key2
143 && key1->both.word == key2->both.word
144 && key1->both.ptr == key2->both.ptr
145 && key1->both.offset == key2->both.offset);
146 }
147
148 extern int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
149 struct futex_q *q, struct futex_hash_bucket **hb);
150 extern void futex_wait_queue(struct futex_hash_bucket *hb, struct futex_q *q,
151 struct hrtimer_sleeper *timeout);
152 extern void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q);
153
154 extern int fault_in_user_writeable(u32 __user *uaddr);
155 extern int futex_cmpxchg_value_locked(u32 *curval, u32 __user *uaddr, u32 uval, u32 newval);
156 extern int futex_get_value_locked(u32 *dest, u32 __user *from);
157 extern struct futex_q *futex_top_waiter(struct futex_hash_bucket *hb, union futex_key *key);
158
159 extern void __futex_unqueue(struct futex_q *q);
160 extern void __futex_queue(struct futex_q *q, struct futex_hash_bucket *hb);
161 extern int futex_unqueue(struct futex_q *q);
162
163 /**
164 * futex_queue() - Enqueue the futex_q on the futex_hash_bucket
165 * @q: The futex_q to enqueue
166 * @hb: The destination hash bucket
167 *
168 * The hb->lock must be held by the caller, and is released here. A call to
169 * futex_queue() is typically paired with exactly one call to futex_unqueue(). The
170 * exceptions involve the PI related operations, which may use futex_unqueue_pi()
171 * or nothing if the unqueue is done as part of the wake process and the unqueue
172 * state is implicit in the state of woken task (see futex_wait_requeue_pi() for
173 * an example).
174 */
futex_queue(struct futex_q * q,struct futex_hash_bucket * hb)175 static inline void futex_queue(struct futex_q *q, struct futex_hash_bucket *hb)
176 __releases(&hb->lock)
177 {
178 __futex_queue(q, hb);
179 spin_unlock(&hb->lock);
180 }
181
182 extern void futex_unqueue_pi(struct futex_q *q);
183
184 extern void wait_for_owner_exiting(int ret, struct task_struct *exiting);
185
186 /*
187 * Reflects a new waiter being added to the waitqueue.
188 */
futex_hb_waiters_inc(struct futex_hash_bucket * hb)189 static inline void futex_hb_waiters_inc(struct futex_hash_bucket *hb)
190 {
191 #ifdef CONFIG_SMP
192 atomic_inc(&hb->waiters);
193 /*
194 * Full barrier (A), see the ordering comment above.
195 */
196 smp_mb__after_atomic();
197 #endif
198 }
199
200 /*
201 * Reflects a waiter being removed from the waitqueue by wakeup
202 * paths.
203 */
futex_hb_waiters_dec(struct futex_hash_bucket * hb)204 static inline void futex_hb_waiters_dec(struct futex_hash_bucket *hb)
205 {
206 #ifdef CONFIG_SMP
207 atomic_dec(&hb->waiters);
208 #endif
209 }
210
futex_hb_waiters_pending(struct futex_hash_bucket * hb)211 static inline int futex_hb_waiters_pending(struct futex_hash_bucket *hb)
212 {
213 #ifdef CONFIG_SMP
214 /*
215 * Full barrier (B), see the ordering comment above.
216 */
217 smp_mb();
218 return atomic_read(&hb->waiters);
219 #else
220 return 1;
221 #endif
222 }
223
224 extern struct futex_hash_bucket *futex_q_lock(struct futex_q *q);
225 extern void futex_q_unlock(struct futex_hash_bucket *hb);
226
227
228 extern int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb,
229 union futex_key *key,
230 struct futex_pi_state **ps,
231 struct task_struct *task,
232 struct task_struct **exiting,
233 int set_waiters);
234
235 extern int refill_pi_state_cache(void);
236 extern void get_pi_state(struct futex_pi_state *pi_state);
237 extern void put_pi_state(struct futex_pi_state *pi_state);
238 extern int fixup_pi_owner(u32 __user *uaddr, struct futex_q *q, int locked);
239
240 /*
241 * Express the locking dependencies for lockdep:
242 */
243 static inline void
double_lock_hb(struct futex_hash_bucket * hb1,struct futex_hash_bucket * hb2)244 double_lock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
245 {
246 if (hb1 > hb2)
247 swap(hb1, hb2);
248
249 spin_lock(&hb1->lock);
250 if (hb1 != hb2)
251 spin_lock_nested(&hb2->lock, SINGLE_DEPTH_NESTING);
252 }
253
254 static inline void
double_unlock_hb(struct futex_hash_bucket * hb1,struct futex_hash_bucket * hb2)255 double_unlock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
256 {
257 spin_unlock(&hb1->lock);
258 if (hb1 != hb2)
259 spin_unlock(&hb2->lock);
260 }
261
262 /* syscalls */
263
264 extern int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, u32
265 val, ktime_t *abs_time, u32 bitset, u32 __user
266 *uaddr2);
267
268 extern int futex_requeue(u32 __user *uaddr1, unsigned int flags,
269 u32 __user *uaddr2, int nr_wake, int nr_requeue,
270 u32 *cmpval, int requeue_pi);
271
272 extern int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,
273 ktime_t *abs_time, u32 bitset);
274
275 /**
276 * struct futex_vector - Auxiliary struct for futex_waitv()
277 * @w: Userspace provided data
278 * @q: Kernel side data
279 *
280 * Struct used to build an array with all data need for futex_waitv()
281 */
282 struct futex_vector {
283 struct futex_waitv w;
284 struct futex_q q;
285 };
286
287 extern int futex_wait_multiple(struct futex_vector *vs, unsigned int count,
288 struct hrtimer_sleeper *to);
289
290 extern int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset);
291
292 extern int futex_wake_op(u32 __user *uaddr1, unsigned int flags,
293 u32 __user *uaddr2, int nr_wake, int nr_wake2, int op);
294
295 extern int futex_unlock_pi(u32 __user *uaddr, unsigned int flags);
296
297 extern int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int trylock);
298
299 #endif /* _FUTEX_H */
300