1 // Copyright (C) 2002-2016 Free Software Foundation, Inc.
2 //
3 // This file is part of GCC.
4 //
5 // GCC is free software; you can redistribute it and/or modify
6 // it under the terms of the GNU General Public License as published by
7 // the Free Software Foundation; either version 3, or (at your option)
8 // any later version.
9
10 // GCC is distributed in the hope that it will be useful,
11 // but WITHOUT ANY WARRANTY; without even the implied warranty of
12 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 // GNU General Public License for more details.
14
15 // Under Section 7 of GPL version 3, you are granted additional
16 // permissions described in the GCC Runtime Library Exception, version
17 // 3.1, as published by the Free Software Foundation.
18
19 // You should have received a copy of the GNU General Public License and
20 // a copy of the GCC Runtime Library Exception along with this program;
21 // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
22 // <http://www.gnu.org/licenses/>.
23
24 // Written by Mark Mitchell, CodeSourcery LLC, <mark@codesourcery.com>
25 // Thread support written by Jason Merrill, Red Hat Inc. <jason@redhat.com>
26
27 #include <bits/c++config.h>
28 #include <cxxabi.h>
29 #include <exception>
30 #include <new>
31 #include <ext/atomicity.h>
32 #include <ext/concurrence.h>
33 #if defined(__GTHREADS) && defined(__GTHREAD_HAS_COND) \
34 && (ATOMIC_INT_LOCK_FREE > 1) && defined(_GLIBCXX_HAVE_LINUX_FUTEX)
35 # include <climits>
36 # include <syscall.h>
37 # include <unistd.h>
38 # define _GLIBCXX_USE_FUTEX
39 # define _GLIBCXX_FUTEX_WAIT 0
40 # define _GLIBCXX_FUTEX_WAKE 1
41 #endif
42
43 // The IA64/generic ABI uses the first byte of the guard variable.
44 // The ARM EABI uses the least significant bit.
45
46 // Thread-safe static local initialization support.
47 #ifdef __GTHREADS
48 # ifndef _GLIBCXX_USE_FUTEX
49 namespace
50 {
51 // A single mutex controlling all static initializations.
52 static __gnu_cxx::__recursive_mutex* static_mutex;
53
54 typedef char fake_recursive_mutex[sizeof(__gnu_cxx::__recursive_mutex)]
55 __attribute__ ((aligned(__alignof__(__gnu_cxx::__recursive_mutex))));
56 fake_recursive_mutex fake_mutex;
57
init()58 static void init()
59 { static_mutex = new (&fake_mutex) __gnu_cxx::__recursive_mutex(); }
60
61 __gnu_cxx::__recursive_mutex&
get_static_mutex()62 get_static_mutex()
63 {
64 static __gthread_once_t once = __GTHREAD_ONCE_INIT;
65 __gthread_once(&once, init);
66 return *static_mutex;
67 }
68
69 // Simple wrapper for exception safety.
70 struct mutex_wrapper
71 {
72 bool unlock;
mutex_wrapper__anonc1bc370c0111::mutex_wrapper73 mutex_wrapper() : unlock(true)
74 { get_static_mutex().lock(); }
75
~mutex_wrapper__anonc1bc370c0111::mutex_wrapper76 ~mutex_wrapper()
77 {
78 if (unlock)
79 static_mutex->unlock();
80 }
81 };
82 }
83 # endif
84
85 # if defined(__GTHREAD_HAS_COND) && !defined(_GLIBCXX_USE_FUTEX)
86 namespace
87 {
88 // A single condition variable controlling all static initializations.
89 static __gnu_cxx::__cond* static_cond;
90
91 // using a fake type to avoid initializing a static class.
92 typedef char fake_cond_t[sizeof(__gnu_cxx::__cond)]
93 __attribute__ ((aligned(__alignof__(__gnu_cxx::__cond))));
94 fake_cond_t fake_cond;
95
init_static_cond()96 static void init_static_cond()
97 { static_cond = new (&fake_cond) __gnu_cxx::__cond(); }
98
99 __gnu_cxx::__cond&
get_static_cond()100 get_static_cond()
101 {
102 static __gthread_once_t once = __GTHREAD_ONCE_INIT;
103 __gthread_once(&once, init_static_cond);
104 return *static_cond;
105 }
106 }
107 # endif
108
109 # ifndef _GLIBCXX_GUARD_TEST_AND_ACQUIRE
110
111 // Test the guard variable with a memory load with
112 // acquire semantics.
113
114 inline bool
__test_and_acquire(__cxxabiv1::__guard * g)115 __test_and_acquire (__cxxabiv1::__guard *g)
116 {
117 unsigned char __c;
118 unsigned char *__p = reinterpret_cast<unsigned char *>(g);
119 __atomic_load (__p, &__c, __ATOMIC_ACQUIRE);
120 (void) __p;
121 return _GLIBCXX_GUARD_TEST(&__c);
122 }
123 # define _GLIBCXX_GUARD_TEST_AND_ACQUIRE(G) __test_and_acquire (G)
124 # endif
125
126 # ifndef _GLIBCXX_GUARD_SET_AND_RELEASE
127
128 // Set the guard variable to 1 with memory order release semantics.
129
130 inline void
__set_and_release(__cxxabiv1::__guard * g)131 __set_and_release (__cxxabiv1::__guard *g)
132 {
133 unsigned char *__p = reinterpret_cast<unsigned char *>(g);
134 unsigned char val = 1;
135 __atomic_store (__p, &val, __ATOMIC_RELEASE);
136 (void) __p;
137 }
138 # define _GLIBCXX_GUARD_SET_AND_RELEASE(G) __set_and_release (G)
139 # endif
140
141 #else /* !__GTHREADS */
142
143 # undef _GLIBCXX_GUARD_TEST_AND_ACQUIRE
144 # undef _GLIBCXX_GUARD_SET_AND_RELEASE
145 # define _GLIBCXX_GUARD_SET_AND_RELEASE(G) _GLIBCXX_GUARD_SET (G)
146
147 #endif /* __GTHREADS */
148
149 //
150 // Here are C++ run-time routines for guarded initialization of static
151 // variables. There are 4 scenarios under which these routines are called:
152 //
153 // 1. Threads not supported (__GTHREADS not defined)
154 // 2. Threads are supported but not enabled at run-time.
155 // 3. Threads enabled at run-time but __gthreads_* are not fully POSIX.
156 // 4. Threads enabled at run-time and __gthreads_* support all POSIX threads
157 // primitives we need here.
158 //
159 // The old code supported scenarios 1-3 but was broken since it used a global
160 // mutex for all threads and had the mutex locked during the whole duration of
161 // initialization of a guarded static variable. The following created a
162 // dead-lock with the old code.
163 //
164 // Thread 1 acquires the global mutex.
165 // Thread 1 starts initializing static variable.
166 // Thread 1 creates thread 2 during initialization.
167 // Thread 2 attempts to acquire mutex to initialize another variable.
168 // Thread 2 blocks since thread 1 is locking the mutex.
169 // Thread 1 waits for result from thread 2 and also blocks. A deadlock.
170 //
171 // The new code here can handle this situation and thus is more robust. However,
172 // we need to use the POSIX thread condition variable, which is not supported
173 // in all platforms, notably older versions of Microsoft Windows. The gthr*.h
174 // headers define a symbol __GTHREAD_HAS_COND for platforms that support POSIX
175 // like condition variables. For platforms that do not support condition
176 // variables, we need to fall back to the old code.
177
178 // If _GLIBCXX_USE_FUTEX, no global mutex or condition variable is used,
179 // only atomic operations are used together with futex syscall.
180 // Valid values of the first integer in guard are:
181 // 0 No thread encountered the guarded init
182 // yet or it has been aborted.
183 // _GLIBCXX_GUARD_BIT The guarded static var has been successfully
184 // initialized.
185 // _GLIBCXX_GUARD_PENDING_BIT The guarded static var is being initialized
186 // and no other thread is waiting for its
187 // initialization.
188 // (_GLIBCXX_GUARD_PENDING_BIT The guarded static var is being initialized
189 // | _GLIBCXX_GUARD_WAITING_BIT) and some other threads are waiting until
190 // it is initialized.
191
192 namespace __cxxabiv1
193 {
194 #ifdef _GLIBCXX_USE_FUTEX
195 namespace
196 {
__guard_test_bit(const int __byte,const int __val)197 static inline int __guard_test_bit (const int __byte, const int __val)
198 {
199 union { int __i; char __c[sizeof (int)]; } __u = { 0 };
200 __u.__c[__byte] = __val;
201 return __u.__i;
202 }
203 }
204 #endif
205
206 static inline int
init_in_progress_flag(__guard * g)207 init_in_progress_flag(__guard* g)
208 { return ((char *)g)[1]; }
209
210 static inline void
set_init_in_progress_flag(__guard * g,int v)211 set_init_in_progress_flag(__guard* g, int v)
212 { ((char *)g)[1] = v; }
213
214 static inline void
throw_recursive_init_exception()215 throw_recursive_init_exception()
216 {
217 #if __cpp_exceptions
218 throw __gnu_cxx::recursive_init_error();
219 #else
220 // Use __builtin_trap so we don't require abort().
221 __builtin_trap();
222 #endif
223 }
224
225 // acquire() is a helper function used to acquire guard if thread support is
226 // not compiled in or is compiled in but not enabled at run-time.
227 static int
acquire(__guard * g)228 acquire(__guard *g)
229 {
230 // Quit if the object is already initialized.
231 if (_GLIBCXX_GUARD_TEST(g))
232 return 0;
233
234 if (init_in_progress_flag(g))
235 throw_recursive_init_exception();
236
237 set_init_in_progress_flag(g, 1);
238 return 1;
239 }
240
241 extern "C"
__cxa_guard_acquire(__guard * g)242 int __cxa_guard_acquire (__guard *g)
243 {
244 #ifdef __GTHREADS
245 // If the target can reorder loads, we need to insert a read memory
246 // barrier so that accesses to the guarded variable happen after the
247 // guard test.
248 if (_GLIBCXX_GUARD_TEST_AND_ACQUIRE (g))
249 return 0;
250
251 # ifdef _GLIBCXX_USE_FUTEX
252 // If __atomic_* and futex syscall are supported, don't use any global
253 // mutex.
254 if (__gthread_active_p ())
255 {
256 int *gi = (int *) (void *) g;
257 const int guard_bit = _GLIBCXX_GUARD_BIT;
258 const int pending_bit = _GLIBCXX_GUARD_PENDING_BIT;
259 const int waiting_bit = _GLIBCXX_GUARD_WAITING_BIT;
260
261 while (1)
262 {
263 int expected(0);
264 if (__atomic_compare_exchange_n(gi, &expected, pending_bit, false,
265 __ATOMIC_ACQ_REL,
266 __ATOMIC_ACQUIRE))
267 {
268 // This thread should do the initialization.
269 return 1;
270 }
271
272 if (expected == guard_bit)
273 {
274 // Already initialized.
275 return 0;
276 }
277
278 if (expected == pending_bit)
279 {
280 // Use acquire here.
281 int newv = expected | waiting_bit;
282 if (!__atomic_compare_exchange_n(gi, &expected, newv, false,
283 __ATOMIC_ACQ_REL,
284 __ATOMIC_ACQUIRE))
285 {
286 if (expected == guard_bit)
287 {
288 // Make a thread that failed to set the
289 // waiting bit exit the function earlier,
290 // if it detects that another thread has
291 // successfully finished initialising.
292 return 0;
293 }
294 if (expected == 0)
295 continue;
296 }
297
298 expected = newv;
299 }
300
301 syscall (SYS_futex, gi, _GLIBCXX_FUTEX_WAIT, expected, 0);
302 }
303 }
304 # else
305 if (__gthread_active_p ())
306 {
307 mutex_wrapper mw;
308
309 while (1) // When this loop is executing, mutex is locked.
310 {
311 # ifdef __GTHREAD_HAS_COND
312 // The static is already initialized.
313 if (_GLIBCXX_GUARD_TEST(g))
314 return 0; // The mutex will be unlocked via wrapper
315
316 if (init_in_progress_flag(g))
317 {
318 // The guarded static is currently being initialized by
319 // another thread, so we release mutex and wait for the
320 // condition variable. We will lock the mutex again after
321 // this.
322 get_static_cond().wait_recursive(&get_static_mutex());
323 }
324 else
325 {
326 set_init_in_progress_flag(g, 1);
327 return 1; // The mutex will be unlocked via wrapper.
328 }
329 # else
330 // This provides compatibility with older systems not supporting
331 // POSIX like condition variables.
332 if (acquire(g))
333 {
334 mw.unlock = false;
335 return 1; // The mutex still locked.
336 }
337 return 0; // The mutex will be unlocked via wrapper.
338 # endif
339 }
340 }
341 # endif
342 #endif
343
344 return acquire (g);
345 }
346
347 extern "C"
__cxa_guard_abort(__guard * g)348 void __cxa_guard_abort (__guard *g) throw ()
349 {
350 #ifdef _GLIBCXX_USE_FUTEX
351 // If __atomic_* and futex syscall are supported, don't use any global
352 // mutex.
353 if (__gthread_active_p ())
354 {
355 int *gi = (int *) (void *) g;
356 const int waiting_bit = _GLIBCXX_GUARD_WAITING_BIT;
357 int old = __atomic_exchange_n (gi, 0, __ATOMIC_ACQ_REL);
358
359 if ((old & waiting_bit) != 0)
360 syscall (SYS_futex, gi, _GLIBCXX_FUTEX_WAKE, INT_MAX);
361 return;
362 }
363 #elif defined(__GTHREAD_HAS_COND)
364 if (__gthread_active_p())
365 {
366 mutex_wrapper mw;
367
368 set_init_in_progress_flag(g, 0);
369
370 // If we abort, we still need to wake up all other threads waiting for
371 // the condition variable.
372 get_static_cond().broadcast();
373 return;
374 }
375 #endif
376
377 set_init_in_progress_flag(g, 0);
378 #if defined(__GTHREADS) && !defined(__GTHREAD_HAS_COND)
379 // This provides compatibility with older systems not supporting POSIX like
380 // condition variables.
381 if (__gthread_active_p ())
382 static_mutex->unlock();
383 #endif
384 }
385
386 extern "C"
__cxa_guard_release(__guard * g)387 void __cxa_guard_release (__guard *g) throw ()
388 {
389 #ifdef _GLIBCXX_USE_FUTEX
390 // If __atomic_* and futex syscall are supported, don't use any global
391 // mutex.
392 if (__gthread_active_p ())
393 {
394 int *gi = (int *) (void *) g;
395 const int guard_bit = _GLIBCXX_GUARD_BIT;
396 const int waiting_bit = _GLIBCXX_GUARD_WAITING_BIT;
397 int old = __atomic_exchange_n (gi, guard_bit, __ATOMIC_ACQ_REL);
398
399 if ((old & waiting_bit) != 0)
400 syscall (SYS_futex, gi, _GLIBCXX_FUTEX_WAKE, INT_MAX);
401 return;
402 }
403 #elif defined(__GTHREAD_HAS_COND)
404 if (__gthread_active_p())
405 {
406 mutex_wrapper mw;
407
408 set_init_in_progress_flag(g, 0);
409 _GLIBCXX_GUARD_SET_AND_RELEASE(g);
410
411 get_static_cond().broadcast();
412 return;
413 }
414 #endif
415
416 set_init_in_progress_flag(g, 0);
417 _GLIBCXX_GUARD_SET_AND_RELEASE (g);
418
419 #if defined(__GTHREADS) && !defined(__GTHREAD_HAS_COND)
420 // This provides compatibility with older systems not supporting POSIX like
421 // condition variables.
422 if (__gthread_active_p())
423 static_mutex->unlock();
424 #endif
425 }
426 }
427