1 /* Manage function descriptors. Generic version.
2 Copyright (C) 1999-2021 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, write to the Free
17 Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
18 02111-1307 USA. */
19
20 #include <libintl.h>
21 #include <unistd.h>
22 #include <string.h>
23 #include <sys/param.h>
24 #include <sys/mman.h>
25 #include <link.h>
26 #include <ldsodefs.h>
27 #include <elf/dynamic-link.h>
28 #include <dl-fptr.h>
29 #include <dl-unmap-segments.h>
30 #include <atomic.h>
31 #include <libc-pointer-arith.h>
32
33 #ifndef ELF_MACHINE_BOOT_FPTR_TABLE_LEN
34 /* ELF_MACHINE_BOOT_FPTR_TABLE_LEN should be greater than the number of
35 dynamic symbols in ld.so. */
36 # define ELF_MACHINE_BOOT_FPTR_TABLE_LEN 256
37 #endif
38
39 #ifndef ELF_MACHINE_LOAD_ADDRESS
40 # error "ELF_MACHINE_LOAD_ADDRESS is not defined."
41 #endif
42
43 #ifndef COMPARE_AND_SWAP
44 # define COMPARE_AND_SWAP(ptr, old, new) \
45 (catomic_compare_and_exchange_bool_acq (ptr, new, old) == 0)
46 #endif
47
ElfW(Addr)48 ElfW(Addr) _dl_boot_fptr_table [ELF_MACHINE_BOOT_FPTR_TABLE_LEN];
49
50 static struct local
51 {
52 struct fdesc_table *root;
53 struct fdesc *free_list;
54 unsigned int npages; /* # of pages to allocate */
55 /* the next to members MUST be consecutive! */
56 struct fdesc_table boot_table;
57 struct fdesc boot_fdescs[1024];
58 }
59 local =
60 {
61 #ifdef SHARED
62 /* Address of .boot_table is not known until runtime. */
63 .root = 0,
64 #else
65 .root = &local.boot_table,
66 #endif
67 .npages = 2,
68 .boot_table =
69 {
70 .len = sizeof (local.boot_fdescs) / sizeof (local.boot_fdescs[0]),
71 .first_unused = 0
72 }
73 };
74
75 /* Create a new fdesc table and return a pointer to the first fdesc
76 entry. The fdesc lock must have been acquired already. */
77
78 static struct fdesc_table *
new_fdesc_table(struct local * l,size_t * size)79 new_fdesc_table (struct local *l, size_t *size)
80 {
81 size_t old_npages = l->npages;
82 size_t new_npages = old_npages + old_npages;
83 struct fdesc_table *new_table;
84
85 /* If someone has just created a new table, we return NULL to tell
86 the caller to use the new table. */
87 if (! COMPARE_AND_SWAP (&l->npages, old_npages, new_npages))
88 return (struct fdesc_table *) NULL;
89
90 *size = old_npages * GLRO(dl_pagesize);
91 new_table = __mmap (NULL, *size,
92 PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0);
93 if (new_table == MAP_FAILED)
94 _dl_signal_error (errno, NULL, NULL,
95 N_("cannot map pages for fdesc table"));
96
97 new_table->len
98 = (*size - sizeof (*new_table)) / sizeof (struct fdesc);
99 new_table->first_unused = 1;
100 return new_table;
101 }
102
103 /* Must call _dl_fptr_init before using any other function. */
104 void
_dl_fptr_init(void)105 _dl_fptr_init (void)
106 {
107 struct local *l;
108
109 ELF_MACHINE_LOAD_ADDRESS (l, local);
110 l->root = &l->boot_table;
111 }
112
113 static ElfW(Addr)
make_fdesc(ElfW (Addr)ip,ElfW (Addr)gp)114 make_fdesc (ElfW(Addr) ip, ElfW(Addr) gp)
115 {
116 struct fdesc *fdesc = NULL;
117 struct fdesc_table *root;
118 unsigned int old;
119 struct local *l;
120
121 ELF_MACHINE_LOAD_ADDRESS (l, local);
122
123 retry:
124 root = l->root;
125 while (1)
126 {
127 old = root->first_unused;
128 if (old >= root->len)
129 break;
130 else if (COMPARE_AND_SWAP (&root->first_unused, old, old + 1))
131 {
132 fdesc = &root->fdesc[old];
133 goto install;
134 }
135 }
136
137 if (l->free_list)
138 {
139 /* Get it from free-list. */
140 do
141 {
142 fdesc = l->free_list;
143 if (fdesc == NULL)
144 goto retry;
145 }
146 while (! COMPARE_AND_SWAP ((ElfW(Addr) *) &l->free_list,
147 (ElfW(Addr)) fdesc, fdesc->ip));
148 }
149 else
150 {
151 /* Create a new fdesc table. */
152 size_t size;
153 struct fdesc_table *new_table = new_fdesc_table (l, &size);
154
155 if (new_table == NULL)
156 goto retry;
157
158 new_table->next = root;
159 if (! COMPARE_AND_SWAP ((ElfW(Addr) *) &l->root,
160 (ElfW(Addr)) root,
161 (ElfW(Addr)) new_table))
162 {
163 /* Someone has just installed a new table. Return NULL to
164 tell the caller to use the new table. */
165 __munmap (new_table, size);
166 goto retry;
167 }
168
169 /* Note that the first entry was reserved while allocating the
170 memory for the new page. */
171 fdesc = &new_table->fdesc[0];
172 }
173
174 install:
175 fdesc->gp = gp;
176 fdesc->ip = ip;
177
178 return (ElfW(Addr)) fdesc;
179 }
180
181
ElfW(Addr)182 static inline ElfW(Addr) * __attribute__ ((always_inline))
183 make_fptr_table (struct link_map *map)
184 {
185 const ElfW(Sym) *symtab = (const void *) D_PTR (map, l_info[DT_SYMTAB]);
186 const char *strtab = (const void *) D_PTR (map, l_info[DT_STRTAB]);
187 ElfW(Addr) *fptr_table;
188 size_t size;
189 size_t len;
190 const ElfW(Sym) *symtabend;
191
192 /* Determine the end of the dynamic symbol table using the hash. */
193 if (map->l_info[DT_HASH] != NULL)
194 symtabend = (symtab + ((Elf_Symndx *) D_PTR (map, l_info[DT_HASH]))[1]);
195 else
196 /* There is no direct way to determine the number of symbols in the
197 dynamic symbol table and no hash table is present. The ELF
198 binary is ill-formed but what shall we do? Use the beginning of
199 the string table which generally follows the symbol table. */
200 symtabend = (const ElfW(Sym) *) strtab;
201
202 len = (((char *) symtabend - (char *) symtab)
203 / map->l_info[DT_SYMENT]->d_un.d_val);
204 size = ALIGN_UP (len * sizeof (fptr_table[0]), GLRO(dl_pagesize));
205
206 /* We don't support systems without MAP_ANON. We avoid using malloc
207 because this might get called before malloc is setup. */
208 fptr_table = __mmap (NULL, size,
209 PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE,
210 -1, 0);
211 if (fptr_table == MAP_FAILED)
212 _dl_signal_error (errno, NULL, NULL,
213 N_("cannot map pages for fptr table"));
214
215 if (COMPARE_AND_SWAP ((ElfW(Addr) *) &map->l_mach.fptr_table,
216 (ElfW(Addr)) NULL, (ElfW(Addr)) fptr_table))
217 map->l_mach.fptr_table_len = len;
218 else
219 __munmap (fptr_table, len * sizeof (fptr_table[0]));
220
221 return map->l_mach.fptr_table;
222 }
223
224
225 ElfW(Addr)
_dl_make_fptr(struct link_map * map,const ElfW (Sym)* sym,ElfW (Addr)ip)226 _dl_make_fptr (struct link_map *map, const ElfW(Sym) *sym,
227 ElfW(Addr) ip)
228 {
229 ElfW(Addr) *ftab = map->l_mach.fptr_table;
230 const ElfW(Sym) *symtab;
231 Elf_Symndx symidx;
232 struct local *l;
233
234 if (__builtin_expect (ftab == NULL, 0))
235 ftab = make_fptr_table (map);
236
237 symtab = (const void *) D_PTR (map, l_info[DT_SYMTAB]);
238 symidx = sym - symtab;
239
240 if (symidx >= map->l_mach.fptr_table_len)
241 _dl_signal_error (0, NULL, NULL,
242 N_("internal error: symidx out of range of fptr table"));
243
244 while (ftab[symidx] == 0)
245 {
246 /* GOT has already been relocated in elf_get_dynamic_info -
247 don't try to relocate it again. */
248 ElfW(Addr) fdesc
249 = make_fdesc (ip, map->l_info[DT_PLTGOT]->d_un.d_ptr);
250
251 if (__builtin_expect (COMPARE_AND_SWAP (&ftab[symidx], (ElfW(Addr)) NULL,
252 fdesc), 1))
253 {
254 /* Noone has updated the entry and the new function
255 descriptor has been installed. */
256 #if 0
257 const char *strtab
258 = (const void *) D_PTR (map, l_info[DT_STRTAB]);
259
260 ELF_MACHINE_LOAD_ADDRESS (l, local);
261 if (l->root != &l->boot_table
262 || l->boot_table.first_unused > 20)
263 _dl_debug_printf ("created fdesc symbol `%s' at %lx\n",
264 strtab + sym->st_name, ftab[symidx]);
265 #endif
266 break;
267 }
268 else
269 {
270 /* We created a duplicated function descriptor. We put it on
271 free-list. */
272 struct fdesc *f = (struct fdesc *) fdesc;
273
274 ELF_MACHINE_LOAD_ADDRESS (l, local);
275
276 do
277 f->ip = (ElfW(Addr)) l->free_list;
278 while (! COMPARE_AND_SWAP ((ElfW(Addr) *) &l->free_list,
279 f->ip, fdesc));
280 }
281 }
282
283 return ftab[symidx];
284 }
285
286
287 void
_dl_unmap(struct link_map * map)288 _dl_unmap (struct link_map *map)
289 {
290 ElfW(Addr) *ftab = map->l_mach.fptr_table;
291 struct fdesc *head = NULL, *tail = NULL;
292 size_t i;
293
294 _dl_unmap_segments (map);
295
296 if (ftab == NULL)
297 return;
298
299 /* String together the fdesc structures that are being freed. */
300 for (i = 0; i < map->l_mach.fptr_table_len; ++i)
301 {
302 if (ftab[i])
303 {
304 *(struct fdesc **) ftab[i] = head;
305 head = (struct fdesc *) ftab[i];
306 if (tail == NULL)
307 tail = head;
308 }
309 }
310
311 /* Prepend the new list to the free_list: */
312 if (tail)
313 do
314 tail->ip = (ElfW(Addr)) local.free_list;
315 while (! COMPARE_AND_SWAP ((ElfW(Addr) *) &local.free_list,
316 tail->ip, (ElfW(Addr)) head));
317
318 __munmap (ftab, (map->l_mach.fptr_table_len
319 * sizeof (map->l_mach.fptr_table[0])));
320
321 map->l_mach.fptr_table = NULL;
322 }
323
324 extern ElfW(Addr) _dl_fixup (struct link_map *, ElfW(Word)) attribute_hidden;
325
326 static inline Elf32_Addr
elf_machine_resolve(void)327 elf_machine_resolve (void)
328 {
329 Elf32_Addr addr;
330
331 asm ("b,l 1f,%0\n"
332 " addil L'_dl_runtime_resolve - ($PIC_pcrel$0 - 1),%0\n"
333 "1: ldo R'_dl_runtime_resolve - ($PIC_pcrel$0 - 5)(%%r1),%0\n"
334 : "=r" (addr) : : "r1");
335
336 return addr;
337 }
338
339 static inline int
_dl_read_access_allowed(unsigned int * addr)340 _dl_read_access_allowed (unsigned int *addr)
341 {
342 int result;
343
344 asm ("proberi (%1),3,%0" : "=r" (result) : "r" (addr) : );
345
346 return result;
347 }
348
349 ElfW(Addr)
_dl_lookup_address(const void * address)350 _dl_lookup_address (const void *address)
351 {
352 ElfW(Addr) addr = (ElfW(Addr)) address;
353 ElfW(Word) reloc_arg;
354 volatile unsigned int *desc;
355 unsigned int *gptr;
356
357 /* Return ADDR if the least-significant two bits of ADDR are not consistent
358 with ADDR being a linker defined function pointer. The normal value for
359 a code address in a backtrace is 3. */
360 if (((unsigned int) addr & 3) != 2)
361 return addr;
362
363 /* Handle special case where ADDR points to page 0. */
364 if ((unsigned int) addr < 4096)
365 return addr;
366
367 /* Clear least-significant two bits from descriptor address. */
368 desc = (unsigned int *) ((unsigned int) addr & ~3);
369 if (!_dl_read_access_allowed (desc))
370 return addr;
371
372 /* First load the relocation offset. */
373 reloc_arg = (ElfW(Word)) desc[1];
374 atomic_full_barrier();
375
376 /* Then load first word of candidate descriptor. It should be a pointer
377 with word alignment and point to memory that can be read. */
378 gptr = (unsigned int *) desc[0];
379 if (((unsigned int) gptr & 3) != 0
380 || !_dl_read_access_allowed (gptr))
381 return addr;
382
383 /* See if descriptor requires resolution. The following trampoline is
384 used in each global offset table for function resolution:
385
386 ldw 0(r20),r21
387 bv r0(r21)
388 ldw 4(r20),r21
389 tramp: b,l .-12,r20
390 depwi 0,31,2,r20
391 .word _dl_runtime_resolve
392 .word "_dl_runtime_resolve ltp"
393 got: .word _DYNAMIC
394 .word "struct link map address" */
395 if (gptr[0] == 0xea9f1fdd /* b,l .-12,r20 */
396 && gptr[1] == 0xd6801c1e /* depwi 0,31,2,r20 */
397 && (ElfW(Addr)) gptr[2] == elf_machine_resolve ())
398 {
399 struct link_map *l = (struct link_map *) gptr[5];
400
401 /* If gp has been resolved, we need to hunt for relocation offset. */
402 if (!(reloc_arg & PA_GP_RELOC))
403 reloc_arg = _dl_fix_reloc_arg (addr, l);
404
405 _dl_fixup (l, reloc_arg);
406 }
407
408 return (ElfW(Addr)) desc[0];
409 }
410