1 /* Assembly macros for 64-bit PowerPC. 2 Copyright (C) 2002-2021 Free Software Foundation, Inc. 3 This file is part of the GNU C Library. 4 5 The GNU C Library is free software; you can redistribute it and/or 6 modify it under the terms of the GNU Lesser General Public 7 License as published by the Free Software Foundation; either 8 version 2.1 of the License, or (at your option) any later version. 9 10 The GNU C Library is distributed in the hope that it will be useful, 11 but WITHOUT ANY WARRANTY; without even the implied warranty of 12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 Lesser General Public License for more details. 14 15 You should have received a copy of the GNU Lesser General Public 16 License along with the GNU C Library; if not, see 17 <https://www.gnu.org/licenses/>. */ 18 19 #include <sysdeps/powerpc/sysdep.h> 20 #include <tls.h> 21 22 #ifdef __ASSEMBLER__ 23 24 /* Stack frame offsets. */ 25 #define FRAME_BACKCHAIN 0 26 #define FRAME_CR_SAVE 8 27 #define FRAME_LR_SAVE 16 28 #if _CALL_ELF != 2 29 #define FRAME_MIN_SIZE 112 30 #define FRAME_MIN_SIZE_PARM 112 31 #define FRAME_TOC_SAVE 40 32 #define FRAME_PARM_SAVE 48 33 #else 34 #define FRAME_MIN_SIZE 32 35 #define FRAME_MIN_SIZE_PARM 96 36 #define FRAME_TOC_SAVE 24 37 #define FRAME_PARM_SAVE 32 38 #endif 39 40 /* Support macros for CALL_MCOUNT. */ 41 .macro SAVE_ARG NARG 42 .if \NARG 43 SAVE_ARG \NARG-1 44 std 2+\NARG,-FRAME_MIN_SIZE_PARM+FRAME_PARM_SAVE-8+8*(\NARG)(1) 45 .endif 46 .endm 47 48 .macro REST_ARG NARG 49 .if \NARG 50 REST_ARG \NARG-1 51 ld 2+\NARG,FRAME_PARM_SAVE-8+8*(\NARG)(1) 52 .endif 53 .endm 54 55 .macro CFI_SAVE_ARG NARG 56 .if \NARG 57 CFI_SAVE_ARG \NARG-1 58 cfi_offset(2+\NARG,-FRAME_MIN_SIZE_PARM+FRAME_PARM_SAVE-8+8*(\NARG)) 59 .endif 60 .endm 61 62 .macro CFI_REST_ARG NARG 63 .if \NARG 64 CFI_REST_ARG \NARG-1 65 cfi_restore(2+\NARG) 66 .endif 67 .endm 68 69 /* If compiled for profiling, call `_mcount' at the start of each function. 70 see ppc-mcount.S for more details. */ 71 .macro CALL_MCOUNT NARG 72 #ifdef PROF 73 mflr r0 74 SAVE_ARG \NARG 75 std r0,FRAME_LR_SAVE(r1) 76 stdu r1,-FRAME_MIN_SIZE_PARM(r1) 77 cfi_adjust_cfa_offset(FRAME_MIN_SIZE_PARM) 78 cfi_offset(lr,FRAME_LR_SAVE) 79 CFI_SAVE_ARG \NARG 80 bl JUMPTARGET (_mcount) 81 #ifndef SHARED 82 nop 83 #endif 84 ld r0,FRAME_MIN_SIZE_PARM+FRAME_LR_SAVE(r1) 85 REST_ARG \NARG 86 mtlr r0 87 addi r1,r1,FRAME_MIN_SIZE_PARM 88 cfi_adjust_cfa_offset(-FRAME_MIN_SIZE_PARM) 89 cfi_restore(lr) 90 CFI_REST_ARG \NARG 91 #endif 92 .endm 93 94 #if _CALL_ELF != 2 95 96 /* Macro to prepare for calling via a function pointer. */ 97 .macro PPC64_LOAD_FUNCPTR PTR 98 ld r12,0(\PTR) 99 ld r2,8(\PTR) 100 mtctr r12 101 ld r11,16(\PTR) 102 .endm 103 104 #ifdef USE_PPC64_OVERLAPPING_OPD 105 # define OPD_ENT(name) .quad BODY_LABEL (name), .TOC.@tocbase 106 #else 107 # define OPD_ENT(name) .quad BODY_LABEL (name), .TOC.@tocbase, 0 108 #endif 109 110 #define ENTRY_1(name) \ 111 .type BODY_LABEL(name),@function; \ 112 .globl name; \ 113 .section ".opd","aw"; \ 114 .p2align 3;FUNC_LABEL(name): \ 115 OPD_ENT (name); \ 116 .previous 117 118 #define FUNC_LABEL(X) X 119 #define BODY_LABEL(X) .LY##X 120 #define ENTRY_2(name) \ 121 .type name,@function; \ 122 ENTRY_1(name) 123 #define END_2(name) \ 124 .size name,.-BODY_LABEL(name); \ 125 .size BODY_LABEL(name),.-BODY_LABEL(name) 126 #define LOCALENTRY(name) 127 128 #else /* _CALL_ELF == 2 */ 129 130 /* Macro to prepare for calling via a function pointer. */ 131 .macro PPC64_LOAD_FUNCPTR PTR 132 mr r12,\PTR 133 mtctr r12 134 .endm 135 136 #define FUNC_LABEL(X) X 137 #define BODY_LABEL(X) X 138 #define ENTRY_2(name) \ 139 .globl name; \ 140 .type name,@function 141 #define END_2(name) \ 142 .size name,.-name 143 #define LOCALENTRY(name) \ 144 1: addis r2,r12,.TOC.-1b@ha; \ 145 addi r2,r2,.TOC.-1b@l; \ 146 .localentry name,.-name 147 148 #endif /* _CALL_ELF */ 149 150 .macro NOPS NARG 151 .if \NARG 152 NOPS \NARG-1 153 nop 154 .endif 155 .endm 156 157 .macro ENTRY_3 name, alignp2=2, nopwords=0 158 .text 159 ENTRY_2(\name) 160 .p2align \alignp2 161 NOPS \nopwords 162 BODY_LABEL(\name): 163 .endm 164 165 /* Use ENTRY_TOCLESS for functions that make no use of r2 and 166 guarantee r2 is unchanged on exit. Any function that has @toc or 167 @got relocs uses r2. Functions that call other functions via the 168 PLT use r2. Use ENTRY for functions that may use or change r2. 169 The first argument is the function name. 170 The optional second argument specifies alignment of the function's 171 code, as the logarithm base two of the byte alignment. For 172 example, a value of four aligns to a sixteen byte boundary. 173 The optional third argument specifies the number of NOPs to emit 174 before the start of the function's code. */ 175 #ifndef PROF 176 #define ENTRY_TOCLESS(name, ...) \ 177 ENTRY_3 name, ## __VA_ARGS__; \ 178 cfi_startproc 179 180 #define ENTRY(name, ...) \ 181 ENTRY_TOCLESS(name, ## __VA_ARGS__); \ 182 LOCALENTRY(name) 183 #else 184 /* The call to _mcount is potentially via the plt, so profiling code 185 is never free of an r2 use. */ 186 #define ENTRY_TOCLESS(name, ...) \ 187 ENTRY_3 name, ## __VA_ARGS__; \ 188 cfi_startproc; \ 189 LOCALENTRY(name) 190 191 #define ENTRY(name, ...) \ 192 ENTRY_TOCLESS(name, ## __VA_ARGS__) 193 #endif 194 195 /* Local labels stripped out by the linker. */ 196 #undef L 197 #define L(x) .L##x 198 199 #define tostring(s) #s 200 #define stringify(s) tostring(s) 201 #define XGLUE(a,b) a##b 202 #define GLUE(a,b) XGLUE(a,b) 203 #define LT_LABEL(name) GLUE(.LT,name) 204 #define LT_LABELSUFFIX(name,suffix) GLUE(GLUE(.LT,name),suffix) 205 206 /* Support Traceback tables */ 207 #define TB_ASM 0x000c000000000000 208 #define TB_GLOBALLINK 0x0000800000000000 209 #define TB_IS_EPROL 0x0000400000000000 210 #define TB_HAS_TBOFF 0x0000200000000000 211 #define TB_INT_PROC 0x0000100000000000 212 #define TB_HAS_CTL 0x0000080000000000 213 #define TB_TOCLESS 0x0000040000000000 214 #define TB_FP_PRESENT 0x0000020000000000 215 #define TB_LOG_ABORT 0x0000010000000000 216 #define TB_INT_HANDL 0x0000008000000000 217 #define TB_NAME_PRESENT 0x0000004000000000 218 #define TB_USES_ALLOCA 0x0000002000000000 219 #define TB_SAVES_CR 0x0000000200000000 220 #define TB_SAVES_LR 0x0000000100000000 221 #define TB_STORES_BC 0x0000000080000000 222 #define TB_FIXUP 0x0000000040000000 223 #define TB_FP_SAVED(fprs) (((fprs) & 0x3f) << 24) 224 #define TB_GPR_SAVED(gprs) (((fprs) & 0x3f) << 16) 225 #define TB_FIXEDPARMS(parms) (((parms) & 0xff) << 8) 226 #define TB_FLOATPARMS(parms) (((parms) & 0x7f) << 1) 227 #define TB_PARMSONSTK 0x0000000000000001 228 229 #define PPC_HIGHER(v) (((v) >> 32) & 0xffff) 230 #define TB_DEFAULT TB_ASM | TB_HAS_TBOFF | TB_NAME_PRESENT 231 232 #define TRACEBACK(name) \ 233 LT_LABEL(name): ; \ 234 .long 0 ; \ 235 .quad TB_DEFAULT ; \ 236 .long LT_LABEL(name)-BODY_LABEL(name) ; \ 237 .short LT_LABELSUFFIX(name,_name_end)-LT_LABELSUFFIX(name,_name_start) ; \ 238 LT_LABELSUFFIX(name,_name_start): ;\ 239 .ascii stringify(name) ; \ 240 LT_LABELSUFFIX(name,_name_end): ; \ 241 .p2align 2 242 243 #define TRACEBACK_MASK(name,mask) \ 244 LT_LABEL(name): ; \ 245 .long 0 ; \ 246 .quad TB_DEFAULT | mask ; \ 247 .long LT_LABEL(name)-BODY_LABEL(name) ; \ 248 .short LT_LABELSUFFIX(name,_name_end)-LT_LABELSUFFIX(name,_name_start) ; \ 249 LT_LABELSUFFIX(name,_name_start): ;\ 250 .ascii stringify(name) ; \ 251 LT_LABELSUFFIX(name,_name_end): ; \ 252 .p2align 2 253 254 /* END generates Traceback tables */ 255 #undef END 256 #define END(name) \ 257 cfi_endproc; \ 258 TRACEBACK(name); \ 259 END_2(name) 260 261 /* This form supports more informative traceback tables */ 262 #define END_GEN_TB(name,mask) \ 263 cfi_endproc; \ 264 TRACEBACK_MASK(name,mask); \ 265 END_2(name) 266 267 /* We will allocate a new frame to save LR and the non-volatile register used to 268 read the TCB when checking for scv support on syscall code. We actually just 269 need the minimum frame size plus room for 1 reg (8 bytes). But the ABI 270 mandates stack frames should be aligned at 16 Bytes, so we end up allocating 271 a bit more space then what will actually be used. */ 272 #define SCV_FRAME_SIZE (FRAME_MIN_SIZE+16) 273 #define SCV_FRAME_NVOLREG_SAVE FRAME_MIN_SIZE 274 275 /* Allocate frame and save register */ 276 #define NVOLREG_SAVE \ 277 stdu r1,-SCV_FRAME_SIZE(r1); \ 278 cfi_adjust_cfa_offset(SCV_FRAME_SIZE); \ 279 std r31,SCV_FRAME_NVOLREG_SAVE(r1); \ 280 cfi_rel_offset(r31,SCV_FRAME_NVOLREG_SAVE); 281 282 /* Restore register and destroy frame */ 283 #define NVOLREG_RESTORE \ 284 ld r31,SCV_FRAME_NVOLREG_SAVE(r1); \ 285 cfi_restore(r31); \ 286 addi r1,r1,SCV_FRAME_SIZE; \ 287 cfi_adjust_cfa_offset(-SCV_FRAME_SIZE); 288 289 /* Check PPC_FEATURE2_SCV bit from hwcap2 in the TCB. If it is not set, scv is 290 not available, then go to JUMPFALSE (label given by the macro's caller). We 291 save the value we read from the TCB in a non-volatile register so we can 292 reuse it later when exiting from the syscall in PSEUDO_RET. Note that for 293 the static case we need an extra check to guarantee the thread pointer has 294 already been initialized, otherwise we may try to access an invalid address 295 if a syscall is called before the TLS has been setup. */ 296 .macro CHECK_SCV_SUPPORT REG JUMPFALSE 297 298 #ifndef SHARED 299 /* Check if thread pointer has already been setup. */ 300 cmpdi r13,0 301 beq \JUMPFALSE 302 #endif 303 304 /* Read PPC_FEATURE2_SCV from TCB and store it in REG */ 305 ld \REG,TCB_HWCAP(PT_THREAD_POINTER) 306 andis. \REG,\REG,PPC_FEATURE2_SCV>>16 307 308 beq \JUMPFALSE 309 .endm 310 311 #if !defined(USE_PPC_SCV) || IS_IN(rtld) 312 # define DO_CALL(syscall) \ 313 li r0,syscall; \ 314 DO_CALL_SC 315 #else 316 /* Before doing the syscall, check if we can use scv. scv is supported by P9 317 and later with Linux v5.9 and later. If so, use it. Otherwise, fallback to 318 sc. We use a non-volatile register to save hwcap2 from the TCB, so we need 319 to save its content beforehand. */ 320 # define DO_CALL(syscall) \ 321 li r0,syscall; \ 322 NVOLREG_SAVE; \ 323 CHECK_SCV_SUPPORT r31 0f; \ 324 DO_CALL_SCV; \ 325 b 1f; \ 326 0: DO_CALL_SC; \ 327 1: 328 #endif /* !defined(USE_PPC_SCV) || IS_IN(rtld) */ 329 330 /* DO_CALL_SC and DO_CALL_SCV expect the syscall number to be in r0. */ 331 #define DO_CALL_SC \ 332 sc 333 334 #define DO_CALL_SCV \ 335 mflr r9; \ 336 std r9,SCV_FRAME_SIZE+FRAME_LR_SAVE(r1); \ 337 cfi_rel_offset(lr,SCV_FRAME_SIZE+FRAME_LR_SAVE); \ 338 .machine "push"; \ 339 .machine "power9"; \ 340 scv 0; \ 341 .machine "pop"; \ 342 ld r9,SCV_FRAME_SIZE+FRAME_LR_SAVE(r1); \ 343 mtlr r9; \ 344 cfi_restore(lr); 345 346 /* ppc64 is always PIC */ 347 #undef JUMPTARGET 348 #define JUMPTARGET(name) FUNC_LABEL(name) 349 350 #define PSEUDO(name, syscall_name, args) \ 351 .section ".text"; \ 352 ENTRY (name); \ 353 DO_CALL (SYS_ify (syscall_name)) 354 355 #ifdef SHARED 356 #define TAIL_CALL_SYSCALL_ERROR \ 357 b JUMPTARGET (NOTOC (__syscall_error)) 358 #else 359 /* Static version might be linked into a large app with a toc exceeding 360 64k. We can't put a toc adjusting stub on a plain branch, so can't 361 tail call __syscall_error. */ 362 #define TAIL_CALL_SYSCALL_ERROR \ 363 .ifdef .Local_syscall_error; \ 364 b .Local_syscall_error; \ 365 .else; \ 366 .Local_syscall_error: \ 367 mflr 0; \ 368 std 0,FRAME_LR_SAVE(1); \ 369 stdu 1,-FRAME_MIN_SIZE(1); \ 370 cfi_adjust_cfa_offset(FRAME_MIN_SIZE); \ 371 cfi_offset(lr,FRAME_LR_SAVE); \ 372 bl JUMPTARGET(__syscall_error); \ 373 nop; \ 374 ld 0,FRAME_MIN_SIZE+FRAME_LR_SAVE(1); \ 375 addi 1,1,FRAME_MIN_SIZE; \ 376 cfi_adjust_cfa_offset(-FRAME_MIN_SIZE); \ 377 mtlr 0; \ 378 cfi_restore(lr); \ 379 blr; \ 380 .endif 381 #endif 382 383 #if !defined(USE_PPC_SCV) || IS_IN(rtld) 384 # define PSEUDO_RET \ 385 RET_SC; \ 386 TAIL_CALL_SYSCALL_ERROR 387 #else 388 /* This should only be called after a DO_CALL. In such cases, r31 contains the 389 value of PPC_FEATURE2_SCV read from hwcap2 by CHECK_SCV_SUPPORT. If it is 390 set, we know we have entered the kernel using scv, so handle the return code 391 accordingly. */ 392 # define PSEUDO_RET \ 393 cmpdi cr5,r31,0; \ 394 NVOLREG_RESTORE; \ 395 beq cr5,0f; \ 396 RET_SCV; \ 397 b 1f; \ 398 0: RET_SC; \ 399 1: TAIL_CALL_SYSCALL_ERROR 400 #endif /* !defined(USE_PPC_SCV) || IS_IN(rtld) */ 401 402 #define RET_SCV \ 403 li r9,-4095; \ 404 cmpld r3,r9; \ 405 bltlr+; \ 406 neg r3,r3; 407 408 #define RET_SC \ 409 bnslr+; 410 411 #define ret PSEUDO_RET 412 413 #undef PSEUDO_END 414 #define PSEUDO_END(name) \ 415 END (name) 416 417 #define PSEUDO_NOERRNO(name, syscall_name, args) \ 418 .section ".text"; \ 419 ENTRY (name); \ 420 DO_CALL (SYS_ify (syscall_name)) 421 422 #if !defined(USE_PPC_SCV) || IS_IN(rtld) 423 # define PSEUDO_RET_NOERRNO \ 424 blr 425 #else 426 /* This should only be called after a DO_CALL. */ 427 # define PSEUDO_RET_NOERRNO \ 428 NVOLREG_RESTORE; \ 429 blr 430 #endif /* !defined(USE_PPC_SCV) || IS_IN(rtld) */ 431 432 #define ret_NOERRNO PSEUDO_RET_NOERRNO 433 434 #undef PSEUDO_END_NOERRNO 435 #define PSEUDO_END_NOERRNO(name) \ 436 END (name) 437 438 #define PSEUDO_ERRVAL(name, syscall_name, args) \ 439 .section ".text"; \ 440 ENTRY (name); \ 441 DO_CALL (SYS_ify (syscall_name)) 442 443 #if !defined(USE_PPC_SCV) || IS_IN(rtld) 444 # define PSEUDO_RET_ERRVAL \ 445 blr 446 #else 447 /* This should only be called after a DO_CALL. */ 448 # define PSEUDO_RET_ERRVAL \ 449 NVOLREG_RESTORE; \ 450 blr 451 #endif /* !defined(USE_PPC_SCV) || IS_IN(rtld) */ 452 453 #define ret_ERRVAL PSEUDO_RET_ERRVAL 454 455 #undef PSEUDO_END_ERRVAL 456 #define PSEUDO_END_ERRVAL(name) \ 457 END (name) 458 459 #ifdef SHARED 460 # if IS_IN (rtld) 461 /* Inside ld.so we use the local alias to avoid runtime GOT 462 relocations. */ 463 # define __GLRO_DEF(var) \ 464 .LC__ ## var: \ 465 .tc _rtld_local_ro[TC],_rtld_local_ro 466 # else 467 # define __GLRO_DEF(var) \ 468 .LC__ ## var: \ 469 .tc _rtld_global_ro[TC],_rtld_global_ro 470 # endif 471 # define __GLRO(rOUT, var, offset) \ 472 ld rOUT,.LC__ ## var@toc(r2); \ 473 lwz rOUT,offset(rOUT) 474 #else 475 # define __GLRO_DEF(var) \ 476 .LC__ ## var: \ 477 .tc _ ## var[TC],_ ## var 478 # define __GLRO(rOUT, var, offset) \ 479 ld rOUT,.LC__ ## var@toc(r2); \ 480 lwz rOUT,0(rOUT) 481 #endif 482 483 #ifdef USE_PPC64_NOTOC 484 # define NOTOC(l) l@notoc 485 #else 486 # define NOTOC(l) l 487 #endif 488 489 #else /* !__ASSEMBLER__ */ 490 491 #if _CALL_ELF != 2 492 493 #define PPC64_LOAD_FUNCPTR(ptr) \ 494 "ld 12,0(" #ptr ")\n" \ 495 "ld 2,8(" #ptr ")\n" \ 496 "mtctr 12\n" \ 497 "ld 11,16(" #ptr ")" 498 499 #ifdef USE_PPC64_OVERLAPPING_OPD 500 # define OPD_ENT(name) ".quad " BODY_PREFIX #name ", .TOC.@tocbase" 501 #else 502 # define OPD_ENT(name) ".quad " BODY_PREFIX #name ", .TOC.@tocbase, 0" 503 #endif 504 505 #define ENTRY_1(name) \ 506 ".type " BODY_PREFIX #name ",@function\n" \ 507 ".globl " #name "\n" \ 508 ".pushsection \".opd\",\"aw\"\n" \ 509 ".p2align 3\n" \ 510 #name ":\n" \ 511 OPD_ENT (name) "\n" \ 512 ".popsection" 513 514 #define DOT_PREFIX "" 515 #define BODY_PREFIX ".LY" 516 #define ENTRY_2(name) \ 517 ".type " #name ",@function\n" \ 518 ENTRY_1(name) 519 #define END_2(name) \ 520 ".size " #name ",.-" BODY_PREFIX #name "\n" \ 521 ".size " BODY_PREFIX #name ",.-" BODY_PREFIX #name 522 #define LOCALENTRY(name) 523 524 #else /* _CALL_ELF */ 525 526 #define PPC64_LOAD_FUNCPTR(ptr) \ 527 "mr 12," #ptr "\n" \ 528 "mtctr 12" 529 530 #define DOT_PREFIX "" 531 #define BODY_PREFIX "" 532 #define ENTRY_2(name) \ 533 ".type " #name ",@function\n" \ 534 ".globl " #name 535 #define END_2(name) \ 536 ".size " #name ",.-" #name 537 #define LOCALENTRY(name) \ 538 "1: addis 2,12,.TOC.-1b@ha\n" \ 539 "addi 2,2,.TOC.-1b@l\n" \ 540 ".localentry " #name ",.-" #name 541 542 #endif /* _CALL_ELF */ 543 544 #endif /* __ASSEMBLER__ */ 545