1 /* Common definition for memcpy, mempcpy and memmove implementation.
2 All versions must be listed in ifunc-impl-list.c.
3 Copyright (C) 2017-2021 Free Software Foundation, Inc.
4 This file is part of the GNU C Library.
5
6 The GNU C Library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public
8 License as published by the Free Software Foundation; either
9 version 2.1 of the License, or (at your option) any later version.
10
11 The GNU C Library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
15
16 You should have received a copy of the GNU Lesser General Public
17 License along with the GNU C Library; if not, see
18 <https://www.gnu.org/licenses/>. */
19
20 #include <init-arch.h>
21
22 extern __typeof (REDIRECT_NAME) OPTIMIZE (erms) attribute_hidden;
23 extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned)
24 attribute_hidden;
25 extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned_erms)
26 attribute_hidden;
27 extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden;
28 extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3_back) attribute_hidden;
29 extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned) attribute_hidden;
30 extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_erms)
31 attribute_hidden;
32 extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_rtm)
33 attribute_hidden;
34 extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_erms_rtm)
35 attribute_hidden;
36 extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned)
37 attribute_hidden;
38 extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned_erms)
39 attribute_hidden;
40 extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned)
41 attribute_hidden;
42 extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned_erms)
43 attribute_hidden;
44 extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_no_vzeroupper)
45 attribute_hidden;
46
47 static inline void *
IFUNC_SELECTOR(void)48 IFUNC_SELECTOR (void)
49 {
50 const struct cpu_features* cpu_features = __get_cpu_features ();
51
52 if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_ERMS)
53 || CPU_FEATURES_ARCH_P (cpu_features, Prefer_FSRM))
54 return OPTIMIZE (erms);
55
56 if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F)
57 && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512))
58 {
59 if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL))
60 {
61 if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
62 return OPTIMIZE (avx512_unaligned_erms);
63
64 return OPTIMIZE (avx512_unaligned);
65 }
66
67 return OPTIMIZE (avx512_no_vzeroupper);
68 }
69
70 if (CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
71 {
72 if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL))
73 {
74 if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
75 return OPTIMIZE (evex_unaligned_erms);
76
77 return OPTIMIZE (evex_unaligned);
78 }
79
80 if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
81 {
82 if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
83 return OPTIMIZE (avx_unaligned_erms_rtm);
84
85 return OPTIMIZE (avx_unaligned_rtm);
86 }
87
88 if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER))
89 {
90 if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
91 return OPTIMIZE (avx_unaligned_erms);
92
93 return OPTIMIZE (avx_unaligned);
94 }
95 }
96
97 if (!CPU_FEATURE_USABLE_P (cpu_features, SSSE3)
98 || CPU_FEATURES_ARCH_P (cpu_features, Fast_Unaligned_Copy))
99 {
100 if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
101 return OPTIMIZE (sse2_unaligned_erms);
102
103 return OPTIMIZE (sse2_unaligned);
104 }
105
106 if (CPU_FEATURES_ARCH_P (cpu_features, Fast_Copy_Backward))
107 return OPTIMIZE (ssse3_back);
108
109 return OPTIMIZE (ssse3);
110 }
111