1/* Vector optimized 32/64 bit S/390 version of memrchr.
2   Copyright (C) 2015-2021 Free Software Foundation, Inc.
3   This file is part of the GNU C Library.
4
5   The GNU C Library is free software; you can redistribute it and/or
6   modify it under the terms of the GNU Lesser General Public
7   License as published by the Free Software Foundation; either
8   version 2.1 of the License, or (at your option) any later version.
9
10   The GNU C Library is distributed in the hope that it will be useful,
11   but WITHOUT ANY WARRANTY; without even the implied warranty of
12   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13   Lesser General Public License for more details.
14
15   You should have received a copy of the GNU Lesser General Public
16   License along with the GNU C Library; if not, see
17   <https://www.gnu.org/licenses/>.  */
18
19#include <ifunc-memrchr.h>
20
21#if HAVE_MEMRCHR_Z13
22
23# include "sysdep.h"
24# include "asm-syntax.h"
25
26	.text
27
28/* void *memrchr (const void *s, int c, size_t n)
29   Scans memory for character c backwards
30   and returns pointer to first c.
31
32   Register usage:
33   -r0=tmp
34   -r1=tmp
35   -r2=s
36   -r3=c
37   -r4=n
38   -r5=s in loop
39
40   -v16=part of s
41   -v17=index of found c
42   -v18=c replicated
43   -v20=permute pattern
44*/
45ENTRY(MEMRCHR_Z13)
46	.machine "z13"
47	.machinemode "zarch_nohighgprs"
48
49# if !defined __s390x__
50	llgfr	%r4,%r4
51# endif /* !defined __s390x__ */
52	clgije	%r4,0,.Lnot_found
53
54	vlvgb	%v18,%r3,0	/* Generate vector which elements are all c.
55				   If c > 255, c will be truncated.  */
56	vrepb	%v18,%v18,0
57
58	llcr	%r3,%r3		/* char c_char = (char) c.  */
59
60	/* check byte n - 1.  */
61	llc	%r0,-1(%r4,%r2)
62	slgfi	%r4,1
63	clrje	%r0,%r3,.Lfound_end
64	jh	.Lnot_found	/* Return NULL if n is now 0.  */
65
66	larl	%r1,.Lpermute_mask /* Load permute mask.  */
67	vl	%v20,0(%r1)
68
69	/* check byte n - 2.  */
70	llc	%r0,-1(%r4,%r2)
71	slgfi	%r4,1
72	clrje	%r0,%r3,.Lfound_end
73	jh	.Lnot_found	/* Return NULL if n is now 0.  */
74
75	clgijhe	%r4,64,.Lloop64	/* If n >= 64 -> loop64.  */
76
77.Llt64:
78	/* Process n < 64 bytes.  */
79	clgijl	%r4,16,.Llt16	/* Jump away if n < 16.  */
80	aghi	%r4,-16
81	vl	%v16,0(%r4,%r2)
82	vfeebs	%v17,%v16,%v18
83	jno	.Lfound0
84	clgijl	%r4,16,.Llt16
85	aghi	%r4,-16
86	vl	%v16,0(%r4,%r2)
87	vfeebs	%v17,%v16,%v18
88	jno	.Lfound0
89	clgijl	%r4,16,.Llt16
90	aghi	%r4,-16
91	vl	%v16,0(%r4,%r2)
92	vfeebs	%v17,%v16,%v18
93	jno	.Lfound0
94.Llt16:
95	clgfi	%r4,0		/* if remaining bytes == 0, return NULL.  */
96	locghie	%r2,0
97	ber	%r14
98
99	aghi	%r4,-1		/* vll needs highest index.  */
100	vll	%v16,%r4,0(%r2)	/* Load remaining bytes.  */
101
102	/* Right-shift of v16 to mask bytes after highest index.  */
103	lhi	%r0,15
104	slr	%r0,%r4		/* Compute byte count for vector shift right.  */
105	sll	%r0,3		/* Convert to bit count.  */
106	vlvgb	%v17,%r0,7
107	vsrlb	%v16,%v16,%v17	/* Vector shift right by byte by number of bytes
108				   specified in bits 1-4 of byte 7 in v17.   */
109	j	.Lfound_permute
110
111.Lfound48:
112	aghi	%r4,16
113.Lfound32:
114	aghi	%r4,16
115.Lfound16:
116	aghi	%r4,16
117.Lfound0:
118	la	%r2,0(%r4,%r2)	/* Set pointer to start of v16.  */
119	lghi	%r4,15		/* Set highest index in v16 to last index.  */
120.Lfound_permute:
121	/* Search for a c in v16 in reversed byte order. v16 contains %r4 + 1
122	   bytes. If v16 was not fully loaded, the bytes are already
123	   right shifted, so that the bytes in v16 can simply be reversed.  */
124	vperm	%v16,%v16,%v16,%v20 /* Permute v16 to reversed order.  */
125	vfeeb	%v16,%v16,%v18	/* Find c in reversed v16.  */
126	vlgvb	%r1,%v16,7	/* Index of c or 16 if not found.  */
127
128	/* Return NULL if there is no c in loaded bytes. */
129	clrjh	%r1,%r4,.Lnot_found
130
131	slgr	%r4,%r1
132.Lfound_end:
133	la	%r2,0(%r4,%r2)	/* Return pointer to c.  */
134	br	%r14
135
136.Lnot_found:
137	lghi	%r2,0
138	br	%r14
139
140.Lpermute_mask:
141	.byte	0x0F,0x0E,0x0D,0x0C,0x0B,0x0A,0x09,0x08
142	.byte	0x07,0x06,0x05,0x04,0x03,0x02,0x01,0x00
143
144.Lloop64:
145	aghi	%r4,-64
146	vl	%v16,48(%r4,%r2) /* Load 16bytes of memory area.  */
147	vfeebs	%v17,%v16,%v18	/* Find c.  */
148	jno	.Lfound48	/* Jump away if c was found.  */
149	vl	%v16,32(%r4,%r2)
150	vfeebs	%v17,%v16,%v18
151	jno	.Lfound32
152	vl	%v16,16(%r4,%r2)
153	vfeebs	%v17,%v16,%v18
154	jno	.Lfound16
155	vl	%v16,0(%r4,%r2)
156	vfeebs	%v17,%v16,%v18
157	jno	.Lfound0
158
159	clgijhe	%r4,64,.Lloop64	/* If n >= 64 -> loop64.  */
160	j	.Llt64
161END(MEMRCHR_Z13)
162
163# if ! HAVE_MEMRCHR_IFUNC
164strong_alias (MEMRCHR_Z13, __memrchr)
165weak_alias (__memrchr, memrchr)
166# endif
167
168#endif /* HAVE_MEMRCHR_Z13  */
169