1/*
2 * Copyright (c) 2013-2022, Arm Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7#include <arch.h>
8#include <asm_macros.S>
9
10	.globl	flush_dcache_range
11	.globl	flush_dcache_to_popa_range
12	.globl	clean_dcache_range
13	.globl	inv_dcache_range
14	.globl	dcsw_op_louis
15	.globl	dcsw_op_all
16	.globl	dcsw_op_level1
17	.globl	dcsw_op_level2
18	.globl	dcsw_op_level3
19
20/*
21 * This macro can be used for implementing various data cache operations `op`
22 */
23.macro do_dcache_maintenance_by_mva op
24	/* Exit early if size is zero */
25	cbz	x1, exit_loop_\op
26	dcache_line_size x2, x3
27	add	x1, x0, x1
28	sub	x3, x2, #1
29	bic	x0, x0, x3
30loop_\op:
31	dc	\op, x0
32	add	x0, x0, x2
33	cmp	x0, x1
34	b.lo	loop_\op
35	dsb	sy
36exit_loop_\op:
37	ret
38.endm
39
40.macro check_plat_can_cmo
41#if CONDITIONAL_CMO
42	mov	x3, x30
43	mov	x2, x0
44	bl	plat_can_cmo
45	mov	x30, x3
46	cbnz	x0, 1f
47	ret
481:
49	mov	 x0, x2
50#endif
51.endm
52	/* ------------------------------------------
53	 * Clean+Invalidate from base address till
54	 * size. 'x0' = addr, 'x1' = size
55	 * ------------------------------------------
56	 */
57func flush_dcache_range
58	check_plat_can_cmo
59	do_dcache_maintenance_by_mva civac
60endfunc flush_dcache_range
61
62	/* ------------------------------------------
63	 * Clean from base address till size.
64	 * 'x0' = addr, 'x1' = size
65	 * ------------------------------------------
66	 */
67func clean_dcache_range
68	check_plat_can_cmo
69	do_dcache_maintenance_by_mva cvac
70endfunc clean_dcache_range
71
72	/* ------------------------------------------
73	 * Invalidate from base address till
74	 * size. 'x0' = addr, 'x1' = size
75	 * ------------------------------------------
76	 */
77func inv_dcache_range
78	check_plat_can_cmo
79	do_dcache_maintenance_by_mva ivac
80endfunc inv_dcache_range
81
82
83	/*
84	 * On implementations with FEAT_MTE2,
85	 * Root firmware must issue DC_CIGDPAPA instead of DC_CIPAPA ,
86	 * in order to additionally clean and invalidate Allocation Tags
87	 * associated with the affected locations.
88	 *
89	 * ------------------------------------------
90	 * Clean+Invalidate by PA to POPA
91	 * from base address till size.
92	 * 'x0' = addr, 'x1' = size
93	 * ------------------------------------------
94	 */
95func flush_dcache_to_popa_range
96	/* Exit early if size is zero */
97	cbz	x1, exit_loop_dc_cipapa
98	check_plat_can_cmo
99	dcache_line_size x2, x3
100	sub	x3, x2, #1
101	bic	x0, x0, x3
102	add	x1, x1, x0
103loop_dc_cipapa:
104	sys	#6, c7, c14, #1, x0 /* DC CIPAPA,<Xt> */
105	add	x0, x0, x2
106	cmp	x0, x1
107	b.lo	loop_dc_cipapa
108	dsb	osh
109exit_loop_dc_cipapa:
110	ret
111endfunc	flush_dcache_to_popa_range
112
113	/* ---------------------------------------------------------------
114	 * Data cache operations by set/way to the level specified
115	 *
116	 * The main function, do_dcsw_op requires:
117	 * x0: The operation type (0-2), as defined in arch.h
118	 * x3: The last cache level to operate on
119	 * x9: clidr_el1
120	 * x10: The cache level to begin operation from
121	 * and will carry out the operation on each data cache from level 0
122	 * to the level in x3 in sequence
123	 *
124	 * The dcsw_op macro sets up the x3 and x9 parameters based on
125	 * clidr_el1 cache information before invoking the main function
126	 * ---------------------------------------------------------------
127	 */
128
129	.macro	dcsw_op shift, fw, ls
130	mrs	x9, clidr_el1
131	ubfx	x3, x9, \shift, \fw
132	lsl	x3, x3, \ls
133	mov	x10, xzr
134	b	do_dcsw_op
135	.endm
136
137func do_dcsw_op
138	cbz	x3, exit
139	mrs	x12, ID_AA64MMFR2_EL1	// stash FEAT_CCIDX identifier in x12
140	ubfx	x12, x12, #ID_AA64MMFR2_EL1_CCIDX_SHIFT, #ID_AA64MMFR2_EL1_CCIDX_LENGTH
141	adr	x14, dcsw_loop_table	// compute inner loop address
142	add	x14, x14, x0, lsl #5	// inner loop is 8x32-bit instructions
143#if ENABLE_BTI
144	add	x14, x14, x0, lsl #2	// inner loop is + "bti j" instruction
145#endif
146	mov	x0, x9
147	mov	w8, #1
148loop1:
149	add	x2, x10, x10, lsr #1	// work out 3x current cache level
150	lsr	x1, x0, x2		// extract cache type bits from clidr
151	and	x1, x1, #7		// mask the bits for current cache only
152	cmp	x1, #2			// see what cache we have at this level
153	b.lo	level_done		// nothing to do if no cache or icache
154
155	msr	csselr_el1, x10		// select current cache level in csselr
156	isb				// isb to sych the new cssr&csidr
157	mrs	x1, ccsidr_el1		// read the new ccsidr
158	and	x2, x1, #7		// extract the length of the cache lines
159	add	x2, x2, #4		// add 4 (line length offset)
160
161	cbz	x12, 1f			// check for FEAT_CCIDX for Associativity
162	ubfx	x4, x1, #3, #21 	// x4 = associativity CCSIDR_EL1[23:3]
163	b 	2f
1641:
165	ubfx	x4, x1, #3, #10 	// x4 = associativity CCSIDR_EL1[12:3]
1662:
167	clz	w5, w4			// bit position of way size increment
168	lsl	w9, w4, w5		// w9 = aligned max way number
169	lsl	w16, w8, w5		// w16 = way number loop decrement
170	orr	w9, w10, w9		// w9 = combine way and cache number
171
172	cbz	x12, 3f			// check for FEAT_CCIDX for NumSets
173	ubfx	x6, x1, #32, #24	// x6 (w6) = numsets CCSIDR_EL1[55:32]
174					// ISA will not allow x->w ubfx
175	b	4f
1763:
177	ubfx	w6, w1, #13, #15	// w6 = numsets CCSIDR_EL1[27:13]
1784:
179	lsl	w17, w8, w2		// w17 = set number loop decrement
180	dsb	sy			// barrier before we start this level
181	br	x14			// jump to DC operation specific loop
182
183	.macro	dcsw_loop _op
184#if ENABLE_BTI
185	bti	j
186#endif
187loop2_\_op:
188	lsl	w7, w6, w2		// w7 = aligned max set number
189
190loop3_\_op:
191	orr	w11, w9, w7		// combine cache, way and set number
192	dc	\_op, x11
193	subs	w7, w7, w17		// decrement set number
194	b.hs	loop3_\_op
195
196	subs	x9, x9, x16		// decrement way number
197	b.hs	loop2_\_op
198
199	b	level_done
200	.endm
201
202level_done:
203	add	x10, x10, #2		// increment cache number
204	cmp	x3, x10
205	b.hi	loop1
206	msr	csselr_el1, xzr		// select cache level 0 in csselr
207	dsb	sy			// barrier to complete final cache operation
208	isb
209exit:
210	ret
211endfunc do_dcsw_op
212
213dcsw_loop_table:
214	dcsw_loop isw
215	dcsw_loop cisw
216	dcsw_loop csw
217
218
219func dcsw_op_louis
220	check_plat_can_cmo
221	dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
222endfunc dcsw_op_louis
223
224
225func dcsw_op_all
226	check_plat_can_cmo
227	dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
228endfunc dcsw_op_all
229
230	/* ---------------------------------------------------------------
231	 *  Helper macro for data cache operations by set/way for the
232	 *  level specified
233	 * ---------------------------------------------------------------
234	 */
235	.macro dcsw_op_level level
236	mrs	x9, clidr_el1
237	mov	x3, \level
238	sub	x10, x3, #2
239	b	do_dcsw_op
240	.endm
241
242	/* ---------------------------------------------------------------
243	 * Data cache operations by set/way for level 1 cache
244	 *
245	 * The main function, do_dcsw_op requires:
246	 * x0: The operation type (0-2), as defined in arch.h
247	 * ---------------------------------------------------------------
248	 */
249func dcsw_op_level1
250	check_plat_can_cmo
251	dcsw_op_level #(1 << LEVEL_SHIFT)
252endfunc dcsw_op_level1
253
254	/* ---------------------------------------------------------------
255	 * Data cache operations by set/way for level 2 cache
256	 *
257	 * The main function, do_dcsw_op requires:
258	 * x0: The operation type (0-2), as defined in arch.h
259	 * ---------------------------------------------------------------
260	 */
261func dcsw_op_level2
262	check_plat_can_cmo
263	dcsw_op_level #(2 << LEVEL_SHIFT)
264endfunc dcsw_op_level2
265
266	/* ---------------------------------------------------------------
267	 * Data cache operations by set/way for level 3 cache
268	 *
269	 * The main function, do_dcsw_op requires:
270	 * x0: The operation type (0-2), as defined in arch.h
271	 * ---------------------------------------------------------------
272	 */
273func dcsw_op_level3
274	check_plat_can_cmo
275	dcsw_op_level #(3 << LEVEL_SHIFT)
276endfunc dcsw_op_level3
277