1/*
2 * Copyright (c) 2016-2021, Arm Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7#include <arch.h>
8#include <asm_macros.S>
9
10	.globl	flush_dcache_range
11	.globl	clean_dcache_range
12	.globl	inv_dcache_range
13	.globl	dcsw_op_louis
14	.globl	dcsw_op_all
15	.globl	dcsw_op_level1
16	.globl	dcsw_op_level2
17	.globl	dcsw_op_level3
18
19/*
20 * This macro can be used for implementing various data cache operations `op`
21 */
22.macro do_dcache_maintenance_by_mva op, coproc, opc1, CRn, CRm, opc2
23	/* Exit early if size is zero */
24	cmp	r1, #0
25	beq	exit_loop_\op
26	dcache_line_size r2, r3
27	add	r1, r0, r1
28	sub	r3, r2, #1
29	bic	r0, r0, r3
30loop_\op:
31	stcopr	r0, \coproc, \opc1, \CRn, \CRm, \opc2
32	add	r0, r0, r2
33	cmp	r0, r1
34	blo	loop_\op
35	dsb	sy
36exit_loop_\op:
37	bx	lr
38.endm
39
40	/* ------------------------------------------
41	 * Clean+Invalidate from base address till
42	 * size. 'r0' = addr, 'r1' = size
43	 * ------------------------------------------
44	 */
45func flush_dcache_range
46	do_dcache_maintenance_by_mva cimvac, DCCIMVAC
47endfunc flush_dcache_range
48
49	/* ------------------------------------------
50	 * Clean from base address till size.
51	 * 'r0' = addr, 'r1' = size
52	 * ------------------------------------------
53	 */
54func clean_dcache_range
55	do_dcache_maintenance_by_mva cmvac, DCCMVAC
56endfunc clean_dcache_range
57
58	/* ------------------------------------------
59	 * Invalidate from base address till
60	 * size. 'r0' = addr, 'r1' = size
61	 * ------------------------------------------
62	 */
63func inv_dcache_range
64	do_dcache_maintenance_by_mva imvac, DCIMVAC
65endfunc inv_dcache_range
66
67	/* ----------------------------------------------------------------
68	 * Data cache operations by set/way to the level specified
69	 *
70	 * The main function, do_dcsw_op requires:
71	 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW),
72	 * as defined in arch.h
73	 * r1: The cache level to begin operation from
74	 * r2: clidr_el1
75	 * r3: The last cache level to operate on
76	 * and will carry out the operation on each data cache from level 0
77	 * to the level in r3 in sequence
78	 *
79	 * The dcsw_op macro sets up the r2 and r3 parameters based on
80	 * clidr_el1 cache information before invoking the main function
81	 * ----------------------------------------------------------------
82	 */
83
84	.macro	dcsw_op shift, fw, ls
85	ldcopr	r2, CLIDR
86	ubfx	r3, r2, \shift, \fw
87	lsl	r3, r3, \ls
88	mov	r1, #0
89	b	do_dcsw_op
90	.endm
91
92func do_dcsw_op
93	push	{r4-r12, lr}
94	ldcopr	r8, ID_MMFR4		// stash FEAT_CCIDX identifier in r8
95	ubfx	r8, r8, #ID_MMFR4_CCIDX_SHIFT, #ID_MMFR4_CCIDX_LENGTH
96	adr	r11, dcsw_loop_table	// compute cache op based on the operation type
97	add	r6, r11, r0, lsl #3	// cache op is 2x32-bit instructions
98loop1:
99	add	r10, r1, r1, LSR #1	// Work out 3x current cache level
100	mov	r12, r2, LSR r10	// extract cache type bits from clidr
101	and	r12, r12, #7		// mask the bits for current cache only
102	cmp	r12, #2			// see what cache we have at this level
103	blo	level_done		// no cache or only instruction cache at this level
104
105	stcopr	r1, CSSELR		// select current cache level in csselr
106	isb				// isb to sych the new cssr&csidr
107	ldcopr	r12, CCSIDR		// read the new ccsidr
108	and	r10, r12, #7		// extract the length of the cache lines
109	add	r10, r10, #4		// add 4 (r10 = line length offset)
110
111	cmp	r8, #0			// check for FEAT_CCIDX for Associativity
112	beq	1f
113	ubfx	r4, r12, #3, #21 	// r4 = associativity CCSIDR[23:3]
114	b	2f
1151:
116	ubfx	r4, r12, #3, #10 	// r4 = associativity CCSIDR[12:3]
1172:
118	clz	r5, r4			// r5 = the bit position of the way size increment
119	mov	r9, r4			// r9 working copy of the aligned max way number
120
121loop2:
122	cmp	r8, #0			// check for FEAT_CCIDX for NumSets
123	beq	3f
124	ldcopr	r12, CCSIDR2		// FEAT_CCIDX numsets is in CCSIDR2
125	ubfx	r7, r12, #0, #24	// r7 = numsets CCSIDR2[23:0]
126	b	loop3
1273:
128	ubfx	r7, r12, #13, #15	// r7 = numsets CCSIDR[27:13]
129loop3:
130	orr	r0, r1, r9, LSL r5	// factor in the way number and cache level into r0
131	orr	r0, r0, r7, LSL r10	// factor in the set number
132
133	blx	r6
134	subs	r7, r7, #1		// decrement the set number
135	bhs	loop3
136	subs	r9, r9, #1		// decrement the way number
137	bhs	loop2
138level_done:
139	add	r1, r1, #2		// increment the cache number
140	cmp	r3, r1
141	// Ensure completion of previous cache maintenance instruction. Note
142	// this also mitigates erratum 814220 on Cortex-A7
143	dsb	sy
144	bhi	loop1
145
146	mov	r6, #0
147	stcopr	r6, CSSELR		//select cache level 0 in csselr
148	dsb	sy
149	isb
150	pop	{r4-r12, pc}
151
152dcsw_loop_table:
153	stcopr	r0, DCISW
154	bx	lr
155	stcopr	r0, DCCISW
156	bx	lr
157	stcopr	r0, DCCSW
158	bx	lr
159
160endfunc do_dcsw_op
161
162	/* ---------------------------------------------------------------
163	 * Data cache operations by set/way till PoU.
164	 *
165	 * The function requires :
166	 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW),
167	 * as defined in arch.h
168	 * ---------------------------------------------------------------
169	 */
170func dcsw_op_louis
171	dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
172endfunc	dcsw_op_louis
173
174	/* ---------------------------------------------------------------
175	 * Data cache operations by set/way till PoC.
176	 *
177	 * The function requires :
178	 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW),
179	 * as defined in arch.h
180	 * ---------------------------------------------------------------
181	 */
182func dcsw_op_all
183	dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
184endfunc	dcsw_op_all
185
186
187	/* ---------------------------------------------------------------
188	 *  Helper macro for data cache operations by set/way for the
189	 *  level specified
190	 * ---------------------------------------------------------------
191	 */
192	.macro	dcsw_op_level level
193	ldcopr	r2, CLIDR
194	mov	r3, \level
195	sub	r1, r3, #2
196	b	do_dcsw_op
197	.endm
198
199	/* ---------------------------------------------------------------
200	 * Data cache operations by set/way for level 1 cache
201	 *
202	 * The main function, do_dcsw_op requires:
203	 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW),
204	 * as defined in arch.h
205	 * ---------------------------------------------------------------
206	 */
207func dcsw_op_level1
208	dcsw_op_level #(1 << LEVEL_SHIFT)
209endfunc dcsw_op_level1
210
211	/* ---------------------------------------------------------------
212	 * Data cache operations by set/way for level 2 cache
213	 *
214	 * The main function, do_dcsw_op requires:
215	 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW),
216	 * as defined in arch.h
217	 * ---------------------------------------------------------------
218	 */
219func dcsw_op_level2
220	dcsw_op_level #(2 << LEVEL_SHIFT)
221endfunc dcsw_op_level2
222
223	/* ---------------------------------------------------------------
224	 * Data cache operations by set/way for level 3 cache
225	 *
226	 * The main function, do_dcsw_op requires:
227	 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW),
228	 * as defined in arch.h
229	 * ---------------------------------------------------------------
230	 */
231func dcsw_op_level3
232	dcsw_op_level #(3 << LEVEL_SHIFT)
233endfunc dcsw_op_level3
234