1/* 2 * Copyright (c) 2016-2021, Arm Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7#include <arch.h> 8#include <asm_macros.S> 9 10 .globl flush_dcache_range 11 .globl clean_dcache_range 12 .globl inv_dcache_range 13 .globl dcsw_op_louis 14 .globl dcsw_op_all 15 .globl dcsw_op_level1 16 .globl dcsw_op_level2 17 .globl dcsw_op_level3 18 19/* 20 * This macro can be used for implementing various data cache operations `op` 21 */ 22.macro do_dcache_maintenance_by_mva op, coproc, opc1, CRn, CRm, opc2 23 /* Exit early if size is zero */ 24 cmp r1, #0 25 beq exit_loop_\op 26 dcache_line_size r2, r3 27 add r1, r0, r1 28 sub r3, r2, #1 29 bic r0, r0, r3 30loop_\op: 31 stcopr r0, \coproc, \opc1, \CRn, \CRm, \opc2 32 add r0, r0, r2 33 cmp r0, r1 34 blo loop_\op 35 dsb sy 36exit_loop_\op: 37 bx lr 38.endm 39 40 /* ------------------------------------------ 41 * Clean+Invalidate from base address till 42 * size. 'r0' = addr, 'r1' = size 43 * ------------------------------------------ 44 */ 45func flush_dcache_range 46 do_dcache_maintenance_by_mva cimvac, DCCIMVAC 47endfunc flush_dcache_range 48 49 /* ------------------------------------------ 50 * Clean from base address till size. 51 * 'r0' = addr, 'r1' = size 52 * ------------------------------------------ 53 */ 54func clean_dcache_range 55 do_dcache_maintenance_by_mva cmvac, DCCMVAC 56endfunc clean_dcache_range 57 58 /* ------------------------------------------ 59 * Invalidate from base address till 60 * size. 'r0' = addr, 'r1' = size 61 * ------------------------------------------ 62 */ 63func inv_dcache_range 64 do_dcache_maintenance_by_mva imvac, DCIMVAC 65endfunc inv_dcache_range 66 67 /* ---------------------------------------------------------------- 68 * Data cache operations by set/way to the level specified 69 * 70 * The main function, do_dcsw_op requires: 71 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), 72 * as defined in arch.h 73 * r1: The cache level to begin operation from 74 * r2: clidr_el1 75 * r3: The last cache level to operate on 76 * and will carry out the operation on each data cache from level 0 77 * to the level in r3 in sequence 78 * 79 * The dcsw_op macro sets up the r2 and r3 parameters based on 80 * clidr_el1 cache information before invoking the main function 81 * ---------------------------------------------------------------- 82 */ 83 84 .macro dcsw_op shift, fw, ls 85 ldcopr r2, CLIDR 86 ubfx r3, r2, \shift, \fw 87 lsl r3, r3, \ls 88 mov r1, #0 89 b do_dcsw_op 90 .endm 91 92func do_dcsw_op 93 push {r4-r12, lr} 94 ldcopr r8, ID_MMFR4 // stash FEAT_CCIDX identifier in r8 95 ubfx r8, r8, #ID_MMFR4_CCIDX_SHIFT, #ID_MMFR4_CCIDX_LENGTH 96 adr r11, dcsw_loop_table // compute cache op based on the operation type 97 add r6, r11, r0, lsl #3 // cache op is 2x32-bit instructions 98loop1: 99 add r10, r1, r1, LSR #1 // Work out 3x current cache level 100 mov r12, r2, LSR r10 // extract cache type bits from clidr 101 and r12, r12, #7 // mask the bits for current cache only 102 cmp r12, #2 // see what cache we have at this level 103 blo level_done // no cache or only instruction cache at this level 104 105 stcopr r1, CSSELR // select current cache level in csselr 106 isb // isb to sych the new cssr&csidr 107 ldcopr r12, CCSIDR // read the new ccsidr 108 and r10, r12, #7 // extract the length of the cache lines 109 add r10, r10, #4 // add 4 (r10 = line length offset) 110 111 cmp r8, #0 // check for FEAT_CCIDX for Associativity 112 beq 1f 113 ubfx r4, r12, #3, #21 // r4 = associativity CCSIDR[23:3] 114 b 2f 1151: 116 ubfx r4, r12, #3, #10 // r4 = associativity CCSIDR[12:3] 1172: 118 clz r5, r4 // r5 = the bit position of the way size increment 119 mov r9, r4 // r9 working copy of the aligned max way number 120 121loop2: 122 cmp r8, #0 // check for FEAT_CCIDX for NumSets 123 beq 3f 124 ldcopr r12, CCSIDR2 // FEAT_CCIDX numsets is in CCSIDR2 125 ubfx r7, r12, #0, #24 // r7 = numsets CCSIDR2[23:0] 126 b loop3 1273: 128 ubfx r7, r12, #13, #15 // r7 = numsets CCSIDR[27:13] 129loop3: 130 orr r0, r1, r9, LSL r5 // factor in the way number and cache level into r0 131 orr r0, r0, r7, LSL r10 // factor in the set number 132 133 blx r6 134 subs r7, r7, #1 // decrement the set number 135 bhs loop3 136 subs r9, r9, #1 // decrement the way number 137 bhs loop2 138level_done: 139 add r1, r1, #2 // increment the cache number 140 cmp r3, r1 141 // Ensure completion of previous cache maintenance instruction. Note 142 // this also mitigates erratum 814220 on Cortex-A7 143 dsb sy 144 bhi loop1 145 146 mov r6, #0 147 stcopr r6, CSSELR //select cache level 0 in csselr 148 dsb sy 149 isb 150 pop {r4-r12, pc} 151 152dcsw_loop_table: 153 stcopr r0, DCISW 154 bx lr 155 stcopr r0, DCCISW 156 bx lr 157 stcopr r0, DCCSW 158 bx lr 159 160endfunc do_dcsw_op 161 162 /* --------------------------------------------------------------- 163 * Data cache operations by set/way till PoU. 164 * 165 * The function requires : 166 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), 167 * as defined in arch.h 168 * --------------------------------------------------------------- 169 */ 170func dcsw_op_louis 171 dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT 172endfunc dcsw_op_louis 173 174 /* --------------------------------------------------------------- 175 * Data cache operations by set/way till PoC. 176 * 177 * The function requires : 178 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), 179 * as defined in arch.h 180 * --------------------------------------------------------------- 181 */ 182func dcsw_op_all 183 dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT 184endfunc dcsw_op_all 185 186 187 /* --------------------------------------------------------------- 188 * Helper macro for data cache operations by set/way for the 189 * level specified 190 * --------------------------------------------------------------- 191 */ 192 .macro dcsw_op_level level 193 ldcopr r2, CLIDR 194 mov r3, \level 195 sub r1, r3, #2 196 b do_dcsw_op 197 .endm 198 199 /* --------------------------------------------------------------- 200 * Data cache operations by set/way for level 1 cache 201 * 202 * The main function, do_dcsw_op requires: 203 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), 204 * as defined in arch.h 205 * --------------------------------------------------------------- 206 */ 207func dcsw_op_level1 208 dcsw_op_level #(1 << LEVEL_SHIFT) 209endfunc dcsw_op_level1 210 211 /* --------------------------------------------------------------- 212 * Data cache operations by set/way for level 2 cache 213 * 214 * The main function, do_dcsw_op requires: 215 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), 216 * as defined in arch.h 217 * --------------------------------------------------------------- 218 */ 219func dcsw_op_level2 220 dcsw_op_level #(2 << LEVEL_SHIFT) 221endfunc dcsw_op_level2 222 223 /* --------------------------------------------------------------- 224 * Data cache operations by set/way for level 3 cache 225 * 226 * The main function, do_dcsw_op requires: 227 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), 228 * as defined in arch.h 229 * --------------------------------------------------------------- 230 */ 231func dcsw_op_level3 232 dcsw_op_level #(3 << LEVEL_SHIFT) 233endfunc dcsw_op_level3 234