/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2019 MediaTek Inc. */ #include #include #include #include #include #include "proc-macros.S" /* * __xxxx_dcache_user_area(start,size) * * Ensure that any D-cache lines for the interval [start, start+size) * are cleaned or/and invalidated to the PoC. * * - start - virtual start address of region (EL0/EL1) * - size - size in question */ ENTRY(__clean_dcache_user_area) uaccess_enable r2 add r1, r1, r0 dcache_line_size r2, r3 sub r3, r2, #1 bic r0, r0, r3 #ifdef CONFIG_ARM_ERRATA_764369 ALT_SMP(W(dsb)) ALT_UP(W(nop)) #endif 1: #if defined(CONFIG_ARM_ERRATA_855873) || defined(CONFIG_ARM_ERRATA_824069) mcr p15, 0, r0, c7, c14, 1 @ clean and invalidate D / U line #else mcr p15, 0, r0, c7, c10, 1 @ clean D / U line #endif add r0, r0, r2 cmp r0, r1 blo 1b dsb st uaccess_disable r2 ret lr ENDPROC(__clean_dcache_user_area) ENTRY(__clean_dcache_area_poc) add r1, r1, r0 b v7_dma_clean_range ENDPROC(__clean_dcache_area_poc) ENTRY(__flush_dcache_user_area) uaccess_enable r2 add r1, r1, r0 dcache_line_size r2, r3 sub r3, r2, #1 bic r0, r0, r3 #ifdef CONFIG_ARM_ERRATA_764369 ALT_SMP(W(dsb)) ALT_UP(W(nop)) #endif 1: mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D / U line add r0, r0, r2 cmp r0, r1 blo 1b dsb st uaccess_disable r2 ret lr ENDPROC(__flush_dcache_user_area) ENTRY(__flush_dcache_area) add r1, r1, r0 b v7_dma_flush_range ENDPROC(__flush_dcache_area) ENTRY(__inval_dcache_user_area) b __flush_dcache_user_area ENDPROC(__inval_dcache_user_area) ENTRY(__inval_dcache_area) add r1, r1, r0 b v7_dma_inv_range ENDPROC(__inval_dcache_area) .text .global __inner_flush_dcache_all .global __inner_flush_dcache_L1 .global __inner_flush_dcache_L2 .global __inner_clean_dcache_all .global __inner_clean_dcache_L1 .global __inner_clean_dcache_L2 .global __inner_inv_dcache_all .global __inner_inv_dcache_L1 .global __inner_inv_dcache_L2 .global __enable_dcache .global __enable_icache .global __enable_cache .global __disable_dcache .global __disable_icache .global __disable_cache .global __disable_dcache__inner_flush_dcache_L1 .global __disable_dcache__inner_flush_dcache_L1__inner_flush_dcache_L2 .global __disable_dcache__inner_flush_dcache_L1__inner_clean_dcache_L2 .global dis_D_inner_fL1L2 .global d_i_dis_flush_all .global dis_D_inner_flush_all .equ C1_IBIT , 0x00001000 .equ C1_CBIT , 0x00000004 __enable_icache: MRC p15,0,r0,c1,c0,0 ORR r0,r0,#C1_IBIT MCR p15,0,r0,c1,c0,0 BX lr __disable_icache: MRC p15,0,r0,c1,c0,0 BIC r0,r0,#C1_IBIT MCR p15,0,r0,c1,c0,0 BX lr __enable_dcache: MRC p15,0,r0,c1,c0,0 ORR r0,r0,#C1_CBIT dsb MCR p15,0,r0,c1,c0,0 dsb isb BX lr __disable_dcache: MRC p15,0,r0,c1,c0,0 BIC r0,r0,#C1_CBIT dsb MCR p15,0,r0,c1,c0,0 dsb isb /* * Erratum:794322,An instruction fetch can be allocated into * the L2 cache after the cache is disabled Status * This erratum can be avoided by inserting both of the following * after the SCTLR.C bit is cleared to 0, * and before the caches are cleaned or invalidated: * 1) A TLBIMVA operation to any address. * 2) A DSB instruction. */ MCR p15,0,r0,c8,c7,1 dsb isb BX lr __enable_cache: MRC p15,0,r0,c1,c0,0 ORR r0,r0,#C1_IBIT ORR r0,r0,#C1_CBIT MCR p15,0,r0,c1,c0,0 BX lr __disable_cache: MRC p15,0,r0,c1,c0,0 BIC r0,r0,#C1_IBIT BIC r0,r0,#C1_CBIT MCR p15,0,r0,c1,c0,0 /* * Erratum:794322,An instruction fetch can be allocated * into the L2 cache after the cache is disabled Status * This erratum can be avoided by inserting both of the * following after the SCTLR.C bit is cleared to 0, * and before the caches are cleaned or invalidated: * 1) A TLBIMVA operation to any address. * 2) A DSB instruction. */ MCR p15,0,r0,c8,c7,1 dsb BX lr __inner_flush_dcache_all: push {r0,r1,r2,r3,r4,r5,r7,r8,r9,r10,r11,r14} dmb mrc p15, 1, r0, c0, c0, 1 ands r3, r0, #0x7000000 mov r3, r3, lsr #23 beq all_finished mov r10, #0 all_loop1: add r2, r10, r10, lsr #1 mov r1, r0, lsr r2 and r1, r1, #7 @ cmp r1, #2 blt all_skip #ifdef CONFIG_ARM_ERRATA_814220 dsb #endif mcr p15, 2, r10, c0, c0, 0 @ isb @ mrc p15, 1, r1, c0, c0, 0 @ and r2, r1, #7 @ add r2, r2, #4 @ ldr r4, =0x3ff ands r4, r4, r1, lsr #3 @ clz r5, r4 @ ldr r7, =0x7fff ands r7, r7, r1, lsr #13 @ all_loop2: mov r9, r4 @ all_loop3: orr r11, r10, r9, lsl r5 @ orr r11, r11, r7, lsl r2 @ #ifdef CONFIG_L1C_OPT #replace DCCISW by DCISW+DCCSW cmp r10, #2 mrsne r1, cpsr @ orrne r8, r1, #PSR_I_BIT | PSR_F_BIT msrne cpsr_c, r8 mcrne p15, 0, r11, c7, c10, 2 @ mcrne p15, 0, r11, c7, c6, 2 @ msrne cpsr_c, r1 mcreq p15, 0, r11, c7, c14, 2 @ #else mcr p15, 0, r11, c7, c14, 2 @ #endif subs r9, r9, #1 @ bge all_loop3 subs r7, r7, #1 @ bge all_loop2 all_skip: add r10, r10, #2 @ cmp r3, r10 bgt all_loop1 all_finished: mov r10, #0 @ mcr p15, 2, r10, c0, c0, 0 @ dsb isb pop {r0,r1,r2,r3,r4,r5,r7,r8,r9,r10,r11,r14} bx lr __inner_flush_dcache_L1: push {r0,r1,r2,r3,r4,r5,r7,r8,r9,r10,r11,r14} dmb @ mrc p15, 1, r0, c0, c0, 1 @ ands r3, r0, #0x7000000 @ mov r3, r3, lsr #23 @ beq L1_finished mov r10, #0 @ L1_loop1: add r2, r10, r10, lsr #1 @ mov r1, r0, lsr r2 @ and r1, r1, #7 @ cmp r1, #2 @ blt L1_skip @ #ifdef CONFIG_ARM_ERRATA_814220 dsb #endif mcr p15, 2, r10, c0, c0, 0 @ isb @ mrc p15, 1, r1, c0, c0, 0 @ and r2, r1, #7 @ add r2, r2, #4 @ ldr r4, =0x3ff ands r4, r4, r1, lsr #3 @ clz r5, r4 @ ldr r7, =0x7fff ands r7, r7, r1, lsr #13 @ L1_loop2: mov r9, r4 @ L1_loop3: orr r11, r10, r9, lsl r5 @ orr r11, r11, r7, lsl r2 @ #ifdef CONFIG_L1C_OPT #replace DCCISW by DCISW+DCCSW mrs r1, cpsr @ orr r8, r1, #PSR_I_BIT | PSR_F_BIT msr cpsr_c, r8 mcr p15, 0, r11, c7, c10, 2 @ mcr p15, 0, r11, c7, c6, 2 @ msr cpsr_c, r1 #else mcr p15, 0, r11, c7, c14, 2 @ #endif subs r9, r9, #1 @ bge L1_loop3 subs r7, r7, #1 @ bge L1_loop2 L1_skip: @ @ @ L1_finished: mov r10, #0 @ mcr p15, 2, r10, c0, c0, 0 @ dsb isb pop {r0,r1,r2,r3,r4,r5,r7,r8,r9,r10,r11,r14} bx lr __inner_flush_dcache_L2: push {r0,r1,r2,r3,r4,r5,r7,r9,r10,r11,r14} @ dmb @ mrc p15, 1, r0, c0, c0, 1 @ ands r3, r0, #0x7000000 @ mov r3, r3, lsr #23 @ beq L2_finished @ mov r10, #2 @ L2_loop1: add r2, r10, r10, lsr #1 @ mov r1, r0, lsr r2 @ and r1, r1, #7 @ cmp r1, #2 @ blt L2_skip @ #ifdef CONFIG_ARM_ERRATA_814220 dsb #endif mcr p15, 2, r10, c0, c0, 0 @ isb @ mrc p15, 1, r1, c0, c0, 0 @ and r2, r1, #7 @ add r2, r2, #4 @ ldr r4, =0x3ff ands r4, r4, r1, lsr #3 @ clz r5, r4 @ ldr r7, =0x7fff ands r7, r7, r1, lsr #13 @ L2_loop2: mov r9, r4 @ L2_loop3: orr r11, r10, r9, lsl r5 @ orr r11, r11, r7, lsl r2 @ mcr p15, 0, r11, c7, c14, 2 @ subs r9, r9, #1 @ bge L2_loop3 subs r7, r7, #1 @ bge L2_loop2 L2_skip: @ @ @ L2_finished: mov r10, #0 @ mcr p15, 2, r10, c0, c0, 0 @ dsb isb @ pop {r0,r1,r2,r3,r4,r5,r7,r9,r10,r11,r14} bx lr __inner_clean_dcache_all: push {r0,r1,r2,r3,r4,r5,r7,r9,r10,r11,r14} @ dmb @ mrc p15, 1, r0, c0, c0, 1 @ ands r3, r0, #0x7000000 @ mov r3, r3, lsr #23 @ beq all_cl_finished mov r10, #0 @ all_cl_loop1: add r2, r10, r10, lsr #1 @ mov r1, r0, lsr r2 @ and r1, r1, #7 @ cmp r1, #2 @ blt all_cl_skip #ifdef CONFIG_ARM_ERRATA_814220 dsb #endif mcr p15, 2, r10, c0, c0, 0 @ isb @ mrc p15, 1, r1, c0, c0, 0 @ and r2, r1, #7 @ add r2, r2, #4 @ ldr r4, =0x3ff ands r4, r4, r1, lsr #3 @ clz r5, r4 @ ldr r7, =0x7fff ands r7, r7, r1, lsr #13 @ all_cl_loop2: mov r9, r4 @ all_cl_loop3: orr r11, r10, r9, lsl r5 @ orr r11, r11, r7, lsl r2 @ mcr p15, 0, r11, c7, c10, 2 @ subs r9, r9, #1 @ bge all_cl_loop3 subs r7, r7, #1 @ bge all_cl_loop2 all_cl_skip: add r10, r10, #2 @ cmp r3, r10 bgt all_cl_loop1 all_cl_finished: mov r10, #0 @ mcr p15, 2, r10, c0, c0, 0 @ dsb isb @ pop {r0,r1,r2,r3,r4,r5,r7,r9,r10,r11,r14} bx lr __inner_clean_dcache_L1: push {r0,r1,r2,r3,r4,r5,r7,r9,r10,r11,r14} @ dmb @ mrc p15, 1, r0, c0, c0, 1 @ ands r3, r0, #0x7000000 @ mov r3, r3, lsr #23 @ beq L1_cl_finished @ mov r10, #0 @ L1_cl_loop1: add r2, r10, r10, lsr #1 @ mov r1, r0, lsr r2 @ and r1, r1, #7 @ cmp r1, #2 @ blt L1_cl_skip @ #ifdef CONFIG_ARM_ERRATA_814220 dsb #endif mcr p15, 2, r10, c0, c0, 0 @ isb @ mrc p15, 1, r1, c0, c0, 0 @ and r2, r1, #7 @ add r2, r2, #4 @ ldr r4, =0x3ff ands r4, r4, r1, lsr #3 @ clz r5, r4 @ ldr r7, =0x7fff ands r7, r7, r1, lsr #13 @ L1_cl_loop2: mov r9, r4 @ L1_cl_loop3: orr r11, r10, r9, lsl r5 @ orr r11, r11, r7, lsl r2 @ mcr p15, 0, r11, c7, c10, 2 @ subs r9, r9, #1 @ bge L1_cl_loop3 subs r7, r7, #1 @ bge L1_cl_loop2 L1_cl_skip: @ @ @ L1_cl_finished: mov r10, #0 @ mcr p15, 2, r10, c0, c0, 0 @ dsb isb @ pop {r0,r1,r2,r3,r4,r5,r7,r9,r10,r11,r14} bx lr __inner_clean_dcache_L2: #if 0 mov r0, sp mcr p15, 0, r0, c7, c14, 1 @ dsb sub r0, r0, #64 mcr p15, 0, r0, c7, c14, 1 @ dsb #endif push {r0,r1,r2,r3,r4,r5,r7,r9,r10,r11,r14} @ #if 0 mov r0, sp mcr p15, 0, r0, c7, c14, 1 @ dsb sub r0, r0, #64 mcr p15, 0, r0, c7, c14, 1 @ dsb #endif dmb @ mrc p15, 1, r0, c0, c0, 1 @ ands r3, r0, #0x7000000 @ mov r3, r3, lsr #23 @ beq L2_cl_finished @ mov r10, #2 @ L2_cl_loop1: add r2, r10, r10, lsr #1 @ mov r1, r0, lsr r2 @ and r1, r1, #7 @ cmp r1, #2 @ blt L2_cl_skip @ #ifdef CONFIG_ARM_ERRATA_814220 dsb #endif mcr p15, 2, r10, c0, c0, 0 @ isb @ mrc p15, 1, r1, c0, c0, 0 @ and r2, r1, #7 @ add r2, r2, #4 @ ldr r4, =0x3ff ands r4, r4, r1, lsr #3 @ clz r5, r4 @ ldr r7, =0x7fff ands r7, r7, r1, lsr #13 @ L2_cl_loop2: mov r9, r4 @ L2_cl_loop3: orr r11, r10, r9, lsl r5 @ orr r11, r11, r7, lsl r2 @ mcr p15, 0, r11, c7, c10, 2 @ subs r9, r9, #1 @ bge L2_cl_loop3 subs r7, r7, #1 @ bge L2_cl_loop2 L2_cl_skip: @ @ @ L2_cl_finished: mov r10, #0 @ mcr p15, 2, r10, c0, c0, 0 @ dsb isb @ pop {r0,r1,r2,r3,r4,r5,r7,r9,r10,r11,r14} bx lr __inner_inv_dcache_all: push {r0,r1,r2,r3,r4,r5,r7,r9,r10,r11,r14} @ dmb @ mrc p15, 1, r0, c0, c0, 1 @ ands r3, r0, #0x7000000 @ mov r3, r3, lsr #23 @ beq all_inv_finished @ mov r10, #0 @ all_inv_loop1: add r2, r10, r10, lsr #1 @ mov r1, r0, lsr r2 @ and r1, r1, #7 @ cmp r1, #2 @ blt all_inv_skip @ #ifdef CONFIG_ARM_ERRATA_814220 dsb #endif mcr p15, 2, r10, c0, c0, 0 @ isb @ mrc p15, 1, r1, c0, c0, 0 @ and r2, r1, #7 @ add r2, r2, #4 @ ldr r4, =0x3ff ands r4, r4, r1, lsr #3 @ clz r5, r4 @ ldr r7, =0x7fff ands r7, r7, r1, lsr #13 @ all_inv_loop2: mov r9, r4 @ all_inv_loop3: orr r11, r10, r9, lsl r5 @ orr r11, r11, r7, lsl r2 @ mcr p15, 0, r11, c7, c6, 2 @ subs r9, r9, #1 @ bge all_inv_loop3 subs r7, r7, #1 @ bge all_inv_loop2 all_inv_skip: add r10, r10, #2 @ cmp r3, r10 bgt all_inv_loop1 all_inv_finished: mov r10, #0 @ mcr p15, 2, r10, c0, c0, 0 @ dsb isb @ pop {r0,r1,r2,r3,r4,r5,r7,r9,r10,r11,r14} bx lr __inner_inv_dcache_L1: push {r0,r1,r2,r3,r4,r5,r7,r9,r10,r11,r14} @ dmb @ mrc p15, 1, r0, c0, c0, 1 @ ands r3, r0, #0x7000000 @ mov r3, r3, lsr #23 @ beq L1_inv_finished @ mov r10, #0 @ L1_inv_loop1: add r2, r10, r10, lsr #1 @ mov r1, r0, lsr r2 @ and r1, r1, #7 @ cmp r1, #2 @ blt L1_inv_skip @ #ifdef CONFIG_ARM_ERRATA_814220 dsb #endif mcr p15, 2, r10, c0, c0, 0 @ isb @ mrc p15, 1, r1, c0, c0, 0 @ and r2, r1, #7 @ add r2, r2, #4 @ ldr r4, =0x3ff ands r4, r4, r1, lsr #3 @ clz r5, r4 @ ldr r7, =0x7fff ands r7, r7, r1, lsr #13 @ L1_inv_loop2: mov r9, r4 @ L1_inv_loop3: orr r11, r10, r9, lsl r5 @ orr r11, r11, r7, lsl r2 @ mcr p15, 0, r11, c7, c6, 2 @ subs r9, r9, #1 @ bge L1_inv_loop3 subs r7, r7, #1 @ bge L1_inv_loop2 L1_inv_skip: @ @ @ L1_inv_finished: mov r10, #0 @ mcr p15, 2, r10, c0, c0, 0 @ dsb isb @ pop {r0,r1,r2,r3,r4,r5,r7,r9,r10,r11,r14} bx lr __inner_inv_dcache_L2: push {r0,r1,r2,r3,r4,r5,r7,r9,r10,r11,r14} @ dmb @ mrc p15, 1, r0, c0, c0, 1 @ ands r3, r0, #0x7000000 @ mov r3, r3, lsr #23 @ beq L2_inv_finished @ mov r10, #2 @ L2_inv_loop1: add r2, r10, r10, lsr #1 @ mov r1, r0, lsr r2 @ and r1, r1, #7 @ cmp r1, #2 @ blt L2_inv_skip @ #ifdef CONFIG_ARM_ERRATA_814220 dsb #endif mcr p15, 2, r10, c0, c0, 0 @ isb @ mrc p15, 1, r1, c0, c0, 0 @ and r2, r1, #7 @ add r2, r2, #4 @ ldr r4, =0x3ff ands r4, r4, r1, lsr #3 @ clz r5, r4 @ ldr r7, =0x7fff ands r7, r7, r1, lsr #13 @ L2_inv_loop2: mov r9, r4 @ L2_inv_loop3: orr r11, r10, r9, lsl r5 @ orr r11, r11, r7, lsl r2 @ mcr p15, 0, r11, c7, c6, 2 @ subs r9, r9, #1 @ bge L2_inv_loop3 subs r7, r7, #1 @ bge L2_inv_loop2 L2_inv_skip: @ @ @ L2_inv_finished: mov r10, #0 @ mcr p15, 2, r10, c0, c0, 0 @ dsb isb @ pop {r0,r1,r2,r3,r4,r5,r7,r9,r10,r11,r14} bx lr __disable_dcache__inner_flush_dcache_L1: /******************************************************************************* * push stack * ******************************************************************************/ push {r0,r1,r2,r3,r4,r5,r7,r9,r10,r11,r14} /******************************************************************************* * __disable_dcache * ******************************************************************************/ MRC p15,0,r0,c1,c0,0 BIC r0,r0,#C1_CBIT dsb MCR p15,0,r0,c1,c0,0 dsb isb /* * Erratum:794322,An instruction fetch can be allocated * into the L2 cache after the cache is disabled Status * This erratum can be avoided by inserting both of the * following after the SCTLR.C bit is cleared to 0, * and before the caches are cleaned or invalidated: * 1) A TLBIMVA operation to any address. * 2) A DSB instruction. */ MCR p15,0,r0,c8,c7,1 dsb isb /******************************************************************************* * __inner_flush_dcache_L1 * ******************************************************************************/ dmb @ mrc p15, 1, r0, c0, c0, 1 @ ands r3, r0, #0x7000000 @ mov r3, r3, lsr #23 @ beq DF1_L1_finished @ mov r10, #0 @ DF1_L1_loop1: add r2, r10, r10, lsr #1 @ mov r1, r0, lsr r2 @ and r1, r1, #7 @ cmp r1, #2 @ blt DF1_L1_skip @ #ifdef CONFIG_ARM_ERRATA_814220 dsb #endif mcr p15, 2, r10, c0, c0, 0 @ isb @ mrc p15, 1, r1, c0, c0, 0 @ and r2, r1, #7 @ add r2, r2, #4 @ ldr r4, =0x3ff ands r4, r4, r1, lsr #3 @ clz r5, r4 @ ldr r7, =0x7fff ands r7, r7, r1, lsr #13 @ DF1_L1_loop2: mov r9, r4 @ DF1_L1_loop3: orr r11, r10, r9, lsl r5 @ orr r11, r11, r7, lsl r2 @ #if 1 mcr p15, 0, r11, c7, c10, 2 @ mcr p15, 0, r11, c7, c6, 2 @ #endif #if 0 mcr p15, 0, r11, c7, c14, 2 @ #endif subs r9, r9, #1 @ bge DF1_L1_loop3 subs r7, r7, #1 @ bge DF1_L1_loop2 DF1_L1_skip: @ @ @ DF1_L1_finished: mov r10, #0 @ mcr p15, 2, r10, c0, c0, 0 @ dsb isb /******************************************************************************* * pop stack * ******************************************************************************/ pop {r0,r1,r2,r3,r4,r5,r7,r9,r10,r11,r14} bx lr __disable_dcache__inner_flush_dcache_L1__inner_flush_dcache_L2: /******************************************************************************* * push stack * ******************************************************************************/ push {r0,r1,r2,r3,r4,r5,r7,r9,r10,r11,r14} /******************************************************************************* * __disable_dcache * ******************************************************************************/ MRC p15,0,r0,c1,c0,0 BIC r0,r0,#C1_CBIT dsb MCR p15,0,r0,c1,c0,0 dsb isb /* * Erratum:794322,An instruction fetch can be allocated * into the L2 cache after the cache is disabled Status * This erratum can be avoided by inserting both of the * following after the SCTLR.C bit is cleared to 0, * and before the caches are cleaned or invalidated: * 1) A TLBIMVA operation to any address. * 2) A DSB instruction. */ MCR p15,0,r0,c8,c7,1 dsb isb /******************************************************************************* * __inner_flush_dcache_L1 * ******************************************************************************/ dmb @ mrc p15, 1, r0, c0, c0, 1 @ ands r3, r0, #0x7000000 @ mov r3, r3, lsr #23 @ beq DF1F2_L1_finished @ mov r10, #0 @ DF1F2_L1_loop1: add r2, r10, r10, lsr #1 @ mov r1, r0, lsr r2 @ and r1, r1, #7 @ cmp r1, #2 @ blt DF1F2_L1_skip @ #ifdef CONFIG_ARM_ERRATA_814220 dsb #endif mcr p15, 2, r10, c0, c0, 0 @ isb @ mrc p15, 1, r1, c0, c0, 0 @ and r2, r1, #7 @ add r2, r2, #4 @ ldr r4, =0x3ff ands r4, r4, r1, lsr #3 @ clz r5, r4 @ ldr r7, =0x7fff ands r7, r7, r1, lsr #13 @ DF1F2_L1_loop2: mov r9, r4 @ DF1F2_L1_loop3: orr r11, r10, r9, lsl r5 @ orr r11, r11, r7, lsl r2 @ #if 1 mcr p15, 0, r11, c7, c10, 2 @ mcr p15, 0, r11, c7, c6, 2 @ #endif #if 0 mcr p15, 0, r11, c7, c14, 2 @ #endif subs r9, r9, #1 @ bge DF1F2_L1_loop3 subs r7, r7, #1 @ bge DF1F2_L1_loop2 DF1F2_L1_skip: @ @ @ DF1F2_L1_finished: mov r10, #0 @ mcr p15, 2, r10, c0, c0, 0 @ dsb isb /******************************************************************************* * clrex * ******************************************************************************/ clrex /******************************************************************************* * __inner_flush_dcache_L2 * ******************************************************************************/ dmb @ mrc p15, 1, r0, c0, c0, 1 @ ands r3, r0, #0x7000000 @ mov r3, r3, lsr #23 @ beq DF1F2_L2_finished @ mov r10, #2 @ DF1F2_L2_loop1: add r2, r10, r10, lsr #1 @ mov r1, r0, lsr r2 @ and r1, r1, #7 @ cmp r1, #2 @ blt DF1F2_L2_skip @ #ifdef CONFIG_ARM_ERRATA_814220 dsb #endif mcr p15, 2, r10, c0, c0, 0 @ isb @ mrc p15, 1, r1, c0, c0, 0 @ and r2, r1, #7 @ add r2, r2, #4 @ ldr r4, =0x3ff ands r4, r4, r1, lsr #3 @ clz r5, r4 @ ldr r7, =0x7fff ands r7, r7, r1, lsr #13 @ DF1F2_L2_loop2: mov r9, r4 @ DF1F2_L2_loop3: orr r11, r10, r9, lsl r5 @ orr r11, r11, r7, lsl r2 @ mcr p15, 0, r11, c7, c14, 2 @ subs r9, r9, #1 @ bge DF1F2_L2_loop3 subs r7, r7, #1 @ bge DF1F2_L2_loop2 DF1F2_L2_skip: @ @ @ DF1F2_L2_finished: mov r10, #0 @ mcr p15, 2, r10, c0, c0, 0 @ dsb isb /******************************************************************************* * pop stack * ******************************************************************************/ pop {r0,r1,r2,r3,r4,r5,r7,r9,r10,r11,r14} bx lr dis_D_inner_fL1L2: __disable_dcache__inner_flush_dcache_L1__inner_clean_dcache_L2: /******************************************************************************* * push stack * ******************************************************************************/ push {r0,r1,r2,r3,r4,r5,r7,r9,r10,r11,r14} /******************************************************************************* * __disable_dcache * ******************************************************************************/ MRC p15,0,r0,c1,c0,0 BIC r0,r0,#C1_CBIT dsb MCR p15,0,r0,c1,c0,0 dsb isb /* * Erratum:794322,An instruction fetch can be allocated * into the L2 cache after the cache is disabled Status * This erratum can be avoided by inserting both of the * following after the SCTLR.C bit is cleared to 0, * and before the caches are cleaned or invalidated: * 1) A TLBIMVA operation to any address. * 2) A DSB instruction. */ MCR p15,0,r0,c8,c7,1 dsb isb /******************************************************************************* * __inner_flush_dcache_L1 * ******************************************************************************/ dmb @ mrc p15, 1, r0, c0, c0, 1 @ ands r3, r0, #0x7000000 @ mov r3, r3, lsr #23 @ beq DF1C2_L1_finished @ mov r10, #0 @ DF1C2_L1_loop1: add r2, r10, r10, lsr #1 @ mov r1, r0, lsr r2 @ and r1, r1, #7 @ cmp r1, #2 @ blt DF1C2_L1_skip @ #ifdef CONFIG_ARM_ERRATA_814220 dsb #endif mcr p15, 2, r10, c0, c0, 0 @ isb @ mrc p15, 1, r1, c0, c0, 0 @ and r2, r1, #7 @ add r2, r2, #4 @ ldr r4, =0x3ff ands r4, r4, r1, lsr #3 @ clz r5, r4 @ ldr r7, =0x7fff ands r7, r7, r1, lsr #13 @ DF1C2_L1_loop2: mov r9, r4 @ DF1C2_L1_loop3: orr r11, r10, r9, lsl r5 @ orr r11, r11, r7, lsl r2 @ #if 1 mcr p15, 0, r11, c7, c10, 2 @ mcr p15, 0, r11, c7, c6, 2 @ #endif #if 0 mcr p15, 0, r11, c7, c14, 2 @ #endif subs r9, r9, #1 @ bge DF1C2_L1_loop3 subs r7, r7, #1 @ bge DF1C2_L1_loop2 DF1C2_L1_skip: @ @ @ DF1C2_L1_finished: mov r10, #0 @ mcr p15, 2, r10, c0, c0, 0 @ dsb isb /******************************************************************************* * clrex * ******************************************************************************/ clrex /******************************************************************************* * __inner_clean_dcache_L2 * ******************************************************************************/ dmb @ mrc p15, 1, r0, c0, c0, 1 @ ands r3, r0, #0x7000000 @ mov r3, r3, lsr #23 @ beq DF1C2_L2_cl_finished @ mov r10, #2 @ DF1C2_L2_cl_loop1: add r2, r10, r10, lsr #1 @ mov r1, r0, lsr r2 @ and r1, r1, #7 @ cmp r1, #2 @ blt DF1C2_L2_cl_skip @ #ifdef CONFIG_ARM_ERRATA_814220 dsb #endif mcr p15, 2, r10, c0, c0, 0 @ isb @ mrc p15, 1, r1, c0, c0, 0 @ and r2, r1, #7 @ add r2, r2, #4 @ ldr r4, =0x3ff ands r4, r4, r1, lsr #3 @ clz r5, r4 @ ldr r7, =0x7fff ands r7, r7, r1, lsr #13 @ DF1C2_L2_cl_loop2: mov r9, r4 @ DF1C2_L2_cl_loop3: orr r11, r10, r9, lsl r5 @ orr r11, r11, r7, lsl r2 @ mcr p15, 0, r11, c7, c10, 2 @ subs r9, r9, #1 @ bge DF1C2_L2_cl_loop3 subs r7, r7, #1 @ bge DF1C2_L2_cl_loop2 DF1C2_L2_cl_skip: @ @ @ DF1C2_L2_cl_finished: mov r10, #0 @ mcr p15, 2, r10, c0, c0, 0 @ dsb isb /******************************************************************************* * pop stack * ******************************************************************************/ pop {r0,r1,r2,r3,r4,r5,r7,r9,r10,r11,r14} bx lr d_i_dis_flush_all: /******************************************************************************* * push stack * ******************************************************************************/ push {r0,r1,r2,r3,r4,r5,r7,r9,r10,r11,r14} /******************************************************************************* * __disable_dcache * ******************************************************************************/ MRC p15,0,r0,c1,c0,0 BIC r0,r0,#C1_CBIT BIC r0,r0,#C1_IBIT dsb MCR p15,0,r0,c1,c0,0 dsb isb /* * Erratum:794322,An instruction fetch can be allocated * into the L2 cache after the cache is disabled Status * This erratum can be avoided by inserting both of the * following after the SCTLR.C bit is cleared to 0, * and before the caches are cleaned or invalidated: * 1) A TLBIMVA operation to any address. * 2) A DSB instruction. */ MCR p15,0,r0,c8,c7,1 dsb isb /******************************************************************************* * __inner_flush_dcache_L1 * ******************************************************************************/ dmb @ mrc p15, 1, r0, c0, c0, 1 @ ands r3, r0, #0x7000000 @ mov r3, r3, lsr #23 @ beq DIF1F2_L1_finished @ mov r10, #0 @ DIF1F2_L1_loop1: add r2, r10, r10, lsr #1 @ mov r1, r0, lsr r2 @ and r1, r1, #7 @ cmp r1, #2 @ blt DIF1F2_L1_skip @ #ifdef CONFIG_ARM_ERRATA_814220 dsb #endif mcr p15, 2, r10, c0, c0, 0 @ isb @ mrc p15, 1, r1, c0, c0, 0 @ and r2, r1, #7 @ add r2, r2, #4 @ ldr r4, =0x3ff ands r4, r4, r1, lsr #3 @ clz r5, r4 @ ldr r7, =0x7fff ands r7, r7, r1, lsr #13 @ DIF1F2_L1_loop2: mov r9, r4 @ DIF1F2_L1_loop3: orr r11, r10, r9, lsl r5 @ orr r11, r11, r7, lsl r2 @ #if 1 mcr p15, 0, r11, c7, c10, 2 @ mcr p15, 0, r11, c7, c6, 2 @ #endif #if 0 mcr p15, 0, r11, c7, c14, 2 @ #endif subs r9, r9, #1 @ bge DIF1F2_L1_loop3 subs r7, r7, #1 @ bge DIF1F2_L1_loop2 DIF1F2_L1_skip: @ @ @ DIF1F2_L1_finished: mov r10, #0 @ mcr p15, 2, r10, c0, c0, 0 @ dsb isb /******************************************************************************* * clrex * ******************************************************************************/ clrex /******************************************************************************* * __inner_flush_dcache_L2 * ******************************************************************************/ dmb @ mrc p15, 1, r0, c0, c0, 1 @ ands r3, r0, #0x7000000 @ mov r3, r3, lsr #23 @ beq DIF1F2_L2_finished @ mov r10, #2 @ DIF1F2_L2_loop1: add r2, r10, r10, lsr #1 @ mov r1, r0, lsr r2 @ and r1, r1, #7 @ cmp r1, #2 @ blt DIF1F2_L2_skip @ #ifdef CONFIG_ARM_ERRATA_814220 dsb #endif mcr p15, 2, r10, c0, c0, 0 @ isb @ mrc p15, 1, r1, c0, c0, 0 @ and r2, r1, #7 @ add r2, r2, #4 @ ldr r4, =0x3ff ands r4, r4, r1, lsr #3 @ clz r5, r4 @ ldr r7, =0x7fff ands r7, r7, r1, lsr #13 @ DIF1F2_L2_loop2: mov r9, r4 @ DIF1F2_L2_loop3: orr r11, r10, r9, lsl r5 @ orr r11, r11, r7, lsl r2 @ mcr p15, 0, r11, c7, c14, 2 @ subs r9, r9, #1 @ bge DIF1F2_L2_loop3 subs r7, r7, #1 @ bge DIF1F2_L2_loop2 DIF1F2_L2_skip: @ @ @ DIF1F2_L2_finished: mov r10, #0 @ mcr p15, 2, r10, c0, c0, 0 @ dsb isb /******************************************************************************* * pop stack * ******************************************************************************/ pop {r0,r1,r2,r3,r4,r5,r7,r9,r10,r11,r14} bx lr dis_D_inner_flush_all: /* disable data cache*/ MRC p15,0,r0,c1,c0,0 BIC r0,r0,#C1_CBIT dsb MCR p15,0,r0,c1,c0,0 dsb isb b v7_flush_dcache_all .end