lib/aarch64/cache_helpers.S - trusted-firmware-a - Git at Google

 /*
  * Copyright (c) 2013-2022, Arm Limited and Contributors. All rights reserved.
  *
  * SPDX-License-Identifier: BSD-3-Clause
  */

 #include <arch.h>
 #include <asm_macros.S>

 	.globl	flush_dcache_range
 	.globl	flush_dcache_to_popa_range
 	.globl	flush_dcache_to_popa_range_mte2
 	.globl	clean_dcache_range
 	.globl	inv_dcache_range
 	.globl	dcsw_op_louis
 	.globl	dcsw_op_all
 	.globl	dcsw_op_level1
 	.globl	dcsw_op_level2
 	.globl	dcsw_op_level3

 /* Opcodes for data cache maintenance by PA instructions. */

 /*
  * sys  #6, c7, c14, #1, x0
  * DC CIPAPA, X0
  */
 #define dc_cipapa_x0	0xd50e7e20

 /*
  * sys #6, c7, c14, #3, x0
  * DC CIDGPAPA, X0
   */
 #define dc_cigdpapa_x0	0xd50e7ea0

 /*
  * This macro can be used for implementing various data cache operations `op`
  */
 .macro do_dcache_maintenance_by_mva op
 	/* Exit early if size is zero */
 	cbz	x1, exit_loop_\op
 	dcache_line_size x2, x3
 	add	x1, x0, x1
 	sub	x3, x2, #1
 	bic	x0, x0, x3
 loop_\op:
 	dc	\op, x0
 	add	x0, x0, x2
 	cmp	x0, x1
 	b.lo	loop_\op
 	dsb	sy
 exit_loop_\op:
 	ret
 .endm

 /* op: the hexadecimal instruction opcode for the cache operation */
 .macro do_dcache_maintenance_instr op
 	/* Exit early if size is zero */
 	cbz	x1, exit_loop_\op
 	dcache_line_size x2, x3
 	sub	x3, x2, #1
 	bic	x0, x0, x3
 	add	x1, x1, x0
 loop_\op:
 	.inst	\op
 	add	x0, x0, x2
 	cmp	x0, x1
 	b.lo	loop_\op
 	dsb	osh
 exit_loop_\op:
 	ret
 .endm

 .macro check_plat_can_cmo
 #if CONDITIONAL_CMO
 	mov	x3, x30
 	mov	x2, x0
 	bl	plat_can_cmo
 	mov	x30, x3
 	cbnz	x0, 1f
 	ret
 1:
 	mov	 x0, x2
 #endif
 .endm

 	/* -------------------------------------------
 	 * DCache Clean+Invalidate by MVA from base
 	 * address till size. 'x0' = addr, 'x1' = size
 	 * -------------------------------------------
 	 */
 func flush_dcache_range
 	check_plat_can_cmo
 	do_dcache_maintenance_by_mva civac
 endfunc flush_dcache_range

 	/* ------------------------------------------
 	 * DCache Clean by MVA from base address till
 	 * size. 'x0' = addr, 'x1' = size
 	 * ------------------------------------------
 	 */
 func clean_dcache_range
 	check_plat_can_cmo
 	do_dcache_maintenance_by_mva cvac
 endfunc clean_dcache_range

 	/* ------------------------------------------
 	 * DCache Invalidate by MVA from base address
 	 * till size. 'x0' = addr, 'x1' = size
 	 * ------------------------------------------
 	 */
 func inv_dcache_range
 	check_plat_can_cmo
 	do_dcache_maintenance_by_mva ivac
 endfunc inv_dcache_range

 	/*
 	 * ------------------------------------------
 	 * DCache Clean+Invalidate by PA to POPA from
 	 * base address till size.
 	 * 'x0' = addr, 'x1' = size
 	 * ------------------------------------------
 	 */
 func flush_dcache_to_popa_range
 	check_plat_can_cmo
 	/* dc cipapa, x0 */
 	do_dcache_maintenance_instr dc_cipapa_x0
 endfunc	flush_dcache_to_popa_range

 	/*
 	 * ------------------------------------------
 	 * Clean+Invalidate by PA to POPA (MTE2)
 	 * from base address till size.
 	 * 'x0' = addr, 'x1' = size
 	 * ------------------------------------------
 	 * On implementations with FEAT_MTE2, Root firmware must issue
 	 * DC_CIGDPAPA instead of DC_CIPAPA, in order to additionally
 	 * clean and invalidate Allocation Tags associated with the
 	 * affected locations.
 	 */
 func flush_dcache_to_popa_range_mte2
 	check_plat_can_cmo
 	/* dc cigdpapa, x0 */
 	do_dcache_maintenance_instr dc_cigdpapa_x0
 endfunc	flush_dcache_to_popa_range_mte2

 	/* ---------------------------------------------------------------
 	 * Data cache operations by set/way to the level specified
 	 *
 	 * The main function, do_dcsw_op requires:
 	 * x0: The operation type (0-2), as defined in arch.h
 	 * x3: The last cache level to operate on
 	 * x9: clidr_el1
 	 * x10: The cache level to begin operation from
 	 * and will carry out the operation on each data cache from level 0
 	 * to the level in x3 in sequence
 	 *
 	 * The dcsw_op macro sets up the x3 and x9 parameters based on
 	 * clidr_el1 cache information before invoking the main function
 	 * ---------------------------------------------------------------
 	 */

 	.macro	dcsw_op shift, fw, ls
 	mrs	x9, clidr_el1
 	ubfx	x3, x9, \shift, \fw
 	lsl	x3, x3, \ls
 	mov	x10, xzr
 	b	do_dcsw_op
 	.endm

 func do_dcsw_op
 	cbz	x3, exit
 	mrs	x12, ID_AA64MMFR2_EL1	// stash FEAT_CCIDX identifier in x12
 	ubfx	x12, x12, #ID_AA64MMFR2_EL1_CCIDX_SHIFT, #ID_AA64MMFR2_EL1_CCIDX_LENGTH
 	adr	x14, dcsw_loop_table	// compute inner loop address
 	add	x14, x14, x0, lsl #5	// inner loop is 8x32-bit instructions
 #if ENABLE_BTI
 	add	x14, x14, x0, lsl #2	// inner loop is + "bti j" instruction
 #endif
 	mov	x0, x9
 	mov	w8, #1
 loop1:
 	add	x2, x10, x10, lsr #1	// work out 3x current cache level
 	lsr	x1, x0, x2		// extract cache type bits from clidr
 	and	x1, x1, #7		// mask the bits for current cache only
 	cmp	x1, #2			// see what cache we have at this level
 	b.lo	level_done		// nothing to do if no cache or icache

 	msr	csselr_el1, x10		// select current cache level in csselr
 	isb				// isb to sych the new cssr&csidr
 	mrs	x1, ccsidr_el1		// read the new ccsidr
 	and	x2, x1, #7		// extract the length of the cache lines
 	add	x2, x2, #4		// add 4 (line length offset)

 	cbz	x12, 1f			// check for FEAT_CCIDX for Associativity
 	ubfx	x4, x1, #3, #21 	// x4 = associativity CCSIDR_EL1[23:3]
 	b 	2f
 1:
 	ubfx	x4, x1, #3, #10 	// x4 = associativity CCSIDR_EL1[12:3]
 2:
 	clz	w5, w4			// bit position of way size increment
 	lsl	w9, w4, w5		// w9 = aligned max way number
 	lsl	w16, w8, w5		// w16 = way number loop decrement
 	orr	w9, w10, w9		// w9 = combine way and cache number

 	cbz	x12, 3f			// check for FEAT_CCIDX for NumSets
 	ubfx	x6, x1, #32, #24	// x6 (w6) = numsets CCSIDR_EL1[55:32]
 					// ISA will not allow x->w ubfx
 	b	4f
 3:
 	ubfx	w6, w1, #13, #15	// w6 = numsets CCSIDR_EL1[27:13]
 4:
 	lsl	w17, w8, w2		// w17 = set number loop decrement
 	dsb	sy			// barrier before we start this level
 	br	x14			// jump to DC operation specific loop

 	.macro	dcsw_loop _op
 #if ENABLE_BTI
 	bti	j
 #endif
 loop2_\_op:
 	lsl	w7, w6, w2		// w7 = aligned max set number

 loop3_\_op:
 	orr	w11, w9, w7		// combine cache, way and set number
 	dc	\_op, x11
 	subs	w7, w7, w17		// decrement set number
 	b.hs	loop3_\_op

 	subs	x9, x9, x16		// decrement way number
 	b.hs	loop2_\_op

 	b	level_done
 	.endm

 level_done:
 	add	x10, x10, #2		// increment cache number
 	cmp	x3, x10
 	b.hi	loop1
 	msr	csselr_el1, xzr		// select cache level 0 in csselr
 	dsb	sy			// barrier to complete final cache operation
 	isb
 exit:
 	ret
 endfunc do_dcsw_op

 dcsw_loop_table:
 	dcsw_loop isw
 	dcsw_loop cisw
 	dcsw_loop csw


 func dcsw_op_louis
 	check_plat_can_cmo
 	dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
 endfunc dcsw_op_louis


 func dcsw_op_all
 	check_plat_can_cmo
 	dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
 endfunc dcsw_op_all

 	/* ---------------------------------------------------------------
 	 *  Helper macro for data cache operations by set/way for the
 	 *  level specified
 	 * ---------------------------------------------------------------
 	 */
 	.macro dcsw_op_level level
 	mrs	x9, clidr_el1
 	mov	x3, \level
 	sub	x10, x3, #2
 	b	do_dcsw_op
 	.endm

 	/* ---------------------------------------------------------------
 	 * Data cache operations by set/way for level 1 cache
 	 *
 	 * The main function, do_dcsw_op requires:
 	 * x0: The operation type (0-2), as defined in arch.h
 	 * ---------------------------------------------------------------
 	 */
 func dcsw_op_level1
 	check_plat_can_cmo
 	dcsw_op_level #(1 << LEVEL_SHIFT)
 endfunc dcsw_op_level1

 	/* ---------------------------------------------------------------
 	 * Data cache operations by set/way for level 2 cache
 	 *
 	 * The main function, do_dcsw_op requires:
 	 * x0: The operation type (0-2), as defined in arch.h
 	 * ---------------------------------------------------------------
 	 */
 func dcsw_op_level2
 	check_plat_can_cmo
 	dcsw_op_level #(2 << LEVEL_SHIFT)
 endfunc dcsw_op_level2

 	/* ---------------------------------------------------------------
 	 * Data cache operations by set/way for level 3 cache
 	 *
 	 * The main function, do_dcsw_op requires:
 	 * x0: The operation type (0-2), as defined in arch.h
 	 * ---------------------------------------------------------------
 	 */
 func dcsw_op_level3
 	check_plat_can_cmo
 	dcsw_op_level #(3 << LEVEL_SHIFT)
 endfunc dcsw_op_level3
	/*
	* Copyright (c) 2013-2022, Arm Limited and Contributors. All rights reserved.
	*
	* SPDX-License-Identifier: BSD-3-Clause
	*/

	#include <arch.h>
	#include <asm_macros.S>

	.globl flush_dcache_range
	.globl flush_dcache_to_popa_range
	.globl flush_dcache_to_popa_range_mte2
	.globl clean_dcache_range
	.globl inv_dcache_range
	.globl dcsw_op_louis
	.globl dcsw_op_all
	.globl dcsw_op_level1
	.globl dcsw_op_level2
	.globl dcsw_op_level3

	/* Opcodes for data cache maintenance by PA instructions. */

	/*
	* sys #6, c7, c14, #1, x0
	* DC CIPAPA, X0
	*/
	#define dc_cipapa_x0 0xd50e7e20

	/*
	* sys #6, c7, c14, #3, x0
	* DC CIDGPAPA, X0
	*/
	#define dc_cigdpapa_x0 0xd50e7ea0

	/*
	* This macro can be used for implementing various data cache operations `op`
	*/
	.macro do_dcache_maintenance_by_mva op
	/* Exit early if size is zero */
	cbz x1, exit_loop_\op
	dcache_line_size x2, x3
	add x1, x0, x1
	sub x3, x2, #1
	bic x0, x0, x3
	loop_\op:
	dc \op, x0
	add x0, x0, x2
	cmp x0, x1
	b.lo loop_\op
	dsb sy
	exit_loop_\op:
	ret
	.endm

	/* op: the hexadecimal instruction opcode for the cache operation */
	.macro do_dcache_maintenance_instr op
	/* Exit early if size is zero */
	cbz x1, exit_loop_\op
	dcache_line_size x2, x3
	sub x3, x2, #1
	bic x0, x0, x3
	add x1, x1, x0
	loop_\op:
	.inst \op
	add x0, x0, x2
	cmp x0, x1
	b.lo loop_\op
	dsb osh
	exit_loop_\op:
	ret
	.endm

	.macro check_plat_can_cmo
	#if CONDITIONAL_CMO
	mov x3, x30
	mov x2, x0
	bl plat_can_cmo
	mov x30, x3
	cbnz x0, 1f
	ret
	1:
	mov x0, x2
	#endif
	.endm

	/* -------------------------------------------
	* DCache Clean+Invalidate by MVA from base
	* address till size. 'x0' = addr, 'x1' = size
	* -------------------------------------------
	*/
	func flush_dcache_range
	check_plat_can_cmo
	do_dcache_maintenance_by_mva civac
	endfunc flush_dcache_range

	/* ------------------------------------------
	* DCache Clean by MVA from base address till
	* size. 'x0' = addr, 'x1' = size
	* ------------------------------------------
	*/
	func clean_dcache_range
	check_plat_can_cmo
	do_dcache_maintenance_by_mva cvac
	endfunc clean_dcache_range

	/* ------------------------------------------
	* DCache Invalidate by MVA from base address
	* till size. 'x0' = addr, 'x1' = size
	* ------------------------------------------
	*/
	func inv_dcache_range
	check_plat_can_cmo
	do_dcache_maintenance_by_mva ivac
	endfunc inv_dcache_range

	/*
	* ------------------------------------------
	* DCache Clean+Invalidate by PA to POPA from
	* base address till size.
	* 'x0' = addr, 'x1' = size
	* ------------------------------------------
	*/
	func flush_dcache_to_popa_range
	check_plat_can_cmo
	/* dc cipapa, x0 */
	do_dcache_maintenance_instr dc_cipapa_x0
	endfunc flush_dcache_to_popa_range

	/*
	* ------------------------------------------
	* Clean+Invalidate by PA to POPA (MTE2)
	* from base address till size.
	* 'x0' = addr, 'x1' = size
	* ------------------------------------------
	* On implementations with FEAT_MTE2, Root firmware must issue
	* DC_CIGDPAPA instead of DC_CIPAPA, in order to additionally
	* clean and invalidate Allocation Tags associated with the
	* affected locations.
	*/
	func flush_dcache_to_popa_range_mte2
	check_plat_can_cmo
	/* dc cigdpapa, x0 */
	do_dcache_maintenance_instr dc_cigdpapa_x0
	endfunc flush_dcache_to_popa_range_mte2

	/* ---------------------------------------------------------------
	* Data cache operations by set/way to the level specified
	*
	* The main function, do_dcsw_op requires:
	* x0: The operation type (0-2), as defined in arch.h
	* x3: The last cache level to operate on
	* x9: clidr_el1
	* x10: The cache level to begin operation from
	* and will carry out the operation on each data cache from level 0
	* to the level in x3 in sequence
	*
	* The dcsw_op macro sets up the x3 and x9 parameters based on
	* clidr_el1 cache information before invoking the main function
	* ---------------------------------------------------------------
	*/

	.macro dcsw_op shift, fw, ls
	mrs x9, clidr_el1
	ubfx x3, x9, \shift, \fw
	lsl x3, x3, \ls
	mov x10, xzr
	b do_dcsw_op
	.endm

	func do_dcsw_op
	cbz x3, exit
	mrs x12, ID_AA64MMFR2_EL1 // stash FEAT_CCIDX identifier in x12
	ubfx x12, x12, #ID_AA64MMFR2_EL1_CCIDX_SHIFT, #ID_AA64MMFR2_EL1_CCIDX_LENGTH
	adr x14, dcsw_loop_table // compute inner loop address
	add x14, x14, x0, lsl #5 // inner loop is 8x32-bit instructions
	#if ENABLE_BTI
	add x14, x14, x0, lsl #2 // inner loop is + "bti j" instruction
	#endif
	mov x0, x9
	mov w8, #1
	loop1:
	add x2, x10, x10, lsr #1 // work out 3x current cache level
	lsr x1, x0, x2 // extract cache type bits from clidr
	and x1, x1, #7 // mask the bits for current cache only
	cmp x1, #2 // see what cache we have at this level
	b.lo level_done // nothing to do if no cache or icache

	msr csselr_el1, x10 // select current cache level in csselr
	isb // isb to sych the new cssr&csidr
	mrs x1, ccsidr_el1 // read the new ccsidr
	and x2, x1, #7 // extract the length of the cache lines
	add x2, x2, #4 // add 4 (line length offset)

	cbz x12, 1f // check for FEAT_CCIDX for Associativity
	ubfx x4, x1, #3, #21 // x4 = associativity CCSIDR_EL1[23:3]
	b 2f
	1:
	ubfx x4, x1, #3, #10 // x4 = associativity CCSIDR_EL1[12:3]
	2:
	clz w5, w4 // bit position of way size increment
	lsl w9, w4, w5 // w9 = aligned max way number
	lsl w16, w8, w5 // w16 = way number loop decrement
	orr w9, w10, w9 // w9 = combine way and cache number

	cbz x12, 3f // check for FEAT_CCIDX for NumSets
	ubfx x6, x1, #32, #24 // x6 (w6) = numsets CCSIDR_EL1[55:32]
	// ISA will not allow x->w ubfx
	b 4f
	3:
	ubfx w6, w1, #13, #15 // w6 = numsets CCSIDR_EL1[27:13]
	4:
	lsl w17, w8, w2 // w17 = set number loop decrement
	dsb sy // barrier before we start this level
	br x14 // jump to DC operation specific loop

	.macro dcsw_loop _op
	#if ENABLE_BTI
	bti j
	#endif
	loop2_\_op:
	lsl w7, w6, w2 // w7 = aligned max set number

	loop3_\_op:
	orr w11, w9, w7 // combine cache, way and set number
	dc \_op, x11
	subs w7, w7, w17 // decrement set number
	b.hs loop3_\_op

	subs x9, x9, x16 // decrement way number
	b.hs loop2_\_op

	b level_done
	.endm

	level_done:
	add x10, x10, #2 // increment cache number
	cmp x3, x10
	b.hi loop1
	msr csselr_el1, xzr // select cache level 0 in csselr
	dsb sy // barrier to complete final cache operation
	isb
	exit:
	ret
	endfunc do_dcsw_op

	dcsw_loop_table:
	dcsw_loop isw
	dcsw_loop cisw
	dcsw_loop csw


	func dcsw_op_louis
	check_plat_can_cmo
	dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
	endfunc dcsw_op_louis


	func dcsw_op_all
	check_plat_can_cmo
	dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
	endfunc dcsw_op_all

	/* ---------------------------------------------------------------
	* Helper macro for data cache operations by set/way for the
	* level specified
	* ---------------------------------------------------------------
	*/
	.macro dcsw_op_level level
	mrs x9, clidr_el1
	mov x3, \level
	sub x10, x3, #2
	b do_dcsw_op
	.endm

	/* ---------------------------------------------------------------
	* Data cache operations by set/way for level 1 cache
	*
	* The main function, do_dcsw_op requires:
	* x0: The operation type (0-2), as defined in arch.h
	* ---------------------------------------------------------------
	*/
	func dcsw_op_level1
	check_plat_can_cmo
	dcsw_op_level #(1 << LEVEL_SHIFT)
	endfunc dcsw_op_level1

	/* ---------------------------------------------------------------
	* Data cache operations by set/way for level 2 cache
	*
	* The main function, do_dcsw_op requires:
	* x0: The operation type (0-2), as defined in arch.h
	* ---------------------------------------------------------------
	*/
	func dcsw_op_level2
	check_plat_can_cmo
	dcsw_op_level #(2 << LEVEL_SHIFT)
	endfunc dcsw_op_level2

	/* ---------------------------------------------------------------
	* Data cache operations by set/way for level 3 cache
	*
	* The main function, do_dcsw_op requires:
	* x0: The operation type (0-2), as defined in arch.h
	* ---------------------------------------------------------------
	*/
	func dcsw_op_level3
	check_plat_can_cmo
	dcsw_op_level #(3 << LEVEL_SHIFT)
	endfunc dcsw_op_level3