drivers/memory/ti-emif-sram-pm.S - linux - Git at Google

 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
  * Low level PM code for TI EMIF
  *
  * Copyright (C) 2016-2017 Texas Instruments Incorporated - http://www.ti.com/
  *	Dave Gerlach
  */

 #include <linux/linkage.h>
 #include <asm/assembler.h>
 #include <asm/memory.h>

 #include "emif.h"
 #include "ti-emif-asm-offsets.h"

 #define EMIF_POWER_MGMT_WAIT_SELF_REFRESH_8192_CYCLES	0x00a0
 #define EMIF_POWER_MGMT_SR_TIMER_MASK			0x00f0
 #define EMIF_POWER_MGMT_SELF_REFRESH_MODE		0x0200
 #define EMIF_POWER_MGMT_SELF_REFRESH_MODE_MASK		0x0700

 #define EMIF_SDCFG_TYPE_DDR2				0x2 << SDRAM_TYPE_SHIFT
 #define EMIF_SDCFG_TYPE_DDR3				0x3 << SDRAM_TYPE_SHIFT
 #define EMIF_STATUS_READY				0x4

 #define AM43XX_EMIF_PHY_CTRL_REG_COUNT                  0x120

 #define EMIF_AM437X_REGISTERS				0x1

 	.arm
 	.align 3
 	.arch armv7-a

 ENTRY(ti_emif_sram)

 /*
  * void ti_emif_save_context(void)
  *
  * Used during suspend to save the context of all required EMIF registers
  * to local memory if the EMIF is going to lose context during the sleep
  * transition. Operates on the VIRTUAL address of the EMIF.
  */
 ENTRY(ti_emif_save_context)
 	stmfd   sp!, {r4 - r11, lr}     @ save registers on stack

 	adr	r4, ti_emif_pm_sram_data
 	ldr	r0, [r4, #EMIF_PM_BASE_ADDR_VIRT_OFFSET]
 	ldr	r2, [r4, #EMIF_PM_REGS_VIRT_OFFSET]

 	/* Save EMIF configuration */
 	ldr	r1, [r0, #EMIF_SDRAM_CONFIG]
 	str	r1, [r2, #EMIF_SDCFG_VAL_OFFSET]

 	ldr	r1, [r0, #EMIF_SDRAM_REFRESH_CONTROL]
 	str	r1, [r2, #EMIF_REF_CTRL_VAL_OFFSET]

 	ldr	r1, [r0, #EMIF_SDRAM_TIMING_1]
 	str     r1, [r2, #EMIF_TIMING1_VAL_OFFSET]

 	ldr	r1, [r0, #EMIF_SDRAM_TIMING_2]
 	str     r1, [r2, #EMIF_TIMING2_VAL_OFFSET]

 	ldr	r1, [r0, #EMIF_SDRAM_TIMING_3]
 	str     r1, [r2, #EMIF_TIMING3_VAL_OFFSET]

 	ldr	r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
 	str     r1, [r2, #EMIF_PMCR_VAL_OFFSET]

 	ldr	r1, [r0, #EMIF_POWER_MANAGEMENT_CTRL_SHDW]
 	str     r1, [r2, #EMIF_PMCR_SHDW_VAL_OFFSET]

 	ldr	r1, [r0, #EMIF_SDRAM_OUTPUT_IMPEDANCE_CALIBRATION_CONFIG]
 	str     r1, [r2, #EMIF_ZQCFG_VAL_OFFSET]

 	ldr	r1, [r0, #EMIF_DDR_PHY_CTRL_1]
 	str     r1, [r2, #EMIF_DDR_PHY_CTLR_1_OFFSET]

 	ldr	r1, [r0, #EMIF_COS_CONFIG]
 	str     r1, [r2, #EMIF_COS_CONFIG_OFFSET]

 	ldr	r1, [r0, #EMIF_PRIORITY_TO_CLASS_OF_SERVICE_MAPPING]
 	str     r1, [r2, #EMIF_PRIORITY_TO_COS_MAPPING_OFFSET]

 	ldr	r1, [r0, #EMIF_CONNECTION_ID_TO_CLASS_OF_SERVICE_1_MAPPING]
 	str     r1, [r2, #EMIF_CONNECT_ID_SERV_1_MAP_OFFSET]

 	ldr	r1, [r0, #EMIF_CONNECTION_ID_TO_CLASS_OF_SERVICE_2_MAPPING]
 	str     r1, [r2, #EMIF_CONNECT_ID_SERV_2_MAP_OFFSET]

 	ldr	r1, [r0, #EMIF_OCP_CONFIG]
 	str     r1, [r2, #EMIF_OCP_CONFIG_VAL_OFFSET]

 	ldr	r5, [r4, #EMIF_PM_CONFIG_OFFSET]
 	cmp	r5, #EMIF_SRAM_AM43_REG_LAYOUT
 	bne	emif_skip_save_extra_regs

 	ldr	r1, [r0, #EMIF_READ_WRITE_LEVELING_RAMP_CONTROL]
 	str     r1, [r2, #EMIF_RD_WR_LEVEL_RAMP_CTRL_OFFSET]

 	ldr	r1, [r0, #EMIF_READ_WRITE_EXECUTION_THRESHOLD]
 	str     r1, [r2, #EMIF_RD_WR_EXEC_THRESH_OFFSET]

 	ldr	r1, [r0, #EMIF_LPDDR2_NVM_TIMING]
 	str     r1, [r2, #EMIF_LPDDR2_NVM_TIM_OFFSET]

 	ldr	r1, [r0, #EMIF_LPDDR2_NVM_TIMING_SHDW]
 	str     r1, [r2, #EMIF_LPDDR2_NVM_TIM_SHDW_OFFSET]

 	ldr	r1, [r0, #EMIF_DLL_CALIB_CTRL]
 	str     r1, [r2, #EMIF_DLL_CALIB_CTRL_VAL_OFFSET]

 	ldr	r1, [r0, #EMIF_DLL_CALIB_CTRL_SHDW]
 	str     r1, [r2, #EMIF_DLL_CALIB_CTRL_VAL_SHDW_OFFSET]

 	/* Loop and save entire block of emif phy regs */
 	mov	r5, #0x0
 	add	r4, r2, #EMIF_EXT_PHY_CTRL_VALS_OFFSET
 	add	r3, r0, #EMIF_EXT_PHY_CTRL_1
 ddr_phy_ctrl_save:
 	ldr	r1, [r3, r5]
 	str	r1, [r4, r5]
 	add	r5, r5, #0x4
 	cmp	r5, #AM43XX_EMIF_PHY_CTRL_REG_COUNT
 	bne	ddr_phy_ctrl_save

 emif_skip_save_extra_regs:
 	ldmfd	sp!, {r4 - r11, pc}	@ restore regs and return
 ENDPROC(ti_emif_save_context)

 /*
  * void ti_emif_restore_context(void)
  *
  * Used during resume to restore the context of all required EMIF registers
  * from local memory after the EMIF has lost context during a sleep transition.
  * Operates on the PHYSICAL address of the EMIF.
  */
 ENTRY(ti_emif_restore_context)
 	adr	r4, ti_emif_pm_sram_data
 	ldr	r0, [r4, #EMIF_PM_BASE_ADDR_PHYS_OFFSET]
 	ldr	r2, [r4, #EMIF_PM_REGS_PHYS_OFFSET]

 	/* Config EMIF Timings */
 	ldr     r1, [r2, #EMIF_DDR_PHY_CTLR_1_OFFSET]
 	str	r1, [r0, #EMIF_DDR_PHY_CTRL_1]
 	str	r1, [r0, #EMIF_DDR_PHY_CTRL_1_SHDW]

 	ldr     r1, [r2, #EMIF_TIMING1_VAL_OFFSET]
 	str	r1, [r0, #EMIF_SDRAM_TIMING_1]
 	str	r1, [r0, #EMIF_SDRAM_TIMING_1_SHDW]

 	ldr     r1, [r2, #EMIF_TIMING2_VAL_OFFSET]
 	str	r1, [r0, #EMIF_SDRAM_TIMING_2]
 	str	r1, [r0, #EMIF_SDRAM_TIMING_2_SHDW]

 	ldr     r1, [r2, #EMIF_TIMING3_VAL_OFFSET]
 	str	r1, [r0, #EMIF_SDRAM_TIMING_3]
 	str	r1, [r0, #EMIF_SDRAM_TIMING_3_SHDW]

 	ldr     r1, [r2, #EMIF_REF_CTRL_VAL_OFFSET]
 	str	r1, [r0, #EMIF_SDRAM_REFRESH_CONTROL]
 	str	r1, [r0, #EMIF_SDRAM_REFRESH_CTRL_SHDW]

 	ldr     r1, [r2, #EMIF_PMCR_VAL_OFFSET]
 	str	r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]

 	ldr     r1, [r2, #EMIF_PMCR_SHDW_VAL_OFFSET]
 	str	r1, [r0, #EMIF_POWER_MANAGEMENT_CTRL_SHDW]

 	ldr     r1, [r2, #EMIF_COS_CONFIG_OFFSET]
 	str	r1, [r0, #EMIF_COS_CONFIG]

 	ldr     r1, [r2, #EMIF_PRIORITY_TO_COS_MAPPING_OFFSET]
 	str	r1, [r0, #EMIF_PRIORITY_TO_CLASS_OF_SERVICE_MAPPING]

 	ldr	r1, [r2, #EMIF_CONNECT_ID_SERV_1_MAP_OFFSET]
 	str	r1, [r0, #EMIF_CONNECTION_ID_TO_CLASS_OF_SERVICE_1_MAPPING]

 	ldr     r1, [r2, #EMIF_CONNECT_ID_SERV_2_MAP_OFFSET]
 	str	r1, [r0, #EMIF_CONNECTION_ID_TO_CLASS_OF_SERVICE_2_MAPPING]

 	ldr     r1, [r2, #EMIF_OCP_CONFIG_VAL_OFFSET]
 	str	r1, [r0, #EMIF_OCP_CONFIG]

 	ldr	r5, [r4, #EMIF_PM_CONFIG_OFFSET]
 	cmp	r5, #EMIF_SRAM_AM43_REG_LAYOUT
 	bne	emif_skip_restore_extra_regs

 	ldr     r1, [r2, #EMIF_RD_WR_LEVEL_RAMP_CTRL_OFFSET]
 	str	r1, [r0, #EMIF_READ_WRITE_LEVELING_RAMP_CONTROL]

 	ldr     r1, [r2, #EMIF_RD_WR_EXEC_THRESH_OFFSET]
 	str	r1, [r0, #EMIF_READ_WRITE_EXECUTION_THRESHOLD]

 	ldr     r1, [r2, #EMIF_LPDDR2_NVM_TIM_OFFSET]
 	str	r1, [r0, #EMIF_LPDDR2_NVM_TIMING]

 	ldr     r1, [r2, #EMIF_LPDDR2_NVM_TIM_SHDW_OFFSET]
 	str	r1, [r0, #EMIF_LPDDR2_NVM_TIMING_SHDW]

 	ldr     r1, [r2, #EMIF_DLL_CALIB_CTRL_VAL_OFFSET]
 	str	r1, [r0, #EMIF_DLL_CALIB_CTRL]

 	ldr     r1, [r2, #EMIF_DLL_CALIB_CTRL_VAL_SHDW_OFFSET]
 	str	r1, [r0, #EMIF_DLL_CALIB_CTRL_SHDW]

 	ldr     r1, [r2, #EMIF_ZQCFG_VAL_OFFSET]
 	str	r1, [r0, #EMIF_SDRAM_OUTPUT_IMPEDANCE_CALIBRATION_CONFIG]

 	/* Loop and restore entire block of emif phy regs */
 	mov	r5, #0x0
 	/* Load ti_emif_regs_amx3 + EMIF_EXT_PHY_CTRL_VALS_OFFSET for address
 	 * to phy register save space
 	 */
 	add	r3, r2, #EMIF_EXT_PHY_CTRL_VALS_OFFSET
 	add	r4, r0, #EMIF_EXT_PHY_CTRL_1
 ddr_phy_ctrl_restore:
 	ldr	r1, [r3, r5]
 	str	r1, [r4, r5]
 	add	r5, r5, #0x4
 	cmp	r5, #AM43XX_EMIF_PHY_CTRL_REG_COUNT
 	bne	ddr_phy_ctrl_restore

 emif_skip_restore_extra_regs:
 	/*
 	 * Output impedence calib needed only for DDR3
 	 * but since the initial state of this will be
 	 * disabled for DDR2 no harm in restoring the
 	 * old configuration
 	 */
 	ldr     r1, [r2, #EMIF_ZQCFG_VAL_OFFSET]
 	str	r1, [r0, #EMIF_SDRAM_OUTPUT_IMPEDANCE_CALIBRATION_CONFIG]

 	/* Write to sdcfg last for DDR2 only */
 	ldr	r1, [r2, #EMIF_SDCFG_VAL_OFFSET]
 	and	r2, r1, #SDRAM_TYPE_MASK
 	cmp	r2, #EMIF_SDCFG_TYPE_DDR2
 	streq	r1, [r0, #EMIF_SDRAM_CONFIG]

 	mov	pc, lr
 ENDPROC(ti_emif_restore_context)

 /*
  * void ti_emif_run_hw_leveling(void)
  *
  * Used during resume to run hardware leveling again and restore the
  * configuration of the EMIF PHY, only for DDR3.
  */
 ENTRY(ti_emif_run_hw_leveling)
 	adr	r4, ti_emif_pm_sram_data
 	ldr	r0, [r4, #EMIF_PM_BASE_ADDR_PHYS_OFFSET]

 	ldr	r3, [r0, #EMIF_READ_WRITE_LEVELING_CONTROL]
 	orr	r3, r3, #RDWRLVLFULL_START
 	ldr	r2, [r0, #EMIF_SDRAM_CONFIG]
 	and	r2, r2, #SDRAM_TYPE_MASK
 	cmp	r2, #EMIF_SDCFG_TYPE_DDR3
 	bne	skip_hwlvl

 	str	r3, [r0, #EMIF_READ_WRITE_LEVELING_CONTROL]

 	/*
 	 * If EMIF registers are touched during initial stage of HW
 	 * leveling sequence there will be an L3 NOC timeout error issued
 	 * as the EMIF will not respond, which is not fatal, but it is
 	 * avoidable. This small wait loop is enough time for this condition
 	 * to clear, even at worst case of CPU running at max speed of 1Ghz.
 	 */
 	mov	r2, #0x2000
 1:
 	subs	r2, r2, #0x1
 	bne	1b

 	/* Bit clears when operation is complete */
 2:	ldr     r1, [r0, #EMIF_READ_WRITE_LEVELING_CONTROL]
 	tst     r1, #RDWRLVLFULL_START
 	bne     2b

 skip_hwlvl:
 	mov	pc, lr
 ENDPROC(ti_emif_run_hw_leveling)

 /*
  * void ti_emif_enter_sr(void)
  *
  * Programs the EMIF to tell the SDRAM to enter into self-refresh
  * mode during a sleep transition. Operates on the VIRTUAL address
  * of the EMIF.
  */
 ENTRY(ti_emif_enter_sr)
 	stmfd   sp!, {r4 - r11, lr}     @ save registers on stack

 	adr	r4, ti_emif_pm_sram_data
 	ldr	r0, [r4, #EMIF_PM_BASE_ADDR_VIRT_OFFSET]
 	ldr	r2, [r4, #EMIF_PM_REGS_VIRT_OFFSET]

 	ldr	r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
 	bic	r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE_MASK
 	orr	r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE
 	str	r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]

 	ldmfd	sp!, {r4 - r11, pc}	@ restore regs and return
 ENDPROC(ti_emif_enter_sr)

 /*
  * void ti_emif_exit_sr(void)
  *
  * Programs the EMIF to tell the SDRAM to exit self-refresh mode
  * after a sleep transition. Operates on the PHYSICAL address of
  * the EMIF.
  */
 ENTRY(ti_emif_exit_sr)
 	adr	r4, ti_emif_pm_sram_data
 	ldr	r0, [r4, #EMIF_PM_BASE_ADDR_PHYS_OFFSET]
 	ldr	r2, [r4, #EMIF_PM_REGS_PHYS_OFFSET]

 	/*
 	 * Toggle EMIF to exit refresh mode:
 	 * if EMIF lost context, PWR_MGT_CTRL is currently 0, writing disable
 	 *   (0x0), wont do diddly squat! so do a toggle from SR(0x2) to disable
 	 *   (0x0) here.
 	 * *If* EMIF did not lose context, nothing broken as we write the same
 	 *   value(0x2) to reg before we write a disable (0x0).
 	 */
 	ldr	r1, [r2, #EMIF_PMCR_VAL_OFFSET]
 	bic	r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE_MASK
 	orr	r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE
 	str	r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
 	bic	r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE_MASK
 	str	r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]

         /* Wait for EMIF to become ready */
 1:	ldr     r1, [r0, #EMIF_STATUS]
 	tst     r1, #EMIF_STATUS_READY
 	beq     1b

 	mov	pc, lr
 ENDPROC(ti_emif_exit_sr)

 /*
  * void ti_emif_abort_sr(void)
  *
  * Disables self-refresh after a failed transition to a low-power
  * state so the kernel can jump back to DDR and follow abort path.
  * Operates on the VIRTUAL address of the EMIF.
  */
 ENTRY(ti_emif_abort_sr)
 	stmfd   sp!, {r4 - r11, lr}     @ save registers on stack

 	adr	r4, ti_emif_pm_sram_data
 	ldr	r0, [r4, #EMIF_PM_BASE_ADDR_VIRT_OFFSET]
 	ldr	r2, [r4, #EMIF_PM_REGS_VIRT_OFFSET]

 	ldr	r1, [r2, #EMIF_PMCR_VAL_OFFSET]
 	bic	r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE_MASK
 	str	r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]

 	/* Wait for EMIF to become ready */
 1:	ldr     r1, [r0, #EMIF_STATUS]
 	tst     r1, #EMIF_STATUS_READY
 	beq     1b

 	ldmfd	sp!, {r4 - r11, pc}	@ restore regs and return
 ENDPROC(ti_emif_abort_sr)

 	.align 3
 ENTRY(ti_emif_pm_sram_data)
 	.space EMIF_PM_DATA_SIZE
 ENTRY(ti_emif_sram_sz)
         .word   . - ti_emif_save_context
	/* SPDX-License-Identifier: GPL-2.0-only */
	/*
	* Low level PM code for TI EMIF
	*
	* Copyright (C) 2016-2017 Texas Instruments Incorporated - http://www.ti.com/
	* Dave Gerlach
	*/

	#include <linux/linkage.h>
	#include <asm/assembler.h>
	#include <asm/memory.h>

	#include "emif.h"
	#include "ti-emif-asm-offsets.h"

	#define EMIF_POWER_MGMT_WAIT_SELF_REFRESH_8192_CYCLES 0x00a0
	#define EMIF_POWER_MGMT_SR_TIMER_MASK 0x00f0
	#define EMIF_POWER_MGMT_SELF_REFRESH_MODE 0x0200
	#define EMIF_POWER_MGMT_SELF_REFRESH_MODE_MASK 0x0700

	#define EMIF_SDCFG_TYPE_DDR2 0x2 << SDRAM_TYPE_SHIFT
	#define EMIF_SDCFG_TYPE_DDR3 0x3 << SDRAM_TYPE_SHIFT
	#define EMIF_STATUS_READY 0x4

	#define AM43XX_EMIF_PHY_CTRL_REG_COUNT 0x120

	#define EMIF_AM437X_REGISTERS 0x1

	.arm
	.align 3
	.arch armv7-a

	ENTRY(ti_emif_sram)

	/*
	* void ti_emif_save_context(void)
	*
	* Used during suspend to save the context of all required EMIF registers
	* to local memory if the EMIF is going to lose context during the sleep
	* transition. Operates on the VIRTUAL address of the EMIF.
	*/
	ENTRY(ti_emif_save_context)
	stmfd sp!, {r4 - r11, lr} @ save registers on stack

	adr r4, ti_emif_pm_sram_data
	ldr r0, [r4, #EMIF_PM_BASE_ADDR_VIRT_OFFSET]
	ldr r2, [r4, #EMIF_PM_REGS_VIRT_OFFSET]

	/* Save EMIF configuration */
	ldr r1, [r0, #EMIF_SDRAM_CONFIG]
	str r1, [r2, #EMIF_SDCFG_VAL_OFFSET]

	ldr r1, [r0, #EMIF_SDRAM_REFRESH_CONTROL]
	str r1, [r2, #EMIF_REF_CTRL_VAL_OFFSET]

	ldr r1, [r0, #EMIF_SDRAM_TIMING_1]
	str r1, [r2, #EMIF_TIMING1_VAL_OFFSET]

	ldr r1, [r0, #EMIF_SDRAM_TIMING_2]
	str r1, [r2, #EMIF_TIMING2_VAL_OFFSET]

	ldr r1, [r0, #EMIF_SDRAM_TIMING_3]
	str r1, [r2, #EMIF_TIMING3_VAL_OFFSET]

	ldr r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
	str r1, [r2, #EMIF_PMCR_VAL_OFFSET]

	ldr r1, [r0, #EMIF_POWER_MANAGEMENT_CTRL_SHDW]
	str r1, [r2, #EMIF_PMCR_SHDW_VAL_OFFSET]

	ldr r1, [r0, #EMIF_SDRAM_OUTPUT_IMPEDANCE_CALIBRATION_CONFIG]
	str r1, [r2, #EMIF_ZQCFG_VAL_OFFSET]

	ldr r1, [r0, #EMIF_DDR_PHY_CTRL_1]
	str r1, [r2, #EMIF_DDR_PHY_CTLR_1_OFFSET]

	ldr r1, [r0, #EMIF_COS_CONFIG]
	str r1, [r2, #EMIF_COS_CONFIG_OFFSET]

	ldr r1, [r0, #EMIF_PRIORITY_TO_CLASS_OF_SERVICE_MAPPING]
	str r1, [r2, #EMIF_PRIORITY_TO_COS_MAPPING_OFFSET]

	ldr r1, [r0, #EMIF_CONNECTION_ID_TO_CLASS_OF_SERVICE_1_MAPPING]
	str r1, [r2, #EMIF_CONNECT_ID_SERV_1_MAP_OFFSET]

	ldr r1, [r0, #EMIF_CONNECTION_ID_TO_CLASS_OF_SERVICE_2_MAPPING]
	str r1, [r2, #EMIF_CONNECT_ID_SERV_2_MAP_OFFSET]

	ldr r1, [r0, #EMIF_OCP_CONFIG]
	str r1, [r2, #EMIF_OCP_CONFIG_VAL_OFFSET]

	ldr r5, [r4, #EMIF_PM_CONFIG_OFFSET]
	cmp r5, #EMIF_SRAM_AM43_REG_LAYOUT
	bne emif_skip_save_extra_regs

	ldr r1, [r0, #EMIF_READ_WRITE_LEVELING_RAMP_CONTROL]
	str r1, [r2, #EMIF_RD_WR_LEVEL_RAMP_CTRL_OFFSET]

	ldr r1, [r0, #EMIF_READ_WRITE_EXECUTION_THRESHOLD]
	str r1, [r2, #EMIF_RD_WR_EXEC_THRESH_OFFSET]

	ldr r1, [r0, #EMIF_LPDDR2_NVM_TIMING]
	str r1, [r2, #EMIF_LPDDR2_NVM_TIM_OFFSET]

	ldr r1, [r0, #EMIF_LPDDR2_NVM_TIMING_SHDW]
	str r1, [r2, #EMIF_LPDDR2_NVM_TIM_SHDW_OFFSET]

	ldr r1, [r0, #EMIF_DLL_CALIB_CTRL]
	str r1, [r2, #EMIF_DLL_CALIB_CTRL_VAL_OFFSET]

	ldr r1, [r0, #EMIF_DLL_CALIB_CTRL_SHDW]
	str r1, [r2, #EMIF_DLL_CALIB_CTRL_VAL_SHDW_OFFSET]

	/* Loop and save entire block of emif phy regs */
	mov r5, #0x0
	add r4, r2, #EMIF_EXT_PHY_CTRL_VALS_OFFSET
	add r3, r0, #EMIF_EXT_PHY_CTRL_1
	ddr_phy_ctrl_save:
	ldr r1, [r3, r5]
	str r1, [r4, r5]
	add r5, r5, #0x4
	cmp r5, #AM43XX_EMIF_PHY_CTRL_REG_COUNT
	bne ddr_phy_ctrl_save

	emif_skip_save_extra_regs:
	ldmfd sp!, {r4 - r11, pc} @ restore regs and return
	ENDPROC(ti_emif_save_context)

	/*
	* void ti_emif_restore_context(void)
	*
	* Used during resume to restore the context of all required EMIF registers
	* from local memory after the EMIF has lost context during a sleep transition.
	* Operates on the PHYSICAL address of the EMIF.
	*/
	ENTRY(ti_emif_restore_context)
	adr r4, ti_emif_pm_sram_data
	ldr r0, [r4, #EMIF_PM_BASE_ADDR_PHYS_OFFSET]
	ldr r2, [r4, #EMIF_PM_REGS_PHYS_OFFSET]

	/* Config EMIF Timings */
	ldr r1, [r2, #EMIF_DDR_PHY_CTLR_1_OFFSET]
	str r1, [r0, #EMIF_DDR_PHY_CTRL_1]
	str r1, [r0, #EMIF_DDR_PHY_CTRL_1_SHDW]

	ldr r1, [r2, #EMIF_TIMING1_VAL_OFFSET]
	str r1, [r0, #EMIF_SDRAM_TIMING_1]
	str r1, [r0, #EMIF_SDRAM_TIMING_1_SHDW]

	ldr r1, [r2, #EMIF_TIMING2_VAL_OFFSET]
	str r1, [r0, #EMIF_SDRAM_TIMING_2]
	str r1, [r0, #EMIF_SDRAM_TIMING_2_SHDW]

	ldr r1, [r2, #EMIF_TIMING3_VAL_OFFSET]
	str r1, [r0, #EMIF_SDRAM_TIMING_3]
	str r1, [r0, #EMIF_SDRAM_TIMING_3_SHDW]

	ldr r1, [r2, #EMIF_REF_CTRL_VAL_OFFSET]
	str r1, [r0, #EMIF_SDRAM_REFRESH_CONTROL]
	str r1, [r0, #EMIF_SDRAM_REFRESH_CTRL_SHDW]

	ldr r1, [r2, #EMIF_PMCR_VAL_OFFSET]
	str r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]

	ldr r1, [r2, #EMIF_PMCR_SHDW_VAL_OFFSET]
	str r1, [r0, #EMIF_POWER_MANAGEMENT_CTRL_SHDW]

	ldr r1, [r2, #EMIF_COS_CONFIG_OFFSET]
	str r1, [r0, #EMIF_COS_CONFIG]

	ldr r1, [r2, #EMIF_PRIORITY_TO_COS_MAPPING_OFFSET]
	str r1, [r0, #EMIF_PRIORITY_TO_CLASS_OF_SERVICE_MAPPING]

	ldr r1, [r2, #EMIF_CONNECT_ID_SERV_1_MAP_OFFSET]
	str r1, [r0, #EMIF_CONNECTION_ID_TO_CLASS_OF_SERVICE_1_MAPPING]

	ldr r1, [r2, #EMIF_CONNECT_ID_SERV_2_MAP_OFFSET]
	str r1, [r0, #EMIF_CONNECTION_ID_TO_CLASS_OF_SERVICE_2_MAPPING]

	ldr r1, [r2, #EMIF_OCP_CONFIG_VAL_OFFSET]
	str r1, [r0, #EMIF_OCP_CONFIG]

	ldr r5, [r4, #EMIF_PM_CONFIG_OFFSET]
	cmp r5, #EMIF_SRAM_AM43_REG_LAYOUT
	bne emif_skip_restore_extra_regs

	ldr r1, [r2, #EMIF_RD_WR_LEVEL_RAMP_CTRL_OFFSET]
	str r1, [r0, #EMIF_READ_WRITE_LEVELING_RAMP_CONTROL]

	ldr r1, [r2, #EMIF_RD_WR_EXEC_THRESH_OFFSET]
	str r1, [r0, #EMIF_READ_WRITE_EXECUTION_THRESHOLD]

	ldr r1, [r2, #EMIF_LPDDR2_NVM_TIM_OFFSET]
	str r1, [r0, #EMIF_LPDDR2_NVM_TIMING]

	ldr r1, [r2, #EMIF_LPDDR2_NVM_TIM_SHDW_OFFSET]
	str r1, [r0, #EMIF_LPDDR2_NVM_TIMING_SHDW]

	ldr r1, [r2, #EMIF_DLL_CALIB_CTRL_VAL_OFFSET]
	str r1, [r0, #EMIF_DLL_CALIB_CTRL]

	ldr r1, [r2, #EMIF_DLL_CALIB_CTRL_VAL_SHDW_OFFSET]
	str r1, [r0, #EMIF_DLL_CALIB_CTRL_SHDW]

	ldr r1, [r2, #EMIF_ZQCFG_VAL_OFFSET]
	str r1, [r0, #EMIF_SDRAM_OUTPUT_IMPEDANCE_CALIBRATION_CONFIG]

	/* Loop and restore entire block of emif phy regs */
	mov r5, #0x0
	/* Load ti_emif_regs_amx3 + EMIF_EXT_PHY_CTRL_VALS_OFFSET for address
	* to phy register save space
	*/
	add r3, r2, #EMIF_EXT_PHY_CTRL_VALS_OFFSET
	add r4, r0, #EMIF_EXT_PHY_CTRL_1
	ddr_phy_ctrl_restore:
	ldr r1, [r3, r5]
	str r1, [r4, r5]
	add r5, r5, #0x4
	cmp r5, #AM43XX_EMIF_PHY_CTRL_REG_COUNT
	bne ddr_phy_ctrl_restore

	emif_skip_restore_extra_regs:
	/*
	* Output impedence calib needed only for DDR3
	* but since the initial state of this will be
	* disabled for DDR2 no harm in restoring the
	* old configuration
	*/
	ldr r1, [r2, #EMIF_ZQCFG_VAL_OFFSET]
	str r1, [r0, #EMIF_SDRAM_OUTPUT_IMPEDANCE_CALIBRATION_CONFIG]

	/* Write to sdcfg last for DDR2 only */
	ldr r1, [r2, #EMIF_SDCFG_VAL_OFFSET]
	and r2, r1, #SDRAM_TYPE_MASK
	cmp r2, #EMIF_SDCFG_TYPE_DDR2
	streq r1, [r0, #EMIF_SDRAM_CONFIG]

	mov pc, lr
	ENDPROC(ti_emif_restore_context)

	/*
	* void ti_emif_run_hw_leveling(void)
	*
	* Used during resume to run hardware leveling again and restore the
	* configuration of the EMIF PHY, only for DDR3.
	*/
	ENTRY(ti_emif_run_hw_leveling)
	adr r4, ti_emif_pm_sram_data
	ldr r0, [r4, #EMIF_PM_BASE_ADDR_PHYS_OFFSET]

	ldr r3, [r0, #EMIF_READ_WRITE_LEVELING_CONTROL]
	orr r3, r3, #RDWRLVLFULL_START
	ldr r2, [r0, #EMIF_SDRAM_CONFIG]
	and r2, r2, #SDRAM_TYPE_MASK
	cmp r2, #EMIF_SDCFG_TYPE_DDR3
	bne skip_hwlvl

	str r3, [r0, #EMIF_READ_WRITE_LEVELING_CONTROL]

	/*
	* If EMIF registers are touched during initial stage of HW
	* leveling sequence there will be an L3 NOC timeout error issued
	* as the EMIF will not respond, which is not fatal, but it is
	* avoidable. This small wait loop is enough time for this condition
	* to clear, even at worst case of CPU running at max speed of 1Ghz.
	*/
	mov r2, #0x2000
	1:
	subs r2, r2, #0x1
	bne 1b

	/* Bit clears when operation is complete */
	2: ldr r1, [r0, #EMIF_READ_WRITE_LEVELING_CONTROL]
	tst r1, #RDWRLVLFULL_START
	bne 2b

	skip_hwlvl:
	mov pc, lr
	ENDPROC(ti_emif_run_hw_leveling)

	/*
	* void ti_emif_enter_sr(void)
	*
	* Programs the EMIF to tell the SDRAM to enter into self-refresh
	* mode during a sleep transition. Operates on the VIRTUAL address
	* of the EMIF.
	*/
	ENTRY(ti_emif_enter_sr)
	stmfd sp!, {r4 - r11, lr} @ save registers on stack

	adr r4, ti_emif_pm_sram_data
	ldr r0, [r4, #EMIF_PM_BASE_ADDR_VIRT_OFFSET]
	ldr r2, [r4, #EMIF_PM_REGS_VIRT_OFFSET]

	ldr r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
	bic r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE_MASK
	orr r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE
	str r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]

	ldmfd sp!, {r4 - r11, pc} @ restore regs and return
	ENDPROC(ti_emif_enter_sr)

	/*
	* void ti_emif_exit_sr(void)
	*
	* Programs the EMIF to tell the SDRAM to exit self-refresh mode
	* after a sleep transition. Operates on the PHYSICAL address of
	* the EMIF.
	*/
	ENTRY(ti_emif_exit_sr)
	adr r4, ti_emif_pm_sram_data
	ldr r0, [r4, #EMIF_PM_BASE_ADDR_PHYS_OFFSET]
	ldr r2, [r4, #EMIF_PM_REGS_PHYS_OFFSET]

	/*
	* Toggle EMIF to exit refresh mode:
	* if EMIF lost context, PWR_MGT_CTRL is currently 0, writing disable
	* (0x0), wont do diddly squat! so do a toggle from SR(0x2) to disable
	* (0x0) here.
	* If EMIF did not lose context, nothing broken as we write the same
	* value(0x2) to reg before we write a disable (0x0).
	*/
	ldr r1, [r2, #EMIF_PMCR_VAL_OFFSET]
	bic r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE_MASK
	orr r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE
	str r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
	bic r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE_MASK
	str r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]

	/* Wait for EMIF to become ready */
	1: ldr r1, [r0, #EMIF_STATUS]
	tst r1, #EMIF_STATUS_READY
	beq 1b

	mov pc, lr
	ENDPROC(ti_emif_exit_sr)

	/*
	* void ti_emif_abort_sr(void)
	*
	* Disables self-refresh after a failed transition to a low-power
	* state so the kernel can jump back to DDR and follow abort path.
	* Operates on the VIRTUAL address of the EMIF.
	*/
	ENTRY(ti_emif_abort_sr)
	stmfd sp!, {r4 - r11, lr} @ save registers on stack

	adr r4, ti_emif_pm_sram_data
	ldr r0, [r4, #EMIF_PM_BASE_ADDR_VIRT_OFFSET]
	ldr r2, [r4, #EMIF_PM_REGS_VIRT_OFFSET]

	ldr r1, [r2, #EMIF_PMCR_VAL_OFFSET]
	bic r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE_MASK
	str r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]

	/* Wait for EMIF to become ready */
	1: ldr r1, [r0, #EMIF_STATUS]
	tst r1, #EMIF_STATUS_READY
	beq 1b

	ldmfd sp!, {r4 - r11, pc} @ restore regs and return
	ENDPROC(ti_emif_abort_sr)

	.align 3
	ENTRY(ti_emif_pm_sram_data)
	.space EMIF_PM_DATA_SIZE
	ENTRY(ti_emif_sram_sz)
	.word . - ti_emif_save_context