diff --git a/arch/arm/cpu/armv7/am33xx/ddr.c b/arch/arm/cpu/armv7/am33xx/ddr.c
index ef1fc4dc13d78386b7b662baba9da010bb728228..6acf30c5db00fe4e363a0fe449e1193cc8306643 100644
--- a/arch/arm/cpu/armv7/am33xx/ddr.c
+++ b/arch/arm/cpu/armv7/am33xx/ddr.c
@@ -292,19 +292,14 @@ static void ext_phy_settings_hwlvl(const struct emif_regs *regs, int nr)
 void config_ddr_phy(const struct emif_regs *regs, int nr)
 {
 	/*
-	 * Disable initialization and refreshes for now until we
-	 * finish programming EMIF regs.
-	 * Also set time between rising edge of DDR_RESET to rising
-	 * edge of DDR_CKE to > 500us per memory spec.
+	 * Disable initialization and refreshes for now until we finish
+	 * programming EMIF regs and set time between rising edge of
+	 * DDR_RESET to rising edge of DDR_CKE to > 500us per memory spec.
+	 * We currently hardcode a value based on a max expected frequency
+	 * of 400MHz.
 	 */
-#ifndef CONFIG_AM43XX
-	setbits_le32(&emif_reg[nr]->emif_sdram_ref_ctrl,
-		     EMIF_REG_INITREF_DIS_MASK);
-#endif
-	if (regs->zq_config)
-		/* Set time between rising edge of DDR_RESET to rising
-		 * edge of DDR_CKE to > 500us per memory spec. */
-		writel(0x00003100, &emif_reg[nr]->emif_sdram_ref_ctrl);
+	writel(EMIF_REG_INITREF_DIS_MASK | 0x3100,
+		&emif_reg[nr]->emif_sdram_ref_ctrl);
 
 	writel(regs->emif_ddr_phy_ctlr_1,
 		&emif_reg[nr]->emif_ddr_phy_ctrl_1);