diff --git a/arch/arm/cpu/at91-common/spl_atmel.c b/arch/arm/cpu/at91-common/spl_atmel.c
index fdea46665f80f76f4e4da7ede1c7e0bb46dcacdb..9cc1111234a16f723519281bbd3fd4f7aae7a531 100644
--- a/arch/arm/cpu/at91-common/spl_atmel.c
+++ b/arch/arm/cpu/at91-common/spl_atmel.c
@@ -51,11 +51,13 @@ static void switch_to_main_crystal_osc(void)
 	while (!(readl(&pmc->mcfr) & AT91_PMC_MAINRDY))
 		;
 
+#ifndef CONFIG_SAMA5D4
 	tmp = readl(&pmc->mor);
 	tmp &= ~AT91_PMC_MOR_MOSCRCEN;
 	tmp &= ~AT91_PMC_MOR_KEY(0xff);
 	tmp |= AT91_PMC_MOR_KEY(0x37);
 	writel(tmp, &pmc->mor);
+#endif
 }
 
 __weak void matrix_init(void)