diff --git a/arch/arm/lib/bootm.c b/arch/arm/lib/bootm.c
index 47ee070593b7fed5094df4312f6994fc2c205cec..304210effdcd510d6efc16646efc5cf0a7a71f9e 100644
--- a/arch/arm/lib/bootm.c
+++ b/arch/arm/lib/bootm.c
@@ -242,7 +242,6 @@ static void boot_prep_linux(bootm_headers_t *images)
 		printf("FDT and ATAGS support not compiled in - hanging\n");
 		hang();
 	}
-	do_nonsec_virt_switch();
 }
 
 /* Subcommand: GO */
@@ -260,8 +259,10 @@ static void boot_jump_linux(bootm_headers_t *images, int flag)
 
 	announce_and_cleanup(fake);
 
-	if (!fake)
+	if (!fake) {
+		do_nonsec_virt_switch();
 		kernel_entry(images->ft_addr);
+	}
 #else
 	unsigned long machid = gd->bd->bi_arch_number;
 	char *s;
@@ -287,8 +288,10 @@ static void boot_jump_linux(bootm_headers_t *images, int flag)
 	else
 		r2 = gd->bd->bi_boot_params;
 
-	if (!fake)
+	if (!fake) {
+		do_nonsec_virt_switch();
 		kernel_entry(0, machid, r2);
+	}
 #endif
 }