diff --git a/arch/arm/mach-uniphier/Makefile b/arch/arm/mach-uniphier/Makefile
index 548cfe76fc5184941de8aa7565b88a58601c018c..ae78548670836b015507ac09b9a309b42bf62dec 100644
--- a/arch/arm/mach-uniphier/Makefile
+++ b/arch/arm/mach-uniphier/Makefile
@@ -4,7 +4,7 @@
 
 ifdef CONFIG_SPL_BUILD
 
-obj-y += init/ bcu/ memconf/ pll/ early-clk/
+obj-y += init/ bcu/ memconf/
 obj-$(CONFIG_MICRO_SUPPORT_CARD) += sbc/
 
 else
@@ -15,13 +15,12 @@ obj-y += board_init.o
 obj-$(CONFIG_BOARD_LATE_INIT) += board_late_init.o
 obj-y += reset.o
 
-obj-y += clk/
-
 endif
 
 obj-y += boards.o
 obj-y += soc_info.o
 obj-y += boot-mode/
+obj-y += clk/
 obj-y += dram/
 obj-y += pinctrl-glue.o
 
diff --git a/arch/arm/mach-uniphier/clk/Makefile b/arch/arm/mach-uniphier/clk/Makefile
index b7227814432fa066bfebe507d139a1e1a3b1373a..233e6591eeea816b5983efbec04660d6a0a3fded 100644
--- a/arch/arm/mach-uniphier/clk/Makefile
+++ b/arch/arm/mach-uniphier/clk/Makefile
@@ -2,6 +2,20 @@
 # SPDX-License-Identifier:	GPL-2.0+
 #
 
+ifdef CONFIG_SPL_BUILD
+
+obj-$(CONFIG_ARCH_UNIPHIER_SLD3)	+= early-clk-ld4.o dpll-sld3.o
+obj-$(CONFIG_ARCH_UNIPHIER_LD4)		+= early-clk-ld4.o dpll-ld4.o
+obj-$(CONFIG_ARCH_UNIPHIER_PRO4)	+= early-clk-ld4.o dpll-pro4.o
+obj-$(CONFIG_ARCH_UNIPHIER_SLD8)	+= early-clk-ld4.o dpll-sld8.o
+obj-$(CONFIG_ARCH_UNIPHIER_PRO5)	+= early-clk-pro5.o
+obj-$(CONFIG_ARCH_UNIPHIER_PXS2)	+= early-clk-pxs2.o
+obj-$(CONFIG_ARCH_UNIPHIER_LD6B)	+= early-clk-pxs2.o
+obj-$(CONFIG_ARCH_UNIPHIER_LD11)	+= early-clk-ld11.o
+obj-$(CONFIG_ARCH_UNIPHIER_LD20)	+= early-clk-ld20.o
+
+else
+
 obj-$(CONFIG_ARCH_UNIPHIER_SLD3)	+= clk-ld4.o pll-sld3.o dpll-tail.o
 obj-$(CONFIG_ARCH_UNIPHIER_LD4)		+= clk-ld4.o pll-ld4.o dpll-tail.o
 obj-$(CONFIG_ARCH_UNIPHIER_PRO4)	+= clk-pro4.o pll-pro4.o dpll-tail.o
@@ -11,3 +25,5 @@ obj-$(CONFIG_ARCH_UNIPHIER_PXS2)	+= clk-pxs2.o
 obj-$(CONFIG_ARCH_UNIPHIER_LD6B)	+= clk-pxs2.o
 obj-$(CONFIG_ARCH_UNIPHIER_LD11)	+= clk-ld11.o
 obj-$(CONFIG_ARCH_UNIPHIER_LD20)	+= clk-ld20.o
+
+endif
diff --git a/arch/arm/mach-uniphier/pll/pll-init-ld4.c b/arch/arm/mach-uniphier/clk/dpll-ld4.c
similarity index 100%
rename from arch/arm/mach-uniphier/pll/pll-init-ld4.c
rename to arch/arm/mach-uniphier/clk/dpll-ld4.c
diff --git a/arch/arm/mach-uniphier/pll/pll-init-pro4.c b/arch/arm/mach-uniphier/clk/dpll-pro4.c
similarity index 100%
rename from arch/arm/mach-uniphier/pll/pll-init-pro4.c
rename to arch/arm/mach-uniphier/clk/dpll-pro4.c
diff --git a/arch/arm/mach-uniphier/pll/pll-init-sld3.c b/arch/arm/mach-uniphier/clk/dpll-sld3.c
similarity index 100%
rename from arch/arm/mach-uniphier/pll/pll-init-sld3.c
rename to arch/arm/mach-uniphier/clk/dpll-sld3.c
diff --git a/arch/arm/mach-uniphier/pll/pll-init-sld8.c b/arch/arm/mach-uniphier/clk/dpll-sld8.c
similarity index 100%
rename from arch/arm/mach-uniphier/pll/pll-init-sld8.c
rename to arch/arm/mach-uniphier/clk/dpll-sld8.c
diff --git a/arch/arm/mach-uniphier/early-clk/early-clk-ld11.c b/arch/arm/mach-uniphier/clk/early-clk-ld11.c
similarity index 100%
rename from arch/arm/mach-uniphier/early-clk/early-clk-ld11.c
rename to arch/arm/mach-uniphier/clk/early-clk-ld11.c
diff --git a/arch/arm/mach-uniphier/early-clk/early-clk-ld20.c b/arch/arm/mach-uniphier/clk/early-clk-ld20.c
similarity index 100%
rename from arch/arm/mach-uniphier/early-clk/early-clk-ld20.c
rename to arch/arm/mach-uniphier/clk/early-clk-ld20.c
diff --git a/arch/arm/mach-uniphier/early-clk/early-clk-ld4.c b/arch/arm/mach-uniphier/clk/early-clk-ld4.c
similarity index 100%
rename from arch/arm/mach-uniphier/early-clk/early-clk-ld4.c
rename to arch/arm/mach-uniphier/clk/early-clk-ld4.c
diff --git a/arch/arm/mach-uniphier/early-clk/early-clk-pro5.c b/arch/arm/mach-uniphier/clk/early-clk-pro5.c
similarity index 100%
rename from arch/arm/mach-uniphier/early-clk/early-clk-pro5.c
rename to arch/arm/mach-uniphier/clk/early-clk-pro5.c
diff --git a/arch/arm/mach-uniphier/early-clk/early-clk-pxs2.c b/arch/arm/mach-uniphier/clk/early-clk-pxs2.c
similarity index 100%
rename from arch/arm/mach-uniphier/early-clk/early-clk-pxs2.c
rename to arch/arm/mach-uniphier/clk/early-clk-pxs2.c
diff --git a/arch/arm/mach-uniphier/early-clk/Makefile b/arch/arm/mach-uniphier/early-clk/Makefile
deleted file mode 100644
index 755a3618b571261d0d3266943ca9ee4261f5e183..0000000000000000000000000000000000000000
--- a/arch/arm/mach-uniphier/early-clk/Makefile
+++ /dev/null
@@ -1,13 +0,0 @@
-#
-# SPDX-License-Identifier:	GPL-2.0+
-#
-
-obj-$(CONFIG_ARCH_UNIPHIER_SLD3)	+= early-clk-ld4.o
-obj-$(CONFIG_ARCH_UNIPHIER_LD4)		+= early-clk-ld4.o
-obj-$(CONFIG_ARCH_UNIPHIER_PRO4)	+= early-clk-ld4.o
-obj-$(CONFIG_ARCH_UNIPHIER_SLD8)	+= early-clk-ld4.o
-obj-$(CONFIG_ARCH_UNIPHIER_PRO5)	+= early-clk-pro5.o
-obj-$(CONFIG_ARCH_UNIPHIER_PXS2)	+= early-clk-pxs2.o
-obj-$(CONFIG_ARCH_UNIPHIER_LD6B)	+= early-clk-pxs2.o
-obj-$(CONFIG_ARCH_UNIPHIER_LD11)	+= early-clk-ld11.o
-obj-$(CONFIG_ARCH_UNIPHIER_LD20)	+= early-clk-ld20.o
diff --git a/arch/arm/mach-uniphier/pll/Makefile b/arch/arm/mach-uniphier/pll/Makefile
deleted file mode 100644
index db22ba4ae1ee14fe4509e0f7785f7d5f489d3a20..0000000000000000000000000000000000000000
--- a/arch/arm/mach-uniphier/pll/Makefile
+++ /dev/null
@@ -1,8 +0,0 @@
-#
-# SPDX-License-Identifier:	GPL-2.0+
-#
-
-obj-$(CONFIG_ARCH_UNIPHIER_SLD3)	+= pll-init-sld3.o
-obj-$(CONFIG_ARCH_UNIPHIER_LD4)		+= pll-init-ld4.o
-obj-$(CONFIG_ARCH_UNIPHIER_PRO4)	+= pll-init-pro4.o
-obj-$(CONFIG_ARCH_UNIPHIER_SLD8)	+= pll-init-sld8.o