diff --git a/common/Makefile b/common/Makefile
index 33c606abb2305560879c007ab4c26e762b5ae027..abca82072665540cc1a043d77c0246fd0beddfe0 100644
--- a/common/Makefile
+++ b/common/Makefile
@@ -32,7 +32,6 @@ COBJS-y += command.o
 COBJS-y += exports.o
 COBJS-$(CONFIG_SYS_HUSH_PARSER) += hush.o
 COBJS-y += s_record.o
-COBJS-$(CONFIG_SERIAL_MULTI) += serial.o
 COBJS-y += xyzModem.o
 COBJS-y += cmd_disk.o
 
diff --git a/drivers/serial/Makefile b/drivers/serial/Makefile
index afaddeaa261b341ed3a8744e5f395a06ac16e03d..a8b500b222fdd560e8233ae840d5d4959b3a643e 100644
--- a/drivers/serial/Makefile
+++ b/drivers/serial/Makefile
@@ -25,6 +25,10 @@ include $(TOPDIR)/config.mk
 
 LIB	:= $(obj)libserial.o
 
+ifndef CONFIG_SPL_BUILD
+COBJS-$(CONFIG_SERIAL_MULTI) += serial.o
+endif
+
 COBJS-$(CONFIG_ALTERA_UART) += altera_uart.o
 COBJS-$(CONFIG_ALTERA_JTAG_UART) += altera_jtag_uart.o
 COBJS-$(CONFIG_ARM_DCC) += arm_dcc.o
diff --git a/common/serial.c b/drivers/serial/serial.c
similarity index 100%
rename from common/serial.c
rename to drivers/serial/serial.c