diff --git a/Makefile b/Makefile
index 3a8ea6750e37ce68982316976868b961e1cfdacb..0e9609f69c187e3563352659feb61b06fe8e9af7 100644
--- a/Makefile
+++ b/Makefile
@@ -271,11 +271,7 @@ LIBS-y += drivers/power/libpower.o \
 	drivers/power/pmic/libpmic.o \
 	drivers/power/battery/libbattery.o
 LIBS-y += drivers/spi/libspi.o
-ifeq ($(CPU),mpc83xx)
-LIBS-y += drivers/qe/libqe.o
-endif
 ifeq ($(CPU),mpc85xx)
-LIBS-y += drivers/qe/libqe.o
 LIBS-y += drivers/net/fm/libfm.o
 endif
 LIBS-y += drivers/serial/libserial.o
diff --git a/drivers/Makefile b/drivers/Makefile
index 95a4b807242816a6c546283de8686d946514a63d..5d03f37a187b8fb5f112578ff69043ea2d0135ce 100644
--- a/drivers/Makefile
+++ b/drivers/Makefile
@@ -13,3 +13,4 @@ obj-y += tpm/
 obj-y += twserial/
 obj-y += video/
 obj-y += watchdog/
+obj-$(CONFIG_QE) += qe/
diff --git a/drivers/qe/Makefile b/drivers/qe/Makefile
index b8c15f8e1f898d0c3a388d60694222bd3b79b510..7f1bd06922f4995b446581713c589d5b30ba670f 100644
--- a/drivers/qe/Makefile
+++ b/drivers/qe/Makefile
@@ -4,5 +4,5 @@
 # SPDX-License-Identifier:	GPL-2.0+
 #
 
-obj-$(and $(CONFIG_QE),$(CONFIG_OF_LIBFDT)) += fdt.o
-obj-$(CONFIG_QE) += qe.o uccf.o uec.o uec_phy.o
+obj-y := qe.o uccf.o uec.o uec_phy.o
+obj-$(CONFIG_OF_LIBFDT) += fdt.o