diff --git a/drivers/crypto/rsa_mod_exp/Makefile b/drivers/crypto/rsa_mod_exp/Makefile
index 915b751dbe110ae7bc41e397658dadb74467913b..ae3dcf3d7ae9843c9eec402daeacc88fdcfd3f71 100644
--- a/drivers/crypto/rsa_mod_exp/Makefile
+++ b/drivers/crypto/rsa_mod_exp/Makefile
@@ -4,4 +4,5 @@
 # SPDX-License-Identifier:	GPL-2.0+
 #
 
-obj-$(CONFIG_RSA) += mod_exp_uclass.o mod_exp_sw.o
+obj-$(CONFIG_RSA) += mod_exp_uclass.o
+obj-$(CONFIG_RSA_SOFTWARE_EXP) += mod_exp_sw.o
diff --git a/lib/rsa/Makefile b/lib/rsa/Makefile
index cc25b3ce6d9232c60ff0b848632265b8a2788626..6867e5054c098c20a64b18cecb128cae9b548079 100644
--- a/lib/rsa/Makefile
+++ b/lib/rsa/Makefile
@@ -7,4 +7,5 @@
 # SPDX-License-Identifier:	GPL-2.0+
 #
 
-obj-$(CONFIG_FIT_SIGNATURE) += rsa-verify.o rsa-checksum.o rsa-mod-exp.o
+obj-$(CONFIG_FIT_SIGNATURE) += rsa-verify.o rsa-checksum.o
+obj-$(CONFIG_RSA_SOFTWARE_EXP) += rsa-mod-exp.o