diff --git a/README b/README
index fefa71c0a6c5f3adf8ed437e6956227a0e502e8b..cac7978afac081129cb0355d9487c674fcd6c6af 100644
--- a/README
+++ b/README
@@ -3176,8 +3176,13 @@ CBFS (Coreboot Filesystem) support
 		This enables the RSA algorithm used for FIT image verification
 		in U-Boot. See doc/uImage.FIT/signature.txt for more information.
 
+		The Modular Exponentiation algorithm in RSA is implemented using
+		driver model. So CONFIG_DM needs to be enabled by default for this
+		library to function.
+
 		The signing part is build into mkimage regardless of this
-		option.
+		option. The software based modular exponentiation is built into
+		mkimage irrespective of this option.
 
 - bootcount support:
 		CONFIG_BOOTCOUNT_LIMIT
diff --git a/configs/am335x_boneblack_vboot_defconfig b/configs/am335x_boneblack_vboot_defconfig
index 5837a0a4da75bfb9750fcf416194a58977a5ee90..51bf370364a3e10981c7a874ba89417daca604d2 100644
--- a/configs/am335x_boneblack_vboot_defconfig
+++ b/configs/am335x_boneblack_vboot_defconfig
@@ -4,3 +4,7 @@ CONFIG_SYS_EXTRA_OPTIONS="EMMC_BOOT,ENABLE_VBOOT"
 +S:CONFIG_TARGET_AM335X_EVM=y
 CONFIG_OF_CONTROL=y
 CONFIG_DEFAULT_DEVICE_TREE="am335x-boneblack"
+CONFIG_FIT=y
+CONFIG_FIT_VERBOSE=y
+CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
diff --git a/configs/ids8313_defconfig b/configs/ids8313_defconfig
index 8479cd42f7edc326ecfb79f04eaafb528cd3752f..0950ec8b77847b72ebfec4119da1be41a3d0b51e 100644
--- a/configs/ids8313_defconfig
+++ b/configs/ids8313_defconfig
@@ -4,3 +4,4 @@ CONFIG_MPC83xx=y
 CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
 CONFIG_TARGET_IDS8313=y
+CONFIG_DM=y
diff --git a/configs/sandbox_defconfig b/configs/sandbox_defconfig
index 0111f252245579c0f22f1b0bcd0930290bf0685c..660063ebf333ae666ff7f1898a67d036c20909ad 100644
--- a/configs/sandbox_defconfig
+++ b/configs/sandbox_defconfig
@@ -3,4 +3,5 @@ CONFIG_OF_HOSTFILE=y
 CONFIG_FIT=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
 CONFIG_DEFAULT_DEVICE_TREE="sandbox"
diff --git a/configs/zynq_microzed_defconfig b/configs/zynq_microzed_defconfig
index b9a6fe522a6bbaaff4062e33bf1cab07f3293fe8..8b985fe5a4294b0867f5a4ef95efe3ae40d2d177 100644
--- a/configs/zynq_microzed_defconfig
+++ b/configs/zynq_microzed_defconfig
@@ -6,4 +6,5 @@ CONFIG_OF_CONTROL=y
 CONFIG_FIT=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
 CONFIG_DEFAULT_DEVICE_TREE="zynq-microzed"
diff --git a/configs/zynq_zc70x_defconfig b/configs/zynq_zc70x_defconfig
index dc8aa84992b0b4c92b22dd7b4a3dd2b063a1809b..cceb32199da1b6291a65cc62fb155c17ee0bcb9e 100644
--- a/configs/zynq_zc70x_defconfig
+++ b/configs/zynq_zc70x_defconfig
@@ -7,3 +7,4 @@ CONFIG_DEFAULT_DEVICE_TREE="zynq-zc702"
 CONFIG_FIT=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
diff --git a/configs/zynq_zc770_xm010_defconfig b/configs/zynq_zc770_xm010_defconfig
index 2f5fa8c59add4e794f2defe8677070dfc7d22698..2935c0dff783fac32b6c0dda91917878e94b7c96 100644
--- a/configs/zynq_zc770_xm010_defconfig
+++ b/configs/zynq_zc770_xm010_defconfig
@@ -8,3 +8,4 @@ CONFIG_DEFAULT_DEVICE_TREE="zynq-zc770-xm010"
 CONFIG_FIT=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
diff --git a/configs/zynq_zc770_xm012_defconfig b/configs/zynq_zc770_xm012_defconfig
index a92d495dfe65466572eaee0a72fbb36dc9613c1a..0401739652fc361d75b073faa1825223a375f780 100644
--- a/configs/zynq_zc770_xm012_defconfig
+++ b/configs/zynq_zc770_xm012_defconfig
@@ -8,3 +8,4 @@ CONFIG_DEFAULT_DEVICE_TREE="zynq-zc770-xm012"
 CONFIG_FIT=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
diff --git a/configs/zynq_zc770_xm013_defconfig b/configs/zynq_zc770_xm013_defconfig
index 3a02f750c64f35a49c1db79e932740fab5e4b864..a95970a917cece43f64ceca2c95f719049558ee9 100644
--- a/configs/zynq_zc770_xm013_defconfig
+++ b/configs/zynq_zc770_xm013_defconfig
@@ -8,3 +8,4 @@ CONFIG_DEFAULT_DEVICE_TREE="zynq-zc770-xm013"
 CONFIG_FIT=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
diff --git a/configs/zynq_zed_defconfig b/configs/zynq_zed_defconfig
index 1d816f68c47914cff2f0ef1745c56011636a3840..0fbc41ab8a8faca00138233a2be3db17eb9eb45a 100644
--- a/configs/zynq_zed_defconfig
+++ b/configs/zynq_zed_defconfig
@@ -7,3 +7,4 @@ CONFIG_DEFAULT_DEVICE_TREE="zynq-zed"
 CONFIG_FIT=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
diff --git a/configs/zynq_zybo_defconfig b/configs/zynq_zybo_defconfig
index 9183629bfbb8a503f67bd662ba19f68462855586..4e66760750c53bb6df093674aa6c60c8c79b3dde 100644
--- a/configs/zynq_zybo_defconfig
+++ b/configs/zynq_zybo_defconfig
@@ -7,3 +7,4 @@ CONFIG_DEFAULT_DEVICE_TREE="zynq-zybo"
 CONFIG_FIT=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
diff --git a/include/configs/am335x_evm.h b/include/configs/am335x_evm.h
index f9bc23b407d22d892a7ed81321be5db28fbf4b94..76ce7deb9508716871e6c65e5a38634e8afc86ab 100644
--- a/include/configs/am335x_evm.h
+++ b/include/configs/am335x_evm.h
@@ -19,13 +19,11 @@
 #include <configs/ti_am335x_common.h>
 
 #ifndef CONFIG_SPL_BUILD
+#ifndef CONFIG_FIT
 # define CONFIG_FIT
+#endif
 # define CONFIG_TIMESTAMP
 # define CONFIG_LZO
-# ifdef CONFIG_ENABLE_VBOOT
-# define CONFIG_FIT_SIGNATURE
-# define CONFIG_RSA
-# endif
 #endif
 
 #define CONFIG_SYS_BOOTM_LEN		(16 << 20)
diff --git a/include/configs/sandbox.h b/include/configs/sandbox.h
index 6fd29b9c73de9d7ffb5365abb572e9772235ece8..e9d3f3226b373f078faaa2c6c5e34932a3729ee1 100644
--- a/include/configs/sandbox.h
+++ b/include/configs/sandbox.h
@@ -23,7 +23,6 @@
 
 #define CONFIG_BOOTSTAGE
 #define CONFIG_BOOTSTAGE_REPORT
-#define CONFIG_DM
 #define CONFIG_CMD_DEMO
 #define CONFIG_CMD_DM
 #define CONFIG_DM_DEMO
diff --git a/lib/rsa/rsa-verify.c b/lib/rsa/rsa-verify.c
index f8bc086fd77c0a7049795f864be82b6bcf95b53d..da45daffd3b434c685a8f4b8aad7050800b9be01 100644
--- a/lib/rsa/rsa-verify.c
+++ b/lib/rsa/rsa-verify.c
@@ -12,6 +12,7 @@
 #include <asm/errno.h>
 #include <asm/types.h>
 #include <asm/unaligned.h>
+#include <dm.h>
 #else
 #include "fdt_host.h"
 #include "mkimage.h"
@@ -43,6 +44,9 @@ static int rsa_verify_key(struct key_prop *prop, const uint8_t *sig,
 	const uint8_t *padding;
 	int pad_len;
 	int ret;
+#if !defined(USE_HOSTCC)
+	struct udevice *mod_exp_dev;
+#endif
 
 	if (!prop || !sig || !hash || !algo)
 		return -EIO;
@@ -63,7 +67,17 @@ static int rsa_verify_key(struct key_prop *prop, const uint8_t *sig,
 
 	uint8_t buf[sig_len];
 
+#if !defined(USE_HOSTCC)
+	ret = uclass_get_device(UCLASS_MOD_EXP, 0, &mod_exp_dev);
+	if (ret) {
+		printf("RSA: Can't find Modular Exp implementation\n");
+		return -EINVAL;
+	}
+
+	ret = rsa_mod_exp(mod_exp_dev, sig, sig_len, prop, buf);
+#else
 	ret = rsa_mod_exp_sw(sig, sig_len, prop, buf);
+#endif
 	if (ret) {
 		debug("Error in Modular exponentation\n");
 		return ret;