diff --git a/drivers/crypto/rsa_mod_exp/Makefile b/drivers/crypto/rsa_mod_exp/Makefile
index 915b751..ae3dcf3 100644
--- a/drivers/crypto/rsa_mod_exp/Makefile
+++ b/drivers/crypto/rsa_mod_exp/Makefile
@@ -4,4 +4,5 @@
 # SPDX-License-Identifier:	GPL-2.0+
 #
 
-obj-$(CONFIG_RSA) += mod_exp_uclass.o mod_exp_sw.o
+obj-$(CONFIG_RSA) += mod_exp_uclass.o
+obj-$(CONFIG_RSA_SOFTWARE_EXP) += mod_exp_sw.o
diff --git a/lib/rsa/Makefile b/lib/rsa/Makefile
index cc25b3c..6867e50 100644
--- a/lib/rsa/Makefile
+++ b/lib/rsa/Makefile
@@ -7,4 +7,5 @@
 # SPDX-License-Identifier:	GPL-2.0+
 #
 
-obj-$(CONFIG_FIT_SIGNATURE) += rsa-verify.o rsa-checksum.o rsa-mod-exp.o
+obj-$(CONFIG_FIT_SIGNATURE) += rsa-verify.o rsa-checksum.o
+obj-$(CONFIG_RSA_SOFTWARE_EXP) += rsa-mod-exp.o