diff --git a/config.mk b/config.mk
index 5a91203..81c2584 100644
--- a/config.mk
+++ b/config.mk
@@ -96,7 +96,6 @@ HOSTCFLAGS	+= $(call os_x_before, 10, 4, "-traditional-cpp")
 HOSTLDFLAGS	+= $(call os_x_before, 10, 5, "-multiply_defined suppress")
 else
 HOSTCC		= gcc
-HOSTLIBS	+= -lssl -lcrypto
 endif
 
 ifeq ($(HOSTOS),cygwin)
@@ -211,6 +210,11 @@ CPPFLAGS += -ffunction-sections -fdata-sections
 LDFLAGS_FINAL += --gc-sections
 endif
 
+# TODO(sjg@chromium.org): Is this correct on Mac OS?
+ifdef CONFIG_FIT_SIGNATURE
+HOSTLIBS	+= -lssl -lcrypto
+endif
+
 ifneq ($(CONFIG_SYS_TEXT_BASE),)
 CPPFLAGS += -DCONFIG_SYS_TEXT_BASE=$(CONFIG_SYS_TEXT_BASE)
 endif
diff --git a/include/image.h b/include/image.h
index 2614918..a7b93db 100644
--- a/include/image.h
+++ b/include/image.h
@@ -46,9 +46,6 @@ struct lmb;
 #define CONFIG_OF_LIBFDT	1
 #define CONFIG_FIT_VERBOSE	1 /* enable fit_format_{error,warning}() */
 
-/* Support FIT image signing on host */
-#define CONFIG_FIT_SIGNATURE
-
 #define IMAGE_ENABLE_IGNORE	0
 #define IMAGE_INDENT_STRING	""
 
diff --git a/tools/Makefile b/tools/Makefile
index cc912fb..46159b2 100644
--- a/tools/Makefile
+++ b/tools/Makefile
@@ -126,7 +126,7 @@ LIBFDT_OBJ_FILES-y += fdt_strerror.o
 LIBFDT_OBJ_FILES-y += fdt_wip.o
 
 # RSA objects
-RSA_OBJ_FILES-y += rsa-sign.o
+RSA_OBJ_FILES-$(CONFIG_FIT_SIGNATURE) += rsa-sign.o
 
 # Generated LCD/video logo
 LOGO_H = $(OBJTREE)/include/bmp_logo.h