diff --git a/arch/arm/cpu/armv7/zynq/Makefile b/arch/arm/cpu/armv7/zynq/Makefile
index 901f2ce..bf29b4d 100644
--- a/arch/arm/cpu/armv7/zynq/Makefile
+++ b/arch/arm/cpu/armv7/zynq/Makefile
@@ -14,4 +14,5 @@ obj-y	+= ddrc.o
 obj-y	+= slcr.o
 obj-y	+= clk.o
 obj-y	+= lowlevel_init.o
+AFLAGS_lowlevel_init.o := -mfpu=neon
 obj-$(CONFIG_SPL_BUILD)	+= spl.o
diff --git a/arch/arm/cpu/armv7/zynq/config.mk b/arch/arm/cpu/armv7/zynq/config.mk
deleted file mode 100644
index 778a377..0000000
--- a/arch/arm/cpu/armv7/zynq/config.mk
+++ /dev/null
@@ -1,7 +0,0 @@
-#
-# Copyright (C) 2013 - 2015 Xilinx, Inc. All rights reserved.
-#
-# SPDX-License-Identifier:      GPL-2.0
-#
-# Allow NEON instructions (needed for lowlevel_init.S with GNU toolchain)
-PLATFORM_RELFLAGS += -mfpu=neon