diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 18c7fb2..a1c18d2 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -112,6 +112,7 @@ source "arch/x86/cpu/braswell/Kconfig"
 source "arch/x86/cpu/broadwell/Kconfig"
 source "arch/x86/cpu/coreboot/Kconfig"
 source "arch/x86/cpu/ivybridge/Kconfig"
+source "arch/x86/cpu/efi/Kconfig"
 source "arch/x86/cpu/qemu/Kconfig"
 source "arch/x86/cpu/quark/Kconfig"
 source "arch/x86/cpu/queensbay/Kconfig"
@@ -772,6 +773,4 @@ config HIGH_TABLE_SIZE
 	  Increse it if the default size does not fit the board's needs.
 	  This is most likely due to a large ACPI DSDT table is used.
 
-source "arch/x86/lib/efi/Kconfig"
-
 endmenu
diff --git a/arch/x86/cpu/Makefile b/arch/x86/cpu/Makefile
index af9e26c..f862d8c 100644
--- a/arch/x86/cpu/Makefile
+++ b/arch/x86/cpu/Makefile
@@ -29,7 +29,7 @@ obj-$(CONFIG_INTEL_BAYTRAIL) += baytrail/
 obj-$(CONFIG_INTEL_BRASWELL) += braswell/
 obj-$(CONFIG_INTEL_BROADWELL) += broadwell/
 obj-$(CONFIG_SYS_COREBOOT) += coreboot/
-obj-$(CONFIG_EFI_APP) += efi/
+obj-$(CONFIG_EFI) += efi/
 obj-$(CONFIG_QEMU) += qemu/
 obj-$(CONFIG_NORTHBRIDGE_INTEL_IVYBRIDGE) += ivybridge/
 obj-$(CONFIG_INTEL_QUARK) += quark/
diff --git a/arch/x86/lib/efi/Kconfig b/arch/x86/cpu/efi/Kconfig
similarity index 100%
rename from arch/x86/lib/efi/Kconfig
rename to arch/x86/cpu/efi/Kconfig
diff --git a/arch/x86/cpu/efi/Makefile b/arch/x86/cpu/efi/Makefile
index 06d0480..32c2dde 100644
--- a/arch/x86/cpu/efi/Makefile
+++ b/arch/x86/cpu/efi/Makefile
@@ -2,5 +2,12 @@
 #
 # Copyright (c) 2015 Google, Inc
 
+ifdef CONFIG_EFI_APP
 obj-y += efi.o
 obj-y += sdram.o
+endif
+
+ifdef CONFIG_EFI_STUB
+obj-y += car.o
+obj-y += payload.o
+endif
diff --git a/arch/x86/lib/efi/car.S b/arch/x86/cpu/efi/car.S
similarity index 100%
rename from arch/x86/lib/efi/car.S
rename to arch/x86/cpu/efi/car.S
diff --git a/arch/x86/lib/efi/efi.c b/arch/x86/cpu/efi/payload.c
similarity index 100%
rename from arch/x86/lib/efi/efi.c
rename to arch/x86/cpu/efi/payload.c
diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile
index 0e054da..ba07ac7 100644
--- a/arch/x86/lib/Makefile
+++ b/arch/x86/lib/Makefile
@@ -14,7 +14,6 @@ endif
 obj-y	+= cmd_boot.o
 obj-$(CONFIG_SEABIOS) += coreboot_table.o
 obj-y	+= early_cmos.o
-obj-$(CONFIG_EFI) += efi/
 obj-y	+= e820.o
 obj-y	+= init_helpers.o
 obj-y	+= interrupts.o
diff --git a/arch/x86/lib/efi/Makefile b/arch/x86/lib/efi/Makefile
deleted file mode 100644
index f6c6523..0000000
--- a/arch/x86/lib/efi/Makefile
+++ /dev/null
@@ -1,7 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0+
-#
-# (C) Copyright 2002-2006
-# Wolfgang Denk, DENX Software Engineering, wd@denx.de.
-
-obj-$(CONFIG_EFI_STUB) += car.o
-obj-$(CONFIG_EFI_STUB) += efi.o
diff --git a/doc/README.u-boot_on_efi b/doc/README.u-boot_on_efi
index d5903c7..0349c56 100644
--- a/doc/README.u-boot_on_efi
+++ b/doc/README.u-boot_on_efi
@@ -235,12 +235,8 @@ Where is the code?
 lib/efi
 	payload stub, application, support code. Mostly arch-neutral
 
-arch/x86/lib/efi
-	helper functions for the fake DRAM init, etc. These can be used by
-	any board that runs as a payload.
-
 arch/x86/cpu/efi
-	x86 support code for running as an EFI application
+	x86 support code for running as an EFI application and payload
 
 board/efi/efi-x86/efi.c
 	x86 board code for running as an EFI application