common: Drop linux/delay.h from common header

Move this uncommon header out of the common header.

Signed-off-by: Simon Glass <sjg@chromium.org>
diff --git a/arch/arm/mach-socfpga/clock_manager_arria10.c b/arch/arm/mach-socfpga/clock_manager_arria10.c
index d7c8eaf..58d5d3f 100644
--- a/arch/arm/mach-socfpga/clock_manager_arria10.c
+++ b/arch/arm/mach-socfpga/clock_manager_arria10.c
@@ -11,6 +11,7 @@
 #include <clk.h>
 #include <dm/device-internal.h>
 #include <asm/arch/clock_manager.h>
+#include <linux/delay.h>
 
 #ifdef CONFIG_SPL_BUILD
 
diff --git a/arch/arm/mach-socfpga/freeze_controller.c b/arch/arm/mach-socfpga/freeze_controller.c
index d48aeec..561d340 100644
--- a/arch/arm/mach-socfpga/freeze_controller.c
+++ b/arch/arm/mach-socfpga/freeze_controller.c
@@ -8,6 +8,7 @@
 #include <asm/io.h>
 #include <asm/arch/clock_manager.h>
 #include <asm/arch/freeze_controller.h>
+#include <linux/delay.h>
 #include <linux/errno.h>
 
 static const struct socfpga_freeze_controller *freeze_controller_base =
diff --git a/arch/arm/mach-socfpga/scan_manager.c b/arch/arm/mach-socfpga/scan_manager.c
index f7ee289..36d6880 100644
--- a/arch/arm/mach-socfpga/scan_manager.c
+++ b/arch/arm/mach-socfpga/scan_manager.c
@@ -9,6 +9,7 @@
 #include <asm/arch/freeze_controller.h>
 #include <asm/arch/scan_manager.h>
 #include <asm/arch/system_manager.h>
+#include <linux/delay.h>
 
 /*
  * Maximum polling loop to wait for IO scan chain engine becomes idle