serial: Make use of the SERIAL define

This is always enabled for U-Boot proper, so simplify the condition
in the common Makefile.

Signed-off-by: Simon Glass <sjg@chromium.org>
diff --git a/common/Makefile b/common/Makefile
index d871113c..c52d4c3 100644
--- a/common/Makefile
+++ b/common/Makefile
@@ -65,15 +65,7 @@
 obj-$(CONFIG_SPD_EEPROM) += ddr_spd.o
 obj-$(CONFIG_HWCONFIG) += hwconfig.o
 obj-$(CONFIG_BOUNCE_BUFFER) += bouncebuf.o
-ifdef CONFIG_SPL_BUILD
-ifdef CONFIG_TPL_BUILD
-obj-$(CONFIG_TPL_SERIAL) += console.o
-else
-obj-$(CONFIG_SPL_SERIAL) += console.o
-endif
-else
-obj-y += console.o
-endif # CONFIG_SPL_BUILD
+obj-$(CONFIG_$(SPL_TPL_)SERIAL) += console.o
 
 obj-$(CONFIG_CROS_EC) += cros_ec.o
 obj-y += dlmalloc.o