diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 009a545..21e6690 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -185,14 +185,6 @@
 	  The Andes PLMT block holds memory-mapped mtime register
 	  associated with timer tick.
 
-config RISCV_RDTIME
-	bool
-	default y if RISCV_SMODE || SPL_RISCV_SMODE
-	help
-	  The provides the riscv_get_time() API that is implemented using the
-	  standard rdtime instruction. This is the case for S-mode U-Boot, and
-	  is useful for processors that support rdtime in M-mode too.
-
 config SYS_MALLOC_F_LEN
 	default 0x1000
 
diff --git a/arch/riscv/cpu/ax25/Kconfig b/arch/riscv/cpu/ax25/Kconfig
index 8d8d71d..5cb5bb5 100644
--- a/arch/riscv/cpu/ax25/Kconfig
+++ b/arch/riscv/cpu/ax25/Kconfig
@@ -3,7 +3,7 @@
 	select ARCH_EARLY_INIT_R
 	imply CPU
 	imply CPU_RISCV
-	imply RISCV_TIMER
+	imply RISCV_TIMER if (RISCV_SMODE || SPL_RISCV_SMODE)
 	imply ANDES_PLIC if (RISCV_MMODE || SPL_RISCV_MMODE)
 	imply ANDES_PLMT if (RISCV_MMODE || SPL_RISCV_MMODE)
 	imply SPL_CPU_SUPPORT
diff --git a/arch/riscv/cpu/fu540/Kconfig b/arch/riscv/cpu/fu540/Kconfig
index 53e1963..ac3f183 100644
--- a/arch/riscv/cpu/fu540/Kconfig
+++ b/arch/riscv/cpu/fu540/Kconfig
@@ -10,7 +10,7 @@
 	select SPL_RAM if SPL
 	imply CPU
 	imply CPU_RISCV
-	imply RISCV_TIMER
+	imply RISCV_TIMER if (RISCV_SMODE || SPL_RISCV_SMODE)
 	imply SIFIVE_CLINT if (RISCV_MMODE || SPL_RISCV_MMODE)
 	imply CMD_CPU
 	imply SPL_CPU_SUPPORT
diff --git a/arch/riscv/cpu/generic/Kconfig b/arch/riscv/cpu/generic/Kconfig
index b2cb155..f4c2e26 100644
--- a/arch/riscv/cpu/generic/Kconfig
+++ b/arch/riscv/cpu/generic/Kconfig
@@ -7,7 +7,7 @@
 	select ARCH_EARLY_INIT_R
 	imply CPU
 	imply CPU_RISCV
-	imply RISCV_TIMER
+	imply RISCV_TIMER if (RISCV_SMODE || SPL_RISCV_SMODE)
 	imply SIFIVE_CLINT if (RISCV_MMODE || SPL_RISCV_MMODE)
 	imply CMD_CPU
 	imply SPL_CPU_SUPPORT
diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile
index 6c503ff..10ac5b0 100644
--- a/arch/riscv/lib/Makefile
+++ b/arch/riscv/lib/Makefile
@@ -15,7 +15,6 @@
 obj-$(CONFIG_ANDES_PLIC) += andes_plic.o
 obj-$(CONFIG_ANDES_PLMT) += andes_plmt.o
 else
-obj-$(CONFIG_RISCV_RDTIME) += rdtime.o
 obj-$(CONFIG_SBI) += sbi.o
 obj-$(CONFIG_SBI_IPI) += sbi_ipi.o
 endif
diff --git a/arch/riscv/lib/rdtime.c b/arch/riscv/lib/rdtime.c
deleted file mode 100644
index e128d7f..0000000
--- a/arch/riscv/lib/rdtime.c
+++ /dev/null
@@ -1,38 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-/*
- * Copyright (C) 2018, Anup Patel <anup@brainfault.org>
- * Copyright (C) 2018, Bin Meng <bmeng.cn@gmail.com>
- *
- * The riscv_get_time() API implementation that is using the
- * standard rdtime instruction.
- */
-
-#include <common.h>
-
-/* Implement the API required by RISC-V timer driver */
-int riscv_get_time(u64 *time)
-{
-#ifdef CONFIG_64BIT
-	u64 n;
-
-	__asm__ __volatile__ (
-		"rdtime %0"
-		: "=r" (n));
-
-	*time = n;
-#else
-	u32 lo, hi, tmp;
-
-	__asm__ __volatile__ (
-		"1:\n"
-		"rdtimeh %0\n"
-		"rdtime %1\n"
-		"rdtimeh %2\n"
-		"bne %0, %2, 1b"
-		: "=&r" (hi), "=&r" (lo), "=&r" (tmp));
-
-	*time = ((u64)hi << 32) | lo;
-#endif
-
-	return 0;
-}