diff --git a/arch/arm/mach-imx/imx9/clock.c b/arch/arm/mach-imx/imx9/clock.c
index a5f95fb..38e4cbb 100644
--- a/arch/arm/mach-imx/imx9/clock.c
+++ b/arch/arm/mach-imx/imx9/clock.c
@@ -206,6 +206,9 @@
 		return -EPERM;
 	}
 
+	/* Clear PLL HW CTRL SEL */
+	setbits_le32(&reg->ctrl.reg_clr, PLL_CTRL_HW_CTRL_SEL);
+
 	/* Bypass the PLL to ref */
 	writel(PLL_CTRL_CLKMUX_BYPASS, &reg->ctrl.reg_set);