diff --git a/drivers/clk/clk-uclass.c b/drivers/clk/clk-uclass.c
index dc3e9d6..f186fcb 100644
--- a/drivers/clk/clk-uclass.c
+++ b/drivers/clk/clk-uclass.c
@@ -329,7 +329,13 @@
 			dev_dbg(dev,
 				"could not get assigned clock %d (err = %d)\n",
 				index, ret);
-			continue;
+			/* Skip if it is empty */
+			if (ret == -ENOENT) {
+				ret = 0;
+				continue;
+			}
+
+			return ret;
 		}
 
 		/* This is clk provider device trying to program itself