buildman: Handle exceptions in threads gracefully
There have been at least a few cases where an exception has occurred in a
thread and resulted in buildman hanging: running out of disk space and
getting a unicode error.
Handle these by collecting a list of exceptions, printing them out and
reporting failure if any are found. Add a test for this.
Signed-off-by: Simon Glass <sjg@chromium.org>
diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py
index 76ffbb6..ddb3eab 100644
--- a/tools/buildman/builderthread.py
+++ b/tools/buildman/builderthread.py
@@ -97,12 +97,15 @@
test_exception: Used for testing; True to raise an exception instead of
reporting the build result
"""
+ def __init__(self, builder, thread_num, mrproper, per_board_out_dir,
+ test_exception=False):
"""Set up a new builder thread"""
threading.Thread.__init__(self)
self.builder = builder
self.thread_num = thread_num
self.mrproper = mrproper
self.per_board_out_dir = per_board_out_dir
+ self.test_exception = test_exception
def Make(self, commit, brd, stage, cwd, *args, **kwargs):
"""Run 'make' on a particular commit and board.
@@ -449,7 +452,12 @@
Args:
result: CommandResult object containing the results of the build
+
+ Raises:
+ ValueError if self.test_exception is true (for testing)
"""
+ if self.test_exception:
+ raise ValueError('test exception')
if self.thread_num != -1:
self.builder.out_queue.put(result)
else:
@@ -547,5 +555,9 @@
"""
while True:
job = self.builder.queue.get()
- self.RunJob(job)
+ try:
+ self.RunJob(job)
+ except Exception as e:
+ print('Thread exception:', e)
+ self.builder.thread_exceptions.append(e)
self.builder.queue.task_done()