Merge patch series "test: Minor fixes to test.py"
Simon Glass <sjg@chromium.org> says:
This series collects together the patches from the Labgrid series which
are not related to Labgrid, or at least can be applied independently of
using Labgrid to run the lab.
Link: https://lore.kernel.org/r/20241010002907.19383-1-sjg@chromium.org
diff --git a/test/py/conftest.py b/test/py/conftest.py
index fc9dd3a..46a410c 100644
--- a/test/py/conftest.py
+++ b/test/py/conftest.py
@@ -24,6 +24,7 @@
import re
from _pytest.runner import runtestprotocol
import sys
+from u_boot_spawn import BootFail, Timeout, Unexpected, handle_exception
# Globals: The HTML log file, and the connection to the U-Boot console.
log = None
@@ -115,14 +116,36 @@
runner.close()
log.status_pass('OK')
-def pytest_xdist_setupnodes(config, specs):
- """Clear out any 'done' file from a previous build"""
- global build_done_file
- build_dir = config.getoption('build_dir')
+def get_details(config):
+ """Obtain salient details about the board and directories to use
+
+ Args:
+ config (pytest.Config): pytest configuration
+
+ Returns:
+ tuple:
+ str: Board type (U-Boot build name)
+ str: Identity for the lab board
+ str: Build directory
+ str: Source directory
+ """
board_type = config.getoption('board_type')
+ board_identity = config.getoption('board_identity')
+ build_dir = config.getoption('build_dir')
+
source_dir = os.path.dirname(os.path.dirname(TEST_PY_DIR))
+ default_build_dir = source_dir + '/build-' + board_type
if not build_dir:
- build_dir = source_dir + '/build-' + board_type
+ build_dir = default_build_dir
+
+ return board_type, board_identity, build_dir, source_dir
+
+def pytest_xdist_setupnodes(config, specs):
+ """Clear out any 'done' file from a previous build"""
+ global build_done_file
+
+ build_dir = get_details(config)[2]
+
build_done_file = Path(build_dir) / 'build.done'
if build_done_file.exists():
os.remove(build_done_file)
@@ -161,17 +184,10 @@
global console
global ubconfig
- source_dir = os.path.dirname(os.path.dirname(TEST_PY_DIR))
+ board_type, board_identity, build_dir, source_dir = get_details(config)
- board_type = config.getoption('board_type')
board_type_filename = board_type.replace('-', '_')
-
- board_identity = config.getoption('board_identity')
board_identity_filename = board_identity.replace('-', '_')
-
- build_dir = config.getoption('build_dir')
- if not build_dir:
- build_dir = source_dir + '/build-' + board_type
mkdir_p(build_dir)
result_dir = config.getoption('result_dir')
@@ -239,6 +255,7 @@
ubconfig.board_identity = board_identity
ubconfig.gdbserver = gdbserver
ubconfig.dtb = build_dir + '/arch/sandbox/dts/test.dtb'
+ ubconfig.connection_ok = True
env_vars = (
'board_type',
@@ -405,8 +422,21 @@
Returns:
The fixture value.
"""
-
- console.ensure_spawned()
+ if not ubconfig.connection_ok:
+ pytest.skip('Cannot get target connection')
+ return None
+ try:
+ console.ensure_spawned()
+ except OSError as err:
+ handle_exception(ubconfig, console, log, err, 'Lab failure', True)
+ except Timeout as err:
+ handle_exception(ubconfig, console, log, err, 'Lab timeout', True)
+ except BootFail as err:
+ handle_exception(ubconfig, console, log, err, 'Boot fail', True,
+ console.get_spawn_output())
+ except Unexpected:
+ handle_exception(ubconfig, console, log, err, 'Unexpected test output',
+ False)
return console
anchors = {}
diff --git a/test/py/u_boot_console_base.py b/test/py/u_boot_console_base.py
index 76a550d..d8d0bdf 100644
--- a/test/py/u_boot_console_base.py
+++ b/test/py/u_boot_console_base.py
@@ -14,6 +14,7 @@
import re
import sys
import u_boot_spawn
+from u_boot_spawn import BootFail, Timeout, Unexpected, handle_exception
# Regexes for text we expect U-Boot to send to the console.
pattern_u_boot_spl_signon = re.compile('(U-Boot SPL \\d{4}\\.\\d{2}[^\r\n]*\\))')
@@ -26,6 +27,9 @@
PAT_ID = 0
PAT_RE = 1
+# Timeout before expecting the console to be ready (in milliseconds)
+TIMEOUT_MS = 30000
+
bad_pattern_defs = (
('spl_signon', pattern_u_boot_spl_signon),
('main_signon', pattern_u_boot_main_signon),
@@ -109,7 +113,7 @@
Can only usefully be called by sub-classes.
Args:
- log: A mulptiplex_log.Logfile object, to which the U-Boot output
+ log: A multiplexed_log.Logfile object, to which the U-Boot output
will be logged.
config: A configuration data structure, as built by conftest.py.
max_fifo_fill: The maximum number of characters to send to U-Boot
@@ -186,13 +190,13 @@
m = self.p.expect([pattern_u_boot_spl_signon] +
self.bad_patterns)
if m != 0:
- raise Exception('Bad pattern found on SPL console: ' +
+ raise BootFail('Bad pattern found on SPL console: ' +
self.bad_pattern_ids[m - 1])
env_spl_banner_times -= 1
m = self.p.expect([pattern_u_boot_main_signon] + self.bad_patterns)
if m != 0:
- raise Exception('Bad pattern found on console: ' +
+ raise BootFail('Bad pattern found on console: ' +
self.bad_pattern_ids[m - 1])
self.u_boot_version_string = self.p.after
while True:
@@ -203,13 +207,9 @@
if m == 1:
self.p.send(' ')
continue
- raise Exception('Bad pattern found on console: ' +
+ raise BootFail('Bad pattern found on console: ' +
self.bad_pattern_ids[m - 2])
- except Exception as ex:
- self.log.error(str(ex))
- self.cleanup_spawn()
- raise
finally:
self.log.timestamp()
@@ -275,7 +275,7 @@
m = self.p.expect([chunk] + self.bad_patterns)
if m != 0:
self.at_prompt = False
- raise Exception('Bad pattern found on console: ' +
+ raise BootFail('Bad pattern found on console: ' +
self.bad_pattern_ids[m - 1])
if not wait_for_prompt:
return
@@ -285,16 +285,20 @@
m = self.p.expect([self.prompt_compiled] + self.bad_patterns)
if m != 0:
self.at_prompt = False
- raise Exception('Bad pattern found on console: ' +
+ raise BootFail('Missing prompt on console: ' +
self.bad_pattern_ids[m - 1])
self.at_prompt = True
self.at_prompt_logevt = self.logstream.logfile.cur_evt
# Only strip \r\n; space/TAB might be significant if testing
# indentation.
return self.p.before.strip('\r\n')
- except Exception as ex:
- self.log.error(str(ex))
- self.cleanup_spawn()
+ except Timeout as exc:
+ handle_exception(self.config, self, self.log, exc, 'Lab failure',
+ True)
+ raise
+ except BootFail as exc:
+ handle_exception(self.config, self, self.log, exc, 'Boot fail',
+ True, self.get_spawn_output())
raise
finally:
self.log.timestamp()
@@ -351,8 +355,9 @@
text = re.escape(text)
m = self.p.expect([text] + self.bad_patterns)
if m != 0:
- raise Exception('Bad pattern found on console: ' +
- self.bad_pattern_ids[m - 1])
+ raise Unexpected(
+ "Unexpected pattern found on console (exp '{text}': " +
+ self.bad_pattern_ids[m - 1])
def drain_console(self):
"""Read from and log the U-Boot console for a short time.
@@ -422,7 +427,7 @@
# Reset the console timeout value as some tests may change
# its default value during the execution
if not self.config.gdbserver:
- self.p.timeout = 30000
+ self.p.timeout = TIMEOUT_MS
return
try:
self.log.start_section('Starting U-Boot')
@@ -433,7 +438,7 @@
# future, possibly per-test to be optimal. This works for 'help'
# on board 'seaboard'.
if not self.config.gdbserver:
- self.p.timeout = 30000
+ self.p.timeout = TIMEOUT_MS
self.p.logfile_read = self.logstream
if expect_reset:
loop_num = 2
diff --git a/test/py/u_boot_spawn.py b/test/py/u_boot_spawn.py
index 97e95e0..24d3690 100644
--- a/test/py/u_boot_spawn.py
+++ b/test/py/u_boot_spawn.py
@@ -8,6 +8,7 @@
import os
import re
import pty
+import pytest
import signal
import select
import time
@@ -16,6 +17,54 @@
class Timeout(Exception):
"""An exception sub-class that indicates that a timeout occurred."""
+class BootFail(Exception):
+ """An exception sub-class that indicates that a boot failure occurred.
+
+ This is used when a bad pattern is seen when waiting for the boot prompt.
+ It is regarded as fatal, to avoid trying to boot the again and again to no
+ avail.
+ """
+
+class Unexpected(Exception):
+ """An exception sub-class that indicates that unexpected test was seen."""
+
+
+def handle_exception(ubconfig, console, log, err, name, fatal, output=''):
+ """Handle an exception from the console
+
+ Exceptions can occur when there is unexpected output or due to the board
+ crashing or hanging. Some exceptions are likely fatal, where retrying will
+ just chew up time to no available. In those cases it is best to cause
+ further tests be skipped.
+
+ Args:
+ ubconfig (ArbitraryAttributeContainer): ubconfig object
+ log (Logfile): Place to log errors
+ console (ConsoleBase): Console to clean up, if fatal
+ err (Exception): Exception which was thrown
+ name (str): Name of problem, to log
+ fatal (bool): True to abort all tests
+ output (str): Extra output to report on boot failure. This can show the
+ target's console output as it tried to boot
+ """
+ msg = f'{name}: '
+ if fatal:
+ msg += 'Marking connection bad - no other tests will run'
+ else:
+ msg += 'Assuming that lab is healthy'
+ print(msg)
+ log.error(msg)
+ log.error(f'Error: {err}')
+
+ if output:
+ msg += f'; output {output}'
+
+ if fatal:
+ ubconfig.connection_ok = False
+ console.cleanup_spawn()
+ pytest.exit(msg)
+
+
class Spawn:
"""Represents the stdio of a freshly created sub-process. Commands may be
sent to the process, and responses waited for.
@@ -137,6 +186,32 @@
os.write(self.fd, data.encode(errors='replace'))
+ def receive(self, num_bytes):
+ """Receive data from the sub-process's stdin.
+
+ Args:
+ num_bytes (int): Maximum number of bytes to read
+
+ Returns:
+ str: The data received
+
+ Raises:
+ ValueError if U-Boot died
+ """
+ try:
+ c = os.read(self.fd, num_bytes).decode(errors='replace')
+ except OSError as err:
+ # With sandbox, try to detect when U-Boot exits when it
+ # shouldn't and explain why. This is much more friendly than
+ # just dying with an I/O error
+ if self.decode_signal and err.errno == 5: # I/O error
+ alive, _, info = self.checkalive()
+ if alive:
+ raise err
+ raise ValueError('U-Boot exited with %s' % info)
+ raise
+ return c
+
def expect(self, patterns):
"""Wait for the sub-process to emit specific data.
@@ -193,18 +268,7 @@
events = self.poll.poll(poll_maxwait)
if not events:
raise Timeout()
- try:
- c = os.read(self.fd, 1024).decode(errors='replace')
- except OSError as err:
- # With sandbox, try to detect when U-Boot exits when it
- # shouldn't and explain why. This is much more friendly than
- # just dying with an I/O error
- if self.decode_signal and err.errno == 5: # I/O error
- alive, _, info = self.checkalive()
- if alive:
- raise err
- raise ValueError('U-Boot exited with %s' % info)
- raise
+ c = self.receive(1024)
if self.logfile_read:
self.logfile_read.write(c)
self.buf += c
diff --git a/test/test-main.c b/test/test-main.c
index 479dbb3..4daca81 100644
--- a/test/test-main.c
+++ b/test/test-main.c
@@ -486,7 +486,7 @@
static int ut_run_test_live_flat(struct unit_test_state *uts,
struct unit_test *test)
{
- int runs;
+ int runs, ret;
if ((test->flags & UTF_OTHER_FDT) && !IS_ENABLED(CONFIG_SANDBOX))
return skip_test(uts);
@@ -496,8 +496,11 @@
if (CONFIG_IS_ENABLED(OF_LIVE)) {
if (!(test->flags & UTF_FLAT_TREE)) {
uts->of_live = true;
- ut_assertok(ut_run_test(uts, test, test->name));
- runs++;
+ ret = ut_run_test(uts, test, test->name);
+ if (ret != -EAGAIN) {
+ ut_assertok(ret);
+ runs++;
+ }
}
}
@@ -521,8 +524,11 @@
(!runs || ut_test_run_on_flattree(test)) &&
!(gd->flags & GD_FLG_FDT_CHANGED)) {
uts->of_live = false;
- ut_assertok(ut_run_test(uts, test, test->name));
- runs++;
+ ret = ut_run_test(uts, test, test->name);
+ if (ret != -EAGAIN) {
+ ut_assertok(ret);
+ runs++;
+ }
}
return 0;