[05/15] framework: if a test with subtests crashes mark the offending subtest

Submitted by Dylan Baker on Jan. 23, 2018, 1:22 a.m.

Details

Message ID 0a858a495cc394fa33c712f95354fb5e9d43501b.1516670408.git-series.dylan@pnwbakers.com
State New
Headers show
Series "Series without cover letter" ( rev: 11 ) in Piglit

Not browsing as part of any series.

Commit Message

Dylan Baker Jan. 23, 2018, 1:22 a.m.
This relies on the fact that subtests are guaranteed to be ordered to
mark the crashing subtest as such. This ensures that the correct status
will be propagated up the totals tree.
---
 framework/test/base.py                |  8 ++++++-
 unittests/framework/test/test_base.py | 36 ++++++++++++++++++++++++++++-
 2 files changed, 44 insertions(+)

Patch hide | download patch | download mbox

diff --git a/framework/test/base.py b/framework/test/base.py
index 2d761ef..134b872 100644
--- a/framework/test/base.py
+++ b/framework/test/base.py
@@ -236,6 +236,14 @@  class Test(object):
         """
         if is_crash_returncode(self.result.returncode):
             self.result.result = status.CRASH
+            if self.result.subtests:
+                # We know because subtests are ordered that the first test with
+                # a status of NOTRUN is the subtest that crashed, mark that
+                # test and move on.
+                for k, v in six.iteritems(self.result.subtests):
+                    if v == status.NOTRUN:
+                        self.result.subtests[k] = status.CRASH
+                        break
         elif self.result.returncode != 0:
             if self.result.result == status.PASS:
                 self.result.result = status.WARN
diff --git a/unittests/framework/test/test_base.py b/unittests/framework/test/test_base.py
index 656d839..b1b2cdb 100644
--- a/unittests/framework/test/test_base.py
+++ b/unittests/framework/test/test_base.py
@@ -33,6 +33,8 @@  except ImportError:
 import pytest
 import six
 
+from six.moves import range
+
 from framework import dmesg
 from framework import log
 from framework import monitoring
@@ -283,6 +285,40 @@  class TestTest(object):
 
             assert test.result.result is status.FAIL
 
+        def test_crash_subtest_before_start(self):
+            """A test for a a test with a subtest, that crashes in the middle
+            of the run.
+            """
+            test = _Test(['foobar'])
+            test.result.returncode = -1
+            for x in (str(y) for y in range(5)):
+                test.result.subtests[x] = status.NOTRUN
+            test.interpret_result()
+
+            assert test.result.result is status.CRASH
+            assert test.result.subtests['0'] is status.CRASH
+            for x in (str(y) for y in range(1, 5)):
+                assert test.result.subtests[x] is status.NOTRUN
+
+        def test_crash_subtest_mid(self):
+            """A test for a a test with a subtest, that crashes in the middle
+            of the run.
+            """
+            test = _Test(['foobar'])
+            test.result.returncode = -1
+            for x in (str(y) for y in range(2)):
+                test.result.subtests[x] = status.PASS
+            for x in (str(y) for y in range(2, 5)):
+                test.result.subtests[x] = status.NOTRUN
+            test.interpret_result()
+
+            assert test.result.result is status.CRASH
+            for x in (str(y) for y in range(2)):
+                assert test.result.subtests[x] is status.PASS
+            assert test.result.subtests['2'] is status.CRASH
+            for x in (str(y) for y in range(3, 5)):
+                assert test.result.subtests[x] is status.NOTRUN
+
 
 class TestWindowResizeMixin(object):
     """Tests for the WindowResizeMixin class."""