[RFC,8/9] backends: implemented loading support for junit.

Submitted by Dylan Baker on April 6, 2015, 9:30 p.m.

Details

Message ID 1428355819-12180-9-git-send-email-baker.dylan.c@gmail.com
State New, archived
Headers show

Not browsing as part of any series.

Commit Message

Dylan Baker April 6, 2015, 9:30 p.m.
This allows junit results to be loaded like any other reuslts, including
for use with the piglit summary generator.

Signed-off-by: Dylan Baker <dylanx.c.baker@intel.com>
---
 framework/backends/junit.py             |  70 +++++++++++++++-
 framework/tests/junit_backends_tests.py | 140 +++++++++++++++++++++++++++++++-
 framework/tests/utils.py                |   5 +-
 3 files changed, 208 insertions(+), 7 deletions(-)

Patch hide | download patch | download mbox

diff --git a/framework/backends/junit.py b/framework/backends/junit.py
index 5ec4e8e..3602f9e 100644
--- a/framework/backends/junit.py
+++ b/framework/backends/junit.py
@@ -1,4 +1,4 @@ 
-# Copyright (c) 2014 Intel Corporation
+# Copyright (c) 2014, 2015 Intel Corporation
 
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -29,7 +29,7 @@  try:
 except ImportError:
     import xml.etree.cElementTree as etree
 
-import framework.grouptools as grouptools
+from framework import grouptools, results, status
 from framework.core import PIGLIT_CONFIG
 from .abstract import FileBackend
 from .register import Registry
@@ -207,9 +207,73 @@  class JUnitBackend(FileBackend):
             self._fsync(f)
 
 
+def _load(results_file):
+    """Load a junit results instance and return a TestrunResult.
+
+    It's worth noting that junit is not as descriptive as piglit's own json
+    format, so some data structures will be empty compared to json.
+
+    This tries to not make too many assumptions about the strucuter of the
+    JUnit document.
+
+    """
+    run_result = results.TestrunResult()
+
+    splitpath = os.path.splitext(results_file)[0].split(os.path.sep)
+    if splitpath[-1] != 'results':
+        run_result.name = splitpath[-1]
+    elif len(splitpath) > 1:
+        run_result.name = splitpath[-2]
+    else:
+        run_result.name = 'junit result'
+
+    tree = etree.parse(results_file).getroot().find('.//testsuite[@name="piglit"]')
+    for test in tree.iterfind('testcase'):
+        result = results.TestResult()
+        # Take the class name minus the 'piglit.' element, replace junit's '.'
+        # separator with piglit's separator, and join the group and test names
+        name = test.attrib['classname'].split('.', 1)[1]
+        name = name.replace('.', grouptools.SEPARATOR)
+        name = grouptools.join(name, test.attrib['name'])
+
+        # Remove the trailing _ if they were added (such as to api and search)
+        if name.endswith('_'):
+            name = name[:-1]
+
+        result['result'] = status.status_lookup(test.attrib['status'])
+        result['time'] = float(test.attrib['time'])
+        result['err'] = test.find('system-err').text
+
+        # The command is prepended to system-out, so we need to separate those
+        # into two separate elements
+        out = test.find('system-out').text.split('\n')
+        result['command'] = out[0]
+        result['out'] = '\n'.join(out[1:])
+
+        run_result.tests[name] = result
+    
+    return run_result
+
+
+def load(results_dir):
+    """Searches for a results file and returns a TestrunResult.
+
+    wraps _load and searches for the result file.
+
+    """
+    if not os.path.isdir(results_dir):
+        return _load(results_dir)
+    elif os.path.exists(os.path.join(results_dir, 'tests')):
+        raise NotImplementedError('resume support of junit not implemented')
+    elif os.path.exists(os.path.join(results_dir, 'results.xml')):
+        return _load(os.path.join(results_dir, 'results.xml'))
+    else:
+        raise Exception("No results found")
+
+
 REGISTRY = Registry(
     extensions=['.xml'],
     backend=JUnitBackend,
-    load=None,
+    load=load,
     meta=lambda x: x,  # The venerable no-op function
 )
diff --git a/framework/tests/junit_backends_tests.py b/framework/tests/junit_backends_tests.py
index 9afef75..d283119 100644
--- a/framework/tests/junit_backends_tests.py
+++ b/framework/tests/junit_backends_tests.py
@@ -31,14 +31,26 @@  except ImportError:
     import xml.etree.cElementTree as etree
 import nose.tools as nt
 
-from framework import results, backends, grouptools
+from framework import results, backends, grouptools, status
 import framework.tests.utils as utils
 from .backends_tests import BACKEND_INITIAL_META
 
 
 JUNIT_SCHEMA = 'framework/tests/schema/junit-7.xsd'
 
-doc_formatter = utils.DocFormatter({'seperator': grouptools.SEPARATOR})
+doc_formatter = utils.DocFormatter({'separator': grouptools.SEPARATOR})
+
+_XML =  """\
+<?xml version='1.0' encoding='utf-8'?>
+  <testsuites>
+    <testsuite name="piglit" tests="1">
+      <testcase classname="piglit.foo.bar" name="a-test" status="pass" time="1.12345">
+        <system-out>this/is/a/command\nThis is stdout</system-out>
+        <system-err>this is stderr</system-err>
+      </testcase>
+    </testsuite>
+  </testsuites>
+"""
 
 
 class TestJunitNoTests(utils.StaticDirectory):
@@ -133,7 +145,7 @@  class TestJUnitMultiTest(TestJUnitSingleTest):
 
 @doc_formatter
 def test_junit_replace():
-    """JUnitBackend.write_test: '{seperator}' is replaced with '.'"""
+    """JUnitBackend.write_test: '{separator}' is replaced with '.'"""
     with utils.tempdir() as tdir:
         test = backends.junit.JUnitBackend(tdir)
         test.initialize(BACKEND_INITIAL_META)
@@ -177,3 +189,125 @@  def test_junit_skips_bad_tests():
             test.finalize()
         except etree.ParseError as e:
             raise AssertionError(e)
+
+
+class TestJUnitLoad(utils.StaticDirectory):
+    """Methods that test loading JUnit results."""
+    __instance = None
+
+    @classmethod
+    def setup_class(cls):
+        super(TestJUnitLoad, cls).setup_class()
+        cls.xml_file = os.path.join(cls.tdir, 'results.xml')
+        
+        with open(cls.xml_file, 'w') as f:
+            f.write(_XML)
+
+        cls.testname = grouptools.join('foo', 'bar', 'a-test')
+
+    @classmethod
+    def xml(cls):
+        if cls.__instance is None:
+            cls.__instance =  backends.junit._load(cls.xml_file)
+        return cls.__instance
+
+    @utils.no_error
+    def test_no_errors(self):
+        """backends.junit._load: Raises no errors for valid junit."""
+        self.xml()
+
+    def test_return_testrunresult(self):
+        """backends.junit._load: returns a TestrunResult instance"""
+        nt.assert_is_instance(self.xml(), results.TestrunResult)
+
+    @doc_formatter
+    def test_replace_sep(self):
+        """backends.junit._load: replaces '.' with '{separator}'"""
+        nt.assert_in(self.testname, self.xml().tests)
+
+    def test_testresult_instance(self):
+        """backends.junit._load: replaces result with TestResult instance."""
+        nt.assert_is_instance(self.xml().tests[self.testname], results.TestResult)
+
+    def test_status_instance(self):
+        """backends.junit._load: a status is found and loaded."""
+        nt.assert_is_instance(self.xml().tests[self.testname]['result'],
+                              status.Status)
+
+    def test_time(self):
+        """backends.junit._load: Time is loaded correctly."""
+        time = self.xml().tests[self.testname]['time']
+        nt.assert_is_instance(time, float)
+        nt.assert_equal(time, 1.12345)
+
+    def test_command(self):
+        """backends.junit._load: command is loaded correctly."""
+        test = self.xml().tests[self.testname]['command']
+        nt.assert_equal(test, 'this/is/a/command')
+
+    def test_out(self):
+        """backends.junit._load: stdout is loaded correctly."""
+        test = self.xml().tests[self.testname]['out']
+        nt.assert_equal(test, 'This is stdout')
+
+    def test_err(self):
+        """backends.junit._load: stderr is loaded correctly."""
+        test = self.xml().tests[self.testname]['err']
+        nt.assert_equal(test, 'this is stderr')
+
+    @utils.no_error
+    def test_load_file(self):
+        """backends.junit.load: Loads a file directly."""
+        backends.junit.REGISTRY.load(self.xml_file)
+
+    @utils.no_error
+    def test_load_dir(self):
+        """backends.junit.load: Loads a directory."""
+        backends.junit.REGISTRY.load(self.tdir)
+
+
+def test_load_file_name():
+    """backend.junit._load: uses the filename for name if filename != 'results'
+    """
+    with utils.tempdir() as tdir:
+        filename = os.path.join(tdir, 'foobar.xml')
+        with open(filename, 'w') as f:
+            f.write(_XML)
+
+        test = backends.junit.REGISTRY.load(filename)
+    nt.assert_equal(test.name, 'foobar')
+
+
+def test_load_folder_name():
+    """backend.junit._load: uses the foldername if the result is 'results'
+    """
+    with utils.tempdir() as tdir:
+        os.mkdir(os.path.join(tdir, 'a cool test'))
+        filename = os.path.join(tdir, 'a cool test', 'results.xml')
+        with open(filename, 'w') as f:
+            f.write(_XML)
+
+        test = backends.junit.REGISTRY.load(filename)
+    nt.assert_equal(test.name, 'a cool test')
+
+
+def test_load_default_name():
+    """backend.junit._load: uses 'junit result' for name as fallback"""
+    curdir = os.getcwd()
+ 
+    # This try/finally ensures that no matter what the directory will be changed
+    # back, this is necessary to ensure that it doesn't spoil the environment
+    # for other tests.
+    try:
+        with utils.tempdir() as tdir:
+            os.chdir(tdir)
+
+            filename = 'results.xml'
+            with open(filename, 'w') as f:
+                f.write(_XML)
+
+            test = backends.junit.REGISTRY.load(filename)
+    finally:
+        os.chdir(curdir)
+
+    nt.assert_equal(test.name, 'junit result')
diff --git a/framework/tests/utils.py b/framework/tests/utils.py
index f59f6eb..ce10cdd 100644
--- a/framework/tests/utils.py
+++ b/framework/tests/utils.py
@@ -25,6 +25,9 @@  in a single place.
 
 """
 
+# TODO: add a chdir decorator that gets the current dir, runs the test in a
+#       try and returns to the start dir in the finally
+
 from __future__ import print_function, absolute_import
 import os
 import sys
@@ -299,7 +302,7 @@  def no_error(func):
         try:
             func(*args, **kwargs)
         except Exception as e:
-            raise TestFailure(e.message)
+            raise TestFailure(*e.args)
 
     return test_wrapper