[v3,27/28] framework: Embed images in results.

Submitted by Dylan Baker on Oct. 31, 2016, 5:50 p.m.

Details

Message ID 1f3f10527dbf63ccca66eb571779be01b6bbc59a.1477936071.git-series.dylan@pnwbakers.com
State New
Headers show
Series "Series without cover letter" ( rev: 1 ) in Piglit

Not browsing as part of any series.

Commit Message

Dylan Baker Oct. 31, 2016, 5:50 p.m.
This helps to make the results format more portable, since it no longer
requires a separate image directory in a hardcoded location.

Signed-off-by: Dylan Baker <dylanx.c.baker@intel.com>
---
 framework/backends/json.py                             |  36 ++-
 framework/results.py                                   |   7 +-
 templates/test_result.mako                             |   8 +-
 tests/xts.py                                           |  24 +-
 unittests/framework/backends/images/LICENSE            |   6 +-
 unittests/framework/backends/images/openlogo-nd-25.png | Bin 0 -> 409 bytes
 unittests/framework/backends/images/ref.png            |   1 +-
 unittests/framework/backends/images/render.png         |   1 +-
 unittests/framework/backends/schema/piglit-10.json     | 135 ++++++++++-
 unittests/framework/backends/shared.py                 |   2 +-
 unittests/framework/backends/test_json_update.py       | 101 +++++++-
 11 files changed, 301 insertions(+), 20 deletions(-)
 create mode 100644 unittests/framework/backends/images/LICENSE
 create mode 100644 unittests/framework/backends/images/openlogo-nd-25.png
 create mode 120000 unittests/framework/backends/images/ref.png
 create mode 120000 unittests/framework/backends/images/render.png
 create mode 100644 unittests/framework/backends/schema/piglit-10.json

Patch hide | download patch | download mbox

diff --git a/framework/backends/json.py b/framework/backends/json.py
index 17002ed..bf64ad9 100644
--- a/framework/backends/json.py
+++ b/framework/backends/json.py
@@ -23,10 +23,10 @@ 
 from __future__ import (
     absolute_import, division, print_function, unicode_literals
 )
+import base64
 import collections
 import functools
 import os
-import posixpath
 import shutil
 import sys
 
@@ -53,7 +53,7 @@  __all__ = [
 ]
 
 # The current version of the JSON results
-CURRENT_JSON_VERSION = 9
+CURRENT_JSON_VERSION = 10
 
 # The minimum JSON format supported
 MINIMUM_SUPPORTED_VERSION = 7
@@ -312,6 +312,7 @@  def _update_results(results, filepath):
         updates = {
             7: _update_seven_to_eight,
             8: _update_eight_to_nine,
+            9: _update_nine_to_ten,
         }
 
         while results['results_version'] < CURRENT_JSON_VERSION:
@@ -387,6 +388,37 @@  def _update_eight_to_nine(result):
     return result
 
 
+def _update_nine_to_ten(result):
+    """Update json results from version 8 to 9.
+
+    This changes the PID feild of the TestResult object to alist of Integers or
+    null rather than a single integer or null.
+
+    """
+    for test in six.itervalues(result['tests']):
+        if 'images' in test:
+            new_list = []
+
+            for image in test['images']:
+                new = {'description': image['image_desc']}
+
+                if 'image_ref' in image:
+                    with open(image['image_ref'], 'rb') as f:
+                        new['reference'] = base64.b64encode(f.read()).decode('utf-8')
+
+                if 'image_render' in image:
+                    with open(image['image_render'], 'rb') as f:
+                        new['rendered'] = base64.b64encode(f.read()).decode('utf-8')
+
+                new_list.append(new)
+
+            test['images'] = new_list
+
+    result['results_version'] = 10
+
+    return result
+
+
 REGISTRY = Registry(
     extensions=['.json'],
     backend=JSONBackend,
diff --git a/framework/results.py b/framework/results.py
index 3c2bb2a..fc1011d 100644
--- a/framework/results.py
+++ b/framework/results.py
@@ -207,6 +207,11 @@  class TestResult(object):
             'dmesg': self.dmesg,
             'pid': self.pid,
         }
+
+        # Images are not required, only add them if they are not empty
+        if self.images:
+            obj['images'] = self.images
+
         return obj
 
     @classmethod
@@ -225,7 +230,7 @@  class TestResult(object):
         inst = cls()
 
         for each in ['returncode', 'command', 'exception', 'environment',
-                     'traceback', 'dmesg', 'pid', 'result']:
+                     'traceback', 'dmesg', 'pid', 'result', 'images']:
             if each in dict_:
                 setattr(inst, each, dict_[each])
 
diff --git a/templates/test_result.mako b/templates/test_result.mako
index ff08797..ab2b2b7 100644
--- a/templates/test_result.mako
+++ b/templates/test_result.mako
@@ -38,11 +38,11 @@ 
               <td>reference</td>
               <td>rendered</td>
             </tr>
-          % for image in images:
+          % for image in value.images:
             <tr>
-              <td>${image['image_desc']}</td>
-              <td><img src="file://${image['image_ref']}" /></td>
-              <td><img src="file://${image['image_render']}" /></td>
+              <td>${image['description']}</td>
+              <td><img alt="Reference Image" src="data:image/png;base64,${image['reference']}" /></td>
+              <td><img alt="Rendered Image" src="data:image/png;base64,${image['rendered']}" /></td>
             </tr>
           % endfor
           </table>
diff --git a/tests/xts.py b/tests/xts.py
index 715ecfa..d2f1e30 100644
--- a/tests/xts.py
+++ b/tests/xts.py
@@ -26,6 +26,7 @@ 
 from __future__ import (
     absolute_import, division, print_function, unicode_literals
 )
+import base64
 import os
 import re
 import subprocess
@@ -50,14 +51,6 @@  class XTSProfile(TestProfile):  # pylint: disable=too-few-public-methods
         """
         XTSTest.RESULTS_PATH = self.results_dir
 
-        try:
-            os.mkdir(os.path.join(self.results_dir, 'images'))
-        except OSError as e:
-            # If the exception is not 'directory already exists', raise the
-            # exception
-            if e.errno != 17:
-                raise
-
 
 class XTSTest(Test):  # pylint: disable=too-few-public-methods
     """ X Test Suite class
@@ -123,7 +116,9 @@  class XTSTest(Test):  # pylint: disable=too-few-public-methods
                 try:
                     out = subprocess.check_output(command, cwd=self.cwd)
                 except OSError:
-                    images.append({'image_desc': 'image processing failed'})
+                    images.append({'description': 'image processing failed',
+                                   'reference': None,
+                                   'rendered': None})
                     continue
 
                 # Each Err*.err log contains a rendered image, and a reference
@@ -139,13 +134,18 @@  class XTSTest(Test):  # pylint: disable=too-few-public-methods
                     self.RESULTS_PATH, 'images', '{1}-{2}-render.png'.format(
                         self.testname, match.group(1)))
 
+                with open(ref_path, 'rb') as f:
+                    ref = base64.b64encode(f.read()).decode('utf-8')
+                with open(render_path, 'rb') as f:
+                    render = base64.b64encode(f.read()).decode('utf-8')
+
                 split = out.splitlines()
                 os.rename(os.path.join(self.cwd, split[0]), render_path)
                 os.rename(os.path.join(self.cwd, split[1]), ref_path)
 
-                images.append({'image_desc': desc,
-                               'image_ref': ref_path,
-                               'image_render': render_path})
+                images.append({'description': desc,
+                               'reference': ref,
+                               'rendered': render})
 
         return images
 
diff --git a/unittests/framework/backends/images/LICENSE b/unittests/framework/backends/images/LICENSE
new file mode 100644
index 0000000..96286c0
--- /dev/null
+++ b/unittests/framework/backends/images/LICENSE
@@ -0,0 +1,6 @@ 
+The Debian Open Use Logo(s) are Copyright (c) 1999 Software in the Public
+Interest, Inc., and are released under the terms of the GNU Lesser General
+Public License, version 3 or any later version, or, at your option, of the
+Creative Commons Attribution-ShareAlike 3.0 Unported License.
+
+openlogo-nd-25.png and it's links are covered by this license.
diff --git a/unittests/framework/backends/images/openlogo-nd-25.png b/unittests/framework/backends/images/openlogo-nd-25.png
new file mode 100644
index 0000000..66f0ba0
Binary files /dev/null and b/unittests/framework/backends/images/openlogo-nd-25.png differ
diff --git a/unittests/framework/backends/images/ref.png b/unittests/framework/backends/images/ref.png
new file mode 120000
index 0000000..c7a1526
--- /dev/null
+++ b/unittests/framework/backends/images/ref.png
@@ -0,0 +1 @@ 
+openlogo-nd-25.png
\ No newline at end of file
diff --git a/unittests/framework/backends/images/render.png b/unittests/framework/backends/images/render.png
new file mode 120000
index 0000000..c7a1526
--- /dev/null
+++ b/unittests/framework/backends/images/render.png
@@ -0,0 +1 @@ 
+openlogo-nd-25.png
\ No newline at end of file
diff --git a/unittests/framework/backends/schema/piglit-10.json b/unittests/framework/backends/schema/piglit-10.json
new file mode 100644
index 0000000..440db14
--- /dev/null
+++ b/unittests/framework/backends/schema/piglit-10.json
@@ -0,0 +1,135 @@ 
+{
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "title": "TestrunResult",
+    "description": "The collection of all results",
+    "type": "object",
+    "properties": {
+        "__type__": { "type": "string" },
+        "clinfo": { "type": ["string", "null"] },
+        "glxinfo": { "type": ["string", "null"] },
+        "lspci": { "type": ["string", "null"] },
+        "wglinfo": { "type": ["string", "null"] },
+        "name": { "type": "string" },
+        "results_version": { "type": "number" },
+        "uname": { "type": [ "string", "null" ] },
+        "time_elapsed": { "$ref": "#/definitions/timeAttribute" },
+        "options": {
+            "descrption": "The options that were invoked with this run. These are implementation specific and not required.",
+            "type": "object",
+            "properties": {
+                "exclude_tests": { 
+                    "type": "array",
+                    "items": { "type": "string" },
+                    "uniqueItems": true
+                },
+                "include_filter": { 
+                    "type": "array",
+                    "items": { "type": "string" }
+                },
+                "exclude_filter": { 
+                    "type": "array",
+                    "items": { "type": "string" }
+                },
+                "sync": { "type": "boolean" },
+                "valgrind": { "type": "boolean" },
+                "monitored": { "type": "boolean" },
+                "dmesg": { "type": "boolean" },
+                "execute": { "type": "boolean" },
+                "concurrent": { "enum": ["none", "all", "some"] },
+                "platform": { "type": "string" },
+                "log_level": { "type": "string" },
+                "env": {
+                    "description": "Environment variables that must be specified",
+                    "type": "object",
+                    "additionalProperties": { "type": "string" }
+                },
+                "profile": {
+                    "type": "array",
+                    "items": { "type": "string" }
+                }
+            }
+        },
+        "totals": {
+            "type": "object",
+            "description": "A calculation of the group totals.",
+            "additionalProperties": {
+                "type": "object",
+                "properties": {
+                    "crash": { "type": "number" },
+                    "dmesg-fail": { "type": "number" },
+                    "dmesg-warn": { "type": "number" },
+                    "fail": { "type": "number" },
+                    "incomplete": { "type": "number" },
+                    "notrun": { "type": "number" },
+                    "pass": { "type": "number" },
+                    "skip": { "type": "number" },
+                    "timeout": { "type": "number" },
+                    "warn": { "type": "number" }
+                },
+                "additionalProperties": false,
+                "required": [ "crash", "dmesg-fail", "dmesg-warn", "fail", "incomplete", "notrun", "pass", "skip", "timeout", "warn" ]
+            }
+        },
+        "tests": {
+            "type": "object",
+            "additionalProperties": {
+                "type": "object",
+                "properties": {
+                    "__type__": { "type": "string" },
+                    "err": { "type": "string" },
+                    "exception": { "type": ["string", "null"] },
+                    "result": {
+                        "type": "string",
+                        "enum": [ "pass", "fail", "crash", "warn", "incomplete", "notrun", "skip", "dmesg-warn", "dmesg-fail" ]
+                    },
+                    "environment": { "type": "string" },
+                    "command": { "type": "string" },
+                    "traceback": { "type": ["string", "null"] },
+                    "out": { "type": "string" },
+                    "dmesg": { "type": "string" },
+                    "pid": {
+                        "type": "array",
+                        "items": { "type": "number" }
+                    },
+                    "returncode": { "type": [ "number", "null" ] },
+                    "time": { "$ref": "#/definitions/timeAttribute" },
+                    "subtests": {
+                        "type": "object",
+                        "properties": { "__type__": { "type": "string" } },
+                        "additionalProperties": { "type": "string" },
+                        "required": [ "__type__" ]
+                    },
+                    "images": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "description": { "type": "string" },
+                                "reference": { "type": "string" },
+                                "rendered": { "type": "string" }
+                            },
+                            "required": [ "description" ],
+                            "additionalProperties": false
+                        }
+                    }
+                },
+                "additionalProperties": false
+            }
+        }
+    },
+    "additionalProperties": false,
+    "required": [ "__type__", "clinfo", "glxinfo", "lspci", "wglinfo", "name", "results_version", "uname", "time_elapsed", "tests" ],
+    "definitions": {
+        "timeAttribute": {
+            "type": "object",
+            "description": "An element containing a start and end time",
+            "properties": {
+                "__type__": { "type": "string" },
+                "start": { "type": "number" },
+                "end": { "type": "number" }
+            },
+            "additionalProperties": false,
+            "required": [ "__type__", "start", "end" ]
+        }
+    }
+}
diff --git a/unittests/framework/backends/shared.py b/unittests/framework/backends/shared.py
index d9f5790..5232053 100644
--- a/unittests/framework/backends/shared.py
+++ b/unittests/framework/backends/shared.py
@@ -42,7 +42,7 @@  INITIAL_METADATA = {
 # changes. This does not contain piglit specifc objects, only strings, floats,
 # ints, and Nones (instead of JSON's null)
 JSON = {
-    "results_version": 9,
+    "results_version": 10,
     "time_elapsed": {
         "start": 1469638791.2351687,
         "__type__": "TimeAttribute",
diff --git a/unittests/framework/backends/test_json_update.py b/unittests/framework/backends/test_json_update.py
index c8e3ee6..b09413d 100644
--- a/unittests/framework/backends/test_json_update.py
+++ b/unittests/framework/backends/test_json_update.py
@@ -23,6 +23,7 @@ 
 from __future__ import (
     absolute_import, division, print_function, unicode_literals
 )
+import base64
 import os
 try:
     import simplejson as json
@@ -201,3 +202,103 @@  class TestV8toV9(object):
         jsonschema.validate(
             json.loads(json.dumps(result, default=backends.json.piglit_encoder)),
             schema)
+
+
+class TestV9toV10(object):
+    """Tests for Version 9 to version 10."""
+
+    ref = os.path.join(os.path.dirname(__file__), 'images', 'ref.png')
+    render = os.path.join(os.path.dirname(__file__), 'images', 'render.png')
+
+    data = {
+        "results_version": 9,
+        "name": "test",
+        "options": {
+            "profile": ['quick'],
+            "dmesg": False,
+            "verbose": False,
+            "platform": "gbm",
+            "sync": False,
+            "valgrind": False,
+            "filter": [],
+            "concurrent": "all",
+            "test_count": 0,
+            "exclude_tests": [],
+            "exclude_filter": [],
+            "env": {},
+        },
+        "lspci": "stuff",
+        "uname": "more stuff",
+        "glxinfo": "and stuff",
+        "wglinfo": "stuff",
+        "clinfo": "stuff",
+        "tests": {
+            'a@test': {
+                "time": {
+                    'start': 1.2,
+                    'end': 1.8,
+                    '__type__': 'TimeAttribute'
+                },
+                'dmesg': '',
+                'result': 'fail',
+                '__type__': 'TestResult',
+                'command': '/a/command',
+                'traceback': None,
+                'out': '',
+                'environment': 'A=variable',
+                'returncode': 0,
+                'err': '',
+                'pid': [5],
+                'subtests': {
+                    '__type__': 'Subtests',
+                },
+                'exception': None,
+                'images': [
+                    {
+                        'image_desc': "Test image",
+                        'image_ref': os.path.join(os.path.dirname(__file__),
+                                                  'images', 'ref.png'),
+                        'image_render': os.path.join(os.path.dirname(__file__),
+                                                     'images', 'render.png'),
+                    },
+                ],
+            }
+        },
+        "time_elapsed": {
+            'start': 1.2,
+            'end': 1.8,
+            '__type__': 'TimeAttribute'
+        },
+        '__type__': 'TestrunResult',
+    }
+
+    @pytest.fixture
+    def result(self, tmpdir):
+        p = tmpdir.join('result.json')
+        p.write(json.dumps(self.data, default=backends.json.piglit_encoder))
+        with p.open('r') as f:
+            return backends.json._update_nine_to_ten(backends.json._load(f))
+
+    def test_images(self, result):
+        """Converts the images attribute correctly."""
+        test = result['tests']['a@test']['images'][0]
+
+        with open(self.ref, 'rb') as f:
+            ref = base64.b64encode(f.read()).decode('utf-8')
+        assert test['reference'] == ref
+
+        with open(self.render, 'rb') as f:
+            rend = base64.b64encode(f.read()).decode('utf-8')
+        assert test['rendered'] == rend
+
+        assert test['description'] == \
+            self.data['tests']['a@test']['images'][0]['image_desc']
+
+    def test_valid(self, result):
+        with open(os.path.join(os.path.dirname(__file__), 'schema',
+                               'piglit-10.json'),
+                  'r') as f:
+            schema = json.load(f)
+        jsonschema.validate(
+            json.loads(json.dumps(result, default=backends.json.piglit_encoder)),
+            schema)