[4/9] framework: Implement boilerplate for removing process isolation

Submitted by Dylan Baker on Oct. 31, 2016, 10:25 p.m.

Details

Message ID 452e02dad54c0289a120748745231425a80c8b1e.1477952711.git-series.dylan@pnwbakers.com
State New
Headers show
Series "Add support to deqp based profiles to remove process isolation" ( rev: 1 ) in Piglit

Not browsing as part of any series.

Commit Message

Dylan Baker Oct. 31, 2016, 10:25 p.m.
This patches adds all of the boilerplate code that is required to get
group-at-a-time working. It adds the command line option, and all of the
switches to choose between creating a profile full of single tests, or
tests that run group at a time.

Signed-off-by: Dylan Baker <dylanx.c.baker@intel.com>
---
 framework/options.py                  |   1 +-
 framework/test/deqp.py                |  93 +++++++++++--
 unittests/framework/test/test_deqp.py | 196 +++++++++++++++++++++------
 3 files changed, 235 insertions(+), 55 deletions(-)

Patch hide | download patch | download mbox

diff --git a/framework/options.py b/framework/options.py
index 5cb88aa..8bf5d9d 100644
--- a/framework/options.py
+++ b/framework/options.py
@@ -181,6 +181,7 @@  class _Options(object):  # pylint: disable=too-many-instance-attributes
     monitored -- True if monitoring is desired. This forces concurrency off
     env -- environment variables set for each test before run
     deqp_mustpass -- True to enable the use of the deqp mustpass list feature.
+    process_isolation -- Whether to enforce process isolation. Default: True
     """
 
     include_filter = _ReListDescriptor('_include_filter', type_=_FilterReList)
diff --git a/framework/test/deqp.py b/framework/test/deqp.py
index 25dd077..1240ee2 100644
--- a/framework/test/deqp.py
+++ b/framework/test/deqp.py
@@ -90,13 +90,24 @@  def select_source(bin_, filename, mustpass, extra_args):
             gen_caselist_txt(bin_, filename, extra_args))
 
 
-def make_profile(test_list, test_class):
+def make_profile(test_list, single=None, group=None):
     """Create a TestProfile instance."""
+    if options.OPTIONS.process_isolation:
+        assert single is not None
+        if isinstance(single, DEQPUnsupportedMode):
+            raise exceptions.PiglitFatalError(single)
+        test_class = single
+    else:
+        assert group is not None
+        if isinstance(group, DEQPUnsupportedMode):
+            raise exceptions.PiglitFatalError(group)
+        test_class = group
+
     profile = TestProfile()
     for testname in test_list:
         # deqp uses '.' as the testgroup separator.
-        piglit_name = testname.replace('.', grouptools.SEPARATOR)
-        profile.test_list[piglit_name] = test_class(testname)
+        piglit_name = testname[0].replace('.', grouptools.SEPARATOR)
+        profile.test_list[piglit_name] = test_class(*testname)
 
     return profile
 
@@ -104,17 +115,34 @@  def make_profile(test_list, test_class):
 def gen_mustpass_tests(mp_list):
     """Return a testlist from the mustpass list."""
     root = et.parse(mp_list).getroot()
-    group = []
+    cur_group = []
 
-    def gen(root):
+    def single(root):
         for elem in root:
             if elem.tag == 'Test':
-                yield '{}.{}'.format('.'.join(group), elem.get('name'))
+                yield ('{}.{}'.format('.'.join(cur_group), elem.get('name')), )
             else:
-                group.append(elem.get('name'))
-                for test in gen(elem):
+                cur_group.append(elem.get('name'))
+                for test in single(elem):
+                    yield test
+                del cur_group[-1]
+
+    def group(root):
+        for elem in root:
+            if elem.tag == 'TestCase':
+                case = '{}.{}'.format('.'.join(cur_group), elem.get('name'))
+                yield (case, ['{}.{}'.format(case, t.get('name'))
+                              for t in elem.findall('.//Test')])
+            elif elem.tag == 'TestSuite':
+                cur_group.append(elem.get('name'))
+                for test in group(elem):
                     yield test
-                del group[-1]
+                del cur_group[-1]
+
+    if options.OPTIONS.process_isolation:
+        gen = single
+    else:
+        gen = group
 
     for test in gen(root):
         yield test
@@ -150,15 +178,54 @@  def gen_caselist_txt(bin_, caselist, extra_args):
 
 def iter_deqp_test_cases(case_file):
     """Iterate over original dEQP testcase names."""
-    with open(case_file, 'r') as caselist_file:
-        for i, line in enumerate(caselist_file):
+    def single(f):
+        """Iterate over the txt file, and yield each test instance."""
+        for i, line in enumerate(f):
             if line.startswith('GROUP:'):
                 continue
             elif line.startswith('TEST:'):
-                yield line[len('TEST:'):].strip()
+                # The group mode yields a tuple, so the single mode needs to as
+                # well.
+                yield (line[len('TEST:'):].strip(), )
             else:
                 raise exceptions.PiglitFatalError(
-                    'deqp: {}:{}: ill-formed line'.format(case_file, i))
+                    'deqp: {}:{}: ill-formed line'.format(f.name, i))
+
+    def group(f):
+        """Iterate over the txt file, and yield each group and its members.
+
+        The group must contain actual members to be yielded.
+        """
+        group = ''
+        tests = []
+
+        for i, line in enumerate(f):
+            if line.startswith('GROUP:'):
+                new = line[len('GROUP:'):].strip()
+                if group != new and tests:
+                    yield (group, tests)
+                    tests = []
+                group = new
+            elif line.startswith('TEST:'):
+                tests.append(line[len('TEST:'):].strip())
+            else:
+                raise exceptions.PiglitFatalError(
+                    'deqp: {}:{}: ill-formed line'.format(f.name, i))
+        # If we get to the end of the file and we have new tests (the would
+        # have been cleared if there weren't any.
+        if tests:
+            yield (group, tests)
+
+    adder = None
+    if options.OPTIONS.process_isolation:
+        adder = single
+    else:
+        adder = group
+    assert adder is not None
+
+    with open(case_file, 'r') as f:
+        for x in adder(f):
+            yield x
 
 
 def format_trie_list(classname, testnames):
diff --git a/unittests/framework/test/test_deqp.py b/unittests/framework/test/test_deqp.py
index 58504fd..985f2c4 100644
--- a/unittests/framework/test/test_deqp.py
+++ b/unittests/framework/test/test_deqp.py
@@ -39,11 +39,12 @@  import six
 
 from framework import exceptions
 from framework import grouptools
+from framework import options
 from framework import profile
 from framework import status
 from framework.test import deqp
 
-# pylint:disable=invalid-name,no-self-use
+# pylint:disable=no-self-use,protected-access
 
 
 class _DEQPTestTest(deqp.DEQPSingleTest):
@@ -116,44 +117,123 @@  class TestGetOptions(object):
 class TestMakeProfile(object):
     """Test deqp.make_profile."""
 
-    @classmethod
-    def setup_class(cls):
-        cls.profile = deqp.make_profile(['this.is.a.deqp.test'], _DEQPTestTest)
+    class TestSingle(object):
+        """Tests for the single mode."""
+
+        @classmethod
+        def setup_class(cls):
+            with mock.patch('framework.test.deqp.options.OPTIONS',
+                            new=options._Options()) as mocked:
+                mocked.process_isolation = True
+                cls.profile = deqp.make_profile([('this.is.a.deqp.test', )],
+                                                single=_DEQPTestTest)
 
-    def test_returns_profile(self):
-        """deqp.make_profile: returns a TestProfile."""
-        assert isinstance(self.profile, profile.TestProfile)
+        def test_returns_profile(self):
+            """deqp.make_profile: returns a TestProfile."""
+            assert isinstance(self.profile, profile.TestProfile)
 
-    def test_replaces_separator(self):
-        """deqp.make_profile: replaces '.' with grouptools.separator"""
-        expected = grouptools.join('this', 'is', 'a', 'deqp', 'test')
-        assert expected in self.profile.test_list
+        def test_replaces_separator(self):
+            """deqp.make_profile: replaces '.' with grouptools.separator"""
+            expected = grouptools.join('this', 'is', 'a', 'deqp', 'test')
+            assert expected in self.profile.test_list
 
+    class TestGroup(object):
+        """Tests for group mode."""
 
-class TestIterDeqpTestCases(object):
-    """Tests for iter_deqp_test_cases."""
+        @classmethod
+        def setup_class(cls):
+            with mock.patch('framework.test.deqp.options.OPTIONS',
+                            new=options._Options()) as mocked:
+                mocked.process_isolation = False
+                cls.profile = deqp.make_profile(
+                    [('this.is.a.deqp', ['this.is.a.deqp.test',
+                                         'this.is.a.deqp.thing'])],
+                    group=_DEQPGroupTrieTest)
 
-    def _do_test(self, write, expect, tmpdir):
-        """Run the acutal test."""
-        p = tmpdir.join('foo')
-        p.write(write)
-        gen = deqp.iter_deqp_test_cases(six.text_type(p))
-        assert next(gen) == expect
+        def test_returns_profile(self):
+            """deqp.make_profile: returns a TestProfile."""
+            assert isinstance(self.profile, profile.TestProfile)
 
-    def test_test_cases(self, tmpdir):
-        """Correctly detects a test line."""
-        self._do_test('TEST: a.deqp.test', 'a.deqp.test', tmpdir)
+        def test_replaces_separator(self):
+            """deqp.make_profile: replaces '.' with grouptools.separator"""
+            expected = grouptools.join('this', 'is', 'a', 'deqp')
+            assert expected in self.profile.test_list
 
-    def test_test_group(self, tmpdir):
-        """Correctly detects a group line."""
-        self._do_test('GROUP: a group\nTEST: a.deqp.test', 'a.deqp.test',
-                      tmpdir)
 
-    def test_bad_entry(self, tmpdir):
-        """A PiglitFatalException is raised if a line is not a TEST or GROUP.
-        """
-        with pytest.raises(exceptions.PiglitFatalError):
-            self._do_test('this will fail', None, tmpdir)
+class TestIterDeqpTestCases(object):
+    """Tests for iter_deqp_test_cases."""
+
+    class TestSingle(object):
+        """Tests for the single mode."""
+
+        @pytest.yield_fixture(autouse=True, scope='class')
+        def group_setup(self):
+            with mock.patch('framework.test.deqp.options.OPTIONS',
+                            new=options._Options()) as mocked:
+                mocked.process_isolation = True
+                yield
+
+        def _do_test(self, write, expect, tmpdir):
+            """Run the acutal test."""
+            p = tmpdir.join('foo')
+            p.write(write)
+            gen = deqp.iter_deqp_test_cases(six.text_type(p))
+            actual = next(gen)
+            assert actual == expect
+
+        def test_test_cases(self, tmpdir):
+            """Correctly detects a test line."""
+            self._do_test('TEST: a.deqp.test', ('a.deqp.test', ), tmpdir)
+
+        def test_test_group(self, tmpdir):
+            """Correctly detects a group line."""
+            self._do_test('GROUP: a group\nTEST: a.deqp.test',
+                          ('a.deqp.test', ), tmpdir)
+
+        def test_bad_entry(self, tmpdir):
+            """A PiglitFatalException is raised if a line is not a TEST or
+            GROUP.
+            """
+            with pytest.raises(exceptions.PiglitFatalError):
+                self._do_test('this will fail', None, tmpdir)
+
+    class TestGroup(object):
+        """Tests for the group mode."""
+
+        @pytest.yield_fixture(autouse=True, scope='class')
+        def group_setup(self):
+            with mock.patch('framework.test.deqp.options.OPTIONS',
+                            new=options._Options()) as mocked:
+                mocked.process_isolation = False
+                yield
+
+        def _do_test(self, write, expect, tmpdir):
+            """Run the acutal test."""
+            p = tmpdir.join('foo')
+            p.write(write)
+            gen = deqp.iter_deqp_test_cases(six.text_type(p))
+            actual = next(gen)
+            assert actual == expect
+
+        def test_test_cases(self, tmpdir):
+            """Correctly detects a test line."""
+            self._do_test(
+                textwrap.dedent("""\
+                    GROUP: a
+                    GROUP: a.deqp
+                    TEST: a.deqp.test
+                    TEST: a.deqp.failure
+                    TEST: a.deqp.foo
+                """),
+                ('a.deqp', ['a.deqp.test', 'a.deqp.failure', 'a.deqp.foo']),
+                tmpdir)
+
+        def test_bad_entry(self, tmpdir):
+            """A PiglitFatalException is raised if a line is not a TEST or
+            GROUP.
+            """
+            with pytest.raises(exceptions.PiglitFatalError):
+                self._do_test('this will fail', None, tmpdir)
 
 
 class TestFormatTrieList(object):
@@ -460,7 +540,7 @@  class TestDEQPGroupTest(object):
 class TestGenMustpassTests(object):
     """Tests for the gen_mustpass_tests function."""
 
-    _xml = textwrap.dedent("""\
+    xml = textwrap.dedent("""\
         <?xml version="1.0" encoding="UTF-8"?>
         <TestPackage name="dEQP-piglit-test" appPackageName="com.freedesktop.org.piglit.deqp" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp" deqp:glesVersion="196608">
             <TestSuite name="dEQP.piglit">
@@ -478,13 +558,45 @@  class TestGenMustpassTests(object):
         </TestPackage>
     """)
 
-    def test_basic(self, tmpdir):
-        p = tmpdir.join('foo.xml')
-        p.write(self._xml)
-        tests = set(deqp.gen_mustpass_tests(six.text_type(p)))
-        assert tests == {
-            'dEQP.piglit.group1.test1',
-            'dEQP.piglit.group1.test2',
-            'dEQP.piglit.nested.group2.test3',
-            'dEQP.piglit.nested.group2.test4',
-        }
+    class TestSingle(object):
+        """Tests for single mode."""
+
+        @pytest.yield_fixture(autouse=True, scope='class')
+        def group_setup(self):
+            with mock.patch('framework.test.deqp.options.OPTIONS',
+                            new=options._Options()) as mocked:
+                mocked.process_isolation = True
+                yield
+
+        def test_basic(self, tmpdir):
+            p = tmpdir.join('foo.xml')
+            p.write(TestGenMustpassTests.xml)
+            tests = list(deqp.gen_mustpass_tests(six.text_type(p)))
+            assert tests == [
+                ('dEQP.piglit.group1.test1', ),
+                ('dEQP.piglit.group1.test2', ),
+                ('dEQP.piglit.nested.group2.test3', ),
+                ('dEQP.piglit.nested.group2.test4', ),
+            ]
+
+    class TestGroup(object):
+        """Tests for groupmode."""
+
+        @pytest.yield_fixture(autouse=True, scope='class')
+        def group_setup(self):
+            with mock.patch('framework.test.deqp.options.OPTIONS',
+                            new=options._Options()) as mocked:
+                mocked.process_isolation = False
+                yield
+
+        def test_basic(self, tmpdir):
+            p = tmpdir.join('foo.xml')
+            p.write(TestGenMustpassTests.xml)
+            tests = list(deqp.gen_mustpass_tests(six.text_type(p)))
+            assert tests == [
+                ('dEQP.piglit.group1',
+                 ['dEQP.piglit.group1.test1', 'dEQP.piglit.group1.test2']),
+                ('dEQP.piglit.nested.group2',
+                 ['dEQP.piglit.nested.group2.test3',
+                  'dEQP.piglit.nested.group2.test4']),
+            ]