tests: Add integration with Khronos CTS OpenGL runner

Submitted by Juan A. Suarez Romero on May 18, 2016, 1:34 p.m.

Details

Message ID 1463578447-27220-1-git-send-email-jasuarez@igalia.com
State New
Headers show
Series "tests: Add integration with Khronos CTS OpenGL runner" ( rev: 1 ) in Piglit

Not browsing as part of any series.

Commit Message

Juan A. Suarez Romero May 18, 2016, 1:34 p.m.
This adds support for running Khronos' deqp-based conformance suite for
OpenGL with piglit.
---
 piglit.conf.example             | 10 ++++++++++
 tests/{cts.py => cts_gl45.py}   | 43 +++++++++++++++++++++++++----------------
 tests/{cts.py => cts_gles31.py} |  2 +-
 3 files changed, 37 insertions(+), 18 deletions(-)
 copy tests/{cts.py => cts_gl45.py} (56%)
 rename tests/{cts.py => cts_gles31.py} (97%)

Patch hide | download patch | download mbox

diff --git a/piglit.conf.example b/piglit.conf.example
index 944d5d9..3cce52a 100644
--- a/piglit.conf.example
+++ b/piglit.conf.example
@@ -95,6 +95,16 @@  testB
 ; overrides the value set here.
 ;extra_args=--deqp-visibility hidden
 
+[cts_gl]
+; path to the cts opengl executable
+; can be overwritten by PIGLIT_CTS_GL_BIN environment variable
+;bin=/home/knuth/cts/cts/glcts
+
+; Space-separated list of extra command line arguments for cts. The
+; option is not required. The environment variable PIGLIT_CTS_GL_EXTRA_ARGS
+; overrides the value set here.
+;extra_args=--deqp-visibility hidden
+
 ; Section for specific oclconform test.  One of these sections is required for
 ; each test list in the oclconform section and must be called:
 ; oclconform-$testname
diff --git a/tests/cts.py b/tests/cts_gl45.py
similarity index 56%
copy from tests/cts.py
copy to tests/cts_gl45.py
index 0e64e1b..607b79b 100644
--- a/tests/cts.py
+++ b/tests/cts_gl45.py
@@ -20,25 +20,24 @@ 
 
 """Piglit integration for Khronos CTS tests.
 
-By default this will run GLES2, GLES3, GLES31, and GLESEXT test cases. Those
-desiring to run only a subset of them should consider using the -t or -x
-options to include or exclude tests.
+By default this will run GL30, GL31, GL32, GL33, GL40, GL41, GL42, GL43, GL44
+and GL45 test cases. Those desiring to run only a subset of them should consider
+using the -t or -x options to include or exclude tests.
 
 For example:
-./piglit run cts -c foo -t ES3- would run only ES3 tests (note the dash to
-exclude ES31 tests)
+./piglit run cts_gl45 -c foo -t GL30- would run only GL30 tests
 
 This integration requires some configuration in piglit.conf, or the use of
 environment variables.
 
 In piglit.conf one should set the following:
-[cts]:bin -- Path to the glcts binary
-[cts]:extra_args -- any extra arguments to be passed to cts (optional)
+[cts_gl]:bin -- Path to the glcts binary
+[cts_gl]:extra_args -- any extra arguments to be passed to cts (optional)
 
 Alternatively (or in addition, since environment variables have precedence),
 one could set:
-PIGLIT_CTS_BIN -- environment equivalent of [cts]:bin
-PIGLIT_CTS_EXTRA_ARGS -- environment equivalent of [cts]:extra_args
+PIGLIT_CTS_GL_BIN -- environment equivalent of [cts_gl]:bin
+PIGLIT_CTS_GL_EXTRA_ARGS -- environment equivalent of [cts_gl]:extra_args
 
 """
 
@@ -51,9 +50,9 @@  from framework.test import deqp
 
 __all__ = ['profile']
 
-_CTS_BIN = deqp.get_option('PIGLIT_CTS_BIN', ('cts', 'bin'))
+_CTS_BIN = deqp.get_option('PIGLIT_CTS_GL_BIN', ('cts_gl', 'bin'))
 
-_EXTRA_ARGS = deqp.get_option('PIGLIT_CTS_EXTRA_ARGS', ('cts', 'extra_args'),
+_EXTRA_ARGS = deqp.get_option('PIGLIT_CTS_GL_EXTRA_ARGS', ('cts_gl', 'extra_args'),
                               default='').split()
 
 
@@ -65,18 +64,28 @@  class DEQPCTSTest(deqp.DEQPBaseTest):
         return super(DEQPCTSTest, self).extra_args + \
             [x for x in _EXTRA_ARGS if not x.startswith('--deqp-case')]
 
-
 # Add all of the suites by default, users can use filters to remove them.
 profile = deqp.make_profile(  # pylint: disable=invalid-name
     itertools.chain(
         deqp.iter_deqp_test_cases(
-            deqp.gen_caselist_txt(_CTS_BIN, 'ES2-CTS-cases.txt', _EXTRA_ARGS)),
+            deqp.gen_caselist_txt(_CTS_BIN, 'GL30-CTS-cases.txt', _EXTRA_ARGS)),
+        deqp.iter_deqp_test_cases(
+            deqp.gen_caselist_txt(_CTS_BIN, 'GL31-CTS-cases.txt', _EXTRA_ARGS)),
+        deqp.iter_deqp_test_cases(
+            deqp.gen_caselist_txt(_CTS_BIN, 'GL32-CTS-cases.txt', _EXTRA_ARGS)),
+        deqp.iter_deqp_test_cases(
+            deqp.gen_caselist_txt(_CTS_BIN, 'GL33-CTS-cases.txt', _EXTRA_ARGS)),
+        deqp.iter_deqp_test_cases(
+            deqp.gen_caselist_txt(_CTS_BIN, 'GL40-CTS-cases.txt', _EXTRA_ARGS)),
+        deqp.iter_deqp_test_cases(
+            deqp.gen_caselist_txt(_CTS_BIN, 'GL41-CTS-cases.txt', _EXTRA_ARGS)),
+        deqp.iter_deqp_test_cases(
+            deqp.gen_caselist_txt(_CTS_BIN, 'GL42-CTS-cases.txt', _EXTRA_ARGS)),
         deqp.iter_deqp_test_cases(
-            deqp.gen_caselist_txt(_CTS_BIN, 'ES3-CTS-cases.txt', _EXTRA_ARGS)),
+            deqp.gen_caselist_txt(_CTS_BIN, 'GL43-CTS-cases.txt', _EXTRA_ARGS)),
         deqp.iter_deqp_test_cases(
-            deqp.gen_caselist_txt(_CTS_BIN, 'ES31-CTS-cases.txt', _EXTRA_ARGS)),
+            deqp.gen_caselist_txt(_CTS_BIN, 'GL44-CTS-cases.txt', _EXTRA_ARGS)),
         deqp.iter_deqp_test_cases(
-            deqp.gen_caselist_txt(_CTS_BIN, 'ESEXT-CTS-cases.txt',
-                                  _EXTRA_ARGS)),
+            deqp.gen_caselist_txt(_CTS_BIN, 'GL45-CTS-cases.txt', _EXTRA_ARGS)),
     ),
     DEQPCTSTest)
diff --git a/tests/cts.py b/tests/cts_gles31.py
similarity index 97%
rename from tests/cts.py
rename to tests/cts_gles31.py
index 0e64e1b..ad28e58 100644
--- a/tests/cts.py
+++ b/tests/cts_gles31.py
@@ -25,7 +25,7 @@  desiring to run only a subset of them should consider using the -t or -x
 options to include or exclude tests.
 
 For example:
-./piglit run cts -c foo -t ES3- would run only ES3 tests (note the dash to
+./piglit run cts_gles31 -c foo -t ES3- would run only ES3 tests (note the dash to
 exclude ES31 tests)
 
 This integration requires some configuration in piglit.conf, or the use of

Comments

Quoting Juan A. Suarez Romero (2016-05-18 06:34:07)
> This adds support for running Khronos' deqp-based conformance suite for
> OpenGL with piglit.
> ---
>  piglit.conf.example             | 10 ++++++++++
>  tests/{cts.py => cts_gl45.py}   | 43 +++++++++++++++++++++++++----------------
>  tests/{cts.py => cts_gles31.py} |  2 +-

Since these run all of the tests I think I'd prefer cts_gles and cts_gl,
but that's a bit of a nit-pick.

>  3 files changed, 37 insertions(+), 18 deletions(-)
>  copy tests/{cts.py => cts_gl45.py} (56%)
>  rename tests/{cts.py => cts_gles31.py} (97%)
> 
> diff --git a/piglit.conf.example b/piglit.conf.example
> index 944d5d9..3cce52a 100644
> --- a/piglit.conf.example
> +++ b/piglit.conf.example
> @@ -95,6 +95,16 @@ testB
>  ; overrides the value set here.
>  ;extra_args=--deqp-visibility hidden
>  
> +[cts_gl]
> +; path to the cts opengl executable
> +; can be overwritten by PIGLIT_CTS_GL_BIN environment variable
> +;bin=/home/knuth/cts/cts/glcts
> +
> +; Space-separated list of extra command line arguments for cts. The
> +; option is not required. The environment variable PIGLIT_CTS_GL_EXTRA_ARGS
> +; overrides the value set here.
> +;extra_args=--deqp-visibility hidden
> +
>  ; Section for specific oclconform test.  One of these sections is required for
>  ; each test list in the oclconform section and must be called:
>  ; oclconform-$testname
> diff --git a/tests/cts.py b/tests/cts_gl45.py
> similarity index 56%
> copy from tests/cts.py
> copy to tests/cts_gl45.py
> index 0e64e1b..607b79b 100644
> --- a/tests/cts.py
> +++ b/tests/cts_gl45.py
> @@ -20,25 +20,24 @@
>  
>  """Piglit integration for Khronos CTS tests.
>  
> -By default this will run GLES2, GLES3, GLES31, and GLESEXT test cases. Those
> -desiring to run only a subset of them should consider using the -t or -x
> -options to include or exclude tests.
> +By default this will run GL30, GL31, GL32, GL33, GL40, GL41, GL42, GL43, GL44
> +and GL45 test cases. Those desiring to run only a subset of them should consider
> +using the -t or -x options to include or exclude tests.
>  
>  For example:
> -./piglit run cts -c foo -t ES3- would run only ES3 tests (note the dash to
> -exclude ES31 tests)
> +./piglit run cts_gl45 -c foo -t GL30- would run only GL30 tests
>  
>  This integration requires some configuration in piglit.conf, or the use of
>  environment variables.
>  
>  In piglit.conf one should set the following:
> -[cts]:bin -- Path to the glcts binary
> -[cts]:extra_args -- any extra arguments to be passed to cts (optional)
> +[cts_gl]:bin -- Path to the glcts binary
> +[cts_gl]:extra_args -- any extra arguments to be passed to cts (optional)

>  
>  Alternatively (or in addition, since environment variables have precedence),
>  one could set:
> -PIGLIT_CTS_BIN -- environment equivalent of [cts]:bin
> -PIGLIT_CTS_EXTRA_ARGS -- environment equivalent of [cts]:extra_args
> +PIGLIT_CTS_GL_BIN -- environment equivalent of [cts_gl]:bin
> +PIGLIT_CTS_GL_EXTRA_ARGS -- environment equivalent of [cts_gl]:extra_args
>  
>  """
>  
> @@ -51,9 +50,9 @@ from framework.test import deqp
>  
>  __all__ = ['profile']
>  
> -_CTS_BIN = deqp.get_option('PIGLIT_CTS_BIN', ('cts', 'bin'))
> +_CTS_BIN = deqp.get_option('PIGLIT_CTS_GL_BIN', ('cts_gl', 'bin'))
>  
> -_EXTRA_ARGS = deqp.get_option('PIGLIT_CTS_EXTRA_ARGS', ('cts', 'extra_args'),
> +_EXTRA_ARGS = deqp.get_option('PIGLIT_CTS_GL_EXTRA_ARGS', ('cts_gl', 'extra_args'),
>                                default='').split()
>  
>  
> @@ -65,18 +64,28 @@ class DEQPCTSTest(deqp.DEQPBaseTest):
>          return super(DEQPCTSTest, self).extra_args + \
>              [x for x in _EXTRA_ARGS if not x.startswith('--deqp-case')]
>  
> -
>  # Add all of the suites by default, users can use filters to remove them.
>  profile = deqp.make_profile(  # pylint: disable=invalid-name
>      itertools.chain(
>          deqp.iter_deqp_test_cases(
> -            deqp.gen_caselist_txt(_CTS_BIN, 'ES2-CTS-cases.txt', _EXTRA_ARGS)),
> +            deqp.gen_caselist_txt(_CTS_BIN, 'GL30-CTS-cases.txt', _EXTRA_ARGS)),
> +        deqp.iter_deqp_test_cases(
> +            deqp.gen_caselist_txt(_CTS_BIN, 'GL31-CTS-cases.txt', _EXTRA_ARGS)),
> +        deqp.iter_deqp_test_cases(
> +            deqp.gen_caselist_txt(_CTS_BIN, 'GL32-CTS-cases.txt', _EXTRA_ARGS)),
> +        deqp.iter_deqp_test_cases(
> +            deqp.gen_caselist_txt(_CTS_BIN, 'GL33-CTS-cases.txt', _EXTRA_ARGS)),
> +        deqp.iter_deqp_test_cases(
> +            deqp.gen_caselist_txt(_CTS_BIN, 'GL40-CTS-cases.txt', _EXTRA_ARGS)),
> +        deqp.iter_deqp_test_cases(
> +            deqp.gen_caselist_txt(_CTS_BIN, 'GL41-CTS-cases.txt', _EXTRA_ARGS)),
> +        deqp.iter_deqp_test_cases(
> +            deqp.gen_caselist_txt(_CTS_BIN, 'GL42-CTS-cases.txt', _EXTRA_ARGS)),
>          deqp.iter_deqp_test_cases(
> -            deqp.gen_caselist_txt(_CTS_BIN, 'ES3-CTS-cases.txt', _EXTRA_ARGS)),
> +            deqp.gen_caselist_txt(_CTS_BIN, 'GL43-CTS-cases.txt', _EXTRA_ARGS)),
>          deqp.iter_deqp_test_cases(
> -            deqp.gen_caselist_txt(_CTS_BIN, 'ES31-CTS-cases.txt', _EXTRA_ARGS)),
> +            deqp.gen_caselist_txt(_CTS_BIN, 'GL44-CTS-cases.txt', _EXTRA_ARGS)),
>          deqp.iter_deqp_test_cases(
> -            deqp.gen_caselist_txt(_CTS_BIN, 'ESEXT-CTS-cases.txt',
> -                                  _EXTRA_ARGS)),
> +            deqp.gen_caselist_txt(_CTS_BIN, 'GL45-CTS-cases.txt', _EXTRA_ARGS)),
>      ),
>      DEQPCTSTest)
> diff --git a/tests/cts.py b/tests/cts_gles31.py
> similarity index 97%
> rename from tests/cts.py
> rename to tests/cts_gles31.py
> index 0e64e1b..ad28e58 100644
> --- a/tests/cts.py
> +++ b/tests/cts_gles31.py
> @@ -25,7 +25,7 @@ desiring to run only a subset of them should consider using the -t or -x
>  options to include or exclude tests.
>  
>  For example:
> -./piglit run cts -c foo -t ES3- would run only ES3 tests (note the dash to
> +./piglit run cts_gles31 -c foo -t ES3- would run only ES3 tests (note the dash to
>  exclude ES31 tests)
>  
>  This integration requires some configuration in piglit.conf, or the use of
> -- 
> 2.5.5
> 
> _______________________________________________
> Piglit mailing list
> Piglit@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/piglit

Other than the one nit this looks good. Please be sure to let Mark Janes
(cc'd) or myself know when you push this, as we'll need to update our CI
since this will change some profile names.

Reviewed-by: Dylan Baker <dylan@pnwbakers.com>
On Wed, 2016-05-18 at 09:20 -0700, Dylan Baker wrote:
> Quoting Juan A. Suarez Romero (2016-05-18 06:34:07)
> > This adds support for running Khronos' deqp-based conformance suite
> for
> > OpenGL with piglit.
> > ---
> >  piglit.conf.example             | 10 ++++++++++
> >  tests/{cts.py => cts_gl45.py}   | 43 +++++++++++++++++++++++++--
> --------------
> >  tests/{cts.py => cts_gles31.py} |  2 +-
> 
> Since these run all of the tests I think I'd prefer cts_gles and
> cts_gl,
> but that's a bit of a nit-pick.

What about the envvars / piglit.conf?

I keep PIGLIT_CTS_BIN for gles, while using PIGLIT_CTS_GL_BIN for gl.

Does it make sense to rename the envvar and use also
PIGLIT_CTS_GLES_BIN / PIGLIT_CTS_GLES_EXTRA_ARGS for gles, and
PIGLIT_CTS_GL_BIN / PIGLIT_CTS_GL_EXTRA_ARGS for gl?


	J.A.
Quoting Juan A. Suarez Romero (2016-05-18 09:27:34)
> On Wed, 2016-05-18 at 09:20 -0700, Dylan Baker wrote:
> > Quoting Juan A. Suarez Romero (2016-05-18 06:34:07)
> > > This adds support for running Khronos' deqp-based conformance suite
> > for
> > > OpenGL with piglit.
> > > ---
> > >  piglit.conf.example             | 10 ++++++++++
> > >  tests/{cts.py => cts_gl45.py}   | 43 +++++++++++++++++++++++++--
> > --------------
> > >  tests/{cts.py => cts_gles31.py} |  2 +-
> > 
> > Since these run all of the tests I think I'd prefer cts_gles and
> > cts_gl,
> > but that's a bit of a nit-pick.
> 
> What about the envvars / piglit.conf?
> 
> I keep PIGLIT_CTS_BIN for gles, while using PIGLIT_CTS_GL_BIN for gl.
> 
> Does it make sense to rename the envvar and use also
> PIGLIT_CTS_GLES_BIN / PIGLIT_CTS_GLES_EXTRA_ARGS for gles, and
> PIGLIT_CTS_GL_BIN / PIGLIT_CTS_GL_EXTRA_ARGS for gl?
> 
> 
>         J.A.

I'm 50/50 on it. While I think it probably is nicer to change them, it
also creates more churn for people using the functionality.

My r-b remains whichever way you decide to go.

Dylan
On Wed, 2016-05-18 at 11:00 -0700, Dylan Baker wrote:
> I'm 50/50 on it. While I think it probably is nicer to change them,
> it
> also creates more churn for people using the functionality.


Then, I'll keep as it is. We can always change it in future, if
required.


Thanks a lot!

	J.A.
On 19 May 2016 at 07:33, Juan A. Suarez Romero <jasuarez@igalia.com> wrote:
> On Wed, 2016-05-18 at 11:00 -0700, Dylan Baker wrote:
>> I'm 50/50 on it. While I think it probably is nicer to change them,
>> it
>> also creates more churn for people using the functionality.
>
>
> Then, I'll keep as it is. We can always change it in future, if
> required.
>
Wouldn't it be better to get all the churn in one go, considering one
already needs to tweak things, manually (the cts.py > cts_gles.py
rename) ?

-Emil
Quoting Emil Velikov (2016-05-19 02:52:35)
> On 19 May 2016 at 07:33, Juan A. Suarez Romero <jasuarez@igalia.com> wrote:
> > On Wed, 2016-05-18 at 11:00 -0700, Dylan Baker wrote:
> >> I'm 50/50 on it. While I think it probably is nicer to change them,
> >> it
> >> also creates more churn for people using the functionality.
> >
> >
> > Then, I'll keep as it is. We can always change it in future, if
> > required.
> >
> Wouldn't it be better to get all the churn in one go, considering one
> already needs to tweak things, manually (the cts.py > cts_gles.py
> rename) ?
> 
> -Emil

Agreed. Either we should change the name now or never do it.

I think I prefer doing it now.

Dylan
Dylan Baker <dylan@pnwbakers.com> writes:

> Quoting Juan A. Suarez Romero (2016-05-18 06:34:07)
>> This adds support for running Khronos' deqp-based conformance suite for
>> OpenGL with piglit.
>> ---
>>  piglit.conf.example             | 10 ++++++++++
>>  tests/{cts.py => cts_gl45.py}   | 43 +++++++++++++++++++++++++----------------
>>  tests/{cts.py => cts_gles31.py} |  2 +-
>
> Since these run all of the tests I think I'd prefer cts_gles and cts_gl,
> but that's a bit of a nit-pick.
>
>>  3 files changed, 37 insertions(+), 18 deletions(-)
>>  copy tests/{cts.py => cts_gl45.py} (56%)
>>  rename tests/{cts.py => cts_gles31.py} (97%)
>> 
>> diff --git a/piglit.conf.example b/piglit.conf.example
>> index 944d5d9..3cce52a 100644
>> --- a/piglit.conf.example
>> +++ b/piglit.conf.example
>> @@ -95,6 +95,16 @@ testB
>>  ; overrides the value set here.
>>  ;extra_args=--deqp-visibility hidden
>>  
>> +[cts_gl]
>> +; path to the cts opengl executable
>> +; can be overwritten by PIGLIT_CTS_GL_BIN environment variable
>> +;bin=/home/knuth/cts/cts/glcts
>> +
>> +; Space-separated list of extra command line arguments for cts. The
>> +; option is not required. The environment variable PIGLIT_CTS_GL_EXTRA_ARGS
>> +; overrides the value set here.
>> +;extra_args=--deqp-visibility hidden
>> +
>>  ; Section for specific oclconform test.  One of these sections is required for
>>  ; each test list in the oclconform section and must be called:
>>  ; oclconform-$testname
>> diff --git a/tests/cts.py b/tests/cts_gl45.py
>> similarity index 56%
>> copy from tests/cts.py
>> copy to tests/cts_gl45.py
>> index 0e64e1b..607b79b 100644
>> --- a/tests/cts.py
>> +++ b/tests/cts_gl45.py
>> @@ -20,25 +20,24 @@
>>  
>>  """Piglit integration for Khronos CTS tests.
>>  
>> -By default this will run GLES2, GLES3, GLES31, and GLESEXT test cases. Those
>> -desiring to run only a subset of them should consider using the -t or -x
>> -options to include or exclude tests.
>> +By default this will run GL30, GL31, GL32, GL33, GL40, GL41, GL42, GL43, GL44
>> +and GL45 test cases. Those desiring to run only a subset of them should consider
>> +using the -t or -x options to include or exclude tests.
>>  
>>  For example:
>> -./piglit run cts -c foo -t ES3- would run only ES3 tests (note the dash to
>> -exclude ES31 tests)
>> +./piglit run cts_gl45 -c foo -t GL30- would run only GL30 tests
>>  
>>  This integration requires some configuration in piglit.conf, or the use of
>>  environment variables.
>>  
>>  In piglit.conf one should set the following:
>> -[cts]:bin -- Path to the glcts binary
>> -[cts]:extra_args -- any extra arguments to be passed to cts (optional)
>> +[cts_gl]:bin -- Path to the glcts binary
>> +[cts_gl]:extra_args -- any extra arguments to be passed to cts (optional)
>
>>  
>>  Alternatively (or in addition, since environment variables have precedence),
>>  one could set:
>> -PIGLIT_CTS_BIN -- environment equivalent of [cts]:bin
>> -PIGLIT_CTS_EXTRA_ARGS -- environment equivalent of [cts]:extra_args
>> +PIGLIT_CTS_GL_BIN -- environment equivalent of [cts_gl]:bin
>> +PIGLIT_CTS_GL_EXTRA_ARGS -- environment equivalent of [cts_gl]:extra_args
>>  
>>  """
>>  
>> @@ -51,9 +50,9 @@ from framework.test import deqp
>>  
>>  __all__ = ['profile']
>>  
>> -_CTS_BIN = deqp.get_option('PIGLIT_CTS_BIN', ('cts', 'bin'))
>> +_CTS_BIN = deqp.get_option('PIGLIT_CTS_GL_BIN', ('cts_gl', 'bin'))
>>  
>> -_EXTRA_ARGS = deqp.get_option('PIGLIT_CTS_EXTRA_ARGS', ('cts', 'extra_args'),
>> +_EXTRA_ARGS = deqp.get_option('PIGLIT_CTS_GL_EXTRA_ARGS', ('cts_gl', 'extra_args'),
>>                                default='').split()
>>  
>>  
>> @@ -65,18 +64,28 @@ class DEQPCTSTest(deqp.DEQPBaseTest):
>>          return super(DEQPCTSTest, self).extra_args + \
>>              [x for x in _EXTRA_ARGS if not x.startswith('--deqp-case')]
>>  
>> -
>>  # Add all of the suites by default, users can use filters to remove them.
>>  profile = deqp.make_profile(  # pylint: disable=invalid-name
>>      itertools.chain(
>>          deqp.iter_deqp_test_cases(
>> -            deqp.gen_caselist_txt(_CTS_BIN, 'ES2-CTS-cases.txt', _EXTRA_ARGS)),
>> +            deqp.gen_caselist_txt(_CTS_BIN, 'GL30-CTS-cases.txt', _EXTRA_ARGS)),
>> +        deqp.iter_deqp_test_cases(
>> +            deqp.gen_caselist_txt(_CTS_BIN, 'GL31-CTS-cases.txt', _EXTRA_ARGS)),
>> +        deqp.iter_deqp_test_cases(
>> +            deqp.gen_caselist_txt(_CTS_BIN, 'GL32-CTS-cases.txt', _EXTRA_ARGS)),
>> +        deqp.iter_deqp_test_cases(
>> +            deqp.gen_caselist_txt(_CTS_BIN, 'GL33-CTS-cases.txt', _EXTRA_ARGS)),
>> +        deqp.iter_deqp_test_cases(
>> +            deqp.gen_caselist_txt(_CTS_BIN, 'GL40-CTS-cases.txt', _EXTRA_ARGS)),
>> +        deqp.iter_deqp_test_cases(
>> +            deqp.gen_caselist_txt(_CTS_BIN, 'GL41-CTS-cases.txt', _EXTRA_ARGS)),
>> +        deqp.iter_deqp_test_cases(
>> +            deqp.gen_caselist_txt(_CTS_BIN, 'GL42-CTS-cases.txt', _EXTRA_ARGS)),
>>          deqp.iter_deqp_test_cases(
>> -            deqp.gen_caselist_txt(_CTS_BIN, 'ES3-CTS-cases.txt', _EXTRA_ARGS)),
>> +            deqp.gen_caselist_txt(_CTS_BIN, 'GL43-CTS-cases.txt', _EXTRA_ARGS)),
>>          deqp.iter_deqp_test_cases(
>> -            deqp.gen_caselist_txt(_CTS_BIN, 'ES31-CTS-cases.txt', _EXTRA_ARGS)),
>> +            deqp.gen_caselist_txt(_CTS_BIN, 'GL44-CTS-cases.txt', _EXTRA_ARGS)),
>>          deqp.iter_deqp_test_cases(
>> -            deqp.gen_caselist_txt(_CTS_BIN, 'ESEXT-CTS-cases.txt',
>> -                                  _EXTRA_ARGS)),
>> +            deqp.gen_caselist_txt(_CTS_BIN, 'GL45-CTS-cases.txt', _EXTRA_ARGS)),
>>      ),
>>      DEQPCTSTest)
>> diff --git a/tests/cts.py b/tests/cts_gles31.py
>> similarity index 97%
>> rename from tests/cts.py
>> rename to tests/cts_gles31.py
>> index 0e64e1b..ad28e58 100644
>> --- a/tests/cts.py
>> +++ b/tests/cts_gles31.py
>> @@ -25,7 +25,7 @@ desiring to run only a subset of them should consider using the -t or -x
>>  options to include or exclude tests.
>>  
>>  For example:
>> -./piglit run cts -c foo -t ES3- would run only ES3 tests (note the dash to
>> +./piglit run cts_gles31 -c foo -t ES3- would run only ES3 tests (note the dash to
>>  exclude ES31 tests)
>>  
>>  This integration requires some configuration in piglit.conf, or the use of
>> -- 
>> 2.5.5
>> 
>> _______________________________________________
>> Piglit mailing list
>> Piglit@lists.freedesktop.org
>> https://lists.freedesktop.org/mailman/listinfo/piglit
>
> Other than the one nit this looks good. Please be sure to let Mark Janes
> (cc'd) or myself know when you push this, as we'll need to update our CI
> since this will change some profile names.

Whoops!  This was pushed on May 23, at which point our CI stopped
running the CTS.  And now we have a handful of regressions to bisect.

> Reviewed-by: Dylan Baker <dylan@pnwbakers.com>
On Thu, 2016-06-02 at 13:24 -0700, Mark Janes wrote:
> > Other than the one nit this looks good. Please be sure to let Mark
> Janes
> > (cc'd) or myself know when you push this, as we'll need to update
> our CI
> > since this will change some profile names.
> 
> Whoops!  This was pushed on May 23, at which point our CI stopped
> running the CTS.  And now we have a handful of regressions to bisect.


Ouch! I totally forgot to ping Mark.


I'm very very sorry for the inconveniences.

	J.A.