framework: Read test files as utf-8

Submitted by Jussi Kukkonen on July 6, 2016, 10:25 a.m.

Details

Message ID 1467800728-14152-1-git-send-email-jussi.kukkonen@intel.com
State New
Headers show
Series "framework: Read test files as utf-8" ( rev: 1 ) in Piglit

Not browsing as part of any series.

Commit Message

Jussi Kukkonen July 6, 2016, 10:25 a.m.
Make sure test files are always considered utf-8 to avoid a
UnicodeDecodeError when e.g. locale is unset.

This requires using io.open() as python2 open() does not support
"encoding". The upside of this is that same code paths should now work
for both python2 and python3.

Signed-off-by: Jussi Kukkonen <jussi.kukkonen@intel.com>
---
 framework/test/glsl_parser_test.py | 11 +++--------
 framework/test/shader_test.py      | 11 +++--------
 2 files changed, 6 insertions(+), 16 deletions(-)

Patch hide | download patch | download mbox

diff --git a/framework/test/glsl_parser_test.py b/framework/test/glsl_parser_test.py
index cd1693c..df08672 100644
--- a/framework/test/glsl_parser_test.py
+++ b/framework/test/glsl_parser_test.py
@@ -26,7 +26,7 @@  from __future__ import (
 )
 import os
 import re
-
+import io
 import six
 
 from framework import exceptions
@@ -98,13 +98,8 @@  class GLSLParserTest(FastSkipMixin, PiglitBaseTest):
         # Parse the config file and get the config section, then write this
         # section to a StringIO and pass that to ConfigParser
         try:
-            with open(filepath, 'r') as testfile:
-                # Python 2 returns a bytes instance, but python 3 returns str
-                # (unicode) instance.
-                if six.PY2:
-                    testfile = testfile.read().decode('utf-8')
-                elif six.PY3:
-                    testfile = testfile.read()
+            with io.open(filepath, mode='r', encoding='utf-8') as testfile:
+                testfile = testfile.read()
                 config = self.__parser(testfile, filepath)
             command = self.__get_command(config, filepath)
         except GLSLParserInternalError as e:
diff --git a/framework/test/shader_test.py b/framework/test/shader_test.py
index 4fb33a0..f3c945c 100644
--- a/framework/test/shader_test.py
+++ b/framework/test/shader_test.py
@@ -27,8 +27,7 @@  from __future__ import (
     absolute_import, division, print_function, unicode_literals
 )
 import re
-
-import six
+import io
 
 from framework import exceptions
 from .opengl import FastSkipMixin
@@ -60,14 +59,10 @@  class ShaderTest(FastSkipMixin, PiglitBaseTest):
         # cost. The first one looks for the start of the config block or raises
         # an exception. The second looks for the GL version or raises an
         # exception
-        with open(filename, 'r') as shader_file:
+        with io.open(filename, mode='r', encoding='utf-8') as shader_file:
             # The mock in python 3.3 doesn't support readlines(), so use
             # read().split() as a workaround
-            if six.PY3:
-                lines = (l for l in shader_file.read().split('\n'))
-            elif six.PY2:
-                lines = (l.decode('utf-8') for l in
-                         shader_file.read().split(b'\n'))
+            lines = (l for l in shader_file.read().split('\n'))
 
             # Find the config section
             for line in lines:

Comments

Quoting Jussi Kukkonen (2016-07-06 03:25:28)
> Make sure test files are always considered utf-8 to avoid a
> UnicodeDecodeError when e.g. locale is unset.
> 
> This requires using io.open() as python2 open() does not support
> "encoding". The upside of this is that same code paths should now work
> for both python2 and python3.
> 
> Signed-off-by: Jussi Kukkonen <jussi.kukkonen@intel.com>
> ---
>  framework/test/glsl_parser_test.py | 11 +++--------
>  framework/test/shader_test.py      | 11 +++--------
>  2 files changed, 6 insertions(+), 16 deletions(-)
> 
> diff --git a/framework/test/glsl_parser_test.py b/framework/test/glsl_parser_test.py
> index cd1693c..df08672 100644
> --- a/framework/test/glsl_parser_test.py
> +++ b/framework/test/glsl_parser_test.py
> @@ -26,7 +26,7 @@ from __future__ import (
>  )
>  import os
>  import re
> -
> +import io
>  import six
>  
>  from framework import exceptions
> @@ -98,13 +98,8 @@ class GLSLParserTest(FastSkipMixin, PiglitBaseTest):
>          # Parse the config file and get the config section, then write this
>          # section to a StringIO and pass that to ConfigParser
>          try:
> -            with open(filepath, 'r') as testfile:
> -                # Python 2 returns a bytes instance, but python 3 returns str
> -                # (unicode) instance.
> -                if six.PY2:
> -                    testfile = testfile.read().decode('utf-8')
> -                elif six.PY3:
> -                    testfile = testfile.read()
> +            with io.open(filepath, mode='r', encoding='utf-8') as testfile:
> +                testfile = testfile.read()
>                  config = self.__parser(testfile, filepath)
>              command = self.__get_command(config, filepath)
>          except GLSLParserInternalError as e:
> diff --git a/framework/test/shader_test.py b/framework/test/shader_test.py
> index 4fb33a0..f3c945c 100644
> --- a/framework/test/shader_test.py
> +++ b/framework/test/shader_test.py
> @@ -27,8 +27,7 @@ from __future__ import (
>      absolute_import, division, print_function, unicode_literals
>  )
>  import re
> -
> -import six
> +import io
>  
>  from framework import exceptions
>  from .opengl import FastSkipMixin
> @@ -60,14 +59,10 @@ class ShaderTest(FastSkipMixin, PiglitBaseTest):
>          # cost. The first one looks for the start of the config block or raises
>          # an exception. The second looks for the GL version or raises an
>          # exception
> -        with open(filename, 'r') as shader_file:
> +        with io.open(filename, mode='r', encoding='utf-8') as shader_file:
>              # The mock in python 3.3 doesn't support readlines(), so use
>              # read().split() as a workaround
> -            if six.PY3:
> -                lines = (l for l in shader_file.read().split('\n'))
> -            elif six.PY2:
> -                lines = (l.decode('utf-8') for l in
> -                         shader_file.read().split(b'\n'))
> +            lines = (l for l in shader_file.read().split('\n'))
>  
>              # Find the config section
>              for line in lines:
> -- 
> 2.8.1
> 
> _______________________________________________
> Piglit mailing list
> Piglit@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/piglit

Also, there are some unittests that were broken by your patch, I've gone
ahead and fixed them and squashed them into your changes and pushed
this.

Thanks for doing this.

Reviewed-by: Dylan Baker <dylan@pnwbakers.com>