[v1,weston,09/11] tests: Add support for comparing output against reference images

Submitted by Bryce Harrington on Nov. 19, 2014, 11:06 p.m.

Details

Message ID 1416438386-4783-10-git-send-email-bryce@osg.samsung.com
State Superseded
Headers show

Not browsing as part of any series.

Commit Message

Bryce Harrington Nov. 19, 2014, 11:06 p.m.
Steal Cairo's files_equal() routine to do byte comparison of the rendered
files.  Note that since the clock time will change run to run we can
only compare against the first frame (which will be black).

Signed-off-by: Bryce Harrington <bryce@osg.samsung.com>
---
 tests/fadein-test.c               | 31 +++++++++++++++++++++++++++----
 tests/weston-test-client-helper.c | 33 +++++++++++++++++++++++++++++++++
 tests/weston-test-client-helper.h |  5 +++++
 3 files changed, 65 insertions(+), 4 deletions(-)

Patch hide | download patch | download mbox

diff --git a/tests/fadein-test.c b/tests/fadein-test.c
index a6c284f..965bb0a 100644
--- a/tests/fadein-test.c
+++ b/tests/fadein-test.c
@@ -24,27 +24,40 @@ 
 
 #include <unistd.h>
 #include <stdio.h>
+#include <stdbool.h>
 
 #include "weston-test-client-helper.h"
 
 char *server_parameters="--use-pixman --width=320 --height=240";
 
 static char*
-output_filename(const char* basename) {
+output_filename(const char* basename, int head) {
 	static const char *path = "./";
 	char *filename;
 
-        if (asprintf(&filename, "%s%s", path, basename) < 0)
+        if (asprintf(&filename, "%s%s-%d.png", path, basename, head) < 0)
 		filename = NULL;
 
 	return filename;
 }
 
+static char*
+reference_filename(const char* basename, int head) {
+        static const char *path = "./tests/reference/";
+        char *filename;
+
+        if (asprintf(&filename, "%s%s-%d.png", path, basename, head) < 0)
+                filename = NULL;
+
+        return filename;
+}
+
 TEST(fadein)
 {
 	struct client *client;
 	char basename[32];
 	char *out_path;
+	char *ref_path;
 	int i;
 
 	client = client_create(100, 100, 100, 100);
@@ -52,11 +65,21 @@  TEST(fadein)
 
 	for (i = 0; i < 6; i++) {
 		snprintf(basename, sizeof basename, "fadein-%02d", i);
-		out_path = output_filename(basename);
+		// FIXME: Iterate over all heads
+		out_path = output_filename(basename, 0);
+		ref_path = reference_filename(basename, 0);
 
-		wl_test_record_screenshot(client->test->wl_test, out_path);
+		// FIXME: Would be preferred to pass in out_path rather than basename here...
+		wl_test_record_screenshot(client->test->wl_test, basename);
 		client_roundtrip(client);
+		if (i == 0) {
+			if (files_equal(out_path, ref_path))
+				printf("%s is correct\n", out_path);
+			else
+				printf("%s doesn't match reference %s\n", out_path, ref_path);
+		}
 		free (out_path);
+		free (ref_path);
 
 		usleep(250000);
 	}
diff --git a/tests/weston-test-client-helper.c b/tests/weston-test-client-helper.c
index 79097fa..72834e4 100644
--- a/tests/weston-test-client-helper.c
+++ b/tests/weston-test-client-helper.c
@@ -22,6 +22,7 @@ 
 
 #include <config.h>
 
+#include <stdbool.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
@@ -624,3 +625,35 @@  client_create(int x, int y, int width, int height)
 
 	return client;
 }
+
+bool
+files_equal(const char *test_filename, const char* ref_filename)
+{
+        FILE *test, *ref;
+        int t, p;
+
+        if (test_filename == NULL || ref_filename == NULL)
+                return false;
+
+        test = fopen (test_filename, "rb");
+        if (test == NULL)
+                return false;
+
+        ref = fopen (ref_filename, "rb");
+        if (ref == NULL) {
+                fclose (test);
+                return false;
+        }
+
+        do {
+                t = getc (test);
+                p = getc (ref);
+                if (t != p)
+                        break;
+        } while (t != EOF && p != EOF);
+
+        fclose (test);
+        fclose (ref);
+
+        return t == p;  /* both EOF */
+}
diff --git a/tests/weston-test-client-helper.h b/tests/weston-test-client-helper.h
index 684afc6..20e08b1 100644
--- a/tests/weston-test-client-helper.h
+++ b/tests/weston-test-client-helper.h
@@ -26,6 +26,8 @@ 
 #include "config.h"
 
 #include <assert.h>
+#include <stdbool.h>
+
 #include "weston-test-runner.h"
 #include "wayland-test-client-protocol.h"
 
@@ -135,4 +137,7 @@  void
 expect_protocol_error(struct client *client,
 		      const struct wl_interface *intf, uint32_t code);
 
+bool
+files_equal(const char *file_1, const char *file_2);
+
 #endif

Comments

On Wed, 19 Nov 2014 15:06:24 -0800
Bryce Harrington <bryce@osg.samsung.com> wrote:

> Steal Cairo's files_equal() routine to do byte comparison of the rendered
> files.  Note that since the clock time will change run to run we can
> only compare against the first frame (which will be black).

Not even the first frame is actually guaranteed to be black, I think.
It's just luck at this point.

> Signed-off-by: Bryce Harrington <bryce@osg.samsung.com>
> ---
>  tests/fadein-test.c               | 31 +++++++++++++++++++++++++++----
>  tests/weston-test-client-helper.c | 33 +++++++++++++++++++++++++++++++++
>  tests/weston-test-client-helper.h |  5 +++++
>  3 files changed, 65 insertions(+), 4 deletions(-)
> 
> diff --git a/tests/fadein-test.c b/tests/fadein-test.c
> index a6c284f..965bb0a 100644
> --- a/tests/fadein-test.c
> +++ b/tests/fadein-test.c
> @@ -24,27 +24,40 @@
>  
>  #include <unistd.h>
>  #include <stdio.h>
> +#include <stdbool.h>
>  
>  #include "weston-test-client-helper.h"
>  
>  char *server_parameters="--use-pixman --width=320 --height=240";
>  
>  static char*
> -output_filename(const char* basename) {
> +output_filename(const char* basename, int head) {
>  	static const char *path = "./";
>  	char *filename;
>  
> -        if (asprintf(&filename, "%s%s", path, basename) < 0)
> +        if (asprintf(&filename, "%s%s-%d.png", path, basename, head) < 0)
>  		filename = NULL;
>  
>  	return filename;
>  }
>  
> +static char*
> +reference_filename(const char* basename, int head) {
> +        static const char *path = "./tests/reference/";
> +        char *filename;
> +
> +        if (asprintf(&filename, "%s%s-%d.png", path, basename, head) < 0)
> +                filename = NULL;
> +
> +        return filename;
> +}
> +
>  TEST(fadein)
>  {
>  	struct client *client;
>  	char basename[32];
>  	char *out_path;
> +	char *ref_path;
>  	int i;
>  
>  	client = client_create(100, 100, 100, 100);
> @@ -52,11 +65,21 @@ TEST(fadein)
>  
>  	for (i = 0; i < 6; i++) {
>  		snprintf(basename, sizeof basename, "fadein-%02d", i);
> -		out_path = output_filename(basename);
> +		// FIXME: Iterate over all heads
> +		out_path = output_filename(basename, 0);
> +		ref_path = reference_filename(basename, 0);
>  
> -		wl_test_record_screenshot(client->test->wl_test, out_path);
> +		// FIXME: Would be preferred to pass in out_path rather than basename here...
> +		wl_test_record_screenshot(client->test->wl_test, basename);
>  		client_roundtrip(client);
> +		if (i == 0) {
> +			if (files_equal(out_path, ref_path))
> +				printf("%s is correct\n", out_path);
> +			else
> +				printf("%s doesn't match reference %s\n", out_path, ref_path);
> +		}
>  		free (out_path);
> +		free (ref_path);
>  
>  		usleep(250000);
>  	}
> diff --git a/tests/weston-test-client-helper.c b/tests/weston-test-client-helper.c
> index 79097fa..72834e4 100644
> --- a/tests/weston-test-client-helper.c
> +++ b/tests/weston-test-client-helper.c
> @@ -22,6 +22,7 @@
>  
>  #include <config.h>
>  
> +#include <stdbool.h>
>  #include <stdlib.h>
>  #include <stdio.h>
>  #include <string.h>
> @@ -624,3 +625,35 @@ client_create(int x, int y, int width, int height)
>  
>  	return client;
>  }
> +
> +bool
> +files_equal(const char *test_filename, const char* ref_filename)
> +{
> +        FILE *test, *ref;
> +        int t, p;
> +
> +        if (test_filename == NULL || ref_filename == NULL)
> +                return false;
> +
> +        test = fopen (test_filename, "rb");
> +        if (test == NULL)
> +                return false;
> +
> +        ref = fopen (ref_filename, "rb");
> +        if (ref == NULL) {
> +                fclose (test);
> +                return false;
> +        }
> +
> +        do {
> +                t = getc (test);
> +                p = getc (ref);
> +                if (t != p)
> +                        break;
> +        } while (t != EOF && p != EOF);
> +
> +        fclose (test);
> +        fclose (ref);
> +
> +        return t == p;  /* both EOF */
> +}

Byte by byte comparison on the raw PNG data, I didn't expect that. :-)

Well, this needs a rewrite with wl_buffer screenshots, and there we
naturally get a chance to apply some fuzz for matching as needed.


Thanks,
pq

> diff --git a/tests/weston-test-client-helper.h b/tests/weston-test-client-helper.h
> index 684afc6..20e08b1 100644
> --- a/tests/weston-test-client-helper.h
> +++ b/tests/weston-test-client-helper.h
> @@ -26,6 +26,8 @@
>  #include "config.h"
>  
>  #include <assert.h>
> +#include <stdbool.h>
> +
>  #include "weston-test-runner.h"
>  #include "wayland-test-client-protocol.h"
>  
> @@ -135,4 +137,7 @@ void
>  expect_protocol_error(struct client *client,
>  		      const struct wl_interface *intf, uint32_t code);
>  
> +bool
> +files_equal(const char *file_1, const char *file_2);
> +
>  #endif