[i-g-t,1/2] lib: Defer application of igt_fixture results

Submitted by Chris Wilson on Nov. 28, 2018, 10:52 p.m.

Details

Message ID 20181128225231.29228-1-chris@chris-wilson.co.uk
State New
Series "Series without cover letter"
Headers show

Commit Message

Chris Wilson Nov. 28, 2018, 10:52 p.m.
If an igt_fixture causes an independent subgroup to fail/skip and that
subgroup is not part of the execution set (--run-subtest) as no subtests
themselves failed or skipped, it should not count towards the overall
test exitcode.

Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=108891
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
---
 lib/igt_core.c | 57 ++++++++++++++++++++++++++++++--------------------
 1 file changed, 34 insertions(+), 23 deletions(-)

Patch hide | download patch | download mbox

diff --git a/lib/igt_core.c b/lib/igt_core.c
index 64883d640..1d47e171f 100644
--- a/lib/igt_core.c
+++ b/lib/igt_core.c
@@ -291,6 +291,10 @@  enum {
 static int igt_exitcode = IGT_EXIT_SUCCESS;
 static const char *command_str;
 
+static bool skipped_one = false;
+static bool succeeded_one = false;
+static bool failed_one = false;
+
 static char* igt_log_domain_filter;
 static struct {
 	char *entries[256];
@@ -919,15 +923,28 @@  bool __igt_run_subtest(const char *subtest_name)
 	}
 
 	if (skip_subtests_henceforth) {
+		const char *result;
+
+		if (skip_subtests_henceforth == FAIL) {
+			if (!failed_one) {
+				igt_exitcode = FAIL;
+				failed_one = true;
+			}
+			result = "FAIL";
+		} else {
+			skipped_one = true;
+			result = "SKIP";
+		}
+
 		printf("%sSubtest %s: %s%s\n",
-		       (!__igt_plain_output) ? "\x1b[1m" : "", subtest_name,
-		       skip_subtests_henceforth == SKIP ?
-		       "SKIP" : "FAIL", (!__igt_plain_output) ? "\x1b[0m" : "");
+		       !__igt_plain_output ? "\x1b[1m" : "",
+		       subtest_name, result,
+		       !__igt_plain_output ? "\x1b[0m" : "");
 		fflush(stdout);
 		if (stderr_needs_sentinel)
-			fprintf(stderr, "Subtest %s: %s\n", subtest_name,
-				skip_subtests_henceforth == SKIP ?
-				"SKIP" : "FAIL");
+			fprintf(stderr, "Subtest %s: %s\n",
+				subtest_name, result);
+
 		return false;
 	}
 
@@ -976,10 +993,6 @@  void __igt_subtest_group_restore(int save)
 	skip_subtests_henceforth = save;
 }
 
-static bool skipped_one = false;
-static bool succeeded_one = false;
-static bool failed_one = false;
-
 static void exit_subtest(const char *) __attribute__((noreturn));
 static void exit_subtest(const char *result)
 {
@@ -1018,7 +1031,6 @@  static void exit_subtest(const char *result)
 void igt_skip(const char *f, ...)
 {
 	va_list args;
-	skipped_one = true;
 
 	assert(!test_child);
 
@@ -1029,6 +1041,7 @@  void igt_skip(const char *f, ...)
 	}
 
 	if (in_subtest) {
+		skipped_one = true;
 		exit_subtest("SKIP");
 	} else if (test_with_subtests) {
 		skip_subtests_henceforth = SKIP;
@@ -1116,11 +1129,6 @@  void igt_fail(int exitcode)
 	if (in_atexit_handler)
 		_exit(IGT_EXIT_FAILURE);
 
-	if (!failed_one)
-		igt_exitcode = exitcode;
-
-	failed_one = true;
-
 	/* Silent exit, parent will do the yelling. */
 	if (test_child)
 		exit(exitcode);
@@ -1128,18 +1136,21 @@  void igt_fail(int exitcode)
 	_igt_log_buffer_dump();
 
 	if (in_subtest) {
+		if (!failed_one) {
+			igt_exitcode = exitcode;
+			failed_one = true;
+		}
+
 		if (exitcode == IGT_EXIT_TIMEOUT)
 			exit_subtest("TIMEOUT");
 		else
 			exit_subtest("FAIL");
+	} else if (test_with_subtests) {
+		skip_subtests_henceforth = FAIL;
+		assert(in_fixture);
+		__igt_fixture_end();
 	} else {
-		assert(igt_can_fail());
-
-		if (in_fixture) {
-			skip_subtests_henceforth = FAIL;
-			__igt_fixture_end();
-		}
-
+		igt_exitcode = exitcode;
 		igt_exit();
 	}
 }

Comments

Daniel Vetter Nov. 29, 2018, 5:16 p.m.
On Wed, Nov 28, 2018 at 10:52:30PM +0000, Chris Wilson wrote:
> If an igt_fixture causes an independent subgroup to fail/skip and that
> subgroup is not part of the execution set (--run-subtest) as no subtests
> themselves failed or skipped, it should not count towards the overall
> test exitcode.
> 
> Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=108891
> Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>

I'm honestly a bit lost on what's going on here. Could you please write a
new library testcase for this one here, as demonstrator? For stuff like
this I really want to lock behaviour down with compile-time testcases.
-Daniel

> ---
>  lib/igt_core.c | 57 ++++++++++++++++++++++++++++++--------------------
>  1 file changed, 34 insertions(+), 23 deletions(-)
> 
> diff --git a/lib/igt_core.c b/lib/igt_core.c
> index 64883d640..1d47e171f 100644
> --- a/lib/igt_core.c
> +++ b/lib/igt_core.c
> @@ -291,6 +291,10 @@ enum {
>  static int igt_exitcode = IGT_EXIT_SUCCESS;
>  static const char *command_str;
>  
> +static bool skipped_one = false;
> +static bool succeeded_one = false;
> +static bool failed_one = false;
> +
>  static char* igt_log_domain_filter;
>  static struct {
>  	char *entries[256];
> @@ -919,15 +923,28 @@ bool __igt_run_subtest(const char *subtest_name)
>  	}
>  
>  	if (skip_subtests_henceforth) {
> +		const char *result;
> +
> +		if (skip_subtests_henceforth == FAIL) {
> +			if (!failed_one) {
> +				igt_exitcode = FAIL;
> +				failed_one = true;
> +			}
> +			result = "FAIL";
> +		} else {
> +			skipped_one = true;
> +			result = "SKIP";
> +		}
> +
>  		printf("%sSubtest %s: %s%s\n",
> -		       (!__igt_plain_output) ? "\x1b[1m" : "", subtest_name,
> -		       skip_subtests_henceforth == SKIP ?
> -		       "SKIP" : "FAIL", (!__igt_plain_output) ? "\x1b[0m" : "");
> +		       !__igt_plain_output ? "\x1b[1m" : "",
> +		       subtest_name, result,
> +		       !__igt_plain_output ? "\x1b[0m" : "");
>  		fflush(stdout);
>  		if (stderr_needs_sentinel)
> -			fprintf(stderr, "Subtest %s: %s\n", subtest_name,
> -				skip_subtests_henceforth == SKIP ?
> -				"SKIP" : "FAIL");
> +			fprintf(stderr, "Subtest %s: %s\n",
> +				subtest_name, result);
> +
>  		return false;
>  	}
>  
> @@ -976,10 +993,6 @@ void __igt_subtest_group_restore(int save)
>  	skip_subtests_henceforth = save;
>  }
>  
> -static bool skipped_one = false;
> -static bool succeeded_one = false;
> -static bool failed_one = false;
> -
>  static void exit_subtest(const char *) __attribute__((noreturn));
>  static void exit_subtest(const char *result)
>  {
> @@ -1018,7 +1031,6 @@ static void exit_subtest(const char *result)
>  void igt_skip(const char *f, ...)
>  {
>  	va_list args;
> -	skipped_one = true;
>  
>  	assert(!test_child);
>  
> @@ -1029,6 +1041,7 @@ void igt_skip(const char *f, ...)
>  	}
>  
>  	if (in_subtest) {
> +		skipped_one = true;
>  		exit_subtest("SKIP");
>  	} else if (test_with_subtests) {
>  		skip_subtests_henceforth = SKIP;
> @@ -1116,11 +1129,6 @@ void igt_fail(int exitcode)
>  	if (in_atexit_handler)
>  		_exit(IGT_EXIT_FAILURE);
>  
> -	if (!failed_one)
> -		igt_exitcode = exitcode;
> -
> -	failed_one = true;
> -
>  	/* Silent exit, parent will do the yelling. */
>  	if (test_child)
>  		exit(exitcode);
> @@ -1128,18 +1136,21 @@ void igt_fail(int exitcode)
>  	_igt_log_buffer_dump();
>  
>  	if (in_subtest) {
> +		if (!failed_one) {
> +			igt_exitcode = exitcode;
> +			failed_one = true;
> +		}
> +
>  		if (exitcode == IGT_EXIT_TIMEOUT)
>  			exit_subtest("TIMEOUT");
>  		else
>  			exit_subtest("FAIL");
> +	} else if (test_with_subtests) {
> +		skip_subtests_henceforth = FAIL;
> +		assert(in_fixture);
> +		__igt_fixture_end();
>  	} else {
> -		assert(igt_can_fail());
> -
> -		if (in_fixture) {
> -			skip_subtests_henceforth = FAIL;
> -			__igt_fixture_end();
> -		}
> -
> +		igt_exitcode = exitcode;
>  		igt_exit();
>  	}
>  }
> -- 
> 2.20.0.rc1
> 
> _______________________________________________
> igt-dev mailing list
> igt-dev@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/igt-dev
Daniel Vetter Nov. 29, 2018, 5:17 p.m.
On Thu, Nov 29, 2018 at 06:16:48PM +0100, Daniel Vetter wrote:
> On Wed, Nov 28, 2018 at 10:52:30PM +0000, Chris Wilson wrote:
> > If an igt_fixture causes an independent subgroup to fail/skip and that
> > subgroup is not part of the execution set (--run-subtest) as no subtests
> > themselves failed or skipped, it should not count towards the overall
> > test exitcode.
> > 
> > Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=108891
> > Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
> 
> I'm honestly a bit lost on what's going on here. Could you please write a
> new library testcase for this one here, as demonstrator? For stuff like
> this I really want to lock behaviour down with compile-time testcases.

Forgot to add: They're in lib/tests/
-Daniel