[i-g-t] runner: Be patient for processes to die

Submitted by Chris Wilson on Nov. 26, 2018, 9:37 p.m.

Details

Message ID 20181126213744.32050-1-chris@chris-wilson.co.uk
State New
Series "runner: Be patient for processes to die"
Headers show

Commit Message

Chris Wilson Nov. 26, 2018, 9:37 p.m.
Some machines are very slow and some processes hog a lot of resources
and so take much longer than a mere 2s to be terminated. Be patient.

Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=108073
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
---
 runner/executor.c | 44 ++++++++++++++++++++++++++++++--------------
 1 file changed, 30 insertions(+), 14 deletions(-)

Patch hide | download patch | download mbox

diff --git a/runner/executor.c b/runner/executor.c
index 2038c3fda..927696044 100644
--- a/runner/executor.c
+++ b/runner/executor.c
@@ -137,27 +137,36 @@  static char *handle_lockdep(void)
 	return NULL;
 }
 
-static char *handle_taint(void)
+
+static unsigned long tainted(unsigned long *taints)
 {
 	const unsigned long bad_taints =
 		0x20  | /* TAINT_PAGE */
 		0x80  | /* TAINT_DIE */
 		0x200; /* TAINT_OOPS */
-	unsigned long taints = 0;
-	char *reason = NULL;
 	FILE *f;
 
+	*taints = 0;
+
 	f = fopen("/proc/sys/kernel/tainted", "r");
 	if (f) {
-		fscanf(f, "%lu", &taints);
+		fscanf(f, "%lu", taints);
 		fclose(f);
 	}
 
-	if (taints & bad_taints)
-		asprintf(&reason,
-			 "Kernel tainted (%#lx -- %lx)",
-			 taints, taints & bad_taints);
+	return *taints & bad_taints;
+}
+
+static char *handle_taint(void)
+{
+	unsigned long taints, bad_taints;
+	char *reason;
+
+	bad_taints = tainted(&taints);
+	if (!bad_taints)
+		return NULL;
 
+	asprintf(&reason, "Kernel tainted (%#lx -- %lx)", taints, bad_taints);
 	return reason;
 }
 
@@ -437,6 +446,7 @@  static int monitor_output(pid_t child,
 	int wd_extra = 10;
 	int killed = 0; /* 0 if not killed, signal number otherwise */
 	struct timespec time_beg, time_end;
+	unsigned long taints = 0;
 	bool aborting = false;
 
 	igt_gettime(&time_beg);
@@ -493,10 +503,8 @@  static int monitor_output(pid_t child,
 		}
 
 		if (n == 0) {
-			intervals_left--;
-			if (intervals_left) {
+			if (--intervals_left)
 				continue;
-			}
 
 			ping_watchdogs();
 
@@ -504,6 +512,7 @@  static int monitor_output(pid_t child,
 			case 0:
 				if (settings->log_level >= LOG_LEVEL_NORMAL) {
 					printf("Timeout. Killing the current test with SIGTERM.\n");
+					fflush(stdout);
 				}
 
 				killed = SIGTERM;
@@ -514,13 +523,14 @@  static int monitor_output(pid_t child,
 				 * Now continue the loop and let the
 				 * dying child be handled normally.
 				 */
-				timeout = 2; /* Timeout for waiting selected by fair dice roll. */
-				watchdogs_set_timeout(20);
+				timeout = 20;
+				watchdogs_set_timeout(120);
 				intervals_left = timeout_intervals = 1;
 				break;
 			case SIGTERM:
 				if (settings->log_level >= LOG_LEVEL_NORMAL) {
 					printf("Timeout. Killing the current test with SIGKILL.\n");
+					fflush(stdout);
 				}
 
 				killed = SIGKILL;
@@ -530,9 +540,15 @@  static int monitor_output(pid_t child,
 				intervals_left = timeout_intervals = 1;
 				break;
 			case SIGKILL:
+				if (kill(0, child) == 0 && !tainted(&taints)) {
+					intervals_left =  1;
+					break;
+				}
+
 				/* Nothing that can be done, really. Let's tell the caller we want to abort. */
 				if (settings->log_level >= LOG_LEVEL_NORMAL) {
-					fprintf(stderr, "Child refuses to die. Aborting.\n");
+					fprintf(stderr, "Child refuses to die, tainted %x. Aborting.\n",
+						taints);
 				}
 				close_watchdogs(settings);
 				free(outbuf);

Comments

Petri Latvala Nov. 27, 2018, 9:32 a.m.
On Mon, Nov 26, 2018 at 09:37:44PM +0000, Chris Wilson wrote:
> Some machines are very slow and some processes hog a lot of resources
> and so take much longer than a mere 2s to be terminated. Be patient.
> 
> Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=108073
> Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
> ---
>  runner/executor.c | 44 ++++++++++++++++++++++++++++++--------------
>  1 file changed, 30 insertions(+), 14 deletions(-)
> 
> diff --git a/runner/executor.c b/runner/executor.c
> index 2038c3fda..927696044 100644
> --- a/runner/executor.c
> +++ b/runner/executor.c
> @@ -137,27 +137,36 @@ static char *handle_lockdep(void)
>  	return NULL;
>  }
>  
> -static char *handle_taint(void)
> +
> +static unsigned long tainted(unsigned long *taints)


An extra newline slipped in here.


Reviewed-by: Petri Latvala <petri.latvala@intel.com>



>  {
>  	const unsigned long bad_taints =
>  		0x20  | /* TAINT_PAGE */
>  		0x80  | /* TAINT_DIE */
>  		0x200; /* TAINT_OOPS */
> -	unsigned long taints = 0;
> -	char *reason = NULL;
>  	FILE *f;
>  
> +	*taints = 0;
> +
>  	f = fopen("/proc/sys/kernel/tainted", "r");
>  	if (f) {
> -		fscanf(f, "%lu", &taints);
> +		fscanf(f, "%lu", taints);
>  		fclose(f);
>  	}
>  
> -	if (taints & bad_taints)
> -		asprintf(&reason,
> -			 "Kernel tainted (%#lx -- %lx)",
> -			 taints, taints & bad_taints);
> +	return *taints & bad_taints;
> +}
> +
> +static char *handle_taint(void)
> +{
> +	unsigned long taints, bad_taints;
> +	char *reason;
> +
> +	bad_taints = tainted(&taints);
> +	if (!bad_taints)
> +		return NULL;
>  
> +	asprintf(&reason, "Kernel tainted (%#lx -- %lx)", taints, bad_taints);
>  	return reason;
>  }
>  
> @@ -437,6 +446,7 @@ static int monitor_output(pid_t child,
>  	int wd_extra = 10;
>  	int killed = 0; /* 0 if not killed, signal number otherwise */
>  	struct timespec time_beg, time_end;
> +	unsigned long taints = 0;
>  	bool aborting = false;
>  
>  	igt_gettime(&time_beg);
> @@ -493,10 +503,8 @@ static int monitor_output(pid_t child,
>  		}
>  
>  		if (n == 0) {
> -			intervals_left--;
> -			if (intervals_left) {
> +			if (--intervals_left)
>  				continue;
> -			}
>  
>  			ping_watchdogs();
>  
> @@ -504,6 +512,7 @@ static int monitor_output(pid_t child,
>  			case 0:
>  				if (settings->log_level >= LOG_LEVEL_NORMAL) {
>  					printf("Timeout. Killing the current test with SIGTERM.\n");
> +					fflush(stdout);
>  				}
>  
>  				killed = SIGTERM;
> @@ -514,13 +523,14 @@ static int monitor_output(pid_t child,
>  				 * Now continue the loop and let the
>  				 * dying child be handled normally.
>  				 */
> -				timeout = 2; /* Timeout for waiting selected by fair dice roll. */
> -				watchdogs_set_timeout(20);
> +				timeout = 20;
> +				watchdogs_set_timeout(120);
>  				intervals_left = timeout_intervals = 1;
>  				break;
>  			case SIGTERM:
>  				if (settings->log_level >= LOG_LEVEL_NORMAL) {
>  					printf("Timeout. Killing the current test with SIGKILL.\n");
> +					fflush(stdout);
>  				}
>  
>  				killed = SIGKILL;
> @@ -530,9 +540,15 @@ static int monitor_output(pid_t child,
>  				intervals_left = timeout_intervals = 1;
>  				break;
>  			case SIGKILL:
> +				if (kill(0, child) == 0 && !tainted(&taints)) {
> +					intervals_left =  1;
> +					break;
> +				}
> +
>  				/* Nothing that can be done, really. Let's tell the caller we want to abort. */
>  				if (settings->log_level >= LOG_LEVEL_NORMAL) {
> -					fprintf(stderr, "Child refuses to die. Aborting.\n");
> +					fprintf(stderr, "Child refuses to die, tainted %x. Aborting.\n",
> +						taints);
>  				}
>  				close_watchdogs(settings);
>  				free(outbuf);
> -- 
> 2.20.0.rc1
> 
> _______________________________________________
> igt-dev mailing list
> igt-dev@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/igt-dev
Chris Wilson Nov. 27, 2018, 9:41 a.m.
Quoting Petri Latvala (2018-11-27 09:32:32)
> On Mon, Nov 26, 2018 at 09:37:44PM +0000, Chris Wilson wrote:
> > Some machines are very slow and some processes hog a lot of resources
> > and so take much longer than a mere 2s to be terminated. Be patient.
> > 
> > Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=108073
> > Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
> > ---
> >  runner/executor.c | 44 ++++++++++++++++++++++++++++++--------------
> >  1 file changed, 30 insertions(+), 14 deletions(-)
> > 
> > diff --git a/runner/executor.c b/runner/executor.c
> > index 2038c3fda..927696044 100644
> > --- a/runner/executor.c
> > +++ b/runner/executor.c
> > @@ -137,27 +137,36 @@ static char *handle_lockdep(void)
> >       return NULL;
> >  }
> >  
> > -static char *handle_taint(void)
> > +
> > +static unsigned long tainted(unsigned long *taints)
> 
> 
> An extra newline slipped in here.
> 
> 
> Reviewed-by: Petri Latvala <petri.latvala@intel.com>

Fingers crossed since CI skipped skl-shards...
-Chris