[Intel-gfx] tests/meta_test: Add a meta test for sanity checks of CI systems

Submitted by Lofstedt, Marta on April 5, 2017, 10:11 a.m.

Details

Message ID 20170405101153.18525-1-marta.lofstedt@intel.com
State New
Headers show
Series "tests/meta_test: Add a meta test for sanity checks of CI systems" ( rev: 1 ) in IGT (deprecated)

Not browsing as part of any series.

Commit Message

Lofstedt, Marta April 5, 2017, 10:11 a.m.
The intention of this test is use it to test that the CI system
that runs IGT is collecting the results correctly.

For: VIZ-10281

Signed-off-by: Marta Lofstedt <marta.lofstedt@intel.com>
---
 tests/Makefile.sources       |   1 +
 tests/intel-ci/meta.testlist |   7 +++
 tests/meta_test.c            | 145 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 153 insertions(+)
 create mode 100644 tests/intel-ci/meta.testlist
 create mode 100644 tests/meta_test.c

Patch hide | download patch | download mbox

diff --git a/tests/Makefile.sources b/tests/Makefile.sources
index 45c21a0..7fa9b8f 100644
--- a/tests/Makefile.sources
+++ b/tests/Makefile.sources
@@ -143,6 +143,7 @@  TESTS_progs_M = \
 	template \
 	vgem_basic \
 	vgem_slow \
+	meta_test \
 	$(NULL)
 
 if HAVE_CHAMELIUM
diff --git a/tests/intel-ci/meta.testlist b/tests/intel-ci/meta.testlist
new file mode 100644
index 0000000..b3e2923
--- /dev/null
+++ b/tests/intel-ci/meta.testlist
@@ -0,0 +1,7 @@ 
+igt@meta_test@pass-result
+igt@meta_test@fail-result
+igt@meta_test@dmesg-pass
+igt@meta_test@dmesg-warn
+igt@meta_test@user-crash
+igt@meta_test@piglit-timeout
+igt@meta_test@generate-panic
diff --git a/tests/meta_test.c b/tests/meta_test.c
new file mode 100644
index 0000000..8a420ba
--- /dev/null
+++ b/tests/meta_test.c
@@ -0,0 +1,145 @@ 
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <stdlib.h>
+#include <signal.h>
+#include "igt.h"
+
+/*
+ * The purpose of this test is to test the CI system that we have
+ * for running the tests. The test should generate all possible
+ * exit states for igt tests.
+ *
+ * Possible exit-states of igt tests:
+ * 1. pass - subtest: pass-result
+ * 2. fail - subtest: fail-result
+ * 3. dmesg warn - subtest: dmesg-fail,
+ *      subtest: dmesg-pass is a sanity check for the dmesg-fail.
+ * 4. crash - subtest: user-crash
+ * 5. piglit timeout - subtest: piglit-timeout
+ * 6. incomplete - subtest: generate-panic
+ *      NOTE: inorder for this to generate the incomplete state
+ *      the kernel must be configured to reboot on panic.
+ *      NOTE: if the tested CI system have features such as
+ *      PSTORE and/or kexec/kdump enabled. This test could be
+ *      used to make sure that the CI system stores the generated
+ *      log/dumps as expected.
+ * 7. incomplete - where user hang is not caught by piglit timeout.
+ *      This would be caught by a user-side softdog daemon,
+ *      such as owatch by ezbench. However, I don't know
+ *      how to trigger this state, so it will not be tested.
+ * 8. incomplete - system requires hard reboot :
+ *      This state could be triggered by calling an evil kernel
+ *      module that was developed hang the system. Such
+ *      a module will not be developed for this purpose,
+ *      so this "exit state" will not be tested.
+ *
+ * TODO: If this test was deployed on a CI system that
+ * was able to pick up testing again after reboot,
+ * such as ezbench, a post-analyze test should be added
+ * that collected and analyzed the result of the tests
+ * run before reboot.
+ */
+
+__attribute__((format(printf, 1, 2)))
+static void kmsg(const char *format, ...)
+#define KERN_EMER"<0>"
+#define KERN_ALERT"<1>"
+#define KERN_CRIT"<2>"
+#define KERN_ERR"<3>"
+#define KERN_WARNING"<4>"
+#define KERN_NOTICE"<5>"
+#define KERN_INFO"<6>"
+#define KERN_DEBUG"<7>"
+{
+	va_list ap;
+	FILE *file;
+
+	file = fopen("/dev/kmsg", "w");
+	if (file == NULL)
+		return;
+
+	va_start(ap, format);
+	vfprintf(file, format, ap);
+	va_end(ap);
+	fclose(file);
+}
+
+static void test_result(bool result)
+{
+	igt_assert_eq(result, true);
+}
+
+static void test_dmesg(bool pass)
+{
+	if (pass)
+		kmsg(KERN_DEBUG "drm: IGT inserted string.");
+	else
+		kmsg(KERN_WARNING "drm: IGT inserted string.");
+}
+
+static void test_user_crash(void)
+{
+	raise(SIGSEGV);
+}
+
+static void test_piglit_timeout(void)
+{
+	sleep(605);
+}
+
+static void test_panic(void)
+{
+	system("echo c > /proc/sysrq-trigger");
+}
+
+igt_main
+{
+
+	igt_fixture {
+		igt_skip_on_f(!getenv("IGT_CI_META_TEST"),
+			      "Only for meta-testing of CI systems");
+	}
+
+	igt_subtest("pass-result")
+		test_result(true);
+
+	igt_subtest("fail-result")
+		test_result(false);
+
+	igt_subtest("dmesg-pass")
+		test_dmesg(true);
+
+	igt_subtest("dmesg-warn")
+		test_dmesg(false);
+
+	igt_subtest("user-crash")
+		test_user_crash();
+
+	igt_subtest("piglit-timeout")
+		test_piglit_timeout();
+
+	igt_subtest("generate-panic")
+		test_panic();
+}

Comments

On Wed, Apr 05, 2017 at 01:11:53PM +0300, Marta Lofstedt wrote:
> The intention of this test is use it to test that the CI system
> that runs IGT is collecting the results correctly.
> 
> For: VIZ-10281
> 
> Signed-off-by: Marta Lofstedt <marta.lofstedt@intel.com>
> ---
>  tests/Makefile.sources       |   1 +
>  tests/intel-ci/meta.testlist |   7 +++
>  tests/meta_test.c            | 145 +++++++++++++++++++++++++++++++++++++++++++
>  3 files changed, 153 insertions(+)
>  create mode 100644 tests/intel-ci/meta.testlist
>  create mode 100644 tests/meta_test.c
> 
> diff --git a/tests/Makefile.sources b/tests/Makefile.sources
> index 45c21a0..7fa9b8f 100644
> --- a/tests/Makefile.sources
> +++ b/tests/Makefile.sources
> @@ -143,6 +143,7 @@ TESTS_progs_M = \
>  	template \
>  	vgem_basic \
>  	vgem_slow \
> +	meta_test \
>  	$(NULL)
>  
>  if HAVE_CHAMELIUM
> diff --git a/tests/intel-ci/meta.testlist b/tests/intel-ci/meta.testlist
> new file mode 100644
> index 0000000..b3e2923
> --- /dev/null
> +++ b/tests/intel-ci/meta.testlist
> @@ -0,0 +1,7 @@
> +igt@meta_test@pass-result
> +igt@meta_test@fail-result
> +igt@meta_test@dmesg-pass
> +igt@meta_test@dmesg-warn
> +igt@meta_test@user-crash
> +igt@meta_test@piglit-timeout
> +igt@meta_test@generate-panic
> diff --git a/tests/meta_test.c b/tests/meta_test.c
> new file mode 100644
> index 0000000..8a420ba
> --- /dev/null
> +++ b/tests/meta_test.c
> @@ -0,0 +1,145 @@
> +/*
> + * Copyright © 2017 Intel Corporation
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the "Software"),
> + * to deal in the Software without restriction, including without limitation
> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice (including the next
> + * paragraph) shall be included in all copies or substantial portions of the
> + * Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
> + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
> + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
> + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
> + * IN THE SOFTWARE.
> + *
> + */
> +
> +#include <stdlib.h>
> +#include <signal.h>
> +#include "igt.h"
> +
> +/*
> + * The purpose of this test is to test the CI system that we have
> + * for running the tests. The test should generate all possible
> + * exit states for igt tests.
> + *
> + * Possible exit-states of igt tests:
> + * 1. pass - subtest: pass-result
> + * 2. fail - subtest: fail-result
> + * 3. dmesg warn - subtest: dmesg-fail,
> + *      subtest: dmesg-pass is a sanity check for the dmesg-fail.


Add some further explanation here about the purpose and why the test
does what it does: The purpose is to check that certain kernel log
activity gets correctly reported in the test result, and that normal
activity doesn't?


> + * 4. crash - subtest: user-crash
> + * 5. piglit timeout - subtest: piglit-timeout
> + * 6. incomplete - subtest: generate-panic
> + *      NOTE: inorder for this to generate the incomplete state
> + *      the kernel must be configured to reboot on panic.
> + *      NOTE: if the tested CI system have features such as
> + *      PSTORE and/or kexec/kdump enabled. This test could be
> + *      used to make sure that the CI system stores the generated
> + *      log/dumps as expected.
> + * 7. incomplete - where user hang is not caught by piglit timeout.
> + *      This would be caught by a user-side softdog daemon,
> + *      such as owatch by ezbench. However, I don't know
> + *      how to trigger this state, so it will not be tested.
> + * 8. incomplete - system requires hard reboot :
> + *      This state could be triggered by calling an evil kernel
> + *      module that was developed hang the system. Such
> + *      a module will not be developed for this purpose,
> + *      so this "exit state" will not be tested.
> + *
> + * TODO: If this test was deployed on a CI system that
> + * was able to pick up testing again after reboot,
> + * such as ezbench, a post-analyze test should be added
> + * that collected and analyzed the result of the tests
> + * run before reboot.
> + */
> +
> +__attribute__((format(printf, 1, 2)))
> +static void kmsg(const char *format, ...)
> +#define KERN_EMER"<0>"
> +#define KERN_ALERT"<1>"
> +#define KERN_CRIT"<2>"
> +#define KERN_ERR"<3>"
> +#define KERN_WARNING"<4>"
> +#define KERN_NOTICE"<5>"
> +#define KERN_INFO"<6>"
> +#define KERN_DEBUG"<7>"
> +{


This is valid C and does what you mean, but can you add some
whitespace between the token and the value anyway? :P







> +	va_list ap;
> +	FILE *file;
> +
> +	file = fopen("/dev/kmsg", "w");
> +	if (file == NULL)
> +		return;
> +
> +	va_start(ap, format);
> +	vfprintf(file, format, ap);
> +	va_end(ap);
> +	fclose(file);
> +}
> +
> +static void test_result(bool result)
> +{
> +	igt_assert_eq(result, true);
> +}
> +
> +static void test_dmesg(bool pass)
> +{
> +	if (pass)
> +		kmsg(KERN_DEBUG "drm: IGT inserted string.");
> +	else
> +		kmsg(KERN_WARNING "drm: IGT inserted string.");
> +}
> +
> +static void test_user_crash(void)
> +{
> +	raise(SIGSEGV);
> +}
> +
> +static void test_piglit_timeout(void)
> +{
> +	sleep(605);
> +}
> +
> +static void test_panic(void)
> +{
> +	system("echo c > /proc/sysrq-trigger");
> +}
> +
> +igt_main
> +{
> +
> +	igt_fixture {
> +		igt_skip_on_f(!getenv("IGT_CI_META_TEST"),
> +			      "Only for meta-testing of CI systems");


Add a newline here.





With those,

Reviewed-by: Petri Latvala <petri.latvala@intel.com>
> -----Original Message-----
> From: Latvala, Petri
> Sent: Wednesday, April 19, 2017 12:08 PM
> To: Lofstedt, Marta <marta.lofstedt@intel.com>
> Cc: intel-gfx@lists.freedesktop.org
> Subject: Re: [PATCH i-g-t] tests/meta_test: Add a meta test for sanity checks
> of CI systems
> 
> On Wed, Apr 05, 2017 at 01:11:53PM +0300, Marta Lofstedt wrote:
> > The intention of this test is use it to test that the CI system that
> > runs IGT is collecting the results correctly.
> >
> > For: VIZ-10281
> >
> > Signed-off-by: Marta Lofstedt <marta.lofstedt@intel.com>
> > ---
> >  tests/Makefile.sources       |   1 +
> >  tests/intel-ci/meta.testlist |   7 +++
> >  tests/meta_test.c            | 145
> +++++++++++++++++++++++++++++++++++++++++++
> >  3 files changed, 153 insertions(+)
> >  create mode 100644 tests/intel-ci/meta.testlist  create mode 100644
> > tests/meta_test.c
> >
> > diff --git a/tests/Makefile.sources b/tests/Makefile.sources index
> > 45c21a0..7fa9b8f 100644
> > --- a/tests/Makefile.sources
> > +++ b/tests/Makefile.sources
> > @@ -143,6 +143,7 @@ TESTS_progs_M = \
> >  	template \
> >  	vgem_basic \
> >  	vgem_slow \
> > +	meta_test \
> >  	$(NULL)
> >
> >  if HAVE_CHAMELIUM
> > diff --git a/tests/intel-ci/meta.testlist
> > b/tests/intel-ci/meta.testlist new file mode 100644 index
> > 0000000..b3e2923
> > --- /dev/null
> > +++ b/tests/intel-ci/meta.testlist
> > @@ -0,0 +1,7 @@
> > +igt@meta_test@pass-result
> > +igt@meta_test@fail-result
> > +igt@meta_test@dmesg-pass
> > +igt@meta_test@dmesg-warn
> > +igt@meta_test@user-crash
> > +igt@meta_test@piglit-timeout
> > +igt@meta_test@generate-panic
> > diff --git a/tests/meta_test.c b/tests/meta_test.c new file mode
> > 100644 index 0000000..8a420ba
> > --- /dev/null
> > +++ b/tests/meta_test.c
> > @@ -0,0 +1,145 @@
> > +/*
> > + * Copyright (c) 2017 Intel Corporation
> > + *
> > + * Permission is hereby granted, free of charge, to any person
> > +obtaining a
> > + * copy of this software and associated documentation files (the
> > +"Software"),
> > + * to deal in the Software without restriction, including without
> > +limitation
> > + * the rights to use, copy, modify, merge, publish, distribute,
> > +sublicense,
> > + * and/or sell copies of the Software, and to permit persons to whom
> > +the
> > + * Software is furnished to do so, subject to the following conditions:
> > + *
> > + * The above copyright notice and this permission notice (including
> > +the next
> > + * paragraph) shall be included in all copies or substantial portions
> > +of the
> > + * Software.
> > + *
> > + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
> KIND,
> > +EXPRESS OR
> > + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> > +MERCHANTABILITY,
> > + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN
> NO EVENT
> > +SHALL
> > + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
> DAMAGES
> > +OR OTHER
> > + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
> OTHERWISE,
> > +ARISING
> > + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
> OR
> > +OTHER DEALINGS
> > + * IN THE SOFTWARE.
> > + *
> > + */
> > +
> > +#include <stdlib.h>
> > +#include <signal.h>
> > +#include "igt.h"
> > +
> > +/*
> > + * The purpose of this test is to test the CI system that we have
> > + * for running the tests. The test should generate all possible
> > + * exit states for igt tests.
> > + *
> > + * Possible exit-states of igt tests:
> > + * 1. pass - subtest: pass-result
> > + * 2. fail - subtest: fail-result
> > + * 3. dmesg warn - subtest: dmesg-fail,
> > + *      subtest: dmesg-pass is a sanity check for the dmesg-fail.
> 
> 
> Add some further explanation here about the purpose and why the test
> does what it does: The purpose is to check that certain kernel log activity gets
> correctly reported in the test result, and that normal activity doesn't?
> 
> 
> > + * 4. crash - subtest: user-crash
> > + * 5. piglit timeout - subtest: piglit-timeout
> > + * 6. incomplete - subtest: generate-panic
> > + *      NOTE: inorder for this to generate the incomplete state
> > + *      the kernel must be configured to reboot on panic.
> > + *      NOTE: if the tested CI system have features such as
> > + *      PSTORE and/or kexec/kdump enabled. This test could be
> > + *      used to make sure that the CI system stores the generated
> > + *      log/dumps as expected.
> > + * 7. incomplete - where user hang is not caught by piglit timeout.
> > + *      This would be caught by a user-side softdog daemon,
> > + *      such as owatch by ezbench. However, I don't know
> > + *      how to trigger this state, so it will not be tested.
> > + * 8. incomplete - system requires hard reboot :
> > + *      This state could be triggered by calling an evil kernel
> > + *      module that was developed hang the system. Such
> > + *      a module will not be developed for this purpose,
> > + *      so this "exit state" will not be tested.
> > + *
> > + * TODO: If this test was deployed on a CI system that
> > + * was able to pick up testing again after reboot,
> > + * such as ezbench, a post-analyze test should be added
> > + * that collected and analyzed the result of the tests
> > + * run before reboot.
> > + */
> > +
> > +__attribute__((format(printf, 1, 2))) static void kmsg(const char
> > +*format, ...) #define KERN_EMER"<0>"
> > +#define KERN_ALERT"<1>"
> > +#define KERN_CRIT"<2>"
> > +#define KERN_ERR"<3>"
> > +#define KERN_WARNING"<4>"
> > +#define KERN_NOTICE"<5>"
> > +#define KERN_INFO"<6>"
> > +#define KERN_DEBUG"<7>"
> > +{
> 
> 
> This is valid C and does what you mean, but can you add some whitespace
> between the token and the value anyway? :P
> 
> 
> 
> 
> 
> 
> 
> > +	va_list ap;
> > +	FILE *file;
> > +
> > +	file = fopen("/dev/kmsg", "w");
> > +	if (file == NULL)
> > +		return;
> > +
> > +	va_start(ap, format);
> > +	vfprintf(file, format, ap);
> > +	va_end(ap);
> > +	fclose(file);
> > +}
> > +
> > +static void test_result(bool result)
> > +{
> > +	igt_assert_eq(result, true);
> > +}
> > +
> > +static void test_dmesg(bool pass)
> > +{
> > +	if (pass)
> > +		kmsg(KERN_DEBUG "drm: IGT inserted string.");
> > +	else
> > +		kmsg(KERN_WARNING "drm: IGT inserted
> string."); }
> > +
> > +static void test_user_crash(void)
> > +{
> > +	raise(SIGSEGV);
> > +}
> > +
> > +static void test_piglit_timeout(void) {
> > +	sleep(605);
> > +}
> > +
> > +static void test_panic(void)
> > +{
> > +	system("echo c > /proc/sysrq-trigger"); }
> > +
> > +igt_main
> > +{
> > +
> > +	igt_fixture {
> > +		igt_skip_on_f(!getenv("IGT_CI_META_TEST"),
> > +			      "Only for meta-testing of CI
> systems");
> 
> 
> Add a newline here.
> 
> 
> 
> 
> 
> With those,
> 
> Reviewed-by: Petri Latvala <petri.latvala@intel.com>

Thanks Petri, V2 is up.