From 8dab298b893a9724297f346c49be528cb9822944 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 7 Oct 2018 19:44:48 +0200 Subject: [PATCH] tests: create testcases on module import if environment variable is set If QUBES_TEST_TEMPLATES or QUBES_TEST_LOAD_ALL is set, create testcases on modules import, instead of waiting until `load_tests` is called. The `QUBES_TEST_TEMPLATES` doesn't require `qubes.xml` access, so it should be safe to do regardless of the environment. The `QUBES_TEST_LOAD_ALL` force loading tests (and reading `qubes.xml`) regardless. This is useful for test runners not supporting load_tests protocol. Or with limited support - for example both default `unittest` runner and `nose2` can either use load_tests protocol _or_ select individual tests. Setting any of those variable allow to run a single test with those runners. With this feature used together load_tests protocol, tests could be registered twice. Avoid this by not listing already defined test classes in create_testcases_for_templates (according to load_tests protocol, those should already be registered). --- qubes/tests/__init__.py | 17 +++++++++++++++++ qubes/tests/integ/backup.py | 11 ++++++++--- qubes/tests/integ/basic.py | 17 ++++++++++------- qubes/tests/integ/dispvm.py | 12 +++++++++--- qubes/tests/integ/dom0_update.py | 11 ++++++++--- qubes/tests/integ/mime.py | 12 +++++++++--- qubes/tests/integ/network.py | 25 ++++++++++++++----------- qubes/tests/integ/pvgrub.py | 10 +++++++--- qubes/tests/integ/salt.py | 12 +++++++++--- qubes/tests/integ/vm_qrexec_gui.py | 10 +++++++--- 10 files changed, 98 insertions(+), 39 deletions(-) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index b5637ddd..98922195 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -1193,12 +1193,29 @@ def create_testcases_for_templates(name, *bases, module, **kwds): for template in list_templates(): clsname = name + '_' + template + if hasattr(module, clsname): + continue cls = type(clsname, bases, {'template': template, **kwds}) cls.__module__ = module.__name__ # XXX I wonder what other __dunder__ attrs did I miss setattr(module, clsname, cls) yield '.'.join((module.__name__, clsname)) +def maybe_create_testcases_on_import(create_testcases_gen): + '''If certain conditions are met, call *create_testcases_gen* to create + testcases for templates tests. The purpose is to use it on integration + tests module(s) import, so the test runner could discover tests without + using load tests protocol. + + The conditions - any of: + - QUBES_TEST_TEMPLATES present in the environment (it's possible to + create test cases without opening qubes.xml) + - QUBES_TEST_LOAD_ALL present in the environment + ''' + if 'QUBES_TEST_TEMPLATES' in os.environ or \ + 'QUBES_TEST_LOAD_ALL' in os.environ: + list(create_testcases_gen()) + def extra_info(obj): '''Return short info identifying object. diff --git a/qubes/tests/integ/backup.py b/qubes/tests/integ/backup.py index 450054d3..73ad8db7 100644 --- a/qubes/tests/integ/backup.py +++ b/qubes/tests/integ/backup.py @@ -641,9 +641,14 @@ class TC_10_BackupVMMixin(BackupTestsMixin): del vms +def create_testcases_for_templates(): + return qubes.tests.create_testcases_for_templates('TC_10_BackupVM', + TC_10_BackupVMMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('TC_10_BackupVM', - TC_10_BackupVMMixin, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) + create_testcases_for_templates())) return tests + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) diff --git a/qubes/tests/integ/basic.py b/qubes/tests/integ/basic.py index 9c67fc1d..f7565460 100644 --- a/qubes/tests/integ/basic.py +++ b/qubes/tests/integ/basic.py @@ -681,17 +681,20 @@ class TC_06_AppVMMixin(object): self.loop.run_until_complete(asyncio.sleep(1)) self.assertFalse(self.vm.is_running()) +def create_testcases_for_templates(): + yield from qubes.tests.create_testcases_for_templates('TC_05_StandaloneVM', + TC_05_StandaloneVMMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + yield from qubes.tests.create_testcases_for_templates('TC_06_AppVM', + TC_06_AppVMMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('TC_05_StandaloneVM', - TC_05_StandaloneVMMixin, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) - tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('TC_06_AppVM', - TC_06_AppVMMixin, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) + create_testcases_for_templates())) return tests +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) + # vim: ts=4 sw=4 et diff --git a/qubes/tests/integ/dispvm.py b/qubes/tests/integ/dispvm.py index 1fd34445..af6f4f7b 100644 --- a/qubes/tests/integ/dispvm.py +++ b/qubes/tests/integ/dispvm.py @@ -294,9 +294,15 @@ class TC_20_DispVMMixin(object): test_txt_content = test_txt_content[3:] self.assertEqual(test_txt_content, b"Test test 2\ntest1\n") + +def create_testcases_for_templates(): + return qubes.tests.create_testcases_for_templates('TC_20_DispVM', + TC_20_DispVMMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('TC_20_DispVM', - TC_20_DispVMMixin, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) + create_testcases_for_templates())) return tests + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) diff --git a/qubes/tests/integ/dom0_update.py b/qubes/tests/integ/dom0_update.py index 516264a3..d103418d 100644 --- a/qubes/tests/integ/dom0_update.py +++ b/qubes/tests/integ/dom0_update.py @@ -380,9 +380,14 @@ Test package 'UNSIGNED package {}-1.0 installed'.format(self.pkg_name)) +def create_testcases_for_templates(): + return qubes.tests.create_testcases_for_templates('TC_00_Dom0Upgrade', + TC_00_Dom0UpgradeMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('TC_00_Dom0Upgrade', - TC_00_Dom0UpgradeMixin, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) + create_testcases_for_templates())) return tests + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) diff --git a/qubes/tests/integ/mime.py b/qubes/tests/integ/mime.py index e2a5ea0f..9a2105d7 100644 --- a/qubes/tests/integ/mime.py +++ b/qubes/tests/integ/mime.py @@ -20,6 +20,7 @@ # License along with this library; if not, see . # # +import os from distutils import spawn import re import subprocess @@ -335,9 +336,14 @@ class TC_50_MimeHandlers: ["Firefox", "Iceweasel", "Navigator"], dispvm=True) +def create_testcases_for_templates(): + return qubes.tests.create_testcases_for_templates('TC_50_MimeHandlers', + TC_50_MimeHandlers, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('TC_50_MimeHandlers', - TC_50_MimeHandlers, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) + create_testcases_for_templates())) return tests + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) diff --git a/qubes/tests/integ/network.py b/qubes/tests/integ/network.py index 8ebb07f0..30f4f32e 100644 --- a/qubes/tests/integ/network.py +++ b/qubes/tests/integ/network.py @@ -1319,17 +1319,20 @@ SHA256: self.assertIn(self.loop.run_until_complete(p.wait()), self.exit_code_ok, '{}: {}\n{}'.format(self.update_cmd, stdout, stderr)) +def create_testcases_for_templates(): + yield from qubes.tests.create_testcases_for_templates('VmNetworking', + VmNetworkingMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + yield from qubes.tests.create_testcases_for_templates('VmIPv6Networking', + VmIPv6NetworkingMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + yield from qubes.tests.create_testcases_for_templates('VmUpdates', + VmUpdatesMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('VmNetworking', - VmNetworkingMixin, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) - tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('VmIPv6Networking', - VmIPv6NetworkingMixin, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) - tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('VmUpdates', - VmUpdatesMixin, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) + create_testcases_for_templates())) return tests + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) diff --git a/qubes/tests/integ/pvgrub.py b/qubes/tests/integ/pvgrub.py index 53a699bd..5c1e7cb1 100644 --- a/qubes/tests/integ/pvgrub.py +++ b/qubes/tests/integ/pvgrub.py @@ -137,10 +137,14 @@ class TC_40_PVGrub(object): self.test_template.run_for_stdio('uname -r')) self.assertEquals(actual_kver.strip(), kver) +def create_testcases_for_templates(): + return qubes.tests.create_testcases_for_templates('TC_40_PVGrub', + TC_40_PVGrub, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('TC_40_PVGrub', - TC_40_PVGrub, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) + create_testcases_for_templates())) return tests + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) diff --git a/qubes/tests/integ/salt.py b/qubes/tests/integ/salt.py index f598c7e7..bfbbd2b2 100644 --- a/qubes/tests/integ/salt.py +++ b/qubes/tests/integ/salt.py @@ -391,9 +391,15 @@ class SaltVMTestMixin(SaltTestMixin): self.assertEqual(stderr, b'') +def create_testcases_for_templates(): + return qubes.tests.create_testcases_for_templates('TC_10_VMSalt', + SaltVMTestMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + + def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('TC_10_VMSalt', - SaltVMTestMixin, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) + create_testcases_for_templates())) return tests + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) diff --git a/qubes/tests/integ/vm_qrexec_gui.py b/qubes/tests/integ/vm_qrexec_gui.py index e70545b4..c93dd49a 100644 --- a/qubes/tests/integ/vm_qrexec_gui.py +++ b/qubes/tests/integ/vm_qrexec_gui.py @@ -1194,10 +1194,14 @@ class TC_10_Generic(qubes.tests.SystemTestCase): 'Flag file created (service was run) even though should be denied,' ' qrexec-client-vm output: {} {}'.format(stdout, stderr)) +def create_testcases_for_templates(): + return qubes.tests.create_testcases_for_templates('TC_00_AppVM', + TC_00_AppVMMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('TC_00_AppVM', - TC_00_AppVMMixin, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) + create_testcases_for_templates())) return tests + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)