tests: create testcases on module import if environment variable is set

If QUBES_TEST_TEMPLATES or QUBES_TEST_LOAD_ALL is set, create testcases
on modules import, instead of waiting until `load_tests` is called.
The `QUBES_TEST_TEMPLATES` doesn't require `qubes.xml` access, so it
should be safe to do regardless of the environment. The
`QUBES_TEST_LOAD_ALL` force loading tests (and reading `qubes.xml`)
regardless.

This is useful for test runners not supporting load_tests protocol. Or
with limited support - for example both default `unittest` runner and
`nose2` can either use load_tests protocol _or_ select individual tests.
Setting any of those variable allow to run a single test with those
runners.

With this feature used together load_tests protocol, tests could be
registered twice. Avoid this by not listing already defined test classes
in create_testcases_for_templates (according to load_tests protocol,
those should already be registered).

qubes/tests/__init__.py

@@ -1193,12 +1193,29 @@ def create_testcases_for_templates(name, *bases, module, **kwds):
 
     for template in list_templates():
         clsname = name + '_' + template
+        if hasattr(module, clsname):
+            continue
         cls = type(clsname, bases, {'template': template, **kwds})
         cls.__module__ = module.__name__
         # XXX I wonder what other __dunder__ attrs did I miss
         setattr(module, clsname, cls)
         yield '.'.join((module.__name__, clsname))
 
+def maybe_create_testcases_on_import(create_testcases_gen):
+    '''If certain conditions are met, call *create_testcases_gen* to create
+    testcases for templates tests. The purpose is to use it on integration
+    tests module(s) import, so the test runner could discover tests without
+    using load tests protocol.
+
+    The conditions - any of:
+     - QUBES_TEST_TEMPLATES present in the environment (it's possible to
+     create test cases without opening qubes.xml)
+     - QUBES_TEST_LOAD_ALL present in the environment
+    '''
+    if 'QUBES_TEST_TEMPLATES' in os.environ or \
+            'QUBES_TEST_LOAD_ALL' in os.environ:
+        list(create_testcases_gen())
+
 def extra_info(obj):
     '''Return short info identifying object.
 

qubes/tests/integ/backup.py

@@ -641,9 +641,14 @@ class TC_10_BackupVMMixin(BackupTestsMixin):
             del vms
 
 
+def create_testcases_for_templates():
+    return qubes.tests.create_testcases_for_templates('TC_10_BackupVM',
+        TC_10_BackupVMMixin, qubes.tests.SystemTestCase,
+        module=sys.modules[__name__])
+
 def load_tests(loader, tests, pattern):
     tests.addTests(loader.loadTestsFromNames(
-        qubes.tests.create_testcases_for_templates('TC_10_BackupVM',
-            TC_10_BackupVMMixin, qubes.tests.SystemTestCase,
-            module=sys.modules[__name__])))
+        create_testcases_for_templates()))
     return tests
+
+qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)

qubes/tests/integ/basic.py

@@ -681,17 +681,20 @@ class TC_06_AppVMMixin(object):
             self.loop.run_until_complete(asyncio.sleep(1))
         self.assertFalse(self.vm.is_running())
 
+def create_testcases_for_templates():
+    yield from qubes.tests.create_testcases_for_templates('TC_05_StandaloneVM',
+            TC_05_StandaloneVMMixin, qubes.tests.SystemTestCase,
+            module=sys.modules[__name__])
+    yield from qubes.tests.create_testcases_for_templates('TC_06_AppVM',
+            TC_06_AppVMMixin, qubes.tests.SystemTestCase,
+            module=sys.modules[__name__])
 
 def load_tests(loader, tests, pattern):
     tests.addTests(loader.loadTestsFromNames(
-        qubes.tests.create_testcases_for_templates('TC_05_StandaloneVM',
-            TC_05_StandaloneVMMixin, qubes.tests.SystemTestCase,
-            module=sys.modules[__name__])))
-    tests.addTests(loader.loadTestsFromNames(
-        qubes.tests.create_testcases_for_templates('TC_06_AppVM',
-            TC_06_AppVMMixin, qubes.tests.SystemTestCase,
-            module=sys.modules[__name__])))
+        create_testcases_for_templates()))
 
     return tests
 
+qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)
+
 # vim: ts=4 sw=4 et

qubes/tests/integ/dispvm.py

@@ -294,9 +294,15 @@ class TC_20_DispVMMixin(object):
             test_txt_content = test_txt_content[3:]
         self.assertEqual(test_txt_content, b"Test test 2\ntest1\n")
 
+
+def create_testcases_for_templates():
+    return qubes.tests.create_testcases_for_templates('TC_20_DispVM',
+        TC_20_DispVMMixin, qubes.tests.SystemTestCase,
+        module=sys.modules[__name__])
+
 def load_tests(loader, tests, pattern):
     tests.addTests(loader.loadTestsFromNames(
-        qubes.tests.create_testcases_for_templates('TC_20_DispVM',
-            TC_20_DispVMMixin, qubes.tests.SystemTestCase,
-            module=sys.modules[__name__])))
+        create_testcases_for_templates()))
     return tests
+
+qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)

qubes/tests/integ/dom0_update.py

@@ -380,9 +380,14 @@ Test package
                          'UNSIGNED package {}-1.0 installed'.format(self.pkg_name))
 
 
+def create_testcases_for_templates():
+    return qubes.tests.create_testcases_for_templates('TC_00_Dom0Upgrade',
+        TC_00_Dom0UpgradeMixin, qubes.tests.SystemTestCase,
+        module=sys.modules[__name__])
+
 def load_tests(loader, tests, pattern):
     tests.addTests(loader.loadTestsFromNames(
-        qubes.tests.create_testcases_for_templates('TC_00_Dom0Upgrade',
-            TC_00_Dom0UpgradeMixin, qubes.tests.SystemTestCase,
-            module=sys.modules[__name__])))
+        create_testcases_for_templates()))
     return tests
+
+qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)

qubes/tests/integ/mime.py

@@ -20,6 +20,7 @@
 # License along with this library; if not, see <https://www.gnu.org/licenses/>.
 #
 #
+import os
 from distutils import spawn
 import re
 import subprocess
@@ -335,9 +336,14 @@ class TC_50_MimeHandlers:
                                         ["Firefox", "Iceweasel", "Navigator"],
                                         dispvm=True)
 
+def create_testcases_for_templates():
+    return qubes.tests.create_testcases_for_templates('TC_50_MimeHandlers',
+        TC_50_MimeHandlers, qubes.tests.SystemTestCase,
+        module=sys.modules[__name__])
+
 def load_tests(loader, tests, pattern):
     tests.addTests(loader.loadTestsFromNames(
-        qubes.tests.create_testcases_for_templates('TC_50_MimeHandlers',
-            TC_50_MimeHandlers, qubes.tests.SystemTestCase,
-            module=sys.modules[__name__])))
+        create_testcases_for_templates()))
     return tests
+
+qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)

qubes/tests/integ/network.py

@@ -1319,17 +1319,20 @@ SHA256:
             self.assertIn(self.loop.run_until_complete(p.wait()), self.exit_code_ok,
                 '{}: {}\n{}'.format(self.update_cmd, stdout, stderr))
 
+def create_testcases_for_templates():
+    yield from qubes.tests.create_testcases_for_templates('VmNetworking',
+        VmNetworkingMixin, qubes.tests.SystemTestCase,
+        module=sys.modules[__name__])
+    yield from qubes.tests.create_testcases_for_templates('VmIPv6Networking',
+        VmIPv6NetworkingMixin, qubes.tests.SystemTestCase,
+        module=sys.modules[__name__])
+    yield from qubes.tests.create_testcases_for_templates('VmUpdates',
+        VmUpdatesMixin, qubes.tests.SystemTestCase,
+        module=sys.modules[__name__])
+
 def load_tests(loader, tests, pattern):
     tests.addTests(loader.loadTestsFromNames(
-        qubes.tests.create_testcases_for_templates('VmNetworking',
-            VmNetworkingMixin, qubes.tests.SystemTestCase,
-            module=sys.modules[__name__])))
-    tests.addTests(loader.loadTestsFromNames(
-        qubes.tests.create_testcases_for_templates('VmIPv6Networking',
-            VmIPv6NetworkingMixin, qubes.tests.SystemTestCase,
-            module=sys.modules[__name__])))
-    tests.addTests(loader.loadTestsFromNames(
-        qubes.tests.create_testcases_for_templates('VmUpdates',
-            VmUpdatesMixin, qubes.tests.SystemTestCase,
-            module=sys.modules[__name__])))
+        create_testcases_for_templates()))
     return tests
+
+qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)

qubes/tests/integ/pvgrub.py

@@ -137,10 +137,14 @@ class TC_40_PVGrub(object):
             self.test_template.run_for_stdio('uname -r'))
         self.assertEquals(actual_kver.strip(), kver)
 
+def create_testcases_for_templates():
+    return qubes.tests.create_testcases_for_templates('TC_40_PVGrub',
+        TC_40_PVGrub, qubes.tests.SystemTestCase,
+        module=sys.modules[__name__])
 
 def load_tests(loader, tests, pattern):
     tests.addTests(loader.loadTestsFromNames(
-        qubes.tests.create_testcases_for_templates('TC_40_PVGrub',
-            TC_40_PVGrub, qubes.tests.SystemTestCase,
-            module=sys.modules[__name__])))
+        create_testcases_for_templates()))
     return tests
+
+qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)

qubes/tests/integ/salt.py

@@ -391,9 +391,15 @@ class SaltVMTestMixin(SaltTestMixin):
             self.assertEqual(stderr, b'')
 
 
+def create_testcases_for_templates():
+    return qubes.tests.create_testcases_for_templates('TC_10_VMSalt',
+        SaltVMTestMixin, qubes.tests.SystemTestCase,
+        module=sys.modules[__name__])
+
+
 def load_tests(loader, tests, pattern):
     tests.addTests(loader.loadTestsFromNames(
-        qubes.tests.create_testcases_for_templates('TC_10_VMSalt',
-            SaltVMTestMixin, qubes.tests.SystemTestCase,
-            module=sys.modules[__name__])))
+        create_testcases_for_templates()))
     return tests
+
+qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)

qubes/tests/integ/vm_qrexec_gui.py

@@ -1194,10 +1194,14 @@ class TC_10_Generic(qubes.tests.SystemTestCase):
             'Flag file created (service was run) even though should be denied,'
             ' qrexec-client-vm output: {} {}'.format(stdout, stderr))
 
+def create_testcases_for_templates():
+    return qubes.tests.create_testcases_for_templates('TC_00_AppVM',
+        TC_00_AppVMMixin, qubes.tests.SystemTestCase,
+        module=sys.modules[__name__])
 
 def load_tests(loader, tests, pattern):
     tests.addTests(loader.loadTestsFromNames(
-        qubes.tests.create_testcases_for_templates('TC_00_AppVM',
-            TC_00_AppVMMixin, qubes.tests.SystemTestCase,
-            module=sys.modules[__name__])))
+        create_testcases_for_templates()))
     return tests
+
+qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)