Qubes/core-admin
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

tests: create testcases on module import if environment variable is set

Browse Source 
If QUBES_TEST_TEMPLATES or QUBES_TEST_LOAD_ALL is set, create testcases
on modules import, instead of waiting until `load_tests` is called.
The `QUBES_TEST_TEMPLATES` doesn't require `qubes.xml` access, so it
should be safe to do regardless of the environment. The
`QUBES_TEST_LOAD_ALL` force loading tests (and reading `qubes.xml`)
regardless.

This is useful for test runners not supporting load_tests protocol. Or
with limited support - for example both default `unittest` runner and
`nose2` can either use load_tests protocol _or_ select individual tests.
Setting any of those variable allow to run a single test with those
runners.

With this feature used together load_tests protocol, tests could be
registered twice. Avoid this by not listing already defined test classes
in create_testcases_for_templates (according to load_tests protocol,
those should already be registered).
This commit is contained in:
Marek Marczykowski-Górecki 2018-10-07 19:44:48 +02:00
parent c8929cfee9
commit 8dab298b89
No known key found for this signature in database
GPG Key ID: 063938BA42CFA724
10 changed files with 98 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
qubes/tests/__init__.py
View File

@ -1193,12 +1193,29 @@ def create_testcases_for_templates(name, *bases, module, **kwds):
for template in list_templates():
clsname = name + '_' + template
if hasattr(module, clsname):
continue
cls = type(clsname, bases, {'template': template, **kwds})
cls.__module__ = module.__name__
# XXX I wonder what other __dunder__ attrs did I miss
setattr(module, clsname, cls)
yield '.'.join((module.__name__, clsname))
def maybe_create_testcases_on_import(create_testcases_gen):
'''If certain conditions are met, call *create_testcases_gen* to create
testcases for templates tests. The purpose is to use it on integration
tests module(s) import, so the test runner could discover tests without
using load tests protocol.
The conditions - any of:
- QUBES_TEST_TEMPLATES present in the environment (it's possible to
create test cases without opening qubes.xml)
- QUBES_TEST_LOAD_ALL present in the environment
'''
if 'QUBES_TEST_TEMPLATES' in os.environ or \
'QUBES_TEST_LOAD_ALL' in os.environ:
list(create_testcases_gen())
def extra_info(obj):
'''Return short info identifying object.

11
qubes/tests/integ/backup.py
View File

@ -641,9 +641,14 @@ class TC_10_BackupVMMixin(BackupTestsMixin):
del vms
def create_testcases_for_templates():
return qubes.tests.create_testcases_for_templates('TC_10_BackupVM',
TC_10_BackupVMMixin, qubes.tests.SystemTestCase,
module=sys.modules[__name__])
def load_tests(loader, tests, pattern):
tests.addTests(loader.loadTestsFromNames(
qubes.tests.create_testcases_for_templates('TC_10_BackupVM',
TC_10_BackupVMMixin, qubes.tests.SystemTestCase,
module=sys.modules[__name__])))
create_testcases_for_templates()))
return tests
qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)

17
qubes/tests/integ/basic.py
View File

@ -681,17 +681,20 @@ class TC_06_AppVMMixin(object):
self.loop.run_until_complete(asyncio.sleep(1))
self.assertFalse(self.vm.is_running())
def create_testcases_for_templates():
yield from qubes.tests.create_testcases_for_templates('TC_05_StandaloneVM',
TC_05_StandaloneVMMixin, qubes.tests.SystemTestCase,
module=sys.modules[__name__])
yield from qubes.tests.create_testcases_for_templates('TC_06_AppVM',
TC_06_AppVMMixin, qubes.tests.SystemTestCase,
module=sys.modules[__name__])
def load_tests(loader, tests, pattern):
tests.addTests(loader.loadTestsFromNames(
qubes.tests.create_testcases_for_templates('TC_05_StandaloneVM',
TC_05_StandaloneVMMixin, qubes.tests.SystemTestCase,
module=sys.modules[__name__])))
tests.addTests(loader.loadTestsFromNames(
qubes.tests.create_testcases_for_templates('TC_06_AppVM',
TC_06_AppVMMixin, qubes.tests.SystemTestCase,
module=sys.modules[__name__])))
create_testcases_for_templates()))
return tests
qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)
# vim: ts=4 sw=4 et

12
qubes/tests/integ/dispvm.py
View File

@ -294,9 +294,15 @@ class TC_20_DispVMMixin(object):
test_txt_content = test_txt_content[3:]
self.assertEqual(test_txt_content, b"Test test 2\ntest1\n")
def create_testcases_for_templates():
return qubes.tests.create_testcases_for_templates('TC_20_DispVM',
TC_20_DispVMMixin, qubes.tests.SystemTestCase,
module=sys.modules[__name__])
def load_tests(loader, tests, pattern):
tests.addTests(loader.loadTestsFromNames(
qubes.tests.create_testcases_for_templates('TC_20_DispVM',
TC_20_DispVMMixin, qubes.tests.SystemTestCase,
module=sys.modules[__name__])))
create_testcases_for_templates()))
return tests
qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)

11
qubes/tests/integ/dom0_update.py
View File

@ -380,9 +380,14 @@ Test package
'UNSIGNED package {}-1.0 installed'.format(self.pkg_name))
def create_testcases_for_templates():
return qubes.tests.create_testcases_for_templates('TC_00_Dom0Upgrade',
TC_00_Dom0UpgradeMixin, qubes.tests.SystemTestCase,
module=sys.modules[__name__])
def load_tests(loader, tests, pattern):
tests.addTests(loader.loadTestsFromNames(
qubes.tests.create_testcases_for_templates('TC_00_Dom0Upgrade',
TC_00_Dom0UpgradeMixin, qubes.tests.SystemTestCase,
module=sys.modules[__name__])))
create_testcases_for_templates()))
return tests
qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)

12
qubes/tests/integ/mime.py
View File

@ -20,6 +20,7 @@
# License along with this library; if not, see <https://www.gnu.org/licenses/>.
#
#
import os
from distutils import spawn
import re
import subprocess
@ -335,9 +336,14 @@ class TC_50_MimeHandlers:
["Firefox", "Iceweasel", "Navigator"],
dispvm=True)
def create_testcases_for_templates():
return qubes.tests.create_testcases_for_templates('TC_50_MimeHandlers',
TC_50_MimeHandlers, qubes.tests.SystemTestCase,
module=sys.modules[__name__])
def load_tests(loader, tests, pattern):
tests.addTests(loader.loadTestsFromNames(
qubes.tests.create_testcases_for_templates('TC_50_MimeHandlers',
TC_50_MimeHandlers, qubes.tests.SystemTestCase,
module=sys.modules[__name__])))
create_testcases_for_templates()))
return tests
qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)

25
qubes/tests/integ/network.py
View File

@ -1319,17 +1319,20 @@ SHA256:
self.assertIn(self.loop.run_until_complete(p.wait()), self.exit_code_ok,
'{}: {}\n{}'.format(self.update_cmd, stdout, stderr))
def create_testcases_for_templates():
yield from qubes.tests.create_testcases_for_templates('VmNetworking',
VmNetworkingMixin, qubes.tests.SystemTestCase,
module=sys.modules[__name__])
yield from qubes.tests.create_testcases_for_templates('VmIPv6Networking',
VmIPv6NetworkingMixin, qubes.tests.SystemTestCase,
module=sys.modules[__name__])
yield from qubes.tests.create_testcases_for_templates('VmUpdates',
VmUpdatesMixin, qubes.tests.SystemTestCase,
module=sys.modules[__name__])
def load_tests(loader, tests, pattern):
tests.addTests(loader.loadTestsFromNames(
qubes.tests.create_testcases_for_templates('VmNetworking',
VmNetworkingMixin, qubes.tests.SystemTestCase,
module=sys.modules[__name__])))
tests.addTests(loader.loadTestsFromNames(
qubes.tests.create_testcases_for_templates('VmIPv6Networking',
VmIPv6NetworkingMixin, qubes.tests.SystemTestCase,
module=sys.modules[__name__])))
tests.addTests(loader.loadTestsFromNames(
qubes.tests.create_testcases_for_templates('VmUpdates',
VmUpdatesMixin, qubes.tests.SystemTestCase,
module=sys.modules[__name__])))
create_testcases_for_templates()))
return tests
qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)

10
qubes/tests/integ/pvgrub.py
View File

@ -137,10 +137,14 @@ class TC_40_PVGrub(object):
self.test_template.run_for_stdio('uname -r'))
self.assertEquals(actual_kver.strip(), kver)
def create_testcases_for_templates():
return qubes.tests.create_testcases_for_templates('TC_40_PVGrub',
TC_40_PVGrub, qubes.tests.SystemTestCase,
module=sys.modules[__name__])
def load_tests(loader, tests, pattern):
tests.addTests(loader.loadTestsFromNames(
qubes.tests.create_testcases_for_templates('TC_40_PVGrub',
TC_40_PVGrub, qubes.tests.SystemTestCase,
module=sys.modules[__name__])))
create_testcases_for_templates()))
return tests
qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)

12
qubes/tests/integ/salt.py
View File

@ -391,9 +391,15 @@ class SaltVMTestMixin(SaltTestMixin):
self.assertEqual(stderr, b'')
def create_testcases_for_templates():
return qubes.tests.create_testcases_for_templates('TC_10_VMSalt',
SaltVMTestMixin, qubes.tests.SystemTestCase,
module=sys.modules[__name__])
def load_tests(loader, tests, pattern):
tests.addTests(loader.loadTestsFromNames(
qubes.tests.create_testcases_for_templates('TC_10_VMSalt',
SaltVMTestMixin, qubes.tests.SystemTestCase,
module=sys.modules[__name__])))
create_testcases_for_templates()))
return tests
qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)

10
qubes/tests/integ/vm_qrexec_gui.py
View File

@ -1194,10 +1194,14 @@ class TC_10_Generic(qubes.tests.SystemTestCase):
'Flag file created (service was run) even though should be denied,'
' qrexec-client-vm output: {} {}'.format(stdout, stderr))
def create_testcases_for_templates():
return qubes.tests.create_testcases_for_templates('TC_00_AppVM',
TC_00_AppVMMixin, qubes.tests.SystemTestCase,
module=sys.modules[__name__])
def load_tests(loader, tests, pattern):
tests.addTests(loader.loadTestsFromNames(
qubes.tests.create_testcases_for_templates('TC_00_AppVM',
TC_00_AppVMMixin, qubes.tests.SystemTestCase,
module=sys.modules[__name__])))
create_testcases_for_templates()))
return tests
qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)