Merge remote-tracking branch 'qubesos/pr/209'

* qubesos/pr/209: qubes/tests: change globals= to module= and fix syntax errors qubes/tests: use loadTestsFromNames for nose2 compat
2018-04-30 01:13:11 +02:00 · 2018-04-30 01:13:11 +02:00 · 39a9e4e422
--- a/qubes/tests/init.py
+++ b/qubes/tests/init.py
@ -1130,6 +1130,47 @@ def list_templates():
            _templates = ()
    return _templates

+def create_testcases_for_templates(name, *bases, module, **kwds):
+    '''Do-it-all helper for generating per-template tests via load_tests proto
+
+    This does several things:
+        - creates per-template classes
+        - adds them to module's :py:func:`globals`
+        - returns an iterable suitable for passing to loader.loadTestsFromNames
+
+    TestCase classes created by this function have implicit `.template`
+    attribute, which contains name of the respective template. They are also
+    named with given prefix, underscore and template name. If template name
+    contains characters not valid as part of Python identifier, they are
+    impossible to get via standard ``.`` operator, though :py:func:`getattr` is
+    still usable.
+
+    >>> class MyTestsMixIn:
+    ...     def test_000_my_test(self):
+    ...         assert self.template.startswith('debian')
+    >>> def load_tests(loader, tests, pattern):
+    ...     tests.addTests(loader.loadTestsFromNames(
+    ...         qubes.tests.create_testcases_for_templates(
+    ...             'TC_00_MyTests', MyTestsMixIn, qubes.tests.SystemTestCase,
+    ...             module=sys.modules[__name__])))
+
+    *NOTE* adding ``module=sys.modules[__name__]`` is *mandatory*, and to allow
+    enforcing this, it uses keyword-only argument syntax, which is only in
+    Python 3.
+    '''
+    # Do not attempt to grab the module from traceback, since we are actually
+    # a generator and loadTestsFromNames may also be a generator, so it's not
+    # possible to correctly guess frame from stack. Explicit is better than
+    # implicit!
+
+    for template in list_templates():
+        clsname = name + '_' + template
+        cls = type(clsname, bases, {'template': template, **kwds})
+        cls.__module__ = module.__name__
+        # XXX I wonder what other __dunder__ attrs did I miss
+        setattr(module, clsname, cls)
+        yield '.'.join((module.__name__, clsname))
+
 def extra_info(obj):
    '''Return short info identifying object.

--- a/qubes/tests/extra.py
+++ b/qubes/tests/extra.py
@ -18,11 +18,12 @@
 # License along with this library; if not, see <https://www.gnu.org/licenses/>.
 #

-import sys
-
 import asyncio
 import subprocess
+import sys
+
 import pkg_resources
+
 import qubes.tests
 import qubes.vm.appvm

@ -206,15 +207,10 @@ def load_tests(loader, tests, pattern):
            'qubes.tests.extra.for_template'):
        try:
            for test_case in entry.load()():
-                for template in qubes.tests.list_templates():
-                    tests.addTests(loader.loadTestsFromTestCase(
-                        type(
-                            '{}_{}_{}'.format(
-                                entry.name, test_case.__name__, template),
-                            (test_case,),
-                            {'template': template}
-                        )
-                    ))
+                test.addTests(loader.loadTestsFromNames(
+                    qubes.tests.create_testcases_for_templates(
+                        test_case.__name__, test_case,
+                        globals=sys.modules[test_case.__module__].__dict__)))
        except Exception as err:  # pylint: disable=broad-except
            def runTest(self):
                raise err
--- a/qubes/tests/integ/backup.py
+++ b/qubes/tests/integ/backup.py
@ -651,11 +651,8 @@ class TC_10_BackupVMMixin(BackupTestsMixin):


 def load_tests(loader, tests, pattern):
-    for template in qubes.tests.list_templates():
-        tests.addTests(loader.loadTestsFromTestCase(
-            type(
-                'TC_10_BackupVM_' + template,
-                (TC_10_BackupVMMixin, qubes.tests.SystemTestCase),
-                {'template': template})))
-
+    tests.addTests(loader.loadTestsFromNames(
+        qubes.tests.create_testcases_for_templates('TC_10_BackupVM',
+            TC_10_BackupVMMixin, qubes.tests.SystemTestCase,
+            globals=globals())))
    return tests
--- a/qubes/tests/integ/basic.py
+++ b/qubes/tests/integ/basic.py
@ -793,17 +793,14 @@ class TC_06_AppVMMixin(object):


 def load_tests(loader, tests, pattern):
-    for template in qubes.tests.list_templates():
-        tests.addTests(loader.loadTestsFromTestCase(
-            type(
-                'TC_05_StandaloneVM_' + template,
-                (TC_05_StandaloneVMMixin, qubes.tests.SystemTestCase),
-                {'template': template})))
-        tests.addTests(loader.loadTestsFromTestCase(
-            type(
-                'TC_06_AppVM_' + template,
-                (TC_06_AppVMMixin, qubes.tests.SystemTestCase),
-                {'template': template})))
+    tests.addTests(loader.loadTestsFromNames(
+        qubes.tests.create_testcases_for_templates('TC_05_StandaloneVM',
+            TC_05_StandaloneVMMixin, qubes.tests.SystemTestCase,
+            globals=globals())))
+    tests.addTests(loader.loadTestsFromNames(
+        qubes.tests.create_testcases_for_templates('TC_06_AppVM',
+            TC_06_AppVMMixin, qubes.tests.SystemTestCase,
+            globals=globals())))

    return tests

--- a/qubes/tests/integ/dispvm.py
+++ b/qubes/tests/integ/dispvm.py
@ -282,11 +282,8 @@ class TC_20_DispVMMixin(object):
        self.assertEqual(test_txt_content, b"Test test 2\ntest1\n")

 def load_tests(loader, tests, pattern):
-    for template in qubes.tests.list_templates():
-        tests.addTests(loader.loadTestsFromTestCase(
-            type(
-                'TC_20_DispVM_' + template,
-                (TC_20_DispVMMixin, qubes.tests.SystemTestCase),
-                {'template': template})))
-
+    tests.addTests(loader.loadTestsFromNames(
+        qubes.tests.create_testcases_for_templates('TC_20_DispVM',
+            TC_20_DispVMMixin, qubes.tests.SystemTestCase,
+            globals=globals())))
    return tests
--- a/qubes/tests/integ/dom0_update.py
+++ b/qubes/tests/integ/dom0_update.py
@ -18,14 +18,14 @@
 # USA.
 #

+import asyncio
 import os
 import shutil
 import subprocess
+import sys
 import tempfile
 import unittest

-import asyncio
-
 import qubes
 import qubes.tests

@ -386,11 +386,8 @@ Test package


 def load_tests(loader, tests, pattern):
-    for template in qubes.tests.list_templates():
-        tests.addTests(loader.loadTestsFromTestCase(
-            type(
-                'TC_00_Dom0Upgrade_' + template,
-                (TC_00_Dom0UpgradeMixin, qubes.tests.SystemTestCase),
-                {'template': template})))
-
+    tests.addTests(loader.loadTestsFromNames(
+        qubes.tests.create_testcases_for_templates('TC_00_Dom0Upgrade',
+            TC_00_Dom0UpgradeMixin, qubes.tests.SystemTestCase,
+            module=sys.modules[__name__])))
    return tests
--- a/qubes/tests/integ/network.py
+++ b/qubes/tests/integ/network.py
@ -25,8 +25,9 @@ import asyncio
 import multiprocessing
 import os
 import subprocess
-import unittest
+import sys
 import time
+import unittest

 import qubes.tests
 import qubes.firewall
@ -1319,20 +1320,16 @@ SHA256:
                '{}: {}\n{}'.format(self.update_cmd, stdout, stderr))

 def load_tests(loader, tests, pattern):
-    for template in qubes.tests.list_templates():
-        tests.addTests(loader.loadTestsFromTestCase(
-            type(
-                'VmNetworking_' + template,
-                (VmNetworkingMixin, qubes.tests.SystemTestCase),
-                {'template': template})))
-        tests.addTests(loader.loadTestsFromTestCase(
-            type(
-                'VmIPv6Networking_' + template,
-                (VmIPv6NetworkingMixin, qubes.tests.SystemTestCase),
-                {'template': template})))
-        tests.addTests(loader.loadTestsFromTestCase(
-            type(
-                'VmUpdates_' + template,
-                (VmUpdatesMixin, qubes.tests.SystemTestCase),
-                {'template': template})))
+    tests.addTests(loader.loadTestsFromNames(
+        qubes.tests.create_testcases_for_templates('VmNetworking',
+            VmNetworkingMixin, qubes.tests.SystemTestCase,
+            module=sys.modules[__name__])))
+    tests.addTests(loader.loadTestsFromNames(
+        qubes.tests.create_testcases_for_templates('VmIPv6Networking',
+            VmIPv6NetworkingMixin, qubes.tests.SystemTestCase,
+            module=sys.modules[__name__])))
+    tests.addTests(loader.loadTestsFromNames(
+        qubes.tests.create_testcases_for_templates('VmUpdates',
+            VmUpdates, qubes.tests.SystemTestCase,
+            module=sys.modules[__name__])))
    return tests
--- a/qubes/tests/integ/pvgrub.py
+++ b/qubes/tests/integ/pvgrub.py
@ -23,8 +23,11 @@

 import os
 import subprocess
+import sys
 import unittest
+
 import qubes.tests
+
@unittest.skipUnless(os.path.exists('/var/lib/qubes/vm-kernels/pvgrub2'),
                     'grub-xen package not installed')
 class TC_40_PVGrub(object):
@ -136,11 +139,8 @@ class TC_40_PVGrub(object):


 def load_tests(loader, tests, pattern):
-    for template in qubes.tests.list_templates():
-        tests.addTests(loader.loadTestsFromTestCase(
-            type(
-                'TC_40_PVGrub_' + template,
-                (TC_40_PVGrub, qubes.tests.SystemTestCase),
-                {'template': template})))
-
+    tests.addTests(loader.loadTestsFromNames(
+        qubes.tests.create_testcases_for_templates('TC_40_PVGrub',
+            TC_40_PVGrub, qubes.tests.SystemTestCase,
+            module=sys.modules[__name__])))
    return tests
--- a/qubes/tests/integ/salt.py
+++ b/qubes/tests/integ/salt.py
@ -17,13 +17,13 @@
 #
 # You should have received a copy of the GNU General Public License along
 # with this program; if not, see <http://www.gnu.org/licenses/>.
-import os
-import subprocess
-import json
-
-import shutil

 import asyncio
+import json
+import os
+import shutil
+import subprocess
+import sys

 import qubes.tests

@ -392,10 +392,8 @@ class SaltVMTestMixin(SaltTestMixin):


 def load_tests(loader, tests, pattern):
-    for template in qubes.tests.list_templates():
-        tests.addTests(loader.loadTestsFromTestCase(
-            type(
-                'TC_10_VMSalt_' + template,
-                (SaltVMTestMixin, qubes.tests.SystemTestCase),
-                {'template': template})))
+    tests.addTests(loader.loadTestsFromNames(
+        qubes.tests.create_testcases_for_templates('TC_10_VMSalt',
+            SaltVMTestMixin, qubes.tests.SystemTestCase,
+            module=sys.modules[__name__])))
    return tests
--- a/qubes/tests/integ/vm_qrexec_gui.py
+++ b/qubes/tests/integ/vm_qrexec_gui.py
@ -23,6 +23,7 @@ import asyncio
 import multiprocessing
 import os
 import subprocess
+import sys
 import unittest

 from distutils import spawn
@ -1012,11 +1013,8 @@ class TC_10_Generic(qubes.tests.SystemTestCase):


 def load_tests(loader, tests, pattern):
-    for template in qubes.tests.list_templates():
-        tests.addTests(loader.loadTestsFromTestCase(
-            type(
-                'TC_00_AppVM_' + template,
-                (TC_00_AppVMMixin, qubes.tests.SystemTestCase),
-                {'template': template})))
-
+    tests.addTests(loader.loadTestsFromNames(
+        qubes.tests.create_testcases_for_templates('TC_00_AppVM',
+            TC_00_AppVMMixin, qubes.tests.SystemTestCase,
+            module=sys.modules[__name__])))
    return tests