Merge branch 'tests-20180913'

* tests-20180913:
  tests: fix time sync test
  tests: wait for DispVM's qubes.VMShell exit
  tests: exclude whonixcheck and NetworkManager from editor window search
  tests: reenable some qrexec tests, convert them to py3k/asyncio
  tests: skip tests not relevant on Whonix
  tests: improve shutdown timeout handling
  tests: drop qvm-prefs tests

qubes/tests/integ/basic.py

@@ -109,7 +109,10 @@ class TC_00_Basic(qubes.tests.SystemTestCase):
         # Type 'poweroff'
         subprocess.check_call(['xdotool', 'search', '--name', self.vm.name,
                                'type', 'poweroff\r'])
-        self.loop.run_until_complete(asyncio.sleep(1))
+        for _ in range(5):
+            if not self.vm.is_running():
+                break
+            self.loop.run_until_complete(asyncio.sleep(1))
         self.assertFalse(self.vm.is_running())
 
     def _test_200_on_domain_start(self, vm, event, **_kwargs):
@@ -205,6 +208,8 @@ class TC_00_Basic(qubes.tests.SystemTestCase):
         if self.test_failure_reason:
             self.fail(self.test_failure_reason)
 
+        while self.vm.get_power_state() != 'Halted':
+            self.loop.run_until_complete(asyncio.sleep(1))
         # and give a chance for both domain-shutdown handlers to execute
         self.loop.run_until_complete(asyncio.sleep(1))
 
@@ -424,127 +429,6 @@ class TC_01_Properties(qubes.tests.SystemTestCase):
             self.loop.run_until_complete(self.vm2.create_on_disk())
 
 
-class TC_02_QvmPrefs(qubes.tests.SystemTestCase):
-    # pylint: disable=attribute-defined-outside-init
-
-    def setUp(self):
-        super(TC_02_QvmPrefs, self).setUp()
-        self.init_default_template()
-        self.sharedopts = ['--qubesxml', qubes.tests.XMLPATH]
-
-    def setup_appvm(self):
-        self.testvm = self.app.add_new_vm(
-            qubes.vm.appvm.AppVM,
-            name=self.make_vm_name("vm"),
-            label='red')
-        self.loop.run_until_complete(self.testvm.create_on_disk())
-        self.app.save()
-
-    def setup_hvm(self):
-        self.testvm = self.app.add_new_vm(
-            qubes.vm.appvm.AppVM,
-            name=self.make_vm_name("hvm"),
-            label='red')
-        self.testvm.virt_mode = 'hvm'
-        self.loop.run_until_complete(self.testvm.create_on_disk())
-        self.app.save()
-
-    def pref_set(self, name, value, valid=True):
-        self.loop.run_until_complete(self._pref_set(name, value, valid))
-
-    @asyncio.coroutine
-    def _pref_set(self, name, value, valid=True):
-        cmd = ['qvm-prefs']
-        if value != '-D':
-            cmd.append('--')
-        cmd.extend((self.testvm.name, name, value))
-        p = yield from asyncio.create_subprocess_exec(*cmd,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE)
-        (stdout, stderr) = yield from p.communicate()
-        if valid:
-            self.assertEqual(p.returncode, 0,
-                              "qvm-prefs .. '{}' '{}' failed: {}{}".format(
-                                  name, value, stdout, stderr
-                              ))
-        else:
-            self.assertNotEquals(p.returncode, 0,
-                                 "qvm-prefs should reject value '{}' for "
-                                 "property '{}'".format(value, name))
-
-    def pref_get(self, name):
-        self.loop.run_until_complete(self._pref_get(name))
-
-    @asyncio.coroutine
-    def _pref_get(self, name):
-        p = yield from asyncio.create_subprocess_exec(
-            'qvm-prefs', *self.sharedopts, '--', self.testvm.name, name,
-            stdout=subprocess.PIPE)
-        (stdout, _) = yield from p.communicate()
-        self.assertEqual(p.returncode, 0)
-        return stdout.strip()
-
-    bool_test_values = [
-        ('true', 'True', True),
-        ('False', 'False', True),
-        ('0', 'False', True),
-        ('1', 'True', True),
-        ('invalid', '', False)
-    ]
-
-    def execute_tests(self, name, values):
-        """
-        Helper function, which executes tests for given property.
-        :param values: list of tuples (value, expected, valid),
-        where 'value' is what should be set and 'expected' is what should
-        qvm-prefs returns as a property value and 'valid' marks valid and
-        invalid values - if it's False, qvm-prefs should reject the value
-        :return: None
-        """
-        for (value, expected, valid) in values:
-            self.pref_set(name, value, valid)
-            if valid:
-                self.assertEqual(self.pref_get(name), expected)
-
-    @unittest.skip('test not converted to core3 API')
-    def test_006_template(self):
-        templates = [tpl for tpl in self.app.domains.values() if
-            isinstance(tpl, qubes.vm.templatevm.TemplateVM)]
-        if not templates:
-            self.skipTest("No templates installed")
-        some_template = templates[0].name
-        self.setup_appvm()
-        self.execute_tests('template', [
-            (some_template, some_template, True),
-            ('invalid', '', False),
-        ])
-
-    @unittest.skip('test not converted to core3 API')
-    def test_014_pcidevs(self):
-        self.setup_appvm()
-        self.execute_tests('pcidevs', [
-            ('[]', '[]', True),
-            ('[ "00:00.0" ]', "['00:00.0']", True),
-            ('invalid', '', False),
-            ('[invalid]', '', False),
-            # TODO:
-            # ('["12:12.0"]', '', False)
-        ])
-
-    @unittest.skip('test not converted to core3 API')
-    def test_024_pv_reject_hvm_props(self):
-        self.setup_appvm()
-        self.execute_tests('guiagent_installed', [('False', '', False)])
-        self.execute_tests('qrexec_installed', [('False', '', False)])
-        self.execute_tests('drive', [('/tmp/drive.img', '', False)])
-        self.execute_tests('timezone', [('localtime', '', False)])
-
-    @unittest.skip('test not converted to core3 API')
-    def test_025_hvm_reject_pv_props(self):
-        self.setup_hvm()
-        self.execute_tests('kernel', [('default', '', False)])
-        self.execute_tests('kernelopts', [('default', '', False)])
-
 class TC_03_QvmRevertTemplateChanges(qubes.tests.SystemTestCase):
     # pylint: disable=attribute-defined-outside-init
 
@@ -791,7 +675,10 @@ class TC_06_AppVMMixin(object):
         # Type 'poweroff'
         subprocess.check_call(['xdotool', 'search', '--name', self.vm.name,
                                'type', 'poweroff\r'])
-        self.loop.run_until_complete(asyncio.sleep(1))
+        for _ in range(5):
+            if not self.vm.is_running():
+                break
+            self.loop.run_until_complete(asyncio.sleep(1))
         self.assertFalse(self.vm.is_running())
 
 

qubes/tests/integ/dispvm.py

@@ -23,6 +23,7 @@ import subprocess
 import tempfile
 import time
 import unittest
+from contextlib import suppress
 
 from distutils import spawn
 
@@ -162,8 +163,15 @@ class TC_20_DispVMMixin(object):
                 self.enter_keys_in_window(window_title, ['Return'])
                 # Wait for window to close
                 self.wait_for_window(window_title, show=False)
-            finally:
                 p.stdin.close()
+                self.loop.run_until_complete(
+                    asyncio.wait_for(p.wait(), 30))
+            except:
+                with suppress(ProcessLookupError):
+                    p.terminate()
+                self.loop.run_until_complete(p.wait())
+                raise
+            finally:
                 del p
         finally:
             self.loop.run_until_complete(dispvm.cleanup())
@@ -267,7 +275,9 @@ class TC_20_DispVMMixin(object):
                 # ignore LibreOffice splash screen and window with no title
                 # set yet
                 if window_title and not window_title.startswith("LibreOffice")\
-                        and not window_title == 'VMapp command':
+                        and not window_title == 'VMapp command' \
+                        and 'whonixcheck' not in window_title \
+                        and not window_title == 'NetworkManager Applet':
                     break
             wait_count += 1
             if wait_count > 100:

qubes/tests/integ/vm_qrexec_gui.py

@@ -114,6 +114,8 @@ class TC_00_AppVMMixin(object):
     def test_011_run_gnome_terminal(self):
         if "minimal" in self.template:
             self.skipTest("Minimal template doesn't have 'gnome-terminal'")
+        if 'whonix' in self.template:
+            self.skipTest("Whonix template doesn't have 'gnome-terminal'")
         self.loop.run_until_complete(self.testvm1.start())
         self.assertEqual(self.testvm1.get_power_state(), "Running")
         self.loop.run_until_complete(self.wait_for_session(self.testvm1))
@@ -221,7 +223,6 @@ class TC_00_AppVMMixin(object):
         self.assertFalse(stderr,
             'Some data was printed to stderr')
 
-    @unittest.skip('#2851, because there is no GUI in vm')
     def test_051_qrexec_simple_eof_reverse(self):
         """Test for EOF transmission VM->dom0"""
 
@@ -239,7 +240,7 @@ class TC_00_AppVMMixin(object):
             p.stdin.write(TEST_DATA)
             yield from p.stdin.drain()
             p.stdin.close()
-            self.assertEqual(stdout.strip(), 'test',
+            self.assertEqual(stdout.strip(), b'test',
                 'Received data differs from what was expected')
             # this may hang in some buggy cases
             self.assertFalse((yield from p.stderr.read()),
@@ -252,15 +253,18 @@ class TC_00_AppVMMixin(object):
                     "probably EOF wasn't transferred from the VM process")
 
         self.loop.run_until_complete(self.testvm1.start())
+        self.loop.run_until_complete(self.wait_for_session(self.testvm1))
         self.loop.run_until_complete(run(self))
 
-    @unittest.skip('#2851, because there is no GUI in vm')
     def test_052_qrexec_vm_service_eof(self):
         """Test for EOF transmission VM(src)->VM(dst)"""
 
         self.loop.run_until_complete(asyncio.wait([
             self.testvm1.start(),
             self.testvm2.start()]))
+        self.loop.run_until_complete(asyncio.wait([
+            self.wait_for_session(self.testvm1),
+            self.wait_for_session(self.testvm2)]))
         self.loop.run_until_complete(self.testvm2.run_for_stdio(
             'cat > /etc/qubes-rpc/test.EOF',
             user='root',
@@ -277,7 +281,7 @@ class TC_00_AppVMMixin(object):
             except asyncio.TimeoutError:
                 self.fail("Timeout, probably EOF wasn't transferred")
 
-        self.assertEqual(stdout, b'test',
+        self.assertEqual(stdout, b'test\n',
             'Received data differs from what was expected')
 
     @unittest.expectedFailure
@@ -398,23 +402,14 @@ class TC_00_AppVMMixin(object):
             except asyncio.TimeoutError:
                 self.fail('Timeout, probably deadlock')
 
-    @unittest.skip('localcmd= argument went away')
     def test_071_qrexec_dom0_simultaneous_write(self):
         """Test for simultaneous write in dom0(src)->VM(dst) connection
 
             Similar to test_070_qrexec_vm_simultaneous_write, but with dom0
             as a source.
         """
-        def run(self):
+
-            result.value = self.testvm2.run_service(
+        self.loop.run_until_complete(self.testvm2.start())
-                "test.write", localcmd="/bin/sh -c '"
-                # first write a lot of data to fill all the buffers
-                "dd if=/dev/zero bs=993 count=10000 iflag=fullblock & "
-                # then after some time start reading
-                "sleep 1; "
-                "dd of=/dev/null bs=993 count=10000 iflag=fullblock; "
-                "wait"
-                "'")
 
         self.create_remote_file(self.testvm2, '/etc/qubes-rpc/test.write', '''\
             # first write a lot of data
@@ -422,58 +417,93 @@ class TC_00_AppVMMixin(object):
             # and only then read something
             dd of=/dev/null bs=993 count=10000 iflag=fullblock
             ''')
-        self.create_local_file('/etc/qubes-rpc/policy/test.write',
-            '{} {} allow'.format(self.testvm1.name, self.testvm2.name))
 
-        t = multiprocessing.Process(target=run, args=(self,))
+        # can't use subprocess.PIPE, because asyncio will claim those FDs
-        t.start()
+        pipe1_r, pipe1_w = os.pipe()
-        t.join(timeout=10)
+        pipe2_r, pipe2_w = os.pipe()
-        if t.is_alive():
+        try:
-            t.terminate()
+            local_proc = self.loop.run_until_complete(
+                asyncio.create_subprocess_shell(
+                    # first write a lot of data to fill all the buffers
+                    "dd if=/dev/zero bs=993 count=10000 iflag=fullblock & "
+                    # then after some time start reading
+                    "sleep 1; "
+                    "dd of=/dev/null bs=993 count=10000 iflag=fullblock; "
+                    "wait", stdin=pipe1_r, stdout=pipe2_w))
+
+            service_proc = self.loop.run_until_complete(self.testvm2.run_service(
+                "test.write", stdin=pipe2_r, stdout=pipe1_w))
+        finally:
+            os.close(pipe1_r)
+            os.close(pipe1_w)
+            os.close(pipe2_r)
+            os.close(pipe2_w)
+
+        try:
+            self.loop.run_until_complete(
+                asyncio.wait_for(service_proc.wait(), timeout=10))
+        except asyncio.TimeoutError:
             self.fail("Timeout, probably deadlock")
-        self.assertEqual(result.value, 0, "Service call failed")
+        else:
+            self.assertEqual(service_proc.returncode, 0,
+                "Service call failed")
+        finally:
+            try:
+                service_proc.terminate()
+            except ProcessLookupError:
+                pass
 
-    @unittest.skip('localcmd= argument went away')
     def test_072_qrexec_to_dom0_simultaneous_write(self):
         """Test for simultaneous write in dom0(src)<-VM(dst) connection
 
             Similar to test_071_qrexec_dom0_simultaneous_write, but with dom0
             as a "hanging" side.
         """
-        result = multiprocessing.Value('i', -1)
-
-        def run(self):
-            result.value = self.testvm2.run_service(
-                "test.write", localcmd="/bin/sh -c '"
-                # first write a lot of data to fill all the buffers
-                "dd if=/dev/zero bs=993 count=10000 iflag=fullblock "
-                # then, only when all written, read something
-                "dd of=/dev/null bs=993 count=10000 iflag=fullblock; "
-                "'")
 
         self.loop.run_until_complete(self.testvm2.start())
-        p = self.testvm2.run("cat > /etc/qubes-rpc/test.write", user="root",
-                             passio_popen=True)
-        # first write a lot of data
-        p.stdin.write(b"dd if=/dev/zero bs=993 count=10000 iflag=fullblock &\n")
-        # and only then read something
-        p.stdin.write(b"dd of=/dev/null bs=993 count=10000 iflag=fullblock\n")
-        p.stdin.write(b"sleep 1; \n")
-        p.stdin.write(b"wait\n")
-        p.stdin.close()
-        p.wait()
-        policy = open("/etc/qubes-rpc/policy/test.write", "w")
-        policy.write("%s %s allow" % (self.testvm1.name, self.testvm2.name))
-        policy.close()
-        self.addCleanup(os.unlink, "/etc/qubes-rpc/policy/test.write")
 
-        t = multiprocessing.Process(target=run, args=(self,))
+        self.create_remote_file(self.testvm2, '/etc/qubes-rpc/test.write', '''\
-        t.start()
+            # first write a lot of data
-        t.join(timeout=10)
+            dd if=/dev/zero bs=993 count=10000 iflag=fullblock &
-        if t.is_alive():
+            # and only then read something
-            t.terminate()
+            dd of=/dev/null bs=993 count=10000 iflag=fullblock
+            sleep 1;
+            wait
+            ''')
+
+        # can't use subprocess.PIPE, because asyncio will claim those FDs
+        pipe1_r, pipe1_w = os.pipe()
+        pipe2_r, pipe2_w = os.pipe()
+        try:
+            local_proc = self.loop.run_until_complete(
+                asyncio.create_subprocess_shell(
+                    # first write a lot of data to fill all the buffers
+                    "dd if=/dev/zero bs=993 count=10000 iflag=fullblock & "
+                    # then, only when all written, read something
+                    "dd of=/dev/null bs=993 count=10000 iflag=fullblock; ",
+                    stdin=pipe1_r, stdout=pipe2_w))
+
+            service_proc = self.loop.run_until_complete(self.testvm2.run_service(
+                "test.write", stdin=pipe2_r, stdout=pipe1_w))
+        finally:
+            os.close(pipe1_r)
+            os.close(pipe1_w)
+            os.close(pipe2_r)
+            os.close(pipe2_w)
+
+        try:
+            self.loop.run_until_complete(
+                asyncio.wait_for(service_proc.wait(), timeout=10))
+        except asyncio.TimeoutError:
             self.fail("Timeout, probably deadlock")
-        self.assertEqual(result.value, 0, "Service call failed")
+        else:
+            self.assertEqual(service_proc.returncode, 0,
+                "Service call failed")
+        finally:
+            try:
+                service_proc.terminate()
+            except ProcessLookupError:
+                pass
 
     def test_080_qrexec_service_argument_allow_default(self):
         """Qrexec service call with argument"""
@@ -746,7 +776,8 @@ class TC_00_AppVMMixin(object):
         if self.template.startswith('whonix-'):
             self.skipTest('qvm-sync-clock disabled for Whonix VMs')
         self.loop.run_until_complete(asyncio.wait([
-            self.testvm1.start()]))
+            self.testvm1.start(),
+            self.testvm2.start(),]))
         start_time = subprocess.check_output(['date', '-u', '+%s'])
 
         try:
@@ -756,11 +787,11 @@ class TC_00_AppVMMixin(object):
             subprocess.check_call(['sudo', 'date', '-s', '2001-01-01T12:34:56'],
                 stdout=subprocess.DEVNULL)
             self.loop.run_until_complete(
-                self.testvm1.run_for_stdio('date -s 2001-01-01T12:34:56',
+                self.testvm2.run_for_stdio('date -s 2001-01-01T12:34:56',
                     user='root'))
 
             self.loop.run_until_complete(
-                self.testvm1.run_for_stdio('qvm-sync-clock',
+                self.testvm2.run_for_stdio('qvm-sync-clock',
                     user='root'))
 
             p = self.loop.run_until_complete(
@@ -769,7 +800,7 @@ class TC_00_AppVMMixin(object):
             self.loop.run_until_complete(p.wait())
             self.assertEqual(p.returncode, 0)
             vm_time, _ = self.loop.run_until_complete(
-                self.testvm1.run_for_stdio('date -u +%s'))
+                self.testvm2.run_for_stdio('date -u +%s'))
             self.assertAlmostEquals(int(vm_time), int(start_time), delta=30)
 
             dom0_time = subprocess.check_output(['date', '-u', '+%s'])
@@ -786,6 +817,8 @@ class TC_00_AppVMMixin(object):
     @unittest.skipUnless(spawn.find_executable('parecord'),
                          "pulseaudio-utils not installed in dom0")
     def test_220_audio_playback(self):
+        if 'whonix-gw' in self.template:
+            self.skipTest('whonix-gw have no audio')
         self.loop.run_until_complete(self.testvm1.start())
         try:
             self.loop.run_until_complete(
@@ -847,6 +880,8 @@ class TC_00_AppVMMixin(object):
     @unittest.skipUnless(spawn.find_executable('parecord'),
                          "pulseaudio-utils not installed in dom0")
     def test_221_audio_record_muted(self):
+        if 'whonix-gw' in self.template:
+            self.skipTest('whonix-gw have no audio')
         self.loop.run_until_complete(self.testvm1.start())
         try:
             self.loop.run_until_complete(
@@ -886,6 +921,8 @@ class TC_00_AppVMMixin(object):
     @unittest.skipUnless(spawn.find_executable('parecord'),
                          "pulseaudio-utils not installed in dom0")
     def test_222_audio_record_unmuted(self):
+        if 'whonix-gw' in self.template:
+            self.skipTest('whonix-gw have no audio')
         self.loop.run_until_complete(self.testvm1.start())
         try:
             self.loop.run_until_complete(
@@ -1028,10 +1065,10 @@ int main(int argc, char **argv) {
             input=allocator_c.encode())
 
         try:
-            stdout, stderr = yield from self.testvm1.run_for_stdio(
+            yield from self.testvm1.run_for_stdio(
                 'gcc allocator.c -o allocator')
-        except subprocess.CalledProcessError:
+        except subprocess.CalledProcessError as e:
-            self.skipTest('allocator compile failed: {}'.format(stderr))
+            self.skipTest('allocator compile failed: {}'.format(e.stderr))
 
         # drop caches to have even more memory pressure
         yield from self.testvm1.run_for_stdio(