diff --git a/qubes/app.py b/qubes/app.py index 06a1a41d..48561bd1 100644 --- a/qubes/app.py +++ b/qubes/app.py @@ -483,7 +483,8 @@ class VMCollection(object): raise qubes.exc.QubesVMNotHaltedError(vm) self.app.fire_event('domain-pre-delete', pre_event=True, vm=vm) try: - vm.libvirt_domain.undefine() + if vm.libvirt_domain: + vm.libvirt_domain.undefine() # pylint: disable=protected-access vm._libvirt_domain = None except libvirt.libvirtError as e: diff --git a/qubes/qmemman/__init__.py b/qubes/qmemman/__init__.py index a4540eaf..1c7ff780 100644 --- a/qubes/qmemman/__init__.py +++ b/qubes/qmemman/__init__.py @@ -140,6 +140,8 @@ class SystemState(object): def clear_outdated_error_markers(self): # Clear outdated errors for i in self.domdict.keys(): + if self.domdict[i].mem_used is None: + continue # clear markers excluding VM from memory balance, if: # - VM have responded to previous request (with some safety margin) # - VM request more memory than it has assigned diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index 8c1deb7b..a9553791 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -60,6 +60,7 @@ import qubes.config import qubes.devices import qubes.events import qubes.exc +import qubes.ext.pci import qubes.vm.standalonevm import qubes.vm.templatevm @@ -378,6 +379,7 @@ class QubesTestCase(unittest.TestCase): self.loop = asyncio.get_event_loop() self.addCleanup(self.cleanup_loop) + self.addCleanup(qubes.ext.pci._cache_get.cache_clear) def cleanup_gc(self): gc.collect() diff --git a/qubes/tests/extra.py b/qubes/tests/extra.py index a60cdfab..98fc6e71 100644 --- a/qubes/tests/extra.py +++ b/qubes/tests/extra.py @@ -62,6 +62,9 @@ class VMWrapper(object): def __eq__(self, other): return self._vm == other + def __hash__(self): + return hash(self._vm) + def start(self): return self._loop.run_until_complete(self._vm.start()) @@ -84,6 +87,27 @@ class VMWrapper(object): stderr=subprocess.PIPE if passio_stderr else None)) return ProcessWrapper(p, self._loop) + def run_service(self, service, wait=True, input=None, user=None, + passio_popen=False, + passio_stderr=False, **kwargs): + if wait: + try: + if isinstance(input, str): + input = input.encode() + self._loop.run_until_complete( + self._vm.run_service_for_stdio(service, + input=input, user=user)) + except subprocess.CalledProcessError as err: + return err.returncode + return 0 + elif passio_popen: + p = self._loop.run_until_complete(self._vm.run_service(service, + user=user, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE if passio_stderr else None)) + return ProcessWrapper(p, self._loop) + class ExtraTestCase(qubes.tests.SystemTestCase): @@ -137,6 +161,30 @@ class ExtraTestCase(qubes.tests.SystemTestCase): """ self.init_networking() + def qrexec_policy(self, service, source, destination, allow=True): + """ + Allow qrexec calls for duration of the test + :param service: service name + :param source: source VM name + :param destination: destination VM name + :return: + """ + + def add_remove_rule(add=True): + with open('/etc/qubes-rpc/policy/{}'.format(service), 'r+') as policy: + policy_rules = policy.readlines() + rule = "{} {} {}\n".format(source, destination, + 'allow' if allow else 'deny') + if add: + policy_rules.insert(0, rule) + else: + policy_rules.remove(rule) + policy.truncate(0) + policy.seek(0) + policy.write(''.join(policy_rules)) + add_remove_rule(add=True) + self.addCleanup(add_remove_rule, add=False) + def load_tests(loader, tests, pattern): for entry in pkg_resources.iter_entry_points('qubes.tests.extra'): diff --git a/qubes/tests/integ/backup.py b/qubes/tests/integ/backup.py index 52ddced9..cf1576d7 100644 --- a/qubes/tests/integ/backup.py +++ b/qubes/tests/integ/backup.py @@ -90,7 +90,10 @@ class BackupTestsMixin(object): block_size = 4096 self.log.debug("Filling %s" % path) - f = open(path, 'wb+') + try: + f = open(path, 'rb+') + except FileNotFoundError: + f = open(path, 'wb+') if size is None: f.seek(0, 2) size = f.tell() diff --git a/qubes/tests/integ/basic.py b/qubes/tests/integ/basic.py index 99ca131a..ed8745fb 100644 --- a/qubes/tests/integ/basic.py +++ b/qubes/tests/integ/basic.py @@ -91,6 +91,7 @@ class TC_00_Basic(qubes.tests.SystemTestCase): self.vm = self.app.add_new_vm('StandaloneVM', label='red', name=vmname) self.loop.run_until_complete(self.vm.create_on_disk()) self.vm.kernel = None + self.vm.virt_mode = 'hvm' iso_path = self.create_bootable_iso() # start the VM using qvm-start tool, to test --cdrom option there diff --git a/qubes/tests/integ/devices_pci.py b/qubes/tests/integ/devices_pci.py index 484202a1..d0344134 100644 --- a/qubes/tests/integ/devices_pci.py +++ b/qubes/tests/integ/devices_pci.py @@ -37,7 +37,10 @@ class TC_00_Devices_PCI(qubes.tests.SystemTestCase): if self._testMethodName not in ['test_000_list']: pcidev = os.environ['QUBES_TEST_PCIDEV'] self.dev = self.app.domains[0].devices['pci'][pcidev] - self.assignment = qubes.devices.DeviceAssignment(backend_domain=self.dev.backend_domain, ident=self.dev.ident, persistent=True) + self.assignment = qubes.devices.DeviceAssignment( + backend_domain=self.dev.backend_domain, + ident=self.dev.ident, + persistent=True) if isinstance(self.dev, qubes.devices.UnknownDevice): self.skipTest('Specified device {} does not exists'.format(pcidev)) self.init_default_template() @@ -45,7 +48,8 @@ class TC_00_Devices_PCI(qubes.tests.SystemTestCase): name=self.make_vm_name('vm'), label='red', ) - self.vm.create_on_disk() + self.loop.run_until_complete( + self.vm.create_on_disk()) self.vm.features['pci-no-strict-reset/' + pcidev] = True self.app.save() @@ -57,11 +61,12 @@ class TC_00_Devices_PCI(qubes.tests.SystemTestCase): l.split(' (')[0].split(' ', 1) for l in p.communicate()[0].decode().splitlines()) for dev in self.app.domains[0].devices['pci']: + lspci_ident = dev.ident.replace('_', ':') self.assertIsInstance(dev, qubes.ext.pci.PCIDevice) self.assertEqual(dev.backend_domain, self.app.domains[0]) - self.assertIn(dev.ident, actual_devices) - self.assertEqual(dev.description, actual_devices[dev.ident]) - actual_devices.pop(dev.ident) + self.assertIn(lspci_ident, actual_devices) + self.assertEqual(dev.description, actual_devices[lspci_ident]) + actual_devices.pop(lspci_ident) if actual_devices: self.fail('Not all devices listed, missing: {}'.format( @@ -76,7 +81,8 @@ class TC_00_Devices_PCI(qubes.tests.SystemTestCase): def test_010_attach_offline_persistent(self): dev_col = self.vm.devices['pci'] self.assertDeviceNotInCollection(self.dev, dev_col) - dev_col.attach(self.assignment) + self.loop.run_until_complete( + dev_col.attach(self.assignment)) self.app.save() self.assertNotIn(self.dev, dev_col.attached()) self.assertIn(self.dev, dev_col.persistent()) @@ -84,12 +90,11 @@ class TC_00_Devices_PCI(qubes.tests.SystemTestCase): self.assertIn(self.dev, dev_col.assignments(persistent=True)) self.assertNotIn(self.dev, dev_col.assignments(persistent=False)) - - self.vm.start() + self.loop.run_until_complete(self.vm.start()) self.assertIn(self.dev, dev_col.attached()) - p = self.vm.run('lspci', passio_popen=True) - (stdout, _) = p.communicate() + (stdout, _) = self.loop.run_until_complete( + self.vm.run_for_stdio('lspci')) self.assertIn(self.dev.description, stdout.decode()) @@ -98,14 +103,17 @@ class TC_00_Devices_PCI(qubes.tests.SystemTestCase): self.assertDeviceNotInCollection(self.dev, dev_col) self.assignment.persistent = False with self.assertRaises(qubes.exc.QubesVMNotRunningError): - dev_col.attach(self.assignment) + self.loop.run_until_complete( + dev_col.attach(self.assignment)) def test_020_attach_online_persistent(self): - self.vm.start() + self.loop.run_until_complete( + self.vm.start()) dev_col = self.vm.devices['pci'] self.assertDeviceNotInCollection(self.dev, dev_col) - dev_col.attach(self.assignment) + self.loop.run_until_complete( + dev_col.attach(self.assignment)) self.assertIn(self.dev, dev_col.attached()) self.assertIn(self.dev, dev_col.persistent()) @@ -115,39 +123,46 @@ class TC_00_Devices_PCI(qubes.tests.SystemTestCase): # give VM kernel some time to discover new device time.sleep(1) - p = self.vm.run('lspci', passio_popen=True) - (stdout, _) = p.communicate() + (stdout, _) = self.loop.run_until_complete( + self.vm.run_for_stdio('lspci')) self.assertIn(self.dev.description, stdout.decode()) def test_021_persist_detach_online_fail(self): dev_col = self.vm.devices['pci'] self.assertDeviceNotInCollection(self.dev, dev_col) - dev_col.attach(self.assignment) + self.loop.run_until_complete( + dev_col.attach(self.assignment)) self.app.save() - self.vm.start() + self.loop.run_until_complete( + self.vm.start()) with self.assertRaises(qubes.exc.QubesVMNotHaltedError): - self.vm.devices['pci'].detach(self.assignment) + self.loop.run_until_complete( + self.vm.devices['pci'].detach(self.assignment)) def test_030_persist_attach_detach_offline(self): dev_col = self.vm.devices['pci'] self.assertDeviceNotInCollection(self.dev, dev_col) - dev_col.attach(self.assignment) + self.loop.run_until_complete( + dev_col.attach(self.assignment)) self.app.save() self.assertNotIn(self.dev, dev_col.attached()) self.assertIn(self.dev, dev_col.persistent()) self.assertIn(self.dev, dev_col.assignments()) self.assertIn(self.dev, dev_col.assignments(persistent=True)) self.assertNotIn(self.dev, dev_col.assignments(persistent=False)) - dev_col.detach(self.assignment) + self.loop.run_until_complete( + dev_col.detach(self.assignment)) self.assertDeviceNotInCollection(self.dev, dev_col) def test_031_attach_detach_online_temp(self): dev_col = self.vm.devices['pci'] - self.vm.start() + self.loop.run_until_complete( + self.vm.start()) self.assignment.persistent = False self.assertDeviceNotInCollection(self.dev, dev_col) - dev_col.attach(self.assignment) + self.loop.run_until_complete( + dev_col.attach(self.assignment)) self.assertIn(self.dev, dev_col.attached()) self.assertNotIn(self.dev, dev_col.persistent()) @@ -159,13 +174,14 @@ class TC_00_Devices_PCI(qubes.tests.SystemTestCase): # give VM kernel some time to discover new device time.sleep(1) - p = self.vm.run('lspci', passio_popen=True) - (stdout, _) = p.communicate() + (stdout, _) = self.loop.run_until_complete( + self.vm.run_for_stdio('lspci')) self.assertIn(self.dev.description, stdout.decode()) - dev_col.detach(self.assignment) + self.loop.run_until_complete( + dev_col.detach(self.assignment)) self.assertDeviceNotInCollection(self.dev, dev_col) - p = self.vm.run('lspci', passio_popen=True) - (stdout, _) = p.communicate() + (stdout, _) = self.loop.run_until_complete( + self.vm.run_for_stdio('lspci')) self.assertNotIn(self.dev.description, stdout.decode()) diff --git a/qubes/tests/integ/vm_qrexec_gui.py b/qubes/tests/integ/vm_qrexec_gui.py index e6ca70e1..e818c8cd 100644 --- a/qubes/tests/integ/vm_qrexec_gui.py +++ b/qubes/tests/integ/vm_qrexec_gui.py @@ -925,15 +925,17 @@ int main(int argc, char **argv) { # it is important to have some changing content there, to generate # content update events (aka damage notify) proc = yield from self.testvm1.run( - 'gnome-terminal --full-screen -e top') + 'xterm -maximized -e top') # help xdotool a little... yield from asyncio.sleep(2) + if proc.returncode is not None: + self.fail('xterm failed to start') # get window ID winid = (yield from asyncio.get_event_loop().run_in_executor(None, subprocess.check_output, ['xdotool', 'search', '--sync', '--onlyvisible', '--class', - self.testvm1.name + ':.*erminal'])).decode() + self.testvm1.name + ':xterm'])).decode() xprop = yield from asyncio.get_event_loop().run_in_executor(None, subprocess.check_output, ['xprop', '-notype', '-id', winid, '_QUBES_VMWINDOWID']) diff --git a/qubes/tools/qmemmand.py b/qubes/tools/qmemmand.py index 24ae5f9d..640e8051 100644 --- a/qubes/tools/qmemmand.py +++ b/qubes/tools/qmemmand.py @@ -242,10 +242,6 @@ def main(): ha_file.setFormatter( logging.Formatter('%(asctime)s %(name)s[%(process)d]: %(message)s')) logging.root.addHandler(ha_stderr) - else: - # close io - sys.stdout.close() - sys.stderr.close() sys.stdin.close() diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 1d73f7ec..493ef397 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -391,8 +391,8 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): virt_mode = qubes.property('virt_mode', type=str, setter=_setter_virt_mode, default=_default_virt_mode, - doc='''Virtualisation mode: full virtualisation ("hvm"), - or paravirtualisation ("pv"), or hybrid ("pvh")''') + doc='''Virtualisation mode: full virtualisation ("HVM"), + or paravirtualisation ("PV"), or hybrid ("PVH")''') installed_by_rpm = qubes.property('installed_by_rpm', type=bool, setter=qubes.property.bool, @@ -570,6 +570,9 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): if self._libvirt_domain is not None: return self._libvirt_domain + if self.app.vmm.offline_mode: + return None + # XXX _update_libvirt_domain? try: self._libvirt_domain = self.app.vmm.libvirt_conn.lookupByUUID( @@ -869,6 +872,9 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): qmemman_client = None try: + if self.virt_mode == 'pvh' and self.kernel is None: + raise qubes.exc.QubesException( + 'virt_mode PVH require kernel to be set') yield from self.storage.verify() if self.netvm is not None: