From 84af7386f5182b7041cf24968e217a2f60c85838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 25 Jan 2016 23:59:56 +0100 Subject: [PATCH 01/54] tests: support VMs cleaned up in tearDownClass, instead of tearDown Usage: VMs with name created by self.make_vm_name(name, class_teardown=True) will be cleaned up in tearDownClass. It should be used only in setUpClass. Fixes QubesOS/qubes-issues#1691 --- tests/__init__.py | 98 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 68 insertions(+), 30 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 7f740e42..cc06ad6c 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -37,7 +37,8 @@ import qubes.backup import qubes.qubes import time -VMPREFIX = 'test-' +VMPREFIX = 'test-inst-' +CLSVMPREFIX = 'test-cls-' #: :py:obj:`True` if running in dom0, :py:obj:`False` otherwise @@ -214,7 +215,7 @@ class SystemTestsMixin(object): self.conn = libvirt.open(qubes.qubes.defaults['libvirt_uri']) - self.remove_test_vms() + self.remove_test_vms(self.qc, self.conn) def tearDown(self): super(SystemTestsMixin, self).tearDown() @@ -226,12 +227,12 @@ class SystemTestsMixin(object): except qubes.qubes.QubesException: pass - self.kill_test_vms() + self.kill_test_vms(self.qc) self.qc.lock_db_for_writing() self.qc.load() - self.remove_test_vms() + self.remove_test_vms(self.qc, self.conn) self.qc.save() self.qc.unlock_db() @@ -239,8 +240,36 @@ class SystemTestsMixin(object): self.conn.close() - def make_vm_name(self, name): - return VMPREFIX + name + @classmethod + def tearDownClass(cls): + super(SystemTestsMixin, cls).tearDownClass() + + qc = qubes.qubes.QubesVmCollection() + qc.lock_db_for_reading() + qc.load() + qc.unlock_db() + + conn = libvirt.open(qubes.qubes.defaults['libvirt_uri']) + + cls.kill_test_vms(qc, prefix=CLSVMPREFIX) + + qc.lock_db_for_writing() + qc.load() + + cls.remove_test_vms(qc, conn, prefix=CLSVMPREFIX) + + qc.save() + qc.unlock_db() + del qc + + conn.close() + + @staticmethod + def make_vm_name(name, class_teardown=False): + if class_teardown: + return CLSVMPREFIX + name + else: + return VMPREFIX + name def save_and_reload_db(self): self.qc.save() @@ -248,19 +277,21 @@ class SystemTestsMixin(object): self.qc.lock_db_for_writing() self.qc.load() - def kill_test_vms(self): + @staticmethod + def kill_test_vms(qc, prefix=VMPREFIX): # do not keep write lock while killing VMs, because that may cause a # deadlock with disk hotplug scripts (namely qvm-template-commit # called when shutting down TemplateVm) - self.qc.lock_db_for_reading() - self.qc.load() - self.qc.unlock_db() - for vm in self.qc.values(): - if vm.name.startswith(VMPREFIX): + qc.lock_db_for_reading() + qc.load() + qc.unlock_db() + for vm in qc.values(): + if vm.name.startswith(prefix): if vm.is_running(): vm.force_shutdown() - def _remove_vm_qubes(self, vm): + @classmethod + def _remove_vm_qubes(cls, qc, conn, vm): vmname = vm.name try: @@ -280,28 +311,30 @@ class SystemTestsMixin(object): except libvirt.libvirtError: pass - self.qc.pop(vm.qid) + qc.pop(vm.qid) del vm # Now ensure it really went away. This may not have happened, # for example if vm.libvirtDomain malfunctioned. try: - dom = self.conn.lookupByName(vmname) + dom = conn.lookupByName(vmname) except: pass else: - self._remove_vm_libvirt(dom) + cls._remove_vm_libvirt(dom) - self._remove_vm_disk(vmname) + cls._remove_vm_disk(vmname) - def _remove_vm_libvirt(self, dom): + @staticmethod + def _remove_vm_libvirt(dom): try: dom.destroy() except libvirt.libvirtError: # not running pass dom.undefine() - def _remove_vm_disk(self, vmname): + @staticmethod + def _remove_vm_disk(vmname): for dirspec in ( 'qubes_appvms_dir', 'qubes_servicevms_dir', @@ -315,10 +348,12 @@ class SystemTestsMixin(object): os.unlink(dirpath) def remove_vms(self, vms): - for vm in vms: self._remove_vm_qubes(vm) + for vm in vms: + self._remove_vm_qubes(self.qc, self.conn, vm) self.save_and_reload_db() - def remove_test_vms(self): + @classmethod + def remove_test_vms(cls, qc, conn, prefix=VMPREFIX): """Aggresively remove any domain that has name in testing namespace. .. warning:: @@ -330,17 +365,20 @@ class SystemTestsMixin(object): # first, remove them Qubes-way something_removed = False - for vm in self.qc.values(): - if vm.name.startswith(VMPREFIX): - self._remove_vm_qubes(vm) + for vm in qc.values(): + if vm.name.startswith(prefix): + cls._remove_vm_qubes(qc, conn, vm) something_removed = True if something_removed: - self.save_and_reload_db() + qc.save() + qc.unlock_db() + qc.lock_db_for_writing() + qc.load() # now remove what was only in libvirt - for dom in self.conn.listAllDomains(): - if dom.name().startswith(VMPREFIX): - self._remove_vm_libvirt(dom) + for dom in conn.listAllDomains(): + if dom.name().startswith(prefix): + cls._remove_vm_libvirt(dom) # finally remove anything that is left on disk vmnames = set() @@ -351,10 +389,10 @@ class SystemTestsMixin(object): dirpath = os.path.join(qubes.qubes.system_path['qubes_base_dir'], qubes.qubes.system_path[dirspec]) for name in os.listdir(dirpath): - if name.startswith(VMPREFIX): + if name.startswith(prefix): vmnames.add(name) for vmname in vmnames: - self._remove_vm_disk(vmname) + cls._remove_vm_disk(vmname) def wait_for_window(self, title, timeout=30, show=True): """ From be00d15d99926e4ea1bfd32ad5b65556d1955893 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 29 Jan 2016 21:30:11 +0100 Subject: [PATCH 02/54] tests: do not crash when trying to log class setup/teardown fail QubesOS/qubes-issues#1691 --- tests/run.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/run.py b/tests/run.py index d4064898..dae4c0b7 100755 --- a/tests/run.py +++ b/tests/run.py @@ -133,7 +133,11 @@ class QubesTestResult(unittest.TestResult): def addError(self, test, err): # pylint: disable=invalid-name super(QubesTestResult, self).addError(test, err) - test.log.critical('ERROR ({err[0].__name__}: {err[1]!r})'.format(err=err)) + try: + test.log.critical( + 'ERROR ({err[0].__name__}: {err[1]!r})'.format(err=err)) + except AttributeError: + pass if self.showAll: self.stream.writeln( '{color[red]}{color[bold]}ERROR{color[normal]} ({})'.format( @@ -157,7 +161,10 @@ class QubesTestResult(unittest.TestResult): def addSkip(self, test, reason): # pylint: disable=invalid-name super(QubesTestResult, self).addSkip(test, reason) - test.log.warning('skipped ({})'.format(reason)) + try: + test.log.warning('skipped ({})'.format(reason)) + except AttributeError: + pass if self.showAll: self.stream.writeln( '{color[cyan]}skipped{color[normal]} ({})'.format( From 0d2e03389f3a0b36c4af0262ddc84ceb1539d141 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 29 Jan 2016 21:31:49 +0100 Subject: [PATCH 03/54] tests: qvm-open-in-vm and qvm-open-in-dvm tests for different file types QubesOS/qubes-issues#1621 --- tests/vm_qrexec_gui.py | 301 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 300 insertions(+), 1 deletion(-) diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py index e6e03eee..ae5299b7 100644 --- a/tests/vm_qrexec_gui.py +++ b/tests/vm_qrexec_gui.py @@ -33,6 +33,7 @@ import time from qubes.qubes import QubesVmCollection, defaults, QubesException import qubes.tests +import re TEST_DATA = "0123456789" * 1024 @@ -733,7 +734,6 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin): # some safety margin for FS metadata self.assertGreater(int(new_size.strip()), 5.8*1024**2) - class TC_05_StandaloneVM(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): def test_000_create_start(self): testvm1 = self.qc.add_new_vm("QubesAppVm", @@ -1204,6 +1204,299 @@ class TC_40_PVGrub(qubes.tests.SystemTestsMixin): (actual_kver, _) = p.communicate() self.assertEquals(actual_kver.strip(), kver) +@unittest.skipUnless( + spawn.find_executable('xprop') and + spawn.find_executable('xdotool') and + spawn.find_executable('wmctrl'), + "xprop or xdotool or wmctrl not installed") +class TC_50_MimeHandlers(qubes.tests.SystemTestsMixin): + @classmethod + def setUpClass(cls): + if cls.template == 'whonix-gw' or 'minimal' in cls.template: + raise unittest.SkipTest( + 'Template {} not supported by this test'.format(cls.template)) + + if cls.template == 'whonix-ws': + # TODO remove when Whonix-based DispVMs will work (Whonix 13?) + raise unittest.SkipTest( + 'Template {} not supported by this test'.format(cls.template)) + + qc = QubesVmCollection() + + cls.kill_test_vms(qc, prefix=qubes.tests.CLSVMPREFIX) + + qc.lock_db_for_writing() + qc.load() + + cls.remove_test_vms(qc, qubes.qubes.vmm.libvirt_conn, + prefix=qubes.tests.CLSVMPREFIX) + + cls.source_vmname = cls.make_vm_name('source', True) + source_vm = qc.add_new_vm("QubesAppVm", + template=qc.get_vm_by_name(cls.template), + name=cls.source_vmname) + source_vm.create_on_disk(verbose=False) + + cls.target_vmname = cls.make_vm_name('target', True) + target_vm = qc.add_new_vm("QubesAppVm", + template=qc.get_vm_by_name(cls.template), + name=cls.target_vmname) + target_vm.create_on_disk(verbose=False) + + qc.save() + qc.unlock_db() + source_vm.start() + target_vm.start() + + # make sure that DispVMs will be started of the same template + retcode = subprocess.call(['/usr/bin/qvm-create-default-dvm', + cls.template], + stderr=open(os.devnull, 'w')) + assert retcode == 0, "Error preparing DispVM" + + def setUp(self): + super(TC_50_MimeHandlers, self).setUp() + self.source_vm = self.qc.get_vm_by_name(self.source_vmname) + self.target_vm = self.qc.get_vm_by_name(self.target_vmname) + + def get_window_class(self, winid, dispvm=False): + (vm_winid, _) = subprocess.Popen( + ['xprop', '-id', winid, '_QUBES_VMWINDOWID'], + stdout=subprocess.PIPE + ).communicate() + vm_winid = vm_winid.split("#")[1].strip('\n" ') + if dispvm: + (vmname, _) = subprocess.Popen( + ['xprop', '-id', winid, '_QUBES_VMNAME'], + stdout=subprocess.PIPE + ).communicate() + vmname = vmname.split("=")[1].strip('\n" ') + window_class = None + while window_class is None: + # XXX to use self.qc.get_vm_by_name would require reloading + # qubes.xml, so use qvm-run instead + xprop = subprocess.Popen( + ['qvm-run', '-p', vmname, 'xprop -id {} WM_CLASS'.format( + vm_winid)], stdout=subprocess.PIPE) + (window_class, _) = xprop.communicate() + if xprop.returncode != 0: + self.skipTest("xprop failed, not installed?") + if 'not found' in window_class: + # WM_CLASS not set yet, wait a little + time.sleep(0.1) + window_class = None + else: + window_class = None + while window_class is None: + xprop = self.target_vm.run( + 'xprop -id {} WM_CLASS'.format(vm_winid), + passio_popen=True) + (window_class, _) = xprop.communicate() + if xprop.returncode != 0: + self.skipTest("xprop failed, not installed?") + if 'not found' in window_class: + # WM_CLASS not set yet, wait a little + time.sleep(0.1) + window_class = None + # output: WM_CLASS(STRING) = "gnome-terminal-server", "Gnome-terminal" + try: + window_class = window_class.split("=")[1].split(",")[0].strip('\n" ') + except IndexError: + raise Exception( + "Unexpected output from xprop: '{}'".format(window_class)) + + return window_class + + def open_file_and_check_viewer(self, filename, expected_app_titles, + expected_app_classes, dispvm=False): + self.qc.unlock_db() + if dispvm: + p = self.source_vm.run("qvm-open-in-dvm {}".format(filename), + passio_popen=True) + vmpattern = "disp*" + else: + p = self.source_vm.run("qvm-open-in-vm {} {}".format( + self.target_vmname, filename), passio_popen=True) + vmpattern = self.target_vmname + if not dispvm: + # For opening a file in DispVM default policy is set to "allow" + self.enter_keys_in_window('Question', ['y']) + wait_count = 0 + winid = None + window_title = None + while True: + search = subprocess.Popen(['xdotool', 'search', + '--onlyvisible', '--class', vmpattern], + stdout=subprocess.PIPE, + stderr=open(os.path.devnull, 'w')) + retcode = search.wait() + if retcode == 0: + winid = search.stdout.read().strip() + # get window title + (window_title, _) = subprocess.Popen( + ['xdotool', 'getwindowname', winid], stdout=subprocess.PIPE). \ + communicate() + window_title = window_title.strip() + # ignore LibreOffice splash screen and window with no title + # set yet + if window_title and not window_title.startswith("LibreOffice")\ + and not window_title == 'VMapp command': + break + wait_count += 1 + if wait_count > 100: + self.fail("Timeout while waiting for editor window") + time.sleep(0.3) + + # get window class + window_class = self.get_window_class(winid, dispvm) + # close the window - we've got the window class, it is no longer needed + subprocess.check_call(['wmctrl', '-i', '-c', winid]) + p.wait() + self.wait_for_window(window_title, show=False) + + def check_matches(obj, patterns): + return any((pat.search(obj) if isinstance(pat, type(re.compile(''))) + else pat in obj) for pat in patterns) + + if not check_matches(window_title, expected_app_titles) and \ + not check_matches(window_class, expected_app_classes): + self.fail("Opening file {} resulted in window '{} ({})', which is " + "none of {!r} ({!r})".format( + filename, window_title, window_class, + expected_app_titles, expected_app_classes)) + + def prepare_txt(self, filename): + p = self.source_vm.run("cat > {}".format(filename), passio_popen=True) + p.stdin.write("This is test\n") + p.stdin.close() + retcode = p.wait() + assert retcode == 0, "Failed to write {} file".format(filename) + + def prepare_pdf(self, filename): + self.prepare_txt("/tmp/source.txt") + cmd = "convert /tmp/source.txt {}".format(filename) + retcode = self.source_vm.run(cmd, wait=True) + assert retcode == 0, "Failed to run '{}'".format(cmd) + + def prepare_doc(self, filename): + self.prepare_txt("/tmp/source.txt") + cmd = "unoconv -f doc -o {} /tmp/source.txt".format(filename) + retcode = self.source_vm.run(cmd, wait=True) + if retcode != 0: + self.skipTest("Failed to run '{}', not installed?".format(cmd)) + + def prepare_pptx(self, filename): + self.prepare_txt("/tmp/source.txt") + cmd = "unoconv -f pptx -o {} /tmp/source.txt".format(filename) + retcode = self.source_vm.run(cmd, wait=True) + if retcode != 0: + self.skipTest("Failed to run '{}', not installed?".format(cmd)) + + def prepare_png(self, filename): + self.prepare_txt("/tmp/source.txt") + cmd = "convert /tmp/source.txt {}".format(filename) + retcode = self.source_vm.run(cmd, wait=True) + if retcode != 0: + self.skipTest("Failed to run '{}', not installed?".format(cmd)) + + def prepare_jpg(self, filename): + self.prepare_txt("/tmp/source.txt") + cmd = "convert /tmp/source.txt {}".format(filename) + retcode = self.source_vm.run(cmd, wait=True) + if retcode != 0: + self.skipTest("Failed to run '{}', not installed?".format(cmd)) + + def test_000_txt(self): + filename = "/home/user/test_file.txt" + self.prepare_txt(filename) + self.open_file_and_check_viewer(filename, ["vim"], + ["gedit", "emacs"]) + + def test_001_pdf(self): + filename = "/home/user/test_file.pdf" + self.prepare_pdf(filename) + self.open_file_and_check_viewer(filename, [], + ["evince"]) + + def test_002_doc(self): + filename = "/home/user/test_file.doc" + self.prepare_doc(filename) + self.open_file_and_check_viewer(filename, [], + ["libreoffice", "abiword"]) + + def test_003_pptx(self): + filename = "/home/user/test_file.pptx" + self.prepare_pptx(filename) + self.open_file_and_check_viewer(filename, [], + ["libreoffice"]) + + def test_004_png(self): + filename = "/home/user/test_file.png" + self.prepare_png(filename) + self.open_file_and_check_viewer(filename, [], + ["shotwell", "eog", "display"]) + + def test_005_jpg(self): + filename = "/home/user/test_file.jpg" + self.prepare_jpg(filename) + self.open_file_and_check_viewer(filename, [], + ["shotwell", "eog", "display"]) + + def test_006_jpeg(self): + filename = "/home/user/test_file.jpeg" + self.prepare_jpg(filename) + self.open_file_and_check_viewer(filename, [], + ["shotwell", "eog", "display"]) + + def test_100_txt_dispvm(self): + filename = "/home/user/test_file.txt" + self.prepare_txt(filename) + self.open_file_and_check_viewer(filename, ["vim"], + ["gedit", "emacs"], + dispvm=True) + + def test_101_pdf_dispvm(self): + filename = "/home/user/test_file.pdf" + self.prepare_pdf(filename) + self.open_file_and_check_viewer(filename, [], + ["evince"], + dispvm=True) + + def test_102_doc_dispvm(self): + filename = "/home/user/test_file.doc" + self.prepare_doc(filename) + self.open_file_and_check_viewer(filename, [], + ["libreoffice", "abiword"], + dispvm=True) + + def test_103_pptx_dispvm(self): + filename = "/home/user/test_file.pptx" + self.prepare_pptx(filename) + self.open_file_and_check_viewer(filename, [], + ["libreoffice"], + dispvm=True) + + def test_104_png_dispvm(self): + filename = "/home/user/test_file.png" + self.prepare_png(filename) + self.open_file_and_check_viewer(filename, [], + ["shotwell", "eog", "display"], + dispvm=True) + + def test_105_jpg_dispvm(self): + filename = "/home/user/test_file.jpg" + self.prepare_jpg(filename) + self.open_file_and_check_viewer(filename, [], + ["shotwell", "eog", "display"], + dispvm=True) + + def test_106_jpeg_dispvm(self): + filename = "/home/user/test_file.jpeg" + self.prepare_jpg(filename) + self.open_file_and_check_viewer(filename, [], + ["shotwell", "eog", "display"], + dispvm=True) + def load_tests(loader, tests, pattern): try: @@ -1233,4 +1526,10 @@ def load_tests(loader, tests, pattern): (TC_40_PVGrub, qubes.tests.QubesTestCase), {'template': template}))) + tests.addTests(loader.loadTestsFromTestCase( + type( + 'TC_50_MimeHandlers_' + template, + (TC_50_MimeHandlers, qubes.tests.QubesTestCase), + {'template': template}))) + return tests From 0d31306b9098ecee5991d3450a7457710a452ac8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 3 Feb 2016 17:22:15 +0100 Subject: [PATCH 04/54] tests: clearing 'updates pending' flag QubesOS/qubes-issues#1685 --- tests/dom0_update.py | 70 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/tests/dom0_update.py b/tests/dom0_update.py index 12fcc95d..c9a8a5c2 100644 --- a/tests/dom0_update.py +++ b/tests/dom0_update.py @@ -41,6 +41,7 @@ class TC_00_Dom0UpgradeMixin(qubes.tests.SystemTestsMixin): pkg_name = 'qubes-test-pkg' dom0_update_common_opts = ['--disablerepo=*', '--enablerepo=test', '--setopt=test.copy_local=1'] + update_flag_path = '/var/lib/qubes/updates/dom0-updates-available' @classmethod def generate_key(cls, keydir): @@ -187,10 +188,18 @@ Test package "test".format(retcode)) def test_000_update(self): + """Dom0 update tests + + Check if package update is: + - detected + - installed + - "updates pending" flag is cleared + """ filename = self.create_pkg(self.tmpdir, self.pkg_name, '1.0') subprocess.check_call(['sudo', 'rpm', '-i', filename]) filename = self.create_pkg(self.tmpdir, self.pkg_name, '2.0') self.send_pkg(filename) + open(self.update_flag_path, 'a').close() logpath = os.path.join(self.tmpdir, 'dom0-update-output.txt') try: @@ -210,6 +219,67 @@ Test package self.pkg_name)], stdout=open(os.devnull, 'w')) self.assertEqual(retcode, 0, 'Package {}-2.0 not installed after ' 'update'.format(self.pkg_name)) + self.assertFalse(os.path.exists(self.update_flag_path), + "'updates pending' flag not cleared") + + def test_005_update_flag_clear(self): + """Check if 'updates pending' flag is creared""" + + # create any pkg (but not install it) to initialize repo in the VM + filename = self.create_pkg(self.tmpdir, self.pkg_name, '1.0') + self.send_pkg(filename) + open(self.update_flag_path, 'a').close() + + logpath = os.path.join(self.tmpdir, 'dom0-update-output.txt') + try: + subprocess.check_call(['sudo', 'qubes-dom0-update', '-y'] + + self.dom0_update_common_opts, + stdout=open(logpath, 'w'), + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError: + self.fail("qubes-dom0-update failed: " + open( + logpath).read()) + + with open(logpath) as f: + dom0_update_output = f.read() + self.assertFalse('Errno' in dom0_update_output or + 'Couldn\'t' in dom0_update_output, + "qubes-dom0-update reported an error: {}". + format(dom0_update_output)) + + self.assertFalse(os.path.exists(self.update_flag_path), + "'updates pending' flag not cleared") + + def test_006_update_flag_clear(self): + """Check if 'updates pending' flag is creared, using --clean""" + + # create any pkg (but not install it) to initialize repo in the VM + filename = self.create_pkg(self.tmpdir, self.pkg_name, '1.0') + self.send_pkg(filename) + open(self.update_flag_path, 'a').close() + + # remove also repodata to test #1685 + shutil.rmtree('/var/lib/qubes/updates/repodata') + logpath = os.path.join(self.tmpdir, 'dom0-update-output.txt') + try: + subprocess.check_call(['sudo', 'qubes-dom0-update', '-y', + '--clean'] + + self.dom0_update_common_opts, + stdout=open(logpath, 'w'), + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError: + self.fail("qubes-dom0-update failed: " + open( + logpath).read()) + + with open(logpath) as f: + dom0_update_output = f.read() + self.assertFalse('Errno' in dom0_update_output or + 'Couldn\'t' in dom0_update_output, + "qubes-dom0-update reported an error: {}". + format(dom0_update_output)) + + self.assertFalse(os.path.exists(self.update_flag_path), + "'updates pending' flag not cleared") def test_010_instal(self): filename = self.create_pkg(self.tmpdir, self.pkg_name, '1.0') From 8ee909f08c94b5513bc2d41670cc24ff6b6993e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 6 Feb 2016 05:34:22 +0100 Subject: [PATCH 05/54] Caught unknown libvirt errors on getting runtime VM info In some cases libvirt doesn't report error code at all. This probably happens in some stage of domain startup/shutdown. Threat this the same as domain not running. Fixes QubesOS/qubes-issues#1537 --- core-modules/000QubesVm.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py index 35f30d6b..12fce513 100644 --- a/core-modules/000QubesVm.py +++ b/core-modules/000QubesVm.py @@ -779,6 +779,8 @@ class QubesVm(object): # libxl_domain_info failed - domain no longer exists elif e.get_error_code() == libvirt.VIR_ERR_INTERNAL_ERROR: return 0 + elif e.get_error_code() is None: # unknown... + return 0 else: print >>sys.stderr, "libvirt error code: {!r}".format( e.get_error_code()) @@ -798,6 +800,8 @@ class QubesVm(object): # libxl_domain_info failed - domain no longer exists elif e.get_error_code() == libvirt.VIR_INTERNAL_ERROR: return 0 + elif e.get_error_code() is None: # unknown... + return 0 else: print >>sys.stderr, "libvirt error code: {!r}".format( e.get_error_code()) @@ -918,6 +922,11 @@ class QubesVm(object): except libvirt.libvirtError as e: if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN: return False + # libxl_domain_info failed - domain no longer exists + elif e.get_error_code() == libvirt.VIR_INTERNAL_ERROR: + return False + elif e.get_error_code() is None: # unknown... + return False else: print >>sys.stderr, "libvirt error code: {!r}".format( e.get_error_code()) @@ -932,6 +941,11 @@ class QubesVm(object): except libvirt.libvirtError as e: if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN: return False + # libxl_domain_info failed - domain no longer exists + elif e.get_error_code() == libvirt.VIR_INTERNAL_ERROR: + return False + elif e.get_error_code() is None: # unknown... + return False else: print >>sys.stderr, "libvirt error code: {!r}".format( e.get_error_code()) From 0b5bd060bbdd07fdc46bab46f2862fd9a81fb63f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 6 Feb 2016 05:36:07 +0100 Subject: [PATCH 06/54] Fix domain renaming when it wasn't defined in libvirt Fixes QubesOS/qubes-issues#1632 --- core-modules/000QubesVm.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py index 12fce513..b714f544 100644 --- a/core-modules/000QubesVm.py +++ b/core-modules/000QubesVm.py @@ -592,8 +592,13 @@ class QubesVm(object): raise QubesException("Cannot rename VM installed by RPM -- first clone VM and then use yum to remove package.") self.pre_rename(name) - if self.libvirt_domain: + try: self.libvirt_domain.undefine() + except libvirt.libvirtError as e: + if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN: + pass + else: + raise if self._qdb_connection: self._qdb_connection.close() self._qdb_connection = None From f13abc2ace8b1128e593c62cbc521442539e01c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 6 Feb 2016 05:36:53 +0100 Subject: [PATCH 07/54] tests: rename domain when it is't defined libvirt Regression test for QubesOS/qubes-issues#1632 --- tests/basic.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/basic.py b/tests/basic.py index 8e362671..30136207 100644 --- a/tests/basic.py +++ b/tests/basic.py @@ -32,6 +32,7 @@ import tempfile import unittest import time from qubes.qubes import QubesVmCollection, QubesException, system_path +import libvirt import qubes.qubes import qubes.tests @@ -114,6 +115,14 @@ class TC_01_Properties(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): '/etc/systemd/system/multi-user.target.wants/' 'qubes-vm@{}.service'.format(self.vmname))) + def test_001_rename_libvirt_undefined(self): + self.vm.libvirt_domain.undefine() + self.vm._libvirt_domain = None + + newname = self.make_vm_name('newname') + with self.assertNotRaises(libvirt.libvirtError): + self.vm.set_name(newname) + def test_010_netvm(self): if self.qc.get_default_netvm() is None: self.skip("Set default NetVM before running this test") From be231421ded49b3505a53dba42a3e60c9c6c6c61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 6 Feb 2016 05:37:27 +0100 Subject: [PATCH 08/54] tests: try to change name to conflicting value QubesOS/qubes-issues#1723 --- tests/basic.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/basic.py b/tests/basic.py index 30136207..f41563a4 100644 --- a/tests/basic.py +++ b/tests/basic.py @@ -268,6 +268,45 @@ class TC_01_Properties(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): self.assertEquals(testvm1.get_firewall_conf(), testvm3.get_firewall_conf()) + def test_020_name_conflict_app(self): + with self.assertRaises(QubesException): + self.vm2 = self.qc.add_new_vm('QubesAppVm', + name=self.vmname, template=self.qc.get_default_template()) + self.vm2.create_on_disk(verbose=False) + + def test_021_name_conflict_hvm(self): + with self.assertRaises(QubesException): + self.vm2 = self.qc.add_new_vm('QubesHVm', + name=self.vmname, template=self.qc.get_default_template()) + self.vm2.create_on_disk(verbose=False) + + def test_022_name_conflict_net(self): + with self.assertRaises(QubesException): + self.vm2 = self.qc.add_new_vm('QubesNetVm', + name=self.vmname, template=self.qc.get_default_template()) + self.vm2.create_on_disk(verbose=False) + + def test_030_rename_conflict_app(self): + vm2name = self.make_vm_name('newname') + + self.vm2 = self.qc.add_new_vm('QubesAppVm', + name=vm2name, template=self.qc.get_default_template()) + self.vm2.create_on_disk(verbose=False) + + with self.assertRaises(QubesException): + self.vm2.set_name(self.vmname) + + def test_031_rename_conflict_net(self): + vm3name = self.make_vm_name('newname') + + self.vm3 = self.qc.add_new_vm('QubesNetVm', + name=vm3name, template=self.qc.get_default_template()) + self.vm3.create_on_disk(verbose=False) + + with self.assertRaises(QubesException): + self.vm3.set_name(self.vmname) + + class TC_02_QvmPrefs(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): def setup_appvm(self): self.testvm = self.qc.add_new_vm( From 9fa090b422f40a29fa4e9a18d7fb2971e4a3b1d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 8 Feb 2016 04:38:26 +0100 Subject: [PATCH 09/54] core: prevent VM renaming over already existing VM Fixes QubesOS/qubes-issues#1723 --- core-modules/000QubesVm.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py index b714f544..0a777c32 100644 --- a/core-modules/000QubesVm.py +++ b/core-modules/000QubesVm.py @@ -591,6 +591,10 @@ class QubesVm(object): if self.installed_by_rpm: raise QubesException("Cannot rename VM installed by RPM -- first clone VM and then use yum to remove package.") + assert self._collection is not None + if self._collection.get_vm_by_name(name): + raise QubesException("VM with this name already exists") + self.pre_rename(name) try: self.libvirt_domain.undefine() From 6378e2bd20bbef0ad6d8d570ed79ccef1b9eb06d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 8 Feb 2016 04:50:05 +0100 Subject: [PATCH 10/54] qvm-run: allow --localcmd with --pass-io, even when --all is given With --localcmd, stdin/out are connected to that local process (instead of a terminal), so it doesn't conflict with --all. QubesOS/qubes-issues#1728 --- qvm-tools/qvm-run | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/qvm-tools/qvm-run b/qvm-tools/qvm-run index f404a6c2..dab8868f 100755 --- a/qvm-tools/qvm-run +++ b/qvm-tools/qvm-run @@ -129,14 +129,14 @@ def main(): (options, args) = parser.parse_args () - if options.passio and options.run_on_all_running: + if (options.passio and not options.localcmd) and options.run_on_all_running: parser.error ("Options --all and --pass-io cannot be used together") if options.passio: options.verbose = False if options.color_output is None: - if os.isatty(sys.stdout.fileno()): + if os.isatty(sys.stdout.fileno()) and not options.localcmd: options.color_output = 31 elif options.color_output is False: options.color_output = None @@ -180,8 +180,6 @@ def main(): continue if (options.unpause and vm.is_paused()) or (not options.unpause and vm.is_running()): vms_list.append (vm) - # disable options incompatible with --all - options.passio = False else: vm = qvm_collection.get_vm_by_name(vmname) if vm is None: From 0e273276d325ebe0a54be1c0d53cf6bfc9d92b16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 8 Feb 2016 05:01:50 +0100 Subject: [PATCH 11/54] qvm-run: warn if --localcmd used without --pass-io --- qvm-tools/qvm-run | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/qvm-tools/qvm-run b/qvm-tools/qvm-run index dab8868f..83223fe2 100755 --- a/qvm-tools/qvm-run +++ b/qvm-tools/qvm-run @@ -132,6 +132,10 @@ def main(): if (options.passio and not options.localcmd) and options.run_on_all_running: parser.error ("Options --all and --pass-io cannot be used together") + if options.localcmd and not options.passio: + print >> sys.stderr, "WARNING: option --localcmd have no effect " \ + "without --pass-io" + if options.passio: options.verbose = False From ae848d536958b2064ec96f96ad96540162cb4683 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 8 Feb 2016 05:02:30 +0100 Subject: [PATCH 12/54] version 3.1.12 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index efd03d13..b48ce58f 100644 --- a/version +++ b/version @@ -1 +1 @@ -3.1.11 +3.1.12 From 2fcd3c6832a83b2965aefbebf89015578224bdd9 Mon Sep 17 00:00:00 2001 From: Zrubi Date: Wed, 10 Feb 2016 14:55:11 +0100 Subject: [PATCH 13/54] new --force option for qvm-sync-clock to be able to bypass time sync errors --- qvm-tools/qvm-sync-clock | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/qvm-tools/qvm-sync-clock b/qvm-tools/qvm-sync-clock index 36e4cdfb..61740b51 100755 --- a/qvm-tools/qvm-sync-clock +++ b/qvm-tools/qvm-sync-clock @@ -22,6 +22,7 @@ # import fcntl +from optparse import OptionParser from qubes.qubes import QubesVmCollection import os.path import os @@ -41,9 +42,11 @@ def get_netvm_of_vm(vm): return netvm def main(): - verbose = False - if len(sys.argv) > 1 and sys.argv[1] in [ '--verbose', '-v' ]: - verbose = True + parser = OptionParser() + parser.add_option ("-v", "--verbose", action="store_true", dest="verbose", default=False) + parser.add_option ("-f", "--force", action="store_true", dest="force", default=False) + + (options, args) = parser.parse_args () lockfile_name = "/var/run/qubes/qvm-sync-clock.lock" if os.path.exists(lockfile_name): @@ -74,23 +77,26 @@ def main(): sys.exit(1) net_vm = get_netvm_of_vm(clock_vm) - if verbose: + if options.verbose: print >> sys.stderr, '--> Waiting for network for ClockVM.' # Ignore retcode, try even if nm-online failed - user can setup network manually # on-online has timeout 30sec by default - net_vm.run('nm-online -x', verbose=verbose, gui=False, wait=True, + net_vm.run('nm-online -x', verbose=options.verbose, gui=False, wait=True, ignore_stderr=True) # Sync clock if clock_vm.run('QUBESRPC qubes.SyncNtpClock dom0', user="root", - verbose=verbose, gui=False, wait=True, ignore_stderr=True) \ + verbose=options.verbose, gui=False, wait=True, ignore_stderr=True) \ != 0: - print >> sys.stderr, 'Time sync failed, aborting!' - sys.exit(1) + if options.force: + print >> sys.stderr, 'Time sync failed! - Sync forced' + else: + print >> sys.stderr, 'Time sync failed! - Exiting' + sys.exit(1) # Use the date format based on RFC2822 to avoid localisation issues - p = clock_vm.run('date -u -Iseconds', verbose=verbose, + p = clock_vm.run('date -u -Iseconds', verbose=options.verbose, gui=False, passio_popen=True, ignore_stderr=True) date_out = p.stdout.read(100) date_out = date_out.strip() @@ -99,18 +105,18 @@ def main(): sys.exit(1) # Sync dom0 time - if verbose: + if options.verbose: print >> sys.stderr, '--> Syncing dom0 clock.' subprocess.check_call(['sudo', 'date', '-u', '-Iseconds', '-s', date_out], - stdout=None if verbose else open(os.devnull, 'w')) + stdout=None if options.verbose else open(os.devnull, 'w')) subprocess.check_call(['sudo', 'hwclock', '--systohc'], - stdout=None if verbose else open(os.devnull, 'w')) + stdout=None if options.verbose else open(os.devnull, 'w')) # Sync other VMs clock for vm in qvm_collection.values(): if vm.is_running() and vm.qid != 0 and vm.qid != clock_vm.qid: - if verbose: + if options.verbose: print >> sys.stderr, '--> Syncing \'%s\' clock.' % vm.name try: vm.run_service("qubes.SetDateTime", user="root", From 9bd651faaa0408e0c139cddfcf63988eb9377241 Mon Sep 17 00:00:00 2001 From: Jon Griffiths Date: Thu, 18 Feb 2016 16:01:08 +1300 Subject: [PATCH 14/54] qvm-prefs: --get option was misspelled --gry --- qvm-tools/qvm-prefs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qvm-tools/qvm-prefs b/qvm-tools/qvm-prefs index dea3aa4f..7e2225a5 100755 --- a/qvm-tools/qvm-prefs +++ b/qvm-tools/qvm-prefs @@ -536,7 +536,7 @@ def main(): default=False) parser.add_option("-s", "--set", action="store_true", dest="do_set", default=False) - parser.add_option ("-g", "--gry", action="store_true", dest="do_get", + parser.add_option ("-g", "--get", action="store_true", dest="do_get", default=False) parser.add_option("--force-root", action="store_true", dest="force_root", default=False, From 5546d679c0cd8e3d7a9ba94678e8b22be8525510 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 21 Feb 2016 12:41:13 +0100 Subject: [PATCH 15/54] dispvm: use try/finally to make sure that qubes.xml is unlocked Even in case of some exception (in which case theoretically it should be unlocked at qfile-daemon-dvm exit, but the script may wait for something). QubesOS/qubes-issues#1636 --- dispvm/qfile-daemon-dvm | 121 ++++++++++++++++++++-------------------- 1 file changed, 60 insertions(+), 61 deletions(-) diff --git a/dispvm/qfile-daemon-dvm b/dispvm/qfile-daemon-dvm index c582f79b..423500b6 100755 --- a/dispvm/qfile-daemon-dvm +++ b/dispvm/qfile-daemon-dvm @@ -51,68 +51,67 @@ class QfileDaemonDvm: qvm_collection = QubesVmCollection() qvm_collection.lock_db_for_writing() - - tar_process = subprocess.Popen( - ['bsdtar', '-C', current_savefile_vmdir, - '-xSUf', os.path.join(current_savefile_vmdir, 'saved-cows.tar')]) - - qvm_collection.load() - print >>sys.stderr, "time=%s, collection loaded" % (str(time.time())) - - vm = qvm_collection.get_vm_by_name(self.name) - if vm is None: - sys.stderr.write('Domain ' + self.name + ' does not exist ?') - qvm_collection.unlock_db() - return None - label = vm.label - if len(sys.argv) > 4 and len(sys.argv[4]) > 0: - assert sys.argv[4] in QubesDispVmLabels.keys(), "Invalid label" - label = QubesDispVmLabels[sys.argv[4]] - disp_templ = self.get_disp_templ() - vm_disptempl = qvm_collection.get_vm_by_name(disp_templ) - if vm_disptempl is None: - sys.stderr.write('Domain ' + disp_templ + ' does not exist ?') - qvm_collection.unlock_db() - return None - dispvm = qvm_collection.add_new_vm('QubesDisposableVm', - disp_template=vm_disptempl, - label=label) - print >>sys.stderr, "time=%s, VM created" % (str(time.time())) - # By default inherit firewall rules from calling VM - disp_firewall_conf = '/var/run/qubes/%s-firewall.xml' % dispvm.name - dispvm.firewall_conf = disp_firewall_conf - if os.path.exists(vm.firewall_conf): - shutil.copy(vm.firewall_conf, disp_firewall_conf) - elif vm.qid == 0 and os.path.exists(vm_disptempl.firewall_conf): - # for DispVM called from dom0, copy use rules from DispVM template - shutil.copy(vm_disptempl.firewall_conf, disp_firewall_conf) - if len(sys.argv) > 5 and len(sys.argv[5]) > 0: - assert os.path.exists(sys.argv[5]), "Invalid firewall.conf location" - dispvm.firewall_conf = sys.argv[5] - if vm.qid != 0: - dispvm.uses_default_netvm = False - # netvm can be changed before restore, - # but cannot be enabled/disabled - if (dispvm.netvm is None) == (vm.dispvm_netvm is None): - dispvm.netvm = vm.dispvm_netvm - # Wait for tar to finish - if tar_process.wait() != 0: - sys.stderr.write('Failed to unpack saved-cows.tar') - qvm_collection.unlock_db() - return None - print >>sys.stderr, "time=%s, VM starting" % (str(time.time())) try: - dispvm.start() - except (MemoryError, QubesException) as e: - tray_notify_error(str(e)) - raise - if vm.qid != 0: - # if need to enable/disable netvm, do it while DispVM is alive - if (dispvm.netvm is None) != (vm.dispvm_netvm is None): - dispvm.netvm = vm.dispvm_netvm - print >>sys.stderr, "time=%s, VM started" % (str(time.time())) - qvm_collection.save() - qvm_collection.unlock_db() + + tar_process = subprocess.Popen( + ['bsdtar', '-C', current_savefile_vmdir, + '-xSUf', os.path.join(current_savefile_vmdir, 'saved-cows.tar')]) + + qvm_collection.load() + print >>sys.stderr, "time=%s, collection loaded" % (str(time.time())) + + vm = qvm_collection.get_vm_by_name(self.name) + if vm is None: + sys.stderr.write('Domain ' + self.name + ' does not exist ?') + return None + label = vm.label + if len(sys.argv) > 4 and len(sys.argv[4]) > 0: + assert sys.argv[4] in QubesDispVmLabels.keys(), "Invalid label" + label = QubesDispVmLabels[sys.argv[4]] + disp_templ = self.get_disp_templ() + vm_disptempl = qvm_collection.get_vm_by_name(disp_templ) + if vm_disptempl is None: + sys.stderr.write('Domain ' + disp_templ + ' does not exist ?') + return None + dispvm = qvm_collection.add_new_vm('QubesDisposableVm', + disp_template=vm_disptempl, + label=label) + print >>sys.stderr, "time=%s, VM created" % (str(time.time())) + # By default inherit firewall rules from calling VM + disp_firewall_conf = '/var/run/qubes/%s-firewall.xml' % dispvm.name + dispvm.firewall_conf = disp_firewall_conf + if os.path.exists(vm.firewall_conf): + shutil.copy(vm.firewall_conf, disp_firewall_conf) + elif vm.qid == 0 and os.path.exists(vm_disptempl.firewall_conf): + # for DispVM called from dom0, copy use rules from DispVM template + shutil.copy(vm_disptempl.firewall_conf, disp_firewall_conf) + if len(sys.argv) > 5 and len(sys.argv[5]) > 0: + assert os.path.exists(sys.argv[5]), "Invalid firewall.conf location" + dispvm.firewall_conf = sys.argv[5] + if vm.qid != 0: + dispvm.uses_default_netvm = False + # netvm can be changed before restore, + # but cannot be enabled/disabled + if (dispvm.netvm is None) == (vm.dispvm_netvm is None): + dispvm.netvm = vm.dispvm_netvm + # Wait for tar to finish + if tar_process.wait() != 0: + sys.stderr.write('Failed to unpack saved-cows.tar') + return None + print >>sys.stderr, "time=%s, VM starting" % (str(time.time())) + try: + dispvm.start() + except (MemoryError, QubesException) as e: + tray_notify_error(str(e)) + raise + if vm.qid != 0: + # if need to enable/disable netvm, do it while DispVM is alive + if (dispvm.netvm is None) != (vm.dispvm_netvm is None): + dispvm.netvm = vm.dispvm_netvm + print >>sys.stderr, "time=%s, VM started" % (str(time.time())) + qvm_collection.save() + finally: + qvm_collection.unlock_db() # Reload firewall rules print >>sys.stderr, "time=%s, reloading firewall" % (str(time.time())) for vm in qvm_collection.values(): From 0121cb2fd4e75d809519afbe8f60a1f31121821e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 22 Feb 2016 17:56:43 +0100 Subject: [PATCH 16/54] qmemman: make memory request handling more defensive If getting memory for new VM fails for any reason, make sure that global lock will be released. Otherwise qmemman will stop functioning at all. QubesOS/qubes-issues#1636 --- qmemman/qmemman_server.py | 60 +++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/qmemman/qmemman_server.py b/qmemman/qmemman_server.py index 7c295243..ff8ae69c 100755 --- a/qmemman/qmemman_server.py +++ b/qmemman/qmemman_server.py @@ -175,37 +175,41 @@ class QMemmanReqHandler(SocketServer.BaseRequestHandler): self.log = logging.getLogger('qmemman.daemon.reqhandler') got_lock = False - # self.request is the TCP socket connected to the client - while True: - self.data = self.request.recv(1024).strip() - self.log.debug('data={!r}'.format(self.data)) - if len(self.data) == 0: - self.log.info('EOF') + try: + # self.request is the TCP socket connected to the client + while True: + self.data = self.request.recv(1024).strip() + self.log.debug('data={!r}'.format(self.data)) + if len(self.data) == 0: + self.log.info('EOF') + if got_lock: + global force_refresh_domain_list + force_refresh_domain_list = True + return + + # XXX something is wrong here: return without release? if got_lock: - global force_refresh_domain_list - force_refresh_domain_list = True - global_lock.release() - self.log.debug('global_lock released') - return + self.log.warning('Second request over qmemman.sock?') + return - # XXX something is wrong here: return without release? + self.log.debug('acquiring global_lock') + global_lock.acquire() + self.log.debug('global_lock acquired') + + got_lock = True + if system_state.do_balloon(int(self.data)): + resp = "OK\n" + else: + resp = "FAIL\n" + self.log.debug('resp={!r}'.format(resp)) + self.request.send(resp) + except BaseException as e: + self.log.exception( + "exception while handling request: {!r}".format(e)) + finally: if got_lock: - self.log.warning('Second request over qmemman.sock?') - return - - self.log.debug('acquiring global_lock') - global_lock.acquire() - self.log.debug('global_lock acquired') - - got_lock = True - if system_state.do_balloon(int(self.data)): - resp = "OK\n" - else: - resp = "FAIL\n" - self.log.debug('resp={!r}'.format(resp)) - self.request.send(resp) - - # XXX no release of lock? + global_lock.release() + self.log.debug('global_lock released') def start_server(server): From 3ece17a8cf61d2df663e7c2a855602505bab043d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 22 Feb 2016 20:56:38 +0100 Subject: [PATCH 17/54] qvm-sync-clock: sync dom0 clock only when NTP sync succeeded, even with --force Otherwise dom0 clock (initially almost ok) may be adjusted to totally invalid timestamp of ClockVM (for example just after resume from S3 sleep). --- qvm-tools/qvm-sync-clock | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/qvm-tools/qvm-sync-clock b/qvm-tools/qvm-sync-clock index 61740b51..96f330d1 100755 --- a/qvm-tools/qvm-sync-clock +++ b/qvm-tools/qvm-sync-clock @@ -94,24 +94,24 @@ def main(): else: print >> sys.stderr, 'Time sync failed! - Exiting' sys.exit(1) + else: + # Use the date format based on RFC2822 to avoid localisation issues + p = clock_vm.run('date -u -Iseconds', verbose=options.verbose, + gui=False, passio_popen=True, ignore_stderr=True) + date_out = p.stdout.read(100) + date_out = date_out.strip() + if not re.match(r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+0000$', date_out): + print >> sys.stderr, 'Invalid date output, aborting!' + sys.exit(1) - # Use the date format based on RFC2822 to avoid localisation issues - p = clock_vm.run('date -u -Iseconds', verbose=options.verbose, - gui=False, passio_popen=True, ignore_stderr=True) - date_out = p.stdout.read(100) - date_out = date_out.strip() - if not re.match(r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+0000$', date_out): - print >> sys.stderr, 'Invalid date output, aborting!' - sys.exit(1) + # Sync dom0 time + if options.verbose: + print >> sys.stderr, '--> Syncing dom0 clock.' - # Sync dom0 time - if options.verbose: - print >> sys.stderr, '--> Syncing dom0 clock.' - - subprocess.check_call(['sudo', 'date', '-u', '-Iseconds', '-s', date_out], - stdout=None if options.verbose else open(os.devnull, 'w')) - subprocess.check_call(['sudo', 'hwclock', '--systohc'], - stdout=None if options.verbose else open(os.devnull, 'w')) + subprocess.check_call(['sudo', 'date', '-u', '-Iseconds', '-s', date_out], + stdout=None if options.verbose else open(os.devnull, 'w')) + subprocess.check_call(['sudo', 'hwclock', '--systohc'], + stdout=None if options.verbose else open(os.devnull, 'w')) # Sync other VMs clock for vm in qvm_collection.values(): From 0fb69d39cac330988f9579c323718deba6543f70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Tue, 23 Feb 2016 16:20:18 +0100 Subject: [PATCH 18/54] tests: qvm-sync-clock is asynchronouse, take it into account --- tests/vm_qrexec_gui.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py index ae5299b7..0b4ac402 100644 --- a/tests/vm_qrexec_gui.py +++ b/tests/vm_qrexec_gui.py @@ -690,6 +690,10 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin): self.assertEquals(retcode, 0, "qvm-sync-clock failed with code {}". format(retcode)) + # qvm-sync-clock is asynchronous - it spawns qubes.SetDateTime + # service, send it timestamp value and exists without waiting for + # actual time set + time.sleep(1) (vm_time, _) = self.testvm1.run("date -u +%s", passio_popen=True).communicate() self.assertAlmostEquals(int(vm_time), int(start_time), delta=30) From d3f83876eb8db5110bff3ed815a26182eb41eb26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Tue, 23 Feb 2016 16:22:02 +0100 Subject: [PATCH 19/54] rpm: require new enough qubes-core-dom0-linux package For qvm-sync-clock --force option. --- rpm_spec/core-dom0.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec index a9d30def..66737895 100644 --- a/rpm_spec/core-dom0.spec +++ b/rpm_spec/core-dom0.spec @@ -47,7 +47,7 @@ Requires(post): systemd-units Requires(preun): systemd-units Requires(postun): systemd-units Requires: python, pciutils, python-inotify, python-daemon -Requires: qubes-core-dom0-linux >= 2.0.24 +Requires: qubes-core-dom0-linux >= 3.1.8 Requires: qubes-db-dom0 Requires: python-lxml Requires: python-psutil From fa66fec684f823bfefe5355813342426ce8a20e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Tue, 23 Feb 2016 16:28:14 +0100 Subject: [PATCH 20/54] version 3.1.13 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index b48ce58f..55f20a1a 100644 --- a/version +++ b/version @@ -1 +1 @@ -3.1.12 +3.1.13 From 4ae52b037b992967cc8958098456565ea1cd8b2d Mon Sep 17 00:00:00 2001 From: Zrubi Date: Thu, 25 Feb 2016 15:05:06 +0100 Subject: [PATCH 21/54] filename sanitization --- qvm-tools/qubes-hcl-report | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qvm-tools/qubes-hcl-report b/qvm-tools/qubes-hcl-report index 047a79d0..e26d9b86 100755 --- a/qvm-tools/qubes-hcl-report +++ b/qvm-tools/qubes-hcl-report @@ -18,7 +18,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -VERSION=2.4 +VERSION=2.5 COPY2VM="dom0" SUPPORT_FILES=0 @@ -103,7 +103,7 @@ XL_VTX=`cat $TEMP_DIR/xl-info |grep xen_caps | grep hvm` XL_VTD=`cat $TEMP_DIR/xl-info |grep virt_caps |grep hvm_directio` PCRS=`find /sys/devices/ -name pcrs` -FILENAME="Qubes-HCL-${BRAND// /_}-${PRODUCT// /_}-$DATE" +FILENAME="Qubes-HCL-${BRAND//+([^[:alnum:]])/_}-${PRODUCT//+([^[:alnum:]])/_}-$DATE" if [[ $XL_VTX ]] then From 26443e1dd210c7a347728ada26e40793fec4be3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 26 Feb 2016 10:59:20 +0100 Subject: [PATCH 22/54] tests: add function to create minimal OS in HVM SystemTestsMixin.prepare_hvm_system_linux creates minimal Linux installation necessary to launch simple shell script. It installs: - grub2 - kernel from dom0 (the same as the running one) - dracut based initramfs, with provided script set as pre-pivot hook Done in preparation for QubesOS/qubes-issues#1659 test --- tests/__init__.py | 73 +++++++++++++++++++++++++++++++++++++++++++++++ tests/hardware.py | 1 + 2 files changed, 74 insertions(+) create mode 100644 tests/hardware.py diff --git a/tests/__init__.py b/tests/__init__.py index cc06ad6c..34f9363b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -22,12 +22,14 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # +from distutils import spawn import multiprocessing import logging import os import shutil import subprocess +import tempfile import unittest import lxml.etree @@ -444,6 +446,77 @@ class SystemTestsMixin(object): timeout -= 1 self.fail("Timeout while waiting for VM {} shutdown".format(vm.name)) + def prepare_hvm_system_linux(self, vm, init_script, extra_files=None): + if not os.path.exists('/usr/lib/grub/i386-pc'): + self.skipTest('grub2 not installed') + if not spawn.find_executable('grub2-install'): + self.skipTest('grub2-tools not installed') + if not spawn.find_executable('dracut'): + self.skipTest('dracut not installed') + # create a single partition + p = subprocess.Popen(['sfdisk', '-q', '-L', vm.storage.root_img], + stdin=subprocess.PIPE, + stdout=open(os.devnull, 'w'), + stderr=subprocess.STDOUT) + p.communicate('2048,\n') + assert p.returncode == 0, 'sfdisk failed' + # TODO: check if root_img is really file, not already block device + p = subprocess.Popen(['sudo', 'losetup', '-f', '-P', '--show', + vm.storage.root_img], stdout=subprocess.PIPE) + (loopdev, _) = p.communicate() + loopdev = loopdev.strip() + looppart = loopdev + 'p1' + assert p.returncode == 0, 'losetup failed' + subprocess.check_call(['sudo', 'mkfs.ext2', '-q', '-F', looppart]) + mountpoint = tempfile.mkdtemp() + subprocess.check_call(['sudo', 'mount', looppart, mountpoint]) + try: + subprocess.check_call(['sudo', 'grub2-install', + '--target', 'i386-pc', + '--modules', 'part_msdos ext2', + '--boot-directory', mountpoint, loopdev], + stderr=open(os.devnull, 'w') + ) + grub_cfg = '{}/grub2/grub.cfg'.format(mountpoint) + subprocess.check_call( + ['sudo', 'chown', '-R', os.getlogin(), mountpoint]) + with open(grub_cfg, 'w') as f: + f.write( + "set timeout=1\n" + "menuentry 'Default' {\n" + " linux /vmlinuz root=/dev/xvda1 " + "rd.driver.blacklist=bochs_drm " + "rd.driver.blacklist=uhci_hcd\n" + " initrd /initrd\n" + "}" + ) + p = subprocess.Popen(['uname', '-r'], stdout=subprocess.PIPE) + (kernel_version, _) = p.communicate() + kernel_version = kernel_version.strip() + kernel = '/boot/vmlinuz-{}'.format(kernel_version) + shutil.copy(kernel, os.path.join(mountpoint, 'vmlinuz')) + init_path = os.path.join(mountpoint, 'init') + with open(init_path, 'w') as f: + f.write(init_script) + os.chmod(init_path, 0755) + dracut_args = [ + '--kver', kernel_version, + '--include', init_path, + '/usr/lib/dracut/hooks/pre-pivot/initscript.sh', + '--no-hostonly', '--nolvmconf', '--nomdadmconf', + ] + if extra_files: + dracut_args += ['--install', ' '.join(extra_files)] + subprocess.check_call( + ['dracut'] + dracut_args + [os.path.join(mountpoint, + 'initrd')], + stderr=open(os.devnull, 'w') + ) + finally: + subprocess.check_call(['sudo', 'umount', mountpoint]) + shutil.rmtree(mountpoint) + subprocess.check_call(['sudo', 'losetup', '-d', loopdev]) + class BackupTestsMixin(SystemTestsMixin): def setUp(self): super(BackupTestsMixin, self).setUp() diff --git a/tests/hardware.py b/tests/hardware.py new file mode 100644 index 00000000..e6b28fd8 --- /dev/null +++ b/tests/hardware.py @@ -0,0 +1 @@ +__author__ = 'user' From 3bef37f881d6b85891a3f3e902678ef88854beb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 26 Feb 2016 11:02:59 +0100 Subject: [PATCH 23/54] tests: PCI passthrough to HVM A simple test which checks if the device is visible there at all. Device set with QUBES_TEST_PCIDEV env variable is used - it should be some unimportant device which can be freely detached from dom0. QubesOS/qubes-issues#1659 --- tests/Makefile | 2 ++ tests/__init__.py | 1 + tests/hardware.py | 76 ++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/tests/Makefile b/tests/Makefile index 2f503a15..696ef568 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -31,3 +31,5 @@ endif cp storage.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) cp storage_xen.py $(DESTDIR)$(PYTHON_TESTSPATH) cp storage_xen.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) + cp hardware.py $(DESTDIR)$(PYTHON_TESTSPATH) + cp hardware.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) diff --git a/tests/__init__.py b/tests/__init__.py index 34f9363b..b8b6fdf8 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -696,6 +696,7 @@ def load_tests(loader, tests, pattern): 'qubes.tests.regressions', 'qubes.tests.storage', 'qubes.tests.storage_xen', + 'qubes.tests.hardware', ): tests.addTests(loader.loadTestsFromName(modname)) diff --git a/tests/hardware.py b/tests/hardware.py index e6b28fd8..8ae282e5 100644 --- a/tests/hardware.py +++ b/tests/hardware.py @@ -1 +1,75 @@ -__author__ = 'user' +#!/usr/bin/python2 +# -*- coding: utf-8 -*- +# +# The Qubes OS Project, http://www.qubes-os.org +# +# Copyright (C) 2016 Marek Marczykowski-Górecki +# +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# +import os + +import qubes.tests +import time +import subprocess +from unittest import expectedFailure + + +class TC_00_HVM(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): + def setUp(self): + super(TC_00_HVM, self).setUp() + self.vm = self.qc.add_new_vm("QubesHVm", + name=self.make_vm_name('vm1')) + self.vm.create_on_disk(verbose=False) + + @expectedFailure + def test_000_pci_passthrough_presence(self): + pcidev = os.environ.get('QUBES_TEST_PCIDEV', None) + if pcidev is None: + self.skipTest('Specify PCI device with QUBES_TEST_PCIDEV ' + 'environment variable') + self.vm.pcidevs = [pcidev] + self.vm.pci_strictreset = False + self.qc.save() + self.qc.unlock_db() + + init_script = ( + "#!/bin/sh\n" + "set -e\n" + "lspci -n > /dev/xvdb\n" + "poweroff\n" + ) + + self.prepare_hvm_system_linux(self.vm, init_script, + ['/usr/sbin/lspci']) + self.vm.start() + timeout = 60 + while timeout > 0: + if not self.vm.is_running(): + break + time.sleep(1) + timeout -= 1 + if self.vm.is_running(): + self.fail("Timeout while waiting for VM shutdown") + + with open(self.vm.storage.private_img, 'r') as f: + lspci_vm = f.read(512).strip('\0') + p = subprocess.Popen(['lspci', '-ns', pcidev], stdout=subprocess.PIPE) + (lspci_host, _) = p.communicate() + # strip BDF, as it is different in VM + pcidev_desc = ' '.join(lspci_host.strip().split(' ')[1:]) + self.assertIn(pcidev_desc, lspci_vm) From 4dd87f2683de435e9436793cd8bb3cd10ca00db0 Mon Sep 17 00:00:00 2001 From: Jon Griffiths Date: Thu, 3 Mar 2016 13:42:13 +1300 Subject: [PATCH 24/54] qvm-create: Don't allow the user to manually create dispNN names --- qvm-tools/qvm-create | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/qvm-tools/qvm-create b/qvm-tools/qvm-create index 6ae002e5..18e45c87 100755 --- a/qvm-tools/qvm-create +++ b/qvm-tools/qvm-create @@ -130,6 +130,10 @@ def main(): if options.offline_mode: vmm.offline_mode = True + if re.match('^disp\d+$', vmname): + print >> sys.stderr, 'The name "{0}" is reserved for internal use.'.format(vmname) + exit(1) + qvm_collection = QubesVmCollection() qvm_collection.lock_db_for_writing() qvm_collection.load() From 280a0743c20f9c466133e94cef0c84483506b772 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 27 Feb 2016 20:11:49 +0100 Subject: [PATCH 25/54] core: typo fix in error handling cont s/VIR_INTERNAL_ERROR/VIR_ERR_INTERNAL_ERROR/ Related to QubesOS/qubes-issues#1537 Fixes QubesOS/qubes-issues#1804 --- core-modules/000QubesVm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py index 0a777c32..f1c44024 100644 --- a/core-modules/000QubesVm.py +++ b/core-modules/000QubesVm.py @@ -807,7 +807,7 @@ class QubesVm(object): if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN: return 0 # libxl_domain_info failed - domain no longer exists - elif e.get_error_code() == libvirt.VIR_INTERNAL_ERROR: + elif e.get_error_code() == libvirt.VIR_ERR_INTERNAL_ERROR: return 0 elif e.get_error_code() is None: # unknown... return 0 @@ -932,7 +932,7 @@ class QubesVm(object): if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN: return False # libxl_domain_info failed - domain no longer exists - elif e.get_error_code() == libvirt.VIR_INTERNAL_ERROR: + elif e.get_error_code() == libvirt.VIR_ERR_INTERNAL_ERROR: return False elif e.get_error_code() is None: # unknown... return False @@ -951,7 +951,7 @@ class QubesVm(object): if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN: return False # libxl_domain_info failed - domain no longer exists - elif e.get_error_code() == libvirt.VIR_INTERNAL_ERROR: + elif e.get_error_code() == libvirt.VIR_ERR_INTERNAL_ERROR: return False elif e.get_error_code() is None: # unknown... return False From 4420b320e7c447feddc870396874a46fba5dc69b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 28 Feb 2016 03:43:04 +0100 Subject: [PATCH 26/54] tests: add expectedFailureIfTemplate decorator Some tests are expected to fail only on some templates (some feature not available in older distribution, some feature not yet ported to another). --- tests/__init__.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/__init__.py b/tests/__init__.py index b8b6fdf8..f19ca22c 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -23,6 +23,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # from distutils import spawn +import functools import multiprocessing import logging @@ -31,6 +32,7 @@ import shutil import subprocess import tempfile import unittest +from unittest.case import _ExpectedFailure, _UnexpectedSuccess import lxml.etree import sys @@ -88,6 +90,32 @@ def skipUnlessGit(test_item): return unittest.skipUnless(in_git, 'outside git tree')(test_item) +def expectedFailureIfTemplate(templates): + """ + Decorator for marking specific test as expected to fail only for some + templates. Template name is compared as substring, so 'whonix' will + handle both 'whonix-ws' and 'whonix-gw'. + templates can be either a single string, or an iterable + """ + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + template = args[0].template + if isinstance(templates, basestring): + should_expect_fail = template in templates + else: + should_expect_fail = any([template in x for x in templates]) + if should_expect_fail: + try: + func(*args, **kwargs) + except Exception: + raise _ExpectedFailure(sys.exc_info()) + raise _UnexpectedSuccess + else: + # Call directly: + func(*args, **kwargs) + return wrapper + return decorator class _AssertNotRaisesContext(object): """A context manager used to implement TestCase.assertNotRaises methods. From 5c877e9850a4cbae1e6e960f8b1b04b953d0c0f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 28 Feb 2016 03:44:33 +0100 Subject: [PATCH 27/54] tests: mark NetworkManager in ProxyVM fail for debian-7 NM there expect some different options in configuration, and since it's old stable, we won't implement it. --- tests/network.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/network.py b/tests/network.py index ce93cb9f..838d6cf1 100644 --- a/tests/network.py +++ b/tests/network.py @@ -121,6 +121,7 @@ class VmNetworkingMixin(qubes.tests.SystemTestsMixin): "Ping by IP from AppVM failed") + @qubes.tests.expectedFailureIfTemplate('debian-7') @unittest.skipUnless(spawn.find_executable('xdotool'), "xdotool not installed") def test_020_simple_proxyvm_nm(self): From fae73e637356483447b5f7e8c59ddd3cb76395ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 29 Feb 2016 13:22:18 +0100 Subject: [PATCH 28/54] qvm-sync-clock: clarify message on NTP call failed and --force used --- qvm-tools/qvm-sync-clock | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qvm-tools/qvm-sync-clock b/qvm-tools/qvm-sync-clock index 96f330d1..ef628551 100755 --- a/qvm-tools/qvm-sync-clock +++ b/qvm-tools/qvm-sync-clock @@ -90,7 +90,8 @@ def main(): verbose=options.verbose, gui=False, wait=True, ignore_stderr=True) \ != 0: if options.force: - print >> sys.stderr, 'Time sync failed! - Sync forced' + print >> sys.stderr, 'Time sync failed! - Syncing with dom0 ' \ + 'anyway as requested' else: print >> sys.stderr, 'Time sync failed! - Exiting' sys.exit(1) From 8497471a7214724eab6e5a86af65547a627a7443 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 29 Feb 2016 13:22:50 +0100 Subject: [PATCH 29/54] tests: fix preparation of "small" backup storage in Debian 7 losetup Debian 7 doesn't deal with /dev/loop-control, so needs existing /dev/loop* devices. Just /dev/loop0 is enough. --- tests/backup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/backup.py b/tests/backup.py index 72463c91..69b5c597 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -184,6 +184,8 @@ class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin): vms = self.create_backup_vms() self.backupvm.start() retcode = self.backupvm.run( + # Debian 7 has too old losetup to handle loop-control device + "mknod /dev/loop0 b 7 0;" "truncate -s 50M /home/user/backup.img && " "mkfs.ext4 -F /home/user/backup.img && " "mkdir /home/user/backup && " From 7ca89688bd1f7e9a731da7ffaa7b1bbd025f45ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 2 Mar 2016 14:10:50 +0100 Subject: [PATCH 30/54] Improve expectedFailureIfTemplate decorator --- tests/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index f19ca22c..e35a3096 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -32,7 +32,7 @@ import shutil import subprocess import tempfile import unittest -from unittest.case import _ExpectedFailure, _UnexpectedSuccess +import unittest.case import lxml.etree import sys @@ -99,21 +99,21 @@ def expectedFailureIfTemplate(templates): """ def decorator(func): @functools.wraps(func) - def wrapper(*args, **kwargs): - template = args[0].template + def wrapper(self, *args, **kwargs): + template = self.template if isinstance(templates, basestring): should_expect_fail = template in templates else: should_expect_fail = any([template in x for x in templates]) if should_expect_fail: try: - func(*args, **kwargs) + func(self, *args, **kwargs) except Exception: - raise _ExpectedFailure(sys.exc_info()) - raise _UnexpectedSuccess + raise unittest.case._ExpectedFailure(sys.exc_info()) + raise unittest.case._UnexpectedSuccess() else: # Call directly: - func(*args, **kwargs) + func(self, *args, **kwargs) return wrapper return decorator From 29c602c9ffd9c6a2020c17f54f4584bbed75f987 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 3 Mar 2016 00:15:09 +0100 Subject: [PATCH 31/54] tests: prefix internal functions with underscore Preparation for API for external tests. QubesOS/qubes-issues#1800 --- tests/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index e35a3096..07858aab 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -245,7 +245,7 @@ class SystemTestsMixin(object): self.conn = libvirt.open(qubes.qubes.defaults['libvirt_uri']) - self.remove_test_vms(self.qc, self.conn) + self._remove_test_vms(self.qc, self.conn) def tearDown(self): super(SystemTestsMixin, self).tearDown() @@ -257,12 +257,12 @@ class SystemTestsMixin(object): except qubes.qubes.QubesException: pass - self.kill_test_vms(self.qc) + self._kill_test_vms(self.qc) self.qc.lock_db_for_writing() self.qc.load() - self.remove_test_vms(self.qc, self.conn) + self._remove_test_vms(self.qc, self.conn) self.qc.save() self.qc.unlock_db() @@ -281,12 +281,12 @@ class SystemTestsMixin(object): conn = libvirt.open(qubes.qubes.defaults['libvirt_uri']) - cls.kill_test_vms(qc, prefix=CLSVMPREFIX) + cls._kill_test_vms(qc, prefix=CLSVMPREFIX) qc.lock_db_for_writing() qc.load() - cls.remove_test_vms(qc, conn, prefix=CLSVMPREFIX) + cls._remove_test_vms(qc, conn, prefix=CLSVMPREFIX) qc.save() qc.unlock_db() @@ -308,7 +308,7 @@ class SystemTestsMixin(object): self.qc.load() @staticmethod - def kill_test_vms(qc, prefix=VMPREFIX): + def _kill_test_vms(qc, prefix=VMPREFIX): # do not keep write lock while killing VMs, because that may cause a # deadlock with disk hotplug scripts (namely qvm-template-commit # called when shutting down TemplateVm) @@ -383,7 +383,7 @@ class SystemTestsMixin(object): self.save_and_reload_db() @classmethod - def remove_test_vms(cls, qc, conn, prefix=VMPREFIX): + def _remove_test_vms(cls, qc, conn, prefix=VMPREFIX): """Aggresively remove any domain that has name in testing namespace. .. warning:: From 3f66da04121fd7dd7b22a80e8276f03e7a8ef407 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 3 Mar 2016 01:07:23 +0100 Subject: [PATCH 32/54] tests: add a function for adding qrexec rules QubesOS/qubes-issues#1800 --- tests/__init__.py | 24 ++++++++++++++++++++++++ tests/extra.py | 1 + 2 files changed, 25 insertions(+) create mode 100644 tests/extra.py diff --git a/tests/__init__.py b/tests/__init__.py index 07858aab..c8a31a6b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -424,6 +424,30 @@ class SystemTestsMixin(object): for vmname in vmnames: cls._remove_vm_disk(vmname) + def qrexec_policy(self, service, source, destination, allow=True): + """ + Allow qrexec calls for duration of the test + :param service: service name + :param source: source VM name + :param destination: destination VM name + :return: + """ + + def add_remove_rule(add=True): + with open('/etc/qubes-rpc/policy/{}'.format(service), 'r+') as policy: + policy_rules = policy.readlines() + rule = "{} {} {}\n".format(source, destination, + 'allow' if allow else 'deny') + if add: + policy_rules.insert(0, rule) + else: + policy_rules.remove(rule) + policy.truncate(0) + policy.seek(0) + policy.write(''.join(policy_rules)) + add_remove_rule(add=True) + self.addCleanup(add_remove_rule, add=False) + def wait_for_window(self, title, timeout=30, show=True): """ Wait for a window with a given title. Depending on show parameter, diff --git a/tests/extra.py b/tests/extra.py new file mode 100644 index 00000000..e6b28fd8 --- /dev/null +++ b/tests/extra.py @@ -0,0 +1 @@ +__author__ = 'user' From 02c601b8307c1704a4a9e9b5332a73e1b0e13a12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 3 Mar 2016 01:09:16 +0100 Subject: [PATCH 33/54] tests: expose API for external tests Fixes QubesOS/qubes-issues#1800 --- tests/__init__.py | 83 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/tests/__init__.py b/tests/__init__.py index c8a31a6b..02d660e3 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -36,6 +36,7 @@ import unittest.case import lxml.etree import sys +import pkg_resources import qubes.backup import qubes.qubes @@ -490,6 +491,12 @@ class SystemTestsMixin(object): subprocess.check_call(command) def shutdown_and_wait(self, vm, timeout=60): + """ + + :param vm: VM object + :param timeout: timeout after which fail the test + :return: + """ vm.shutdown() while timeout > 0: if not vm.is_running(): @@ -733,6 +740,48 @@ class BackupTestsMixin(SystemTestsMixin): f.truncate(size) f.close() +class ExtraTestMixin(SystemTestsMixin): + + template = None + + def setUp(self): + super(ExtraTestMixin, self).setUp() + self.qc.unlock_db() + + def create_vms(self, names): + """ + Create AppVMs for the duration of the test. Will be automatically + removed after completing the test. + :param names: list of VM names to create (each of them will be + prefixed with some test specific string) + :return: list of created VM objects + """ + self.qc.lock_db_for_writing() + self.qc.load() + if self.template: + template = self.qc.get_vm_by_name(self.template) + else: + template = self.qc.get_default_template() + for vmname in names: + vm = self.qc.add_new_vm("QubesAppVm", + name=self.make_vm_name(vmname), + template=template) + vm.create_on_disk(verbose=False) + self.save_and_reload_db() + + # get objects after reload + vms = [] + for vmname in names: + vms.append(self.qc.get_vm_by_name(self.make_vm_name(vmname))) + return vms + + def enable_network(self): + """ + Enable access to the network. Must be called before creating VMs. + """ + # nothing to do in core2 + pass + def load_tests(loader, tests, pattern): # discard any tests from this module, because it hosts base classes @@ -752,6 +801,40 @@ def load_tests(loader, tests, pattern): ): tests.addTests(loader.loadTestsFromName(modname)) + for entry in pkg_resources.iter_entry_points('qubes.tests.extra'): + for test_case in entry(): + tests.addTests(loader.loadTestsFromTestCase( + type( + entry.name + '_' + test_case.__name__, + (test_case, ExtraTestMixin, QubesTestCase), + {} + ) + )) + + + try: + qc = qubes.qubes.QubesVmCollection() + qc.lock_db_for_reading() + qc.load() + qc.unlock_db() + templates = [vm.name for vm in qc.values() if + isinstance(vm, qubes.qubes.QubesTemplateVm)] + except OSError: + templates = [] + + for entry in pkg_resources.iter_entry_points( + 'qubes.tests.extra.for_template'): + for test_case in entry(): + for template in templates: + tests.addTests(loader.loadTestsFromTestCase( + type( + entry.name + '_' + test_case.__name__, + (test_case, ExtraTestMixin, + QubesTestCase), + {'template': template} + ) + )) + return tests From bc240bd742e28837aaccea4e2d1866abc23641a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 3 Mar 2016 11:48:21 +0100 Subject: [PATCH 34/54] tests: move external test loader to qubes.tests.extra module This way, tests will appear as from 'extra' module. Besides the move, some minor changes: - add missing self.qc.unlock_db() in create_vms() - suffix per-template tests with template name QubesOS/qubes-issues#1800 --- tests/Makefile | 2 + tests/__init__.py | 78 +------------------------------- tests/extra.py | 110 +++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 112 insertions(+), 78 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index 696ef568..993c741a 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -33,3 +33,5 @@ endif cp storage_xen.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) cp hardware.py $(DESTDIR)$(PYTHON_TESTSPATH) cp hardware.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) + cp extra.py $(DESTDIR)$(PYTHON_TESTSPATH) + cp extra.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) diff --git a/tests/__init__.py b/tests/__init__.py index 02d660e3..1aff0021 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -740,48 +740,6 @@ class BackupTestsMixin(SystemTestsMixin): f.truncate(size) f.close() -class ExtraTestMixin(SystemTestsMixin): - - template = None - - def setUp(self): - super(ExtraTestMixin, self).setUp() - self.qc.unlock_db() - - def create_vms(self, names): - """ - Create AppVMs for the duration of the test. Will be automatically - removed after completing the test. - :param names: list of VM names to create (each of them will be - prefixed with some test specific string) - :return: list of created VM objects - """ - self.qc.lock_db_for_writing() - self.qc.load() - if self.template: - template = self.qc.get_vm_by_name(self.template) - else: - template = self.qc.get_default_template() - for vmname in names: - vm = self.qc.add_new_vm("QubesAppVm", - name=self.make_vm_name(vmname), - template=template) - vm.create_on_disk(verbose=False) - self.save_and_reload_db() - - # get objects after reload - vms = [] - for vmname in names: - vms.append(self.qc.get_vm_by_name(self.make_vm_name(vmname))) - return vms - - def enable_network(self): - """ - Enable access to the network. Must be called before creating VMs. - """ - # nothing to do in core2 - pass - def load_tests(loader, tests, pattern): # discard any tests from this module, because it hosts base classes @@ -798,43 +756,9 @@ def load_tests(loader, tests, pattern): 'qubes.tests.storage', 'qubes.tests.storage_xen', 'qubes.tests.hardware', + 'qubes.tests.extra', ): tests.addTests(loader.loadTestsFromName(modname)) - - for entry in pkg_resources.iter_entry_points('qubes.tests.extra'): - for test_case in entry(): - tests.addTests(loader.loadTestsFromTestCase( - type( - entry.name + '_' + test_case.__name__, - (test_case, ExtraTestMixin, QubesTestCase), - {} - ) - )) - - - try: - qc = qubes.qubes.QubesVmCollection() - qc.lock_db_for_reading() - qc.load() - qc.unlock_db() - templates = [vm.name for vm in qc.values() if - isinstance(vm, qubes.qubes.QubesTemplateVm)] - except OSError: - templates = [] - - for entry in pkg_resources.iter_entry_points( - 'qubes.tests.extra.for_template'): - for test_case in entry(): - for template in templates: - tests.addTests(loader.loadTestsFromTestCase( - type( - entry.name + '_' + test_case.__name__, - (test_case, ExtraTestMixin, - QubesTestCase), - {'template': template} - ) - )) - return tests diff --git a/tests/extra.py b/tests/extra.py index e6b28fd8..31943b3c 100644 --- a/tests/extra.py +++ b/tests/extra.py @@ -1 +1,109 @@ -__author__ = 'user' +#!/usr/bin/python2 -O +# vim: fileencoding=utf-8 + +# +# The Qubes OS Project, https://www.qubes-os.org/ +# +# Copyright (C) 2016 +# Marek Marczykowski-Górecki +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +import pkg_resources +import qubes.tests +import qubes.qubes + + +class ExtraTestMixin(qubes.tests.SystemTestsMixin): + + template = None + + def setUp(self): + super(ExtraTestMixin, self).setUp() + self.qc.unlock_db() + + def create_vms(self, names): + """ + Create AppVMs for the duration of the test. Will be automatically + removed after completing the test. + :param names: list of VM names to create (each of them will be + prefixed with some test specific string) + :return: list of created VM objects + """ + self.qc.lock_db_for_writing() + self.qc.load() + if self.template: + template = self.qc.get_vm_by_name(self.template) + else: + template = self.qc.get_default_template() + for vmname in names: + vm = self.qc.add_new_vm("QubesAppVm", + name=self.make_vm_name(vmname), + template=template) + vm.create_on_disk(verbose=False) + self.save_and_reload_db() + self.qc.unlock_db() + + # get objects after reload + vms = [] + for vmname in names: + vms.append(self.qc.get_vm_by_name(self.make_vm_name(vmname))) + return vms + + def enable_network(self): + """ + Enable access to the network. Must be called before creating VMs. + """ + # nothing to do in core2 + pass + + +def load_tests(loader, tests, pattern): + for entry in pkg_resources.iter_entry_points('qubes.tests.extra'): + for test_case in entry(): + tests.addTests(loader.loadTestsFromTestCase( + type( + entry.name + '_' + test_case.__name__, + (test_case, ExtraTestMixin, qubes.tests.QubesTestCase), + {} + ) + )) + + try: + qc = qubes.qubes.QubesVmCollection() + qc.lock_db_for_reading() + qc.load() + qc.unlock_db() + templates = [vm.name for vm in qc.values() if + isinstance(vm, qubes.qubes.QubesTemplateVm)] + except OSError: + templates = [] + + for entry in pkg_resources.iter_entry_points( + 'qubes.tests.extra.for_template'): + for test_case in entry.load()(): + for template in templates: + tests.addTests(loader.loadTestsFromTestCase( + type( + '{}_{}_{}'.format( + entry.name, test_case.__name__, template), + (test_case, ExtraTestMixin, + qubes.tests.QubesTestCase), + {'template': template} + ) + )) + + return tests From 65255626e22a8c1031bc51c4dd714d98f0f42614 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 7 Mar 2016 03:54:21 +0100 Subject: [PATCH 35/54] core: add separate QubesDB entry for primary DNS address Fixes QubesOS/qubes-issues#1817 --- core-modules/000QubesVm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py index f1c44024..43b20a73 100644 --- a/core-modules/000QubesVm.py +++ b/core-modules/000QubesVm.py @@ -1105,6 +1105,7 @@ class QubesVm(object): if self.is_netvm(): self.qdb.write("/qubes-netvm-gateway", self.gateway) + self.qdb.write("/qubes-netvm-primary-dns", self.gateway) self.qdb.write("/qubes-netvm-secondary-dns", self.secondary_dns) self.qdb.write("/qubes-netvm-netmask", self.netmask) self.qdb.write("/qubes-netvm-network", self.network) @@ -1113,6 +1114,7 @@ class QubesVm(object): self.qdb.write("/qubes-ip", self.ip) self.qdb.write("/qubes-netmask", self.netvm.netmask) self.qdb.write("/qubes-gateway", self.netvm.gateway) + self.qdb.write("/qubes-primary-dns", self.netvm.gateway) self.qdb.write("/qubes-secondary-dns", self.netvm.secondary_dns) tzname = self.get_timezone() From 9ac0cf85779ba9a469f70fd92634979c055db59f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 7 Mar 2016 03:55:23 +0100 Subject: [PATCH 36/54] version 3.1.14 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 55f20a1a..2a399f7d 100644 --- a/version +++ b/version @@ -1 +1 @@ -3.1.13 +3.1.14 From 63407b689ed87e4d517b4e62c7c5f8a811170847 Mon Sep 17 00:00:00 2001 From: Jon Griffiths Date: Wed, 9 Mar 2016 00:50:06 +1300 Subject: [PATCH 37/54] qvm-backup/qvm-backup-restore: Allow reading pass phrase from file or stdin Passing -p/--passphrase-file uses the first line of the file passed, or stdin if '-' is given. This works with piped input too. --- qvm-tools/qvm-backup | 35 +++++++++++++++++++++-------------- qvm-tools/qvm-backup-restore | 24 ++++++++++++++++++------ 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/qvm-tools/qvm-backup b/qvm-tools/qvm-backup index aa69f9c6..854b7e31 100755 --- a/qvm-tools/qvm-backup +++ b/qvm-tools/qvm-backup @@ -30,6 +30,7 @@ import qubes.backup import os import sys import getpass +from locale import getpreferredencoding def print_progress(progress): print >> sys.stderr, "\r-> Backing up files: {0}%...".format (progress), @@ -51,6 +52,10 @@ def main(): parser.add_option ("--no-encrypt", action="store_true", dest="no_encrypt", default=False, help="Skip encryption even if sending the backup to VM") + parser.add_option ("-p", "--passphrase-file", action="store", + dest="pass_file", default=None, + help="File containing the pass phrase to use, or '-' " + "to read it from stdin") parser.add_option ("-E", "--enc-algo", action="store", dest="crypto_algorithm", default=None, help="Specify non-default encryption algorithm. For " @@ -156,24 +161,26 @@ def main(): if not options.encrypt: print >>sys.stderr, "WARNING: encryption will not be used" - prompt = raw_input ("Do you want to proceed? [y/N] ") - if not (prompt == "y" or prompt == "Y"): - exit (0) + if options.pass_file is not None: + f = open(options.pass_file) if options.pass_file != "-" else sys.stdin + passphrase = f.readline().rstrip() + if f is not sys.stdin: + f.close() - if options.encrypt: - passphrase = getpass.getpass("Please enter the pass phrase that will " - "be used to encrypt and verify the " - "backup: ") else: - passphrase = getpass.getpass("Please enter the pass phrase that will " - "be used to verify the backup: ") + if raw_input("Do you want to proceed? [y/N] ").upper() != "Y": + exit(0) - passphrase2 = getpass.getpass("Enter again for verification: ") - if passphrase != passphrase2: - print >>sys.stderr, "ERROR: Password mismatch" - exit(1) + s = ("Please enter the pass phrase that will be used to {}verify " + "the backup: ").format('encrypt and ' if options.encrypt else '') + passphrase = getpass.getpass(s) - passphrase = passphrase.decode(sys.stdin.encoding) + if getpass.getpass("Enter again for verification: ") != passphrase: + print >>sys.stderr, "ERROR: Password mismatch" + exit(1) + + encoding = sys.stdin.encoding or getpreferredencoding() + passphrase = passphrase.decode(encoding) kwargs = {} if options.hmac_algorithm: diff --git a/qvm-tools/qvm-backup-restore b/qvm-tools/qvm-backup-restore index bcfdfad0..d3ff527c 100755 --- a/qvm-tools/qvm-backup-restore +++ b/qvm-tools/qvm-backup-restore @@ -31,6 +31,7 @@ from qubes.backup import backup_restore_do import qubes.backup import sys from optparse import OptionParser +from locale import getpreferredencoding import os import sys @@ -81,6 +82,10 @@ def main(): parser.add_option ("-e", "--encrypted", action="store_true", dest="decrypt", default=False, help="The backup is encrypted") + parser.add_option ("-p", "--passphrase-file", action="store", + dest="pass_file", default=None, + help="File containing the pass phrase to use, or '-' to read it from stdin") + parser.add_option ("-z", "--compressed", action="store_true", dest="compressed", default=False, help="The backup is compressed") @@ -128,8 +133,16 @@ def main(): print >>sys.stderr, "ERROR: VM {0} does not exist".format(options.appvm) exit(1) - passphrase = getpass.getpass("Please enter the pass phrase that will be used to decrypt/verify the backup: ") - passphrase = passphrase.decode(sys.stdin.encoding) + if options.pass_file is not None: + f = open(options.pass_file) if options.pass_file != "-" else sys.stdin + passphrase = f.readline().rstrip() + if f is not sys.stdin: + f.close() + else: + passphrase = getpass.getpass("Please enter the pass phrase to decrypt/verify the backup: ") + + encoding = sys.stdin.encoding or getpreferredencoding() + passphrase = passphrase.decode(encoding) print >> sys.stderr, "Checking backup content..." @@ -244,10 +257,9 @@ def main(): print >> sys.stderr, "Continuing as directed" print >> sys.stderr, "While restoring user homedir, existing files/dirs will be backed up in 'home-pre-restore-' dir" - prompt = raw_input ("Do you want to proceed? [y/N] ") - if not (prompt == "y" or prompt == "Y"): - exit (0) - + if options.pass_file is None: + if raw_input("Do you want to proceed? [y/N] ").upper() != "Y": + exit(0) try: backup_restore_do(restore_info, From e33b958bdd5d279602ccce1fce877bba1608242b Mon Sep 17 00:00:00 2001 From: Patrick Schleizer Date: Sun, 13 Mar 2016 01:15:46 +0000 Subject: [PATCH 38/54] implemented dom0 qubes.GetRandomizedTime Required for fixing 'sys-whonix doesn't connect to Tor after system suspend'. https://github.com/QubesOS/qubes-issues/issues/1764 --- Makefile | 2 + .../qubes.GetRandomizedTime.policy | 6 ++ qubes-rpc/qubes.GetRandomizedTime | 80 +++++++++++++++++++ 3 files changed, 88 insertions(+) create mode 100644 qubes-rpc-policy/qubes.GetRandomizedTime.policy create mode 100755 qubes-rpc/qubes.GetRandomizedTime diff --git a/Makefile b/Makefile index 2282173e..2d3ee271 100644 --- a/Makefile +++ b/Makefile @@ -76,8 +76,10 @@ endif cp qubes-rpc-policy/qubes.NotifyUpdates.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.NotifyUpdates cp qubes-rpc-policy/qubes.NotifyTools.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.NotifyTools cp qubes-rpc-policy/qubes.GetImageRGBA.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.GetImageRGBA + cp qubes-rpc-policy/qubes.GetRandomizedTime.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.GetTime cp qubes-rpc/qubes.NotifyUpdates $(DESTDIR)/etc/qubes-rpc/ cp qubes-rpc/qubes.NotifyTools $(DESTDIR)/etc/qubes-rpc/ + cp qubes-rpc/qubes.GetRandomizedTime $(DESTDIR)/etc/qubes-rpc/ cp qubes-rpc/qubes-notify-updates $(DESTDIR)/usr/libexec/qubes/ cp qubes-rpc/qubes-notify-tools $(DESTDIR)/usr/libexec/qubes/ mkdir -p "$(DESTDIR)$(FILESDIR)" diff --git a/qubes-rpc-policy/qubes.GetRandomizedTime.policy b/qubes-rpc-policy/qubes.GetRandomizedTime.policy new file mode 100644 index 00000000..0f00b0b6 --- /dev/null +++ b/qubes-rpc-policy/qubes.GetRandomizedTime.policy @@ -0,0 +1,6 @@ +## Note that policy parsing stops at the first match, +## so adding anything below "$anyvm $anyvm action" line will have no effect + +## Please use a single # to start your custom comments + +$anyvm dom0 allow diff --git a/qubes-rpc/qubes.GetRandomizedTime b/qubes-rpc/qubes.GetRandomizedTime new file mode 100755 index 00000000..54d78d1b --- /dev/null +++ b/qubes-rpc/qubes.GetRandomizedTime @@ -0,0 +1,80 @@ +#!/bin/bash + +# The Qubes OS Project, http://www.qubes-os.org +# +# Copyright (C) 2016 Patrick Schleizer +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +## Similar code as Boot Clock Randomization. +## https://www.whonix.org/wiki/Boot_Clock_Randomization + +set -e + +## Get a random 0 or 1. +## Will use this to decide to use plus or minus. +## +## Thanks to +## http://linux.byexamples.com/archives/128/generating-random-numbers/ +ZERO_OR_ONE="$(( 0+($(od -An -N2 -i /dev/random) )%(0+2) ))" + +## Create a random number between 0 and 180. +DELAY="$(( $(od -An -N2 -i /dev/random)%(180-0+1) ))" + +## Create a random number between 0 and 999999999. +## +## Thanks to +## https://stackoverflow.com/questions/22887891/how-can-i-get-a-random-dev-random-number-between-0-and-999999999-in-bash +NANOSECONDS="$(shuf -i0-999999999 -n1 --random-source=/dev/random)" + +## Examples NANOSECONDS: +## 117752805 +## 38653957 + +## Add leading zeros, because `date` expects 9 digits. +NANOSECONDS="$(printf '%0*d\n' 9 "$NANOSECONDS")" + +## Using +## printf '%0*d\n' 9 "38653957" +## 38653957 +## becomes +## 038653957 + +## Examples NANOSECONDS: +## 117752805 +## 038653957 + +if [ "$ZERO_OR_ONE" = "0" ]; then + PLUS_OR_MINUS="-" +elif [ "$ZERO_OR_ONE" = "1" ]; then + PLUS_OR_MINUS="+" +else + exit 2 +fi + +#OLD_TIME="$(date)" +#OLD_TIME_NANOSECONDS="$(date +%s.%N)" + +OLD_UNIXTIME="$(date +%s)" + +NEW_TIME="$(( $OLD_UNIXTIME $PLUS_OR_MINUS $DELAY ))" + +NEW_TIME_NANOSECONDS="$NEW_TIME.$NANOSECONDS" + +echo "$NEW_TIME_NANOSECONDS" + +## Testing the `date` syntax: +## date --date @1396733199.112834496 +## date --date "@$NEW_TIME_NANOSECONDS" From 524888d2fdfea9b83b0725f9c1ea9620990cdf0d Mon Sep 17 00:00:00 2001 From: Patrick Schleizer Date: Sun, 13 Mar 2016 01:52:03 +0000 Subject: [PATCH 39/54] use shuf rather than od because it is more readable Thanks to @marmarek for the suggestion! https://github.com/QubesOS/qubes-core-admin/pull/23/files#r55930643 --- qubes-rpc/qubes.GetRandomizedTime | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/qubes-rpc/qubes.GetRandomizedTime b/qubes-rpc/qubes.GetRandomizedTime index 54d78d1b..53341f21 100755 --- a/qubes-rpc/qubes.GetRandomizedTime +++ b/qubes-rpc/qubes.GetRandomizedTime @@ -25,13 +25,10 @@ set -e ## Get a random 0 or 1. ## Will use this to decide to use plus or minus. -## -## Thanks to -## http://linux.byexamples.com/archives/128/generating-random-numbers/ -ZERO_OR_ONE="$(( 0+($(od -An -N2 -i /dev/random) )%(0+2) ))" +ZERO_OR_ONE="$(shuf -i0-1 -n1 --random-source=/dev/random)" ## Create a random number between 0 and 180. -DELAY="$(( $(od -An -N2 -i /dev/random)%(180-0+1) ))" +DELAY="$(shuf -i0-180 -n1 --random-source=/dev/random)" ## Create a random number between 0 and 999999999. ## From b24ba307c3dd6b6f38601cfc1cf1f130e082d640 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 13 Mar 2016 03:36:20 +0100 Subject: [PATCH 40/54] tests: move warning to the module level --- tests/__init__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 1aff0021..e30200fa 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -22,6 +22,14 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # + +""" +.. warning:: + The test suite hereby claims any domain whose name starts with + :py:data:`VMPREFIX` as fair game. This is needed to enforce sane + test executing environment. If you have domains named ``test-*``, + don't run the tests. +""" from distutils import spawn import functools @@ -387,11 +395,6 @@ class SystemTestsMixin(object): def _remove_test_vms(cls, qc, conn, prefix=VMPREFIX): """Aggresively remove any domain that has name in testing namespace. - .. warning:: - The test suite hereby claims any domain whose name starts with - :py:data:`VMPREFIX` as fair game. This is needed to enforce sane - test executing environment. If you have domains named ``test-*``, - don't run the tests. """ # first, remove them Qubes-way From 63f0966cdd0bc8f53ea789f89a4a929bb1cb2d55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 13 Mar 2016 03:37:04 +0100 Subject: [PATCH 41/54] tests: fix MimeHandlers after "29c602c tests: prefix internal functions with underscore" --- tests/vm_qrexec_gui.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py index 0b4ac402..5c25dddc 100644 --- a/tests/vm_qrexec_gui.py +++ b/tests/vm_qrexec_gui.py @@ -1227,12 +1227,12 @@ class TC_50_MimeHandlers(qubes.tests.SystemTestsMixin): qc = QubesVmCollection() - cls.kill_test_vms(qc, prefix=qubes.tests.CLSVMPREFIX) + cls._kill_test_vms(qc, prefix=qubes.tests.CLSVMPREFIX) qc.lock_db_for_writing() qc.load() - cls.remove_test_vms(qc, qubes.qubes.vmm.libvirt_conn, + cls._remove_test_vms(qc, qubes.qubes.vmm.libvirt_conn, prefix=qubes.tests.CLSVMPREFIX) cls.source_vmname = cls.make_vm_name('source', True) From 01b667427fa2076e4f766464fb966cfc541d1695 Mon Sep 17 00:00:00 2001 From: Axon Date: Mon, 14 Mar 2016 12:13:04 +0000 Subject: [PATCH 42/54] Nicely print list of excluded VMs Prints a sorted list of VMs not selected for backup, one name per line. (Fixes QubesOS/qubes-issues#1023) --- core/backup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/backup.py b/core/backup.py index 7ba8808f..77c81362 100644 --- a/core/backup.py +++ b/core/backup.py @@ -357,8 +357,8 @@ def backup_prepare(vms_list=None, exclude_list=None, vms_not_for_backup = [vm.name for vm in qvm_collection.values() if not vm.backup_content] - print_callback("VMs not selected for backup: %s" % " ".join( - vms_not_for_backup)) + print_callback("VMs not selected for backup:\n%s" % "\n".join(sorted( + vms_not_for_backup))) if there_are_running_vms: raise QubesException("Please shutdown all VMs before proceeding.") From 2a46ebb205927193b8bb63fa0d1ea278297e235c Mon Sep 17 00:00:00 2001 From: Patrick Schleizer Date: Mon, 14 Mar 2016 22:47:46 +0100 Subject: [PATCH 43/54] fixed wrong target filename --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2d3ee271..189989b5 100644 --- a/Makefile +++ b/Makefile @@ -76,7 +76,7 @@ endif cp qubes-rpc-policy/qubes.NotifyUpdates.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.NotifyUpdates cp qubes-rpc-policy/qubes.NotifyTools.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.NotifyTools cp qubes-rpc-policy/qubes.GetImageRGBA.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.GetImageRGBA - cp qubes-rpc-policy/qubes.GetRandomizedTime.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.GetTime + cp qubes-rpc-policy/qubes.GetRandomizedTime.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.GetRandomizedTime cp qubes-rpc/qubes.NotifyUpdates $(DESTDIR)/etc/qubes-rpc/ cp qubes-rpc/qubes.NotifyTools $(DESTDIR)/etc/qubes-rpc/ cp qubes-rpc/qubes.GetRandomizedTime $(DESTDIR)/etc/qubes-rpc/ From cf5730934a5f51fb69629229c55e2c911dc3fb9e Mon Sep 17 00:00:00 2001 From: Patrick Schleizer Date: Mon, 14 Mar 2016 22:50:46 +0100 Subject: [PATCH 44/54] added to rpm_spec/core-dom0.spec --- rpm_spec/core-dom0.spec | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec index 66737895..c3584744 100644 --- a/rpm_spec/core-dom0.spec +++ b/rpm_spec/core-dom0.spec @@ -247,8 +247,10 @@ fi %attr(0664,root,qubes) %config(noreplace) /etc/qubes-rpc/policy/qubes.NotifyTools %attr(0664,root,qubes) %config(noreplace) /etc/qubes-rpc/policy/qubes.NotifyUpdates %attr(0664,root,qubes) %config(noreplace) /etc/qubes-rpc/policy/qubes.VMShell +%attr(0664,root,qubes) %config(noreplace) /etc/qubes-rpc/policy/qubes.GetRandomizedTime /etc/qubes-rpc/qubes.NotifyTools /etc/qubes-rpc/qubes.NotifyUpdates +/etc/qubes-rpc/qubes.GetRandomizedTime %attr(2770,root,qubes) %dir /var/log/qubes %attr(0770,root,qubes) %dir /var/run/qubes /etc/xdg/autostart/qubes-guid.desktop From 5f3ffbbe367031143326a59264f1d48b6953eb3c Mon Sep 17 00:00:00 2001 From: Jon Griffiths Date: Fri, 11 Mar 2016 16:36:44 +1300 Subject: [PATCH 45/54] Disable debug packages for core-dom0 Leave the 'proper' fix of making this package noarch commented out for now, to allow this to be merged. Comments as per the parallel submit to qubes-artwork. --- rpm_spec/core-dom0.spec | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec index c3584744..a9b5188b 100644 --- a/rpm_spec/core-dom0.spec +++ b/rpm_spec/core-dom0.spec @@ -27,6 +27,9 @@ %{!?version: %define version %(cat version)} +# debug_package hack should be removed when BuildArch:noarch is enabled below +%define debug_package %{nil} + %define _dracutmoddir /usr/lib/dracut/modules.d %if %{fedora} < 17 %define _dracutmoddir /usr/share/dracut/modules.d @@ -43,6 +46,8 @@ License: GPL URL: http://www.qubes-os.org BuildRequires: ImageMagick BuildRequires: systemd-units +# FIXME: Enable this and disable debug_package +#BuildArch: noarch Requires(post): systemd-units Requires(preun): systemd-units Requires(postun): systemd-units From 54eee12b51d41925c59c0d4f276d82bd2697677c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 27 Mar 2016 19:25:17 +0200 Subject: [PATCH 46/54] tests: setup qrexec policy instead of clicking in confirmation dialog xdotool is unreliable so the less of it, the better. --- tests/vm_qrexec_gui.py | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py index 5c25dddc..538509c2 100644 --- a/tests/vm_qrexec_gui.py +++ b/tests/vm_qrexec_gui.py @@ -529,16 +529,14 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin): self.fail("Timeout, probably deadlock") self.assertEqual(result.value, 0, "Service call failed") - @unittest.skipUnless(spawn.find_executable('xdotool'), - "xdotool not installed") def test_100_qrexec_filecopy(self): self.testvm1.start() self.testvm2.start() + self.qrexec_policy('qubes.Filecopy', self.testvm1.name, + self.testvm2.name) p = self.testvm1.run("qvm-copy-to-vm %s /etc/passwd" % self.testvm2.name, passio_popen=True, passio_stderr=True) - # Confirm transfer - self.enter_keys_in_window('Question', ['y']) p.wait() self.assertEqual(p.returncode, 0, "qvm-copy-to-vm failed: %s" % p.stderr.read()) @@ -548,15 +546,13 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin): wait=True) self.assertEqual(retcode, 0, "file differs") - @unittest.skipUnless(spawn.find_executable('xdotool'), - "xdotool not installed") def test_101_qrexec_filecopy_with_autostart(self): self.testvm1.start() + self.qrexec_policy('qubes.Filecopy', self.testvm1.name, + self.testvm2.name) p = self.testvm1.run("qvm-copy-to-vm %s /etc/passwd" % self.testvm2.name, passio_popen=True, passio_stderr=True) - # Confirm transfer - self.enter_keys_in_window('Question', ['y']) p.wait() self.assertEqual(p.returncode, 0, "qvm-copy-to-vm failed: %s" % p.stderr.read()) @@ -571,15 +567,13 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin): wait=True) self.assertEqual(retcode, 0, "file differs") - @unittest.skipUnless(spawn.find_executable('xdotool'), - "xdotool not installed") def test_110_qrexec_filecopy_deny(self): self.testvm1.start() self.testvm2.start() + self.qrexec_policy('qubes.Filecopy', self.testvm1.name, + self.testvm2.name, allow=False) p = self.testvm1.run("qvm-copy-to-vm %s /etc/passwd" % self.testvm2.name, passio_popen=True) - # Deny transfer - self.enter_keys_in_window('Question', ['n']) p.wait() self.assertNotEqual(p.returncode, 0, "qvm-copy-to-vm unexpectedly " "succeeded") @@ -591,15 +585,13 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin): @unittest.skip("Xen gntalloc driver crashes when page is mapped in the " "same domain") - @unittest.skipUnless(spawn.find_executable('xdotool'), - "xdotool not installed") def test_120_qrexec_filecopy_self(self): self.testvm1.start() + self.qrexec_policy('qubes.Filecopy', self.testvm1.name, + self.testvm1.name) p = self.testvm1.run("qvm-copy-to-vm %s /etc/passwd" % self.testvm1.name, passio_popen=True, passio_stderr=True) - # Confirm transfer - self.enter_keys_in_window('Question', ['y']) p.wait() self.assertEqual(p.returncode, 0, "qvm-copy-to-vm failed: %s" % p.stderr.read()) @@ -614,6 +606,8 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin): def test_130_qrexec_filemove_disk_full(self): self.testvm1.start() self.testvm2.start() + self.qrexec_policy('qubes.Filecopy', self.testvm1.name, + self.testvm2.name) # Prepare test file prepare_cmd = ("yes teststring | dd of=testfile bs=1M " "count=50 iflag=fullblock") @@ -634,8 +628,6 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin): p = self.testvm1.run("qvm-move-to-vm %s testfile" % self.testvm2.name, passio_popen=True, passio_stderr=True) - # Confirm transfer - self.enter_keys_in_window('Question', ['y']) # Close GUI error message self.enter_keys_in_window('Error', ['Return']) p.wait() @@ -1319,12 +1311,11 @@ class TC_50_MimeHandlers(qubes.tests.SystemTestsMixin): passio_popen=True) vmpattern = "disp*" else: + self.qrexec_policy('qubes.Filecopy', self.source_vm.name, + self.target_vmname) p = self.source_vm.run("qvm-open-in-vm {} {}".format( self.target_vmname, filename), passio_popen=True) vmpattern = self.target_vmname - if not dispvm: - # For opening a file in DispVM default policy is set to "allow" - self.enter_keys_in_window('Question', ['y']) wait_count = 0 winid = None window_title = None From b396629d4438c0bb8d5352c0136e80c086162a44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 28 Mar 2016 01:19:01 +0200 Subject: [PATCH 47/54] tests: qvm-move-to-vm --- tests/vm_qrexec_gui.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py index 538509c2..1a924b21 100644 --- a/tests/vm_qrexec_gui.py +++ b/tests/vm_qrexec_gui.py @@ -546,6 +546,27 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin): wait=True) self.assertEqual(retcode, 0, "file differs") + def test_105_qrexec_filemove(self): + self.testvm1.start() + self.testvm2.start() + self.qrexec_policy('qubes.Filecopy', self.testvm1.name, + self.testvm2.name) + retcode = self.testvm1.run("cp /etc/passwd passwd", wait=True) + assert retcode == 0, "Failed to prepare source file" + p = self.testvm1.run("qvm-move-to-vm %s passwd" % + self.testvm2.name, passio_popen=True, + passio_stderr=True) + p.wait() + self.assertEqual(p.returncode, 0, "qvm-move-to-vm failed: %s" % + p.stderr.read()) + retcode = self.testvm2.run("diff /etc/passwd " + "/home/user/QubesIncoming/{}/passwd".format( + self.testvm1.name), + wait=True) + self.assertEqual(retcode, 0, "file differs") + retcode = self.testvm1.run("test -f passwd", wait=True) + self.assertEqual(retcode, 1, "source file not removed") + def test_101_qrexec_filecopy_with_autostart(self): self.testvm1.start() self.qrexec_policy('qubes.Filecopy', self.testvm1.name, From 5566f31a4202c14199cccb3db52d893dc62d7aa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 28 Mar 2016 01:19:23 +0200 Subject: [PATCH 48/54] tests: qrexec service argument QubesOS/qubes-issues#1876 --- tests/vm_qrexec_gui.py | 109 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py index 1a924b21..db514249 100644 --- a/tests/vm_qrexec_gui.py +++ b/tests/vm_qrexec_gui.py @@ -529,6 +529,115 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin): self.fail("Timeout, probably deadlock") self.assertEqual(result.value, 0, "Service call failed") + def test_080_qrexec_service_argument_allow_default(self): + """Qrexec service call with argument""" + self.testvm1.start() + self.testvm2.start() + p = self.testvm2.run("cat > /etc/qubes-rpc/test.Argument", user="root", + passio_popen=True) + p.communicate("/bin/echo $1") + + with open("/etc/qubes-rpc/policy/test.Argument", "w") as policy: + policy.write("%s %s allow" % (self.testvm1.name, self.testvm2.name)) + self.addCleanup(os.unlink, "/etc/qubes-rpc/policy/test.Argument") + + p = self.testvm1.run("/usr/lib/qubes/qrexec-client-vm {} " + "test.Argument+argument".format(self.testvm2.name), + passio_popen=True) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, "argument\n") + + def test_081_qrexec_service_argument_allow_specific(self): + """Qrexec service call with argument - allow only specific value""" + self.testvm1.start() + self.testvm2.start() + p = self.testvm2.run("cat > /etc/qubes-rpc/test.Argument", user="root", + passio_popen=True) + p.communicate("/bin/echo $1") + + with open("/etc/qubes-rpc/policy/test.Argument", "w") as policy: + policy.write("$anyvm $anyvm deny") + self.addCleanup(os.unlink, "/etc/qubes-rpc/policy/test.Argument") + + with open("/etc/qubes-rpc/policy/test.Argument+argument", "w") as \ + policy: + policy.write("%s %s allow" % (self.testvm1.name, self.testvm2.name)) + self.addCleanup(os.unlink, + "/etc/qubes-rpc/policy/test.Argument+argument") + + p = self.testvm1.run("/usr/lib/qubes/qrexec-client-vm {} " + "test.Argument+argument".format(self.testvm2.name), + passio_popen=True) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, "argument\n") + + def test_082_qrexec_service_argument_deny_specific(self): + """Qrexec service call with argument - deny specific value""" + self.testvm1.start() + self.testvm2.start() + p = self.testvm2.run("cat > /etc/qubes-rpc/test.Argument", user="root", + passio_popen=True) + p.communicate("/bin/echo $1") + + with open("/etc/qubes-rpc/policy/test.Argument", "w") as policy: + policy.write("$anyvm $anyvm allow") + self.addCleanup(os.unlink, "/etc/qubes-rpc/policy/test.Argument") + + with open("/etc/qubes-rpc/policy/test.Argument+argument", "w") as \ + policy: + policy.write("%s %s deny" % (self.testvm1.name, self.testvm2.name)) + self.addCleanup(os.unlink, + "/etc/qubes-rpc/policy/test.Argument+argument") + + p = self.testvm1.run("/usr/lib/qubes/qrexec-client-vm {} " + "test.Argument+argument".format(self.testvm2.name), + passio_popen=True) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, "") + self.assertEqual(p.returncode, 1, "Service request should be denied") + + def test_083_qrexec_service_argument_specific_implementation(self): + """Qrexec service call with argument - argument specific + implementatation""" + self.testvm1.start() + self.testvm2.start() + p = self.testvm2.run("cat > /etc/qubes-rpc/test.Argument", user="root", + passio_popen=True) + p.communicate("/bin/echo $1") + + p = self.testvm2.run("cat > /etc/qubes-rpc/test.Argument+argument", + user="root", passio_popen=True) + p.communicate("/bin/echo specific: $1") + + with open("/etc/qubes-rpc/policy/test.Argument", "w") as policy: + policy.write("%s %s allow" % (self.testvm1.name, self.testvm2.name)) + self.addCleanup(os.unlink, "/etc/qubes-rpc/policy/test.Argument") + + p = self.testvm1.run("/usr/lib/qubes/qrexec-client-vm {} " + "test.Argument+argument".format(self.testvm2.name), + passio_popen=True) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, "specific: argument\n") + + def test_084_qrexec_service_argument_extra_env(self): + """Qrexec service call with argument - extra env variables""" + self.testvm1.start() + self.testvm2.start() + p = self.testvm2.run("cat > /etc/qubes-rpc/test.Argument", user="root", + passio_popen=True) + p.communicate("/bin/echo $QREXEC_SERVICE_FULL_NAME " + "$QREXEC_SERVICE_ARGUMENT") + + with open("/etc/qubes-rpc/policy/test.Argument", "w") as policy: + policy.write("%s %s allow" % (self.testvm1.name, self.testvm2.name)) + self.addCleanup(os.unlink, "/etc/qubes-rpc/policy/test.Argument") + + p = self.testvm1.run("/usr/lib/qubes/qrexec-client-vm {} " + "test.Argument+argument".format(self.testvm2.name), + passio_popen=True) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, "test.Argument+argument argument\n") + def test_100_qrexec_filecopy(self): self.testvm1.start() self.testvm2.start() From efb5c1fd251cd8d9c78137e23e4d33c638ba9a65 Mon Sep 17 00:00:00 2001 From: Danny Fullerton Date: Mon, 28 Mar 2016 13:45:23 -0400 Subject: [PATCH 49/54] Fix --rename-conflicting option. The code was fully implemented - and tested - but didn't worked from the cli. --- qvm-tools/qvm-backup-restore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qvm-tools/qvm-backup-restore b/qvm-tools/qvm-backup-restore index d3ff527c..00ad7ba4 100755 --- a/qvm-tools/qvm-backup-restore +++ b/qvm-tools/qvm-backup-restore @@ -115,6 +115,8 @@ def main(): restore_options['use-default-netvm'] = True if options.replace_template: restore_options['replace-template'] = options.replace_template + if options.rename_conflicting: + restore_options['rename-conflicting'] = True if not options.dom0_home: restore_options['dom0-home'] = False if options.ignore_username_mismatch: From e863b0ea50aae344597a6c64f5f60cb272503e73 Mon Sep 17 00:00:00 2001 From: Danny Fullerton Date: Mon, 28 Mar 2016 20:33:34 -0400 Subject: [PATCH 50/54] Add missing options to qvm-backup-restore doc. --- doc/qvm-tools/qvm-backup-restore.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/qvm-tools/qvm-backup-restore.rst b/doc/qvm-tools/qvm-backup-restore.rst index e078f7fa..d9e5f4c6 100644 --- a/doc/qvm-tools/qvm-backup-restore.rst +++ b/doc/qvm-tools/qvm-backup-restore.rst @@ -16,6 +16,8 @@ OPTIONS ======= -h, --help Show this help message and exit +--verify-only + Do not restore the data, only verify backup integrity --skip-broken Do not restore VMs that have missing templates or netvms --ignore-missing @@ -32,6 +34,14 @@ OPTIONS Do not restore dom0 user home dir --ignore-username-mismatch Ignore dom0 username mismatch while restoring homedir +-d APPVM, --dest-vm=APPVM + The AppVM to send backups to +-e, --encrypted + The backup is encrypted +-z, --compressed + The backup is compressed +--debug + Enable (a lot of) debug output AUTHORS ======= From 47e81a525edda4f4c16c04006bb81db90be0f099 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Tue, 29 Mar 2016 11:25:11 +0200 Subject: [PATCH 51/54] version 3.2.0 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 2a399f7d..944880fa 100644 --- a/version +++ b/version @@ -1 +1 @@ -3.1.14 +3.2.0 From 046149e0f4e8c94753ca7a2d55a40d954e917fd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 1 Apr 2016 02:53:04 +0200 Subject: [PATCH 52/54] core: fix vm.run_service 'wait' argument handling 1. wait=False isn't supportet together with localcmd (explicit, or implicit via 'input') - qrexec-client refuses such combination 2. When using localcmd, qrexec-client exists as soon as the local command terminates, not necessary remote. This may not be desired effect when used with wait=True (the default), so do not use localcmd in such a case Found while debugging tests for qubes.USBAttach/qubes.USBDetach - with wait=True broken, there were a lot of race conditions. Related to QubesOS/qubes-issues#531 --- core-modules/000QubesVm.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py index 43b20a73..227ab8ac 100644 --- a/core-modules/000QubesVm.py +++ b/core-modules/000QubesVm.py @@ -1676,13 +1676,17 @@ class QubesVm(object): if bool(input) + bool(passio_popen) + bool(localcmd) > 1: raise ValueError("'input', 'passio_popen', 'localcmd' cannot be " "used together") + if not wait and (localcmd or input): + raise ValueError("Cannot use wait=False with input or " + "localcmd specified") if localcmd: return self.run("QUBESRPC %s %s" % (service, source), localcmd=localcmd, user=user, wait=wait, gui=gui) elif input: - return self.run("QUBESRPC %s %s" % (service, source), - localcmd="echo %s" % input, user=user, wait=wait, - gui=gui) + p = self.run("QUBESRPC %s %s" % (service, source), + user=user, wait=wait, gui=gui, passio_popen=True) + p.communicate(input) + return p.returncode else: return self.run("QUBESRPC %s %s" % (service, source), passio_popen=passio_popen, user=user, wait=wait, From 6901fde56a4a8e325454df9f418b585d0b3f0337 Mon Sep 17 00:00:00 2001 From: Axon Date: Fri, 1 Apr 2016 19:06:26 +0000 Subject: [PATCH 53/54] Fix typos Most of these typos were corrected in the qubes-doc repo as part of a larger typo fixing sweet, but those changes were reverted when the pages were regerenated from source. --- doc/qvm-tools/qvm-add-appvm.rst | 2 +- doc/qvm-tools/qvm-prefs.rst | 6 +++--- doc/qvm-tools/qvm-revert-template-changes.rst | 2 +- doc/qvm-tools/qvm-service.rst | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/qvm-tools/qvm-add-appvm.rst b/doc/qvm-tools/qvm-add-appvm.rst index 5fefab79..6d37b57d 100644 --- a/doc/qvm-tools/qvm-add-appvm.rst +++ b/doc/qvm-tools/qvm-add-appvm.rst @@ -6,7 +6,7 @@ NAME ==== qvm-add-appvm - add an already installed appvm to the Qubes DB -WARNING: Noramlly you would not need this command, and you would use qvm-create instead! +WARNING: Normally you should not need this command, and you should use qvm-create instead! :Date: 2012-04-10 diff --git a/doc/qvm-tools/qvm-prefs.rst b/doc/qvm-tools/qvm-prefs.rst index 3120f143..12bef70e 100644 --- a/doc/qvm-tools/qvm-prefs.rst +++ b/doc/qvm-tools/qvm-prefs.rst @@ -134,7 +134,7 @@ mac Can be used to force specific of virtual ethernet card in the VM. Setting to ``auto`` will use automatic-generated MAC - based on VM id. Especially - useful when some licencing depending on static MAC address. + useful when licensing requires a static MAC address. For template-based HVM ``auto`` mode means to clone template MAC. default_user @@ -147,7 +147,7 @@ debug Accepted values: ``on``, ``off`` Enables debug mode for VM. This can be used to turn on/off verbose logging - in many qubes components at once (gui virtualization, VM kernel, some other + in many Qubes components at once (gui virtualization, VM kernel, some other services). For template-based HVM, enabling debug mode also disables automatic reset root.img (actually volatile.img) before each VM startup, so changes made to @@ -172,7 +172,7 @@ guiagent_installed This HVM have gui agent installed. This option disables full screen GUI virtualization and enables per-window seemless GUI mode. This option will be automatically turned on during Qubes Windows Tools installation, but if - you install qubes gui agent in some other OS, you need to turn this option + you install Qubes gui agent in some other OS, you need to turn this option on manually. You can turn this option off to troubleshoot some early HVM OS boot problems (enter safe mode etc), but the option will be automatically enabled at first VM normal startup (and will take effect from the next diff --git a/doc/qvm-tools/qvm-revert-template-changes.rst b/doc/qvm-tools/qvm-revert-template-changes.rst index f250cda2..358964c6 100644 --- a/doc/qvm-tools/qvm-revert-template-changes.rst +++ b/doc/qvm-tools/qvm-revert-template-changes.rst @@ -17,7 +17,7 @@ OPTIONS -h, --help Show this help message and exit --force - Do not prompt for comfirmation + Do not prompt for confirmation AUTHORS ======= diff --git a/doc/qvm-tools/qvm-service.rst b/doc/qvm-tools/qvm-service.rst index c67c6670..f85b6c08 100644 --- a/doc/qvm-tools/qvm-service.rst +++ b/doc/qvm-tools/qvm-service.rst @@ -30,7 +30,7 @@ OPTIONS SUPPORTED SERVICES ================== -This list can be incomplete as VM can implement any additional service without knowlege of qubes-core code. +This list can be incomplete as VM can implement any additional service without knowledge of qubes-core code. meminfo-writer Default: enabled everywhere excluding NetVM @@ -38,7 +38,7 @@ meminfo-writer This service reports VM memory usage to dom0, which effectively enables dynamic memory management for the VM. *Note:* this service is enforced to be set by dom0 code. If you try to - remove it (reset to defult state), will be recreated with the rule: enabled + remove it (reset to default state), will be recreated with the rule: enabled if VM have no PCI devices assigned, otherwise disabled. qubes-dvm From 5b2df611e6aaa99e80b323ab269108050c56ee74 Mon Sep 17 00:00:00 2001 From: Axon Date: Fri, 1 Apr 2016 19:07:16 +0000 Subject: [PATCH 54/54] Clarify descriptions of qvm-backup-restore options --- doc/qvm-tools/qvm-backup-restore.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/qvm-tools/qvm-backup-restore.rst b/doc/qvm-tools/qvm-backup-restore.rst index d9e5f4c6..90201d63 100644 --- a/doc/qvm-tools/qvm-backup-restore.rst +++ b/doc/qvm-tools/qvm-backup-restore.rst @@ -21,21 +21,21 @@ OPTIONS --skip-broken Do not restore VMs that have missing templates or netvms --ignore-missing - Ignore missing templates or netvms, restore VMs anyway + Ignore missing templates and netvms, and restore the VMs anyway --skip-conflicting Do not restore VMs that are already present on the host --force-root - Force to run, even with root privileges + Force to run with root privileges --replace-template=REPLACE_TEMPLATE - Restore VMs using another template, syntax: old-template-name:new-template-name (might be repeated) + Restore VMs using another template, syntax: old-template-name:new-template-name (can be repeated) -x EXCLUDE, --exclude=EXCLUDE - Skip restore of specified VM (might be repeated) + Skip restore of specified VM (can be repeated) --skip-dom0-home - Do not restore dom0 user home dir + Do not restore dom0's user home directory --ignore-username-mismatch - Ignore dom0 username mismatch while restoring homedir + Ignore dom0 username mismatch when restoring dom0's user home directory -d APPVM, --dest-vm=APPVM - The AppVM to send backups to + Restore from a backup located in a specific AppVM -e, --encrypted The backup is encrypted -z, --compressed