From c9a55cc198488f65b412a9c59da3b2d6fe549f5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 21 May 2016 03:30:53 +0200 Subject: [PATCH 1/9] tests: use offline mode QubesOS/qubes-issues#2008 --- qubes/tests/vm/mix/net.py | 1 + qubes/tests/vm/qubesvm.py | 1 + 2 files changed, 2 insertions(+) diff --git a/qubes/tests/vm/mix/net.py b/qubes/tests/vm/mix/net.py index dc15c933..dfd844d1 100644 --- a/qubes/tests/vm/mix/net.py +++ b/qubes/tests/vm/mix/net.py @@ -35,6 +35,7 @@ class TC_00_NetVMMixin( def setUp(self): super(TC_00_NetVMMixin, self).setUp() self.app = qubes.tests.vm.TestApp() + self.app.vmm.offline_mode = True def setup_netvms(self, vm): # usage of QubesVM here means that those tests should be after diff --git a/qubes/tests/vm/qubesvm.py b/qubes/tests/vm/qubesvm.py index f4cb0db1..a58c2a23 100644 --- a/qubes/tests/vm/qubesvm.py +++ b/qubes/tests/vm/qubesvm.py @@ -146,6 +146,7 @@ class QubesVMTestsMixin(object): def setUp(self): super(QubesVMTestsMixin, self).setUp() self.app = qubes.tests.vm.TestApp() + self.app.vmm.offline_mode = True def get_vm(self, **kwargs): return qubes.vm.qubesvm.QubesVM(self.app, None, From 2dacb3a54205bb679e65e4e6fe090e7d09907fe7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 15 Jun 2016 19:03:11 +0200 Subject: [PATCH 2/9] backup: drop/resolve minor "TODO" comments --- qubes/backup.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/qubes/backup.py b/qubes/backup.py index a467e701..7802ac8b 100644 --- a/qubes/backup.py +++ b/qubes/backup.py @@ -298,12 +298,7 @@ class Backup(object): self.log = logging.getLogger('qubes.backup') - # FIXME: drop this legacy feature? - if isinstance(self.compressed, basestring): - self.compression_filter = self.compressed - self.compressed = True - else: - self.compression_filter = DEFAULT_COMPRESSION_FILTER + self.compression_filter = DEFAULT_COMPRESSION_FILTER if exclude_list is None: exclude_list = [] @@ -920,7 +915,6 @@ class ExtractWorker2(Process): self.decryptor_process, self.tar2_process]: if process: - # FIXME: kill()? try: process.terminate() except OSError: @@ -972,7 +966,6 @@ class ExtractWorker2(Process): elif not self.tar2_process: # Extracting of the current archive failed, skip to the next # archive - # TODO: some debug option to preserve it? os.remove(filename) continue else: @@ -1157,7 +1150,6 @@ class ExtractWorker3(ExtractWorker2): elif not self.tar2_process: # Extracting of the current archive failed, skip to the next # archive - # TODO: some debug option to preserve it? os.remove(filename) continue else: @@ -1652,7 +1644,6 @@ class BackupRestore(object): "Extracting data: " + size_to_human(vms_size) + " to restore") # retrieve backup from the backup stream (either VM, or dom0 file) - # TODO: add some safety margin in vms_size? (retrieve_proc, filelist_pipe, error_pipe) = \ self._start_retrieval_process(vms_dirs, limit_count, vms_size) @@ -1722,9 +1713,13 @@ class BackupRestore(object): MAX_STDERR_BYTES))) # wait for other processes (if any) for proc in self.processes_to_kill_on_cancel: - # FIXME check 'vmproc' exit code? proc.wait() + if vmproc.returncode != 0: + raise qubes.exc.QubesException( + "Backup completed, but VM receiving it reported an error " + "(exit code {})".format(vmproc.returncode)) + if filename and filename != "EOF": raise qubes.exc.QubesException( "Premature end of archive, the last file was %s" % filename) From 91404cc647d14a75cd2d79c9a51777fb2a7ce479 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 15 Jun 2016 19:10:19 +0200 Subject: [PATCH 3/9] backup: collect files to backup once --- qubes/backup.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/qubes/backup.py b/qubes/backup.py index 7802ac8b..ddec5c5a 100644 --- a/qubes/backup.py +++ b/qubes/backup.py @@ -310,6 +310,8 @@ class Backup(object): self.vms_for_backup = [vm for vm in vms_list if vm.name not in exclude_list] + self._files_to_backup = self.get_files_to_backup() + def __del__(self): if self.tmpdir and os.path.exists(self.tmpdir): shutil.rmtree(self.tmpdir) @@ -399,7 +401,7 @@ class Backup(object): summary += fmt.format('-') summary += "\n" - files_to_backup = self.get_files_to_backup() + files_to_backup = self._files_to_backup for qid, vm_info in files_to_backup.iteritems(): s = "" @@ -506,8 +508,7 @@ class Backup(object): qubes_xml = os.path.join(self.tmpdir, 'qubes.xml') backup_app = qubes.Qubes(qubes_xml) - # FIXME: cache it earlier? - files_to_backup = self.get_files_to_backup() + files_to_backup = self._files_to_backup # make sure backup_content isn't set initially for vm in backup_app.domains: vm.features['backup-content'] = False @@ -572,14 +573,13 @@ class Backup(object): for f in header_files: to_send.put(f) - vm_files_to_backup = self.get_files_to_backup() qubes_xml_info = self.VMToBackup( None, [self.FileToBackup(qubes_xml, '')], '' ) for vm_info in itertools.chain([qubes_xml_info], - vm_files_to_backup.itervalues()): + files_to_backup.itervalues()): for file_info in vm_info.files: self.log.debug("Backing up {}".format(file_info)) From 98effef6061dca3843f634f1c541fd9323c0a02c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 15 Jun 2016 19:10:52 +0200 Subject: [PATCH 4/9] backup: add option to use default kernel for restored VMs --- qubes/backup.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/qubes/backup.py b/qubes/backup.py index ddec5c5a..7e3f0193 100644 --- a/qubes/backup.py +++ b/qubes/backup.py @@ -1237,6 +1237,9 @@ class BackupRestoreOptions(object): #: set template to default if the one referenced in backup do not # exists on the host self.use_default_template = True + #: use default kernel if the one referenced in backup do not exists + # on the host + self.use_default_kernel = True #: restore dom0 home self.dom0_home = True #: dictionary how what templates should be used instead of those @@ -1271,6 +1274,8 @@ class BackupRestore(object): MISSING_NETVM = object() #: TemplateVM used by the VM does not exists on the host MISSING_TEMPLATE = object() + #: Kernel used by the VM does not exists on the host + MISSING_KERNEL = object() def __init__(self, vm): self.vm = vm @@ -1824,6 +1829,19 @@ class BackupRestore(object): else: vm_info.problems.add(self.VMToRestore.MISSING_NETVM) + # check kernel + if hasattr(vm_info.vm, 'kernel'): + installed_kernels = os.listdir(os.path.join( + qubes.config.qubes_base_dir, + qubes.config.system_path['qubes_kernels_base_dir'])) + if not vm_info.vm.property_is_default('kernel') \ + and vm_info.vm.kernel \ + and vm_info.vm.kernel not in installed_kernels: + if self.options.use_default_kernel: + vm_info.vm.kernel = qubes.property.DEFAULT + else: + vm_info.problems.add(self.VMToRestore.MISSING_KERNEL) + return restore_info def _is_vm_included_in_backup_v1(self, check_vm): @@ -2158,16 +2176,6 @@ class BackupRestore(object): del self.app.domains[new_vm.qid] continue - if hasattr(vm, 'kernel'): - # TODO: add a setting for this? - if not vm.property_is_default('kernel') and vm.kernel and \ - vm.kernel not in \ - os.listdir(os.path.join(qubes.config.qubes_base_dir, - qubes.config.system_path[ - 'qubes_kernels_base_dir'])): - self.log.warning("Kernel %s not installed, " - "using default one" % vm.kernel) - vm.kernel = qubes.property.DEFAULT # remove no longer needed backup metadata if 'backup-content' in vm.features: del vm.features['backup-content'] From 9cdf99436054a15949ef6ac0cbbedc7c5b4828ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 16 Jun 2016 16:57:44 +0200 Subject: [PATCH 5/9] Minor fixes --- qubes/ext/r3compatibility.py | 3 +-- qubes/utils.py | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/qubes/ext/r3compatibility.py b/qubes/ext/r3compatibility.py index b1ccaeb5..240370ba 100644 --- a/qubes/ext/r3compatibility.py +++ b/qubes/ext/r3compatibility.py @@ -117,8 +117,7 @@ class R3Compatibility(qubes.ext.Extension): self.write_services(vm) - # FIXME use event after creating Xen domain object, but before "resume" - @qubes.ext.handler('domain-start') + @qubes.ext.handler('domain-spawn') def on_domain_started(self, vm, event, **kwargs): # pylint: disable=unused-argument if vm.netvm: diff --git a/qubes/utils.py b/qubes/utils.py index 66fcc40d..8f0e3121 100644 --- a/qubes/utils.py +++ b/qubes/utils.py @@ -87,8 +87,6 @@ def format_doc(docstring): config_section=None, enable_exit_status=None) return pub.writer.document.astext() -# FIXME those are wrong, k/M/G are SI prefixes and means 10**3 -# maybe adapt https://code.activestate.com/recipes/578019 def parse_size(size): units = [ ('K', 1000), ('KB', 1000), From 5eea4737259688872fd249b794da62a2f154dfbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 16 Jun 2016 16:58:15 +0200 Subject: [PATCH 6/9] core2migration: add a skeleton for dispvm_netvm migration QubesOS/qubes-issues#2075 --- qubes/core2migration.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/qubes/core2migration.py b/qubes/core2migration.py index 3816525c..9f93391e 100644 --- a/qubes/core2migration.py +++ b/qubes/core2migration.py @@ -110,7 +110,40 @@ class Core2Qubes(qubes.Qubes): else: vm.netvm = int(netvm_qid) - # TODO: dispvm_netvm + def set_dispvm_netvm_dependency(self, element): + kwargs = {} + attr_list = ("qid", "uses_default_netvm", "netvm_qid") + + for attribute in attr_list: + kwargs[attribute] = element.get(attribute) + + vm = self.domains[int(kwargs["qid"])] + + if element.get("uses_default_dispvm_netvm") is None: + uses_default_dispvm_netvm = True + else: + uses_default_dispvm_netvm = ( + True if element.get("uses_default_dispvm_netvm") == "True" + else False) + if not uses_default_dispvm_netvm: + dispvm_netvm_qid = element.get("dispvm_netvm_qid") + if dispvm_netvm_qid is None or dispvm_netvm_qid == "none": + dispvm_netvm = None + else: + dispvm_netvm = self.domains[int(dispvm_netvm_qid)] + else: + dispvm_netvm = vm.netvm + + if dispvm_netvm: + dispvm_tpl_name = 'disp-{}'.format(dispvm_netvm.name) + else: + dispvm_tpl_name = 'disp-no-netvm' + + if dispvm_tpl_name not in self.domains: + vm = self.add_new_vm(qubes.vm.appvm.AppVM, + name=dispvm_tpl_name) + # TODO: add support for #2075 + # TODO: set qrexec policy based on dispvm_netvm value def import_core2_vm(self, element): vm_class_name = element.tag From 2ff6aa456ee62edeaf182010f775380e1f82fd12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 16 Jun 2016 21:08:44 +0200 Subject: [PATCH 7/9] Provide fake CPUs count and total memory in offline mode --- qubes/app.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/qubes/app.py b/qubes/app.py index 85792b36..30570e6e 100644 --- a/qubes/app.py +++ b/qubes/app.py @@ -248,6 +248,8 @@ class QubesHost(object): def memory_total(self): '''Total memory, in kbytes''' + if self.app.vmm.offline_mode: + return 2**64-1 self._fetch() return self._total_mem @@ -256,6 +258,9 @@ class QubesHost(object): def no_cpus(self): '''Number of CPUs''' + if self.app.vmm.offline_mode: + return 42 + self._fetch() return self._no_cpus From 4e797663e929608740f7798641f91baffe644b2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 16 Jun 2016 21:09:48 +0200 Subject: [PATCH 8/9] tests: make storage tests working outside of dom0 --- qubes/tests/storage.py | 9 ++++-- qubes/tests/storage_file.py | 63 ++++++++++++++++++++++++++++++------- 2 files changed, 58 insertions(+), 14 deletions(-) diff --git a/qubes/tests/storage.py b/qubes/tests/storage.py index 9d289712..2b027e50 100644 --- a/qubes/tests/storage.py +++ b/qubes/tests/storage.py @@ -58,13 +58,18 @@ class TestDisposableVM(TestVM): def is_disposablevm(self): return True +class TestApp(qubes.Qubes): + def __init__(self, *args, **kwargs): + super(TestApp, self).__init__('/tmp/qubes-test.xml', + load=False, offline_mode=True, **kwargs) + self.load_initial_values() -class TC_00_Pool(SystemTestsMixin, QubesTestCase): +class TC_00_Pool(QubesTestCase): """ This class tests the utility methods from :mod:``qubes.storage`` """ def setUp(self): super(TC_00_Pool, self).setUp() - self.init_default_template() + self.app = TestApp() def test_000_unknown_pool_driver(self): # :pylint: disable=protected-access diff --git a/qubes/tests/storage_file.py b/qubes/tests/storage_file.py index 14005490..f44aabab 100644 --- a/qubes/tests/storage_file.py +++ b/qubes/tests/storage_file.py @@ -30,10 +30,40 @@ from qubes.tests.storage import TestVM # :pylint: disable=invalid-name +class TestApp(qubes.Qubes): + def __init__(self, *args, **kwargs): + super(TestApp, self).__init__('/tmp/qubes-test.xml', + load=False, offline_mode=True, **kwargs) + self.load_initial_values() + self.pools['linux-kernel'].dir_path = '/tmp/qubes-test-kernel' + dummy_kernel = os.path.join( + self.pools['linux-kernel'].dir_path, 'dummy') + os.makedirs(dummy_kernel) + open(os.path.join(dummy_kernel, 'vmlinuz'), 'w').close() + open(os.path.join(dummy_kernel, 'modules.img'), 'w').close() + open(os.path.join(dummy_kernel, 'initramfs'), 'w').close() + self.default_kernel = 'dummy' -class TC_00_FilePool(SystemTestsMixin, QubesTestCase): + def cleanup(self): + shutil.rmtree(self.pools['linux-kernel'].dir_path) + + def create_dummy_template(self): + self.add_new_vm(qubes.vm.templatevm.TemplateVM, + name='test-template', label='red', + memory=1024, maxmem=1024) + self.default_template = 'test-template' + +class TC_00_FilePool(QubesTestCase): """ This class tests some properties of the 'default' pool. """ + def setUp(self): + super(TC_00_FilePool, self).setUp() + self.app = TestApp() + + def tearDown(self): + self.app.cleanup() + super(TC_00_FilePool, self).tearDown() + def test000_default_pool_dir(self): """ The predefined dir for the default pool should be ``/var/lib/qubes`` @@ -52,27 +82,29 @@ class TC_00_FilePool(SystemTestsMixin, QubesTestCase): def _init_app_vm(self): """ Return initalised, but not created, AppVm. """ vmname = self.make_vm_name('appvm') - self.init_default_template() + self.app.create_dummy_template() return self.app.add_new_vm(qubes.vm.appvm.AppVM, name=vmname, template=self.app.default_template, label='red') -class TC_01_FileVolumes(SystemTestsMixin, QubesTestCase): - POOL_DIR = '/var/lib/qubes/test-pool' +class TC_01_FileVolumes(QubesTestCase): + POOL_DIR = '/tmp/test-pool' POOL_NAME = 'test-pool' POOL_CONF = {'driver': 'file', 'dir_path': POOL_DIR, 'name': POOL_NAME} def setUp(self): """ Add a test file based storage pool """ super(TC_01_FileVolumes, self).setUp() - self.init_default_template() + self.app = TestApp() + self.app.create_dummy_template() self.app.add_pool(**self.POOL_CONF) def tearDown(self): """ Remove the file based storage pool after testing """ self.app.remove_pool("test-pool") + self.app.cleanup() super(TC_01_FileVolumes, self).tearDown() shutil.rmtree(self.POOL_DIR, ignore_errors=True) @@ -199,29 +231,36 @@ class TC_01_FileVolumes(SystemTestsMixin, QubesTestCase): self.assertEquals(b_dev.path, expected) -@qubes.tests.skipUnlessDom0 -class TC_03_FilePool(SystemTestsMixin, QubesTestCase): +class TC_03_FilePool(QubesTestCase): """ Test the paths for the default file based pool (``FilePool``). """ - POOL_DIR = '/var/lib/qubes/test-pool' - APPVMS_DIR = '/var/lib/qubes/test-pool/appvms' - TEMPLATES_DIR = '/var/lib/qubes/test-pool/vm-templates' - SERVICE_DIR = '/var/lib/qubes/test-pool/servicevms' + POOL_DIR = '/tmp/test-pool' + APPVMS_DIR = '/tmp/test-pool/appvms' + TEMPLATES_DIR = '/tmp/test-pool/vm-templates' + SERVICE_DIR = '/tmp/test-pool/servicevms' POOL_NAME = 'test-pool' POOL_CONFIG = {'driver': 'file', 'dir_path': POOL_DIR, 'name': POOL_NAME} def setUp(self): """ Add a test file based storage pool """ super(TC_03_FilePool, self).setUp() - self.init_default_template() + self._orig_qubes_base_dir = qubes.config.system_path['qubes_base_dir'] + qubes.config.system_path['qubes_base_dir'] = '/tmp/qubes-test' + self.app = TestApp() + self.app.create_dummy_template() self.app.add_pool(**self.POOL_CONFIG) def tearDown(self): """ Remove the file based storage pool after testing """ self.app.remove_pool("test-pool") + self.app.cleanup() super(TC_03_FilePool, self).tearDown() shutil.rmtree(self.POOL_DIR, ignore_errors=True) + if os.path.exists('/tmp/qubes-test'): + shutil.rmtree('/tmp/qubes-test') + qubes.config.system_path['qubes_base_dir'] = self._orig_qubes_base_dir + def test_001_pool_exists(self): """ Check if the storage pool was added to the storage pool config """ From dcdb62721b9c1640ea806a3bf6599ad26e2825e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 16 Jun 2016 21:23:18 +0200 Subject: [PATCH 9/9] tests: mark TC_01_FileVolumes.test_003_read_volume with expected failure --- qubes/tests/storage_file.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qubes/tests/storage_file.py b/qubes/tests/storage_file.py index f44aabab..2d6d1aff 100644 --- a/qubes/tests/storage_file.py +++ b/qubes/tests/storage_file.py @@ -21,6 +21,7 @@ import shutil import qubes.storage import qubes.tests.storage +import unittest from qubes.config import defaults from qubes.storage import Storage from qubes.storage.file import (OriginFile, ReadOnlyFile, ReadWriteFile, @@ -152,6 +153,7 @@ class TC_01_FileVolumes(QubesTestCase): self.assertEqual(result.pool, self.POOL_NAME) self.assertEqual(result.size, defaults['root_img_size']) + @unittest.expectedFailure def test_003_read_volume(self): template = self.app.default_template original_path = template.volumes['root'].vid