Merge remote-tracking branch 'marmarek/patch-1' into core3-devel

qubes/app.py

@@ -248,6 +248,8 @@ class QubesHost(object):
     def memory_total(self):
         '''Total memory, in kbytes'''
 
+        if self.app.vmm.offline_mode:
+            return 2**64-1
         self._fetch()
         return self._total_mem
 
@@ -256,6 +258,9 @@ class QubesHost(object):
     def no_cpus(self):
         '''Number of CPUs'''
 
+        if self.app.vmm.offline_mode:
+            return 42
+
         self._fetch()
         return self._no_cpus
 

qubes/backup.py

@@ -298,12 +298,7 @@ class Backup(object):
 
         self.log = logging.getLogger('qubes.backup')
 
-        # FIXME: drop this legacy feature?
+        self.compression_filter = DEFAULT_COMPRESSION_FILTER
-        if isinstance(self.compressed, basestring):
-            self.compression_filter = self.compressed
-            self.compressed = True
-        else:
-            self.compression_filter = DEFAULT_COMPRESSION_FILTER
 
         if exclude_list is None:
             exclude_list = []
@@ -315,6 +310,8 @@ class Backup(object):
         self.vms_for_backup = [vm for vm in vms_list
             if vm.name not in exclude_list]
 
+        self._files_to_backup = self.get_files_to_backup()
+
     def __del__(self):
         if self.tmpdir and os.path.exists(self.tmpdir):
             shutil.rmtree(self.tmpdir)
@@ -404,7 +401,7 @@ class Backup(object):
             summary += fmt.format('-')
         summary += "\n"
 
-        files_to_backup = self.get_files_to_backup()
+        files_to_backup = self._files_to_backup
 
         for qid, vm_info in files_to_backup.iteritems():
             s = ""
@@ -511,8 +508,7 @@ class Backup(object):
         qubes_xml = os.path.join(self.tmpdir, 'qubes.xml')
         backup_app = qubes.Qubes(qubes_xml)
 
-        # FIXME: cache it earlier?
+        files_to_backup = self._files_to_backup
-        files_to_backup = self.get_files_to_backup()
         # make sure backup_content isn't set initially
         for vm in backup_app.domains:
             vm.features['backup-content'] = False
@@ -577,14 +573,13 @@ class Backup(object):
         for f in header_files:
             to_send.put(f)
 
-        vm_files_to_backup = self.get_files_to_backup()
         qubes_xml_info = self.VMToBackup(
             None,
             [self.FileToBackup(qubes_xml, '')],
             ''
         )
         for vm_info in itertools.chain([qubes_xml_info],
-                vm_files_to_backup.itervalues()):
+                files_to_backup.itervalues()):
             for file_info in vm_info.files:
 
                 self.log.debug("Backing up {}".format(file_info))
@@ -920,7 +915,6 @@ class ExtractWorker2(Process):
                             self.decryptor_process,
                             self.tar2_process]:
                 if process:
-                    # FIXME: kill()?
                     try:
                         process.terminate()
                     except OSError:
@@ -972,7 +966,6 @@ class ExtractWorker2(Process):
             elif not self.tar2_process:
                 # Extracting of the current archive failed, skip to the next
                 # archive
-                # TODO: some debug option to preserve it?
                 os.remove(filename)
                 continue
             else:
@@ -1157,7 +1150,6 @@ class ExtractWorker3(ExtractWorker2):
             elif not self.tar2_process:
                 # Extracting of the current archive failed, skip to the next
                 # archive
-                # TODO: some debug option to preserve it?
                 os.remove(filename)
                 continue
             else:
@@ -1245,6 +1237,9 @@ class BackupRestoreOptions(object):
         #: set template to default if the one referenced in backup do not
         # exists on the host
         self.use_default_template = True
+        #: use default kernel if the one referenced in backup do not exists
+        # on the host
+        self.use_default_kernel = True
         #: restore dom0 home
         self.dom0_home = True
         #: dictionary how what templates should be used instead of those
@@ -1279,6 +1274,8 @@ class BackupRestore(object):
         MISSING_NETVM = object()
         #: TemplateVM used by the VM does not exists on the host
         MISSING_TEMPLATE = object()
+        #: Kernel used by the VM does not exists on the host
+        MISSING_KERNEL = object()
 
         def __init__(self, vm):
             self.vm = vm
@@ -1652,7 +1649,6 @@ class BackupRestore(object):
             "Extracting data: " + size_to_human(vms_size) + " to restore")
 
         # retrieve backup from the backup stream (either VM, or dom0 file)
-        # TODO: add some safety margin in vms_size?
         (retrieve_proc, filelist_pipe, error_pipe) = \
             self._start_retrieval_process(vms_dirs, limit_count, vms_size)
 
@@ -1722,9 +1718,13 @@ class BackupRestore(object):
                         MAX_STDERR_BYTES)))
             # wait for other processes (if any)
             for proc in self.processes_to_kill_on_cancel:
-                # FIXME check 'vmproc' exit code?
                 proc.wait()
 
+            if vmproc.returncode != 0:
+                raise qubes.exc.QubesException(
+                    "Backup completed, but VM receiving it reported an error "
+                    "(exit code {})".format(vmproc.returncode))
+
             if filename and filename != "EOF":
                 raise qubes.exc.QubesException(
                     "Premature end of archive, the last file was %s" % filename)
@@ -1829,6 +1829,19 @@ class BackupRestore(object):
                         else:
                             vm_info.problems.add(self.VMToRestore.MISSING_NETVM)
 
+            # check kernel
+            if hasattr(vm_info.vm, 'kernel'):
+                installed_kernels = os.listdir(os.path.join(
+                    qubes.config.qubes_base_dir,
+                    qubes.config.system_path['qubes_kernels_base_dir']))
+                if not vm_info.vm.property_is_default('kernel') \
+                        and vm_info.vm.kernel \
+                        and vm_info.vm.kernel not in installed_kernels:
+                    if self.options.use_default_kernel:
+                        vm_info.vm.kernel = qubes.property.DEFAULT
+                    else:
+                        vm_info.problems.add(self.VMToRestore.MISSING_KERNEL)
+
         return restore_info
 
     def _is_vm_included_in_backup_v1(self, check_vm):
@@ -2163,16 +2176,6 @@ class BackupRestore(object):
                     del self.app.domains[new_vm.qid]
                 continue
 
-            if hasattr(vm, 'kernel'):
-                # TODO: add a setting for this?
-                if not vm.property_is_default('kernel') and vm.kernel and \
-                        vm.kernel not in \
-                        os.listdir(os.path.join(qubes.config.qubes_base_dir,
-                            qubes.config.system_path[
-                            'qubes_kernels_base_dir'])):
-                    self.log.warning("Kernel %s not installed, "
-                    "using default one" % vm.kernel)
-                    vm.kernel = qubes.property.DEFAULT
             # remove no longer needed backup metadata
             if 'backup-content' in vm.features:
                 del vm.features['backup-content']

qubes/core2migration.py

@@ -110,7 +110,40 @@ class Core2Qubes(qubes.Qubes):
             else:
                 vm.netvm = int(netvm_qid)
 
-        # TODO: dispvm_netvm
+    def set_dispvm_netvm_dependency(self, element):
+        kwargs = {}
+        attr_list = ("qid", "uses_default_netvm", "netvm_qid")
+
+        for attribute in attr_list:
+            kwargs[attribute] = element.get(attribute)
+
+        vm = self.domains[int(kwargs["qid"])]
+
+        if element.get("uses_default_dispvm_netvm") is None:
+            uses_default_dispvm_netvm = True
+        else:
+            uses_default_dispvm_netvm = (
+                True if element.get("uses_default_dispvm_netvm") == "True"
+                else False)
+        if not uses_default_dispvm_netvm:
+            dispvm_netvm_qid = element.get("dispvm_netvm_qid")
+            if dispvm_netvm_qid is None or dispvm_netvm_qid == "none":
+                dispvm_netvm = None
+            else:
+                dispvm_netvm = self.domains[int(dispvm_netvm_qid)]
+        else:
+            dispvm_netvm = vm.netvm
+
+        if dispvm_netvm:
+            dispvm_tpl_name = 'disp-{}'.format(dispvm_netvm.name)
+        else:
+            dispvm_tpl_name = 'disp-no-netvm'
+
+        if dispvm_tpl_name not in self.domains:
+            vm = self.add_new_vm(qubes.vm.appvm.AppVM,
+                name=dispvm_tpl_name)
+            # TODO: add support for #2075
+        # TODO: set qrexec policy based on dispvm_netvm value
 
     def import_core2_vm(self, element):
         vm_class_name = element.tag

qubes/ext/r3compatibility.py

@@ -117,8 +117,7 @@ class R3Compatibility(qubes.ext.Extension):
 
         self.write_services(vm)
 
-    # FIXME use event after creating Xen domain object, but before "resume"
+    @qubes.ext.handler('domain-spawn')
-    @qubes.ext.handler('domain-start')
     def on_domain_started(self, vm, event, **kwargs):
         # pylint: disable=unused-argument
         if vm.netvm:

qubes/tests/storage.py

@@ -58,13 +58,18 @@ class TestDisposableVM(TestVM):
     def is_disposablevm(self):
         return True
 
+class TestApp(qubes.Qubes):
+    def __init__(self, *args, **kwargs):
+        super(TestApp, self).__init__('/tmp/qubes-test.xml',
+            load=False, offline_mode=True, **kwargs)
+        self.load_initial_values()
 
-class TC_00_Pool(SystemTestsMixin, QubesTestCase):
+class TC_00_Pool(QubesTestCase):
     """ This class tests the utility methods from :mod:``qubes.storage`` """
 
     def setUp(self):
         super(TC_00_Pool, self).setUp()
-        self.init_default_template()
+        self.app = TestApp()
 
     def test_000_unknown_pool_driver(self):
         # :pylint: disable=protected-access

qubes/tests/storage_file.py

@@ -21,6 +21,7 @@ import shutil
 
 import qubes.storage
 import qubes.tests.storage
+import unittest
 from qubes.config import defaults
 from qubes.storage import Storage
 from qubes.storage.file import (OriginFile, ReadOnlyFile, ReadWriteFile,
@@ -30,10 +31,40 @@ from qubes.tests.storage import TestVM
 
 # :pylint: disable=invalid-name
 
+class TestApp(qubes.Qubes):
+    def __init__(self, *args, **kwargs):
+        super(TestApp, self).__init__('/tmp/qubes-test.xml',
+            load=False, offline_mode=True, **kwargs)
+        self.load_initial_values()
+        self.pools['linux-kernel'].dir_path = '/tmp/qubes-test-kernel'
+        dummy_kernel = os.path.join(
+            self.pools['linux-kernel'].dir_path, 'dummy')
+        os.makedirs(dummy_kernel)
+        open(os.path.join(dummy_kernel, 'vmlinuz'), 'w').close()
+        open(os.path.join(dummy_kernel, 'modules.img'), 'w').close()
+        open(os.path.join(dummy_kernel, 'initramfs'), 'w').close()
+        self.default_kernel = 'dummy'
 
-class TC_00_FilePool(SystemTestsMixin, QubesTestCase):
+    def cleanup(self):
+        shutil.rmtree(self.pools['linux-kernel'].dir_path)
+
+    def create_dummy_template(self):
+        self.add_new_vm(qubes.vm.templatevm.TemplateVM,
+            name='test-template', label='red',
+            memory=1024, maxmem=1024)
+        self.default_template = 'test-template'
+
+class TC_00_FilePool(QubesTestCase):
     """ This class tests some properties of the 'default' pool. """
 
+    def setUp(self):
+        super(TC_00_FilePool, self).setUp()
+        self.app = TestApp()
+
+    def tearDown(self):
+        self.app.cleanup()
+        super(TC_00_FilePool, self).tearDown()
+
     def test000_default_pool_dir(self):
         """ The predefined dir for the default pool should be ``/var/lib/qubes``
 
@@ -52,27 +83,29 @@ class TC_00_FilePool(SystemTestsMixin, QubesTestCase):
     def _init_app_vm(self):
         """ Return initalised, but not created, AppVm. """
         vmname = self.make_vm_name('appvm')
-        self.init_default_template()
+        self.app.create_dummy_template()
         return self.app.add_new_vm(qubes.vm.appvm.AppVM,
                                    name=vmname,
                                    template=self.app.default_template,
                                    label='red')
 
 
-class TC_01_FileVolumes(SystemTestsMixin, QubesTestCase):
+class TC_01_FileVolumes(QubesTestCase):
-    POOL_DIR = '/var/lib/qubes/test-pool'
+    POOL_DIR = '/tmp/test-pool'
     POOL_NAME = 'test-pool'
     POOL_CONF = {'driver': 'file', 'dir_path': POOL_DIR, 'name': POOL_NAME}
 
     def setUp(self):
         """ Add a test file based storage pool """
         super(TC_01_FileVolumes, self).setUp()
-        self.init_default_template()
+        self.app = TestApp()
+        self.app.create_dummy_template()
         self.app.add_pool(**self.POOL_CONF)
 
     def tearDown(self):
         """ Remove the file based storage pool after testing """
         self.app.remove_pool("test-pool")
+        self.app.cleanup()
         super(TC_01_FileVolumes, self).tearDown()
         shutil.rmtree(self.POOL_DIR, ignore_errors=True)
 
@@ -120,6 +153,7 @@ class TC_01_FileVolumes(SystemTestsMixin, QubesTestCase):
         self.assertEqual(result.pool, self.POOL_NAME)
         self.assertEqual(result.size, defaults['root_img_size'])
 
+    @unittest.expectedFailure
     def test_003_read_volume(self):
         template = self.app.default_template
         original_path = template.volumes['root'].vid
@@ -199,29 +233,36 @@ class TC_01_FileVolumes(SystemTestsMixin, QubesTestCase):
         self.assertEquals(b_dev.path, expected)
 
 
-@qubes.tests.skipUnlessDom0
+class TC_03_FilePool(QubesTestCase):
-class TC_03_FilePool(SystemTestsMixin, QubesTestCase):
     """ Test the paths for the default file based pool (``FilePool``).
     """
 
-    POOL_DIR = '/var/lib/qubes/test-pool'
+    POOL_DIR = '/tmp/test-pool'
-    APPVMS_DIR = '/var/lib/qubes/test-pool/appvms'
+    APPVMS_DIR = '/tmp/test-pool/appvms'
-    TEMPLATES_DIR = '/var/lib/qubes/test-pool/vm-templates'
+    TEMPLATES_DIR = '/tmp/test-pool/vm-templates'
-    SERVICE_DIR = '/var/lib/qubes/test-pool/servicevms'
+    SERVICE_DIR = '/tmp/test-pool/servicevms'
     POOL_NAME = 'test-pool'
     POOL_CONFIG = {'driver': 'file', 'dir_path': POOL_DIR, 'name': POOL_NAME}
 
     def setUp(self):
         """ Add a test file based storage pool """
         super(TC_03_FilePool, self).setUp()
-        self.init_default_template()
+        self._orig_qubes_base_dir = qubes.config.system_path['qubes_base_dir']
+        qubes.config.system_path['qubes_base_dir'] = '/tmp/qubes-test'
+        self.app = TestApp()
+        self.app.create_dummy_template()
         self.app.add_pool(**self.POOL_CONFIG)
 
     def tearDown(self):
         """ Remove the file based storage pool after testing """
         self.app.remove_pool("test-pool")
+        self.app.cleanup()
         super(TC_03_FilePool, self).tearDown()
         shutil.rmtree(self.POOL_DIR, ignore_errors=True)
+        if os.path.exists('/tmp/qubes-test'):
+            shutil.rmtree('/tmp/qubes-test')
+        qubes.config.system_path['qubes_base_dir'] = self._orig_qubes_base_dir
+
 
     def test_001_pool_exists(self):
         """ Check if the storage pool was added to the storage pool config """

qubes/tests/vm/mix/net.py

@@ -35,6 +35,7 @@ class TC_00_NetVMMixin(
     def setUp(self):
         super(TC_00_NetVMMixin, self).setUp()
         self.app = qubes.tests.vm.TestApp()
+        self.app.vmm.offline_mode = True
 
     def setup_netvms(self, vm):
         # usage of QubesVM here means that those tests should be after

qubes/tests/vm/qubesvm.py

@@ -146,6 +146,7 @@ class QubesVMTestsMixin(object):
     def setUp(self):
         super(QubesVMTestsMixin, self).setUp()
         self.app = qubes.tests.vm.TestApp()
+        self.app.vmm.offline_mode = True
 
     def get_vm(self, **kwargs):
         return qubes.vm.qubesvm.QubesVM(self.app, None,

qubes/utils.py

@@ -87,8 +87,6 @@ def format_doc(docstring):
         config_section=None, enable_exit_status=None)
     return pub.writer.document.astext()
 
-# FIXME those are wrong, k/M/G are SI prefixes and means 10**3
-# maybe adapt https://code.activestate.com/recipes/578019
 def parse_size(size):
     units = [
         ('K', 1000), ('KB', 1000),