Refactor Storage, Pool and XenPool

- Remove all *_dev_config methods
 - Checks if a storage image exists moved to XenPool
 - Storage.remove wraps Pool.remove()
 - Stop volumes on domain sutdown/kill
 - Warn when using deprecated methods

qubes/storage/__init__.py

@@ -39,7 +39,6 @@ import qubes.exc
 import qubes.utils
 from qubes.devices import BlockDevice
 
-BLKSIZE = 512
 STORAGE_ENTRY_POINT = 'qubes.storage'
 
 
@@ -93,91 +92,20 @@ class Storage(object):
     in mind.
     '''
 
-    modules_dev = 'xvdd'
-
     def __init__(self, vm):
         #: Domain for which we manage storage
         self.vm = vm
+        self.log = self.vm.log
         #: Additional drive (currently used only by HVM)
         self.drive = None
-        for name, conf in self.vm.volume_config.items():
+        self.pools = {}
-            assert 'pool' in conf
+        if hasattr(vm, 'volume_config'):
-            pool = get_pool(conf['pool'], self.vm)
+            for name, conf in self.vm.volume_config.items():
-            self.vm.volumes[name] = pool.init_volume(conf)
+                assert 'pool' in conf, "Pool missing in volume_config" % str(
-
+                    conf)
-    @property
+                pool = self.vm.app.get_pool(conf['pool'])
-    def root_img(self):
+                self.vm.volumes[name] = pool.init_volume(self.vm, conf)
-        pool = self.get_pool()
+                self.pools[name] = pool
-        return pool.root_img
-
-    @property
-    def private_img(self):
-        pool = self.get_pool()
-        return pool.private_img
-
-    @property
-    def volatile_img(self):
-        pool = self.get_pool()
-        return pool.volatile_img
-
-    @property
-    def rootcow_img(self):
-        pool = self.get_pool()
-        return pool.rootcow_img
-
-    def get_config_params(self):
-        args = {}
-        args['rootdev'] = self.root_dev_config()
-        args['privatedev'] = self.private_dev_config()
-        args['volatiledev'] = self.volatile_dev_config()
-        args['otherdevs'] = self.other_dev_config()
-
-        args['kerneldir'] = self.kernels_dir
-
-        return args
-
-    def root_dev_config(self):
-        pool = self.get_pool()
-        return pool.root_dev_config()
-
-    def private_dev_config(self):
-        pool = self.get_pool()
-        return pool.private_dev_config()
-
-    def volatile_dev_config(self):
-        pool = self.get_pool()
-        return pool.volatile_dev_config()
-
-    def modules_dev_config(self):
-        return self.format_disk_dev(self.modules_img,
-                                    'kernel',
-                                    rw=self.modules_img_rw)
-
-    def other_dev_config(self):
-        if self.modules_img is not None:
-            return self.modules_dev_config()
-        elif self.drive is not None:
-            (drive_type, drive_domain, drive_path) = self.drive.split(":")
-            if drive_type == 'hd':
-                drive_type = 'disk'
-
-            rw = (drive_type == 'disk')
-
-            if drive_domain.lower() == "dom0":
-                drive_domain = None
-
-            return self.format_disk_dev(drive_path,
-                                        'other',
-                                        rw=rw,
-                                        devtype=drive_type,
-                                        domain=drive_domain)
-
-        else:
-            return ''
-
-    def format_disk_dev(self, path, name, script=None, rw=True, devtype='disk',
-                        domain=None):
-        return BlockDevice(path, name, script, rw, domain, devtype)
 
     @property
     def kernels_dir(self):
@@ -186,51 +114,33 @@ class Storage(object):
         If :py:attr:`self.vm.kernel` is :py:obj:`None`, the this points inside
         :py:attr:`self.vm.dir_path`
         '''
-        return os.path.join(qubes.config.system_path['qubes_base_dir'],
+        assert 'kernel' in self.vm.volumes, "VM has no kernel pool"
-            qubes.config.system_path['qubes_kernels_base_dir'], self.vm.kernel)\
+        return self.vm.volumes['kernel'].kernels_dir
-            if self.vm.kernel is not None \
-        else os.path.join(self.vm.dir_path,
-            qubes.config.vm_files['kernels_subdir'])
-
-    @property
-    def modules_img(self):
-        '''Path to image with modules.
-
-        Depending on domain, this may be global or inside domain's dir.
-        '''
-
-        modules_path = os.path.join(self.kernels_dir, 'modules.img')
-
-        if os.path.exists(modules_path):
-            return modules_path
-        else:
-            return None
-
-
-    @property
-    def modules_img_rw(self):
-        ''':py:obj:`True` if module image should be mounted RW, :py:obj:`False`
-        otherwise.'''
-        return self.vm.kernel is None
-
 
     def get_disk_utilization(self):
-        return get_disk_usage(self.vm.dir_path)
+        ''' Returns summed up disk utilization for all domain volumes '''
+        result = 0
+        for volume in self.vm.volumes.values():
+            result += volume.usage
+        return result
 
+    # TODO Remove this wrapper
     def get_disk_utilization_private_img(self):
-        # pylint: disable=invalid-name
+        # pylint: disable=invalid-name,missing-docstring
-        return get_disk_usage(self.private_img)
+        return self.vm.volume['private'].usage
 
+    # TODO Remove this wrapper
     def get_private_img_sz(self):
-        if not os.path.exists(self.private_img):
+        # :pylint: disable=missing-docstring
-            return 0
+        return self.vm.volume['private'].size
 
-        return os.path.getsize(self.private_img)
+    def resize(self, volume, size):
-
+        ''' Resize volume '''
-    def resize_private_img(self, size):
+        self.get_pool(volume).resize(volume, size)
-        raise NotImplementedError()
 
+    # TODO rename it to create()
     def create_on_disk(self, source_template=None):
+        # :pylint: disable=missing-docstring
         if source_template is None and hasattr(self.vm, 'template'):
             source_template = self.vm.template
 
@@ -238,14 +148,17 @@ class Storage(object):
 
         self.vm.log.info('Creating directory: {0}'.format(self.vm.dir_path))
         os.makedirs(self.vm.dir_path)
-        pool = self.get_pool()
+        for name, volume in self.vm.volumes.items():
-        pool.create_on_disk_private_img(source_template)
+            source_volume = None
-        pool.create_on_disk_root_img(source_template)
+            if source_template and hasattr(source_template, 'volumes'):
-        pool.reset_volatile_storage()
+                source_volume = source_template.volumes[name]
+            self.get_pool(volume).create(volume, source_volume=source_volume)
 
         os.umask(old_umask)
 
+    # TODO migrate this
     def clone_disk_files(self, src_vm):
+        # :pylint: disable=missing-docstring
         self.vm.log.info('Creating directory: {0}'.format(self.vm.dir_path))
         os.mkdir(self.vm.dir_path)
 
@@ -255,14 +168,15 @@ class Storage(object):
             self._copy_file(src_vm.private_img, self.vm.private_img)
 
         if src_vm.updateable and hasattr(src_vm, 'root_img'):
-            self.vm.log.info('Copying the root image: {} -> {}'.format(
+            self.vm.log.info(
-                src_vm.root_img, self.root_img))
+                'Copying the root image: {} -> {}'.format(
-            self._copy_file(src_vm.root_img, self.root_img)
+                    src_vm.volume['root'].path_origin,
-
+                    self.vm.volume['root'].path_origin)
-            # TODO: modules?
+            )
-            # XXX which modules? -woju
+            self._copy_file(src_vm.volume['root'].path_origin,
-
+                            self.vm.volume['root'].path_origin)
 
+    # TODO migrate this
     @staticmethod
     def rename(newpath, oldpath):
         '''Move storage directory, most likely during domain's rename.
@@ -280,133 +194,87 @@ class Storage(object):
 
         os.rename(oldpath, newpath)
 
-
     def verify_files(self):
+        '''Verify that the storage is sane.
+
+        On success, returns normally. On failure, raises exception.
+        '''
         if not os.path.exists(self.vm.dir_path):
-            raise qubes.exc.QubesVMError(self.vm,
+            raise qubes.exc.QubesVMError(
+                self.vm,
                 'VM directory does not exist: {}'.format(self.vm.dir_path))
 
-        if hasattr(self.vm, 'root_img') and not os.path.exists(self.root_img):
+    def remove(self):
-            raise qubes.exc.QubesVMError(self.vm,
+        for name, volume in self.vm.volumes.items():
-                'VM root image file does not exist: {}'.format(self.root_img))
+            self.log.info('Removing volume %s: %s' % (name, volume.vid))
-
+            self.get_pool(volume).remove(volume)
-        if hasattr(self.vm, 'private_img') \
-                and not os.path.exists(self.private_img):
-            raise qubes.exc.QubesVMError(self.vm,
-                'VM private image file does not exist: {}'.format(
-                    self.private_img))
-
-        if self.modules_img is not None \
-                and not os.path.exists(self.modules_img):
-            raise qubes.exc.QubesVMError(self.vm,
-                'VM kernel modules image does not exists: {}'.format(
-                    self.modules_img))
-
-
-    def remove_from_disk(self):
         shutil.rmtree(self.vm.dir_path)
 
+    def start(self):
+        ''' Execute the start method on each pool '''
+        for volume in self.vm.volumes.values():
+            self.get_pool(volume).start(volume)
 
+    def stop(self):
+        ''' Execute the start method on each pool '''
+        for volume in self.vm.volumes.values():
+            self.get_pool(volume).stop(volume)
 
-    def prepare_for_vm_startup(self):
+    def get_pool(self, volume):
-        pool = get_pool(self.vm.pool_name, self.vm)
+        ''' Helper function '''
-        pool.reset_volatile_storage()
+        assert isinstance(volume, Volume), "You need to pass a Volume"
-
+        return self.pools[volume.name]
-        if hasattr(self.vm, 'private_img') \
-                and not os.path.exists(self.private_img):
-            self.vm.log.info('Creating empty VM private image file: {0}'.format(
-                pool.private_img))
-            pool.create_on_disk_private_img()
-
-    def get_pool(self):
-        return get_pool(self.vm.pool_name, self.vm)
 
     def commit_template_changes(self):
-        pool = self.get_pool()
+        for volume in self.vm.volumes.values():
-        pool.commit_template_changes()
+            if volume.volume_type == 'origin':
+                self.get_pool(volume).commit_template_changes(volume)
 
-def get_disk_usage_one(st):
-    '''Extract disk usage of one inode from its stat_result struct.
-
-    If known, get real disk usage, as written to device by filesystem, not
-    logical file size. Those values may be different for sparse files.
-
-    :param os.stat_result st: stat result
-    :returns: disk usage
-    '''
-    try:
-        return st.st_blocks * BLKSIZE
-    except AttributeError:
-        return st.st_size
-
-
-def get_disk_usage(path):
-    '''Get real disk usage of given path (file or directory).
-
-    When *path* points to directory, then it is evaluated recursively.
-
-    This function tries estiate real disk usage. See documentation of
-    :py:func:`get_disk_usage_one`.
-
-    :param str path: path to evaluate
-    :returns: disk usage
-    '''
-    try:
-        st = os.lstat(path)
-    except OSError:
-        return 0
-
-    ret = get_disk_usage_one(st)
-
-    # if path is not a directory, this is skipped
-    for dirpath, dirnames, filenames in os.walk(path):
-        for name in dirnames + filenames:
-            ret += get_disk_usage_one(os.lstat(os.path.join(dirpath, name)))
-
-    return ret
 
 class Pool(object):
+    ''' A Pool is used to manage different kind of volumes (File
+        based/LVM/Btrfs/...).
+
+        3rd Parties providing own storage implementations will need to extend
+        this class.
+    '''
     private_img_size = qubes.config.defaults['private_img_size']
     root_img_size = qubes.config.defaults['root_img_size']
 
-    def __init__(self, vm=None, name=None, **kwargs):
+    def __init__(self, name=None, **kwargs):
-        assert vm
+        # :pylint: disable=unused-argument
         assert name, "Pool name is missing"
-        self.vm = vm
         self.name = name
 
-    def root_dev_config(self):
+    def create(self, volume, source_volume):
-        raise NotImplementedError()
+        ''' Create the given volume on disk or copy from provided
+            `source_volume`.
+        '''
+        raise NotImplementedError("Pool %s has create() not implemented" %
+                                  self.name)
 
-    def private_dev_config(self):
+    def commit_template_changes(self, volume):
-        raise NotImplementedError()
+        ''' Update origin device '''
+        raise NotImplementedError(
+            "Pool %s has commit_template_changes() not implemented" %
+            self.name)
 
-    def volatile_dev_config(self):
+    @property
-        raise NotImplementedError()
+    def config(self):
+        ''' Returns the pool config to be written to qubes.xml '''
+        raise NotImplementedError("Pool %s has config() not implemented" %
+                                  self.name)
 
-    def create_on_disk_private_img(self, source_template=None):
-        raise NotImplementedError()
-
-    def create_on_disk_root_img(self, source_template=None):
-        raise NotImplementedError()
-
-    def create_dir_if_not_exists(self, path):
-        """ Check if a directory exists in if not create it.
-
-            This method does not create any parent directories.
-        """
-        if not os.path.exists(path):
-            os.mkdir(path)
-
-    def commit_template_changes(self):
-        raise NotImplementedError()
     @staticmethod
     def _copy_file(source, destination):
         '''Effective file copy, preserving sparse files etc.
         '''
         # TODO: Windows support
-
         # We prefer to use Linux's cp, because it nicely handles sparse files
+        assert os.path.exists(source), \
+            "Missing the source %s to copy from" % source
+        assert not os.path.exists(destination), \
+            "Destination %s already exists" % destination
         try:
             subprocess.check_call(['cp', '--reflink=auto', source, destination
                                    ])
@@ -414,34 +282,34 @@ class Pool(object):
             raise IOError('Error while copying {!r} to {!r}'.format(
                 source, destination))
 
-    def format_disk_dev(self, path, vdev, script=None, rw=True, devtype='disk',
+    def remove(self, volume):
-                        domain=None):
+        ''' Remove volume'''
+        raise NotImplementedError("Pool %s has remove() not implemented" %
+                                  self.name)
 
-        return BlockDevice(path, vdev, script, rw, domain, devtype)
+    def clone(self, source, target):
-    def reset_volatile_storage(self):
+        ''' Clone volume '''
-        # Re-create only for template based VMs
+        raise NotImplementedError("Pool %s has clone() not implemented" %
-        try:
+                                  self.name)
-            if self.vm.template is not None and self.volatile_img:
-                if os.path.exists(self.volatile_img):
-                    os.remove(self.volatile_img)
-        except AttributeError: # self.vm.template
-            pass
 
-        # For StandaloneVM create it only if not already exists
+    def start(self, volume):
-        # (eg after backup-restore)
+        ''' Do what ever is needed on start '''
-        if hasattr(self, 'volatile_img') \
+        raise NotImplementedError("Pool %s has start() not implemented" %
-                and not os.path.exists(self.volatile_img):
+                                  self.name)
-            self.vm.log.info(
+
-                'Creating volatile image: {0}'.format(self.volatile_img))
+    def stop(self, volume):
-            subprocess.check_call(
+        ''' Do what ever is needed on stop'''
-                [qubes.config.system_path["prepare_volatile_img_cmd"],
+        raise NotImplementedError("Pool %s has stop() not implemented" %
-                    self.volatile_img,
+                                  self.name)
-                    str(self.root_img_size / 1024 / 1024)])
 
     def init_volume(self, volume_config):
-        raise NotImplementedError()
+        ''' Initialize a :py:class:`qubes.storage.Volume` from `volume_config`.
+        '''
+        raise NotImplementedError("Pool %s has init_volume() not implemented" %
+                                  self.name)
+
 
 def pool_drivers():
     """ Return a list of EntryPoints names """
     return [ep.name
-        for ep in pkg_resources.iter_entry_points(STORAGE_ENTRY_POINT)]
+            for ep in pkg_resources.iter_entry_points(STORAGE_ENTRY_POINT)]

qubes/storage/xen.py

@@ -31,137 +31,70 @@ import os.path
 import re
 import subprocess
 
-import qubes
-import qubes.config
-import qubes.vm.templatevm
 from qubes.storage import Pool, StoragePoolException, Volume
 
+BLKSIZE = 512
+
 
 class XenPool(Pool):
+    ''' File based 'original' disk implementation '''
 
-    root_dev = 'xvda'
+    def __init__(self, name=None, dir_path=None):
-    private_dev = 'xvdb'
+        super(XenPool, self).__init__(name=name)
-    volatile_dev = 'xvdc'
-
-    def __init__(self, vm=None, name=None, dir_path=None):
-        super(XenPool, self).__init__(vm=vm, name=name)
         assert dir_path, "No pool dir_path specified"
         self.dir_path = os.path.normpath(dir_path)
 
-        self.create_dir_if_not_exists(self.dir_path)
+        create_dir_if_not_exists(self.dir_path)
         appvms_path = os.path.join(self.dir_path, 'appvms')
-        self.create_dir_if_not_exists(appvms_path)
+        create_dir_if_not_exists(appvms_path)
         vm_templates_path = os.path.join(self.dir_path, 'vm-templates')
-        self.create_dir_if_not_exists(vm_templates_path)
+        create_dir_if_not_exists(vm_templates_path)
 
-    @property
+    def create(self, volume, source_volume=None):
-    def private_img(self):
+        _type = volume.volume_type
-        '''Path to the private image'''
+        size = volume.size
-        return self.abspath(qubes.config.vm_files['private_img'])
+        if _type == 'origin':
+            create_sparse_file(volume.path_origin, size)
+            create_sparse_file(volume.path_cow, size)
+        elif _type in ['read-write'] and source_volume:
+            copy_file(source_volume.path, volume.path)
+        elif _type in ['read-write', 'volatile']:
+            create_sparse_file(volume.path, size)
 
-    @property
+        return volume
-    def root_img(self):
-        '''Path to the root image'''
-        return self.vm.template.storage.root_img \
-            if hasattr(self.vm, 'template') and self.vm.template \
-            else self.abspath(qubes.config.vm_files['root_img'])
 
-    @property
+    def resize(self, volume, size):
-    def rootcow_img(self):
+        ''' Expands volume, throws
-        '''Path to the root COW image'''
+            :py:class:`qubst.storage.StoragePoolException` if given size is
+            less than current_size
+        '''
+        _type = volume.volume_type
+        if _type not in ['origin', 'read-write', 'volatile']:
+            raise StoragePoolException('Can not resize a %s volume %s' %
+                                       (_type, volume.vid))
 
-        if isinstance(self.vm, qubes.vm.templatevm.TemplateVM):
+        if size <= volume.size:
-            return self.abspath(qubes.config.vm_files['rootcow_img'])
+            raise StoragePoolException(
+                'For your own safety, shrinking of %s is'
+                ' disabled. If you really know what you'
+                ' are doing, use `truncate` on %s manually.' %
+                (volume.name, volume.vid))
 
-        return None
+        if _type == 'origin':
+            path = volume.path_origin
+        elif _type in ['read-write', 'volatile']:
+            path = volume.path
 
-    @property
+        if size <= volume.size:
-    def volatile_img(self):
+            raise StoragePoolException('Can not shring volume %s' %
-        '''Path to the volatile image'''
+                                       volume.name)
-        return self.abspath(qubes.config.vm_files['volatile_img'])
 
-    def root_dev_config(self):
+        with open(path, 'a+b') as fd:
-        dev_name = 'root'
+            fd.truncate(size)
-        if isinstance(self.vm, qubes.vm.templatevm.TemplateVM):
-            return self.format_disk_dev(
-                '{root}:{rootcow}'.format(
-                    root=self.root_img,
-                    rootcow=self.rootcow_img),
-                dev_name,
-                script='block-origin')
-
-        elif self.vm.hvm and hasattr(self.vm, 'template'):
-            # HVM template-based VM - only one device-mapper layer, in dom0
-            # (root+volatile)
-            # HVM detection based on 'kernel' property is massive hack,
-            # but taken from assumption that VM needs Qubes-specific kernel
-            # (actually initramfs) to assemble the second layer of device-mapper
-            return self.format_disk_dev(
-                '{root}:{volatile}'.format(
-                    root=self.vm.template.storage.root_img,
-                    volatile=self.volatile_img),
-                dev_name,
-                script='block-snapshot')
-
-        elif hasattr(self.vm, 'template'):
-            # any other template-based VM - two device-mapper layers: one
-            # in dom0 (here) from root+root-cow, and another one from
-            # this+volatile.img
-            path = '{root}:{template_rootcow}'.format(
-                root=self.root_img,
-                template_rootcow=self.vm.template.storage.rootcow_img)
-            return self.format_disk_dev(path=path,
-                                        vdev=self.root_dev,
-                                        script='block-snapshot',
-                                        rw=False)
-
-        else:
-            # standalone qube
-            return self.format_disk_dev(self.root_img, dev_name)
-
-    def private_dev_config(self):
-        return self.format_disk_dev(self.private_img, 'private')
-
-    def volatile_dev_config(self):
-        return self.format_disk_dev(self.volatile_img, 'volatile')
-
-    def create_on_disk_private_img(self, source_template=None):
-        if not os.path.exists(self.target_dir):
-            os.makedirs(self.target_dir)
-        if source_template is None:
-            f_private = open(self.private_img, 'a+b')
-            f_private.truncate(self.private_img_size)
-            f_private.close()
-
-        else:
-            self.vm.log.info("Copying the template's private image: {}".format(
-                source_template.storage.private_img))
-            self._copy_file(source_template.storage.private_img, self.private_img)
-
-    def create_on_disk_root_img(self, source_template=None):
-        if not os.path.exists(self.target_dir):
-            os.makedirs(self.target_dir)
-        if source_template is None:
-            fd = open(self.root_img, 'a+b')
-            fd.truncate(self.root_img_size)
-            fd.close()
-
-        elif self.vm.updateable:
-            # if not updateable, just use template's disk
-            self.vm.log.info(
-                "--> Copying the template's root image: {}".format(
-                    source_template.storage.root_img))
-            self._copy_file(source_template.storage.root_img, self.root_img)
-
-    def resize_private_img(self, size):
-        fd = open(self.private_img, 'a+b')
-        fd.truncate(size)
-        fd.close()
 
         # find loop device if any
-        p = subprocess.Popen(
+        p = subprocess.Popen(['sudo', 'losetup', '--associated', path],
-            ['sudo', 'losetup', '--associated', self.private_img],
+                             stdout=subprocess.PIPE)
-            stdout=subprocess.PIPE)
         result = p.communicate()
 
         m = re.match(r'^(/dev/loop\d+):\s', result[0])
@@ -172,106 +105,46 @@ class XenPool(Pool):
             subprocess.check_call(['sudo', 'losetup', '--set-capacity',
                                    loop_dev])
 
-    def commit_template_changes(self):
+    def commit_template_changes(self, volume):
-        assert isinstance(self.vm, qubes.vm.templatevm.TemplateVM)
+        if volume.volume_type != 'origin':
+            return volume
 
-        # TODO: move rootcow_img to this class; the same for vm.is_outdated()
+        if os.path.exists(volume.path_cow):
-        if os.path.exists(self.vm.rootcow_img):
+            os.rename(volume.path_cow, volume.path_cow + '.old')
-            os.rename(self.vm.rootcow_img, self.vm.rootcow_img + '.old')
 
         old_umask = os.umask(002)
-        f_cow = open(self.vm.rootcow_img, 'w')
+        with open(volume.path_cow, 'w') as f_cow:
-        f_root = open(self.root_img, 'r')
+            f_cow.truncate(volume.size)
-        f_root.seek(0, os.SEEK_END)
-        # make empty sparse file of the same size as root.img
-        f_cow.truncate(f_root.tell())
-        f_cow.close()
-        f_root.close()
         os.umask(old_umask)
         return volume
 
     def start(self, volume):
         if volume.volume_type == 'volatile':
             self._reset_volume(volume)
+        if volume.volume_type in ['origin', 'snapshot']:
+            _check_path(volume.path_origin)
+            _check_path(volume.path_cow)
+        else:
+            _check_path(volume.path)
+
         return volume
 
+    def stop(self, volume):
+        pass
+
     def _reset_volume(self, volume):
         ''' Remove and recreate a volatile volume '''
         assert volume.volume_type == 'volatile', "Not a volatile volume"
 
-        size = self.vm.volume_config[volume.name]['size']
+        assert volume.size
-        assert size
 
-        if os.path.exists(volume.path):
+        _remove_if_exists(volume)
-            os.remove(volume.path)
 
         with open(volume.path, "w") as f_volatile:
             f_volatile.truncate(volume.size)
         return volume
 
-    def reset_volatile_storage(self):
+    def target_dir(self, vm):
-        try:
-            # no template set, in any way (Standalone VM, Template VM)
-            if self.vm.template is None:
-                raise AttributeError
-
-            # template-based HVM with only one device-mapper layer -
-            # volatile.img used as upper layer on root.img, no root-cow.img
-            # intermediate layer
-            if self.vm.hvm:
-                if os.path.exists(self.volatile_img):
-                    if self.vm.debug:
-                        if os.path.getmtime(self.vm.template.storage.root_img) \
-                                > os.path.getmtime(self.volatile_img):
-                            self.vm.log.warning(
-                                'Template have changed, resetting root.img')
-                        else:
-                            self.vm.log.warning(
-                                'Debug mode: not resetting root.img; if you'
-                                ' want to force root.img reset, either'
-                                ' update template VM, or remove volatile.img'
-                                ' file.')
-                            return
-
-                    os.remove(self.volatile_img)
-
-                # FIXME stat on f_root; with open() ...
-                f_volatile = open(self.volatile_img, "w")
-                f_root = open(self.vm.template.storage.root_img, "r")
-                # make empty sparse file of the same size as root.img
-                f_root.seek(0, os.SEEK_END)
-                f_volatile.truncate(f_root.tell())
-                f_volatile.close()
-                f_root.close()
-                return  # XXX why is that? super() does not run
-        except AttributeError:  # self.vm.template
-            pass
-
-        super(XenPool, self).reset_volatile_storage()
-
-    def prepare_for_vm_startup(self):
-        super(XenPool, self).prepare_for_vm_startup()
-
-        if self.drive is not None:
-            # pylint: disable=unused-variable
-            (drive_type, drive_domain, drive_path) = self.drive.split(":")
-
-            if drive_domain.lower() != "dom0":
-                # XXX "VM '{}' holding '{}' does not exists".format(
-                drive_vm = self.vm.app.domains[drive_domain]
-
-                if not drive_vm.is_running():
-                    raise qubes.exc.QubesVMNotRunningError(
-                        drive_vm, 'VM {!r} holding {!r} isn\'t running'.format(
-                            drive_domain, drive_path))
-
-        if self.rootcow_img and not os.path.exists(self.rootcow_img):
-            self.commit_template_changes()
-
-    # XXX there is also a class attribute on the domain classes which does
-    # exactly that -- which one should prevail?
-    @property
-    def target_dir(self):
         """ Returns the path to vmdir depending on the type of the VM.
 
             The default QubesOS file storage saves the vm images in three
@@ -288,7 +161,6 @@ class XenPool(Pool):
                 string (str) absolute path to the directory where the vm files
                              are stored
         """
-        vm = self.vm
         if vm.is_template():
             subdir = 'vm-templates'
         elif vm.is_disposablevm():
@@ -300,16 +172,10 @@ class XenPool(Pool):
 
         return os.path.join(self.dir_path, subdir, vm.name)
 
-    def abspath(self, file_name):
+    def init_volume(self, vm, volume_config):
-        return os.path.join(self.target_dir, file_name)
-
-    def init_volume(self, volume_config):
         assert 'volume_type' in volume_config, "Volume type missing " \
             + str(volume_config)
-        target_dir = self.target_dir
-        assert target_dir, "Pool target_dir not set"
         volume_type = volume_config['volume_type']
-        volume_config['target_dir'] = target_dir
         known_types = {
             'read-write': ReadWriteFile,
             'read-only': ReadOnlyFile,
@@ -320,125 +186,124 @@ class XenPool(Pool):
         if volume_type not in known_types:
             raise StoragePoolException("Unknown volume type " + volume_type)
 
-        if volume_type == 'snapshot':
+        if volume_type in ['snapshot', 'read-only']:
-            path = qubes.storage.get_pool(volume_config['pool'], self.vm.template).target_dir
+            origin_pool = vm.app.get_pool(volume_config['pool'])
-            volume_config['vid'] = os.path.join(path, volume_config['name'] + '.img')
+            assert isinstance(origin_pool,
+                              XenPool), 'Origin volume not a xen volume'
+            volume_config['target_dir'] = origin_pool.target_dir(vm.template)
+            name = volume_config['name']
+            volume_config['size'] = vm.template.volume_config[name]['size']
+        else:
+            volume_config['target_dir'] = self.target_dir(vm)
 
         return known_types[volume_type](**volume_config)
 
 
-class SizeMixIn(Volume):
+class XenVolume(Volume):
+    ''' Parent class for the xen volumes implementation '''
 
+    def __init__(self, target_dir, **kwargs):
+        self.target_dir = target_dir
+        assert self.target_dir, "target_dir not specified"
+        super(XenVolume, self).__init__(**kwargs)
+
+
+class SizeMixIn(XenVolume):
+    ''' A mix in which expects a `size` param to be > 0 on initialization and
+        provides a usage property wrapper.
+    '''
     def __init__(self, name=None, pool=None, vid=None, target_dir=None, size=0,
                  **kwargs):
         assert size > 0, 'Size for volume ' + name + ' is <=0'
         super(SizeMixIn, self).__init__(name=name,
                                         pool=pool,
                                         vid=vid,
+                                        size=size,
                                         **kwargs)
-        self._size = size
         self.target_dir = target_dir
 
     @property
-    def size(self):
+    def usage(self):
-        if self.vid and os.path.exists(self.vid):
+        ''' Returns the actualy used space '''
-            return qubes.storage.get_disk_usage(self.vid)
+        return get_disk_usage(self.vid)
-        else:
+
-            return self._size
 
 
 class ReadWriteFile(SizeMixIn):
-    # :pylint: disable=too-few-public-methods
+    # :pylint: disable=missing-docstring
     def __init__(self, **kwargs):
         super(ReadWriteFile, self).__init__(**kwargs)
         self.path = os.path.join(self.target_dir, self.name + '.img')
         self.vid = self.path
 
-    @property
-    def size(self):
-        if self.vid and os.path.exists(self.vid):
-            return qubes.storage.get_disk_usage(self.vid)
-        else:
-            return self._size
-
-    def create(self):
-        create_file(self.path, self.size)
-
-    @property
-    def created(self):
-        return os.path.exists(self.path)
-
 
 class ReadOnlyFile(Volume):
+    # :pylint: disable=missing-docstring
+    usage = 0
 
     def __init__(self, name=None, pool=None, vid=None, target_dir=None,
-                 **kwargs):
+                 size=0, **kwargs):
+        # :pylint: disable=unused-argument
         assert os.path.exists(vid), "read-only volume missing vid"
         super(ReadOnlyFile, self).__init__(name=name,
-                                         pool=pool,
+                                           pool=pool,
-                                         vid=vid,
+                                           vid=vid,
-                                         **kwargs)
+                                           size=size,
+                                           **kwargs)
         self.path = self.vid
 
-    @property
-    def size(self):
-        return qubes.storage.get_disk_usage(self.vid)
-
 
 class OriginFile(SizeMixIn):
+    # :pylint: disable=missing-docstring
     script = 'block-origin'
 
     def __init__(self, **kwargs):
         super(OriginFile, self).__init__(**kwargs)
-        self.path = os.path.join(self.target_dir, self.name + '.img')
+        self.path_origin = os.path.join(self.target_dir, self.name + '.img')
         self.path_cow = os.path.join(self.target_dir, self.name + '-cow.img')
-        self.vid = self.path
+        self.path = '%s:%s' % (self.path_origin, self.path_cow)
-
+        self.vid = self.path_origin
-    def create(self):
-        create_file(self.path, self.size)
-        create_file(self.path_cow, self.size)
 
     def commit(self):
         raise NotImplementedError
 
     @property
-    def size(self):
+    def usage(self):
-        if self.vid and os.path.exists(self.vid):
+        result = 0
-            return qubes.storage.get_disk_usage(self.vid)
+        if os.path.exists(self.path_origin):
-        else:
+            result += get_disk_usage(self.path_origin)
-            return self._size
+        if os.path.exists(self.path_cow):
-
+            result += get_disk_usage(self.path_cow)
-    @property
+        return result
-    def created(self):
-        return os.path.exists(self.path) and os.path.exists(self.path_cow)
 
 
 class SnapshotFile(Volume):
-    # :pylint: disable=too-few-public-methods
+    # :pylint: disable=missing-docstring
     script = 'block-snapshot'
     rw = False
+    usage = 0
 
     def __init__(self, name=None, pool=None, vid=None, target_dir=None,
-                 **kwargs):
+                 size=None, **kwargs):
-        assert vid, "SnapshotVolume missing a vid to OriginVolume"
+        assert size
-        assert os.path.exists(vid), "OriginVolume does not exist"
         super(SnapshotFile, self).__init__(name=name,
-                                         pool=pool,
+                                           pool=pool,
-                                         vid=vid,
+                                           vid=vid,
-                                         **kwargs)
+                                           size=size,
-        self.path = os.path.join(target_dir, name + '.img')
+                                           **kwargs)
+        self.path_origin = os.path.join(target_dir, name + '.img')
         self.path_cow = os.path.join(target_dir, name + '-cow.img')
+        self.path = '%s:%s' % (self.path_origin, self.path_cow)
+        self.vid = self.path_origin
 
     @property
     def created(self):
-        return os.path.exists(self.path) and os.path.exists(self.path_cow)
+        return os.path.exists(self.path_origin) and os.path.exists(
-
+            self.path_cow)
-    @property
-    def size(self):
-        return qubes.storage.get_disk_usage(self.vid)
 
 
 class VolatileFile(SizeMixIn):
+    # :pylint: disable=missing-docstring
 
     def __init__(self, **kwargs):
         super(VolatileFile, self).__init__(**kwargs)
@@ -446,8 +311,94 @@ class VolatileFile(SizeMixIn):
         self.vid = self.path
 
 
-def create_file(path, size):
+def create_sparse_file(path, size):
+    ''' Create an empty sparse file '''
     if os.path.exists(path):
         raise IOError("Volume %s already exists", path)
+    parent_dir = os.path.dirname(path)
+    if not os.path.exists(parent_dir):
+        os.makedirs(parent_dir)
     with open(path, 'a+b') as fh:
         fh.truncate(size)
+
+
+def get_disk_usage_one(st):
+    '''Extract disk usage of one inode from its stat_result struct.
+
+    If known, get real disk usage, as written to device by filesystem, not
+    logical file size. Those values may be different for sparse files.
+
+    :param os.stat_result st: stat result
+    :returns: disk usage
+    '''
+    try:
+        return st.st_blocks * BLKSIZE
+    except AttributeError:
+        return st.st_size
+
+
+def get_disk_usage(path):
+    '''Get real disk usage of given path (file or directory).
+
+    When *path* points to directory, then it is evaluated recursively.
+
+    This function tries estiate real disk usage. See documentation of
+    :py:func:`get_disk_usage_one`.
+
+    :param str path: path to evaluate
+    :returns: disk usage
+    '''
+    try:
+        st = os.lstat(path)
+    except OSError:
+        return 0
+
+    ret = get_disk_usage_one(st)
+
+    # if path is not a directory, this is skipped
+    for dirpath, dirnames, filenames in os.walk(path):
+        for name in dirnames + filenames:
+            ret += get_disk_usage_one(os.lstat(os.path.join(dirpath, name)))
+
+    return ret
+
+
+def create_dir_if_not_exists(path):
+    """ Check if a directory exists in if not create it.
+
+        This method does not create any parent directories.
+    """
+    if not os.path.exists(path):
+        os.mkdir(path)
+
+
+def copy_file(source, destination):
+    '''Effective file copy, preserving sparse files etc.
+    '''
+    # TODO: Windows support
+    # We prefer to use Linux's cp, because it nicely handles sparse files
+    assert os.path.exists(source), \
+        "Missing the source %s to copy from" % source
+    assert not os.path.exists(destination), \
+        "Destination %s already exists" % destination
+
+    parent_dir = os.path.dirname(destination)
+    if not os.path.exists(parent_dir):
+        os.makedirs(parent_dir)
+
+    try:
+        subprocess.check_call(['cp', '--reflink=auto', source, destination])
+    except subprocess.CalledProcessError:
+        raise IOError('Error while copying {!r} to {!r}'.format(source,
+                                                                destination))
+
+
+def _remove_if_exists(volume):
+    if os.path.exists(volume.path):
+        os.remove(volume.path)
+
+
+def _check_path(path):
+    ''' Raise an StoragePoolException if ``path`` does not exist'''
+    if not os.path.exists(path):
+        raise StoragePoolException('Missing image file: %s' % path)

qubes/vm/qubesvm.py

@@ -333,10 +333,10 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
 
     @property
     def block_devices(self):
-        return [self.storage.root_dev_config(),
+        ''' Return all :py:class:`qubes.devices.BlockDevice`s for current domain
-                self.storage.private_dev_config(),
+            for serialization in the libvirt XML template as <disk>.
-                self.storage.volatile_dev_config(),
+        '''
-                self.storage.other_dev_config()]
+        return [v.block_device() for v in self.volumes.values()]
 
     @property
     def qdb(self):
@@ -353,21 +353,27 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
     def private_img(self):
         '''Location of private image of the VM (that contains :file:`/rw` \
         and :file:`/home`).'''
-        return self.storage.private_img
+        warnings.warn("volatile_img is deprecated, use volumes['private'].vid",
+                      DeprecationWarning)
+        return self.volumes['private'].vid
 
 
     # XXX this should go to to AppVM? or TemplateVM?
     @property
     def root_img(self):
         '''Location of root image.'''
-        return self.storage.root_img
+        warnings.warn("root_img is deprecated, use volumes['root'].vid",
+                      DeprecationWarning)
+        return self.volumes['root'].vid
 
 
     # XXX and this should go to exactly where? DispVM has it.
     @property
     def volatile_img(self):
         '''Volatile image that overlays :py:attr:`root_img`.'''
-        return self.storage.volatile_img
+        warnings.warn("volatile_img is deprecated, use volumes['volatile'].vid",
+                      DeprecationWarning)
+        return self.volumes['volatile'].vid
 
 
     # XXX shouldn't this go elsewhere?
@@ -425,8 +431,10 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
     # constructor
     #
 
-    def __init__(self, app, xml, **kwargs):
+    def __init__(self, app, xml, volume_config={}, **kwargs):
         super(QubesVM, self).__init__(app, xml, **kwargs)
+        if hasattr(self, 'volume_config'):
+            dict_merge(self.volume_config, volume_config)
 
         import qubes.vm.adminvm # pylint: disable=redefined-outer-name
 
@@ -637,7 +645,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
                     self.netvm.start(start_guid=start_guid,
                         notify_function=notify_function)
 
-        self.storage.prepare_for_vm_startup()
+        self.storage.start()
         self._update_libvirt_domain()
 
         qmemman_client = self.request_memory(mem_required)
@@ -729,6 +737,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
                     exc_info=1)
 
         self.libvirt_domain.shutdown()
+        self.storage.stop()
 
 
     def kill(self):
@@ -742,6 +751,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
             raise qubes.exc.QubesVMNotStartedError(self)
 
         self.libvirt_domain.destroy()
+        self.storage.stop()
 
 
     def force_shutdown(self, *args, **kwargs):
@@ -1025,7 +1035,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
         p.communicate(input=self.default_user)
 
 
-    # TODO move to storage
+    # TODO rename to create
     def create_on_disk(self, source_template=None):
         '''Create files needed for VM.
 
@@ -1115,11 +1125,11 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
         while self.is_running(): #1696
             time.sleep(1)
 
-
     def remove_from_disk(self):
         '''Remove domain remnants from disk.'''
         self.fire_event('domain-remove-from-disk')
-        self.storage.remove_from_disk()
+        self.storage.remove()
+        shutil.rmtree(self.vm.dir_path)
 
 
     def clone_disk_files(self, src):
@@ -1454,68 +1464,69 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
 
     # XXX shouldn't this go only to vms that have root image?
     def get_disk_utilization_root_img(self):
-        '''Get space that is actually ocuppied by :py:attr:`root_img`.
+        '''Get space that is actually ocuppied by :py:attr:`volumes['root']`.
-
-        Root image is a sparse file, so it is probably much less than logical
-        available space.
 
+           This is a temporary wrapper for backwards compatibility. You should
+           call directly :py:attr:`volumes[name].utilization`
         :returns: domain's real disk image size [FIXME unit]
         :rtype: FIXME
 
         .. seealso:: :py:meth:`get_root_img_sz`
         '''
 
-        return qubes.storage.get_disk_usage(self.root_img)
+        warnings.warn(
+            "get_disk_utilization_root_img is deprecated, use volumes['root'].utilization",
+            DeprecationWarning)
+        return qubes.storage.get_disk_usage(self.volumes['root'].utilization)
 
 
     # XXX shouldn't this go only to vms that have root image?
     def get_root_img_sz(self):
-        '''Get image size of :py:attr:`root_img`.
+        '''Get the size of the :py:attr:`volumes['root']`.
-
-        Root image is a sparse file, so it is probably much more than ocuppied
-        physical space.
 
+        This is a temporary wrapper for backwards compatibility. You should
+        call directly :py:attr:`volumes[name].size`
         :returns: domain's virtual disk size [FIXME unit]
         :rtype: FIXME
 
         .. seealso:: :py:meth:`get_disk_utilization_root_img`
         '''
 
-        if not os.path.exists(self.root_img):
+        warnings.warn(
-            return 0
+            "get_disk_root_img_sz is deprecated, use volumes['root'].size",
-
+            DeprecationWarning)
-        return os.path.getsize(self.root_img)
+        return qubes.storage.get_disk_usage(self.volumes['root'].size)
-
 
     def get_disk_utilization_private_img(self):
-        '''Get space that is actually ocuppied by :py:attr:`private_img`.
+        '''Get space that is actually ocuppied by :py:attr:`volumes['private']`.
-
-        Private image is a sparse file, so it is probably much less than
-        logical available space.
 
+           This is a temporary wrapper for backwards compatibility. You should
+           call directly :py:attr:`volumes[name].utilization`
         :returns: domain's real disk image size [FIXME unit]
         :rtype: FIXME
+        '''
 
-        .. seealso:: :py:meth:`get_private_img_sz`
+        warnings.warn(
-        ''' # pylint: disable=invalid-name
+            "get_disk_utilization_private_img is deprecated, use volumes['private'].utilization",
-
+            DeprecationWarning)
-        return qubes.storage.get_disk_usage(self.private_img)
+        return qubes.storage.get_disk_usage(self.volumes[
-
+            'private'].utilization)
 
     def get_private_img_sz(self):
-        '''Get image size of :py:attr:`private_img`.
+        '''Get the size of the :py:attr:`volumes['private']`.
-
-        Private image is a sparse file, so it is probably much more than
-        ocuppied physical space.
 
+        This is a temporary wrapper for backwards compatibility. You should
+        call directly :py:attr:`volumes[name].size`
         :returns: domain's virtual disk size [FIXME unit]
         :rtype: FIXME
 
         .. seealso:: :py:meth:`get_disk_utilization_private_img`
         '''
 
-        return self.storage.get_private_img_sz()
+        warnings.warn(
-
+            "get_disk_private_img_sz is deprecated, use volumes['private'].size",
+            DeprecationWarning)
+        return qubes.storage.get_disk_usage(self.volumes['private'].size)
 
     def get_disk_utilization(self):
         '''Return total space actually occuppied by all files belonging to \
@@ -1527,7 +1538,6 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
 
         return qubes.storage.get_disk_usage(self.dir_path)
 
-
     # TODO move to storage
     def verify_files(self):
         '''Verify that files accessed by this machine are sane.
@@ -1751,3 +1761,20 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
 #           if self.is_qrexec_running():
 #               #TODO: kill qrexec daemon
 #               pass
+
+
+def dict_merge(dct, merge_dct):
+    """ Recursive dict merge. Inspired by :meth:``dict.update()``, instead of
+    updating only top-level keys, dict_merge recurses down into dicts nested
+    to an arbitrary depth, updating keys. The ``merge_dct`` is merged into
+    ``dct``. (Source https://gist.github.com/angstwad/bf22d1822c38a92ec0a9)
+    :param dct: dict onto which the merge is executed
+    :param merge_dct: dct merged into dct
+    :return: None
+    """
+    for k, v in merge_dct.iteritems():
+        if (k in dct and isinstance(dct[k], dict)
+                and isinstance(merge_dct[k], dict)):
+            dict_merge(dct[k], merge_dct[k])
+        else:
+            dct[k] = merge_dct[k]

qubes/vm/templatevm.py

@@ -1,13 +1,16 @@
 #!/usr/bin/python2 -O
 # vim: fileencoding=utf-8
 
+import warnings
+
 import qubes
 import qubes.config
 import qubes.vm.qubesvm
 from qubes.config import defaults
+from qubes.vm.qubesvm import QubesVM
 
 
-class TemplateVM(qubes.vm.qubesvm.QubesVM):
+class TemplateVM(QubesVM):
     '''Template for AppVM'''
 
     dir_path_prefix = qubes.config.system_path['qubes_templates_dir']
@@ -15,7 +18,9 @@ class TemplateVM(qubes.vm.qubesvm.QubesVM):
     @property
     def rootcow_img(self):
         '''COW image'''
-        return self.storage.rootcow_img
+        warnings.warn("rootcow_img is deprecated, use "
+                      "volumes['root'].path_origin", DeprecationWarning)
+        return self.volumes['root'].path_cow
 
     @property
     def appvms(self):
@@ -67,6 +72,4 @@ class TemplateVM(qubes.vm.qubesvm.QubesVM):
             assert not self.is_running(), \
                 'Attempt to commit changes on running Template VM!'
 
-        self.log.info('Commiting template update; COW: {}'.format(
-            self.rootcow_img))
         self.storage.commit_template_changes()