Merge remote-tracking branch 'woju/pull/5/head' into core3-devel

Makefile                    |   1 -
doc/manpages/qvm-create.rst |   5 +
etc/storage.conf            |  13 -
qubes/__init__.py           |  82 +++++-
qubes/config.py             |  14 +-
qubes/devices.py            |  15 ++
qubes/storage/__init__.py   | 591 +++++++++++++---------------------------
qubes/storage/kernels.py    | 109 ++++++++
qubes/storage/xen.py        | 643 +++++++++++++++++++++++++++++---------------
qubes/tests/__init__.py     |  13 +-
qubes/tests/int/basic.py    |   5 +-
qubes/tests/storage.py      |  70 ++---
qubes/tests/storage_xen.py  | 366 +++++++++++++++----------
qubes/tools/qvm_create.py   |  18 +-
qubes/vm/appvm.py           |  38 ++-
qubes/vm/qubesvm.py         | 192 ++++++-------
qubes/vm/templatevm.py      |  45 +++-
rpm_spec/core-dom0.spec     |   2 +-
setup.py                    |   1 +
templates/libvirt/xen.xml   |  32 ++-
20 files changed, 1313 insertions(+), 942 deletions(-)

Makefile

@@ -60,7 +60,6 @@ endif
 #	$(MAKE) install -C tests
 	$(MAKE) install -C relaxng
 	mkdir -p $(DESTDIR)/etc/qubes
-	cp etc/storage.conf $(DESTDIR)/etc/qubes/
 ifeq ($(BACKEND_VMM),xen)
 	# Currently supported only on xen
 	cp etc/qmemman.conf $(DESTDIR)/etc/qubes/

doc/manpages/qvm-create.rst

@@ -53,6 +53,10 @@ Options
    Use provided :file:`root.img` instead of default/empty one (file will be
    *moved*). This option is mutually exclusive with :option:`--root-copy-from`.
 
+.. option:: --pool=POOL_NAME:VOLUME_NAME, -P POOL_NAME:VOLUME_NAME
+
+    Specify the pool to use for a volume
+
 Options for internal use
 ------------------------
 
@@ -71,5 +75,6 @@ Authors
 | Rafal Wojtczuk <rafal at invisiblethingslab dot com>
 | Marek Marczykowski <marmarek at invisiblethingslab dot com>
 | Wojtek Porczyk <woju at invisiblethingslab dot com>
+| Bahtiar `kalkin-` Gadimov <bahtiar at gadimov dot de> 
 
 .. vim: ts=3 sw=3 et tw=80

etc/storage.conf

@@ -1,13 +0,0 @@
-[default] ; poolname
-driver=xen ; the default xen storage
-; class = qubes.storage.xen.XenStorage ; class always overwrites the driver
-; 
-; To use our own pool driver it needs to provide `qubes.storage` entry_point
-; see also: https://pythonhosted.org/setuptools/setuptools.html#dynamic-discovery-of-services-and-plugins
-; class name
-; [pool-b]
-; driver = foo
-;
-; [test-dummy]
-; driver=dummy
-

qubes/__init__.py

@@ -36,8 +36,6 @@ __author__ = 'Invisible Things Lab'
 __license__ = 'GPLv2 or later'
 __version__ = 'R3'
 
-import ast
-import atexit
 import collections
 import errno
 import grp
@@ -47,12 +45,9 @@ import os.path
 import sys
 import tempfile
 import time
-import warnings
 
 import __builtin__
 
-import docutils.core
-import docutils.io
 import jinja2
 import lxml.etree
 import pkg_resources
@@ -1181,7 +1176,6 @@ class Qubes(PropertyHolder):
         default=True,
         doc='check for updates inside qubes')
 
-
     def __init__(self, store=None, load=True, **kwargs):
         #: logger instance for logging global messages
         self.log = logging.getLogger('app')
@@ -1194,6 +1188,9 @@ class Qubes(PropertyHolder):
         #: collection of all available labels for VMs
         self.labels = {}
 
+        #: collection of all pools
+        self.pools = {}
+
         #: Connection to VMM
         self.vmm = VMMConnection()
 
@@ -1235,7 +1232,7 @@ class Qubes(PropertyHolder):
         '''
 
         try:
-            fd = os.open(self._store, os.O_RDWR) # no O_CREAT
+            fd = os.open(self._store, os.O_RDWR)  # no O_CREAT
         except OSError as e:
             if e.errno != errno.ENOENT:
                 raise
@@ -1256,11 +1253,19 @@ class Qubes(PropertyHolder):
 
         self.xml = lxml.etree.parse(fh)
 
-        # stage 1: load labels
+        # stage 1: load labels and pools
         for node in self.xml.xpath('./labels/label'):
             label = Label.fromxml(node)
             self.labels[label.index] = label
 
+        for node in self.xml.xpath('./pools/pool'):
+            name = node.get('name')
+            assert name, "Pool name '%s' is invalid " % name
+            try:
+                self.pools[name] = self._get_pool(**node.attrib)
+            except qubes.exc.QubesException as e:
+                self.log.error(e.message)
+
         # stage 2: load VMs
         for node in self.xml.xpath('./domains/domain'):
             # pylint: disable=no-member
@@ -1270,7 +1275,7 @@ class Qubes(PropertyHolder):
             vm.init_log()
             self.domains.add(vm)
 
-        if not 0 in self.domains:
+        if 0 not in self.domains:
             self.domains.add(qubes.vm.adminvm.AdminVM(
                 self, None, qid=0, name='dom0'))
 
@@ -1310,11 +1315,17 @@ class Qubes(PropertyHolder):
         fh.close()
         del fh
 
-
     def __xml__(self):
         element = lxml.etree.Element('qubes')
 
         element.append(self.xml_labels())
+
+        pools_xml = lxml.etree.Element('pools')
+        for pool in self.pools.values():
+            pools_xml.append(pool.__xml__())
+
+        element.append(pools_xml)
+
         element.append(self.xml_properties())
 
         domains = lxml.etree.Element('domains')
@@ -1395,6 +1406,10 @@ class Qubes(PropertyHolder):
             7: Label(7, '0x75507b', 'purple'),
             8: Label(8, '0x000000', 'black'),
         }
+
+        for name, config in qubes.config.defaults['pool_configs'].items():
+            self.pools[name] = self._get_pool(**config)
+
         self.domains.add(
             qubes.vm.adminvm.AdminVM(self, None, qid=0, name='dom0'))
         self.save()
@@ -1413,7 +1428,6 @@ class Qubes(PropertyHolder):
             labels.append(label.__xml__())
         return labels
 
-
     def get_vm_class(self, clsname):
         '''Find the class for a domain.
 
@@ -1473,6 +1487,52 @@ class Qubes(PropertyHolder):
 
         raise KeyError(label)
 
+    def add_pool(self, **kwargs):
+        """ Add a storage pool to config."""
+        name = kwargs['name']
+        assert name not in self.pools.keys(), \
+            "Pool named %s already exists" % name
+        pool = self._get_pool(**kwargs)
+        pool.setup()
+        self.pools[name] = pool
+
+    def remove_pool(self, name):
+        """ Remove a storage pool from config file.  """
+        try:
+            pool = self.pools[name]
+            del self.pools[name]
+            pool.destroy()
+        except KeyError:
+            return
+
+
+    def get_pool(self, name):
+        '''  Returns a :py:class:`qubes.storage.Pool` instance '''
+        try:
+            return self.pools[name]
+        except KeyError:
+            raise qubes.exc.QubesException('Unknown storage pool ' + name)
+
+    def _get_pool(self, **kwargs):
+        try:
+            name = kwargs['name']
+            assert name, 'Name needs to be an non empty string'
+        except KeyError:
+            raise qubes.exc.QubesException('No pool name for pool')
+
+        try:
+            driver = kwargs['driver']
+        except KeyError:
+            raise qubes.exc.QubesException('No driver specified for pool ' +
+                                           name)
+        try:
+            klass = qubes.get_entry_point_one(
+                qubes.storage.STORAGE_ENTRY_POINT, driver)
+            del kwargs['driver']
+            return klass(**kwargs)
+        except KeyError:
+            raise qubes.exc.QubesException('Driver %s for pool %s' %
+                                           (driver, name))
 
     @qubes.events.handler('domain-pre-delete')
     def on_domain_pre_deleted(self, event, vm):

qubes/config.py

@@ -28,6 +28,8 @@
 # make a real /etc/qubes/master.conf or whatever
 #
 
+import os.path
+
 '''Constants which can be configured in one place'''
 
 qubes_base_dir = "/var/lib/qubes"
@@ -83,7 +85,17 @@ defaults = {
     'private_img_size': 2*1024*1024*1024,
     'root_img_size': 10*1024*1024*1024,
 
-    'pool_config': {'dir_path': '/var/lib/qubes'},
+    'pool_configs': {
+        'default': {'dir_path': qubes_base_dir,
+                    'driver': 'xen',
+                    'name': 'default'},
+        'linux-kernel': {
+            'dir_path': os.path.join(qubes_base_dir,
+                                     system_path['qubes_kernels_base_dir']),
+            'driver': 'linux-kernel',
+            'name': 'linux-kernel'
+        }
+    },
 
     # how long (in sec) to wait for VMs to shutdown,
     # before killing them (when used qvm-run with --wait option),

qubes/devices.py

@@ -6,6 +6,7 @@
 #
 # Copyright (C) 2010-2016  Joanna Rutkowska <joanna@invisiblethingslab.com>
 # Copyright (C) 2015-2016  Wojtek Porczyk <woju@invisiblethingslab.com>
+# Copyright (C) 2016       Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -26,6 +27,7 @@ import re
 
 import qubes
 
+
 class DeviceCollection(object):
     '''Bag for devices.
 
@@ -121,3 +123,16 @@ class RegexDevice(str):
 class PCIDevice(RegexDevice):
     regex = re.compile(
         r'^(?P<bus>[0-9a-f]+):(?P<device>[0-9a-f]+)\.(?P<function>[0-9a-f]+)$')
+
+
+class BlockDevice(object):
+    def __init__(self, path, name, script=None, rw=True, domain=None,
+                 devtype='disk'):
+        assert name, 'Missing device name'
+        assert path, 'Missing device path'
+        self.path = path
+        self.name = name
+        self.rw = rw
+        self.script = script
+        self.domain = domain
+        self.devtype = devtype

qubes/storage/__init__.py

@@ -23,24 +23,22 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 #
-
 """ Qubes storage system"""
 
 from __future__ import absolute_import
 
-import ConfigParser
 import os
 import os.path
 import shutil
-import subprocess
 
 import pkg_resources
 import qubes
 import qubes.exc
 import qubes.utils
+from qubes.devices import BlockDevice
+
+import lxml.etree
 
-BLKSIZE = 512
-CONFIG_FILE = '/etc/qubes/storage.conf'
 STORAGE_ENTRY_POINT = 'qubes.storage'
 
 
@@ -48,84 +46,70 @@ class StoragePoolException(qubes.exc.QubesException):
     pass
 
 
-class Storage(object):
-    '''Class for handling VM virtual disks.
-
-    This is base class for all other implementations, mostly with Xen on Linux
-    in mind.
+class Volume(object):
+    ''' Encapsulates all data about a volume for serialization to qubes.xml and
+        libvirt config.
     '''
 
-    root_img = None
-    private_img = None
-    volatile_img = None
+    devtype = 'disk'
+    domain = None
+    path = None
+    rw = True
+    script = None
+    usage = 0
 
-    modules_dev = None
+    def __init__(self, name, pool, volume_type, vid=None, size=0, **kwargs):
+        super(Volume, self).__init__(**kwargs)
+        self.name = str(name)
+        self.pool = str(pool)
+        self.vid = vid
+        self.size = size
+        self.volume_type = volume_type
 
-    def __init__(self, vm, private_img_size=None, root_img_size=None):
+    def __xml__(self):
+        return lxml.etree.Element('volume', **self.config)
 
-        #: Domain for which we manage storage
-        self.vm = vm
+    @property
+    def config(self):
+        ''' return config data for serialization to qubes.xml '''
+        return {'name': self.name,
+                'pool': self.pool,
+                'volume_type': self.volume_type}
+
+    def __repr__(self):
+        return '{}(name={!s}, pool={!r}, vid={!r}, volume_type={!r})'.format(
+            self.__class__.__name__, self.name, self.pool, self.vid,
+            self.volume_type)
+
+    def block_device(self):
+        ''' Return :py:class:`qubes.devices.BlockDevice` for serialization in
+            the libvirt XML template as <disk>.
+        '''
+        return BlockDevice(self.path, self.name, self.script, self.rw,
+                           self.domain, self.devtype)
 
-        #: Size of the private image
-        self.private_img_size = private_img_size \
-            if private_img_size is not None \
-            else qubes.config.defaults['private_img_size']
 
-        #: Size of the root image
-        self.root_img_size = root_img_size \
-            if root_img_size is not None \
-            else qubes.config.defaults['root_img_size']
+class Storage(object):
+    ''' Class for handling VM virtual disks.
+
+    This is base class for all other implementations, mostly with Xen on Linux
+    in mind.
+    '''
 
+    def __init__(self, vm):
+        #: Domain for which we manage storage
+        self.vm = vm
+        self.log = self.vm.log
         #: Additional drive (currently used only by HVM)
         self.drive = None
-
-    def get_config_params(self):
-        args = {}
-        args['rootdev'] = self.root_dev_config()
-        args['privatedev'] = self.private_dev_config()
-        args['volatiledev'] = self.volatile_dev_config()
-        args['otherdevs'] = self.other_dev_config()
-
-        args['kerneldir'] = self.kernels_dir
-
-        return args
-
-    def root_dev_config(self):
-        raise NotImplementedError()
-
-    def private_dev_config(self):
-        raise NotImplementedError()
-
-    def volatile_dev_config(self):
-        raise NotImplementedError()
-
-    def other_dev_config(self):
-        if self.modules_img is not None:
-            return self.format_disk_dev(self.modules_img, self.modules_dev,
-                rw=self.modules_img_rw)
-        elif self.drive is not None:
-            (drive_type, drive_domain, drive_path) = self.drive.split(":")
-            if drive_type == 'hd':
-                drive_type = 'disk'
-
-            rw = (drive_type == 'disk')
-
-            if drive_domain.lower() == "dom0":
-                drive_domain = None
-
-            return self.format_disk_dev(drive_path,
-                self.modules_dev,
-                rw=rw,
-                devtype=drive_type,
-                domain=drive_domain)
-
-        else:
-            return ''
-
-    def format_disk_dev(self, path, vdev, script=None, rw=True, devtype='disk',
-            domain=None):
-        raise NotImplementedError()
-
+        self.pools = {}
+        if hasattr(vm, 'volume_config'):
+            for name, conf in self.vm.volume_config.items():
+                assert 'pool' in conf, "Pool missing in volume_config" % str(
+                    conf)
+                pool = self.vm.app.get_pool(conf['pool'])
+                self.vm.volumes[name] = pool.init_volume(self.vm, conf)
+                self.pools[name] = pool
 
     @property
     def kernels_dir(self):
@@ -134,370 +118,183 @@ class Storage(object):
         If :py:attr:`self.vm.kernel` is :py:obj:`None`, the this points inside
         :py:attr:`self.vm.dir_path`
         '''
-        return os.path.join(qubes.config.system_path['qubes_base_dir'],
-            qubes.config.system_path['qubes_kernels_base_dir'], self.vm.kernel)\
-            if self.vm.kernel is not None \
-        else os.path.join(self.vm.dir_path,
-            qubes.config.vm_files['kernels_subdir'])
-
-
-    @property
-    def modules_img(self):
-        '''Path to image with modules.
-
-        Depending on domain, this may be global or inside domain's dir.
-        '''
-
-        modules_path = os.path.join(self.kernels_dir, 'modules.img')
-
-        if os.path.exists(modules_path):
-            return modules_path
-        else:
-            return None
-
-
-    @property
-    def modules_img_rw(self):
-        ''':py:obj:`True` if module image should be mounted RW, :py:obj:`False`
-        otherwise.'''
-        return self.vm.kernel is None
-
-
-    def abspath(self, path, rel=None):
-        '''Make absolute path.
-
-        If given path is relative, it is interpreted as relative to
-        :py:attr:`self.vm.dir_path` or given *rel*.
-        '''
-        return path if os.path.isabs(path) \
-            else os.path.join(rel or self.vm.dir_path, path)
-
-
-    @staticmethod
-    def _copy_file(source, destination):
-        '''Effective file copy, preserving sparse files etc.
-        '''
-        # TODO: Windows support
-
-        # We prefer to use Linux's cp, because it nicely handles sparse files
-        try:
-            subprocess.check_call(['cp', '--reflink=auto', source, destination])
-        except subprocess.CalledProcessError:
-            raise IOError('Error while copying {!r} to {!r}'.format(
-                source, destination))
+        assert 'kernel' in self.vm.volumes, "VM has no kernel pool"
+        return self.vm.volumes['kernel'].kernels_dir
 
     def get_disk_utilization(self):
-        return get_disk_usage(self.vm.dir_path)
+        ''' Returns summed up disk utilization for all domain volumes '''
+        result = 0
+        for volume in self.vm.volumes.values():
+            result += volume.usage
+        return result
 
+    # TODO Remove this wrapper
     def get_disk_utilization_private_img(self):
-        # pylint: disable=invalid-name
-        return get_disk_usage(self.private_img)
+        # pylint: disable=invalid-name,missing-docstring
+        return self.vm.volume['private'].usage
 
+    # TODO Remove this wrapper
     def get_private_img_sz(self):
-        if not os.path.exists(self.private_img):
-            return 0
-
-        return os.path.getsize(self.private_img)
-
-    def resize_private_img(self, size):
-        raise NotImplementedError()
-
-    def create_on_disk_private_img(self, source_template=None):
-        raise NotImplementedError()
+        # :pylint: disable=missing-docstring
+        return self.vm.volume['private'].size
 
-    def create_on_disk_root_img(self, source_template=None):
-        raise NotImplementedError()
+    def resize(self, volume, size):
+        ''' Resize volume '''
+        self.get_pool(volume).resize(volume, size)
 
+    # TODO rename it to create()
     def create_on_disk(self, source_template=None):
+        # :pylint: disable=missing-docstring
         if source_template is None and hasattr(self.vm, 'template'):
             source_template = self.vm.template
 
         old_umask = os.umask(002)
 
-        self.vm.log.info('Creating directory: {0}'.format(self.vm.dir_path))
-        os.mkdir(self.vm.dir_path)
-        self.create_on_disk_private_img(source_template)
-        self.create_on_disk_root_img(source_template)
-        self.reset_volatile_storage()
+        self.log.info('Creating directory: {0}'.format(self.vm.dir_path))
+        os.makedirs(self.vm.dir_path)
+        for name, volume in self.vm.volumes.items():
+            source_volume = None
+            if source_template and hasattr(source_template, 'volumes'):
+                source_volume = source_template.volumes[name]
+            self.get_pool(volume).create(volume, source_volume=source_volume)
 
         os.umask(old_umask)
 
-    def clone_disk_files(self, src_vm):
+    def clone(self, src_vm):
         self.vm.log.info('Creating directory: {0}'.format(self.vm.dir_path))
-        os.mkdir(self.vm.dir_path)
-
-        if hasattr(src_vm, 'private_img'):
-            self.vm.log.info('Copying the private image: {} -> {}'.format(
-                src_vm.private_img, self.vm.private_img))
-            self._copy_file(src_vm.private_img, self.vm.private_img)
-
-        if src_vm.updateable and hasattr(src_vm, 'root_img'):
-            self.vm.log.info('Copying the root image: {} -> {}'.format(
-                src_vm.root_img, self.root_img))
-            self._copy_file(src_vm.root_img, self.root_img)
-
-            # TODO: modules?
-            # XXX which modules? -woju
-
-
-    @staticmethod
-    def rename(newpath, oldpath):
-        '''Move storage directory, most likely during domain's rename.
-
-        .. note::
-            The arguments are in different order than in :program:`cp` utility.
+        if not os.path.exists(self.vm.dir_path):
+            self.log.info('Creating directory: {0}'.format(self.vm.dir_path))
+            os.makedirs(self.vm.dir_path)
+        for name, target in self.vm.volumes.items():
+            pool = self.get_pool(target)
+            source = src_vm.volumes[name]
+            volume = pool.clone(source, target)
+            assert volume, "%s.clone() returned '%s'" % (pool.__class__,
+                                                         volume)
+            self.vm.volumes[name] = volume
+
+    def rename(self, old_name, new_name):
+        ''' Notify the pools that the domain was renamed '''
+        volumes = self.vm.volumes
+        for name, volume in volumes.items():
+            pool = self.get_pool(volume)
+            volumes[name] = pool.rename(volume, old_name, new_name)
 
-        .. versionchange:: 4.0
-            This is now dummy method that just passes everything to
-            :py:func:`os.rename`.
+    def verify_files(self):
+        '''Verify that the storage is sane.
 
-        :param str newpath: New path
-        :param str oldpath: Old path
+        On success, returns normally. On failure, raises exception.
         '''
-
-        os.rename(oldpath, newpath)
-
-
-    def verify_files(self):
         if not os.path.exists(self.vm.dir_path):
-            raise qubes.exc.QubesVMError(self.vm,
+            raise qubes.exc.QubesVMError(
+                self.vm,
                 'VM directory does not exist: {}'.format(self.vm.dir_path))
 
-        if hasattr(self.vm, 'root_img') and not os.path.exists(self.root_img):
-            raise qubes.exc.QubesVMError(self.vm,
-                'VM root image file does not exist: {}'.format(self.root_img))
-
-        if hasattr(self.vm, 'private_img') \
-                and not os.path.exists(self.private_img):
-            raise qubes.exc.QubesVMError(self.vm,
-                'VM private image file does not exist: {}'.format(
-                    self.private_img))
-
-        if self.modules_img is not None \
-                and not os.path.exists(self.modules_img):
-            raise qubes.exc.QubesVMError(self.vm,
-                'VM kernel modules image does not exists: {}'.format(
-                    self.modules_img))
-
-
-    def remove_from_disk(self):
+    def remove(self):
+        for name, volume in self.vm.volumes.items():
+            self.log.info('Removing volume %s: %s' % (name, volume.vid))
+            self.get_pool(volume).remove(volume)
         shutil.rmtree(self.vm.dir_path)
 
+    def start(self):
+        ''' Execute the start method on each pool '''
+        for volume in self.vm.volumes.values():
+            self.get_pool(volume).start(volume)
 
-    def reset_volatile_storage(self):
-        # Re-create only for template based VMs
-        try:
-            if self.vm.template is not None and self.volatile_img:
-                if os.path.exists(self.volatile_img):
-                    os.remove(self.volatile_img)
-        except AttributeError: # self.vm.template
-            pass
-
-        # For StandaloneVM create it only if not already exists
-        # (eg after backup-restore)
-        if hasattr(self, 'volatile_img') \
-                and not os.path.exists(self.volatile_img):
-            self.vm.log.info(
-                'Creating volatile image: {0}'.format(self.volatile_img))
-            subprocess.check_call(
-                [qubes.config.system_path["prepare_volatile_img_cmd"],
-                    self.volatile_img,
-                    str(self.root_img_size / 1024 / 1024)])
-
-
-    def prepare_for_vm_startup(self):
-        self.reset_volatile_storage()
-
-        if hasattr(self.vm, 'private_img') \
-                and not os.path.exists(self.private_img):
-            self.vm.log.info('Creating empty VM private image file: {0}'.format(
-                self.private_img))
-            self.create_on_disk_private_img()
-
-
-def get_disk_usage_one(st):
-    '''Extract disk usage of one inode from its stat_result struct.
-
-    If known, get real disk usage, as written to device by filesystem, not
-    logical file size. Those values may be different for sparse files.
-
-    :param os.stat_result st: stat result
-    :returns: disk usage
-    '''
-    try:
-        return st.st_blocks * BLKSIZE
-    except AttributeError:
-        return st.st_size
-
-
-def get_disk_usage(path):
-    '''Get real disk usage of given path (file or directory).
-
-    When *path* points to directory, then it is evaluated recursively.
-
-    This function tries estiate real disk usage. See documentation of
-    :py:func:`get_disk_usage_one`.
-
-    :param str path: path to evaluate
-    :returns: disk usage
-    '''
-    try:
-        st = os.lstat(path)
-    except OSError:
-        return 0
-
-    ret = get_disk_usage_one(st)
-
-    # if path is not a directory, this is skipped
-    for dirpath, dirnames, filenames in os.walk(path):
-        for name in dirnames + filenames:
-            ret += get_disk_usage_one(os.lstat(os.path.join(dirpath, name)))
-
-    return ret
-
+    def stop(self):
+        ''' Execute the start method on each pool '''
+        for volume in self.vm.volumes.values():
+            self.get_pool(volume).stop(volume)
 
-def get_pool(name, vm):
-    """ Instantiates the storage for the specified vm """
-    config = _get_storage_config_parser()
+    def get_pool(self, volume):
+        ''' Helper function '''
+        assert isinstance(volume, Volume), "You need to pass a Volume"
+        return self.pools[volume.name]
 
-    klass = _get_pool_klass(name, config)
+    def commit_template_changes(self):
+        for volume in self.vm.volumes.values():
+            if volume.volume_type == 'origin':
+                self.get_pool(volume).commit_template_changes(volume)
 
-    keys = [k for k in config.options(name) if k != 'driver' and k != 'class']
-    values = [config.get(name, o) for o in keys]
-    config_kwargs = dict(zip(keys, values))
-
-    if name == 'default':
-        kwargs = qubes.config.defaults['pool_config'].copy()
-        kwargs.update(keys)
-    else:
-        kwargs = config_kwargs
-
-    return klass(vm, **kwargs)
-
-
-def pool_exists(name):
-    """ Check if the specified pool exists """
-    try:
-        _get_pool_klass(name)
-        return True
-    except StoragePoolException:
-        return False
-
-def add_pool(name, **kwargs):
-    """ Add a storage pool to config."""
-    config = _get_storage_config_parser()
-    config.add_section(name)
-    for key, value in kwargs.iteritems():
-        config.set(name, key, value)
-    _write_config(config)
-
-def remove_pool(name):
-    """ Remove a storage pool from config file.  """
-    config = _get_storage_config_parser()
-    config.remove_section(name)
-    _write_config(config)
-
-def _write_config(config):
-    with open(CONFIG_FILE, 'w') as configfile:
-        config.write(configfile)
-
-def _get_storage_config_parser():
-    """ Instantiates a `ConfigParaser` for specified storage config file.
-
-        Returns:
-            RawConfigParser
-    """
-    config = ConfigParser.RawConfigParser()
-    config.read(CONFIG_FILE)
-    return config
-
-
-def _get_pool_klass(name, config=None):
-    """ Returns the storage klass for the specified pool.
-
-        Args:
-            name: The pool name.
-            config: If ``config`` is not specified
-                    `_get_storage_config_parser()` is called.
-
-        Returns:
-            type: A class inheriting from `QubesVmStorage`
-    """
-    if config is None:
-        config = _get_storage_config_parser()
-
-    if not config.has_section(name):
-        raise StoragePoolException('Uknown storage pool ' + name)
-    elif not config.has_option(name, 'driver'):
-        raise StoragePoolException('No driver specified for pool ' + name)
-
-
-    driver = config.get(name, 'driver')
-    try:
-        return qubes.get_entry_point_one(STORAGE_ENTRY_POINT, driver)
-    except KeyError:
-        raise StoragePoolException('Driver %s for pool %s' % (driver, name))
 
 class Pool(object):
-    def __init__(self, vm, dir_path):
-        assert vm is not None
-        assert dir_path is not None
-
-        self.vm = vm
-        self.dir_path = dir_path
-
-        self.create_dir_if_not_exists(self.dir_path)
-
-        self.vmdir = self.vmdir_path(vm, self.dir_path)
-
-        appvms_path = os.path.join(self.dir_path, 'appvms')
-        self.create_dir_if_not_exists(appvms_path)
-
-        servicevms_path = os.path.join(self.dir_path, 'servicevms')
-        self.create_dir_if_not_exists(servicevms_path)
+    ''' A Pool is used to manage different kind of volumes (File
+        based/LVM/Btrfs/...).
 
-        vm_templates_path = os.path.join(self.dir_path, 'vm-templates')
-        self.create_dir_if_not_exists(vm_templates_path)
-
-    # XXX there is also a class attribute on the domain classes which does
-    # exactly that -- which one should prevail?
-    def vmdir_path(self, vm, pool_dir):
-        """ Returns the path to vmdir depending on the type of the VM.
-
-            The default QubesOS file storage saves the vm images in three
-            different directories depending on the ``QubesVM`` type:
-
-            * ``appvms`` for ``QubesAppVm`` or ``QubesHvm``
-            * ``vm-templates`` for ``QubesTemplateVm`` or ``QubesTemplateHvm``
+        3rd Parties providing own storage implementations will need to extend
+        this class.
+    '''
+    private_img_size = qubes.config.defaults['private_img_size']
+    root_img_size = qubes.config.defaults['root_img_size']
 
-            Args:
-                vm: a QubesVM
-                pool_dir: the root directory of the pool
+    def __init__(self, name, **kwargs):
+        super(Pool, self).__init__(**kwargs)
+        self.name = name
+        kwargs['name'] = self.name
 
-            Returns:
-                string (str) absolute path to the directory where the vm files
-                             are stored
-        """
-        if vm.is_template():
-            subdir = 'vm-templates'
-        elif vm.is_disposablevm():
-            subdir = 'appvms'
-            return os.path.join(pool_dir, subdir, vm.template.name + '-dvm')
-        else:
-            subdir = 'appvms'
+    def __xml__(self):
+        return lxml.etree.Element('pool', **self.config)
 
-        return os.path.join(pool_dir, subdir, vm.name)
+    def create(self, volume, source_volume):
+        ''' Create the given volume on disk or copy from provided
+            `source_volume`.
+        '''
+        raise NotImplementedError("Pool %s has create() not implemented" %
+                                  self.name)
 
-    def create_dir_if_not_exists(self, path):
-        """ Check if a directory exists in if not create it.
+    def commit_template_changes(self, volume):
+        ''' Update origin device '''
+        raise NotImplementedError(
+            "Pool %s has commit_template_changes() not implemented" %
+            self.name)
 
-            This method does not create any parent directories.
-        """
-        if not os.path.exists(path):
-            os.mkdir(path)
+    @property
+    def config(self):
+        ''' Returns the pool config to be written to qubes.xml '''
+        raise NotImplementedError("Pool %s has config() not implemented" %
+                                  self.name)
+
+    def clone(self, source, target):
+        ''' Clone volume '''
+        raise NotImplementedError("Pool %s has clone() not implemented" %
+                                  self.name)
+
+    def destroy(self):
+        raise NotImplementedError("Pool %s has destroy() not implemented" %
+                                  self.name)
+
+    def remove(self, volume):
+        ''' Remove volume'''
+        raise NotImplementedError("Pool %s has remove() not implemented" %
+                                  self.name)
+
+    def rename(self, volume, old_name, new_name):
+        ''' Called when the domain changes its name '''
+        raise NotImplementedError("Pool %s has rename() not implemented" %
+                                  self.name)
+
+    def start(self, volume):
+        ''' Do what ever is needed on start '''
+        raise NotImplementedError("Pool %s has start() not implemented" %
+                                  self.name)
+
+    def setup(self):
+        raise NotImplementedError("Pool %s has setup() not implemented" %
+                                  self.name)
+
+    def stop(self, volume):
+        ''' Do what ever is needed on stop'''
+        raise NotImplementedError("Pool %s has stop() not implemented" %
+                                  self.name)
+
+    def init_volume(self, volume_config):
+        ''' Initialize a :py:class:`qubes.storage.Volume` from `volume_config`.
+        '''
+        raise NotImplementedError("Pool %s has init_volume() not implemented" %
+                                  self.name)
 
 
 def pool_drivers():
     """ Return a list of EntryPoints names """
     return [ep.name
-        for ep in pkg_resources.iter_entry_points(STORAGE_ENTRY_POINT)]
+            for ep in pkg_resources.iter_entry_points(STORAGE_ENTRY_POINT)]

qubes/storage/kernels.py

@@ -0,0 +1,109 @@
+#!/usr/bin/python2
+# -*- encoding: utf8 -*-
+#
+# The Qubes OS Project, http://www.qubes-os.org
+#
+# Copyright (C) 2010-2015  Joanna Rutkowska <joanna@invisiblethingslab.com>
+# Copyright (C) 2015       Wojtek Porczyk <woju@invisiblethingslab.com>
+# Copyright (C) 2016       Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+import os
+
+from qubes.storage import Pool, StoragePoolException, Volume
+
+
+class LinuxModules(Volume):
+    rw = False
+
+    def __init__(self, target_dir, kernel_version, **kwargs):
+        super(LinuxModules, self).__init__(**kwargs)
+        self.kernels_dir = os.path.join(target_dir, kernel_version)
+        self.path = os.path.join(self.kernels_dir, 'modules.img')
+        self.vid = self.path
+        self.vmlinuz = os.path.join(self.kernels_dir, 'vmlinuz')
+        self.initramfs = os.path.join(self.kernels_dir, 'initramfs')
+
+
+class LinuxKernel(Pool):
+    driver = 'linux-kernel'
+
+    def __init__(self, name=None, dir_path=None):
+        assert dir_path, 'Missing dir_path'
+        super(LinuxKernel, self).__init__(name=name)
+        self.dir_path = dir_path
+
+    def init_volume(self, vm, volume_config):
+        assert 'volume_type' in volume_config, "Volume type missing " \
+            + str(volume_config)
+        volume_type = volume_config['volume_type']
+        if volume_type != 'read-only':
+            raise StoragePoolException("Unknown volume type " + volume_type)
+
+        volume = LinuxModules(self.dir_path, vm.kernel, **volume_config)
+
+        _check_path(volume.path)
+        _check_path(volume.vmlinuz)
+        _check_path(volume.initramfs)
+
+        return volume
+
+    def clone(self, source, target):
+        return target
+
+    def create(self, volume, source_volume):
+        return volume
+
+    def commit_template_changes(self, volume):
+        return volume
+
+    @property
+    def config(self):
+        return {
+            'name': self.name,
+            'dir_path': self.dir_path,
+            'driver': LinuxKernel.driver,
+        }
+
+    def destroy(self):
+        pass
+
+    def remove(self, volume):
+        pass
+
+    def rename(self, volume, old_name, new_name):
+        return volume
+
+    def setup(self):
+        pass
+
+    def start(self, volume):
+        path = volume.path
+        if not os.path.exists(path):
+            raise StoragePoolException('Missing kernel modules: %s' % path)
+
+        return volume
+
+    def stop(self, volume):
+        pass
+
+
+def _check_path(path):
+    ''' Raise an :py:class:`qubes.storage.StoragePoolException` if ``path`` does
+        not exist.
+    '''
+    if not os.path.exists(path):
+        raise StoragePoolException('Missing file: %s' % path)

qubes/storage/xen.py

@@ -31,264 +31,471 @@ import os.path
 import re
 import subprocess
 
-import lxml.etree
+from qubes.storage import Pool, StoragePoolException, Volume
 
-import qubes
-import qubes.config
-import qubes.storage
-import qubes.vm.templatevm
+BLKSIZE = 512
 
 
-class XenStorage(qubes.storage.Storage):
-    '''Class for VM storage of Xen VMs.
-    '''
+class XenPool(Pool):
+    ''' File based 'original' disk implementation '''
+    driver = 'xen'
+
+    def __init__(self, name=None, dir_path=None):
+        super(XenPool, self).__init__(name=name)
+        assert dir_path, "No pool dir_path specified"
+        self.dir_path = os.path.normpath(dir_path)
+
+    def clone(self, source, target):
+        ''' Clones the volume if the `source.pool` if the source is a
+            :py:class:`XenVolume`.
+        '''
+        if issubclass(XenVolume, source.__class__):
+            raise StoragePoolException('Volumes %s and %s use different pools'
+                                       % (source.__class__, target.__class__))
+
+        if source.volume_type not in ['origin', 'read-write']:
+            return target
+
+        copy_file(source.vid, target.vid)
+        return target
+
+    def create(self, volume, source_volume=None):
+        _type = volume.volume_type
+        size = volume.size
+        if _type == 'origin':
+            create_sparse_file(volume.path_origin, size)
+            create_sparse_file(volume.path_cow, size)
+        elif _type in ['read-write'] and source_volume:
+            copy_file(source_volume.path, volume.path)
+        elif _type in ['read-write', 'volatile']:
+            create_sparse_file(volume.path, size)
+
+        return volume
+
+    @property
+    def config(self):
+        return {
+            'name': self.name,
+            'dir_path': self.dir_path,
+            'driver': XenPool.driver,
+        }
+
+    def resize(self, volume, size):
+        ''' Expands volume, throws
+            :py:class:`qubst.storage.StoragePoolException` if given size is
+            less than current_size
+        '''
+        _type = volume.volume_type
+        if _type not in ['origin', 'read-write', 'volatile']:
+            raise StoragePoolException('Can not resize a %s volume %s' %
+                                       (_type, volume.vid))
+
+        if size <= volume.size:
+            raise StoragePoolException(
+                'For your own safety, shrinking of %s is'
+                ' disabled. If you really know what you'
+                ' are doing, use `truncate` on %s manually.' %
+                (volume.name, volume.vid))
+
+        if _type == 'origin':
+            path = volume.path_origin
+        elif _type in ['read-write', 'volatile']:
+            path = volume.path
+
+        with open(path, 'a+b') as fd:
+            fd.truncate(size)
+
+        self._resize_loop_device(path)
+
+    def remove(self, volume):
+        if volume.volume_type in ['read-write', 'volatile']:
+            _remove_if_exists(volume.vid)
+        elif volume.volume_type == 'origin':
+            _remove_if_exists(volume.vid)
+            _remove_if_exists(volume.path_cow)
+
+    def rename(self, volume, old_name, new_name):
+        assert issubclass(volume.__class__, XenVolume)
+        old_dir = os.path.dirname(volume.path)
+        new_dir = os.path.join(os.path.dirname(old_dir), new_name)
+
+        if not os.path.exists(new_dir):
+            os.makedirs(new_dir)
+
+        if volume.volume_type == 'read-write':
+            volume.rename_target_dir(new_name, new_dir)
+        elif volume.volume_type == 'read-only':
+            volume.rename_target_dir(old_name, new_dir)
+        elif volume.volume_type in ['origin', 'volatile']:
+            volume.rename_target_dir(new_dir)
+
+        return volume
+
+    def _resize_loop_device(self, path):
+        # find loop device if any
+        p = subprocess.Popen(
+            ['sudo', 'losetup', '--associated', path],
+            stdout=subprocess.PIPE)
+        result = p.communicate()
+
+        m = re.match(r'^(/dev/loop\d+):\s', result[0])
+        if m is not None:
+            loop_dev = m.group(1)
+
+            # resize loop device
+            subprocess.check_call(['sudo', 'losetup', '--set-capacity',
+                                   loop_dev])
+
+    def commit_template_changes(self, volume):
+        if volume.volume_type != 'origin':
+            return volume
+
+        if os.path.exists(volume.path_cow):
+            os.rename(volume.path_cow, volume.path_cow + '.old')
+
+        old_umask = os.umask(002)
+        with open(volume.path_cow, 'w') as f_cow:
+            f_cow.truncate(volume.size)
+        os.umask(old_umask)
+        return volume
+
+    def destroy(self):
+        pass
+
+    def setup(self):
+        create_dir_if_not_exists(self.dir_path)
+        appvms_path = os.path.join(self.dir_path, 'appvms')
+        create_dir_if_not_exists(appvms_path)
+        vm_templates_path = os.path.join(self.dir_path, 'vm-templates')
+        create_dir_if_not_exists(vm_templates_path)
+
+    def start(self, volume):
+        if volume.volume_type == 'volatile':
+            self._reset_volume(volume)
+        if volume.volume_type in ['origin', 'snapshot']:
+            _check_path(volume.path_origin)
+            _check_path(volume.path_cow)
+        else:
+            _check_path(volume.path)
+
+        return volume
+
+    def stop(self, volume):
+        pass
+
+    def _reset_volume(self, volume):
+        ''' Remove and recreate a volatile volume '''
+        assert volume.volume_type == 'volatile', "Not a volatile volume"
+
+        assert volume.size
+
+        _remove_if_exists(volume)
+
+        with open(volume.path, "w") as f_volatile:
+            f_volatile.truncate(volume.size)
+        return volume
 
-    root_dev = 'xvda'
-    private_dev = 'xvdb'
-    volatile_dev = 'xvdc'
-    modules_dev = 'xvdd'
+    def target_dir(self, vm):
+        """ Returns the path to vmdir depending on the type of the VM.
 
-    def __init__(self, vm, vmdir, **kwargs):
-        """ Instantiate the storage.
+            The default QubesOS file storage saves the vm images in three
+            different directories depending on the ``QubesVM`` type:
+
+            * ``appvms`` for ``QubesAppVm`` or ``QubesHvm``
+            * ``vm-templates`` for ``QubesTemplateVm`` or ``QubesTemplateHvm``
 
             Args:
                 vm: a QubesVM
-                vmdir: the root directory of the pool
+                pool_dir: the root directory of the pool
+
+            Returns:
+                string (str) absolute path to the directory where the vm files
+                             are stored
         """
-        assert vm is not None
-        assert vmdir is not None
+        if vm.is_template():
+            subdir = 'vm-templates'
+        elif vm.is_disposablevm():
+            subdir = 'appvms'
+            return os.path.join(self.dir_path, subdir,
+                                vm.template.name + '-dvm')
+        else:
+            subdir = 'appvms'
+
+        return os.path.join(self.dir_path, subdir, vm.name)
+
+    def init_volume(self, vm, volume_config):
+        assert 'volume_type' in volume_config, "Volume type missing " \
+            + str(volume_config)
+        volume_type = volume_config['volume_type']
+        known_types = {
+            'read-write': ReadWriteFile,
+            'read-only': ReadOnlyFile,
+            'origin': OriginFile,
+            'snapshot': SnapshotFile,
+            'volatile': VolatileFile,
+        }
+        if volume_type not in known_types:
+            raise StoragePoolException("Unknown volume type " + volume_type)
+
+        if volume_type in ['snapshot', 'read-only']:
+            origin_pool = vm.app.get_pool(volume_config['pool'])
+            assert isinstance(origin_pool,
+                              XenPool), 'Origin volume not a xen volume'
+            volume_config['target_dir'] = origin_pool.target_dir(vm.template)
+            name = volume_config['name']
+            volume_config['size'] = vm.template.volume_config[name]['size']
+        else:
+            volume_config['target_dir'] = self.target_dir(vm)
 
-        super(XenStorage, self).__init__(vm, **kwargs)
+        return known_types[volume_type](**volume_config)
 
-        self.vmdir = vmdir
 
+class XenVolume(Volume):
+    ''' Parent class for the xen volumes implementation which expects a
+        `target_dir` param on initialization.
+    '''
 
-    @property
-    def private_img(self):
-        '''Path to the private image'''
-        return self.abspath(qubes.config.vm_files['private_img'])
+    def __init__(self, target_dir, **kwargs):
+        self.target_dir = target_dir
+        assert self.target_dir, "target_dir not specified"
+        super(XenVolume, self).__init__(**kwargs)
 
 
-    @property
-    def root_img(self):
-        '''Path to the root image'''
-        return self.vm.template.storage.root_img \
-            if hasattr(self.vm, 'template') and self.vm.template \
-            else self.abspath(qubes.config.vm_files['root_img'])
+class SizeMixIn(XenVolume):
+    ''' A mix in which expects a `size` param to be > 0 on initialization and
+        provides a usage property wrapper.
+    '''
 
+    def __init__(self, size=0, **kwargs):
+        assert size, 'Empty size provided'
+        assert size > 0, 'Size for volume ' + kwargs['name'] + ' is <=0'
+        super(SizeMixIn, self).__init__(size=int(size), **kwargs)
 
     @property
-    def rootcow_img(self):
-        '''Path to the root COW image'''
+    def usage(self):
+        ''' Returns the actualy used space '''
+        return get_disk_usage(self.vid)
 
-        if isinstance(self.vm, qubes.vm.templatevm.TemplateVM):
-            return self.abspath(qubes.config.vm_files['rootcow_img'])
+    @property
+    def config(self):
+        ''' return config data for serialization to qubes.xml '''
+        return {'name': self.name,
+                'pool': self.pool,
+                'size': str(self.size),
+                'volume_type': self.volume_type}
+
+
+class ReadWriteFile(SizeMixIn):
+    ''' Represents a readable & writable file image based volume '''
+    def __init__(self, **kwargs):
+        super(ReadWriteFile, self).__init__(**kwargs)
+        self.path = os.path.join(self.target_dir, self.name + '.img')
+        self.vid = self.path
+
+    def rename_target_dir(self, new_name, new_dir):
+        ''' Called by :py:class:`XenPool` when a domain changes it's name '''
+        old_path = self.path
+        file_name = os.path.basename(self.path)
+        new_path = os.path.join(new_dir, file_name)
+
+        os.rename(old_path, new_path)
+        self.target_dir = new_dir
+        self.path = new_path
+        self.vid = self.path
+
+
+class ReadOnlyFile(XenVolume):
+    ''' Represents a readonly file image based volume '''
+    usage = 0
+
+    def __init__(self, size=0, **kwargs):
+        super(ReadOnlyFile, self).__init__(size=int(size), **kwargs)
+        self.path = self.vid
+
+    def rename_target_dir(self, old_name, new_dir):
+        """ Called by :py:class:`XenPool` when a domain changes it's name.
+
+        Only copies the volume if it belongs to the domain being renamed.
+        Currently if a volume is in a directory named the same as the domain,
+        it's ”owned” by the domain.
+        """
+        if os.path.basename(self.target_dir) == old_name:
+            file_name = os.path.basename(self.path)
+            new_path = os.path.join(new_dir, file_name)
+            old_path = self.path
 
-        return None
+            os.rename(old_path, new_path)
 
+            self.target_dir = new_dir
+            self.path = new_path
+            self.vid = self.path
 
-    @property
-    def volatile_img(self):
-        '''Path to the volatile image'''
-        return self.abspath(qubes.config.vm_files['volatile_img'])
-
-
-    def format_disk_dev(self, path, vdev, script=None, rw=True, devtype='disk',
-            domain=None):
-        if path is None:
-            return ''
-
-        element = lxml.etree.Element('disk')
-        element.set('type', 'block')
-        element.set('device', devtype)
-
-        element.append(lxml.etree.Element('driver', name='phy'))
-        element.append(lxml.etree.Element('source', dev=path))
-        element.append(lxml.etree.Element('target', dev=vdev))
-
-        if not rw:
-            element.append(lxml.etree.Element('readonly'))
-        if domain is not None:
-            # XXX vm.name?
-            element.append(lxml.etree.Element('domain', name=domain))
-        if script:
-            element.append(lxml.etree.Element('script', path=script))
-
-        # TODO return element
-        return lxml.etree.tostring(element)
-
-
-    def root_dev_config(self):
-        if isinstance(self.vm, qubes.vm.templatevm.TemplateVM):
-            return self.format_disk_dev(
-                '{root}:{rootcow}'.format(
-                    root=self.root_img,
-                    rootcow=self.rootcow_img),
-                self.root_dev,
-                script='block-origin')
-
-        elif self.vm.hvm and hasattr(self.vm, 'template'):
-            # HVM template-based VM - only one device-mapper layer, in dom0
-            # (root+volatile)
-            # HVM detection based on 'kernel' property is massive hack,
-            # but taken from assumption that VM needs Qubes-specific kernel
-            # (actually initramfs) to assemble the second layer of device-mapper
-            return self.format_disk_dev(
-                '{root}:{volatile}'.format(
-                    root=self.vm.template.storage.root_img,
-                    volatile=self.volatile_img),
-                self.root_dev,
-                script='block-snapshot')
-
-        elif hasattr(self.vm, 'template'):
-            # any other template-based VM - two device-mapper layers: one
-            # in dom0 (here) from root+root-cow, and another one from
-            # this+volatile.img
-            return self.format_disk_dev(
-                '{root}:{template_rootcow}'.format(
-                    root=self.root_img,
-                    template_rootcow=self.vm.template.storage.rootcow_img),
-                self.root_dev,
-                script='block-snapshot',
-                rw=False)
 
-        else:
-            # standalone qube
-            return self.format_disk_dev(self.root_img, self.root_dev)
+class OriginFile(SizeMixIn):
+    ''' Represents a readable, writeable & snapshotable file image based volume.
 
+        This is used for TemplateVM's
+    '''
 
-    def private_dev_config(self):
-        return self.format_disk_dev(self.private_img, self.private_dev)
+    script = 'block-origin'
+
+    def __init__(self, **kwargs):
+        super(OriginFile, self).__init__(**kwargs)
+        self.path_origin = os.path.join(self.target_dir, self.name + '.img')
+        self.path_cow = os.path.join(self.target_dir, self.name + '-cow.img')
+        self.path = '%s:%s' % (self.path_origin, self.path_cow)
+        self.vid = self.path_origin
+
+    def commit(self):
+        ''' Commit Template changes '''
+        raise NotImplementedError
+
+    def rename_target_dir(self, new_dir):
+        ''' Called by :py:class:`XenPool` when a domain changes it's name '''
+        old_path_origin = self.path_origin
+        old_path_cow = self.path_cow
+        new_path_origin = os.path.join(new_dir, self.name + '.img')
+        new_path_cow = os.path.join(new_dir, self.name + '-cow.img')
+        os.rename(old_path_origin, new_path_origin)
+        os.rename(old_path_cow, new_path_cow)
+        self.target_dir = new_dir
+        self.path_origin = new_path_origin
+        self.path_cow = new_path_cow
+        self.path = '%s:%s' % (self.path_origin, self.path_cow)
+        self.vid = self.path_origin
 
-    def volatile_dev_config(self):
-        return self.format_disk_dev(self.volatile_img, self.volatile_dev)
+    @property
+    def usage(self):
+        result = 0
+        if os.path.exists(self.path_origin):
+            result += get_disk_usage(self.path_origin)
+        if os.path.exists(self.path_cow):
+            result += get_disk_usage(self.path_cow)
+        return result
+
+
+class SnapshotFile(XenVolume):
+    ''' Represents a readonly snapshot of an :py:class:`OriginFile` volume '''
+    script = 'block-snapshot'
+    rw = False
+    usage = 0
+
+    def __init__(self, name=None, size=None, **kwargs):
+        assert size
+        super(SnapshotFile, self).__init__(name=name, size=int(size), **kwargs)
+        self.path_origin = os.path.join(self.target_dir, name + '.img')
+        self.path_cow = os.path.join(self.target_dir, name + '-cow.img')
+        self.path = '%s:%s' % (self.path_origin, self.path_cow)
+        self.vid = self.path_origin
+
+
+class VolatileFile(SizeMixIn):
+    ''' Represents a readable & writeable file based volume, which will be
+        discarded and recreated at each startup.
+    '''
+    def __init__(self, **kwargs):
+        super(VolatileFile, self).__init__(**kwargs)
+        self.path = os.path.join(self.target_dir, self.name + '.img')
+        self.vid = self.path
+
+    def rename_target_dir(self, new_dir):
+        ''' Called by :py:class:`XenPool` when a domain changes it's name '''
+        _remove_if_exists(self)
+        file_name = os.path.basename(self.path)
+        self.target_dir = new_dir
+        new_path = os.path.join(new_dir, file_name)
+        self.path = new_path
+        self.vid = self.path
+
+
+def create_sparse_file(path, size):
+    ''' Create an empty sparse file '''
+    if os.path.exists(path):
+        raise IOError("Volume %s already exists", path)
+    parent_dir = os.path.dirname(path)
+    if not os.path.exists(parent_dir):
+        os.makedirs(parent_dir)
+    with open(path, 'a+b') as fh:
+        fh.truncate(size)
+
+
+def get_disk_usage_one(st):
+    '''Extract disk usage of one inode from its stat_result struct.
+
+    If known, get real disk usage, as written to device by filesystem, not
+    logical file size. Those values may be different for sparse files.
+
+    :param os.stat_result st: stat result
+    :returns: disk usage
+    '''
+    try:
+        return st.st_blocks * BLKSIZE
+    except AttributeError:
+        return st.st_size
 
 
-    def create_on_disk_private_img(self, source_template=None):
-        if source_template is None:
-            f_private = open(self.private_img, 'a+b')
-            f_private.truncate(self.private_img_size)
-            f_private.close()
+def get_disk_usage(path):
+    '''Get real disk usage of given path (file or directory).
 
-        else:
-            self.vm.log.info("Copying the template's private image: {}".format(
-                source_template.storage.private_img))
-            self._copy_file(source_template.storage.private_img,
-                self.private_img)
+    When *path* points to directory, then it is evaluated recursively.
 
+    This function tries estiate real disk usage. See documentation of
+    :py:func:`get_disk_usage_one`.
 
-    def create_on_disk_root_img(self, source_template=None):
-        if source_template is None:
-            fd = open(self.root_img, 'a+b')
-            fd.truncate(self.root_img_size)
-            fd.close()
+    :param str path: path to evaluate
+    :returns: disk usage
+    '''
+    try:
+        st = os.lstat(path)
+    except OSError:
+        return 0
 
-        elif self.vm.updateable:
-            # if not updateable, just use template's disk
-            self.vm.log.info("--> Copying the template's root image: {}".format(
-                source_template.storage.root_img))
-            self._copy_file(source_template.storage.root_img, self.root_img)
+    ret = get_disk_usage_one(st)
 
+    # if path is not a directory, this is skipped
+    for dirpath, dirnames, filenames in os.walk(path):
+        for name in dirnames + filenames:
+            ret += get_disk_usage_one(os.lstat(os.path.join(dirpath, name)))
 
-    def resize_private_img(self, size):
-        fd = open(self.private_img, 'a+b')
-        fd.truncate(size)
-        fd.close()
+    return ret
 
-        # find loop device if any
-        p = subprocess.Popen(
-            ['sudo', 'losetup', '--associated', self.private_img],
-            stdout=subprocess.PIPE)
-        result = p.communicate()
 
-        m = re.match(r'^(/dev/loop\d+):\s', result[0])
-        if m is not None:
-            loop_dev = m.group(1)
+def create_dir_if_not_exists(path):
+    """ Check if a directory exists in if not create it.
 
-            # resize loop device
-            subprocess.check_call(
-                ['sudo', 'losetup', '--set-capacity', loop_dev])
+        This method does not create any parent directories.
+    """
+    if not os.path.exists(path):
+        os.mkdir(path)
 
 
-    def commit_template_changes(self):
-        assert isinstance(self.vm, qubes.vm.templatevm.TemplateVM)
+def copy_file(source, destination):
+    '''Effective file copy, preserving sparse files etc.
+    '''
+    # TODO: Windows support
+    # We prefer to use Linux's cp, because it nicely handles sparse files
+    assert os.path.exists(source), \
+        "Missing the source %s to copy from" % source
+    assert not os.path.exists(destination), \
+        "Destination %s already exists" % destination
 
-        # TODO: move rootcow_img to this class; the same for vm.is_outdated()
-        if os.path.exists(self.vm.rootcow_img):
-            os.rename(self.vm.rootcow_img, self.vm.rootcow_img + '.old')
+    parent_dir = os.path.dirname(destination)
+    if not os.path.exists(parent_dir):
+        os.makedirs(parent_dir)
 
-        old_umask = os.umask(002)
-        f_cow = open(self.vm.rootcow_img, 'w')
-        f_root = open(self.root_img, 'r')
-        f_root.seek(0, os.SEEK_END)
-        # make empty sparse file of the same size as root.img
-        f_cow.truncate(f_root.tell())
-        f_cow.close()
-        f_root.close()
-        os.umask(old_umask)
+    try:
+        subprocess.check_call(['cp', '--reflink=auto', source, destination])
+    except subprocess.CalledProcessError:
+        raise IOError('Error while copying {!r} to {!r}'.format(source,
+                                                                destination))
+
+
+def _remove_if_exists(volume):
+    if os.path.exists(volume.path):
+        os.remove(volume.path)
 
 
-    def reset_volatile_storage(self):
-        try:
-            # no template set, in any way (Standalone VM, Template VM)
-            if self.vm.template is None:
-                raise AttributeError
-
-            # template-based HVM with only one device-mapper layer -
-            # volatile.img used as upper layer on root.img, no root-cow.img
-            # intermediate layer
-            if self.vm.hvm:
-                if os.path.exists(self.volatile_img):
-                    if self.vm.debug:
-                        if os.path.getmtime(self.vm.template.storage.root_img) \
-                                > os.path.getmtime(self.volatile_img):
-                            self.vm.log.warning(
-                                'Template have changed, resetting root.img')
-                        else:
-                            self.vm.log.warning(
-                                'Debug mode: not resetting root.img; if you'
-                                ' want to force root.img reset, either'
-                                ' update template VM, or remove volatile.img'
-                                ' file.')
-                            return
-
-                    os.remove(self.volatile_img)
-
-                # FIXME stat on f_root; with open() ...
-                f_volatile = open(self.volatile_img, "w")
-                f_root = open(self.vm.template.storage.root_img, "r")
-                # make empty sparse file of the same size as root.img
-                f_root.seek(0, os.SEEK_END)
-                f_volatile.truncate(f_root.tell())
-                f_volatile.close()
-                f_root.close()
-                return # XXX why is that? super() does not run
-        except AttributeError: # self.vm.template
-            pass
-
-        super(XenStorage, self).reset_volatile_storage()
-
-
-    def prepare_for_vm_startup(self):
-        super(XenStorage, self).prepare_for_vm_startup()
-
-        if self.drive is not None:
-            # pylint: disable=unused-variable
-            (drive_type, drive_domain, drive_path) = self.drive.split(":")
-
-            if drive_domain.lower() != "dom0":
-                # XXX "VM '{}' holding '{}' does not exists".format(
-                drive_vm = self.vm.app.domains[drive_domain]
-
-                if not drive_vm.is_running():
-                    raise qubes.exc.QubesVMNotRunningError(drive_vm,
-                        'VM {!r} holding {!r} isn\'t running'.format(
-                            drive_domain, drive_path))
-
-        if self.rootcow_img and not os.path.exists(self.rootcow_img):
-            self.commit_template_changes()
-
-class XenPool(qubes.storage.Pool):
-    def get_storage(self):
-        """ Returns an instantiated ``XenStorage``. """
-        return XenStorage(self.vm, vmdir=self.vmdir)
+def _check_path(path):
+    ''' Raise an StoragePoolException if ``path`` does not exist'''
+    if not os.path.exists(path):
+        raise StoragePoolException('Missing image file: %s' % path)

qubes/tests/__init__.py

@@ -820,7 +820,7 @@ class BackupTestsMixin(SystemTestsMixin):
             name=vmname, template=template, provides_network=True, label='red')
         testnet.create_on_disk()
         vms.append(testnet)
-        self.fill_image(testnet.private_img, 20*1024*1024)
+        self.fill_image(testnet.volumes['private'].vid, 20*1024*1024)
 
         vmname = self.make_vm_name('test1')
         if self.verbose:
@@ -831,16 +831,17 @@ class BackupTestsMixin(SystemTestsMixin):
         testvm1.netvm = testnet
         testvm1.create_on_disk()
         vms.append(testvm1)
-        self.fill_image(testvm1.private_img, 100*1024*1024)
+        self.fill_image(testvm1.volumes['private'].vid, 100*1024*1024)
 
         vmname = self.make_vm_name('testhvm1')
         if self.verbose:
             print >>sys.stderr, "-> Creating %s" % vmname
         testvm2 = self.app.add_new_vm(qubes.vm.standalonevm.StandaloneVM,
-            name=vmname,
-            hvm=True, label='red')
-        testvm2.create_on_disk()
-        self.fill_image(testvm2.root_img, 1024*1024*1024, True)
+                                      name=vmname,
+                                      hvm=True,
+                                      label='red')
+        testvm2.create_on_disk(verbose=self.verbose)
+        self.fill_image(testvm2.volumes['root'].vid, 1024 * 1024 * 1024, True)
         vms.append(testvm2)
 
         vmname = self.make_vm_name('template')

qubes/tests/int/basic.py

@@ -392,8 +392,9 @@ class TC_03_QvmRevertTemplateChanges(qubes.tests.SystemTestsMixin,
         self.save_and_reload_db()
 
     def get_rootimg_checksum(self):
-        p = subprocess.Popen(['sha1sum', self.test_template.root_img],
-                             stdout=subprocess.PIPE)
+        p = subprocess.Popen(
+            ['sha1sum', self.test_template.volumes['root'].vid],
+            stdout=subprocess.PIPE)
         return p.communicate()[0]
 
     def _do_test(self):

qubes/tests/storage.py

@@ -17,9 +17,12 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import qubes.log
-from qubes.storage import StoragePoolException, pool_drivers
+from qubes.exc import QubesException
+from qubes.storage import pool_drivers
 from qubes.storage.xen import XenPool
-from qubes.tests import QubesTestCase
+from qubes.tests import QubesTestCase, SystemTestsMixin
+
+# :pylint: disable=invalid-name
 
 
 class TestApp(qubes.tests.TestEmitter):
@@ -27,30 +30,26 @@ class TestApp(qubes.tests.TestEmitter):
 
 
 class TestVM(object):
-    def __init__(self, app, qid, name, pool_name, template=None):
-        super(TestVM, self).__init__()
-        self.app = app
-        self.qid = qid
-        self.name = name
-        self.pool_name = pool_name
-        self.template = template
-        self.hvm = False
-        self.storage = qubes.storage.get_pool(self.pool_name,
-                                              self).get_storage()
+    def __init__(self, test, template=None):
+        self.app = test.app
+        self.name = test.make_vm_name('appvm')
         self.log = qubes.log.get_vm_logger(self.name)
 
+        if template:
+            self.template = template
+
     def is_template(self):
+        # :pylint: disable=no-self-use
         return False
 
     def is_disposablevm(self):
+        # :pylint: disable=no-self-use
         return False
 
-    @property
-    def dir_path(self):
-        return self.storage.vmdir
-
 
 class TestTemplateVM(TestVM):
+    dir_path_prefix = qubes.config.system_path['qubes_templates_dir']
+
     def is_template(self):
         return True
 
@@ -60,46 +59,47 @@ class TestDisposableVM(TestVM):
         return True
 
 
-class TC_00_Pool(QubesTestCase):
+class TC_00_Pool(SystemTestsMixin, QubesTestCase):
     """ This class tests the utility methods from :mod:``qubes.storage`` """
 
     def setUp(self):
         super(TC_00_Pool, self).setUp()
+        self.init_default_template()
 
     def test_000_unknown_pool_driver(self):
         # :pylint: disable=protected-access
         """ Expect an exception when unknown pool is requested"""
-        with self.assertRaises(StoragePoolException):
-            qubes.storage._get_pool_klass('foo-bar')
+        with self.assertRaises(QubesException):
+            self.app.get_pool('foo-bar')
 
     def test_001_all_pool_drivers(self):
-        """ The only predefined pool driver is file """
-        self.assertEquals(["xen"], pool_drivers())
+        """ The only predefined pool driver is xen """
+        self.assertEquals(['linux-kernel', 'xen'], pool_drivers())
 
     def test_002_get_pool_klass(self):
         """ Expect the default pool to be `XenPool` """
         # :pylint: disable=protected-access
-        result = qubes.storage._get_pool_klass('default')
-        self.assertTrue(result is XenPool)
+        result = self.app.get_pool('default')
+        self.assertIsInstance(result, XenPool)
 
     def test_003_pool_exists_default(self):
         """ Expect the default pool to exists """
-        self.assertTrue(qubes.storage.pool_exists('default'))
-
-    def test_004_pool_exists_random(self):
-        """ Expect this pool to not a exist """
-        self.assertFalse(qubes.storage.pool_exists(
-            'asdh312096r832598213iudhas'))
+        self.assertPoolExists('default')
 
-    def test_005_add_remove_pool(self):
+    def test_004_add_remove_pool(self):
         """ Tries to adding and removing a pool. """
         pool_name = 'asdjhrp89132'
 
         # make sure it's really does not exist
-        qubes.storage.remove_pool(pool_name)
+        self.app.remove_pool(pool_name)
+        self.assertFalse(self.assertPoolExists(pool_name))
+
+        self.app.add_pool(name=pool_name, driver='xen', dir_path='/tmp/asdjhrp89132')
+        self.assertTrue(self.assertPoolExists(pool_name))
 
-        qubes.storage.add_pool(pool_name, driver='xen')
-        self.assertTrue(qubes.storage.pool_exists(pool_name))
+        self.app.remove_pool(pool_name)
+        self.assertFalse(self.assertPoolExists(pool_name))
 
-        qubes.storage.remove_pool(pool_name)
-        self.assertFalse(qubes.storage.pool_exists(pool_name))
+    def assertPoolExists(self, pool):
+        """ Check if specified pool exists """
+        return pool in self.app.pools.keys()

qubes/tests/storage_xen.py

@@ -18,13 +18,21 @@
 
 import os
 import shutil
-import unittest
+
 import qubes.storage
 import qubes.tests.storage
+from qubes.config import defaults
+from qubes.storage import Storage
+from qubes.storage.xen import (OriginFile, ReadOnlyFile, ReadWriteFile,
+                               SnapshotFile, VolatileFile)
 from qubes.tests import QubesTestCase, SystemTestsMixin
-from qubes.storage.xen import XenStorage
+from qubes.tests.storage import TestVM
+
+
+# :pylint: disable=invalid-name
+
 
-class TC_00_XenPool(QubesTestCase):
+class TC_00_XenPool(SystemTestsMixin, QubesTestCase):
 
     """ This class tests some properties of the 'default' pool. """
 
@@ -34,32 +42,166 @@ class TC_00_XenPool(QubesTestCase):
             .. sealso::
                Data :data:``qubes.qubes.defaults['pool_config']``.
         """
-        vm = self._init_app_vm()
-        result = qubes.storage.get_pool("default", vm).dir_path
+        result = self.app.get_pool("default").dir_path
         expected = '/var/lib/qubes'
         self.assertEquals(result, expected)
 
     def test001_default_storage_class(self):
-        """ Check when using default pool the Storage is ``XenStorage``. """
+        """ Check when using default pool the Storage is ``Storage``. """
         result = self._init_app_vm().storage
-        self.assertIsInstance(result, XenStorage)
-
-    def test_002_default_pool_name(self):
-        """ Default pool_name is 'default'. """
-        vm = self._init_app_vm()
-        self.assertEquals(vm.pool_name, "default")
+        self.assertIsInstance(result, Storage)
 
     def _init_app_vm(self):
         """ Return initalised, but not created, AppVm. """
-        app = qubes.tests.storage.TestApp()
         vmname = self.make_vm_name('appvm')
-        template = qubes.tests.storage.TestTemplateVM(app, 1,
-            self.make_vm_name('template'), 'default')
-        return qubes.tests.storage.TestVM(app, qid=2, name=vmname,
-            template=template, pool_name='default')
+        self.init_default_template()
+        return self.app.add_new_vm(qubes.vm.appvm.AppVM,
+                                   name=vmname,
+                                   template=self.app.default_template,
+                                   label='red')
+
+
+class TC_01_XenVolumes(SystemTestsMixin, QubesTestCase):
+    POOL_DIR = '/var/lib/qubes/test-pool'
+    POOL_NAME = 'test-pool'
+    POOL_CONF = {'driver': 'xen', 'dir_path': POOL_DIR, 'name': POOL_NAME}
+
+    def setUp(self):
+        """ Add a test file based storage pool """
+        super(TC_01_XenVolumes, self).setUp()
+        self.init_default_template()
+        self.app.add_pool(**self.POOL_CONF)
+
+    def tearDown(self):
+        """ Remove the file based storage pool after testing """
+        self.app.remove_pool("test-pool")
+        super(TC_01_XenVolumes, self).tearDown()
+        shutil.rmtree(self.POOL_DIR, ignore_errors=True)
+
+    def test_000_origin_volume(self):
+        config = {
+            'name': 'root',
+            'pool': self.POOL_NAME,
+            'volume_type': 'origin',
+            'size': defaults['root_img_size'],
+        }
+        vm = TestVM(self)
+        result = self.app.get_pool(self.POOL_NAME).init_volume(vm, config)
+        self.assertIsInstance(result, OriginFile)
+        self.assertEqual(result.name, 'root')
+        self.assertEqual(result.pool, self.POOL_NAME)
+        self.assertEqual(result.size, defaults['root_img_size'])
+
+    def test_001_snapshot_volume(self):
+        original_path = '/var/lib/qubes/vm-templates/fedora-23/root.img'
+        original_size = qubes.config.defaults['root_img_size']
+        config = {
+            'name': 'root',
+            'pool': 'default',
+            'volume_type': 'snapshot',
+            'vid': original_path,
+        }
+        vm = TestVM(self, template=self.app.default_template)
+        result = self.app.get_pool(self.POOL_NAME).init_volume(vm, config)
+        self.assertIsInstance(result, SnapshotFile)
+        self.assertEqual(result.name, 'root')
+        self.assertEqual(result.pool, 'default')
+        self.assertEqual(result.size, original_size)
+
+    def test_002_read_write_volume(self):
+        config = {
+            'name': 'root',
+            'pool': self.POOL_NAME,
+            'volume_type': 'read-write',
+            'size': defaults['root_img_size'],
+        }
+        vm = TestVM(self)
+        result = self.app.get_pool(self.POOL_NAME).init_volume(vm, config)
+        self.assertIsInstance(result, ReadWriteFile)
+        self.assertEqual(result.name, 'root')
+        self.assertEqual(result.pool, self.POOL_NAME)
+        self.assertEqual(result.size, defaults['root_img_size'])
+
+    def test_003_read_volume(self):
+        template = self.app.default_template
+        original_path = template.volumes['root'].vid
+        original_size = qubes.config.defaults['root_img_size']
+        config = {
+            'name': 'root',
+            'pool': 'default',
+            'volume_type': 'read-only',
+            'vid': original_path
+        }
+        vm = TestVM(self, template=template)
+
+        result = self.app.get_pool(self.POOL_NAME).init_volume(vm, config)
+        self.assertIsInstance(result, ReadOnlyFile)
+        self.assertEqual(result.name, 'root')
+        self.assertEqual(result.pool, 'default')
+        self.assertEqual(result.size, original_size)
+
+    def test_004_volatile_volume(self):
+        config = {
+            'name': 'root',
+            'pool': self.POOL_NAME,
+            'volume_type': 'volatile',
+            'size': defaults['root_img_size'],
+        }
+        vm = TestVM(self)
+        result = self.app.get_pool(self.POOL_NAME).init_volume(vm, config)
+        self.assertIsInstance(result, VolatileFile)
+        self.assertEqual(result.name, 'root')
+        self.assertEqual(result.pool, self.POOL_NAME)
+        self.assertEqual(result.size, defaults['root_img_size'])
+
+    def test_005_appvm_volumes(self):
+        ''' Check if AppVM volumes are propertly initialized '''
+        vmname = self.make_vm_name('appvm')
+        vm = self.app.add_new_vm(qubes.vm.appvm.AppVM,
+                                 name=vmname,
+                                 template=self.app.default_template,
+                                 label='red')
+
+        volumes = vm.volumes
+        self.assertIsInstance(volumes['root'], SnapshotFile)
+        self.assertIsInstance(volumes['private'], ReadWriteFile)
+        self.assertIsInstance(volumes['volatile'], VolatileFile)
+        expected = vm.template.dir_path + '/root.img:' + vm.template.dir_path \
+            + '/root-cow.img'
+        self.assertVolumePath(vm, 'root', expected, rw=False)
+        expected = vm.dir_path + '/private.img'
+        self.assertVolumePath(vm, 'private', expected, rw=True)
+        expected = vm.dir_path + '/volatile.img'
+        self.assertVolumePath(vm, 'volatile', expected, rw=True)
+
+    def test_006_template_volumes(self):
+        ''' Check if TemplateVM volumes are propertly initialized '''
+        vmname = self.make_vm_name('appvm')
+        vm = self.app.add_new_vm(qubes.vm.templatevm.TemplateVM,
+                                 name=vmname,
+                                 label='red')
+
+        volumes = vm.volumes
+        self.assertIsInstance(volumes['root'], OriginFile)
+        self.assertIsInstance(volumes['private'], ReadWriteFile)
+        self.assertIsInstance(volumes['volatile'], VolatileFile)
+        expected = vm.dir_path + '/root.img:' + vm.dir_path + '/root-cow.img'
+        self.assertVolumePath(vm, 'root', expected, rw=True)
+        expected = vm.dir_path + '/private.img'
+        self.assertVolumePath(vm, 'private', expected, rw=True)
+        expected = vm.dir_path + '/volatile.img'
+        self.assertVolumePath(vm, 'volatile', expected, rw=True)
+
+    def assertVolumePath(self, vm, dev_name, expected, rw=True):
+        # :pylint: disable=invalid-name
+        volumes = vm.volumes
+        b_dev = volumes[dev_name].block_device()
+        self.assertEqual(b_dev.rw, rw)
+        self.assertEquals(b_dev.path, expected)
+
 
 @qubes.tests.skipUnlessDom0
-class TC_01_XenPool(QubesTestCase):
+class TC_03_XenPool(SystemTestsMixin, QubesTestCase):
 
     """ Test the paths for the default Xen file based storage (``XenStorage``).
     """
@@ -68,172 +210,124 @@ class TC_01_XenPool(QubesTestCase):
     APPVMS_DIR = '/var/lib/qubes/test-pool/appvms'
     TEMPLATES_DIR = '/var/lib/qubes/test-pool/vm-templates'
     SERVICE_DIR = '/var/lib/qubes/test-pool/servicevms'
+    POOL_NAME = 'test-pool'
+    POOL_CONFIG = {'driver': 'xen', 'dir_path': POOL_DIR, 'name': POOL_NAME}
 
     def setUp(self):
         """ Add a test file based storage pool """
-        super(TC_01_XenPool, self).setUp()
-        qubes.storage.add_pool('test-pool', driver='xen',
-                               dir_path=self.POOL_DIR)
-        self.app = qubes.tests.storage.TestApp()
-        self.template = qubes.tests.storage.TestTemplateVM(self.app, 1,
-            self.make_vm_name('template'), 'default')
-
+        super(TC_03_XenPool, self).setUp()
+        self.init_default_template()
+        self.app.add_pool(**self.POOL_CONFIG)
 
     def tearDown(self):
         """ Remove the file based storage pool after testing """
-        super(TC_01_XenPool, self).tearDown()
-        qubes.storage.remove_pool("test-pool")
+        self.app.remove_pool("test-pool")
+        super(TC_03_XenPool, self).tearDown()
         shutil.rmtree(self.POOL_DIR, ignore_errors=True)
 
     def test_001_pool_exists(self):
         """ Check if the storage pool was added to the storage pool config """
-        self.assertTrue(qubes.storage.pool_exists('test-pool'))
+        self.assertIn('test-pool', self.app.pools.keys())
 
     def test_002_pool_dir_create(self):
         """ Check if the storage pool dir and subdirs were created """
-
         # The dir should not exists before
-        self.assertFalse(os.path.exists(self.POOL_DIR))
+        pool_name = 'foo'
+        pool_dir = '/tmp/foo'
+        appvms_dir = '/tmp/foo/appvms'
+        templates_dir = '/tmp/foo/vm-templates'
 
-        vmname = self.make_vm_name('appvm')
-        qubes.tests.storage.TestVM(self.app, qid=2, name=vmname,
-            template=self.template, pool_name='test-pool')
+        self.assertFalse(os.path.exists(pool_dir))
 
-        self.assertTrue(os.path.exists(self.POOL_DIR))
-        self.assertTrue(os.path.exists(self.APPVMS_DIR))
-        self.assertTrue(os.path.exists(self.SERVICE_DIR))
-        self.assertTrue(os.path.exists(self.TEMPLATES_DIR))
+        self.app.add_pool(name=pool_name, dir_path=pool_dir, driver='xen')
 
-    def test_003_pool_dir(self):
-        """ Check if the vm storage pool_dir is the same as specified """
-        vmname = self.make_vm_name('appvm')
-        vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname,
-            template=self.template, pool_name='test-pool')
-        result = qubes.storage.get_pool('test-pool', vm).dir_path
-        self.assertEquals(self.POOL_DIR, result)
+        self.assertTrue(os.path.exists(pool_dir))
+        self.assertTrue(os.path.exists(appvms_dir))
+        self.assertTrue(os.path.exists(templates_dir))
 
-    def test_004_app_vmdir(self):
-        """ Check the vm storage dir for an AppVm"""
-        vmname = self.make_vm_name('appvm')
-        vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname,
-            template=self.template, pool_name='test-pool')
-
-        expected = os.path.join(self.APPVMS_DIR, vm.name)
-        result = vm.storage.vmdir
-        self.assertEquals(expected, result)
-
-    def test_005_hvm_vmdir(self):
-        """ Check the vm storage dir for a HVM"""
-        vmname = self.make_vm_name('hvm')
-        vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname,
-            template=self.template, pool_name='test-pool')
-        vm.hvm = True
-
-        expected = os.path.join(self.APPVMS_DIR, vm.name)
-        result = vm.storage.vmdir
-        self.assertEquals(expected, result)
-
-    @unittest.skip('TODO - servicevms dir?')
-    def test_006_net_vmdir(self):
-        """ Check the vm storage dir for a Netvm"""
-        vmname = self.make_vm_name('hvm')
-        vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname,
-            template=self.template, pool_name='test-pool')
-
-        expected = os.path.join(self.SERVICE_DIR, vm.name)
-        result = vm.storage.vmdir
-        self.assertEquals(expected, result)
-
-    @unittest.skip('TODO - servicevms dir?')
-    def test_007_proxy_vmdir(self):
-        """ Check the vm storage dir for a ProxyVm"""
-        vmname = self.make_vm_name('proxyvm')
-        vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname,
-            template=self.template, pool_name='test-pool')
-
-        expected = os.path.join(self.SERVICE_DIR, vm.name)
-        result = vm.storage.vmdir
-        self.assertEquals(expected, result)
-
-    def test_008_admin_vmdir(self):
-        """ Check the vm storage dir for a AdminVm"""
-        # TODO How to test AdminVm?
-        pass
-
-    def test_009_template_vmdir(self):
-        """ Check the vm storage dir for a TemplateVm"""
-        vmname = self.make_vm_name('templatevm')
-        vm = qubes.tests.storage.TestTemplateVM(self.app, qid=2, name=vmname,
-            pool_name='test-pool')
-
-        expected = os.path.join(self.TEMPLATES_DIR, vm.name)
-        result = vm.storage.vmdir
-        self.assertEquals(expected, result)
-
-    def test_010_template_hvm_vmdir(self):
-        """ Check the vm storage dir for a TemplateHVm"""
-        vmname = self.make_vm_name('templatehvm')
-        vm = qubes.tests.storage.TestTemplateVM(self.app, qid=2, name=vmname,
-            pool_name='test-pool')
-
-        expected = os.path.join(self.TEMPLATES_DIR, vm.name)
-        result = vm.storage.vmdir
-        self.assertEquals(expected, result)
+        shutil.rmtree(pool_dir, ignore_errors=True)
 
     def test_011_appvm_file_images(self):
         """ Check if all the needed image files are created for an AppVm"""
 
         vmname = self.make_vm_name('appvm')
-        vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname,
-            pool_name='test-pool')
+        vm = self.app.add_new_vm(qubes.vm.appvm.AppVM,
+                                 name=vmname,
+                                 template=self.app.default_template,
+                                 volume_config={
+                                     'private': {
+                                         'pool': 'test-pool'
+                                     },
+                                     'volatile': {
+                                         'pool': 'test-pool'
+                                     }
+                                 },
+                                 label='red')
         vm.storage.create_on_disk()
 
         expected_vmdir = os.path.join(self.APPVMS_DIR, vm.name)
-        self.assertEqualsAndExists(vm.storage.vmdir, expected_vmdir)
 
         expected_private_path = os.path.join(expected_vmdir, 'private.img')
-        self.assertEqualsAndExists(vm.storage.private_img,
+        self.assertEqualsAndExists(vm.volumes['private'].path,
                                    expected_private_path)
 
         expected_volatile_path = os.path.join(expected_vmdir, 'volatile.img')
-        self.assertEqualsAndExists(vm.storage.volatile_img,
+        self.assertEqualsAndExists(vm.volumes['volatile'].path,
                                    expected_volatile_path)
 
-    def test_012_hvm_file_images(self):
-        """ Check if all the needed image files are created for a HVm"""
-
-        vmname = self.make_vm_name('hvm')
-        vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname,
-            pool_name='test-pool')
-        vm.hvm = True
-        vm.storage.create_on_disk()
-
-        expected_vmdir = os.path.join(self.APPVMS_DIR, vm.name)
-        self.assertEqualsAndExists(vm.storage.vmdir, expected_vmdir)
+    def test_013_template_file_images(self):
+        """ Check if root.img, private.img, volatile.img and root-cow.img are
+            created propertly by the storage system
+        """
+        vmname = self.make_vm_name('tmvm')
+        vm = self.app.add_new_vm(qubes.vm.templatevm.TemplateVM,
+                                 name=vmname,
+                                 volume_config={
+                                     'root': {
+                                         'pool': 'test-pool'
+                                     },
+                                     'private': {
+                                         'pool': 'test-pool'
+                                     },
+                                     'volatile': {
+                                         'pool': 'test-pool'
+                                     }
+                                 },
+                                 label='red')
+        vm.create_on_disk()
+
+        expected_vmdir = os.path.join(self.TEMPLATES_DIR, vm.name)
+
+        expected_root_origin_path = os.path.join(expected_vmdir, 'root.img')
+        expected_root_cow_path = os.path.join(expected_vmdir, 'root-cow.img')
+        expected_root_path = '%s:%s' % (expected_root_origin_path,
+                                        expected_root_cow_path)
+        self.assertEquals(vm.volumes['root'].path, expected_root_path)
+        self.assertExist(vm.volumes['root'].path_origin)
 
         expected_private_path = os.path.join(expected_vmdir, 'private.img')
-        self.assertEqualsAndExists(vm.storage.private_img,
+        self.assertEqualsAndExists(vm.volumes['private'].path,
                                    expected_private_path)
 
-        expected_root_path = os.path.join(expected_vmdir, 'root.img')
-        self.assertEqualsAndExists(vm.storage.root_img, expected_root_path)
-
         expected_volatile_path = os.path.join(expected_vmdir, 'volatile.img')
-        self.assertEqualsAndExists(vm.storage.volatile_img,
+        self.assertEqualsAndExists(vm.volumes['volatile'].path,
                                    expected_volatile_path)
 
-    @unittest.skip('test not implemented') # TODO
-    def test_013_template_based_file_images(self):
-        pass
+        vm.storage.commit_template_changes()
+        expected_rootcow_path = os.path.join(expected_vmdir, 'root-cow.img')
+        self.assertEqualsAndExists(vm.volumes['root'].path_cow,
+                                   expected_rootcow_path)
 
     def assertEqualsAndExists(self, result_path, expected_path):
         """ Check if the ``result_path``, matches ``expected_path`` and exists.
 
             See also: :meth:``assertExist``
         """
+        # :pylint: disable=invalid-name
         self.assertEquals(result_path, expected_path)
         self.assertExist(result_path)
 
     def assertExist(self, path):
         """ Assert that the given path exists. """
-        self.assertTrue(os.path.exists(path))
+        # :pylint: disable=invalid-name
+        self.assertTrue(os.path.exists(path), "Path %s does not exist" % path)

qubes/tools/qvm_create.py

@@ -46,9 +46,10 @@ parser.add_argument('--property', '--prop', '-p',
     action=qubes.tools.PropertyAction,
     help='set domain\'s property, like "internal", "memory" or "vcpus"')
 
-parser.add_argument('--pool-name', '--pool', '-P',
-    action=qubes.tools.SinglePropertyAction,
-    help='specify the storage pool to use')
+parser.add_argument('--pool', '-P',
+                    action='append',
+                    metavar='POOL_NAME:VOLUME_NAME',
+                    help='specify the pool to use for a volume')
 
 parser.add_argument('--template', '-t',
     action=qubes.tools.SinglePropertyAction,
@@ -79,6 +80,17 @@ parser.add_argument('name', metavar='VMNAME',
 def main(args=None):
     args = parser.parse_args(args)
 
+    if args.pool:
+        args.properties['volume_config'] = {}
+        for pool_vol in args.pool:
+            try:
+                pool_name, volume_name = pool_vol.split(':')
+                config = {'pool': pool_name, 'name': volume_name}
+                args.properties['volume_config'][volume_name] = config
+            except ValueError:
+                parser.error(
+                    'Pool argument must be of form: -P pool_name:volume_name')
+
     if 'label' not in args.properties:
         parser.error('--label option is mandatory')
 

qubes/vm/appvm.py

@@ -3,16 +3,44 @@
 
 import qubes.events
 import qubes.vm.qubesvm
+from qubes.config import defaults
+
 
 class AppVM(qubes.vm.qubesvm.QubesVM):
     '''Application VM'''
 
-    template = qubes.VMProperty('template', load_stage=4,
-        vmclass=qubes.vm.templatevm.TemplateVM,
-        ls_width=31,
-        doc='Template, on which this AppVM is based.')
+    template = qubes.VMProperty('template',
+                                load_stage=4,
+                                vmclass=qubes.vm.templatevm.TemplateVM,
+                                ls_width=31,
+                                doc='Template, on which this AppVM is based.')
 
     def __init__(self, *args, **kwargs):
+        self.volumes = {}
+        self.volume_config = {
+            'root': {
+                'name': 'root',
+                'pool': 'default',
+                'volume_type': 'snapshot',
+            },
+            'private': {
+                'name': 'private',
+                'pool': 'default',
+                'volume_type': 'read-write',
+                'size': defaults['private_img_size'],
+            },
+            'volatile': {
+                'name': 'volatile',
+                'pool': 'default',
+                'volume_type': 'volatile',
+                'size': defaults['root_img_size'],
+            },
+            'kernel': {
+                'name': 'kernel',
+                'pool': 'linux-kernel',
+                'volume_type': 'read-only',
+            }
+        }
         super(AppVM, self).__init__(*args, **kwargs)
 
     @qubes.events.handler('domain-load')
@@ -20,4 +48,4 @@ class AppVM(qubes.vm.qubesvm.QubesVM):
         # pylint: disable=unused-argument
         # Some additional checks for template based VM
         assert self.template
-        #self.template.appvms.add(self) # XXX
+        # self.template.appvms.add(self) # XXX

qubes/vm/qubesvm.py

@@ -29,6 +29,7 @@ from __future__ import absolute_import
 import base64
 import datetime
 import itertools
+import lxml
 import os
 import os.path
 import re
@@ -209,7 +210,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
         default='default',
         doc='storage pool for this qube devices')
 
-    dir_path = property((lambda self: self.storage.vmdir),
+    dir_path = property( (lambda self: os.path.join(qubes.config.system_path['qubes_base_dir'], self.dir_path_prefix, self.name)),
         doc='Root directory for files related to this domain')
 
     # XXX swallowed uses_default_kernel
@@ -331,6 +332,12 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
                 raise
         return self._libvirt_domain
 
+    @property
+    def block_devices(self):
+        ''' Return all :py:class:`qubes.devices.BlockDevice`s for current domain
+            for serialization in the libvirt XML template as <disk>.
+        '''
+        return [v.block_device() for v in self.volumes.values()]
 
     @property
     def qdb(self):
@@ -347,21 +354,27 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
     def private_img(self):
         '''Location of private image of the VM (that contains :file:`/rw` \
         and :file:`/home`).'''
-        return self.storage.private_img
+        warnings.warn("volatile_img is deprecated, use volumes['private'].vid",
+                      DeprecationWarning)
+        return self.volumes['private'].vid
 
 
     # XXX this should go to to AppVM? or TemplateVM?
     @property
     def root_img(self):
         '''Location of root image.'''
-        return self.storage.root_img
+        warnings.warn("root_img is deprecated, use volumes['root'].vid",
+                      DeprecationWarning)
+        return self.volumes['root'].vid
 
 
     # XXX and this should go to exactly where? DispVM has it.
     @property
     def volatile_img(self):
         '''Volatile image that overlays :py:attr:`root_img`.'''
-        return self.storage.volatile_img
+        warnings.warn("volatile_img is deprecated, use volumes['volatile'].vid",
+                      DeprecationWarning)
+        return self.volumes['volatile'].vid
 
 
     # XXX shouldn't this go elsewhere?
@@ -419,12 +432,27 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
     # constructor
     #
 
-    def __init__(self, app, xml, **kwargs):
+    def __init__(self, app, xml, volume_config={}, **kwargs):
         super(QubesVM, self).__init__(app, xml, **kwargs)
+        if hasattr(self, 'volume_config'):
+            if xml is not None:
+                for node in xml.xpath('volume-config/volume'):
+                    name = node.get('name')
+                    assert name
+                    for k, v in node.items():
+                        self.volume_config[name][k] = v
+
+            for name, conf in volume_config.items():
+                for k, v in conf.items():
+                    self.volume_config[name][k] = v
+        elif volume_config:
+            raise TypeError(
+                'volume_config specified, but {} did not expect that.' %
+                self.__class__.__name__)
 
-        import qubes.vm.adminvm # pylint: disable=redefined-outer-name
+        import qubes.vm.adminvm  # pylint: disable=redefined-outer-name
 
-        #Init private attrs
+        # Init private attrs
 
         self._libvirt_domain = None
         self._qdb_connection = None
@@ -458,13 +486,23 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
                 self.features['check-updates'] = None
 
         # will be initialized after loading all the properties
-        self.storage = None
 
         # fire hooks
         if xml is None:
             self.events_enabled = True
         self.fire_event('domain-init')
 
+    def __xml__(self):
+        element = super(QubesVM, self).__xml__()
+        if hasattr(self, 'volumes'):
+            volume_config_node = lxml.etree.Element('volume-config')
+            for volume in self.volumes.values():
+                volume_config_node.append(volume.__xml__())
+
+            element.append(volume_config_node)
+
+                
+        return element
 
     #
     # event handlers
@@ -477,8 +515,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
             self.uuid = uuid.uuid4()
 
         # Initialize VM image storage class
-        self.storage = qubes.storage.get_pool(
-            self.pool_name, self).get_storage()
+        self.storage = qubes.storage.Storage(self)
 
 
     @qubes.events.handler('property-set:label')
@@ -532,18 +569,18 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
             self._qdb_connection.close()
             self._qdb_connection = None
 
-        self.storage.rename(
-            os.path.join(qubes.config.system_path['qubes_base_dir'],
-                self.dir_path_prefix, new_name),
-            os.path.join(qubes.config.system_path['qubes_base_dir'],
-                self.dir_path_prefix, old_name))
+        self.storage.rename(old_name, new_name)
+
+        prefix = os.path.join(qubes.config.system_path['qubes_base_dir'], self.dir_path_prefix)
+        old_config = os.path.join(prefix, old_name, old_name + '.conf')
+        new_config = os.path.join(prefix, new_name, new_name + '.conf')
+        os.rename(old_config, new_config)
 
         self._update_libvirt_domain()
 
         if self.autostart:
             self.autostart = self.autostart
 
-
     @qubes.events.handler('property-pre-set:autostart')
     def on_property_pre_set_autostart(self, event, prop, name, value,
             oldvalue=None):
@@ -633,7 +670,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
                     self.netvm.start(start_guid=start_guid,
                         notify_function=notify_function)
 
-        self.storage.prepare_for_vm_startup()
+        self.storage.start()
         self._update_libvirt_domain()
 
         qmemman_client = self.request_memory(mem_required)
@@ -725,6 +762,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
                     exc_info=1)
 
         self.libvirt_domain.shutdown()
+        self.storage.stop()
 
 
     def kill(self):
@@ -738,6 +776,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
             raise qubes.exc.QubesVMNotStartedError(self)
 
         self.libvirt_domain.destroy()
+        self.storage.stop()
 
 
     def force_shutdown(self, *args, **kwargs):
@@ -1021,7 +1060,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
         p.communicate(input=self.default_user)
 
 
-    # TODO move to storage
+    # TODO rename to create
     def create_on_disk(self, source_template=None):
         '''Create files needed for VM.
 
@@ -1050,11 +1089,11 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
     def resize_private_img(self, size):
         '''Resize private image.'''
 
-        if size >= self.get_private_img_sz():
-            raise qubes.exc.QubesValueError('Cannot shrink private.img')
+        warnings.warn(
+            "resize_private_img is deprecated, use volumes[name].resize()",
+            DeprecationWarning)
 
-        # resize the image
-        self.storage.resize_private_img(size)
+        self.volumes['private'].resize(size)
 
         # and then the filesystem
         # FIXME move this to qubes.storage.xen.XenVMStorage
@@ -1070,29 +1109,12 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
         if retcode != 0:
             raise qubes.exc.QubesException('resize2fs failed')
 
-
-    # TODO move to storage
     def resize_root_img(self, size, allow_start=False):
-        if hasattr(self, 'template'):
-            raise qubes.exc.QubesVMError(self,
-                'Cannot resize root.img of template based qube. Resize the'
-                ' root.img of the template instead.')
-
-        # TODO self.is_halted
-        if self.is_running():
-            raise qubes.exc.QubesVMNotHaltedError(self,
-                'Cannot resize root.img of a running qube')
-
-        if size < self.get_root_img_sz():
-            raise qubes.exc.QubesValueError(
-                'For your own safety, shrinking of root.img is disabled. If you'
-                ' really know what you are doing, use `truncate` manually.')
-
-        with open(self.root_img, 'a+b') as fd:
-            fd.truncate(size)
+        warnings.warn(
+            "resize_root_img is deprecated, use volumes[name].resize()",
+            DeprecationWarning)
 
-        if False: #self.hvm:
-            return
+        self.volumes['root'].resize(size)
 
         if not allow_start:
             raise qubes.exc.QubesException(
@@ -1111,12 +1133,11 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
         while self.is_running(): #1696
             time.sleep(1)
 
-
     def remove_from_disk(self):
         '''Remove domain remnants from disk.'''
         self.fire_event('domain-remove-from-disk')
-        self.storage.remove_from_disk()
-
+        self.storage.remove()
+        shutil.rmtree(self.vm.dir_path)
 
     def clone_disk_files(self, src):
         '''Clone files from other vm.
@@ -1124,11 +1145,14 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
         :param qubes.vm.qubesvm.QubesVM src: source VM
         '''
 
-        if src.is_running(): # XXX what about paused?
+        if src.is_running():  # XXX what about paused?
             raise qubes.exc.QubesVMNotHaltedError(
                 self, 'Cannot clone a running domain {!r}'.format(self.name))
 
-        self.storage.clone_disk_files(src)
+        if hasattr(src, 'volume_config'):
+            self.volume_config = src.volume_config
+        self.storage = qubes.storage.Storage(self)
+        self.storage.clone(src)
 
         if src.icon_path is not None \
                 and os.path.exists(src.dir_path) \
@@ -1450,68 +1474,69 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
 
     # XXX shouldn't this go only to vms that have root image?
     def get_disk_utilization_root_img(self):
-        '''Get space that is actually ocuppied by :py:attr:`root_img`.
-
-        Root image is a sparse file, so it is probably much less than logical
-        available space.
+        '''Get space that is actually ocuppied by :py:attr:`volumes['root']`.
 
+           This is a temporary wrapper for backwards compatibility. You should
+           call directly :py:attr:`volumes[name].utilization`
         :returns: domain's real disk image size [FIXME unit]
         :rtype: FIXME
 
         .. seealso:: :py:meth:`get_root_img_sz`
         '''
 
-        return qubes.storage.get_disk_usage(self.root_img)
+        warnings.warn(
+            "get_disk_utilization_root_img is deprecated, use volumes['root'].utilization",
+            DeprecationWarning)
+        return qubes.storage.get_disk_usage(self.volumes['root'].utilization)
 
 
     # XXX shouldn't this go only to vms that have root image?
     def get_root_img_sz(self):
-        '''Get image size of :py:attr:`root_img`.
-
-        Root image is a sparse file, so it is probably much more than ocuppied
-        physical space.
+        '''Get the size of the :py:attr:`volumes['root']`.
 
+        This is a temporary wrapper for backwards compatibility. You should
+        call directly :py:attr:`volumes[name].size`
         :returns: domain's virtual disk size [FIXME unit]
         :rtype: FIXME
 
         .. seealso:: :py:meth:`get_disk_utilization_root_img`
         '''
 
-        if not os.path.exists(self.root_img):
-            return 0
-
-        return os.path.getsize(self.root_img)
-
+        warnings.warn(
+            "get_disk_root_img_sz is deprecated, use volumes['root'].size",
+            DeprecationWarning)
+        return qubes.storage.get_disk_usage(self.volumes['root'].size)
 
     def get_disk_utilization_private_img(self):
-        '''Get space that is actually ocuppied by :py:attr:`private_img`.
-
-        Private image is a sparse file, so it is probably much less than
-        logical available space.
+        '''Get space that is actually ocuppied by :py:attr:`volumes['private']`.
 
+           This is a temporary wrapper for backwards compatibility. You should
+           call directly :py:attr:`volumes[name].utilization`
         :returns: domain's real disk image size [FIXME unit]
         :rtype: FIXME
+        '''
 
-        .. seealso:: :py:meth:`get_private_img_sz`
-        ''' # pylint: disable=invalid-name
-
-        return qubes.storage.get_disk_usage(self.private_img)
-
+        warnings.warn(
+            "get_disk_utilization_private_img is deprecated, use volumes['private'].utilization",
+            DeprecationWarning)
+        return qubes.storage.get_disk_usage(self.volumes[
+            'private'].utilization)
 
     def get_private_img_sz(self):
-        '''Get image size of :py:attr:`private_img`.
-
-        Private image is a sparse file, so it is probably much more than
-        ocuppied physical space.
+        '''Get the size of the :py:attr:`volumes['private']`.
 
+        This is a temporary wrapper for backwards compatibility. You should
+        call directly :py:attr:`volumes[name].size`
         :returns: domain's virtual disk size [FIXME unit]
         :rtype: FIXME
 
         .. seealso:: :py:meth:`get_disk_utilization_private_img`
         '''
 
-        return self.storage.get_private_img_sz()
-
+        warnings.warn(
+            "get_disk_private_img_sz is deprecated, use volumes['private'].size",
+            DeprecationWarning)
+        return qubes.storage.get_disk_usage(self.volumes['private'].size)
 
     def get_disk_utilization(self):
         '''Return total space actually occuppied by all files belonging to \
@@ -1523,7 +1548,6 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
 
         return qubes.storage.get_disk_usage(self.dir_path)
 
-
     # TODO move to storage
     def verify_files(self):
         '''Verify that files accessed by this machine are sane.
@@ -1533,18 +1557,6 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
 
         self.storage.verify_files()
 
-        if not os.path.exists(
-                os.path.join(self.storage.kernels_dir, 'vmlinuz')):
-            raise qubes.exc.QubesException(
-                'VM kernel does not exist: {0}'.format(
-                    os.path.join(self.storage.kernels_dir, 'vmlinuz')))
-
-        if not os.path.exists(
-                os.path.join(self.storage.kernels_dir, 'initramfs')):
-            raise qubes.exc.QubesException(
-                'VM initramfs does not exist: {0}'.format(
-                    os.path.join(self.storage.kernels_dir, 'initramfs')))
-
         self.fire_event('domain-verify-files')
 
         return True

qubes/vm/templatevm.py

@@ -1,11 +1,16 @@
 #!/usr/bin/python2 -O
 # vim: fileencoding=utf-8
 
+import warnings
+
 import qubes
 import qubes.config
 import qubes.vm.qubesvm
+from qubes.config import defaults
+from qubes.vm.qubesvm import QubesVM
+
 
-class TemplateVM(qubes.vm.qubesvm.QubesVM):
+class TemplateVM(QubesVM):
     '''Template for AppVM'''
 
     dir_path_prefix = qubes.config.system_path['qubes_templates_dir']
@@ -13,7 +18,9 @@ class TemplateVM(qubes.vm.qubesvm.QubesVM):
     @property
     def rootcow_img(self):
         '''COW image'''
-        return self.storage.rootcow_img
+        warnings.warn("rootcow_img is deprecated, use "
+                      "volumes['root'].path_origin", DeprecationWarning)
+        return self.volumes['root'].path_cow
 
     @property
     def appvms(self):
@@ -23,20 +30,40 @@ class TemplateVM(qubes.vm.qubesvm.QubesVM):
 
     def __init__(self, *args, **kwargs):
         assert 'template' not in kwargs, "A TemplateVM can not have a template"
+        self.volumes = {}
+        self.volume_config = {
+            'root': {
+                'name': 'root',
+                'pool': 'default',
+                'volume_type': 'origin',
+                'size': defaults['root_img_size'],
+            },
+            'private': {
+                'name': 'private',
+                'pool': 'default',
+                'volume_type': 'read-write',
+                'size': defaults['private_img_size'],
+            },
+            'volatile': {
+                'name': 'volatile',
+                'pool': 'default',
+                'size': defaults['root_img_size'],
+                'volume_type': 'volatile',
+            },
+            'kernel': {
+                'name': 'kernel',
+                'pool': 'linux-kernel',
+                'volume_type': 'read-only',
+            }
+        }
         super(TemplateVM, self).__init__(*args, **kwargs)
 
-        # Some additional checks for template based VM
-        # TODO find better way
-#       assert self.root_img is not None, "Missing root_img for standalone VM!"
-
-
     def clone_disk_files(self, src):
         super(TemplateVM, self).clone_disk_files(src)
 
         # Create root-cow.img
         self.commit_changes()
 
-
     def commit_changes(self):
         '''Commit changes to template'''
         self.log.debug('commit_changes()')
@@ -45,6 +72,4 @@ class TemplateVM(qubes.vm.qubesvm.QubesVM):
             assert not self.is_running(), \
                 'Attempt to commit changes on running Template VM!'
 
-        self.log.info(
-            'Commiting template update; COW: {}'.format(self.rootcow_img))
         self.storage.commit_template_changes()

rpm_spec/core-dom0.spec

@@ -197,7 +197,6 @@ fi
 %files
 %defattr(-,root,root,-)
 %config(noreplace) %attr(0664,root,qubes) %{_sysconfdir}/qubes/qmemman.conf
-%config(noreplace) %attr(0664,root,qubes) %{_sysconfdir}/qubes/storage.conf
 /usr/bin/qvm-*
 /usr/bin/qubes-*
 /usr/bin/qmemmand
@@ -234,6 +233,7 @@ fi
 %dir %{python_sitelib}/qubes/storage
 %{python_sitelib}/qubes/storage/__init__.py*
 %{python_sitelib}/qubes/storage/xen.py*
+%{python_sitelib}/qubes/storage/kernels.py*
 
 %dir %{python_sitelib}/qubes/tools
 %{python_sitelib}/qubes/tools/__init__.py*

setup.py

@@ -45,5 +45,6 @@ if __name__ == '__main__':
             ],
             'qubes.storage': [
                 'xen = qubes.storage.xen:XenPool',
+                'linux-kernel = qubes.storage.kernels:LinuxKernel',
             ]
         })

templates/libvirt/xen.xml

@@ -45,12 +45,27 @@
 	<on_reboot>destroy</on_reboot>
 	<on_crash>destroy</on_crash>
 	<devices>
-        {#
-		{% for device in vm.storage %}
-            <disk type="block" device="{{ device.type }}">
+        {% set i = 0 %}
+        {# TODO Allow more volumes out of the box #}
+        {% set dd = ['e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p',
+            'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y']
+        %}
+		{% for device in vm.block_devices %}
+            <disk type="block" device="{{ device.devtype }}">
                 <driver name="phy" />
                 <source dev="{{ device.path }}" />
-                <target dev="{{ device.vdev }}" />
+                {% if device.name == 'root' %}
+                    <target dev="xvda" />
+                {% elif device.name == 'private' %}
+                    <target dev="xvdb" />
+                {% elif device.name == 'volatile' %}
+                    <target dev="xvdc" />
+                {% elif device.name == 'kernel' %}
+                    <target dev="xvdd" />
+                {% else %}
+                    <target dev="xvd{{dd[i]}}" />
+                    {% set i = i + 1 %}
+                {% endif %}
 
                 {% if not device.rw %}
                     <readonly />
@@ -65,15 +80,6 @@
                 {% endif %}
             </disk>
 		{% endfor %}
-        #}
-
-        {{ vm.storage.root_dev_config() }}
-        {% if not prepare_dvm %}{{ vm.storage.private_dev_config() }}{% endif %}
-        {{ vm.storage.other_dev_config() }}
-
-		{% if not vm.hvm %}
-            {{ vm.storage.volatile_dev_config() }}
-		{% endif %}
 
 		{% if vm.netvm %}
 			<interface type="ethernet">