diff --git a/Makefile b/Makefile index 01edd38a..99d1491d 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,6 @@ endif # $(MAKE) install -C tests $(MAKE) install -C relaxng mkdir -p $(DESTDIR)/etc/qubes - cp etc/storage.conf $(DESTDIR)/etc/qubes/ ifeq ($(BACKEND_VMM),xen) # Currently supported only on xen cp etc/qmemman.conf $(DESTDIR)/etc/qubes/ diff --git a/doc/manpages/qvm-create.rst b/doc/manpages/qvm-create.rst index c5ec3cee..c287b4b6 100644 --- a/doc/manpages/qvm-create.rst +++ b/doc/manpages/qvm-create.rst @@ -53,6 +53,10 @@ Options Use provided :file:`root.img` instead of default/empty one (file will be *moved*). This option is mutually exclusive with :option:`--root-copy-from`. +.. option:: --pool=POOL_NAME:VOLUME_NAME, -P POOL_NAME:VOLUME_NAME + + Specify the pool to use for a volume + Options for internal use ------------------------ @@ -71,5 +75,6 @@ Authors | Rafal Wojtczuk | Marek Marczykowski | Wojtek Porczyk +| Bahtiar `kalkin-` Gadimov .. vim: ts=3 sw=3 et tw=80 diff --git a/etc/storage.conf b/etc/storage.conf deleted file mode 100644 index 78fd2a3a..00000000 --- a/etc/storage.conf +++ /dev/null @@ -1,13 +0,0 @@ -[default] ; poolname -driver=xen ; the default xen storage -; class = qubes.storage.xen.XenStorage ; class always overwrites the driver -; -; To use our own pool driver it needs to provide `qubes.storage` entry_point -; see also: https://pythonhosted.org/setuptools/setuptools.html#dynamic-discovery-of-services-and-plugins -; class name -; [pool-b] -; driver = foo -; -; [test-dummy] -; driver=dummy - diff --git a/qubes/__init__.py b/qubes/__init__.py index a5e5c7eb..d14dd520 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -36,8 +36,6 @@ __author__ = 'Invisible Things Lab' __license__ = 'GPLv2 or later' __version__ = 'R3' -import ast -import atexit import collections import errno import grp @@ -47,12 +45,9 @@ import os.path import sys import tempfile import time -import warnings import __builtin__ -import docutils.core -import docutils.io import jinja2 import lxml.etree import pkg_resources @@ -1181,7 +1176,6 @@ class Qubes(PropertyHolder): default=True, doc='check for updates inside qubes') - def __init__(self, store=None, load=True, **kwargs): #: logger instance for logging global messages self.log = logging.getLogger('app') @@ -1194,6 +1188,9 @@ class Qubes(PropertyHolder): #: collection of all available labels for VMs self.labels = {} + #: collection of all pools + self.pools = {} + #: Connection to VMM self.vmm = VMMConnection() @@ -1235,7 +1232,7 @@ class Qubes(PropertyHolder): ''' try: - fd = os.open(self._store, os.O_RDWR) # no O_CREAT + fd = os.open(self._store, os.O_RDWR) # no O_CREAT except OSError as e: if e.errno != errno.ENOENT: raise @@ -1256,11 +1253,19 @@ class Qubes(PropertyHolder): self.xml = lxml.etree.parse(fh) - # stage 1: load labels + # stage 1: load labels and pools for node in self.xml.xpath('./labels/label'): label = Label.fromxml(node) self.labels[label.index] = label + for node in self.xml.xpath('./pools/pool'): + name = node.get('name') + assert name, "Pool name '%s' is invalid " % name + try: + self.pools[name] = self._get_pool(**node.attrib) + except qubes.exc.QubesException as e: + self.log.error(e.message) + # stage 2: load VMs for node in self.xml.xpath('./domains/domain'): # pylint: disable=no-member @@ -1270,7 +1275,7 @@ class Qubes(PropertyHolder): vm.init_log() self.domains.add(vm) - if not 0 in self.domains: + if 0 not in self.domains: self.domains.add(qubes.vm.adminvm.AdminVM( self, None, qid=0, name='dom0')) @@ -1310,11 +1315,17 @@ class Qubes(PropertyHolder): fh.close() del fh - def __xml__(self): element = lxml.etree.Element('qubes') element.append(self.xml_labels()) + + pools_xml = lxml.etree.Element('pools') + for pool in self.pools.values(): + pools_xml.append(pool.__xml__()) + + element.append(pools_xml) + element.append(self.xml_properties()) domains = lxml.etree.Element('domains') @@ -1395,6 +1406,10 @@ class Qubes(PropertyHolder): 7: Label(7, '0x75507b', 'purple'), 8: Label(8, '0x000000', 'black'), } + + for name, config in qubes.config.defaults['pool_configs'].items(): + self.pools[name] = self._get_pool(**config) + self.domains.add( qubes.vm.adminvm.AdminVM(self, None, qid=0, name='dom0')) self.save() @@ -1413,7 +1428,6 @@ class Qubes(PropertyHolder): labels.append(label.__xml__()) return labels - def get_vm_class(self, clsname): '''Find the class for a domain. @@ -1473,6 +1487,52 @@ class Qubes(PropertyHolder): raise KeyError(label) + def add_pool(self, **kwargs): + """ Add a storage pool to config.""" + name = kwargs['name'] + assert name not in self.pools.keys(), \ + "Pool named %s already exists" % name + pool = self._get_pool(**kwargs) + pool.setup() + self.pools[name] = pool + + def remove_pool(self, name): + """ Remove a storage pool from config file. """ + try: + pool = self.pools[name] + del self.pools[name] + pool.destroy() + except KeyError: + return + + + def get_pool(self, name): + ''' Returns a :py:class:`qubes.storage.Pool` instance ''' + try: + return self.pools[name] + except KeyError: + raise qubes.exc.QubesException('Unknown storage pool ' + name) + + def _get_pool(self, **kwargs): + try: + name = kwargs['name'] + assert name, 'Name needs to be an non empty string' + except KeyError: + raise qubes.exc.QubesException('No pool name for pool') + + try: + driver = kwargs['driver'] + except KeyError: + raise qubes.exc.QubesException('No driver specified for pool ' + + name) + try: + klass = qubes.get_entry_point_one( + qubes.storage.STORAGE_ENTRY_POINT, driver) + del kwargs['driver'] + return klass(**kwargs) + except KeyError: + raise qubes.exc.QubesException('Driver %s for pool %s' % + (driver, name)) @qubes.events.handler('domain-pre-delete') def on_domain_pre_deleted(self, event, vm): diff --git a/qubes/config.py b/qubes/config.py index a09e2f35..80a2981b 100644 --- a/qubes/config.py +++ b/qubes/config.py @@ -28,6 +28,8 @@ # make a real /etc/qubes/master.conf or whatever # +import os.path + '''Constants which can be configured in one place''' qubes_base_dir = "/var/lib/qubes" @@ -83,7 +85,17 @@ defaults = { 'private_img_size': 2*1024*1024*1024, 'root_img_size': 10*1024*1024*1024, - 'pool_config': {'dir_path': '/var/lib/qubes'}, + 'pool_configs': { + 'default': {'dir_path': qubes_base_dir, + 'driver': 'xen', + 'name': 'default'}, + 'linux-kernel': { + 'dir_path': os.path.join(qubes_base_dir, + system_path['qubes_kernels_base_dir']), + 'driver': 'linux-kernel', + 'name': 'linux-kernel' + } + }, # how long (in sec) to wait for VMs to shutdown, # before killing them (when used qvm-run with --wait option), diff --git a/qubes/devices.py b/qubes/devices.py index 0911344f..b9799b50 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -6,6 +6,7 @@ # # Copyright (C) 2010-2016 Joanna Rutkowska # Copyright (C) 2015-2016 Wojtek Porczyk +# Copyright (C) 2016 Bahtiar `kalkin-` Gadimov # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -26,6 +27,7 @@ import re import qubes + class DeviceCollection(object): '''Bag for devices. @@ -121,3 +123,16 @@ class RegexDevice(str): class PCIDevice(RegexDevice): regex = re.compile( r'^(?P[0-9a-f]+):(?P[0-9a-f]+)\.(?P[0-9a-f]+)$') + + +class BlockDevice(object): + def __init__(self, path, name, script=None, rw=True, domain=None, + devtype='disk'): + assert name, 'Missing device name' + assert path, 'Missing device path' + self.path = path + self.name = name + self.rw = rw + self.script = script + self.domain = domain + self.devtype = devtype diff --git a/qubes/storage/__init__.py b/qubes/storage/__init__.py index 4ddbe34b..5250b20c 100644 --- a/qubes/storage/__init__.py +++ b/qubes/storage/__init__.py @@ -23,24 +23,22 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # - """ Qubes storage system""" from __future__ import absolute_import -import ConfigParser import os import os.path import shutil -import subprocess import pkg_resources import qubes import qubes.exc import qubes.utils +from qubes.devices import BlockDevice + +import lxml.etree -BLKSIZE = 512 -CONFIG_FILE = '/etc/qubes/storage.conf' STORAGE_ENTRY_POINT = 'qubes.storage' @@ -48,84 +46,70 @@ class StoragePoolException(qubes.exc.QubesException): pass +class Volume(object): + ''' Encapsulates all data about a volume for serialization to qubes.xml and + libvirt config. + ''' + + devtype = 'disk' + domain = None + path = None + rw = True + script = None + usage = 0 + + def __init__(self, name, pool, volume_type, vid=None, size=0, **kwargs): + super(Volume, self).__init__(**kwargs) + self.name = str(name) + self.pool = str(pool) + self.vid = vid + self.size = size + self.volume_type = volume_type + + def __xml__(self): + return lxml.etree.Element('volume', **self.config) + + @property + def config(self): + ''' return config data for serialization to qubes.xml ''' + return {'name': self.name, + 'pool': self.pool, + 'volume_type': self.volume_type} + + def __repr__(self): + return '{}(name={!s}, pool={!r}, vid={!r}, volume_type={!r})'.format( + self.__class__.__name__, self.name, self.pool, self.vid, + self.volume_type) + + def block_device(self): + ''' Return :py:class:`qubes.devices.BlockDevice` for serialization in + the libvirt XML template as . + ''' + return BlockDevice(self.path, self.name, self.script, self.rw, + self.domain, self.devtype) + + class Storage(object): - '''Class for handling VM virtual disks. + ''' Class for handling VM virtual disks. This is base class for all other implementations, mostly with Xen on Linux in mind. ''' - root_img = None - private_img = None - volatile_img = None - - modules_dev = None - - def __init__(self, vm, private_img_size=None, root_img_size=None): - + def __init__(self, vm): #: Domain for which we manage storage self.vm = vm - - #: Size of the private image - self.private_img_size = private_img_size \ - if private_img_size is not None \ - else qubes.config.defaults['private_img_size'] - - #: Size of the root image - self.root_img_size = root_img_size \ - if root_img_size is not None \ - else qubes.config.defaults['root_img_size'] - + self.log = self.vm.log #: Additional drive (currently used only by HVM) self.drive = None - - def get_config_params(self): - args = {} - args['rootdev'] = self.root_dev_config() - args['privatedev'] = self.private_dev_config() - args['volatiledev'] = self.volatile_dev_config() - args['otherdevs'] = self.other_dev_config() - - args['kerneldir'] = self.kernels_dir - - return args - - def root_dev_config(self): - raise NotImplementedError() - - def private_dev_config(self): - raise NotImplementedError() - - def volatile_dev_config(self): - raise NotImplementedError() - - def other_dev_config(self): - if self.modules_img is not None: - return self.format_disk_dev(self.modules_img, self.modules_dev, - rw=self.modules_img_rw) - elif self.drive is not None: - (drive_type, drive_domain, drive_path) = self.drive.split(":") - if drive_type == 'hd': - drive_type = 'disk' - - rw = (drive_type == 'disk') - - if drive_domain.lower() == "dom0": - drive_domain = None - - return self.format_disk_dev(drive_path, - self.modules_dev, - rw=rw, - devtype=drive_type, - domain=drive_domain) - - else: - return '' - - def format_disk_dev(self, path, vdev, script=None, rw=True, devtype='disk', - domain=None): - raise NotImplementedError() - + self.pools = {} + if hasattr(vm, 'volume_config'): + for name, conf in self.vm.volume_config.items(): + assert 'pool' in conf, "Pool missing in volume_config" % str( + conf) + pool = self.vm.app.get_pool(conf['pool']) + self.vm.volumes[name] = pool.init_volume(self.vm, conf) + self.pools[name] = pool @property def kernels_dir(self): @@ -134,370 +118,183 @@ class Storage(object): If :py:attr:`self.vm.kernel` is :py:obj:`None`, the this points inside :py:attr:`self.vm.dir_path` ''' - return os.path.join(qubes.config.system_path['qubes_base_dir'], - qubes.config.system_path['qubes_kernels_base_dir'], self.vm.kernel)\ - if self.vm.kernel is not None \ - else os.path.join(self.vm.dir_path, - qubes.config.vm_files['kernels_subdir']) - - - @property - def modules_img(self): - '''Path to image with modules. - - Depending on domain, this may be global or inside domain's dir. - ''' - - modules_path = os.path.join(self.kernels_dir, 'modules.img') - - if os.path.exists(modules_path): - return modules_path - else: - return None - - - @property - def modules_img_rw(self): - ''':py:obj:`True` if module image should be mounted RW, :py:obj:`False` - otherwise.''' - return self.vm.kernel is None - - - def abspath(self, path, rel=None): - '''Make absolute path. - - If given path is relative, it is interpreted as relative to - :py:attr:`self.vm.dir_path` or given *rel*. - ''' - return path if os.path.isabs(path) \ - else os.path.join(rel or self.vm.dir_path, path) - - - @staticmethod - def _copy_file(source, destination): - '''Effective file copy, preserving sparse files etc. - ''' - # TODO: Windows support - - # We prefer to use Linux's cp, because it nicely handles sparse files - try: - subprocess.check_call(['cp', '--reflink=auto', source, destination]) - except subprocess.CalledProcessError: - raise IOError('Error while copying {!r} to {!r}'.format( - source, destination)) + assert 'kernel' in self.vm.volumes, "VM has no kernel pool" + return self.vm.volumes['kernel'].kernels_dir def get_disk_utilization(self): - return get_disk_usage(self.vm.dir_path) + ''' Returns summed up disk utilization for all domain volumes ''' + result = 0 + for volume in self.vm.volumes.values(): + result += volume.usage + return result + # TODO Remove this wrapper def get_disk_utilization_private_img(self): - # pylint: disable=invalid-name - return get_disk_usage(self.private_img) + # pylint: disable=invalid-name,missing-docstring + return self.vm.volume['private'].usage + # TODO Remove this wrapper def get_private_img_sz(self): - if not os.path.exists(self.private_img): - return 0 + # :pylint: disable=missing-docstring + return self.vm.volume['private'].size - return os.path.getsize(self.private_img) - - def resize_private_img(self, size): - raise NotImplementedError() - - def create_on_disk_private_img(self, source_template=None): - raise NotImplementedError() - - def create_on_disk_root_img(self, source_template=None): - raise NotImplementedError() + def resize(self, volume, size): + ''' Resize volume ''' + self.get_pool(volume).resize(volume, size) + # TODO rename it to create() def create_on_disk(self, source_template=None): + # :pylint: disable=missing-docstring if source_template is None and hasattr(self.vm, 'template'): source_template = self.vm.template old_umask = os.umask(002) - self.vm.log.info('Creating directory: {0}'.format(self.vm.dir_path)) - os.mkdir(self.vm.dir_path) - self.create_on_disk_private_img(source_template) - self.create_on_disk_root_img(source_template) - self.reset_volatile_storage() + self.log.info('Creating directory: {0}'.format(self.vm.dir_path)) + os.makedirs(self.vm.dir_path) + for name, volume in self.vm.volumes.items(): + source_volume = None + if source_template and hasattr(source_template, 'volumes'): + source_volume = source_template.volumes[name] + self.get_pool(volume).create(volume, source_volume=source_volume) os.umask(old_umask) - def clone_disk_files(self, src_vm): + def clone(self, src_vm): self.vm.log.info('Creating directory: {0}'.format(self.vm.dir_path)) - os.mkdir(self.vm.dir_path) - - if hasattr(src_vm, 'private_img'): - self.vm.log.info('Copying the private image: {} -> {}'.format( - src_vm.private_img, self.vm.private_img)) - self._copy_file(src_vm.private_img, self.vm.private_img) - - if src_vm.updateable and hasattr(src_vm, 'root_img'): - self.vm.log.info('Copying the root image: {} -> {}'.format( - src_vm.root_img, self.root_img)) - self._copy_file(src_vm.root_img, self.root_img) - - # TODO: modules? - # XXX which modules? -woju - - - @staticmethod - def rename(newpath, oldpath): - '''Move storage directory, most likely during domain's rename. - - .. note:: - The arguments are in different order than in :program:`cp` utility. - - .. versionchange:: 4.0 - This is now dummy method that just passes everything to - :py:func:`os.rename`. - - :param str newpath: New path - :param str oldpath: Old path - ''' - - os.rename(oldpath, newpath) + if not os.path.exists(self.vm.dir_path): + self.log.info('Creating directory: {0}'.format(self.vm.dir_path)) + os.makedirs(self.vm.dir_path) + for name, target in self.vm.volumes.items(): + pool = self.get_pool(target) + source = src_vm.volumes[name] + volume = pool.clone(source, target) + assert volume, "%s.clone() returned '%s'" % (pool.__class__, + volume) + self.vm.volumes[name] = volume + def rename(self, old_name, new_name): + ''' Notify the pools that the domain was renamed ''' + volumes = self.vm.volumes + for name, volume in volumes.items(): + pool = self.get_pool(volume) + volumes[name] = pool.rename(volume, old_name, new_name) def verify_files(self): + '''Verify that the storage is sane. + + On success, returns normally. On failure, raises exception. + ''' if not os.path.exists(self.vm.dir_path): - raise qubes.exc.QubesVMError(self.vm, + raise qubes.exc.QubesVMError( + self.vm, 'VM directory does not exist: {}'.format(self.vm.dir_path)) - if hasattr(self.vm, 'root_img') and not os.path.exists(self.root_img): - raise qubes.exc.QubesVMError(self.vm, - 'VM root image file does not exist: {}'.format(self.root_img)) - - if hasattr(self.vm, 'private_img') \ - and not os.path.exists(self.private_img): - raise qubes.exc.QubesVMError(self.vm, - 'VM private image file does not exist: {}'.format( - self.private_img)) - - if self.modules_img is not None \ - and not os.path.exists(self.modules_img): - raise qubes.exc.QubesVMError(self.vm, - 'VM kernel modules image does not exists: {}'.format( - self.modules_img)) - - - def remove_from_disk(self): + def remove(self): + for name, volume in self.vm.volumes.items(): + self.log.info('Removing volume %s: %s' % (name, volume.vid)) + self.get_pool(volume).remove(volume) shutil.rmtree(self.vm.dir_path) + def start(self): + ''' Execute the start method on each pool ''' + for volume in self.vm.volumes.values(): + self.get_pool(volume).start(volume) - def reset_volatile_storage(self): - # Re-create only for template based VMs - try: - if self.vm.template is not None and self.volatile_img: - if os.path.exists(self.volatile_img): - os.remove(self.volatile_img) - except AttributeError: # self.vm.template - pass + def stop(self): + ''' Execute the start method on each pool ''' + for volume in self.vm.volumes.values(): + self.get_pool(volume).stop(volume) - # For StandaloneVM create it only if not already exists - # (eg after backup-restore) - if hasattr(self, 'volatile_img') \ - and not os.path.exists(self.volatile_img): - self.vm.log.info( - 'Creating volatile image: {0}'.format(self.volatile_img)) - subprocess.check_call( - [qubes.config.system_path["prepare_volatile_img_cmd"], - self.volatile_img, - str(self.root_img_size / 1024 / 1024)]) + def get_pool(self, volume): + ''' Helper function ''' + assert isinstance(volume, Volume), "You need to pass a Volume" + return self.pools[volume.name] + def commit_template_changes(self): + for volume in self.vm.volumes.values(): + if volume.volume_type == 'origin': + self.get_pool(volume).commit_template_changes(volume) - def prepare_for_vm_startup(self): - self.reset_volatile_storage() - - if hasattr(self.vm, 'private_img') \ - and not os.path.exists(self.private_img): - self.vm.log.info('Creating empty VM private image file: {0}'.format( - self.private_img)) - self.create_on_disk_private_img() - - -def get_disk_usage_one(st): - '''Extract disk usage of one inode from its stat_result struct. - - If known, get real disk usage, as written to device by filesystem, not - logical file size. Those values may be different for sparse files. - - :param os.stat_result st: stat result - :returns: disk usage - ''' - try: - return st.st_blocks * BLKSIZE - except AttributeError: - return st.st_size - - -def get_disk_usage(path): - '''Get real disk usage of given path (file or directory). - - When *path* points to directory, then it is evaluated recursively. - - This function tries estiate real disk usage. See documentation of - :py:func:`get_disk_usage_one`. - - :param str path: path to evaluate - :returns: disk usage - ''' - try: - st = os.lstat(path) - except OSError: - return 0 - - ret = get_disk_usage_one(st) - - # if path is not a directory, this is skipped - for dirpath, dirnames, filenames in os.walk(path): - for name in dirnames + filenames: - ret += get_disk_usage_one(os.lstat(os.path.join(dirpath, name))) - - return ret - - -def get_pool(name, vm): - """ Instantiates the storage for the specified vm """ - config = _get_storage_config_parser() - - klass = _get_pool_klass(name, config) - - keys = [k for k in config.options(name) if k != 'driver' and k != 'class'] - values = [config.get(name, o) for o in keys] - config_kwargs = dict(zip(keys, values)) - - if name == 'default': - kwargs = qubes.config.defaults['pool_config'].copy() - kwargs.update(keys) - else: - kwargs = config_kwargs - - return klass(vm, **kwargs) - - -def pool_exists(name): - """ Check if the specified pool exists """ - try: - _get_pool_klass(name) - return True - except StoragePoolException: - return False - -def add_pool(name, **kwargs): - """ Add a storage pool to config.""" - config = _get_storage_config_parser() - config.add_section(name) - for key, value in kwargs.iteritems(): - config.set(name, key, value) - _write_config(config) - -def remove_pool(name): - """ Remove a storage pool from config file. """ - config = _get_storage_config_parser() - config.remove_section(name) - _write_config(config) - -def _write_config(config): - with open(CONFIG_FILE, 'w') as configfile: - config.write(configfile) - -def _get_storage_config_parser(): - """ Instantiates a `ConfigParaser` for specified storage config file. - - Returns: - RawConfigParser - """ - config = ConfigParser.RawConfigParser() - config.read(CONFIG_FILE) - return config - - -def _get_pool_klass(name, config=None): - """ Returns the storage klass for the specified pool. - - Args: - name: The pool name. - config: If ``config`` is not specified - `_get_storage_config_parser()` is called. - - Returns: - type: A class inheriting from `QubesVmStorage` - """ - if config is None: - config = _get_storage_config_parser() - - if not config.has_section(name): - raise StoragePoolException('Uknown storage pool ' + name) - elif not config.has_option(name, 'driver'): - raise StoragePoolException('No driver specified for pool ' + name) - - - driver = config.get(name, 'driver') - try: - return qubes.get_entry_point_one(STORAGE_ENTRY_POINT, driver) - except KeyError: - raise StoragePoolException('Driver %s for pool %s' % (driver, name)) class Pool(object): - def __init__(self, vm, dir_path): - assert vm is not None - assert dir_path is not None + ''' A Pool is used to manage different kind of volumes (File + based/LVM/Btrfs/...). - self.vm = vm - self.dir_path = dir_path + 3rd Parties providing own storage implementations will need to extend + this class. + ''' + private_img_size = qubes.config.defaults['private_img_size'] + root_img_size = qubes.config.defaults['root_img_size'] - self.create_dir_if_not_exists(self.dir_path) + def __init__(self, name, **kwargs): + super(Pool, self).__init__(**kwargs) + self.name = name + kwargs['name'] = self.name - self.vmdir = self.vmdir_path(vm, self.dir_path) + def __xml__(self): + return lxml.etree.Element('pool', **self.config) - appvms_path = os.path.join(self.dir_path, 'appvms') - self.create_dir_if_not_exists(appvms_path) + def create(self, volume, source_volume): + ''' Create the given volume on disk or copy from provided + `source_volume`. + ''' + raise NotImplementedError("Pool %s has create() not implemented" % + self.name) - servicevms_path = os.path.join(self.dir_path, 'servicevms') - self.create_dir_if_not_exists(servicevms_path) + def commit_template_changes(self, volume): + ''' Update origin device ''' + raise NotImplementedError( + "Pool %s has commit_template_changes() not implemented" % + self.name) - vm_templates_path = os.path.join(self.dir_path, 'vm-templates') - self.create_dir_if_not_exists(vm_templates_path) + @property + def config(self): + ''' Returns the pool config to be written to qubes.xml ''' + raise NotImplementedError("Pool %s has config() not implemented" % + self.name) - # XXX there is also a class attribute on the domain classes which does - # exactly that -- which one should prevail? - def vmdir_path(self, vm, pool_dir): - """ Returns the path to vmdir depending on the type of the VM. + def clone(self, source, target): + ''' Clone volume ''' + raise NotImplementedError("Pool %s has clone() not implemented" % + self.name) - The default QubesOS file storage saves the vm images in three - different directories depending on the ``QubesVM`` type: + def destroy(self): + raise NotImplementedError("Pool %s has destroy() not implemented" % + self.name) - * ``appvms`` for ``QubesAppVm`` or ``QubesHvm`` - * ``vm-templates`` for ``QubesTemplateVm`` or ``QubesTemplateHvm`` + def remove(self, volume): + ''' Remove volume''' + raise NotImplementedError("Pool %s has remove() not implemented" % + self.name) - Args: - vm: a QubesVM - pool_dir: the root directory of the pool + def rename(self, volume, old_name, new_name): + ''' Called when the domain changes its name ''' + raise NotImplementedError("Pool %s has rename() not implemented" % + self.name) - Returns: - string (str) absolute path to the directory where the vm files - are stored - """ - if vm.is_template(): - subdir = 'vm-templates' - elif vm.is_disposablevm(): - subdir = 'appvms' - return os.path.join(pool_dir, subdir, vm.template.name + '-dvm') - else: - subdir = 'appvms' + def start(self, volume): + ''' Do what ever is needed on start ''' + raise NotImplementedError("Pool %s has start() not implemented" % + self.name) - return os.path.join(pool_dir, subdir, vm.name) + def setup(self): + raise NotImplementedError("Pool %s has setup() not implemented" % + self.name) - def create_dir_if_not_exists(self, path): - """ Check if a directory exists in if not create it. + def stop(self, volume): + ''' Do what ever is needed on stop''' + raise NotImplementedError("Pool %s has stop() not implemented" % + self.name) - This method does not create any parent directories. - """ - if not os.path.exists(path): - os.mkdir(path) + def init_volume(self, volume_config): + ''' Initialize a :py:class:`qubes.storage.Volume` from `volume_config`. + ''' + raise NotImplementedError("Pool %s has init_volume() not implemented" % + self.name) def pool_drivers(): """ Return a list of EntryPoints names """ return [ep.name - for ep in pkg_resources.iter_entry_points(STORAGE_ENTRY_POINT)] + for ep in pkg_resources.iter_entry_points(STORAGE_ENTRY_POINT)] diff --git a/qubes/storage/kernels.py b/qubes/storage/kernels.py new file mode 100644 index 00000000..febbc13c --- /dev/null +++ b/qubes/storage/kernels.py @@ -0,0 +1,109 @@ +#!/usr/bin/python2 +# -*- encoding: utf8 -*- +# +# The Qubes OS Project, http://www.qubes-os.org +# +# Copyright (C) 2010-2015 Joanna Rutkowska +# Copyright (C) 2015 Wojtek Porczyk +# Copyright (C) 2016 Bahtiar `kalkin-` Gadimov +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +import os + +from qubes.storage import Pool, StoragePoolException, Volume + + +class LinuxModules(Volume): + rw = False + + def __init__(self, target_dir, kernel_version, **kwargs): + super(LinuxModules, self).__init__(**kwargs) + self.kernels_dir = os.path.join(target_dir, kernel_version) + self.path = os.path.join(self.kernels_dir, 'modules.img') + self.vid = self.path + self.vmlinuz = os.path.join(self.kernels_dir, 'vmlinuz') + self.initramfs = os.path.join(self.kernels_dir, 'initramfs') + + +class LinuxKernel(Pool): + driver = 'linux-kernel' + + def __init__(self, name=None, dir_path=None): + assert dir_path, 'Missing dir_path' + super(LinuxKernel, self).__init__(name=name) + self.dir_path = dir_path + + def init_volume(self, vm, volume_config): + assert 'volume_type' in volume_config, "Volume type missing " \ + + str(volume_config) + volume_type = volume_config['volume_type'] + if volume_type != 'read-only': + raise StoragePoolException("Unknown volume type " + volume_type) + + volume = LinuxModules(self.dir_path, vm.kernel, **volume_config) + + _check_path(volume.path) + _check_path(volume.vmlinuz) + _check_path(volume.initramfs) + + return volume + + def clone(self, source, target): + return target + + def create(self, volume, source_volume): + return volume + + def commit_template_changes(self, volume): + return volume + + @property + def config(self): + return { + 'name': self.name, + 'dir_path': self.dir_path, + 'driver': LinuxKernel.driver, + } + + def destroy(self): + pass + + def remove(self, volume): + pass + + def rename(self, volume, old_name, new_name): + return volume + + def setup(self): + pass + + def start(self, volume): + path = volume.path + if not os.path.exists(path): + raise StoragePoolException('Missing kernel modules: %s' % path) + + return volume + + def stop(self, volume): + pass + + +def _check_path(path): + ''' Raise an :py:class:`qubes.storage.StoragePoolException` if ``path`` does + not exist. + ''' + if not os.path.exists(path): + raise StoragePoolException('Missing file: %s' % path) diff --git a/qubes/storage/xen.py b/qubes/storage/xen.py index 57c0ff66..2071e45e 100644 --- a/qubes/storage/xen.py +++ b/qubes/storage/xen.py @@ -31,173 +31,110 @@ import os.path import re import subprocess -import lxml.etree +from qubes.storage import Pool, StoragePoolException, Volume -import qubes -import qubes.config -import qubes.storage -import qubes.vm.templatevm +BLKSIZE = 512 -class XenStorage(qubes.storage.Storage): - '''Class for VM storage of Xen VMs. - ''' +class XenPool(Pool): + ''' File based 'original' disk implementation ''' + driver = 'xen' - root_dev = 'xvda' - private_dev = 'xvdb' - volatile_dev = 'xvdc' - modules_dev = 'xvdd' + def __init__(self, name=None, dir_path=None): + super(XenPool, self).__init__(name=name) + assert dir_path, "No pool dir_path specified" + self.dir_path = os.path.normpath(dir_path) - def __init__(self, vm, vmdir, **kwargs): - """ Instantiate the storage. + def clone(self, source, target): + ''' Clones the volume if the `source.pool` if the source is a + :py:class:`XenVolume`. + ''' + if issubclass(XenVolume, source.__class__): + raise StoragePoolException('Volumes %s and %s use different pools' + % (source.__class__, target.__class__)) - Args: - vm: a QubesVM - vmdir: the root directory of the pool - """ - assert vm is not None - assert vmdir is not None + if source.volume_type not in ['origin', 'read-write']: + return target - super(XenStorage, self).__init__(vm, **kwargs) + copy_file(source.vid, target.vid) + return target - self.vmdir = vmdir + def create(self, volume, source_volume=None): + _type = volume.volume_type + size = volume.size + if _type == 'origin': + create_sparse_file(volume.path_origin, size) + create_sparse_file(volume.path_cow, size) + elif _type in ['read-write'] and source_volume: + copy_file(source_volume.path, volume.path) + elif _type in ['read-write', 'volatile']: + create_sparse_file(volume.path, size) + return volume @property - def private_img(self): - '''Path to the private image''' - return self.abspath(qubes.config.vm_files['private_img']) + def config(self): + return { + 'name': self.name, + 'dir_path': self.dir_path, + 'driver': XenPool.driver, + } + def resize(self, volume, size): + ''' Expands volume, throws + :py:class:`qubst.storage.StoragePoolException` if given size is + less than current_size + ''' + _type = volume.volume_type + if _type not in ['origin', 'read-write', 'volatile']: + raise StoragePoolException('Can not resize a %s volume %s' % + (_type, volume.vid)) - @property - def root_img(self): - '''Path to the root image''' - return self.vm.template.storage.root_img \ - if hasattr(self.vm, 'template') and self.vm.template \ - else self.abspath(qubes.config.vm_files['root_img']) + if size <= volume.size: + raise StoragePoolException( + 'For your own safety, shrinking of %s is' + ' disabled. If you really know what you' + ' are doing, use `truncate` on %s manually.' % + (volume.name, volume.vid)) + if _type == 'origin': + path = volume.path_origin + elif _type in ['read-write', 'volatile']: + path = volume.path - @property - def rootcow_img(self): - '''Path to the root COW image''' + with open(path, 'a+b') as fd: + fd.truncate(size) - if isinstance(self.vm, qubes.vm.templatevm.TemplateVM): - return self.abspath(qubes.config.vm_files['rootcow_img']) + self._resize_loop_device(path) - return None + def remove(self, volume): + if volume.volume_type in ['read-write', 'volatile']: + _remove_if_exists(volume.vid) + elif volume.volume_type == 'origin': + _remove_if_exists(volume.vid) + _remove_if_exists(volume.path_cow) + def rename(self, volume, old_name, new_name): + assert issubclass(volume.__class__, XenVolume) + old_dir = os.path.dirname(volume.path) + new_dir = os.path.join(os.path.dirname(old_dir), new_name) - @property - def volatile_img(self): - '''Path to the volatile image''' - return self.abspath(qubes.config.vm_files['volatile_img']) + if not os.path.exists(new_dir): + os.makedirs(new_dir) + if volume.volume_type == 'read-write': + volume.rename_target_dir(new_name, new_dir) + elif volume.volume_type == 'read-only': + volume.rename_target_dir(old_name, new_dir) + elif volume.volume_type in ['origin', 'volatile']: + volume.rename_target_dir(new_dir) - def format_disk_dev(self, path, vdev, script=None, rw=True, devtype='disk', - domain=None): - if path is None: - return '' - - element = lxml.etree.Element('disk') - element.set('type', 'block') - element.set('device', devtype) - - element.append(lxml.etree.Element('driver', name='phy')) - element.append(lxml.etree.Element('source', dev=path)) - element.append(lxml.etree.Element('target', dev=vdev)) - - if not rw: - element.append(lxml.etree.Element('readonly')) - if domain is not None: - # XXX vm.name? - element.append(lxml.etree.Element('domain', name=domain)) - if script: - element.append(lxml.etree.Element('script', path=script)) - - # TODO return element - return lxml.etree.tostring(element) - - - def root_dev_config(self): - if isinstance(self.vm, qubes.vm.templatevm.TemplateVM): - return self.format_disk_dev( - '{root}:{rootcow}'.format( - root=self.root_img, - rootcow=self.rootcow_img), - self.root_dev, - script='block-origin') - - elif self.vm.hvm and hasattr(self.vm, 'template'): - # HVM template-based VM - only one device-mapper layer, in dom0 - # (root+volatile) - # HVM detection based on 'kernel' property is massive hack, - # but taken from assumption that VM needs Qubes-specific kernel - # (actually initramfs) to assemble the second layer of device-mapper - return self.format_disk_dev( - '{root}:{volatile}'.format( - root=self.vm.template.storage.root_img, - volatile=self.volatile_img), - self.root_dev, - script='block-snapshot') - - elif hasattr(self.vm, 'template'): - # any other template-based VM - two device-mapper layers: one - # in dom0 (here) from root+root-cow, and another one from - # this+volatile.img - return self.format_disk_dev( - '{root}:{template_rootcow}'.format( - root=self.root_img, - template_rootcow=self.vm.template.storage.rootcow_img), - self.root_dev, - script='block-snapshot', - rw=False) - - else: - # standalone qube - return self.format_disk_dev(self.root_img, self.root_dev) - - - def private_dev_config(self): - return self.format_disk_dev(self.private_img, self.private_dev) - - def volatile_dev_config(self): - return self.format_disk_dev(self.volatile_img, self.volatile_dev) - - - def create_on_disk_private_img(self, source_template=None): - if source_template is None: - f_private = open(self.private_img, 'a+b') - f_private.truncate(self.private_img_size) - f_private.close() - - else: - self.vm.log.info("Copying the template's private image: {}".format( - source_template.storage.private_img)) - self._copy_file(source_template.storage.private_img, - self.private_img) - - - def create_on_disk_root_img(self, source_template=None): - if source_template is None: - fd = open(self.root_img, 'a+b') - fd.truncate(self.root_img_size) - fd.close() - - elif self.vm.updateable: - # if not updateable, just use template's disk - self.vm.log.info("--> Copying the template's root image: {}".format( - source_template.storage.root_img)) - self._copy_file(source_template.storage.root_img, self.root_img) - - - def resize_private_img(self, size): - fd = open(self.private_img, 'a+b') - fd.truncate(size) - fd.close() + return volume + def _resize_loop_device(self, path): # find loop device if any p = subprocess.Popen( - ['sudo', 'losetup', '--associated', self.private_img], + ['sudo', 'losetup', '--associated', path], stdout=subprocess.PIPE) result = p.communicate() @@ -206,89 +143,359 @@ class XenStorage(qubes.storage.Storage): loop_dev = m.group(1) # resize loop device - subprocess.check_call( - ['sudo', 'losetup', '--set-capacity', loop_dev]) + subprocess.check_call(['sudo', 'losetup', '--set-capacity', + loop_dev]) + def commit_template_changes(self, volume): + if volume.volume_type != 'origin': + return volume - def commit_template_changes(self): - assert isinstance(self.vm, qubes.vm.templatevm.TemplateVM) - - # TODO: move rootcow_img to this class; the same for vm.is_outdated() - if os.path.exists(self.vm.rootcow_img): - os.rename(self.vm.rootcow_img, self.vm.rootcow_img + '.old') + if os.path.exists(volume.path_cow): + os.rename(volume.path_cow, volume.path_cow + '.old') old_umask = os.umask(002) - f_cow = open(self.vm.rootcow_img, 'w') - f_root = open(self.root_img, 'r') - f_root.seek(0, os.SEEK_END) - # make empty sparse file of the same size as root.img - f_cow.truncate(f_root.tell()) - f_cow.close() - f_root.close() + with open(volume.path_cow, 'w') as f_cow: + f_cow.truncate(volume.size) os.umask(old_umask) + return volume + + def destroy(self): + pass + + def setup(self): + create_dir_if_not_exists(self.dir_path) + appvms_path = os.path.join(self.dir_path, 'appvms') + create_dir_if_not_exists(appvms_path) + vm_templates_path = os.path.join(self.dir_path, 'vm-templates') + create_dir_if_not_exists(vm_templates_path) + + def start(self, volume): + if volume.volume_type == 'volatile': + self._reset_volume(volume) + if volume.volume_type in ['origin', 'snapshot']: + _check_path(volume.path_origin) + _check_path(volume.path_cow) + else: + _check_path(volume.path) + + return volume + + def stop(self, volume): + pass + + def _reset_volume(self, volume): + ''' Remove and recreate a volatile volume ''' + assert volume.volume_type == 'volatile', "Not a volatile volume" + + assert volume.size + + _remove_if_exists(volume) + + with open(volume.path, "w") as f_volatile: + f_volatile.truncate(volume.size) + return volume + + def target_dir(self, vm): + """ Returns the path to vmdir depending on the type of the VM. + + The default QubesOS file storage saves the vm images in three + different directories depending on the ``QubesVM`` type: + + * ``appvms`` for ``QubesAppVm`` or ``QubesHvm`` + * ``vm-templates`` for ``QubesTemplateVm`` or ``QubesTemplateHvm`` + + Args: + vm: a QubesVM + pool_dir: the root directory of the pool + + Returns: + string (str) absolute path to the directory where the vm files + are stored + """ + if vm.is_template(): + subdir = 'vm-templates' + elif vm.is_disposablevm(): + subdir = 'appvms' + return os.path.join(self.dir_path, subdir, + vm.template.name + '-dvm') + else: + subdir = 'appvms' + + return os.path.join(self.dir_path, subdir, vm.name) + + def init_volume(self, vm, volume_config): + assert 'volume_type' in volume_config, "Volume type missing " \ + + str(volume_config) + volume_type = volume_config['volume_type'] + known_types = { + 'read-write': ReadWriteFile, + 'read-only': ReadOnlyFile, + 'origin': OriginFile, + 'snapshot': SnapshotFile, + 'volatile': VolatileFile, + } + if volume_type not in known_types: + raise StoragePoolException("Unknown volume type " + volume_type) + + if volume_type in ['snapshot', 'read-only']: + origin_pool = vm.app.get_pool(volume_config['pool']) + assert isinstance(origin_pool, + XenPool), 'Origin volume not a xen volume' + volume_config['target_dir'] = origin_pool.target_dir(vm.template) + name = volume_config['name'] + volume_config['size'] = vm.template.volume_config[name]['size'] + else: + volume_config['target_dir'] = self.target_dir(vm) + + return known_types[volume_type](**volume_config) - def reset_volatile_storage(self): - try: - # no template set, in any way (Standalone VM, Template VM) - if self.vm.template is None: - raise AttributeError +class XenVolume(Volume): + ''' Parent class for the xen volumes implementation which expects a + `target_dir` param on initialization. + ''' - # template-based HVM with only one device-mapper layer - - # volatile.img used as upper layer on root.img, no root-cow.img - # intermediate layer - if self.vm.hvm: - if os.path.exists(self.volatile_img): - if self.vm.debug: - if os.path.getmtime(self.vm.template.storage.root_img) \ - > os.path.getmtime(self.volatile_img): - self.vm.log.warning( - 'Template have changed, resetting root.img') - else: - self.vm.log.warning( - 'Debug mode: not resetting root.img; if you' - ' want to force root.img reset, either' - ' update template VM, or remove volatile.img' - ' file.') - return - - os.remove(self.volatile_img) - - # FIXME stat on f_root; with open() ... - f_volatile = open(self.volatile_img, "w") - f_root = open(self.vm.template.storage.root_img, "r") - # make empty sparse file of the same size as root.img - f_root.seek(0, os.SEEK_END) - f_volatile.truncate(f_root.tell()) - f_volatile.close() - f_root.close() - return # XXX why is that? super() does not run - except AttributeError: # self.vm.template - pass - - super(XenStorage, self).reset_volatile_storage() + def __init__(self, target_dir, **kwargs): + self.target_dir = target_dir + assert self.target_dir, "target_dir not specified" + super(XenVolume, self).__init__(**kwargs) - def prepare_for_vm_startup(self): - super(XenStorage, self).prepare_for_vm_startup() +class SizeMixIn(XenVolume): + ''' A mix in which expects a `size` param to be > 0 on initialization and + provides a usage property wrapper. + ''' - if self.drive is not None: - # pylint: disable=unused-variable - (drive_type, drive_domain, drive_path) = self.drive.split(":") + def __init__(self, size=0, **kwargs): + assert size, 'Empty size provided' + assert size > 0, 'Size for volume ' + kwargs['name'] + ' is <=0' + super(SizeMixIn, self).__init__(size=int(size), **kwargs) - if drive_domain.lower() != "dom0": - # XXX "VM '{}' holding '{}' does not exists".format( - drive_vm = self.vm.app.domains[drive_domain] + @property + def usage(self): + ''' Returns the actualy used space ''' + return get_disk_usage(self.vid) - if not drive_vm.is_running(): - raise qubes.exc.QubesVMNotRunningError(drive_vm, - 'VM {!r} holding {!r} isn\'t running'.format( - drive_domain, drive_path)) + @property + def config(self): + ''' return config data for serialization to qubes.xml ''' + return {'name': self.name, + 'pool': self.pool, + 'size': str(self.size), + 'volume_type': self.volume_type} - if self.rootcow_img and not os.path.exists(self.rootcow_img): - self.commit_template_changes() -class XenPool(qubes.storage.Pool): - def get_storage(self): - """ Returns an instantiated ``XenStorage``. """ - return XenStorage(self.vm, vmdir=self.vmdir) +class ReadWriteFile(SizeMixIn): + ''' Represents a readable & writable file image based volume ''' + def __init__(self, **kwargs): + super(ReadWriteFile, self).__init__(**kwargs) + self.path = os.path.join(self.target_dir, self.name + '.img') + self.vid = self.path + + def rename_target_dir(self, new_name, new_dir): + ''' Called by :py:class:`XenPool` when a domain changes it's name ''' + old_path = self.path + file_name = os.path.basename(self.path) + new_path = os.path.join(new_dir, file_name) + + os.rename(old_path, new_path) + self.target_dir = new_dir + self.path = new_path + self.vid = self.path + + +class ReadOnlyFile(XenVolume): + ''' Represents a readonly file image based volume ''' + usage = 0 + + def __init__(self, size=0, **kwargs): + super(ReadOnlyFile, self).__init__(size=int(size), **kwargs) + self.path = self.vid + + def rename_target_dir(self, old_name, new_dir): + """ Called by :py:class:`XenPool` when a domain changes it's name. + + Only copies the volume if it belongs to the domain being renamed. + Currently if a volume is in a directory named the same as the domain, + it's ”owned” by the domain. + """ + if os.path.basename(self.target_dir) == old_name: + file_name = os.path.basename(self.path) + new_path = os.path.join(new_dir, file_name) + old_path = self.path + + os.rename(old_path, new_path) + + self.target_dir = new_dir + self.path = new_path + self.vid = self.path + + +class OriginFile(SizeMixIn): + ''' Represents a readable, writeable & snapshotable file image based volume. + + This is used for TemplateVM's + ''' + + script = 'block-origin' + + def __init__(self, **kwargs): + super(OriginFile, self).__init__(**kwargs) + self.path_origin = os.path.join(self.target_dir, self.name + '.img') + self.path_cow = os.path.join(self.target_dir, self.name + '-cow.img') + self.path = '%s:%s' % (self.path_origin, self.path_cow) + self.vid = self.path_origin + + def commit(self): + ''' Commit Template changes ''' + raise NotImplementedError + + def rename_target_dir(self, new_dir): + ''' Called by :py:class:`XenPool` when a domain changes it's name ''' + old_path_origin = self.path_origin + old_path_cow = self.path_cow + new_path_origin = os.path.join(new_dir, self.name + '.img') + new_path_cow = os.path.join(new_dir, self.name + '-cow.img') + os.rename(old_path_origin, new_path_origin) + os.rename(old_path_cow, new_path_cow) + self.target_dir = new_dir + self.path_origin = new_path_origin + self.path_cow = new_path_cow + self.path = '%s:%s' % (self.path_origin, self.path_cow) + self.vid = self.path_origin + + @property + def usage(self): + result = 0 + if os.path.exists(self.path_origin): + result += get_disk_usage(self.path_origin) + if os.path.exists(self.path_cow): + result += get_disk_usage(self.path_cow) + return result + + +class SnapshotFile(XenVolume): + ''' Represents a readonly snapshot of an :py:class:`OriginFile` volume ''' + script = 'block-snapshot' + rw = False + usage = 0 + + def __init__(self, name=None, size=None, **kwargs): + assert size + super(SnapshotFile, self).__init__(name=name, size=int(size), **kwargs) + self.path_origin = os.path.join(self.target_dir, name + '.img') + self.path_cow = os.path.join(self.target_dir, name + '-cow.img') + self.path = '%s:%s' % (self.path_origin, self.path_cow) + self.vid = self.path_origin + + +class VolatileFile(SizeMixIn): + ''' Represents a readable & writeable file based volume, which will be + discarded and recreated at each startup. + ''' + def __init__(self, **kwargs): + super(VolatileFile, self).__init__(**kwargs) + self.path = os.path.join(self.target_dir, self.name + '.img') + self.vid = self.path + + def rename_target_dir(self, new_dir): + ''' Called by :py:class:`XenPool` when a domain changes it's name ''' + _remove_if_exists(self) + file_name = os.path.basename(self.path) + self.target_dir = new_dir + new_path = os.path.join(new_dir, file_name) + self.path = new_path + self.vid = self.path + + +def create_sparse_file(path, size): + ''' Create an empty sparse file ''' + if os.path.exists(path): + raise IOError("Volume %s already exists", path) + parent_dir = os.path.dirname(path) + if not os.path.exists(parent_dir): + os.makedirs(parent_dir) + with open(path, 'a+b') as fh: + fh.truncate(size) + + +def get_disk_usage_one(st): + '''Extract disk usage of one inode from its stat_result struct. + + If known, get real disk usage, as written to device by filesystem, not + logical file size. Those values may be different for sparse files. + + :param os.stat_result st: stat result + :returns: disk usage + ''' + try: + return st.st_blocks * BLKSIZE + except AttributeError: + return st.st_size + + +def get_disk_usage(path): + '''Get real disk usage of given path (file or directory). + + When *path* points to directory, then it is evaluated recursively. + + This function tries estiate real disk usage. See documentation of + :py:func:`get_disk_usage_one`. + + :param str path: path to evaluate + :returns: disk usage + ''' + try: + st = os.lstat(path) + except OSError: + return 0 + + ret = get_disk_usage_one(st) + + # if path is not a directory, this is skipped + for dirpath, dirnames, filenames in os.walk(path): + for name in dirnames + filenames: + ret += get_disk_usage_one(os.lstat(os.path.join(dirpath, name))) + + return ret + + +def create_dir_if_not_exists(path): + """ Check if a directory exists in if not create it. + + This method does not create any parent directories. + """ + if not os.path.exists(path): + os.mkdir(path) + + +def copy_file(source, destination): + '''Effective file copy, preserving sparse files etc. + ''' + # TODO: Windows support + # We prefer to use Linux's cp, because it nicely handles sparse files + assert os.path.exists(source), \ + "Missing the source %s to copy from" % source + assert not os.path.exists(destination), \ + "Destination %s already exists" % destination + + parent_dir = os.path.dirname(destination) + if not os.path.exists(parent_dir): + os.makedirs(parent_dir) + + try: + subprocess.check_call(['cp', '--reflink=auto', source, destination]) + except subprocess.CalledProcessError: + raise IOError('Error while copying {!r} to {!r}'.format(source, + destination)) + + +def _remove_if_exists(volume): + if os.path.exists(volume.path): + os.remove(volume.path) + + +def _check_path(path): + ''' Raise an StoragePoolException if ``path`` does not exist''' + if not os.path.exists(path): + raise StoragePoolException('Missing image file: %s' % path) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index 1505a07d..538a02bb 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -820,7 +820,7 @@ class BackupTestsMixin(SystemTestsMixin): name=vmname, template=template, provides_network=True, label='red') testnet.create_on_disk() vms.append(testnet) - self.fill_image(testnet.private_img, 20*1024*1024) + self.fill_image(testnet.volumes['private'].vid, 20*1024*1024) vmname = self.make_vm_name('test1') if self.verbose: @@ -831,16 +831,17 @@ class BackupTestsMixin(SystemTestsMixin): testvm1.netvm = testnet testvm1.create_on_disk() vms.append(testvm1) - self.fill_image(testvm1.private_img, 100*1024*1024) + self.fill_image(testvm1.volumes['private'].vid, 100*1024*1024) vmname = self.make_vm_name('testhvm1') if self.verbose: print >>sys.stderr, "-> Creating %s" % vmname testvm2 = self.app.add_new_vm(qubes.vm.standalonevm.StandaloneVM, - name=vmname, - hvm=True, label='red') - testvm2.create_on_disk() - self.fill_image(testvm2.root_img, 1024*1024*1024, True) + name=vmname, + hvm=True, + label='red') + testvm2.create_on_disk(verbose=self.verbose) + self.fill_image(testvm2.volumes['root'].vid, 1024 * 1024 * 1024, True) vms.append(testvm2) vmname = self.make_vm_name('template') diff --git a/qubes/tests/int/basic.py b/qubes/tests/int/basic.py index 5a107fdc..e740eeb2 100644 --- a/qubes/tests/int/basic.py +++ b/qubes/tests/int/basic.py @@ -392,8 +392,9 @@ class TC_03_QvmRevertTemplateChanges(qubes.tests.SystemTestsMixin, self.save_and_reload_db() def get_rootimg_checksum(self): - p = subprocess.Popen(['sha1sum', self.test_template.root_img], - stdout=subprocess.PIPE) + p = subprocess.Popen( + ['sha1sum', self.test_template.volumes['root'].vid], + stdout=subprocess.PIPE) return p.communicate()[0] def _do_test(self): diff --git a/qubes/tests/storage.py b/qubes/tests/storage.py index 8cb9f9fc..5fa82222 100644 --- a/qubes/tests/storage.py +++ b/qubes/tests/storage.py @@ -17,9 +17,12 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import qubes.log -from qubes.storage import StoragePoolException, pool_drivers +from qubes.exc import QubesException +from qubes.storage import pool_drivers from qubes.storage.xen import XenPool -from qubes.tests import QubesTestCase +from qubes.tests import QubesTestCase, SystemTestsMixin + +# :pylint: disable=invalid-name class TestApp(qubes.tests.TestEmitter): @@ -27,30 +30,26 @@ class TestApp(qubes.tests.TestEmitter): class TestVM(object): - def __init__(self, app, qid, name, pool_name, template=None): - super(TestVM, self).__init__() - self.app = app - self.qid = qid - self.name = name - self.pool_name = pool_name - self.template = template - self.hvm = False - self.storage = qubes.storage.get_pool(self.pool_name, - self).get_storage() + def __init__(self, test, template=None): + self.app = test.app + self.name = test.make_vm_name('appvm') self.log = qubes.log.get_vm_logger(self.name) + if template: + self.template = template + def is_template(self): + # :pylint: disable=no-self-use return False def is_disposablevm(self): + # :pylint: disable=no-self-use return False - @property - def dir_path(self): - return self.storage.vmdir - class TestTemplateVM(TestVM): + dir_path_prefix = qubes.config.system_path['qubes_templates_dir'] + def is_template(self): return True @@ -60,46 +59,47 @@ class TestDisposableVM(TestVM): return True -class TC_00_Pool(QubesTestCase): +class TC_00_Pool(SystemTestsMixin, QubesTestCase): """ This class tests the utility methods from :mod:``qubes.storage`` """ def setUp(self): super(TC_00_Pool, self).setUp() + self.init_default_template() def test_000_unknown_pool_driver(self): # :pylint: disable=protected-access """ Expect an exception when unknown pool is requested""" - with self.assertRaises(StoragePoolException): - qubes.storage._get_pool_klass('foo-bar') + with self.assertRaises(QubesException): + self.app.get_pool('foo-bar') def test_001_all_pool_drivers(self): - """ The only predefined pool driver is file """ - self.assertEquals(["xen"], pool_drivers()) + """ The only predefined pool driver is xen """ + self.assertEquals(['linux-kernel', 'xen'], pool_drivers()) def test_002_get_pool_klass(self): """ Expect the default pool to be `XenPool` """ # :pylint: disable=protected-access - result = qubes.storage._get_pool_klass('default') - self.assertTrue(result is XenPool) + result = self.app.get_pool('default') + self.assertIsInstance(result, XenPool) def test_003_pool_exists_default(self): """ Expect the default pool to exists """ - self.assertTrue(qubes.storage.pool_exists('default')) + self.assertPoolExists('default') - def test_004_pool_exists_random(self): - """ Expect this pool to not a exist """ - self.assertFalse(qubes.storage.pool_exists( - 'asdh312096r832598213iudhas')) - - def test_005_add_remove_pool(self): + def test_004_add_remove_pool(self): """ Tries to adding and removing a pool. """ pool_name = 'asdjhrp89132' # make sure it's really does not exist - qubes.storage.remove_pool(pool_name) + self.app.remove_pool(pool_name) + self.assertFalse(self.assertPoolExists(pool_name)) - qubes.storage.add_pool(pool_name, driver='xen') - self.assertTrue(qubes.storage.pool_exists(pool_name)) + self.app.add_pool(name=pool_name, driver='xen', dir_path='/tmp/asdjhrp89132') + self.assertTrue(self.assertPoolExists(pool_name)) - qubes.storage.remove_pool(pool_name) - self.assertFalse(qubes.storage.pool_exists(pool_name)) + self.app.remove_pool(pool_name) + self.assertFalse(self.assertPoolExists(pool_name)) + + def assertPoolExists(self, pool): + """ Check if specified pool exists """ + return pool in self.app.pools.keys() diff --git a/qubes/tests/storage_xen.py b/qubes/tests/storage_xen.py index 5954fba6..0dd9557c 100644 --- a/qubes/tests/storage_xen.py +++ b/qubes/tests/storage_xen.py @@ -18,13 +18,21 @@ import os import shutil -import unittest + import qubes.storage import qubes.tests.storage +from qubes.config import defaults +from qubes.storage import Storage +from qubes.storage.xen import (OriginFile, ReadOnlyFile, ReadWriteFile, + SnapshotFile, VolatileFile) from qubes.tests import QubesTestCase, SystemTestsMixin -from qubes.storage.xen import XenStorage +from qubes.tests.storage import TestVM -class TC_00_XenPool(QubesTestCase): + +# :pylint: disable=invalid-name + + +class TC_00_XenPool(SystemTestsMixin, QubesTestCase): """ This class tests some properties of the 'default' pool. """ @@ -34,32 +42,166 @@ class TC_00_XenPool(QubesTestCase): .. sealso:: Data :data:``qubes.qubes.defaults['pool_config']``. """ - vm = self._init_app_vm() - result = qubes.storage.get_pool("default", vm).dir_path + result = self.app.get_pool("default").dir_path expected = '/var/lib/qubes' self.assertEquals(result, expected) def test001_default_storage_class(self): - """ Check when using default pool the Storage is ``XenStorage``. """ + """ Check when using default pool the Storage is ``Storage``. """ result = self._init_app_vm().storage - self.assertIsInstance(result, XenStorage) - - def test_002_default_pool_name(self): - """ Default pool_name is 'default'. """ - vm = self._init_app_vm() - self.assertEquals(vm.pool_name, "default") + self.assertIsInstance(result, Storage) def _init_app_vm(self): """ Return initalised, but not created, AppVm. """ - app = qubes.tests.storage.TestApp() vmname = self.make_vm_name('appvm') - template = qubes.tests.storage.TestTemplateVM(app, 1, - self.make_vm_name('template'), 'default') - return qubes.tests.storage.TestVM(app, qid=2, name=vmname, - template=template, pool_name='default') + self.init_default_template() + return self.app.add_new_vm(qubes.vm.appvm.AppVM, + name=vmname, + template=self.app.default_template, + label='red') + + +class TC_01_XenVolumes(SystemTestsMixin, QubesTestCase): + POOL_DIR = '/var/lib/qubes/test-pool' + POOL_NAME = 'test-pool' + POOL_CONF = {'driver': 'xen', 'dir_path': POOL_DIR, 'name': POOL_NAME} + + def setUp(self): + """ Add a test file based storage pool """ + super(TC_01_XenVolumes, self).setUp() + self.init_default_template() + self.app.add_pool(**self.POOL_CONF) + + def tearDown(self): + """ Remove the file based storage pool after testing """ + self.app.remove_pool("test-pool") + super(TC_01_XenVolumes, self).tearDown() + shutil.rmtree(self.POOL_DIR, ignore_errors=True) + + def test_000_origin_volume(self): + config = { + 'name': 'root', + 'pool': self.POOL_NAME, + 'volume_type': 'origin', + 'size': defaults['root_img_size'], + } + vm = TestVM(self) + result = self.app.get_pool(self.POOL_NAME).init_volume(vm, config) + self.assertIsInstance(result, OriginFile) + self.assertEqual(result.name, 'root') + self.assertEqual(result.pool, self.POOL_NAME) + self.assertEqual(result.size, defaults['root_img_size']) + + def test_001_snapshot_volume(self): + original_path = '/var/lib/qubes/vm-templates/fedora-23/root.img' + original_size = qubes.config.defaults['root_img_size'] + config = { + 'name': 'root', + 'pool': 'default', + 'volume_type': 'snapshot', + 'vid': original_path, + } + vm = TestVM(self, template=self.app.default_template) + result = self.app.get_pool(self.POOL_NAME).init_volume(vm, config) + self.assertIsInstance(result, SnapshotFile) + self.assertEqual(result.name, 'root') + self.assertEqual(result.pool, 'default') + self.assertEqual(result.size, original_size) + + def test_002_read_write_volume(self): + config = { + 'name': 'root', + 'pool': self.POOL_NAME, + 'volume_type': 'read-write', + 'size': defaults['root_img_size'], + } + vm = TestVM(self) + result = self.app.get_pool(self.POOL_NAME).init_volume(vm, config) + self.assertIsInstance(result, ReadWriteFile) + self.assertEqual(result.name, 'root') + self.assertEqual(result.pool, self.POOL_NAME) + self.assertEqual(result.size, defaults['root_img_size']) + + def test_003_read_volume(self): + template = self.app.default_template + original_path = template.volumes['root'].vid + original_size = qubes.config.defaults['root_img_size'] + config = { + 'name': 'root', + 'pool': 'default', + 'volume_type': 'read-only', + 'vid': original_path + } + vm = TestVM(self, template=template) + + result = self.app.get_pool(self.POOL_NAME).init_volume(vm, config) + self.assertIsInstance(result, ReadOnlyFile) + self.assertEqual(result.name, 'root') + self.assertEqual(result.pool, 'default') + self.assertEqual(result.size, original_size) + + def test_004_volatile_volume(self): + config = { + 'name': 'root', + 'pool': self.POOL_NAME, + 'volume_type': 'volatile', + 'size': defaults['root_img_size'], + } + vm = TestVM(self) + result = self.app.get_pool(self.POOL_NAME).init_volume(vm, config) + self.assertIsInstance(result, VolatileFile) + self.assertEqual(result.name, 'root') + self.assertEqual(result.pool, self.POOL_NAME) + self.assertEqual(result.size, defaults['root_img_size']) + + def test_005_appvm_volumes(self): + ''' Check if AppVM volumes are propertly initialized ''' + vmname = self.make_vm_name('appvm') + vm = self.app.add_new_vm(qubes.vm.appvm.AppVM, + name=vmname, + template=self.app.default_template, + label='red') + + volumes = vm.volumes + self.assertIsInstance(volumes['root'], SnapshotFile) + self.assertIsInstance(volumes['private'], ReadWriteFile) + self.assertIsInstance(volumes['volatile'], VolatileFile) + expected = vm.template.dir_path + '/root.img:' + vm.template.dir_path \ + + '/root-cow.img' + self.assertVolumePath(vm, 'root', expected, rw=False) + expected = vm.dir_path + '/private.img' + self.assertVolumePath(vm, 'private', expected, rw=True) + expected = vm.dir_path + '/volatile.img' + self.assertVolumePath(vm, 'volatile', expected, rw=True) + + def test_006_template_volumes(self): + ''' Check if TemplateVM volumes are propertly initialized ''' + vmname = self.make_vm_name('appvm') + vm = self.app.add_new_vm(qubes.vm.templatevm.TemplateVM, + name=vmname, + label='red') + + volumes = vm.volumes + self.assertIsInstance(volumes['root'], OriginFile) + self.assertIsInstance(volumes['private'], ReadWriteFile) + self.assertIsInstance(volumes['volatile'], VolatileFile) + expected = vm.dir_path + '/root.img:' + vm.dir_path + '/root-cow.img' + self.assertVolumePath(vm, 'root', expected, rw=True) + expected = vm.dir_path + '/private.img' + self.assertVolumePath(vm, 'private', expected, rw=True) + expected = vm.dir_path + '/volatile.img' + self.assertVolumePath(vm, 'volatile', expected, rw=True) + + def assertVolumePath(self, vm, dev_name, expected, rw=True): + # :pylint: disable=invalid-name + volumes = vm.volumes + b_dev = volumes[dev_name].block_device() + self.assertEqual(b_dev.rw, rw) + self.assertEquals(b_dev.path, expected) + @qubes.tests.skipUnlessDom0 -class TC_01_XenPool(QubesTestCase): +class TC_03_XenPool(SystemTestsMixin, QubesTestCase): """ Test the paths for the default Xen file based storage (``XenStorage``). """ @@ -68,172 +210,124 @@ class TC_01_XenPool(QubesTestCase): APPVMS_DIR = '/var/lib/qubes/test-pool/appvms' TEMPLATES_DIR = '/var/lib/qubes/test-pool/vm-templates' SERVICE_DIR = '/var/lib/qubes/test-pool/servicevms' + POOL_NAME = 'test-pool' + POOL_CONFIG = {'driver': 'xen', 'dir_path': POOL_DIR, 'name': POOL_NAME} def setUp(self): """ Add a test file based storage pool """ - super(TC_01_XenPool, self).setUp() - qubes.storage.add_pool('test-pool', driver='xen', - dir_path=self.POOL_DIR) - self.app = qubes.tests.storage.TestApp() - self.template = qubes.tests.storage.TestTemplateVM(self.app, 1, - self.make_vm_name('template'), 'default') - + super(TC_03_XenPool, self).setUp() + self.init_default_template() + self.app.add_pool(**self.POOL_CONFIG) def tearDown(self): """ Remove the file based storage pool after testing """ - super(TC_01_XenPool, self).tearDown() - qubes.storage.remove_pool("test-pool") + self.app.remove_pool("test-pool") + super(TC_03_XenPool, self).tearDown() shutil.rmtree(self.POOL_DIR, ignore_errors=True) def test_001_pool_exists(self): """ Check if the storage pool was added to the storage pool config """ - self.assertTrue(qubes.storage.pool_exists('test-pool')) + self.assertIn('test-pool', self.app.pools.keys()) def test_002_pool_dir_create(self): """ Check if the storage pool dir and subdirs were created """ - # The dir should not exists before - self.assertFalse(os.path.exists(self.POOL_DIR)) + pool_name = 'foo' + pool_dir = '/tmp/foo' + appvms_dir = '/tmp/foo/appvms' + templates_dir = '/tmp/foo/vm-templates' - vmname = self.make_vm_name('appvm') - qubes.tests.storage.TestVM(self.app, qid=2, name=vmname, - template=self.template, pool_name='test-pool') + self.assertFalse(os.path.exists(pool_dir)) - self.assertTrue(os.path.exists(self.POOL_DIR)) - self.assertTrue(os.path.exists(self.APPVMS_DIR)) - self.assertTrue(os.path.exists(self.SERVICE_DIR)) - self.assertTrue(os.path.exists(self.TEMPLATES_DIR)) + self.app.add_pool(name=pool_name, dir_path=pool_dir, driver='xen') - def test_003_pool_dir(self): - """ Check if the vm storage pool_dir is the same as specified """ - vmname = self.make_vm_name('appvm') - vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname, - template=self.template, pool_name='test-pool') - result = qubes.storage.get_pool('test-pool', vm).dir_path - self.assertEquals(self.POOL_DIR, result) + self.assertTrue(os.path.exists(pool_dir)) + self.assertTrue(os.path.exists(appvms_dir)) + self.assertTrue(os.path.exists(templates_dir)) - def test_004_app_vmdir(self): - """ Check the vm storage dir for an AppVm""" - vmname = self.make_vm_name('appvm') - vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname, - template=self.template, pool_name='test-pool') - - expected = os.path.join(self.APPVMS_DIR, vm.name) - result = vm.storage.vmdir - self.assertEquals(expected, result) - - def test_005_hvm_vmdir(self): - """ Check the vm storage dir for a HVM""" - vmname = self.make_vm_name('hvm') - vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname, - template=self.template, pool_name='test-pool') - vm.hvm = True - - expected = os.path.join(self.APPVMS_DIR, vm.name) - result = vm.storage.vmdir - self.assertEquals(expected, result) - - @unittest.skip('TODO - servicevms dir?') - def test_006_net_vmdir(self): - """ Check the vm storage dir for a Netvm""" - vmname = self.make_vm_name('hvm') - vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname, - template=self.template, pool_name='test-pool') - - expected = os.path.join(self.SERVICE_DIR, vm.name) - result = vm.storage.vmdir - self.assertEquals(expected, result) - - @unittest.skip('TODO - servicevms dir?') - def test_007_proxy_vmdir(self): - """ Check the vm storage dir for a ProxyVm""" - vmname = self.make_vm_name('proxyvm') - vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname, - template=self.template, pool_name='test-pool') - - expected = os.path.join(self.SERVICE_DIR, vm.name) - result = vm.storage.vmdir - self.assertEquals(expected, result) - - def test_008_admin_vmdir(self): - """ Check the vm storage dir for a AdminVm""" - # TODO How to test AdminVm? - pass - - def test_009_template_vmdir(self): - """ Check the vm storage dir for a TemplateVm""" - vmname = self.make_vm_name('templatevm') - vm = qubes.tests.storage.TestTemplateVM(self.app, qid=2, name=vmname, - pool_name='test-pool') - - expected = os.path.join(self.TEMPLATES_DIR, vm.name) - result = vm.storage.vmdir - self.assertEquals(expected, result) - - def test_010_template_hvm_vmdir(self): - """ Check the vm storage dir for a TemplateHVm""" - vmname = self.make_vm_name('templatehvm') - vm = qubes.tests.storage.TestTemplateVM(self.app, qid=2, name=vmname, - pool_name='test-pool') - - expected = os.path.join(self.TEMPLATES_DIR, vm.name) - result = vm.storage.vmdir - self.assertEquals(expected, result) + shutil.rmtree(pool_dir, ignore_errors=True) def test_011_appvm_file_images(self): """ Check if all the needed image files are created for an AppVm""" vmname = self.make_vm_name('appvm') - vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname, - pool_name='test-pool') + vm = self.app.add_new_vm(qubes.vm.appvm.AppVM, + name=vmname, + template=self.app.default_template, + volume_config={ + 'private': { + 'pool': 'test-pool' + }, + 'volatile': { + 'pool': 'test-pool' + } + }, + label='red') vm.storage.create_on_disk() expected_vmdir = os.path.join(self.APPVMS_DIR, vm.name) - self.assertEqualsAndExists(vm.storage.vmdir, expected_vmdir) expected_private_path = os.path.join(expected_vmdir, 'private.img') - self.assertEqualsAndExists(vm.storage.private_img, + self.assertEqualsAndExists(vm.volumes['private'].path, expected_private_path) expected_volatile_path = os.path.join(expected_vmdir, 'volatile.img') - self.assertEqualsAndExists(vm.storage.volatile_img, + self.assertEqualsAndExists(vm.volumes['volatile'].path, expected_volatile_path) - def test_012_hvm_file_images(self): - """ Check if all the needed image files are created for a HVm""" + def test_013_template_file_images(self): + """ Check if root.img, private.img, volatile.img and root-cow.img are + created propertly by the storage system + """ + vmname = self.make_vm_name('tmvm') + vm = self.app.add_new_vm(qubes.vm.templatevm.TemplateVM, + name=vmname, + volume_config={ + 'root': { + 'pool': 'test-pool' + }, + 'private': { + 'pool': 'test-pool' + }, + 'volatile': { + 'pool': 'test-pool' + } + }, + label='red') + vm.create_on_disk() - vmname = self.make_vm_name('hvm') - vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname, - pool_name='test-pool') - vm.hvm = True - vm.storage.create_on_disk() + expected_vmdir = os.path.join(self.TEMPLATES_DIR, vm.name) - expected_vmdir = os.path.join(self.APPVMS_DIR, vm.name) - self.assertEqualsAndExists(vm.storage.vmdir, expected_vmdir) + expected_root_origin_path = os.path.join(expected_vmdir, 'root.img') + expected_root_cow_path = os.path.join(expected_vmdir, 'root-cow.img') + expected_root_path = '%s:%s' % (expected_root_origin_path, + expected_root_cow_path) + self.assertEquals(vm.volumes['root'].path, expected_root_path) + self.assertExist(vm.volumes['root'].path_origin) expected_private_path = os.path.join(expected_vmdir, 'private.img') - self.assertEqualsAndExists(vm.storage.private_img, + self.assertEqualsAndExists(vm.volumes['private'].path, expected_private_path) - expected_root_path = os.path.join(expected_vmdir, 'root.img') - self.assertEqualsAndExists(vm.storage.root_img, expected_root_path) - expected_volatile_path = os.path.join(expected_vmdir, 'volatile.img') - self.assertEqualsAndExists(vm.storage.volatile_img, + self.assertEqualsAndExists(vm.volumes['volatile'].path, expected_volatile_path) - @unittest.skip('test not implemented') # TODO - def test_013_template_based_file_images(self): - pass + vm.storage.commit_template_changes() + expected_rootcow_path = os.path.join(expected_vmdir, 'root-cow.img') + self.assertEqualsAndExists(vm.volumes['root'].path_cow, + expected_rootcow_path) def assertEqualsAndExists(self, result_path, expected_path): """ Check if the ``result_path``, matches ``expected_path`` and exists. See also: :meth:``assertExist`` """ + # :pylint: disable=invalid-name self.assertEquals(result_path, expected_path) self.assertExist(result_path) def assertExist(self, path): """ Assert that the given path exists. """ - self.assertTrue(os.path.exists(path)) + # :pylint: disable=invalid-name + self.assertTrue(os.path.exists(path), "Path %s does not exist" % path) diff --git a/qubes/tools/qvm_create.py b/qubes/tools/qvm_create.py index 3ee077d6..83f1fb35 100644 --- a/qubes/tools/qvm_create.py +++ b/qubes/tools/qvm_create.py @@ -46,9 +46,10 @@ parser.add_argument('--property', '--prop', '-p', action=qubes.tools.PropertyAction, help='set domain\'s property, like "internal", "memory" or "vcpus"') -parser.add_argument('--pool-name', '--pool', '-P', - action=qubes.tools.SinglePropertyAction, - help='specify the storage pool to use') +parser.add_argument('--pool', '-P', + action='append', + metavar='POOL_NAME:VOLUME_NAME', + help='specify the pool to use for a volume') parser.add_argument('--template', '-t', action=qubes.tools.SinglePropertyAction, @@ -79,6 +80,17 @@ parser.add_argument('name', metavar='VMNAME', def main(args=None): args = parser.parse_args(args) + if args.pool: + args.properties['volume_config'] = {} + for pool_vol in args.pool: + try: + pool_name, volume_name = pool_vol.split(':') + config = {'pool': pool_name, 'name': volume_name} + args.properties['volume_config'][volume_name] = config + except ValueError: + parser.error( + 'Pool argument must be of form: -P pool_name:volume_name') + if 'label' not in args.properties: parser.error('--label option is mandatory') diff --git a/qubes/vm/appvm.py b/qubes/vm/appvm.py index aa2e3966..d310ca99 100644 --- a/qubes/vm/appvm.py +++ b/qubes/vm/appvm.py @@ -3,16 +3,44 @@ import qubes.events import qubes.vm.qubesvm +from qubes.config import defaults + class AppVM(qubes.vm.qubesvm.QubesVM): '''Application VM''' - template = qubes.VMProperty('template', load_stage=4, - vmclass=qubes.vm.templatevm.TemplateVM, - ls_width=31, - doc='Template, on which this AppVM is based.') + template = qubes.VMProperty('template', + load_stage=4, + vmclass=qubes.vm.templatevm.TemplateVM, + ls_width=31, + doc='Template, on which this AppVM is based.') def __init__(self, *args, **kwargs): + self.volumes = {} + self.volume_config = { + 'root': { + 'name': 'root', + 'pool': 'default', + 'volume_type': 'snapshot', + }, + 'private': { + 'name': 'private', + 'pool': 'default', + 'volume_type': 'read-write', + 'size': defaults['private_img_size'], + }, + 'volatile': { + 'name': 'volatile', + 'pool': 'default', + 'volume_type': 'volatile', + 'size': defaults['root_img_size'], + }, + 'kernel': { + 'name': 'kernel', + 'pool': 'linux-kernel', + 'volume_type': 'read-only', + } + } super(AppVM, self).__init__(*args, **kwargs) @qubes.events.handler('domain-load') @@ -20,4 +48,4 @@ class AppVM(qubes.vm.qubesvm.QubesVM): # pylint: disable=unused-argument # Some additional checks for template based VM assert self.template - #self.template.appvms.add(self) # XXX + # self.template.appvms.add(self) # XXX diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 09519010..6b5135ec 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -29,6 +29,7 @@ from __future__ import absolute_import import base64 import datetime import itertools +import lxml import os import os.path import re @@ -209,7 +210,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): default='default', doc='storage pool for this qube devices') - dir_path = property((lambda self: self.storage.vmdir), + dir_path = property( (lambda self: os.path.join(qubes.config.system_path['qubes_base_dir'], self.dir_path_prefix, self.name)), doc='Root directory for files related to this domain') # XXX swallowed uses_default_kernel @@ -331,6 +332,12 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): raise return self._libvirt_domain + @property + def block_devices(self): + ''' Return all :py:class:`qubes.devices.BlockDevice`s for current domain + for serialization in the libvirt XML template as . + ''' + return [v.block_device() for v in self.volumes.values()] @property def qdb(self): @@ -347,21 +354,27 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): def private_img(self): '''Location of private image of the VM (that contains :file:`/rw` \ and :file:`/home`).''' - return self.storage.private_img + warnings.warn("volatile_img is deprecated, use volumes['private'].vid", + DeprecationWarning) + return self.volumes['private'].vid # XXX this should go to to AppVM? or TemplateVM? @property def root_img(self): '''Location of root image.''' - return self.storage.root_img + warnings.warn("root_img is deprecated, use volumes['root'].vid", + DeprecationWarning) + return self.volumes['root'].vid # XXX and this should go to exactly where? DispVM has it. @property def volatile_img(self): '''Volatile image that overlays :py:attr:`root_img`.''' - return self.storage.volatile_img + warnings.warn("volatile_img is deprecated, use volumes['volatile'].vid", + DeprecationWarning) + return self.volumes['volatile'].vid # XXX shouldn't this go elsewhere? @@ -419,12 +432,27 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): # constructor # - def __init__(self, app, xml, **kwargs): + def __init__(self, app, xml, volume_config={}, **kwargs): super(QubesVM, self).__init__(app, xml, **kwargs) + if hasattr(self, 'volume_config'): + if xml is not None: + for node in xml.xpath('volume-config/volume'): + name = node.get('name') + assert name + for k, v in node.items(): + self.volume_config[name][k] = v - import qubes.vm.adminvm # pylint: disable=redefined-outer-name + for name, conf in volume_config.items(): + for k, v in conf.items(): + self.volume_config[name][k] = v + elif volume_config: + raise TypeError( + 'volume_config specified, but {} did not expect that.' % + self.__class__.__name__) - #Init private attrs + import qubes.vm.adminvm # pylint: disable=redefined-outer-name + + # Init private attrs self._libvirt_domain = None self._qdb_connection = None @@ -458,13 +486,23 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): self.features['check-updates'] = None # will be initialized after loading all the properties - self.storage = None # fire hooks if xml is None: self.events_enabled = True self.fire_event('domain-init') + def __xml__(self): + element = super(QubesVM, self).__xml__() + if hasattr(self, 'volumes'): + volume_config_node = lxml.etree.Element('volume-config') + for volume in self.volumes.values(): + volume_config_node.append(volume.__xml__()) + + element.append(volume_config_node) + + + return element # # event handlers @@ -477,8 +515,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): self.uuid = uuid.uuid4() # Initialize VM image storage class - self.storage = qubes.storage.get_pool( - self.pool_name, self).get_storage() + self.storage = qubes.storage.Storage(self) @qubes.events.handler('property-set:label') @@ -532,18 +569,18 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): self._qdb_connection.close() self._qdb_connection = None - self.storage.rename( - os.path.join(qubes.config.system_path['qubes_base_dir'], - self.dir_path_prefix, new_name), - os.path.join(qubes.config.system_path['qubes_base_dir'], - self.dir_path_prefix, old_name)) + self.storage.rename(old_name, new_name) + + prefix = os.path.join(qubes.config.system_path['qubes_base_dir'], self.dir_path_prefix) + old_config = os.path.join(prefix, old_name, old_name + '.conf') + new_config = os.path.join(prefix, new_name, new_name + '.conf') + os.rename(old_config, new_config) self._update_libvirt_domain() if self.autostart: self.autostart = self.autostart - @qubes.events.handler('property-pre-set:autostart') def on_property_pre_set_autostart(self, event, prop, name, value, oldvalue=None): @@ -633,7 +670,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): self.netvm.start(start_guid=start_guid, notify_function=notify_function) - self.storage.prepare_for_vm_startup() + self.storage.start() self._update_libvirt_domain() qmemman_client = self.request_memory(mem_required) @@ -725,6 +762,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): exc_info=1) self.libvirt_domain.shutdown() + self.storage.stop() def kill(self): @@ -738,6 +776,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): raise qubes.exc.QubesVMNotStartedError(self) self.libvirt_domain.destroy() + self.storage.stop() def force_shutdown(self, *args, **kwargs): @@ -1021,7 +1060,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): p.communicate(input=self.default_user) - # TODO move to storage + # TODO rename to create def create_on_disk(self, source_template=None): '''Create files needed for VM. @@ -1050,11 +1089,11 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): def resize_private_img(self, size): '''Resize private image.''' - if size >= self.get_private_img_sz(): - raise qubes.exc.QubesValueError('Cannot shrink private.img') + warnings.warn( + "resize_private_img is deprecated, use volumes[name].resize()", + DeprecationWarning) - # resize the image - self.storage.resize_private_img(size) + self.volumes['private'].resize(size) # and then the filesystem # FIXME move this to qubes.storage.xen.XenVMStorage @@ -1070,29 +1109,12 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): if retcode != 0: raise qubes.exc.QubesException('resize2fs failed') - - # TODO move to storage def resize_root_img(self, size, allow_start=False): - if hasattr(self, 'template'): - raise qubes.exc.QubesVMError(self, - 'Cannot resize root.img of template based qube. Resize the' - ' root.img of the template instead.') + warnings.warn( + "resize_root_img is deprecated, use volumes[name].resize()", + DeprecationWarning) - # TODO self.is_halted - if self.is_running(): - raise qubes.exc.QubesVMNotHaltedError(self, - 'Cannot resize root.img of a running qube') - - if size < self.get_root_img_sz(): - raise qubes.exc.QubesValueError( - 'For your own safety, shrinking of root.img is disabled. If you' - ' really know what you are doing, use `truncate` manually.') - - with open(self.root_img, 'a+b') as fd: - fd.truncate(size) - - if False: #self.hvm: - return + self.volumes['root'].resize(size) if not allow_start: raise qubes.exc.QubesException( @@ -1111,12 +1133,11 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): while self.is_running(): #1696 time.sleep(1) - def remove_from_disk(self): '''Remove domain remnants from disk.''' self.fire_event('domain-remove-from-disk') - self.storage.remove_from_disk() - + self.storage.remove() + shutil.rmtree(self.vm.dir_path) def clone_disk_files(self, src): '''Clone files from other vm. @@ -1124,11 +1145,14 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): :param qubes.vm.qubesvm.QubesVM src: source VM ''' - if src.is_running(): # XXX what about paused? + if src.is_running(): # XXX what about paused? raise qubes.exc.QubesVMNotHaltedError( self, 'Cannot clone a running domain {!r}'.format(self.name)) - self.storage.clone_disk_files(src) + if hasattr(src, 'volume_config'): + self.volume_config = src.volume_config + self.storage = qubes.storage.Storage(self) + self.storage.clone(src) if src.icon_path is not None \ and os.path.exists(src.dir_path) \ @@ -1450,68 +1474,69 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): # XXX shouldn't this go only to vms that have root image? def get_disk_utilization_root_img(self): - '''Get space that is actually ocuppied by :py:attr:`root_img`. - - Root image is a sparse file, so it is probably much less than logical - available space. + '''Get space that is actually ocuppied by :py:attr:`volumes['root']`. + This is a temporary wrapper for backwards compatibility. You should + call directly :py:attr:`volumes[name].utilization` :returns: domain's real disk image size [FIXME unit] :rtype: FIXME .. seealso:: :py:meth:`get_root_img_sz` ''' - return qubes.storage.get_disk_usage(self.root_img) + warnings.warn( + "get_disk_utilization_root_img is deprecated, use volumes['root'].utilization", + DeprecationWarning) + return qubes.storage.get_disk_usage(self.volumes['root'].utilization) # XXX shouldn't this go only to vms that have root image? def get_root_img_sz(self): - '''Get image size of :py:attr:`root_img`. - - Root image is a sparse file, so it is probably much more than ocuppied - physical space. + '''Get the size of the :py:attr:`volumes['root']`. + This is a temporary wrapper for backwards compatibility. You should + call directly :py:attr:`volumes[name].size` :returns: domain's virtual disk size [FIXME unit] :rtype: FIXME .. seealso:: :py:meth:`get_disk_utilization_root_img` ''' - if not os.path.exists(self.root_img): - return 0 - - return os.path.getsize(self.root_img) - + warnings.warn( + "get_disk_root_img_sz is deprecated, use volumes['root'].size", + DeprecationWarning) + return qubes.storage.get_disk_usage(self.volumes['root'].size) def get_disk_utilization_private_img(self): - '''Get space that is actually ocuppied by :py:attr:`private_img`. - - Private image is a sparse file, so it is probably much less than - logical available space. + '''Get space that is actually ocuppied by :py:attr:`volumes['private']`. + This is a temporary wrapper for backwards compatibility. You should + call directly :py:attr:`volumes[name].utilization` :returns: domain's real disk image size [FIXME unit] :rtype: FIXME + ''' - .. seealso:: :py:meth:`get_private_img_sz` - ''' # pylint: disable=invalid-name - - return qubes.storage.get_disk_usage(self.private_img) - + warnings.warn( + "get_disk_utilization_private_img is deprecated, use volumes['private'].utilization", + DeprecationWarning) + return qubes.storage.get_disk_usage(self.volumes[ + 'private'].utilization) def get_private_img_sz(self): - '''Get image size of :py:attr:`private_img`. - - Private image is a sparse file, so it is probably much more than - ocuppied physical space. + '''Get the size of the :py:attr:`volumes['private']`. + This is a temporary wrapper for backwards compatibility. You should + call directly :py:attr:`volumes[name].size` :returns: domain's virtual disk size [FIXME unit] :rtype: FIXME .. seealso:: :py:meth:`get_disk_utilization_private_img` ''' - return self.storage.get_private_img_sz() - + warnings.warn( + "get_disk_private_img_sz is deprecated, use volumes['private'].size", + DeprecationWarning) + return qubes.storage.get_disk_usage(self.volumes['private'].size) def get_disk_utilization(self): '''Return total space actually occuppied by all files belonging to \ @@ -1523,7 +1548,6 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): return qubes.storage.get_disk_usage(self.dir_path) - # TODO move to storage def verify_files(self): '''Verify that files accessed by this machine are sane. @@ -1533,18 +1557,6 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): self.storage.verify_files() - if not os.path.exists( - os.path.join(self.storage.kernels_dir, 'vmlinuz')): - raise qubes.exc.QubesException( - 'VM kernel does not exist: {0}'.format( - os.path.join(self.storage.kernels_dir, 'vmlinuz'))) - - if not os.path.exists( - os.path.join(self.storage.kernels_dir, 'initramfs')): - raise qubes.exc.QubesException( - 'VM initramfs does not exist: {0}'.format( - os.path.join(self.storage.kernels_dir, 'initramfs'))) - self.fire_event('domain-verify-files') return True diff --git a/qubes/vm/templatevm.py b/qubes/vm/templatevm.py index f0c7fe2f..39432ac8 100644 --- a/qubes/vm/templatevm.py +++ b/qubes/vm/templatevm.py @@ -1,11 +1,16 @@ #!/usr/bin/python2 -O # vim: fileencoding=utf-8 +import warnings + import qubes import qubes.config import qubes.vm.qubesvm +from qubes.config import defaults +from qubes.vm.qubesvm import QubesVM -class TemplateVM(qubes.vm.qubesvm.QubesVM): + +class TemplateVM(QubesVM): '''Template for AppVM''' dir_path_prefix = qubes.config.system_path['qubes_templates_dir'] @@ -13,7 +18,9 @@ class TemplateVM(qubes.vm.qubesvm.QubesVM): @property def rootcow_img(self): '''COW image''' - return self.storage.rootcow_img + warnings.warn("rootcow_img is deprecated, use " + "volumes['root'].path_origin", DeprecationWarning) + return self.volumes['root'].path_cow @property def appvms(self): @@ -23,20 +30,40 @@ class TemplateVM(qubes.vm.qubesvm.QubesVM): def __init__(self, *args, **kwargs): assert 'template' not in kwargs, "A TemplateVM can not have a template" + self.volumes = {} + self.volume_config = { + 'root': { + 'name': 'root', + 'pool': 'default', + 'volume_type': 'origin', + 'size': defaults['root_img_size'], + }, + 'private': { + 'name': 'private', + 'pool': 'default', + 'volume_type': 'read-write', + 'size': defaults['private_img_size'], + }, + 'volatile': { + 'name': 'volatile', + 'pool': 'default', + 'size': defaults['root_img_size'], + 'volume_type': 'volatile', + }, + 'kernel': { + 'name': 'kernel', + 'pool': 'linux-kernel', + 'volume_type': 'read-only', + } + } super(TemplateVM, self).__init__(*args, **kwargs) - # Some additional checks for template based VM - # TODO find better way -# assert self.root_img is not None, "Missing root_img for standalone VM!" - - def clone_disk_files(self, src): super(TemplateVM, self).clone_disk_files(src) # Create root-cow.img self.commit_changes() - def commit_changes(self): '''Commit changes to template''' self.log.debug('commit_changes()') @@ -45,6 +72,4 @@ class TemplateVM(qubes.vm.qubesvm.QubesVM): assert not self.is_running(), \ 'Attempt to commit changes on running Template VM!' - self.log.info( - 'Commiting template update; COW: {}'.format(self.rootcow_img)) self.storage.commit_template_changes() diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec index 942e3f66..7f0b7b48 100644 --- a/rpm_spec/core-dom0.spec +++ b/rpm_spec/core-dom0.spec @@ -197,7 +197,6 @@ fi %files %defattr(-,root,root,-) %config(noreplace) %attr(0664,root,qubes) %{_sysconfdir}/qubes/qmemman.conf -%config(noreplace) %attr(0664,root,qubes) %{_sysconfdir}/qubes/storage.conf /usr/bin/qvm-* /usr/bin/qubes-* /usr/bin/qmemmand @@ -234,6 +233,7 @@ fi %dir %{python_sitelib}/qubes/storage %{python_sitelib}/qubes/storage/__init__.py* %{python_sitelib}/qubes/storage/xen.py* +%{python_sitelib}/qubes/storage/kernels.py* %dir %{python_sitelib}/qubes/tools %{python_sitelib}/qubes/tools/__init__.py* diff --git a/setup.py b/setup.py index 533981f1..ad3a0f90 100644 --- a/setup.py +++ b/setup.py @@ -45,5 +45,6 @@ if __name__ == '__main__': ], 'qubes.storage': [ 'xen = qubes.storage.xen:XenPool', + 'linux-kernel = qubes.storage.kernels:LinuxKernel', ] }) diff --git a/templates/libvirt/xen.xml b/templates/libvirt/xen.xml index 13b15e51..a16ff581 100644 --- a/templates/libvirt/xen.xml +++ b/templates/libvirt/xen.xml @@ -45,12 +45,27 @@ destroy destroy - {# - {% for device in vm.storage %} - + {% set i = 0 %} + {# TODO Allow more volumes out of the box #} + {% set dd = ['e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', + 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y'] + %} + {% for device in vm.block_devices %} + - + {% if device.name == 'root' %} + + {% elif device.name == 'private' %} + + {% elif device.name == 'volatile' %} + + {% elif device.name == 'kernel' %} + + {% else %} + + {% set i = i + 1 %} + {% endif %} {% if not device.rw %} @@ -65,15 +80,6 @@ {% endif %} {% endfor %} - #} - - {{ vm.storage.root_dev_config() }} - {% if not prepare_dvm %}{{ vm.storage.private_dev_config() }}{% endif %} - {{ vm.storage.other_dev_config() }} - - {% if not vm.hvm %} - {{ vm.storage.volatile_dev_config() }} - {% endif %} {% if vm.netvm %}