From bdfb85ac1926ac987c471af817dcfe50157bce2c Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 15 Apr 2016 20:40:53 +0200 Subject: [PATCH] Refactor Storage, Pool and XenPool - Remove all *_dev_config methods - Checks if a storage image exists moved to XenPool - Storage.remove wraps Pool.remove() - Stop volumes on domain sutdown/kill - Warn when using deprecated methods --- qubes/storage/__init__.py | 368 ++++++++++------------------- qubes/storage/xen.py | 473 +++++++++++++++++--------------------- qubes/vm/qubesvm.py | 111 +++++---- qubes/vm/templatevm.py | 11 +- 4 files changed, 406 insertions(+), 557 deletions(-) diff --git a/qubes/storage/__init__.py b/qubes/storage/__init__.py index 3bfb4762..da257d69 100644 --- a/qubes/storage/__init__.py +++ b/qubes/storage/__init__.py @@ -39,7 +39,6 @@ import qubes.exc import qubes.utils from qubes.devices import BlockDevice -BLKSIZE = 512 STORAGE_ENTRY_POINT = 'qubes.storage' @@ -93,91 +92,20 @@ class Storage(object): in mind. ''' - modules_dev = 'xvdd' - def __init__(self, vm): #: Domain for which we manage storage self.vm = vm + self.log = self.vm.log #: Additional drive (currently used only by HVM) self.drive = None - for name, conf in self.vm.volume_config.items(): - assert 'pool' in conf - pool = get_pool(conf['pool'], self.vm) - self.vm.volumes[name] = pool.init_volume(conf) - - @property - def root_img(self): - pool = self.get_pool() - return pool.root_img - - @property - def private_img(self): - pool = self.get_pool() - return pool.private_img - - @property - def volatile_img(self): - pool = self.get_pool() - return pool.volatile_img - - @property - def rootcow_img(self): - pool = self.get_pool() - return pool.rootcow_img - - def get_config_params(self): - args = {} - args['rootdev'] = self.root_dev_config() - args['privatedev'] = self.private_dev_config() - args['volatiledev'] = self.volatile_dev_config() - args['otherdevs'] = self.other_dev_config() - - args['kerneldir'] = self.kernels_dir - - return args - - def root_dev_config(self): - pool = self.get_pool() - return pool.root_dev_config() - - def private_dev_config(self): - pool = self.get_pool() - return pool.private_dev_config() - - def volatile_dev_config(self): - pool = self.get_pool() - return pool.volatile_dev_config() - - def modules_dev_config(self): - return self.format_disk_dev(self.modules_img, - 'kernel', - rw=self.modules_img_rw) - - def other_dev_config(self): - if self.modules_img is not None: - return self.modules_dev_config() - elif self.drive is not None: - (drive_type, drive_domain, drive_path) = self.drive.split(":") - if drive_type == 'hd': - drive_type = 'disk' - - rw = (drive_type == 'disk') - - if drive_domain.lower() == "dom0": - drive_domain = None - - return self.format_disk_dev(drive_path, - 'other', - rw=rw, - devtype=drive_type, - domain=drive_domain) - - else: - return '' - - def format_disk_dev(self, path, name, script=None, rw=True, devtype='disk', - domain=None): - return BlockDevice(path, name, script, rw, domain, devtype) + self.pools = {} + if hasattr(vm, 'volume_config'): + for name, conf in self.vm.volume_config.items(): + assert 'pool' in conf, "Pool missing in volume_config" % str( + conf) + pool = self.vm.app.get_pool(conf['pool']) + self.vm.volumes[name] = pool.init_volume(self.vm, conf) + self.pools[name] = pool @property def kernels_dir(self): @@ -186,51 +114,33 @@ class Storage(object): If :py:attr:`self.vm.kernel` is :py:obj:`None`, the this points inside :py:attr:`self.vm.dir_path` ''' - return os.path.join(qubes.config.system_path['qubes_base_dir'], - qubes.config.system_path['qubes_kernels_base_dir'], self.vm.kernel)\ - if self.vm.kernel is not None \ - else os.path.join(self.vm.dir_path, - qubes.config.vm_files['kernels_subdir']) - - @property - def modules_img(self): - '''Path to image with modules. - - Depending on domain, this may be global or inside domain's dir. - ''' - - modules_path = os.path.join(self.kernels_dir, 'modules.img') - - if os.path.exists(modules_path): - return modules_path - else: - return None - - - @property - def modules_img_rw(self): - ''':py:obj:`True` if module image should be mounted RW, :py:obj:`False` - otherwise.''' - return self.vm.kernel is None - + assert 'kernel' in self.vm.volumes, "VM has no kernel pool" + return self.vm.volumes['kernel'].kernels_dir def get_disk_utilization(self): - return get_disk_usage(self.vm.dir_path) + ''' Returns summed up disk utilization for all domain volumes ''' + result = 0 + for volume in self.vm.volumes.values(): + result += volume.usage + return result + # TODO Remove this wrapper def get_disk_utilization_private_img(self): - # pylint: disable=invalid-name - return get_disk_usage(self.private_img) + # pylint: disable=invalid-name,missing-docstring + return self.vm.volume['private'].usage + # TODO Remove this wrapper def get_private_img_sz(self): - if not os.path.exists(self.private_img): - return 0 + # :pylint: disable=missing-docstring + return self.vm.volume['private'].size - return os.path.getsize(self.private_img) - - def resize_private_img(self, size): - raise NotImplementedError() + def resize(self, volume, size): + ''' Resize volume ''' + self.get_pool(volume).resize(volume, size) + # TODO rename it to create() def create_on_disk(self, source_template=None): + # :pylint: disable=missing-docstring if source_template is None and hasattr(self.vm, 'template'): source_template = self.vm.template @@ -238,14 +148,17 @@ class Storage(object): self.vm.log.info('Creating directory: {0}'.format(self.vm.dir_path)) os.makedirs(self.vm.dir_path) - pool = self.get_pool() - pool.create_on_disk_private_img(source_template) - pool.create_on_disk_root_img(source_template) - pool.reset_volatile_storage() + for name, volume in self.vm.volumes.items(): + source_volume = None + if source_template and hasattr(source_template, 'volumes'): + source_volume = source_template.volumes[name] + self.get_pool(volume).create(volume, source_volume=source_volume) os.umask(old_umask) + # TODO migrate this def clone_disk_files(self, src_vm): + # :pylint: disable=missing-docstring self.vm.log.info('Creating directory: {0}'.format(self.vm.dir_path)) os.mkdir(self.vm.dir_path) @@ -255,14 +168,15 @@ class Storage(object): self._copy_file(src_vm.private_img, self.vm.private_img) if src_vm.updateable and hasattr(src_vm, 'root_img'): - self.vm.log.info('Copying the root image: {} -> {}'.format( - src_vm.root_img, self.root_img)) - self._copy_file(src_vm.root_img, self.root_img) - - # TODO: modules? - # XXX which modules? -woju - + self.vm.log.info( + 'Copying the root image: {} -> {}'.format( + src_vm.volume['root'].path_origin, + self.vm.volume['root'].path_origin) + ) + self._copy_file(src_vm.volume['root'].path_origin, + self.vm.volume['root'].path_origin) + # TODO migrate this @staticmethod def rename(newpath, oldpath): '''Move storage directory, most likely during domain's rename. @@ -280,133 +194,87 @@ class Storage(object): os.rename(oldpath, newpath) - def verify_files(self): + '''Verify that the storage is sane. + + On success, returns normally. On failure, raises exception. + ''' if not os.path.exists(self.vm.dir_path): - raise qubes.exc.QubesVMError(self.vm, + raise qubes.exc.QubesVMError( + self.vm, 'VM directory does not exist: {}'.format(self.vm.dir_path)) - if hasattr(self.vm, 'root_img') and not os.path.exists(self.root_img): - raise qubes.exc.QubesVMError(self.vm, - 'VM root image file does not exist: {}'.format(self.root_img)) - - if hasattr(self.vm, 'private_img') \ - and not os.path.exists(self.private_img): - raise qubes.exc.QubesVMError(self.vm, - 'VM private image file does not exist: {}'.format( - self.private_img)) - - if self.modules_img is not None \ - and not os.path.exists(self.modules_img): - raise qubes.exc.QubesVMError(self.vm, - 'VM kernel modules image does not exists: {}'.format( - self.modules_img)) - - - def remove_from_disk(self): + def remove(self): + for name, volume in self.vm.volumes.items(): + self.log.info('Removing volume %s: %s' % (name, volume.vid)) + self.get_pool(volume).remove(volume) shutil.rmtree(self.vm.dir_path) + def start(self): + ''' Execute the start method on each pool ''' + for volume in self.vm.volumes.values(): + self.get_pool(volume).start(volume) + def stop(self): + ''' Execute the start method on each pool ''' + for volume in self.vm.volumes.values(): + self.get_pool(volume).stop(volume) - def prepare_for_vm_startup(self): - pool = get_pool(self.vm.pool_name, self.vm) - pool.reset_volatile_storage() - - if hasattr(self.vm, 'private_img') \ - and not os.path.exists(self.private_img): - self.vm.log.info('Creating empty VM private image file: {0}'.format( - pool.private_img)) - pool.create_on_disk_private_img() - - def get_pool(self): - return get_pool(self.vm.pool_name, self.vm) + def get_pool(self, volume): + ''' Helper function ''' + assert isinstance(volume, Volume), "You need to pass a Volume" + return self.pools[volume.name] def commit_template_changes(self): - pool = self.get_pool() - pool.commit_template_changes() + for volume in self.vm.volumes.values(): + if volume.volume_type == 'origin': + self.get_pool(volume).commit_template_changes(volume) -def get_disk_usage_one(st): - '''Extract disk usage of one inode from its stat_result struct. - - If known, get real disk usage, as written to device by filesystem, not - logical file size. Those values may be different for sparse files. - - :param os.stat_result st: stat result - :returns: disk usage - ''' - try: - return st.st_blocks * BLKSIZE - except AttributeError: - return st.st_size - - -def get_disk_usage(path): - '''Get real disk usage of given path (file or directory). - - When *path* points to directory, then it is evaluated recursively. - - This function tries estiate real disk usage. See documentation of - :py:func:`get_disk_usage_one`. - - :param str path: path to evaluate - :returns: disk usage - ''' - try: - st = os.lstat(path) - except OSError: - return 0 - - ret = get_disk_usage_one(st) - - # if path is not a directory, this is skipped - for dirpath, dirnames, filenames in os.walk(path): - for name in dirnames + filenames: - ret += get_disk_usage_one(os.lstat(os.path.join(dirpath, name))) - - return ret class Pool(object): + ''' A Pool is used to manage different kind of volumes (File + based/LVM/Btrfs/...). + + 3rd Parties providing own storage implementations will need to extend + this class. + ''' private_img_size = qubes.config.defaults['private_img_size'] root_img_size = qubes.config.defaults['root_img_size'] - def __init__(self, vm=None, name=None, **kwargs): - assert vm + def __init__(self, name=None, **kwargs): + # :pylint: disable=unused-argument assert name, "Pool name is missing" - self.vm = vm self.name = name - def root_dev_config(self): - raise NotImplementedError() + def create(self, volume, source_volume): + ''' Create the given volume on disk or copy from provided + `source_volume`. + ''' + raise NotImplementedError("Pool %s has create() not implemented" % + self.name) - def private_dev_config(self): - raise NotImplementedError() + def commit_template_changes(self, volume): + ''' Update origin device ''' + raise NotImplementedError( + "Pool %s has commit_template_changes() not implemented" % + self.name) - def volatile_dev_config(self): - raise NotImplementedError() + @property + def config(self): + ''' Returns the pool config to be written to qubes.xml ''' + raise NotImplementedError("Pool %s has config() not implemented" % + self.name) - def create_on_disk_private_img(self, source_template=None): - raise NotImplementedError() - - def create_on_disk_root_img(self, source_template=None): - raise NotImplementedError() - - def create_dir_if_not_exists(self, path): - """ Check if a directory exists in if not create it. - - This method does not create any parent directories. - """ - if not os.path.exists(path): - os.mkdir(path) - - def commit_template_changes(self): - raise NotImplementedError() @staticmethod def _copy_file(source, destination): '''Effective file copy, preserving sparse files etc. ''' # TODO: Windows support - # We prefer to use Linux's cp, because it nicely handles sparse files + assert os.path.exists(source), \ + "Missing the source %s to copy from" % source + assert not os.path.exists(destination), \ + "Destination %s already exists" % destination try: subprocess.check_call(['cp', '--reflink=auto', source, destination ]) @@ -414,34 +282,34 @@ class Pool(object): raise IOError('Error while copying {!r} to {!r}'.format( source, destination)) - def format_disk_dev(self, path, vdev, script=None, rw=True, devtype='disk', - domain=None): + def remove(self, volume): + ''' Remove volume''' + raise NotImplementedError("Pool %s has remove() not implemented" % + self.name) - return BlockDevice(path, vdev, script, rw, domain, devtype) - def reset_volatile_storage(self): - # Re-create only for template based VMs - try: - if self.vm.template is not None and self.volatile_img: - if os.path.exists(self.volatile_img): - os.remove(self.volatile_img) - except AttributeError: # self.vm.template - pass + def clone(self, source, target): + ''' Clone volume ''' + raise NotImplementedError("Pool %s has clone() not implemented" % + self.name) - # For StandaloneVM create it only if not already exists - # (eg after backup-restore) - if hasattr(self, 'volatile_img') \ - and not os.path.exists(self.volatile_img): - self.vm.log.info( - 'Creating volatile image: {0}'.format(self.volatile_img)) - subprocess.check_call( - [qubes.config.system_path["prepare_volatile_img_cmd"], - self.volatile_img, - str(self.root_img_size / 1024 / 1024)]) + def start(self, volume): + ''' Do what ever is needed on start ''' + raise NotImplementedError("Pool %s has start() not implemented" % + self.name) + + def stop(self, volume): + ''' Do what ever is needed on stop''' + raise NotImplementedError("Pool %s has stop() not implemented" % + self.name) def init_volume(self, volume_config): - raise NotImplementedError() + ''' Initialize a :py:class:`qubes.storage.Volume` from `volume_config`. + ''' + raise NotImplementedError("Pool %s has init_volume() not implemented" % + self.name) + def pool_drivers(): """ Return a list of EntryPoints names """ return [ep.name - for ep in pkg_resources.iter_entry_points(STORAGE_ENTRY_POINT)] + for ep in pkg_resources.iter_entry_points(STORAGE_ENTRY_POINT)] diff --git a/qubes/storage/xen.py b/qubes/storage/xen.py index dc1bced6..a50cd23f 100644 --- a/qubes/storage/xen.py +++ b/qubes/storage/xen.py @@ -31,137 +31,70 @@ import os.path import re import subprocess -import qubes -import qubes.config -import qubes.vm.templatevm from qubes.storage import Pool, StoragePoolException, Volume +BLKSIZE = 512 + class XenPool(Pool): + ''' File based 'original' disk implementation ''' - root_dev = 'xvda' - private_dev = 'xvdb' - volatile_dev = 'xvdc' - - def __init__(self, vm=None, name=None, dir_path=None): - super(XenPool, self).__init__(vm=vm, name=name) + def __init__(self, name=None, dir_path=None): + super(XenPool, self).__init__(name=name) assert dir_path, "No pool dir_path specified" self.dir_path = os.path.normpath(dir_path) - self.create_dir_if_not_exists(self.dir_path) + create_dir_if_not_exists(self.dir_path) appvms_path = os.path.join(self.dir_path, 'appvms') - self.create_dir_if_not_exists(appvms_path) + create_dir_if_not_exists(appvms_path) vm_templates_path = os.path.join(self.dir_path, 'vm-templates') - self.create_dir_if_not_exists(vm_templates_path) + create_dir_if_not_exists(vm_templates_path) - @property - def private_img(self): - '''Path to the private image''' - return self.abspath(qubes.config.vm_files['private_img']) + def create(self, volume, source_volume=None): + _type = volume.volume_type + size = volume.size + if _type == 'origin': + create_sparse_file(volume.path_origin, size) + create_sparse_file(volume.path_cow, size) + elif _type in ['read-write'] and source_volume: + copy_file(source_volume.path, volume.path) + elif _type in ['read-write', 'volatile']: + create_sparse_file(volume.path, size) - @property - def root_img(self): - '''Path to the root image''' - return self.vm.template.storage.root_img \ - if hasattr(self.vm, 'template') and self.vm.template \ - else self.abspath(qubes.config.vm_files['root_img']) + return volume - @property - def rootcow_img(self): - '''Path to the root COW image''' + def resize(self, volume, size): + ''' Expands volume, throws + :py:class:`qubst.storage.StoragePoolException` if given size is + less than current_size + ''' + _type = volume.volume_type + if _type not in ['origin', 'read-write', 'volatile']: + raise StoragePoolException('Can not resize a %s volume %s' % + (_type, volume.vid)) - if isinstance(self.vm, qubes.vm.templatevm.TemplateVM): - return self.abspath(qubes.config.vm_files['rootcow_img']) + if size <= volume.size: + raise StoragePoolException( + 'For your own safety, shrinking of %s is' + ' disabled. If you really know what you' + ' are doing, use `truncate` on %s manually.' % + (volume.name, volume.vid)) - return None + if _type == 'origin': + path = volume.path_origin + elif _type in ['read-write', 'volatile']: + path = volume.path - @property - def volatile_img(self): - '''Path to the volatile image''' - return self.abspath(qubes.config.vm_files['volatile_img']) + if size <= volume.size: + raise StoragePoolException('Can not shring volume %s' % + volume.name) - def root_dev_config(self): - dev_name = 'root' - if isinstance(self.vm, qubes.vm.templatevm.TemplateVM): - return self.format_disk_dev( - '{root}:{rootcow}'.format( - root=self.root_img, - rootcow=self.rootcow_img), - dev_name, - script='block-origin') - - elif self.vm.hvm and hasattr(self.vm, 'template'): - # HVM template-based VM - only one device-mapper layer, in dom0 - # (root+volatile) - # HVM detection based on 'kernel' property is massive hack, - # but taken from assumption that VM needs Qubes-specific kernel - # (actually initramfs) to assemble the second layer of device-mapper - return self.format_disk_dev( - '{root}:{volatile}'.format( - root=self.vm.template.storage.root_img, - volatile=self.volatile_img), - dev_name, - script='block-snapshot') - - elif hasattr(self.vm, 'template'): - # any other template-based VM - two device-mapper layers: one - # in dom0 (here) from root+root-cow, and another one from - # this+volatile.img - path = '{root}:{template_rootcow}'.format( - root=self.root_img, - template_rootcow=self.vm.template.storage.rootcow_img) - return self.format_disk_dev(path=path, - vdev=self.root_dev, - script='block-snapshot', - rw=False) - - else: - # standalone qube - return self.format_disk_dev(self.root_img, dev_name) - - def private_dev_config(self): - return self.format_disk_dev(self.private_img, 'private') - - def volatile_dev_config(self): - return self.format_disk_dev(self.volatile_img, 'volatile') - - def create_on_disk_private_img(self, source_template=None): - if not os.path.exists(self.target_dir): - os.makedirs(self.target_dir) - if source_template is None: - f_private = open(self.private_img, 'a+b') - f_private.truncate(self.private_img_size) - f_private.close() - - else: - self.vm.log.info("Copying the template's private image: {}".format( - source_template.storage.private_img)) - self._copy_file(source_template.storage.private_img, self.private_img) - - def create_on_disk_root_img(self, source_template=None): - if not os.path.exists(self.target_dir): - os.makedirs(self.target_dir) - if source_template is None: - fd = open(self.root_img, 'a+b') - fd.truncate(self.root_img_size) - fd.close() - - elif self.vm.updateable: - # if not updateable, just use template's disk - self.vm.log.info( - "--> Copying the template's root image: {}".format( - source_template.storage.root_img)) - self._copy_file(source_template.storage.root_img, self.root_img) - - def resize_private_img(self, size): - fd = open(self.private_img, 'a+b') - fd.truncate(size) - fd.close() + with open(path, 'a+b') as fd: + fd.truncate(size) # find loop device if any - p = subprocess.Popen( - ['sudo', 'losetup', '--associated', self.private_img], - stdout=subprocess.PIPE) + p = subprocess.Popen(['sudo', 'losetup', '--associated', path], + stdout=subprocess.PIPE) result = p.communicate() m = re.match(r'^(/dev/loop\d+):\s', result[0]) @@ -172,106 +105,46 @@ class XenPool(Pool): subprocess.check_call(['sudo', 'losetup', '--set-capacity', loop_dev]) - def commit_template_changes(self): - assert isinstance(self.vm, qubes.vm.templatevm.TemplateVM) + def commit_template_changes(self, volume): + if volume.volume_type != 'origin': + return volume - # TODO: move rootcow_img to this class; the same for vm.is_outdated() - if os.path.exists(self.vm.rootcow_img): - os.rename(self.vm.rootcow_img, self.vm.rootcow_img + '.old') + if os.path.exists(volume.path_cow): + os.rename(volume.path_cow, volume.path_cow + '.old') old_umask = os.umask(002) - f_cow = open(self.vm.rootcow_img, 'w') - f_root = open(self.root_img, 'r') - f_root.seek(0, os.SEEK_END) - # make empty sparse file of the same size as root.img - f_cow.truncate(f_root.tell()) - f_cow.close() - f_root.close() + with open(volume.path_cow, 'w') as f_cow: + f_cow.truncate(volume.size) os.umask(old_umask) return volume def start(self, volume): if volume.volume_type == 'volatile': self._reset_volume(volume) + if volume.volume_type in ['origin', 'snapshot']: + _check_path(volume.path_origin) + _check_path(volume.path_cow) + else: + _check_path(volume.path) + return volume + def stop(self, volume): + pass + def _reset_volume(self, volume): ''' Remove and recreate a volatile volume ''' assert volume.volume_type == 'volatile', "Not a volatile volume" - size = self.vm.volume_config[volume.name]['size'] - assert size + assert volume.size - if os.path.exists(volume.path): - os.remove(volume.path) + _remove_if_exists(volume) with open(volume.path, "w") as f_volatile: f_volatile.truncate(volume.size) return volume - def reset_volatile_storage(self): - try: - # no template set, in any way (Standalone VM, Template VM) - if self.vm.template is None: - raise AttributeError - - # template-based HVM with only one device-mapper layer - - # volatile.img used as upper layer on root.img, no root-cow.img - # intermediate layer - if self.vm.hvm: - if os.path.exists(self.volatile_img): - if self.vm.debug: - if os.path.getmtime(self.vm.template.storage.root_img) \ - > os.path.getmtime(self.volatile_img): - self.vm.log.warning( - 'Template have changed, resetting root.img') - else: - self.vm.log.warning( - 'Debug mode: not resetting root.img; if you' - ' want to force root.img reset, either' - ' update template VM, or remove volatile.img' - ' file.') - return - - os.remove(self.volatile_img) - - # FIXME stat on f_root; with open() ... - f_volatile = open(self.volatile_img, "w") - f_root = open(self.vm.template.storage.root_img, "r") - # make empty sparse file of the same size as root.img - f_root.seek(0, os.SEEK_END) - f_volatile.truncate(f_root.tell()) - f_volatile.close() - f_root.close() - return # XXX why is that? super() does not run - except AttributeError: # self.vm.template - pass - - super(XenPool, self).reset_volatile_storage() - - def prepare_for_vm_startup(self): - super(XenPool, self).prepare_for_vm_startup() - - if self.drive is not None: - # pylint: disable=unused-variable - (drive_type, drive_domain, drive_path) = self.drive.split(":") - - if drive_domain.lower() != "dom0": - # XXX "VM '{}' holding '{}' does not exists".format( - drive_vm = self.vm.app.domains[drive_domain] - - if not drive_vm.is_running(): - raise qubes.exc.QubesVMNotRunningError( - drive_vm, 'VM {!r} holding {!r} isn\'t running'.format( - drive_domain, drive_path)) - - if self.rootcow_img and not os.path.exists(self.rootcow_img): - self.commit_template_changes() - - # XXX there is also a class attribute on the domain classes which does - # exactly that -- which one should prevail? - @property - def target_dir(self): + def target_dir(self, vm): """ Returns the path to vmdir depending on the type of the VM. The default QubesOS file storage saves the vm images in three @@ -288,7 +161,6 @@ class XenPool(Pool): string (str) absolute path to the directory where the vm files are stored """ - vm = self.vm if vm.is_template(): subdir = 'vm-templates' elif vm.is_disposablevm(): @@ -300,16 +172,10 @@ class XenPool(Pool): return os.path.join(self.dir_path, subdir, vm.name) - def abspath(self, file_name): - return os.path.join(self.target_dir, file_name) - - def init_volume(self, volume_config): + def init_volume(self, vm, volume_config): assert 'volume_type' in volume_config, "Volume type missing " \ + str(volume_config) - target_dir = self.target_dir - assert target_dir, "Pool target_dir not set" volume_type = volume_config['volume_type'] - volume_config['target_dir'] = target_dir known_types = { 'read-write': ReadWriteFile, 'read-only': ReadOnlyFile, @@ -320,125 +186,124 @@ class XenPool(Pool): if volume_type not in known_types: raise StoragePoolException("Unknown volume type " + volume_type) - if volume_type == 'snapshot': - path = qubes.storage.get_pool(volume_config['pool'], self.vm.template).target_dir - volume_config['vid'] = os.path.join(path, volume_config['name'] + '.img') + if volume_type in ['snapshot', 'read-only']: + origin_pool = vm.app.get_pool(volume_config['pool']) + assert isinstance(origin_pool, + XenPool), 'Origin volume not a xen volume' + volume_config['target_dir'] = origin_pool.target_dir(vm.template) + name = volume_config['name'] + volume_config['size'] = vm.template.volume_config[name]['size'] + else: + volume_config['target_dir'] = self.target_dir(vm) return known_types[volume_type](**volume_config) -class SizeMixIn(Volume): +class XenVolume(Volume): + ''' Parent class for the xen volumes implementation ''' + def __init__(self, target_dir, **kwargs): + self.target_dir = target_dir + assert self.target_dir, "target_dir not specified" + super(XenVolume, self).__init__(**kwargs) + + +class SizeMixIn(XenVolume): + ''' A mix in which expects a `size` param to be > 0 on initialization and + provides a usage property wrapper. + ''' def __init__(self, name=None, pool=None, vid=None, target_dir=None, size=0, **kwargs): assert size > 0, 'Size for volume ' + name + ' is <=0' super(SizeMixIn, self).__init__(name=name, pool=pool, vid=vid, + size=size, **kwargs) - self._size = size self.target_dir = target_dir @property - def size(self): - if self.vid and os.path.exists(self.vid): - return qubes.storage.get_disk_usage(self.vid) - else: - return self._size + def usage(self): + ''' Returns the actualy used space ''' + return get_disk_usage(self.vid) + class ReadWriteFile(SizeMixIn): - # :pylint: disable=too-few-public-methods + # :pylint: disable=missing-docstring def __init__(self, **kwargs): super(ReadWriteFile, self).__init__(**kwargs) self.path = os.path.join(self.target_dir, self.name + '.img') self.vid = self.path - @property - def size(self): - if self.vid and os.path.exists(self.vid): - return qubes.storage.get_disk_usage(self.vid) - else: - return self._size - - def create(self): - create_file(self.path, self.size) - - @property - def created(self): - return os.path.exists(self.path) - class ReadOnlyFile(Volume): + # :pylint: disable=missing-docstring + usage = 0 def __init__(self, name=None, pool=None, vid=None, target_dir=None, - **kwargs): + size=0, **kwargs): + # :pylint: disable=unused-argument assert os.path.exists(vid), "read-only volume missing vid" super(ReadOnlyFile, self).__init__(name=name, - pool=pool, - vid=vid, - **kwargs) + pool=pool, + vid=vid, + size=size, + **kwargs) self.path = self.vid - @property - def size(self): - return qubes.storage.get_disk_usage(self.vid) - class OriginFile(SizeMixIn): + # :pylint: disable=missing-docstring script = 'block-origin' def __init__(self, **kwargs): super(OriginFile, self).__init__(**kwargs) - self.path = os.path.join(self.target_dir, self.name + '.img') + self.path_origin = os.path.join(self.target_dir, self.name + '.img') self.path_cow = os.path.join(self.target_dir, self.name + '-cow.img') - self.vid = self.path - - def create(self): - create_file(self.path, self.size) - create_file(self.path_cow, self.size) + self.path = '%s:%s' % (self.path_origin, self.path_cow) + self.vid = self.path_origin def commit(self): raise NotImplementedError @property - def size(self): - if self.vid and os.path.exists(self.vid): - return qubes.storage.get_disk_usage(self.vid) - else: - return self._size - - @property - def created(self): - return os.path.exists(self.path) and os.path.exists(self.path_cow) + def usage(self): + result = 0 + if os.path.exists(self.path_origin): + result += get_disk_usage(self.path_origin) + if os.path.exists(self.path_cow): + result += get_disk_usage(self.path_cow) + return result class SnapshotFile(Volume): - # :pylint: disable=too-few-public-methods + # :pylint: disable=missing-docstring script = 'block-snapshot' rw = False + usage = 0 def __init__(self, name=None, pool=None, vid=None, target_dir=None, - **kwargs): - assert vid, "SnapshotVolume missing a vid to OriginVolume" - assert os.path.exists(vid), "OriginVolume does not exist" + size=None, **kwargs): + assert size super(SnapshotFile, self).__init__(name=name, - pool=pool, - vid=vid, - **kwargs) - self.path = os.path.join(target_dir, name + '.img') + pool=pool, + vid=vid, + size=size, + **kwargs) + self.path_origin = os.path.join(target_dir, name + '.img') self.path_cow = os.path.join(target_dir, name + '-cow.img') + self.path = '%s:%s' % (self.path_origin, self.path_cow) + self.vid = self.path_origin @property def created(self): - return os.path.exists(self.path) and os.path.exists(self.path_cow) - - @property - def size(self): - return qubes.storage.get_disk_usage(self.vid) + return os.path.exists(self.path_origin) and os.path.exists( + self.path_cow) class VolatileFile(SizeMixIn): + # :pylint: disable=missing-docstring def __init__(self, **kwargs): super(VolatileFile, self).__init__(**kwargs) @@ -446,8 +311,94 @@ class VolatileFile(SizeMixIn): self.vid = self.path -def create_file(path, size): +def create_sparse_file(path, size): + ''' Create an empty sparse file ''' if os.path.exists(path): raise IOError("Volume %s already exists", path) + parent_dir = os.path.dirname(path) + if not os.path.exists(parent_dir): + os.makedirs(parent_dir) with open(path, 'a+b') as fh: fh.truncate(size) + + +def get_disk_usage_one(st): + '''Extract disk usage of one inode from its stat_result struct. + + If known, get real disk usage, as written to device by filesystem, not + logical file size. Those values may be different for sparse files. + + :param os.stat_result st: stat result + :returns: disk usage + ''' + try: + return st.st_blocks * BLKSIZE + except AttributeError: + return st.st_size + + +def get_disk_usage(path): + '''Get real disk usage of given path (file or directory). + + When *path* points to directory, then it is evaluated recursively. + + This function tries estiate real disk usage. See documentation of + :py:func:`get_disk_usage_one`. + + :param str path: path to evaluate + :returns: disk usage + ''' + try: + st = os.lstat(path) + except OSError: + return 0 + + ret = get_disk_usage_one(st) + + # if path is not a directory, this is skipped + for dirpath, dirnames, filenames in os.walk(path): + for name in dirnames + filenames: + ret += get_disk_usage_one(os.lstat(os.path.join(dirpath, name))) + + return ret + + +def create_dir_if_not_exists(path): + """ Check if a directory exists in if not create it. + + This method does not create any parent directories. + """ + if not os.path.exists(path): + os.mkdir(path) + + +def copy_file(source, destination): + '''Effective file copy, preserving sparse files etc. + ''' + # TODO: Windows support + # We prefer to use Linux's cp, because it nicely handles sparse files + assert os.path.exists(source), \ + "Missing the source %s to copy from" % source + assert not os.path.exists(destination), \ + "Destination %s already exists" % destination + + parent_dir = os.path.dirname(destination) + if not os.path.exists(parent_dir): + os.makedirs(parent_dir) + + try: + subprocess.check_call(['cp', '--reflink=auto', source, destination]) + except subprocess.CalledProcessError: + raise IOError('Error while copying {!r} to {!r}'.format(source, + destination)) + + +def _remove_if_exists(volume): + if os.path.exists(volume.path): + os.remove(volume.path) + + +def _check_path(path): + ''' Raise an StoragePoolException if ``path`` does not exist''' + if not os.path.exists(path): + raise StoragePoolException('Missing image file: %s' % path) diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index c6203121..91be2f7d 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -333,10 +333,10 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): @property def block_devices(self): - return [self.storage.root_dev_config(), - self.storage.private_dev_config(), - self.storage.volatile_dev_config(), - self.storage.other_dev_config()] + ''' Return all :py:class:`qubes.devices.BlockDevice`s for current domain + for serialization in the libvirt XML template as . + ''' + return [v.block_device() for v in self.volumes.values()] @property def qdb(self): @@ -353,21 +353,27 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): def private_img(self): '''Location of private image of the VM (that contains :file:`/rw` \ and :file:`/home`).''' - return self.storage.private_img + warnings.warn("volatile_img is deprecated, use volumes['private'].vid", + DeprecationWarning) + return self.volumes['private'].vid # XXX this should go to to AppVM? or TemplateVM? @property def root_img(self): '''Location of root image.''' - return self.storage.root_img + warnings.warn("root_img is deprecated, use volumes['root'].vid", + DeprecationWarning) + return self.volumes['root'].vid # XXX and this should go to exactly where? DispVM has it. @property def volatile_img(self): '''Volatile image that overlays :py:attr:`root_img`.''' - return self.storage.volatile_img + warnings.warn("volatile_img is deprecated, use volumes['volatile'].vid", + DeprecationWarning) + return self.volumes['volatile'].vid # XXX shouldn't this go elsewhere? @@ -425,8 +431,10 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): # constructor # - def __init__(self, app, xml, **kwargs): + def __init__(self, app, xml, volume_config={}, **kwargs): super(QubesVM, self).__init__(app, xml, **kwargs) + if hasattr(self, 'volume_config'): + dict_merge(self.volume_config, volume_config) import qubes.vm.adminvm # pylint: disable=redefined-outer-name @@ -637,7 +645,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): self.netvm.start(start_guid=start_guid, notify_function=notify_function) - self.storage.prepare_for_vm_startup() + self.storage.start() self._update_libvirt_domain() qmemman_client = self.request_memory(mem_required) @@ -729,6 +737,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): exc_info=1) self.libvirt_domain.shutdown() + self.storage.stop() def kill(self): @@ -742,6 +751,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): raise qubes.exc.QubesVMNotStartedError(self) self.libvirt_domain.destroy() + self.storage.stop() def force_shutdown(self, *args, **kwargs): @@ -1025,7 +1035,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): p.communicate(input=self.default_user) - # TODO move to storage + # TODO rename to create def create_on_disk(self, source_template=None): '''Create files needed for VM. @@ -1115,11 +1125,11 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): while self.is_running(): #1696 time.sleep(1) - def remove_from_disk(self): '''Remove domain remnants from disk.''' self.fire_event('domain-remove-from-disk') - self.storage.remove_from_disk() + self.storage.remove() + shutil.rmtree(self.vm.dir_path) def clone_disk_files(self, src): @@ -1454,68 +1464,69 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): # XXX shouldn't this go only to vms that have root image? def get_disk_utilization_root_img(self): - '''Get space that is actually ocuppied by :py:attr:`root_img`. - - Root image is a sparse file, so it is probably much less than logical - available space. + '''Get space that is actually ocuppied by :py:attr:`volumes['root']`. + This is a temporary wrapper for backwards compatibility. You should + call directly :py:attr:`volumes[name].utilization` :returns: domain's real disk image size [FIXME unit] :rtype: FIXME .. seealso:: :py:meth:`get_root_img_sz` ''' - return qubes.storage.get_disk_usage(self.root_img) + warnings.warn( + "get_disk_utilization_root_img is deprecated, use volumes['root'].utilization", + DeprecationWarning) + return qubes.storage.get_disk_usage(self.volumes['root'].utilization) # XXX shouldn't this go only to vms that have root image? def get_root_img_sz(self): - '''Get image size of :py:attr:`root_img`. - - Root image is a sparse file, so it is probably much more than ocuppied - physical space. + '''Get the size of the :py:attr:`volumes['root']`. + This is a temporary wrapper for backwards compatibility. You should + call directly :py:attr:`volumes[name].size` :returns: domain's virtual disk size [FIXME unit] :rtype: FIXME .. seealso:: :py:meth:`get_disk_utilization_root_img` ''' - if not os.path.exists(self.root_img): - return 0 - - return os.path.getsize(self.root_img) - + warnings.warn( + "get_disk_root_img_sz is deprecated, use volumes['root'].size", + DeprecationWarning) + return qubes.storage.get_disk_usage(self.volumes['root'].size) def get_disk_utilization_private_img(self): - '''Get space that is actually ocuppied by :py:attr:`private_img`. - - Private image is a sparse file, so it is probably much less than - logical available space. + '''Get space that is actually ocuppied by :py:attr:`volumes['private']`. + This is a temporary wrapper for backwards compatibility. You should + call directly :py:attr:`volumes[name].utilization` :returns: domain's real disk image size [FIXME unit] :rtype: FIXME + ''' - .. seealso:: :py:meth:`get_private_img_sz` - ''' # pylint: disable=invalid-name - - return qubes.storage.get_disk_usage(self.private_img) - + warnings.warn( + "get_disk_utilization_private_img is deprecated, use volumes['private'].utilization", + DeprecationWarning) + return qubes.storage.get_disk_usage(self.volumes[ + 'private'].utilization) def get_private_img_sz(self): - '''Get image size of :py:attr:`private_img`. - - Private image is a sparse file, so it is probably much more than - ocuppied physical space. + '''Get the size of the :py:attr:`volumes['private']`. + This is a temporary wrapper for backwards compatibility. You should + call directly :py:attr:`volumes[name].size` :returns: domain's virtual disk size [FIXME unit] :rtype: FIXME .. seealso:: :py:meth:`get_disk_utilization_private_img` ''' - return self.storage.get_private_img_sz() - + warnings.warn( + "get_disk_private_img_sz is deprecated, use volumes['private'].size", + DeprecationWarning) + return qubes.storage.get_disk_usage(self.volumes['private'].size) def get_disk_utilization(self): '''Return total space actually occuppied by all files belonging to \ @@ -1527,7 +1538,6 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): return qubes.storage.get_disk_usage(self.dir_path) - # TODO move to storage def verify_files(self): '''Verify that files accessed by this machine are sane. @@ -1751,3 +1761,20 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): # if self.is_qrexec_running(): # #TODO: kill qrexec daemon # pass + + +def dict_merge(dct, merge_dct): + """ Recursive dict merge. Inspired by :meth:``dict.update()``, instead of + updating only top-level keys, dict_merge recurses down into dicts nested + to an arbitrary depth, updating keys. The ``merge_dct`` is merged into + ``dct``. (Source https://gist.github.com/angstwad/bf22d1822c38a92ec0a9) + :param dct: dict onto which the merge is executed + :param merge_dct: dct merged into dct + :return: None + """ + for k, v in merge_dct.iteritems(): + if (k in dct and isinstance(dct[k], dict) + and isinstance(merge_dct[k], dict)): + dict_merge(dct[k], merge_dct[k]) + else: + dct[k] = merge_dct[k] diff --git a/qubes/vm/templatevm.py b/qubes/vm/templatevm.py index 7eadc134..39432ac8 100644 --- a/qubes/vm/templatevm.py +++ b/qubes/vm/templatevm.py @@ -1,13 +1,16 @@ #!/usr/bin/python2 -O # vim: fileencoding=utf-8 +import warnings + import qubes import qubes.config import qubes.vm.qubesvm from qubes.config import defaults +from qubes.vm.qubesvm import QubesVM -class TemplateVM(qubes.vm.qubesvm.QubesVM): +class TemplateVM(QubesVM): '''Template for AppVM''' dir_path_prefix = qubes.config.system_path['qubes_templates_dir'] @@ -15,7 +18,9 @@ class TemplateVM(qubes.vm.qubesvm.QubesVM): @property def rootcow_img(self): '''COW image''' - return self.storage.rootcow_img + warnings.warn("rootcow_img is deprecated, use " + "volumes['root'].path_origin", DeprecationWarning) + return self.volumes['root'].path_cow @property def appvms(self): @@ -67,6 +72,4 @@ class TemplateVM(qubes.vm.qubesvm.QubesVM): assert not self.is_running(), \ 'Attempt to commit changes on running Template VM!' - self.log.info('Commiting template update; COW: {}'.format( - self.rootcow_img)) self.storage.commit_template_changes()