# # The Qubes OS Project, https://www.qubes-os.org/ # # Copyright (C) 2015 Joanna Rutkowska # Copyright (C) 2013-2015 Marek Marczykowski-Górecki # # Copyright (C) 2015 Wojtek Porczyk # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, see . # ''' This module contains pool implementations backed by file images''' import asyncio import os import os.path import re import subprocess from contextlib import suppress import qubes.storage import qubes.utils BLKSIZE = 512 # 256 KiB chunk, same as in block-snapshot script. Header created by # struct.pack('<4I', 0x70416e53, 1, 1, 256) mimicking write_header() # in linux/drivers/md/dm-snap-persistent.c EMPTY_SNAPSHOT = b'SnAp\x01\x00\x00\x00\x01\x00\x00\x00\x00\x01\x00\x00' \ + bytes(262128) class FilePool(qubes.storage.Pool): ''' File based 'original' disk implementation Volumes are stored in sparse files. Additionally device-mapper is used for applying copy-on-write layer. Quick reference on device-mapper layers: snap_on_start save_on_stop layout yes yes not supported no yes snapshot-origin(volume.img, volume-cow.img) yes no snapshot( snapshot(source.img, source-cow.img), volume-cow.img) no no volume.img directly ''' # pylint: disable=protected-access driver = 'file' def __init__(self, *, name, revisions_to_keep=1, dir_path): super().__init__(name=name, revisions_to_keep=revisions_to_keep) self.dir_path = os.path.normpath(dir_path) self._volumes = [] @property def config(self): return { 'name': self.name, 'dir_path': self.dir_path, 'driver': FilePool.driver, 'revisions_to_keep': self.revisions_to_keep } def init_volume(self, vm, volume_config): if volume_config.get('snap_on_start', False) and \ volume_config.get('save_on_stop', False): raise NotImplementedError( 'snap_on_start + save_on_stop not supported by file driver') volume_config['dir_path'] = self.dir_path if 'vid' not in volume_config: volume_config['vid'] = os.path.join( self._vid_prefix(vm), volume_config['name']) try: if not volume_config.get('save_on_stop', False): volume_config['revisions_to_keep'] = 0 except KeyError: pass if 'revisions_to_keep' not in volume_config: volume_config['revisions_to_keep'] = self.revisions_to_keep volume_config['pool'] = self volume = FileVolume(**volume_config) self._volumes += [volume] return volume @property def revisions_to_keep(self): return self._revisions_to_keep @revisions_to_keep.setter def revisions_to_keep(self, value): value = int(value) if value > 1: raise NotImplementedError( 'FilePool supports maximum 1 volume revision to keep') self._revisions_to_keep = value def destroy(self): pass def setup(self): create_dir_if_not_exists(self.dir_path) appvms_path = os.path.join(self.dir_path, 'appvms') create_dir_if_not_exists(appvms_path) vm_templates_path = os.path.join(self.dir_path, 'vm-templates') create_dir_if_not_exists(vm_templates_path) @staticmethod def _vid_prefix(vm): ''' Helper to create a prefix for the vid for volume ''' # FIX Remove this if we drop the file backend import qubes.vm.templatevm # pylint: disable=redefined-outer-name import qubes.vm.dispvm # pylint: disable=redefined-outer-name if isinstance(vm, qubes.vm.templatevm.TemplateVM): subdir = 'vm-templates' else: subdir = 'appvms' return os.path.join(subdir, vm.name) def target_dir(self, vm): """ Returns the path to vmdir depending on the type of the VM. The default QubesOS file storage saves the vm images in three different directories depending on the ``QubesVM`` type: * ``appvms`` for ``QubesAppVm`` or ``QubesHvm`` * ``vm-templates`` for ``QubesTemplateVm`` or ``QubesTemplateHvm`` Args: vm: a QubesVM pool_dir: the root directory of the pool Returns: string (str) absolute path to the directory where the vm files are stored """ return os.path.join(self.dir_path, self._vid_prefix(vm)) def list_volumes(self): return self._volumes @property def size(self): try: statvfs = os.statvfs(self.dir_path) return statvfs.f_frsize * statvfs.f_blocks except FileNotFoundError: return 0 @property def usage(self): try: statvfs = os.statvfs(self.dir_path) return statvfs.f_frsize * (statvfs.f_blocks - statvfs.f_bfree) except FileNotFoundError: return 0 def included_in(self, app): ''' Check if there is pool containing this one - either as a filesystem or its LVM volume''' return qubes.storage.search_pool_containing_dir( [pool for pool in app.pools.values() if pool is not self], self.dir_path) class FileVolume(qubes.storage.Volume): ''' Parent class for the xen volumes implementation which expects a `target_dir` param on initialization. ''' _marker_running = object() _marker_exported = object() def __init__(self, dir_path, **kwargs): self.dir_path = dir_path assert self.dir_path, "dir_path not specified" self._revisions_to_keep = 0 self._export_lock = None super().__init__(**kwargs) if self.snap_on_start: img_name = self.source.vid + '-cow.img' self.path_source_cow = os.path.join(self.dir_path, img_name) @property def revisions_to_keep(self): return self._revisions_to_keep @revisions_to_keep.setter def revisions_to_keep(self, value): if int(value) > 1: raise NotImplementedError( 'FileVolume supports maximum 1 volume revision to keep') self._revisions_to_keep = int(value) def create(self): assert isinstance(self.size, int) and self.size > 0, \ 'Volume size must be > 0' if not self.snap_on_start: create_sparse_file(self.path, self.size) def remove(self): if not self.snap_on_start: _remove_if_exists(self.path) if self.snap_on_start or self.save_on_stop: _remove_if_exists(self.path_cow) def is_outdated(self): return False # avoid spamming the log with NotImplementedError def is_dirty(self): if self.save_on_stop: with suppress(FileNotFoundError), open(self.path_cow, 'rb') as cow: cow_used = os.fstat(cow.fileno()).st_blocks * BLKSIZE return (cow_used > 0 and (cow_used > len(EMPTY_SNAPSHOT) or cow.read(len(EMPTY_SNAPSHOT)) != EMPTY_SNAPSHOT or cow_used > cow.seek(0, os.SEEK_HOLE))) return False def resize(self, size): ''' Expands volume, throws :py:class:`qubst.storage.qubes.storage.StoragePoolException` if given size is less than current_size ''' # pylint: disable=no-self-use if not self.rw: msg = 'Can not resize reađonly volume {!s}'.format(self) raise qubes.storage.StoragePoolException(msg) if size < self.size: raise qubes.storage.StoragePoolException( 'For your own safety, shrinking of %s is' ' disabled. If you really know what you' ' are doing, use `truncate` on %s manually.' % (self.name, self.vid)) with open(self.path, 'a+b') as fd: fd.truncate(size) p = subprocess.Popen(['losetup', '--associated', self.path], stdout=subprocess.PIPE) result = p.communicate() m = re.match(r'^(/dev/loop\d+):\s', result[0].decode()) if m is not None: loop_dev = m.group(1) # resize loop device subprocess.check_call(['losetup', '--set-capacity', loop_dev]) self._size = size def commit(self): msg = 'Tried to commit a non commitable volume {!r}'.format(self) assert self.save_on_stop and self.rw, msg if os.path.exists(self.path_cow): if self.revisions_to_keep: old_path = self.path_cow + '.old' os.rename(self.path_cow, old_path) else: os.unlink(self.path_cow) create_sparse_file(self.path_cow, self.size) return self def export(self): if self._export_lock is not None: assert self._export_lock is FileVolume._marker_running, \ 'nested calls to export()' raise qubes.storage.StoragePoolException( 'file pool cannot export running volumes') if self.is_dirty(): raise qubes.storage.StoragePoolException( 'file pool cannot export dirty volumes') self._export_lock = FileVolume._marker_exported return self.path def export_end(self, path): assert self._export_lock is not FileVolume._marker_running, \ 'ending an export on a running volume?' self._export_lock = None @asyncio.coroutine def import_volume(self, src_volume): if src_volume.snap_on_start: raise qubes.storage.StoragePoolException( "Can not import snapshot volume {!s} in to pool {!s} ".format( src_volume, self)) if self.save_on_stop: _remove_if_exists(self.path) path = yield from qubes.utils.coro_maybe(src_volume.export()) try: copy_file(path, self.path) finally: yield from qubes.utils.coro_maybe(src_volume.export_end(path)) return self def import_data(self, size): if not self.save_on_stop: raise qubes.storage.StoragePoolException( "Can not import into save_on_stop=False volume {!s}".format( self)) create_sparse_file(self.path_import, size) return self.path_import def import_data_end(self, success): if success: os.rename(self.path_import, self.path) else: os.unlink(self.path_import) return self def reset(self): ''' Remove and recreate a volatile volume ''' assert not self.snap_on_start and not self.save_on_stop, \ "Not a volatile volume" assert isinstance(self.size, int) and self.size > 0, \ 'Volatile volume size must be > 0' _remove_if_exists(self.path) create_sparse_file(self.path, self.size) return self def start(self): if self._export_lock is not None: assert self._export_lock is FileVolume._marker_exported, \ 'nested calls to start()' raise qubes.storage.StoragePoolException( 'file pool cannot start a VM with an exported volume') self._export_lock = FileVolume._marker_running if not self.save_on_stop and not self.snap_on_start: self.reset() else: if not self.save_on_stop: # make sure previous snapshot is removed - even if VM # shutdown routine wasn't called (power interrupt or so) _remove_if_exists(self.path_cow) if not os.path.exists(self.path_cow): create_sparse_file(self.path_cow, self.size) if not self.snap_on_start: _check_path(self.path) if hasattr(self, 'path_source_cow'): if not os.path.exists(self.path_source_cow): create_sparse_file(self.path_source_cow, self.size) return self def stop(self): assert self._export_lock is not FileVolume._marker_exported, \ 'trying to stop exported file volume?' if self.save_on_stop: self.commit() elif self.snap_on_start: _remove_if_exists(self.path_cow) else: _remove_if_exists(self.path) self._export_lock = None return self @property def path(self): if self.snap_on_start: return os.path.join(self.dir_path, self.source.vid + '.img') return os.path.join(self.dir_path, self.vid + '.img') @property def path_cow(self): img_name = self.vid + '-cow.img' return os.path.join(self.dir_path, img_name) @property def path_import(self): img_name = self.vid + '-import.img' return os.path.join(self.dir_path, img_name) def verify(self): ''' Verifies the volume. ''' if not os.path.exists(self.path) and \ (self.snap_on_start or self.save_on_stop): msg = 'Missing image file: {!s}.'.format(self.path) raise qubes.storage.StoragePoolException(msg) return True @property def script(self): if not self.snap_on_start and not self.save_on_stop: return None if not self.snap_on_start and self.save_on_stop: return 'block-origin' if self.snap_on_start: return 'block-snapshot' return None def block_device(self): ''' Return :py:class:`qubes.storage.BlockDevice` for serialization in the libvirt XML template as . ''' path = self.path if self.snap_on_start: path += ":" + self.path_source_cow if self.snap_on_start or self.save_on_stop: path += ":" + self.path_cow return qubes.storage.BlockDevice(path, self.name, self.script, self.rw, self.domain, self.devtype) @property def revisions(self): if not hasattr(self, 'path_cow'): return {} old_revision = self.path_cow + '.old' # pylint: disable=no-member if not os.path.exists(old_revision): return {} seconds = os.path.getctime(old_revision) iso_date = qubes.storage.isodate(seconds).split('.', 1)[0] return {'old': iso_date} @property def size(self): with suppress(FileNotFoundError): self._size = os.path.getsize(self.path) return self._size @size.setter def size(self, _): raise qubes.storage.StoragePoolException( "You shouldn't use volume size setter, use resize method instead") @property def usage(self): ''' Returns the actualy used space ''' usage = 0 if self.save_on_stop or self.snap_on_start: usage = get_disk_usage(self.path_cow) if self.save_on_stop or not self.snap_on_start: usage += get_disk_usage(self.path) return usage def create_sparse_file(path, size): ''' Create an empty sparse file ''' if os.path.exists(path): raise IOError("Volume %s already exists" % path) parent_dir = os.path.dirname(path) if not os.path.exists(parent_dir): os.makedirs(parent_dir) with open(path, 'a+b') as fh: fh.truncate(size) def get_disk_usage_one(st): '''Extract disk usage of one inode from its stat_result struct. If known, get real disk usage, as written to device by filesystem, not logical file size. Those values may be different for sparse files. :param os.stat_result st: stat result :returns: disk usage ''' try: return st.st_blocks * BLKSIZE except AttributeError: return st.st_size def get_disk_usage(path): '''Get real disk usage of given path (file or directory). When *path* points to directory, then it is evaluated recursively. This function tries estimate real disk usage. See documentation of :py:func:`get_disk_usage_one`. :param str path: path to evaluate :returns: disk usage ''' try: st = os.lstat(path) except OSError: return 0 ret = get_disk_usage_one(st) # if path is not a directory, this is skipped for dirpath, dirnames, filenames in os.walk(path): for name in dirnames + filenames: ret += get_disk_usage_one(os.lstat(os.path.join(dirpath, name))) return ret def create_dir_if_not_exists(path): """ Check if a directory exists in if not create it. This method does not create any parent directories. """ if not os.path.exists(path): os.mkdir(path) def copy_file(source, destination): '''Effective file copy, preserving sparse files etc.''' # We prefer to use Linux's cp, because it nicely handles sparse files assert os.path.exists(source), \ "Missing the source %s to copy from" % source assert not os.path.exists(destination), \ "Destination %s already exists" % destination parent_dir = os.path.dirname(destination) if not os.path.exists(parent_dir): os.makedirs(parent_dir) try: cmd = ['cp', '--sparse=always', '--reflink=auto', source, destination] subprocess.check_call(cmd) except subprocess.CalledProcessError: raise IOError('Error while copying {!r} to {!r}'.format(source, destination)) def _remove_if_exists(path): ''' Removes a file if it exist, silently succeeds if file does not exist ''' if os.path.exists(path): os.remove(path) def _check_path(path): ''' Raise an StoragePoolException if ``path`` does not exist''' if not os.path.exists(path): msg = 'Missing image file: %s' % path raise qubes.storage.StoragePoolException(msg)