diff --git a/Makefile b/Makefile index 92a4aac6..be880645 100644 --- a/Makefile +++ b/Makefile @@ -117,11 +117,13 @@ DATADIR ?= /var/lib/qubes STATEDIR ?= /var/run/qubes LOGDIR ?= /var/log/qubes FILESDIR ?= /usr/share/qubes +DOCDIR ?= /usr/share/doc/qubes else ifeq ($(OS),Windows_NT) DATADIR ?= c:/qubes STATEDIR ?= c:/qubes/state LOGDIR ?= c:/qubes/log FILESDIR ?= c:/program files/Invisible Things Lab/Qubes +DOCDIR ?= c:/qubes/doc endif help: @@ -214,6 +216,9 @@ endif cp -r templates "$(DESTDIR)$(FILESDIR)/templates" rm -f "$(DESTDIR)$(FILESDIR)/templates/README" + mkdir -p "$(DESTDIR)$(DOCDIR)" + cp qubes/storage/callback.json.example "$(DESTDIR)$(DOCDIR)/qubes_callback.json.example" + mkdir -p $(DESTDIR)$(DATADIR) mkdir -p $(DESTDIR)$(DATADIR)/vm-templates mkdir -p $(DESTDIR)$(DATADIR)/appvms diff --git a/qubes/storage/callback.json.example b/qubes/storage/callback.json.example new file mode 100644 index 00000000..6f56c808 --- /dev/null +++ b/qubes/storage/callback.json.example @@ -0,0 +1,99 @@ +{ +"conf_id": + { + "bdriver": "Backend pool driver to proxy. This can be a class name or a full module path to a class implementing `qubes.storage.Pool`.", + "bdriver_args": { + "arg name 1": "arg value 1", + "arg name 2": "arg value 2" + }, + "cmd": "Default command to call when the [pre|post]_[op] operations are not specified (default: None). The command is called as such: `[cmd] [name] [bdriver] [operation] [ctor params]`. [name]: name of the pool, [operation]: any of the `[pre|post]_` operations from below including its arguments, [bdriver]: backend driver of the pool, [ctor params]: Parameters passed to the `bdriver` constructor in JSON format. Each parameter is on a single line for easy parsing.", + "signal_back": "Boolean (true|false) to allow the executed commands to send signals back to the callback driver (default: false). Signals must be on a dedicated line on stdout. Currently only `SIGNAL_setup` is supported. When found, it causes the callback driver to re-setup the backend pool.", + "pre_sinit": "Command to call before one-time storage initialization/first usage (default: None). Called exactly once for every `qubesd` start. Can be used to override `cmd`. Pass `-` to ignore this callback entirely even if `cmd` is specified.", + "pre_setup": "Called before creating a new pool. Can be used to override `cmd`. Pass `-` to ignore this callback entirely even if `cmd` is specified.", + "post_destroy": "Called after removing an existing pool. Same as above otherwise.", + "pre_volume_create": "Called before creating a volume for the pool. Same as above otherwise.", + "post_volume_create": "Called after creating a volume for the pool. Same as above otherwise.", + "post_volume_remove": "Called after removing a volume of the pool. Same as above otherwise.", + "pre_volume_resize": "Called before resizing a volume of the pool. Same as above otherwise.", + "pre_volume_start": "Called before starting a volume of the pool. Same as above otherwise.", + "post_volume_start": "Called after starting a volume of the pool. Same as above otherwise.", + "post_volume_stop": "Called after stopping a volume of the pool. Same as above otherwise.", + "pre_volume_import": "Called before importing a volume from another. Same as above otherwise.", + "post_volume_import": "Called after importing a volume from another. Same as above otherwise.", + "pre_volume_import_data": "Called before overwriting this volume with new data. Same as above otherwise.", + "post_volume_import_data_end": "Called after finishing an `import_data` action. Same as above otherwise.", + "pre_volume_export": "Called before exporting a volume. Same as above otherwise.", + "post_volume_export_end": "Called after a volume export completed. Same as above otherwise.", + "description": "Optional description for your personal reference." + }, +"testing-fail-missing-all": + { + "description": "For testing purposes only." + }, +"testing-fail-missing-bdriver-args": + { + "bdriver": "file", + "description": "For testing purposes only." + }, +"testing-succ-file-01": + { + "bdriver": "file", + "bdriver_args": { + "dir_path": "/mnt/test01", + "revisions_to_keep": 0 + }, + "description": "For testing purposes only." + }, +"testing-succ-file-02": + { + "bdriver": "file", + "bdriver_args": { + "dir_path": "/mnt/test02" + }, + "cmd": "testCbLogArgs", + "description": "For testing purposes only." + }, +"testing-succ-file-03": + { + "bdriver": "file", + "bdriver_args": { + "dir_path": "/mnt/test03" + }, + "cmd": "exit 1", + "pre_sinit": "testCbLogArgs pre_sinit", + "pre_setup": "testCbLogArgs pre_setup", + "post_destroy": "-", + "description": "For testing purposes only." + }, +"testing-succ-file-luks": + { + "bdriver": "file", + "bdriver_args": { + "dir_path": "/mnt/test_luks" + }, + "pre_setup": "set -e -o pipefail ; [ ! -e /mnt/test.key ] ; [ ! -e /mnt/test.luks ] ; dd if=/dev/random bs=100 of=/mnt/test.key iflag=fullblock count=1 ; dd if=/dev/urandom bs=1M of=/mnt/test.luks iflag=fullblock count=2048 ; cryptsetup luksFormat -q --key-file /mnt/test.key /mnt/test.luks ; cryptsetup open -q --key-file /mnt/test.key /mnt/test.luks test-luks ; mkfs -t ext4 /dev/mapper/test-luks ; mkdir -p /mnt/test_luks ; mount /dev/mapper/test-luks /mnt/test_luks", + "pre_sinit": "set -e -o pipefail ; if [[ \"$(findmnt -S /dev/mapper/test-luks -n -o TARGET)\" != \"/mnt/test_luks\" ]] ; then cryptsetup status test-luks > /dev/null || cryptsetup open -q --key-file /mnt/test.key /mnt/test.luks test-luks ; mount /dev/mapper/test-luks /mnt/test_luks ; else exit 0 ; fi", + "post_destroy": "umount /mnt/test_luks && cryptsetup close test-luks ; set -e -o pipefail ; dd if=/dev/urandom bs=100 of=/mnt/test.key iflag=fullblock count=1 ; rm -f /mnt/test.key ; rm -f /mnt/test.luks ; rmdir /mnt/test_luks", + "pre_volume_create": "[[ \"$(findmnt -S /dev/mapper/test-luks -n -o TARGET)\" == \"/mnt/test_luks\" ]]", + "pre_volume_import": "[[ \"$(findmnt -S /dev/mapper/test-luks -n -o TARGET)\" == \"/mnt/test_luks\" ]]", + "pre_volume_import_data": "[[ \"$(findmnt -S /dev/mapper/test-luks -n -o TARGET)\" == \"/mnt/test_luks\" ]]", + "post_volume_import_data_end": "[[ \"$(findmnt -S /dev/mapper/test-luks -n -o TARGET)\" == \"/mnt/test_luks\" ]]", + "description": "For testing purposes only: Showcasing seemless use of encrypted VM pools. For personal use, a dedicated script is recommended." + }, +"testing-succ-file-luks-eph": + { + "bdriver": "file", + "bdriver_args": { + "dir_path": "/mnt/test_eph" + }, + "signal_back": true, + "pre_setup": "set -e -o pipefail ; if [[ \"$(findmnt -T /mnt/ram -o SOURCE -n)\" != \"ramfs\" ]] ; then mkdir -p /mnt/ram ; mount -t ramfs ramfs /mnt/ram ; fi ; if [[ \"$(findmnt -S /dev/mapper/test-eph -n -o TARGET)\" != \"/mnt/test_eph\" ]] ; then if [ -f /mnt/ram/teph.key ] ; then cryptsetup status test-eph > /dev/null || cryptsetup open -q --key-file /mnt/ram/teph.key /mnt/teph.luks test-eph ; else dd if=/dev/random bs=100 of=/mnt/ram/teph.key iflag=fullblock count=1 ; if [ ! -f /mnt/teph.luks ] ; then dd if=/dev/urandom bs=1M of=/mnt/teph.luks iflag=fullblock count=2048 ; fi ; cryptsetup luksFormat -q --key-file /mnt/ram/teph.key /mnt/teph.luks ;cryptsetup open -q --key-file /mnt/ram/teph.key /mnt/teph.luks test-eph ; mkfs -t ext4 /dev/mapper/test-eph ; echo SIGNAL_setup ; fi ; mkdir -p /mnt/test_eph ; mount /dev/mapper/test-eph /mnt/test_eph ; fi ; exit 0", + "pre_sinit": "set -e -o pipefail ; if [[ \"$(findmnt -T /mnt/ram -o SOURCE -n)\" != \"ramfs\" ]] ; then mkdir -p /mnt/ram ; mount -t ramfs ramfs /mnt/ram ; fi ; if [[ \"$(findmnt -S /dev/mapper/test-eph -n -o TARGET)\" != \"/mnt/test_eph\" ]] ; then if [ -f /mnt/ram/teph.key ] ; then cryptsetup status test-eph > /dev/null || cryptsetup open -q --key-file /mnt/ram/teph.key /mnt/teph.luks test-eph ; else dd if=/dev/random bs=100 of=/mnt/ram/teph.key iflag=fullblock count=1 ; if [ ! -f /mnt/teph.luks ] ; then dd if=/dev/urandom bs=1M of=/mnt/teph.luks iflag=fullblock count=2048 ; fi ; cryptsetup luksFormat -q --key-file /mnt/ram/teph.key /mnt/teph.luks ;cryptsetup open -q --key-file /mnt/ram/teph.key /mnt/teph.luks test-eph ; mkfs -t ext4 /dev/mapper/test-eph ; echo SIGNAL_setup ; fi ; mkdir -p /mnt/test_eph ; mount /dev/mapper/test-eph /mnt/test_eph ; fi ; exit 0", + "post_destroy": "umount /mnt/test_eph && cryptsetup close test-eph ; set -e -o pipefail ; if [ -f /mnt/ram/teph.key ] ; then dd if=/dev/urandom bs=100 of=/mnt/ram/teph.key iflag=fullblock count=1 ; rm -f /mnt/ram/teph.key ; fi ; rm -f /mnt/teph.luks ; rmdir /mnt/test_eph", + "pre_volume_create": "[[ \"$(findmnt -S /dev/mapper/test-eph -n -o TARGET)\" == \"/mnt/test_eph\" ]]", + "pre_volume_import": "[[ \"$(findmnt -S /dev/mapper/test-eph -n -o TARGET)\" == \"/mnt/test_eph\" ]]", + "pre_volume_import_data": "[[ \"$(findmnt -S /dev/mapper/test-eph -n -o TARGET)\" == \"/mnt/test_eph\" ]]", + "post_volume_import_data_end": "[[ \"$(findmnt -S /dev/mapper/test-eph -n -o TARGET)\" == \"/mnt/test_eph\" ]]", + "description": "For testing purposes only: Showcasing seemless use of encrypted disposable (upon boot) VM pools with keys held in RAM only. For personal use, a dedicated script is recommended." + } +} diff --git a/qubes/storage/callback.py b/qubes/storage/callback.py new file mode 100644 index 00000000..84e49570 --- /dev/null +++ b/qubes/storage/callback.py @@ -0,0 +1,622 @@ +# +# The Qubes OS Project, https://www.qubes-os.org/ +# +# Copyright (C) 2020 David Hobach +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, see . +# + +# pylint: disable=line-too-long + +import logging +import subprocess +import json +import asyncio +import locale +from shlex import quote +from qubes.utils import coro_maybe + +import qubes.storage + +class UnhandledSignalException(qubes.storage.StoragePoolException): + def __init__(self, pool, signal): + super().__init__('The pool %s failed to handle the signal %s, likely because it was run from synchronous code.' % (pool.name, signal)) + +class CallbackPool(qubes.storage.Pool): + ''' Proxy storage pool driver adding callback functionality to other pool drivers. + + This way, users can extend storage pool drivers with custom functionality using the programming language of their choice. + + All configuration for this pool driver must be done in `/etc/qubes_callback.json`. Each configuration ID `conf_id` can be used + to create a callback pool with e.g. `qvm-pool -o conf_id=your_conf_id -a pool_name callback`. + Check `/usr/share/doc/qubes/qubes_callback.json.example` for an overview of the available options. + + Example applications of this driver: + - custom pool mounts + - encryption + - debugging + + + **Integration tests**: + (all of these tests assume the `qubes_callback.json.example` configuration) + + Tests that should **fail**: + ``` + qvm-pool -a test callback + qvm-pool -o conf_id=non-existing -a test callback + qvm-pool -o conf_id=conf_id -a test callback + qvm-pool -o conf_id=testing-fail-missing-all -a test callback + qvm-pool -o conf_id=testing-fail-missing-bdriver-args -a test callback + ``` + + Tests that should **work**: + ``` + qvm-pool -o conf_id=testing-succ-file-01 -a test callback + qvm-pool + ls /mnt/test01 + qvm-pool -r test && sudo rm -rf /mnt/test01 + + echo '#!/bin/bash'$'\n''i=1 ; for arg in "$@" ; do echo "$i: $arg" >> /tmp/callback.log ; (( i++)) ; done ; exit 0' > /usr/bin/testCbLogArgs && chmod +x /usr/bin/testCbLogArgs + rm -f /tmp/callback.log + qvm-pool -o conf_id=testing-succ-file-02 -a test callback + qvm-pool + ls /mnt/test02 + less /tmp/callback.log (pre_setup should be there) + qvm-create -l red -P test test-vm + cat /tmp/callback.log (2x pre_volume_create + 2x post_volume_create should be added) + qvm-start test-vm + qvm-volume | grep test-vm + grep test-vm /var/lib/qubes/qubes.xml + ls /mnt/test02/appvms/ + cat /tmp/callback.log (2x pre_volume_start & 2x post_volume_start should be added) + qvm-shutdown test-vm + cat /tmp/callback.log (2x post_volume_stop should be added) + #reboot + cat /tmp/callback.log (it should not exist) + qvm-start test-vm + cat /tmp/callback.log (pre_sinit & 2x pre_volume_start & 2x post_volume_start should be added) + qvm-shutdown --wait test-vm && qvm-remove test-vm + qvm-pool -r test && sudo rm -rf /mnt/test02 + less /tmp/callback.log (2x post_volume_stop, 2x post_volume_remove, post_destroy should be added) + + qvm-pool -o conf_id=testing-succ-file-02 -a test callback + qvm-create -l red -P test test-dvm + qvm-prefs test-dvm template_for_dispvms True + qvm-run --dispvm test-dvm xterm + grep -E 'test-dvm|disp' /var/lib/qubes/qubes.xml + qvm-volume | grep -E 'test-dvm|disp' (unexpected by most users: Qubes OS places only the private volume on the pool, cf. #5933) + ls /mnt/test02/appvms/ + cat /tmp/callback.log + #close the disposable VM + qvm-remove test-dvm + qvm-pool -r test && sudo rm -rf /mnt/test02 + + qvm-pool -o conf_id=testing-succ-file-03 -a test callback + qvm-pool + ls /mnt/test03 + less /tmp/callback.log (pre_setup should be there, no more arguments) + qvm-pool -r test && sudo rm -rf /mnt/test03 + less /tmp/callback.log (nothing should have been added) + + #luks pool test: + #(make sure /mnt/test.key & /mnt/test.luks don't exist) + qvm-pool -o conf_id=testing-succ-file-luks -a tluks callback + ls /mnt/ + qvm-pool + sudo cryptsetup status test-luks + sudo mount | grep test_luks + ls /mnt/test_luks/ + qvm-create -l red -P tluks test-luks (journalctl -b0 should show two pre_volume_create callbacks) + ls /mnt/test_luks/appvms/test-luks/ + qvm-volume | grep test-luks + qvm-start test-luks + #reboot + grep luks /var/lib/qubes/qubes.xml + sudo cryptsetup status test-luks (should be inactive due to late pre_sinit!) + qvm-start test-luks + sudo mount | grep test_luks + qvm-shutdown --wait test-luks + qvm-remove test-luks + qvm-pool -r tluks + sudo cryptsetup status test-luks + ls -l /mnt/ + + #ephemeral luks pool test (key in RAM / lost on reboot): + qvm-pool -o conf_id=testing-succ-file-luks-eph -a teph callback (executes setup() twice due to signal_back) + ls /mnt/ + ls /mnt/ram + md5sum /mnt/ram/teph.key (1) + sudo mount|grep -E 'ram|test' + sudo cryptsetup status test-eph + qvm-create -l red -P teph test-eph (should execute two pre_volume_create callbacks) + qvm-volume | grep test-eph + ls /mnt/test_eph/appvms/test-eph/ (should have private.img and volatile.img) + ls /var/lib/qubes/appvms/test-eph (should only have the icon) + qvm-start test-eph + #reboot + ls /mnt/ram (should be empty) + ls /mnt/ + sudo mount|grep -E 'ram|test' (should be empty) + qvm-ls | grep eph (should still have test-eph) + grep eph /var/lib/qubes/qubes.xml (should still have test-eph) + qvm-remove test-eph (should create a new encrypted pool backend) + sudo cryptsetup status test-eph + grep eph /var/lib/qubes/qubes.xml (only the pool should be left) + ls /mnt/test_eph/ (should have the appvms directory etc.) + qvm-create -l red -P teph test-eph2 + ls /mnt/test_eph/appvms/ + ls /mnt/ram + qvm-start test-eph2 + md5sum /mnt/ram/teph.key ((2), different than in (1)) + qvm-shutdown --wait test-eph2 + systemctl restart qubesd + qvm-start test-eph2 (trigger storage re-init) + md5sum /mnt/ram/teph.key (same as in (2)) + qvm-shutdown --wait test-eph2 + sudo umount /mnt/test_eph + qvm-create -l red -P teph test-eph-fail (must fail with error in journalctl) + ls /mnt/test_eph/ (should be empty) + systemctl restart qubesd + qvm-remove test-eph2 + qvm-create -l red -P teph test-eph3 + md5sum /mnt/ram/teph.key (same as in (2)) + sudo mount|grep -E 'ram|test' + ls /mnt/test_eph/appvms/test-eph3 + qvm-remove test-eph3 + qvm-ls | grep test-eph + qvm-pool -r teph + grep eph /var/lib/qubes/qubes.xml (nothing should be left) + qvm-pool + ls /mnt/ + ls /mnt/ram/ (should be empty) + ``` + ''' + + def __init__(self, *, name, conf_id): + '''Constructor. + :param conf_id: Identifier as found inside the user-controlled configuration at `/etc/qubes_callback.json`. + Non-ASCII, non-alphanumeric characters may be disallowed. + **Security Note**: Depending on your RPC policy (admin.pool.Add) this constructor and its parameters + may be called from an untrusted VM (not by default though). In those cases it may be security-relevant + not to pick easily guessable `conf_id` values for your configuration as untrusted VMs may otherwise + execute callbacks meant for other pools. + :raise StoragePoolException: For user configuration issues. + ''' + #NOTE: attribute names **must** start with `_cb_` unless they are meant to be stored as self._cb_impl attributes + self._cb_ctor_done = False #: Boolean to indicate whether or not `__init__` successfully ran through. + self._cb_log = logging.getLogger('qubes.storage.callback') #: Logger instance. + if not isinstance(conf_id, str): + raise qubes.storage.StoragePoolException('conf_id is no String. VM attack?!') + self._cb_conf_id = conf_id #: Configuration ID as passed to `__init__()`. + + config_path = '/etc/qubes_callback.json' + with open(config_path) as json_file: + conf_all = json.load(json_file) + if not isinstance(conf_all, dict): + raise qubes.storage.StoragePoolException('The file %s is supposed to define a dict.' % config_path) + + try: + self._cb_conf = conf_all[self._cb_conf_id] #: Dictionary holding all configuration for the given _cb_conf_id. + except KeyError: + #we cannot throw KeyErrors as we'll otherwise generate incorrect error messages @qubes.app._get_pool() + raise qubes.storage.StoragePoolException('The specified conf_id %s could not be found inside %s.' % (self._cb_conf_id, config_path)) + + try: + bdriver = self._cb_conf['bdriver'] + except KeyError: + raise qubes.storage.StoragePoolException('Missing bdriver for the conf_id %s inside %s.' % (self._cb_conf_id, config_path)) + + self._cb_cmd_arg = json.dumps(self._cb_conf, sort_keys=True, indent=2) #: Full configuration as string in the format required by _callback(). + + try: + cls = qubes.utils.get_entry_point_one(qubes.storage.STORAGE_ENTRY_POINT, bdriver) + except KeyError: + raise qubes.storage.StoragePoolException('The driver %s was not found on your system.' % bdriver) + + if not issubclass(cls, qubes.storage.Pool): + raise qubes.storage.StoragePoolException('The class %s must be a subclass of qubes.storage.Pool.' % cls) + + self._cb_requires_init = self._check_init() #: Boolean indicating whether late storage initialization yet has to be done or not. + self._cb_init_lock = asyncio.Lock() #: Lock ensuring that late storage initialization is only run exactly once. + bdriver_args = self._cb_conf.get('bdriver_args', {}) + self._cb_impl = cls(name=name, **bdriver_args) #: Instance of the backend pool driver. + + super().__init__(name=name, revisions_to_keep=int(bdriver_args.get('revisions_to_keep', 1))) + self._cb_ctor_done = True + + def _check_init(self): + ''' Whether or not this object requires late storage initialization via callback. ''' + cmd = self._cb_conf.get('pre_sinit') + if not cmd: + cmd = self._cb_conf.get('cmd') + return bool(cmd and cmd != '-') + + @asyncio.coroutine + def _init(self, callback=True): + ''' Late storage initialization on first use for e.g. decryption on first usage request. + :param callback: Whether to trigger the `pre_sinit` callback or not. + ''' + with (yield from self._cb_init_lock): + if self._cb_requires_init: + if callback: + yield from self._callback('pre_sinit') + self._cb_requires_init = False + + @asyncio.coroutine + def _assert_initialized(self, **kwargs): + if self._cb_requires_init: + yield from self._init(**kwargs) + + @asyncio.coroutine + def _callback(self, cb, cb_args=None): + '''Run a callback. + :param cb: Callback identifier string. + :param cb_args: Optional list of arguments to pass to the command as last arguments. + Only passed on for the generic command specified as `cmd`, not for `on_xyz` callbacks. + :return: Nothing. + ''' + if self._cb_ctor_done: + cmd = self._cb_conf.get(cb) + args = [] #on_xyz callbacks should never receive arguments + if not cmd: + if cb_args is None: + cb_args = [] + cmd = self._cb_conf.get('cmd') + args = [self.name, self._cb_conf['bdriver'], cb, self._cb_conf_id, self._cb_cmd_arg, *cb_args] + if cmd and cmd != '-': + args = ' '.join(quote(str(a)) for a in args) + cmd = ' '.join(filter(None, [cmd, args])) + self._cb_log.info('callback driver executing (%s, %s %s): %s', self._cb_conf_id, cb, cb_args, cmd) + cmd_arr = ['/bin/bash', '-c', cmd] + proc = yield from asyncio.create_subprocess_exec(*cmd_arr, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = yield from proc.communicate() + encoding = locale.getpreferredencoding() + stdout = stdout.decode(encoding) + stderr = stderr.decode(encoding) + if proc.returncode != 0: + raise subprocess.CalledProcessError(returncode=proc.returncode, cmd=cmd, output=stdout, stderr=stderr) + self._cb_log.debug('callback driver stdout (%s, %s %s): %s', self._cb_conf_id, cb, cb_args, stdout) + self._cb_log.debug('callback driver stderr (%s, %s %s): %s', self._cb_conf_id, cb, cb_args, stderr) + if self._cb_conf.get('signal_back', False) is True: + yield from self._process_signals(stdout) + + @asyncio.coroutine + def _process_signals(self, out): + '''Process any signals found inside a string. + :param out: String to check for signals. Each signal must be on a dedicated line. + They are executed in the order they are found. Callbacks are not triggered. + ''' + for line in out.splitlines(): + if line == 'SIGNAL_setup': + self._cb_log.info('callback driver processing SIGNAL_setup for %s', self._cb_conf_id) + #NOTE: calling our own methods may lead to a deadlock / qubesd freeze due to `self._assert_initialized()` / `self._cb_init_lock` + yield from coro_maybe(self._cb_impl.setup()) + + @property + def backend_class(self): + '''Class of the first non-CallbackPool backend Pool.''' + if isinstance(self._cb_impl, CallbackPool): + return self._cb_impl.backend_class + return self._cb_impl.__class__ + + @property + def config(self): + return { + 'name': self.name, + 'driver': 'callback', + 'conf_id': self._cb_conf_id, + } + + @asyncio.coroutine + def destroy(self): + yield from self._assert_initialized() + ret = yield from coro_maybe(self._cb_impl.destroy()) + yield from self._callback('post_destroy') + return ret + + def init_volume(self, vm, volume_config): + ret = CallbackVolume(self, self._cb_impl.init_volume(vm, volume_config)) + volume_config['pool'] = self + return ret + + @asyncio.coroutine + def setup(self): + yield from self._assert_initialized(callback=False) #setup is assumed to include storage initialization + yield from self._callback('pre_setup') + return (yield from coro_maybe(self._cb_impl.setup())) + + @property + def volumes(self): + for vol in self._cb_impl.volumes: + yield CallbackVolume(self, vol) + + def list_volumes(self): + for vol in self._cb_impl.list_volumes(): + yield CallbackVolume(self, vol) + + def get_volume(self, vid): + return CallbackVolume(self, self._cb_impl.get_volume(vid)) + + def included_in(self, app): + if self._cb_requires_init: + return None + return self._cb_impl.included_in(app) + + @property + def size(self): + if self._cb_requires_init: + return None + return self._cb_impl.size + + @property + def usage(self): + if self._cb_requires_init: + return None + return self._cb_impl.usage + + @property + def usage_details(self): + if self._cb_requires_init: + return {} + return self._cb_impl.usage_details + + #shadow all qubes.storage.Pool class attributes as instance properties + #NOTE: this will cause a subtle difference to using an actual _cb_impl instance: CallbackPool.private_img_size will return a property object, Pool.private_img_size the actual value + @property + def private_img_size(self): + return self._cb_impl.private_img_size + + @private_img_size.setter + def private_img_size(self, private_img_size): + self._cb_impl.private_img_size = private_img_size + + @property + def root_img_size(self): + return self._cb_impl.root_img_size + + @root_img_size.setter + def root_img_size(self, root_img_size): + self._cb_impl.root_img_size = root_img_size + + #remaining method & attribute delegation ("delegation pattern") + #Convention: The methods of this object have priority over the delegated object's methods. All attributes are + # passed to the delegated object unless their name starts with '_cb_'. + def __getattr__(self, name): + #NOTE: This method is only called when an attribute cannot be resolved locally (not part of the instance, + # not part of the class tree). It is also called for methods that cannot be resolved. + return getattr(self._cb_impl, name) + + def __setattr__(self, name, value): + #NOTE: This method is called on every attribute assignment. + if name.startswith('_cb_'): + super().__setattr__(name, value) + else: + setattr(self._cb_impl, name, value) + + def __delattr__(self, name): + if name.startswith('_cb_'): + super().__delattr__(name) + else: + delattr(self._cb_impl, name) + +class CallbackVolume(qubes.storage.Volume): + ''' Proxy volume adding callback functionality to other volumes. + + Required to support the `pre_sinit` and other callbacks. + ''' + + def __init__(self, pool, impl): + '''Constructor. + :param pool: `CallbackPool` of this volume + :param impl: `qubes.storage.Volume` object to wrap + ''' + # pylint: disable=super-init-not-called + #NOTE: we must *not* call super().__init__() as it would prevent attribute delegation + assert isinstance(impl, qubes.storage.Volume), 'impl must be a qubes.storage.Volume instance. Found a %s instance.' % impl.__class__ + assert isinstance(pool, CallbackPool), 'pool must use a qubes.storage.CallbackPool instance. Found a %s instance.' % pool.__class__ + impl.pool = pool #enforce the CallbackPool instance as the parent pool of the volume + self._cb_pool = pool #: CallbackPool instance the Volume belongs to. + self._cb_impl = impl #: Backend volume implementation instance. + + @asyncio.coroutine + def _assert_initialized(self, **kwargs): + yield from self._cb_pool._assert_initialized(**kwargs) # pylint: disable=protected-access + + @asyncio.coroutine + def _callback(self, cb, cb_args=None, **kwargs): + if cb_args is None: + cb_args = [] + vol_args = [self.name, self.vid, self.source, *cb_args] + yield from self._cb_pool._callback(cb, cb_args=vol_args, **kwargs) # pylint: disable=protected-access + + @property + def backend_class(self): + '''Class of the first non-CallbackVolume backend Volume.''' + if isinstance(self._cb_impl, CallbackVolume): + return self._cb_impl.backend_class + return self._cb_impl.__class__ + + @asyncio.coroutine + def create(self): + yield from self._assert_initialized() + yield from self._callback('pre_volume_create') + ret = yield from coro_maybe(self._cb_impl.create()) + yield from self._callback('post_volume_create') + return ret + + @asyncio.coroutine + def remove(self): + yield from self._assert_initialized() + ret = yield from coro_maybe(self._cb_impl.remove()) + yield from self._callback('post_volume_remove') + return ret + + @asyncio.coroutine + def resize(self, size): + yield from self._assert_initialized() + yield from self._callback('pre_volume_resize', cb_args=[size]) + return (yield from coro_maybe(self._cb_impl.resize(size))) + + @asyncio.coroutine + def start(self): + yield from self._assert_initialized() + yield from self._callback('pre_volume_start') + ret = yield from coro_maybe(self._cb_impl.start()) + yield from self._callback('post_volume_start') + return ret + + @asyncio.coroutine + def stop(self): + yield from self._assert_initialized() + ret = yield from coro_maybe(self._cb_impl.stop()) + yield from self._callback('post_volume_stop') + return ret + + @asyncio.coroutine + def import_data(self, size): + yield from self._assert_initialized() + yield from self._callback('pre_volume_import_data', cb_args=[size]) + return (yield from coro_maybe(self._cb_impl.import_data(size))) + + @asyncio.coroutine + def import_data_end(self, success): + yield from self._assert_initialized() + ret = yield from coro_maybe(self._cb_impl.import_data_end(success)) + yield from self._callback('post_volume_import_data_end', cb_args=[success]) + return ret + + @asyncio.coroutine + def import_volume(self, src_volume): + yield from self._assert_initialized() + yield from self._callback('pre_volume_import', cb_args=[src_volume.vid]) + ret = yield from coro_maybe(self._cb_impl.import_volume(src_volume)) + yield from self._callback('post_volume_import', cb_args=[src_volume.vid]) + return ret + + def is_dirty(self): + # pylint: disable=protected-access + if self._cb_pool._cb_requires_init: + return False + return self._cb_impl.is_dirty() + + def is_outdated(self): + # pylint: disable=protected-access + if self._cb_pool._cb_requires_init: + return False + return self._cb_impl.is_outdated() + + @property + def revisions(self): + return self._cb_impl.revisions + + @property + def size(self): + return self._cb_impl.size + + @size.setter + def size(self, size): + self._cb_impl.size = size + + @property + def config(self): + return self._cb_impl.config + + def block_device(self): + # pylint: disable=protected-access + if self._cb_pool._cb_requires_init: + # usually Volume.start() is called beforehand + # --> we should be initialized in 99% of cases + return None + return self._cb_impl.block_device() + + @asyncio.coroutine + def export(self): + yield from self._assert_initialized() + yield from self._callback('pre_volume_export') + return (yield from coro_maybe(self._cb_impl.export())) + + @asyncio.coroutine + def export_end(self, path): + yield from self._assert_initialized() + ret = yield from coro_maybe(self._cb_impl.export_end(path)) + yield from self._callback('post_volume_export_end', cb_args=[path]) + return ret + + @asyncio.coroutine + def verify(self): + yield from self._assert_initialized() + return (yield from coro_maybe(self._cb_impl.verify())) + + @asyncio.coroutine + def revert(self, revision=None): + yield from self._assert_initialized() + return (yield from coro_maybe(self._cb_impl.revert(revision=revision))) + + #shadow all qubes.storage.Volume class attributes as instance properties + #NOTE: this will cause a subtle difference to using an actual _cb_impl instance: CallbackVolume.devtype will return a property object, Volume.devtype the actual value + @property + def devtype(self): + return self._cb_impl.devtype + + @devtype.setter + def devtype(self, devtype): + self._cb_impl.devtype = devtype + + @property + def domain(self): + return self._cb_impl.domain + + @domain.setter + def domain(self, domain): + self._cb_impl.domain = domain + + @property + def path(self): + return self._cb_impl.path + + @path.setter + def path(self, path): + self._cb_impl.path = path + + @property + def script(self): + return self._cb_impl.script + + @script.setter + def script(self, script): + self._cb_impl.script = script + + @property + def usage(self): + return self._cb_impl.usage + + @usage.setter + def usage(self, usage): + self._cb_impl.usage = usage + + #remaining method & attribute delegation + def __getattr__(self, name): + return getattr(self._cb_impl, name) + + def __setattr__(self, name, value): + if name.startswith('_cb_'): + super().__setattr__(name, value) + else: + setattr(self._cb_impl, name, value) + + def __delattr__(self, name): + if name.startswith('_cb_'): + super().__delattr__(name) + else: + delattr(self._cb_impl, name) diff --git a/qubes/storage/lvm.py b/qubes/storage/lvm.py index aab81667..99a89f47 100644 --- a/qubes/storage/lvm.py +++ b/qubes/storage/lvm.py @@ -509,7 +509,7 @@ class ThinVolume(qubes.storage.Volume): # HACK: neat trick to speed up testing if you have same physical thin # pool assigned to two qubes-pools i.e: qubes_dom0 and test-lvm # pylint: disable=line-too-long - if isinstance(src_volume.pool, ThinPool) and \ + if hasattr(src_volume.pool, 'thin_pool') and \ src_volume.pool.thin_pool == self.pool.thin_pool: # NOQA yield from self._commit(src_volume.path[len('/dev/'):], keep=True) else: diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index 8af3dd24..076135b5 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -1412,6 +1412,7 @@ def load_tests(loader, tests, pattern): # pylint: disable=unused-argument 'qubes.tests.storage_file', 'qubes.tests.storage_reflink', 'qubes.tests.storage_lvm', + 'qubes.tests.storage_callback', 'qubes.tests.storage_kernels', 'qubes.tests.ext', 'qubes.tests.vm.qubesvm', diff --git a/qubes/tests/storage.py b/qubes/tests/storage.py index eaa345be..72cec71c 100644 --- a/qubes/tests/storage.py +++ b/qubes/tests/storage.py @@ -116,7 +116,7 @@ class TC_00_Pool(QubesTestCase): def test_001_all_pool_drivers(self): """ Expect all our pool drivers (and only them) """ self.assertCountEqual( - ['linux-kernel', 'lvm_thin', 'file', 'file-reflink'], + ['linux-kernel', 'lvm_thin', 'file', 'file-reflink', 'callback'], pool_drivers()) def test_002_get_pool_klass(self): diff --git a/qubes/tests/storage_callback.py b/qubes/tests/storage_callback.py new file mode 100644 index 00000000..6cc0371a --- /dev/null +++ b/qubes/tests/storage_callback.py @@ -0,0 +1,378 @@ +# +# The Qubes OS Project, http://www.qubes-os.org +# +# Copyright (C) 2020 David Hobach +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, see . +# +''' Tests for the callback storage driver. + + They are mostly based upon the lvm storage driver tests. +''' +# pylint: disable=line-too-long + +import os +import json +import subprocess +import qubes.tests +import qubes.tests.storage +import qubes.tests.storage_lvm +from qubes.tests.storage_lvm import skipUnlessLvmPoolExists +from qubes.storage.callback import CallbackPool, CallbackVolume + +CB_CONF = '/etc/qubes_callback.json' +LOG_BIN = '/tmp/testCbLogArgs' + +CB_DATA = {'utest-callback-01': { + 'bdriver': 'lvm_thin', + 'bdriver_args': { + 'volume_group': qubes.tests.storage_lvm.DEFAULT_LVM_POOL.split('/')[0], + 'thin_pool': qubes.tests.storage_lvm.DEFAULT_LVM_POOL.split('/')[1] + }, + 'description': 'For unit testing of the callback pool driver.' + }, + 'utest-callback-02': { + 'bdriver': 'lvm_thin', + 'bdriver_args': { + 'volume_group': qubes.tests.storage_lvm.DEFAULT_LVM_POOL.split('/')[0], + 'thin_pool': qubes.tests.storage_lvm.DEFAULT_LVM_POOL.split('/')[1] + }, + 'cmd': LOG_BIN, + 'description': 'For unit testing of the callback pool driver.' + }, + 'utest-callback-03': { + 'bdriver': 'lvm_thin', + 'bdriver_args': { + 'volume_group': qubes.tests.storage_lvm.DEFAULT_LVM_POOL.split('/')[0], + 'thin_pool': qubes.tests.storage_lvm.DEFAULT_LVM_POOL.split('/')[1] + }, + 'cmd': 'exit 1', + 'pre_sinit': LOG_BIN + ' pre_sinit', + 'pre_setup': LOG_BIN + ' pre_setup', + 'pre_volume_create': LOG_BIN + ' pre_volume_create', + 'post_volume_create': LOG_BIN + ' post_volume_create', + 'pre_volume_import_data': LOG_BIN + ' pre_volume_import_data', + 'post_volume_import_data_end': LOG_BIN + ' post_volume_import_data_end', + 'post_volume_remove': LOG_BIN + ' post_volume_remove', + 'post_destroy': '-', + 'description': 'For unit testing of the callback pool driver.' + }, + 'testing-fail-missing-all': { + }, + 'testing-fail-missing-bdriver-args': { + 'bdriver': 'file', + 'description': 'For unit testing of the callback pool driver.' + }, + 'testing-fail-incorrect-bdriver': { + 'bdriver': 'nonexisting-bdriver', + 'bdriver_args': { + 'foo': 'bar', + 'bla': 'blub' + }, + 'cmd': 'echo foo', + 'description': 'For unit testing of the callback pool driver.' + }, + } + +class CallbackBase: + ''' Mixin base class for callback tests. Has no base class. ''' + conf_id = None + pool_name = 'test-callback' + + @classmethod + def setUpClass(cls, conf_id='utest-callback-01'): + conf = {'name': cls.pool_name, + 'driver': 'callback', + 'conf_id': conf_id} + cls.conf_id = conf_id + + assert not(os.path.exists(CB_CONF)), '%s must NOT exist. Please delete it, if you do not need it.' % CB_CONF + + sudo = [] if os.getuid() == 0 else ['sudo'] + subprocess.run(sudo + ['install', '-m', '666', '/dev/null', CB_CONF], check=True) + + with open(CB_CONF, 'w') as outfile: + json.dump(CB_DATA, outfile) + super().setUpClass(pool_class=CallbackPool, volume_class=CallbackVolume, pool_conf=conf) + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + + sudo = [] if os.getuid() == 0 else ['sudo'] + subprocess.run(sudo + ['rm', '-f', CB_CONF], check=True) + + def setUp(self, init_pool=True): + super().setUp(init_pool=init_pool) + if init_pool: + #tests from other pools will assume that they're fully initialized after calling __init__() + self.loop.run_until_complete(self.pool._assert_initialized()) + + def test_000_000_callback_test_init(self): + ''' Check whether the test init did work. ''' + if hasattr(self, 'pool'): + self.assertIsInstance(self.pool, CallbackPool) + self.assertEqual(self.pool.backend_class, qubes.storage.lvm.ThinPool) + self.assertTrue(os.path.isfile(CB_CONF)) + +@skipUnlessLvmPoolExists +class TC_00_CallbackPool(CallbackBase, qubes.tests.storage_lvm.TC_00_ThinPool): + pass + +@skipUnlessLvmPoolExists +class TC_01_CallbackPool(CallbackBase, qubes.tests.storage_lvm.TC_01_ThinPool): + pass + +@skipUnlessLvmPoolExists +class TC_02_cb_StorageHelpers(CallbackBase, qubes.tests.storage_lvm.TC_02_StorageHelpers): + pass + +class LoggingCallbackBase(CallbackBase): + ''' Mixin base class that sets up LOG_BIN and removes `LoggingCallbackBase.test_log`, if needed. ''' + test_log = '/tmp/cb_tests.log' + test_log_expected = None #dict: class + test name --> test index (int, 0..x) --> expected _additional_ log content + volume_name = 'volume_name' + xml_path = '/tmp/qubes-test-callback.xml' + + @classmethod + def setUpClass(cls, conf_id=None, log_expected=None): + script = """#!/bin/bash +i=1 +for arg in "$@" ; do + echo "$i: $arg" >> "LOG_OUT" + (( i++)) +done +exit 0 +""" + script = script.replace('LOG_OUT', cls.test_log) + with open(LOG_BIN, 'w') as f: + f.write(script) + os.chmod(LOG_BIN, 0o775) + + cls.test_log_expected = log_expected + super().setUpClass(conf_id=conf_id) + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + os.remove(LOG_BIN) + + def setUp(self, init_pool=False): + assert not(os.path.exists(self.test_log)), '%s must NOT exist. Please delete it, if you do not need it.' % self.test_log + self.maxDiff = None + + xml = """ + + + + + + + + + + + + linux-kernel + + + + + + + black + + + + + + + """ + xml = xml.replace('CONF_ID', self.conf_id) + xml = xml.replace('POOL_NAME', self.pool_name) + with open(self.xml_path, 'w') as f: + f.write(xml) + self.app = qubes.Qubes(self.xml_path, + clockvm=None, + updatevm=None, + offline_mode=True, + ) + os.environ['QUBES_XML_PATH'] = self.xml_path + super().setUp(init_pool=init_pool) + + def tearDown(self): + super().tearDown() + os.unlink(self.app.store) + self.app.close() + del self.app + for attr in dir(self): + if isinstance(getattr(self, attr), qubes.vm.BaseVM): + delattr(self, attr) + + if os.path.exists(self.test_log): + os.remove(self.test_log) + + if os.path.exists(self.xml_path): + os.remove(self.xml_path) + + def assertLogContent(self, expected): + ''' Assert that the log matches the given string. + :param expected: Expected content of the log file (String). + ''' + try: + with open(self.test_log, 'r') as f: + found = f.read() + except FileNotFoundError: + found = '' + if expected != '': + expected = expected + '\n' + self.assertEqual(found, expected) + + def assertLog(self, test_name, ind=0): + ''' Assert that the log matches the expected status. + :param test_name: Name of the test. + :param ind: Index inside `test_log_expected` to check against (Integer starting at 0). + ''' + d = self.test_log_expected[str(self.__class__) + test_name] + expected = [] + for i in range(ind+1): + expected = expected + [d[i]] + expected = filter(None, expected) + self.assertLogContent('\n'.join(expected)) + + def test_001_callbacks(self): + ''' create a lvm pool with additional callbacks ''' + config = { + 'name': self.volume_name, + 'pool': self.pool_name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 2, + 'size': qubes.config.defaults['root_img_size'], + } + new_size = 2 * qubes.config.defaults['root_img_size'] + + test_name = 'test_001_callbacks' + self.assertLog(test_name, 0) + self.init_pool() + self.assertFalse(self.created_pool) + self.assertIsInstance(self.pool, CallbackPool) + self.assertLog(test_name, 1) + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + self.assertLog(test_name, 2) + self.loop.run_until_complete(volume.create()) + self.assertLog(test_name, 3) + self.loop.run_until_complete(volume.import_data(new_size)) + self.assertLog(test_name, 4) + self.loop.run_until_complete(volume.import_data_end(True)) + self.assertLog(test_name, 5) + self.assertEqual(volume.size, new_size) + self.loop.run_until_complete(volume.remove()) + self.assertLog(test_name, 6) + +@skipUnlessLvmPoolExists +class TC_91_CallbackPool(LoggingCallbackBase, qubes.tests.storage_lvm.ThinPoolBase): + ''' Tests for the actual callback functionality. + conf_id = utest-callback-02 + ''' + + @classmethod + def setUpClass(cls): + conf_id = 'utest-callback-02' + name = cls.pool_name + bdriver = (CB_DATA[conf_id])['bdriver'] + ctor_params = json.dumps(CB_DATA[conf_id], sort_keys=True, indent=2) + vname = cls.volume_name + vid = '{0}/vm-test-inst-appvm-{1}'.format(qubes.tests.storage_lvm.DEFAULT_LVM_POOL.split('/')[0], vname) + vsize = 2 * qubes.config.defaults['root_img_size'] + log_expected = \ + {str(cls) + 'test_001_callbacks': + {0: '', + 1: '', + 2: '', + 3: '1: {0}\n2: {1}\n3: pre_sinit\n4: {2}\n5: {3}\n1: {0}\n2: {1}\n3: pre_volume_create\n4: {2}\n5: {3}\n6: {4}\n7: {5}\n8: None\n1: {0}\n2: {1}\n3: post_volume_create\n4: {2}\n5: {3}\n6: {4}\n7: {5}\n8: None'.format(name, bdriver, conf_id, ctor_params, vname, vid), + 4: '1: {0}\n2: {1}\n3: pre_volume_import_data\n4: {2}\n5: {3}\n6: {4}\n7: {5}\n8: None\n9: {6}'.format(name, bdriver, conf_id, ctor_params, vname, vid, vsize), + 5: '1: {0}\n2: {1}\n3: post_volume_import_data_end\n4: {2}\n5: {3}\n6: {4}\n7: {5}\n8: None\n9: {6}'.format(name, bdriver, conf_id, ctor_params, vname, vid, True), + 6: '1: {0}\n2: {1}\n3: post_volume_remove\n4: {2}\n5: {3}\n6: {4}\n7: {5}\n8: None'.format(name, bdriver, conf_id, ctor_params, vname, vid), + } + } + super().setUpClass(conf_id=conf_id, log_expected=log_expected) + +@skipUnlessLvmPoolExists +class TC_92_CallbackPool(LoggingCallbackBase, qubes.tests.storage_lvm.ThinPoolBase): + ''' Tests for the actual callback functionality. + conf_id = utest-callback-03 + ''' + + @classmethod + def setUpClass(cls): + log_expected = \ + {str(cls) + 'test_001_callbacks': + {0: '', + 1: '', + 2: '', + 3: '1: pre_sinit\n1: pre_volume_create\n1: post_volume_create', + 4: '1: pre_volume_import_data', + 5: '1: post_volume_import_data_end', + 6: '1: post_volume_remove', + } + } + super().setUpClass(conf_id='utest-callback-03', log_expected=log_expected) + + def test_002_failing_callback(self): + ''' Make sure that we check the exit code of executed callbacks. ''' + config = { + 'name': self.volume_name, + 'pool': self.pool_name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 2, + 'size': qubes.config.defaults['root_img_size'], + } + self.init_pool() + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + with self.assertRaises(subprocess.CalledProcessError) as cm: + #should trigger the `exit 1` of `cmd` + self.loop.run_until_complete(volume.start()) + self.assertTrue('exit status 1' in str(cm.exception)) + + def test_003_errors(self): + ''' Make sure we error out on common user & dev mistakes. ''' + #missing conf_id + with self.assertRaises(qubes.storage.StoragePoolException): + cb = CallbackPool(name='some-name', conf_id='') + + #invalid conf_id + with self.assertRaises(qubes.storage.StoragePoolException): + cb = CallbackPool(name='some-name', conf_id='nonexisting-id') + + #incorrect backend driver + with self.assertRaises(qubes.storage.StoragePoolException): + cb = CallbackPool(name='some-name', conf_id='testing-fail-incorrect-bdriver') + + #missing config entries + with self.assertRaises(qubes.storage.StoragePoolException): + cb = CallbackPool(name='some-name', conf_id='testing-fail-missing-all') + + #missing bdriver args + with self.assertRaises(TypeError): + cb = CallbackPool(name='some-name', conf_id='testing-fail-missing-bdriver-args') + +class TC_93_CallbackPool(qubes.tests.QubesTestCase): + def test_001_missing_conf(self): + ''' A missing config file must cause errors. ''' + with self.assertRaises(FileNotFoundError): + cb = CallbackPool(name='some-name', conf_id='nonexisting-id') diff --git a/qubes/tests/storage_lvm.py b/qubes/tests/storage_lvm.py index bef958e9..9f2dea06 100644 --- a/qubes/tests/storage_lvm.py +++ b/qubes/tests/storage_lvm.py @@ -68,16 +68,26 @@ POOL_CONF = {'name': 'test-lvm', class ThinPoolBase(qubes.tests.QubesTestCase): ''' Sanity tests for :py:class:`qubes.storage.lvm.ThinPool` ''' + pool_class = None + volume_class = None + pool_conf = None created_pool = False - def setUp(self): + @classmethod + def setUpClass(cls, pool_class=ThinPool, volume_class=ThinVolume, pool_conf=None): + ''' Other test classes (e.g. callback) may use this to test their own config. ''' + conf = pool_conf + if not conf: + conf = POOL_CONF + + cls.pool_class = pool_class + cls.volume_class = volume_class + cls.pool_conf = conf + + def setUp(self, init_pool=True): super(ThinPoolBase, self).setUp() - volume_group, thin_pool = DEFAULT_LVM_POOL.split('/', 1) - self.pool = self._find_pool(volume_group, thin_pool) - if not self.pool: - self.pool = self.loop.run_until_complete( - self.app.add_pool(**POOL_CONF)) - self.created_pool = True + if init_pool: + self.init_pool() def cleanup_test_volumes(self): p = self.loop.run_until_complete(asyncio.create_subprocess_exec( @@ -96,18 +106,26 @@ class ThinPoolBase(qubes.tests.QubesTestCase): def tearDown(self): ''' Remove the default lvm pool if it was created only for this test ''' - self.cleanup_test_volumes() + if hasattr(self, 'pool'): + self.cleanup_test_volumes() if self.created_pool: self.loop.run_until_complete(self.app.remove_pool(self.pool.name)) super(ThinPoolBase, self).tearDown() + def init_pool(self): + volume_group, thin_pool = DEFAULT_LVM_POOL.split('/', 1) + self.pool = self._find_pool(volume_group, thin_pool) + if not self.pool: + self.pool = self.loop.run_until_complete( + self.app.add_pool(**self.pool_conf)) + self.created_pool = True def _find_pool(self, volume_group, thin_pool): ''' Returns the pool matching the specified ``volume_group`` & ``thin_pool``, or None. ''' pools = [p for p in self.app.pools.values() - if issubclass(p.__class__, ThinPool)] + if issubclass(p.__class__, self.pool_class)] for pool in pools: if pool.volume_group == volume_group \ and pool.thin_pool == thin_pool: @@ -118,7 +136,7 @@ class ThinPoolBase(qubes.tests.QubesTestCase): class TC_00_ThinPool(ThinPoolBase): ''' Sanity tests for :py:class:`qubes.storage.lvm.ThinPool` ''' - def setUp(self): + def setUp(self, **kwargs): xml_path = '/tmp/qubes-test.xml' self.app = qubes.Qubes.create_empty_store(store=xml_path, clockvm=None, @@ -126,7 +144,7 @@ class TC_00_ThinPool(ThinPoolBase): offline_mode=True, ) os.environ['QUBES_XML_PATH'] = xml_path - super(TC_00_ThinPool, self).setUp() + super().setUp(**kwargs) def tearDown(self): super(TC_00_ThinPool, self).tearDown() @@ -155,7 +173,7 @@ class TC_00_ThinPool(ThinPoolBase): } vm = qubes.tests.storage.TestVM(self) volume = self.app.get_pool(self.pool.name).init_volume(vm, config) - self.assertIsInstance(volume, ThinVolume) + self.assertIsInstance(volume, self.volume_class) self.assertEqual(volume.name, 'root') self.assertEqual(volume.pool, self.pool.name) self.assertEqual(volume.size, qubes.config.defaults['root_img_size']) @@ -175,7 +193,7 @@ class TC_00_ThinPool(ThinPoolBase): } vm = qubes.tests.storage.TestVM(self) volume = self.app.get_pool(self.pool.name).init_volume(vm, config) - self.assertIsInstance(volume, ThinVolume) + self.assertIsInstance(volume, self.volume_class) self.assertEqual(volume.name, 'root') self.assertEqual(volume.pool, self.pool.name) self.assertEqual(volume.size, qubes.config.defaults['root_img_size']) @@ -952,7 +970,7 @@ class TC_00_ThinPool(ThinPoolBase): } volume = self.app.get_pool(self.pool.name).init_volume( vm, config_snapshot) - self.assertIsInstance(volume, ThinVolume) + self.assertIsInstance(volume, self.volume_class) self.assertEqual(volume.name, 'root2') self.assertEqual(volume.pool, self.pool.name) self.assertEqual(volume.size, qubes.config.defaults['root_img_size']) @@ -1059,8 +1077,8 @@ class TC_00_ThinPool(ThinPoolBase): class TC_01_ThinPool(ThinPoolBase, qubes.tests.SystemTestCase): ''' Sanity tests for :py:class:`qubes.storage.lvm.ThinPool` ''' - def setUp(self): - super(TC_01_ThinPool, self).setUp() + def setUp(self, **kwargs): + super().setUp(**kwargs) self.init_default_template() def test_004_import(self): @@ -1107,7 +1125,7 @@ class TC_01_ThinPool(ThinPoolBase, qubes.tests.SystemTestCase): @skipUnlessLvmPoolExists class TC_02_StorageHelpers(ThinPoolBase): - def setUp(self): + def setUp(self, **kwargs): xml_path = '/tmp/qubes-test.xml' self.app = qubes.Qubes.create_empty_store(store=xml_path, clockvm=None, @@ -1115,7 +1133,7 @@ class TC_02_StorageHelpers(ThinPoolBase): offline_mode=True, ) os.environ['QUBES_XML_PATH'] = xml_path - super(TC_02_StorageHelpers, self).setUp() + super().setUp(**kwargs) # reset cache qubes.storage.DirectoryThinPool._thin_pool = {} diff --git a/rpm_spec/core-dom0.spec.in b/rpm_spec/core-dom0.spec.in index 84ab0038..979b2bfa 100644 --- a/rpm_spec/core-dom0.spec.in +++ b/rpm_spec/core-dom0.spec.in @@ -402,6 +402,8 @@ done %{python3_sitelib}/qubes/storage/reflink.py %{python3_sitelib}/qubes/storage/kernels.py %{python3_sitelib}/qubes/storage/lvm.py +%{python3_sitelib}/qubes/storage/callback.py +%doc /usr/share/doc/qubes/qubes_callback.json.example %dir %{python3_sitelib}/qubes/tools %dir %{python3_sitelib}/qubes/tools/__pycache__ @@ -452,6 +454,7 @@ done %{python3_sitelib}/qubes/tests/storage_reflink.py %{python3_sitelib}/qubes/tests/storage_kernels.py %{python3_sitelib}/qubes/tests/storage_lvm.py +%{python3_sitelib}/qubes/tests/storage_callback.py %{python3_sitelib}/qubes/tests/tarwriter.py %dir %{python3_sitelib}/qubes/tests/vm diff --git a/setup.py b/setup.py index a5d34c36..277eb66d 100644 --- a/setup.py +++ b/setup.py @@ -83,6 +83,7 @@ if __name__ == '__main__': 'file-reflink = qubes.storage.reflink:ReflinkPool', 'linux-kernel = qubes.storage.kernels:LinuxKernel', 'lvm_thin = qubes.storage.lvm:ThinPool', + 'callback = qubes.storage.callback:CallbackPool', ], 'qubes.tests.storage': [ 'test = qubes.tests.storage:TestPool', @@ -90,5 +91,6 @@ if __name__ == '__main__': 'file-reflink = qubes.storage.reflink:ReflinkPool', 'linux-kernel = qubes.storage.kernels:LinuxKernel', 'lvm_thin = qubes.storage.lvm:ThinPool', + 'callback = qubes.storage.callback:CallbackPool', ], })