Merge remote-tracking branch 'origin/pr/354'
* origin/pr/354: (35 commits) tests/lvm: re-introduce POOL_CONF tests/lvm & callback: remove explicit class references storage/callback: remove the "word of caution" storage/callback: comment fixes storage/callback: add the config ID as callback argument storage/callback: some callbacks added & removed tests/lvm & callback: Refactoring Revert "storage/callback: do not run sync code async" tests/callback: ensure missing conf causes errors storage/callback: do not run sync code async tests/callback: added callback-specific tests storage/callback: async Volume.export() & added Volume.export_end() storage/lvm: make the "hack" work with CallbackPool instances storage/callback: add the backend_class property tests/callback: add them to the rpm build storage/callback: various fixes tests/callback: add rudimentary tests for the callback driver tests/lvm: make the tests re-usable for other drivers storage/callback: fix issues detected by pylint storage/callback: volume callbacks now also rceive the source volume as argument (if there's any) ...
This commit is contained in:
Marek Marczykowski-Górecki
commit
39ef189a93
5
Makefile
5
Makefile
@ -117,11 +117,13 @@ DATADIR ?= /var/lib/qubes
|
||||
STATEDIR ?= /var/run/qubes
|
||||
LOGDIR ?= /var/log/qubes
|
||||
FILESDIR ?= /usr/share/qubes
|
||||
DOCDIR ?= /usr/share/doc/qubes
|
||||
else ifeq ($(OS),Windows_NT)
|
||||
DATADIR ?= c:/qubes
|
||||
STATEDIR ?= c:/qubes/state
|
||||
LOGDIR ?= c:/qubes/log
|
||||
FILESDIR ?= c:/program files/Invisible Things Lab/Qubes
|
||||
DOCDIR ?= c:/qubes/doc
|
||||
endif
|
||||
|
||||
help:
|
||||
@ -214,6 +216,9 @@ endif
|
||||
cp -r templates "$(DESTDIR)$(FILESDIR)/templates"
|
||||
rm -f "$(DESTDIR)$(FILESDIR)/templates/README"
|
||||
|
||||
mkdir -p "$(DESTDIR)$(DOCDIR)"
|
||||
cp qubes/storage/callback.json.example "$(DESTDIR)$(DOCDIR)/qubes_callback.json.example"
|
||||
|
||||
mkdir -p $(DESTDIR)$(DATADIR)
|
||||
mkdir -p $(DESTDIR)$(DATADIR)/vm-templates
|
||||
mkdir -p $(DESTDIR)$(DATADIR)/appvms
|
||||
|
||||
99
qubes/storage/callback.json.example
Normal file
99
qubes/storage/callback.json.example
Normal file
@ -0,0 +1,99 @@
|
||||
{
|
||||
"conf_id":
|
||||
{
|
||||
"bdriver": "Backend pool driver to proxy. This can be a class name or a full module path to a class implementing `qubes.storage.Pool`.",
|
||||
"bdriver_args": {
|
||||
"arg name 1": "arg value 1",
|
||||
"arg name 2": "arg value 2"
|
||||
},
|
||||
"cmd": "Default command to call when the [pre|post]_[op] operations are not specified (default: None). The command is called as such: `[cmd] [name] [bdriver] [operation] [ctor params]`. [name]: name of the pool, [operation]: any of the `[pre|post]_` operations from below including its arguments, [bdriver]: backend driver of the pool, [ctor params]: Parameters passed to the `bdriver` constructor in JSON format. Each parameter is on a single line for easy parsing.",
|
||||
"signal_back": "Boolean (true|false) to allow the executed commands to send signals back to the callback driver (default: false). Signals must be on a dedicated line on stdout. Currently only `SIGNAL_setup` is supported. When found, it causes the callback driver to re-setup the backend pool.",
|
||||
"pre_sinit": "Command to call before one-time storage initialization/first usage (default: None). Called exactly once for every `qubesd` start. Can be used to override `cmd`. Pass `-` to ignore this callback entirely even if `cmd` is specified.",
|
||||
"pre_setup": "Called before creating a new pool. Can be used to override `cmd`. Pass `-` to ignore this callback entirely even if `cmd` is specified.",
|
||||
"post_destroy": "Called after removing an existing pool. Same as above otherwise.",
|
||||
"pre_volume_create": "Called before creating a volume for the pool. Same as above otherwise.",
|
||||
"post_volume_create": "Called after creating a volume for the pool. Same as above otherwise.",
|
||||
"post_volume_remove": "Called after removing a volume of the pool. Same as above otherwise.",
|
||||
"pre_volume_resize": "Called before resizing a volume of the pool. Same as above otherwise.",
|
||||
"pre_volume_start": "Called before starting a volume of the pool. Same as above otherwise.",
|
||||
"post_volume_start": "Called after starting a volume of the pool. Same as above otherwise.",
|
||||
"post_volume_stop": "Called after stopping a volume of the pool. Same as above otherwise.",
|
||||
"pre_volume_import": "Called before importing a volume from another. Same as above otherwise.",
|
||||
"post_volume_import": "Called after importing a volume from another. Same as above otherwise.",
|
||||
"pre_volume_import_data": "Called before overwriting this volume with new data. Same as above otherwise.",
|
||||
"post_volume_import_data_end": "Called after finishing an `import_data` action. Same as above otherwise.",
|
||||
"pre_volume_export": "Called before exporting a volume. Same as above otherwise.",
|
||||
"post_volume_export_end": "Called after a volume export completed. Same as above otherwise.",
|
||||
"description": "Optional description for your personal reference."
|
||||
},
|
||||
"testing-fail-missing-all":
|
||||
{
|
||||
"description": "For testing purposes only."
|
||||
},
|
||||
"testing-fail-missing-bdriver-args":
|
||||
{
|
||||
"bdriver": "file",
|
||||
"description": "For testing purposes only."
|
||||
},
|
||||
"testing-succ-file-01":
|
||||
{
|
||||
"bdriver": "file",
|
||||
"bdriver_args": {
|
||||
"dir_path": "/mnt/test01",
|
||||
"revisions_to_keep": 0
|
||||
},
|
||||
"description": "For testing purposes only."
|
||||
},
|
||||
"testing-succ-file-02":
|
||||
{
|
||||
"bdriver": "file",
|
||||
"bdriver_args": {
|
||||
"dir_path": "/mnt/test02"
|
||||
},
|
||||
"cmd": "testCbLogArgs",
|
||||
"description": "For testing purposes only."
|
||||
},
|
||||
"testing-succ-file-03":
|
||||
{
|
||||
"bdriver": "file",
|
||||
"bdriver_args": {
|
||||
"dir_path": "/mnt/test03"
|
||||
},
|
||||
"cmd": "exit 1",
|
||||
"pre_sinit": "testCbLogArgs pre_sinit",
|
||||
"pre_setup": "testCbLogArgs pre_setup",
|
||||
"post_destroy": "-",
|
||||
"description": "For testing purposes only."
|
||||
},
|
||||
"testing-succ-file-luks":
|
||||
{
|
||||
"bdriver": "file",
|
||||
"bdriver_args": {
|
||||
"dir_path": "/mnt/test_luks"
|
||||
},
|
||||
"pre_setup": "set -e -o pipefail ; [ ! -e /mnt/test.key ] ; [ ! -e /mnt/test.luks ] ; dd if=/dev/random bs=100 of=/mnt/test.key iflag=fullblock count=1 ; dd if=/dev/urandom bs=1M of=/mnt/test.luks iflag=fullblock count=2048 ; cryptsetup luksFormat -q --key-file /mnt/test.key /mnt/test.luks ; cryptsetup open -q --key-file /mnt/test.key /mnt/test.luks test-luks ; mkfs -t ext4 /dev/mapper/test-luks ; mkdir -p /mnt/test_luks ; mount /dev/mapper/test-luks /mnt/test_luks",
|
||||
"pre_sinit": "set -e -o pipefail ; if [[ \"$(findmnt -S /dev/mapper/test-luks -n -o TARGET)\" != \"/mnt/test_luks\" ]] ; then cryptsetup status test-luks > /dev/null || cryptsetup open -q --key-file /mnt/test.key /mnt/test.luks test-luks ; mount /dev/mapper/test-luks /mnt/test_luks ; else exit 0 ; fi",
|
||||
"post_destroy": "umount /mnt/test_luks && cryptsetup close test-luks ; set -e -o pipefail ; dd if=/dev/urandom bs=100 of=/mnt/test.key iflag=fullblock count=1 ; rm -f /mnt/test.key ; rm -f /mnt/test.luks ; rmdir /mnt/test_luks",
|
||||
"pre_volume_create": "[[ \"$(findmnt -S /dev/mapper/test-luks -n -o TARGET)\" == \"/mnt/test_luks\" ]]",
|
||||
"pre_volume_import": "[[ \"$(findmnt -S /dev/mapper/test-luks -n -o TARGET)\" == \"/mnt/test_luks\" ]]",
|
||||
"pre_volume_import_data": "[[ \"$(findmnt -S /dev/mapper/test-luks -n -o TARGET)\" == \"/mnt/test_luks\" ]]",
|
||||
"post_volume_import_data_end": "[[ \"$(findmnt -S /dev/mapper/test-luks -n -o TARGET)\" == \"/mnt/test_luks\" ]]",
|
||||
"description": "For testing purposes only: Showcasing seemless use of encrypted VM pools. For personal use, a dedicated script is recommended."
|
||||
},
|
||||
"testing-succ-file-luks-eph":
|
||||
{
|
||||
"bdriver": "file",
|
||||
"bdriver_args": {
|
||||
"dir_path": "/mnt/test_eph"
|
||||
},
|
||||
"signal_back": true,
|
||||
"pre_setup": "set -e -o pipefail ; if [[ \"$(findmnt -T /mnt/ram -o SOURCE -n)\" != \"ramfs\" ]] ; then mkdir -p /mnt/ram ; mount -t ramfs ramfs /mnt/ram ; fi ; if [[ \"$(findmnt -S /dev/mapper/test-eph -n -o TARGET)\" != \"/mnt/test_eph\" ]] ; then if [ -f /mnt/ram/teph.key ] ; then cryptsetup status test-eph > /dev/null || cryptsetup open -q --key-file /mnt/ram/teph.key /mnt/teph.luks test-eph ; else dd if=/dev/random bs=100 of=/mnt/ram/teph.key iflag=fullblock count=1 ; if [ ! -f /mnt/teph.luks ] ; then dd if=/dev/urandom bs=1M of=/mnt/teph.luks iflag=fullblock count=2048 ; fi ; cryptsetup luksFormat -q --key-file /mnt/ram/teph.key /mnt/teph.luks ;cryptsetup open -q --key-file /mnt/ram/teph.key /mnt/teph.luks test-eph ; mkfs -t ext4 /dev/mapper/test-eph ; echo SIGNAL_setup ; fi ; mkdir -p /mnt/test_eph ; mount /dev/mapper/test-eph /mnt/test_eph ; fi ; exit 0",
|
||||
"pre_sinit": "set -e -o pipefail ; if [[ \"$(findmnt -T /mnt/ram -o SOURCE -n)\" != \"ramfs\" ]] ; then mkdir -p /mnt/ram ; mount -t ramfs ramfs /mnt/ram ; fi ; if [[ \"$(findmnt -S /dev/mapper/test-eph -n -o TARGET)\" != \"/mnt/test_eph\" ]] ; then if [ -f /mnt/ram/teph.key ] ; then cryptsetup status test-eph > /dev/null || cryptsetup open -q --key-file /mnt/ram/teph.key /mnt/teph.luks test-eph ; else dd if=/dev/random bs=100 of=/mnt/ram/teph.key iflag=fullblock count=1 ; if [ ! -f /mnt/teph.luks ] ; then dd if=/dev/urandom bs=1M of=/mnt/teph.luks iflag=fullblock count=2048 ; fi ; cryptsetup luksFormat -q --key-file /mnt/ram/teph.key /mnt/teph.luks ;cryptsetup open -q --key-file /mnt/ram/teph.key /mnt/teph.luks test-eph ; mkfs -t ext4 /dev/mapper/test-eph ; echo SIGNAL_setup ; fi ; mkdir -p /mnt/test_eph ; mount /dev/mapper/test-eph /mnt/test_eph ; fi ; exit 0",
|
||||
"post_destroy": "umount /mnt/test_eph && cryptsetup close test-eph ; set -e -o pipefail ; if [ -f /mnt/ram/teph.key ] ; then dd if=/dev/urandom bs=100 of=/mnt/ram/teph.key iflag=fullblock count=1 ; rm -f /mnt/ram/teph.key ; fi ; rm -f /mnt/teph.luks ; rmdir /mnt/test_eph",
|
||||
"pre_volume_create": "[[ \"$(findmnt -S /dev/mapper/test-eph -n -o TARGET)\" == \"/mnt/test_eph\" ]]",
|
||||
"pre_volume_import": "[[ \"$(findmnt -S /dev/mapper/test-eph -n -o TARGET)\" == \"/mnt/test_eph\" ]]",
|
||||
"pre_volume_import_data": "[[ \"$(findmnt -S /dev/mapper/test-eph -n -o TARGET)\" == \"/mnt/test_eph\" ]]",
|
||||
"post_volume_import_data_end": "[[ \"$(findmnt -S /dev/mapper/test-eph -n -o TARGET)\" == \"/mnt/test_eph\" ]]",
|
||||
"description": "For testing purposes only: Showcasing seemless use of encrypted disposable (upon boot) VM pools with keys held in RAM only. For personal use, a dedicated script is recommended."
|
||||
}
|
||||
}
|
||||
622
qubes/storage/callback.py
Normal file
622
qubes/storage/callback.py
Normal file
@ -0,0 +1,622 @@
|
||||
#
|
||||
# The Qubes OS Project, https://www.qubes-os.org/
|
||||
#
|
||||
# Copyright (C) 2020 David Hobach <david@hobach.de>
|
||||
#
|
||||
# This library is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU Lesser General Public
|
||||
# License as published by the Free Software Foundation; either
|
||||
# version 2.1 of the License, or (at your option) any later version.
|
||||
#
|
||||
# This library is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public
|
||||
# License along with this library; if not, see <https://www.gnu.org/licenses/>.
|
||||
#
|
||||
|
||||
# pylint: disable=line-too-long
|
||||
|
||||
import logging
|
||||
import subprocess
|
||||
import json
|
||||
import asyncio
|
||||
import locale
|
||||
from shlex import quote
|
||||
from qubes.utils import coro_maybe
|
||||
|
||||
import qubes.storage
|
||||
|
||||
class UnhandledSignalException(qubes.storage.StoragePoolException):
|
||||
def __init__(self, pool, signal):
|
||||
super().__init__('The pool %s failed to handle the signal %s, likely because it was run from synchronous code.' % (pool.name, signal))
|
||||
|
||||
class CallbackPool(qubes.storage.Pool):
|
||||
''' Proxy storage pool driver adding callback functionality to other pool drivers.
|
||||
|
||||
This way, users can extend storage pool drivers with custom functionality using the programming language of their choice.
|
||||
|
||||
All configuration for this pool driver must be done in `/etc/qubes_callback.json`. Each configuration ID `conf_id` can be used
|
||||
to create a callback pool with e.g. `qvm-pool -o conf_id=your_conf_id -a pool_name callback`.
|
||||
Check `/usr/share/doc/qubes/qubes_callback.json.example` for an overview of the available options.
|
||||
|
||||
Example applications of this driver:
|
||||
- custom pool mounts
|
||||
- encryption
|
||||
- debugging
|
||||
|
||||
|
||||
**Integration tests**:
|
||||
(all of these tests assume the `qubes_callback.json.example` configuration)
|
||||
|
||||
Tests that should **fail**:
|
||||
```
|
||||
qvm-pool -a test callback
|
||||
qvm-pool -o conf_id=non-existing -a test callback
|
||||
qvm-pool -o conf_id=conf_id -a test callback
|
||||
qvm-pool -o conf_id=testing-fail-missing-all -a test callback
|
||||
qvm-pool -o conf_id=testing-fail-missing-bdriver-args -a test callback
|
||||
```
|
||||
|
||||
Tests that should **work**:
|
||||
```
|
||||
qvm-pool -o conf_id=testing-succ-file-01 -a test callback
|
||||
qvm-pool
|
||||
ls /mnt/test01
|
||||
qvm-pool -r test && sudo rm -rf /mnt/test01
|
||||
|
||||
echo '#!/bin/bash'$'\n''i=1 ; for arg in "$@" ; do echo "$i: $arg" >> /tmp/callback.log ; (( i++)) ; done ; exit 0' > /usr/bin/testCbLogArgs && chmod +x /usr/bin/testCbLogArgs
|
||||
rm -f /tmp/callback.log
|
||||
qvm-pool -o conf_id=testing-succ-file-02 -a test callback
|
||||
qvm-pool
|
||||
ls /mnt/test02
|
||||
less /tmp/callback.log (pre_setup should be there)
|
||||
qvm-create -l red -P test test-vm
|
||||
cat /tmp/callback.log (2x pre_volume_create + 2x post_volume_create should be added)
|
||||
qvm-start test-vm
|
||||
qvm-volume | grep test-vm
|
||||
grep test-vm /var/lib/qubes/qubes.xml
|
||||
ls /mnt/test02/appvms/
|
||||
cat /tmp/callback.log (2x pre_volume_start & 2x post_volume_start should be added)
|
||||
qvm-shutdown test-vm
|
||||
cat /tmp/callback.log (2x post_volume_stop should be added)
|
||||
#reboot
|
||||
cat /tmp/callback.log (it should not exist)
|
||||
qvm-start test-vm
|
||||
cat /tmp/callback.log (pre_sinit & 2x pre_volume_start & 2x post_volume_start should be added)
|
||||
qvm-shutdown --wait test-vm && qvm-remove test-vm
|
||||
qvm-pool -r test && sudo rm -rf /mnt/test02
|
||||
less /tmp/callback.log (2x post_volume_stop, 2x post_volume_remove, post_destroy should be added)
|
||||
|
||||
qvm-pool -o conf_id=testing-succ-file-02 -a test callback
|
||||
qvm-create -l red -P test test-dvm
|
||||
qvm-prefs test-dvm template_for_dispvms True
|
||||
qvm-run --dispvm test-dvm xterm
|
||||
grep -E 'test-dvm|disp' /var/lib/qubes/qubes.xml
|
||||
qvm-volume | grep -E 'test-dvm|disp' (unexpected by most users: Qubes OS places only the private volume on the pool, cf. #5933)
|
||||
ls /mnt/test02/appvms/
|
||||
cat /tmp/callback.log
|
||||
#close the disposable VM
|
||||
qvm-remove test-dvm
|
||||
qvm-pool -r test && sudo rm -rf /mnt/test02
|
||||
|
||||
qvm-pool -o conf_id=testing-succ-file-03 -a test callback
|
||||
qvm-pool
|
||||
ls /mnt/test03
|
||||
less /tmp/callback.log (pre_setup should be there, no more arguments)
|
||||
qvm-pool -r test && sudo rm -rf /mnt/test03
|
||||
less /tmp/callback.log (nothing should have been added)
|
||||
|
||||
#luks pool test:
|
||||
#(make sure /mnt/test.key & /mnt/test.luks don't exist)
|
||||
qvm-pool -o conf_id=testing-succ-file-luks -a tluks callback
|
||||
ls /mnt/
|
||||
qvm-pool
|
||||
sudo cryptsetup status test-luks
|
||||
sudo mount | grep test_luks
|
||||
ls /mnt/test_luks/
|
||||
qvm-create -l red -P tluks test-luks (journalctl -b0 should show two pre_volume_create callbacks)
|
||||
ls /mnt/test_luks/appvms/test-luks/
|
||||
qvm-volume | grep test-luks
|
||||
qvm-start test-luks
|
||||
#reboot
|
||||
grep luks /var/lib/qubes/qubes.xml
|
||||
sudo cryptsetup status test-luks (should be inactive due to late pre_sinit!)
|
||||
qvm-start test-luks
|
||||
sudo mount | grep test_luks
|
||||
qvm-shutdown --wait test-luks
|
||||
qvm-remove test-luks
|
||||
qvm-pool -r tluks
|
||||
sudo cryptsetup status test-luks
|
||||
ls -l /mnt/
|
||||
|
||||
#ephemeral luks pool test (key in RAM / lost on reboot):
|
||||
qvm-pool -o conf_id=testing-succ-file-luks-eph -a teph callback (executes setup() twice due to signal_back)
|
||||
ls /mnt/
|
||||
ls /mnt/ram
|
||||
md5sum /mnt/ram/teph.key (1)
|
||||
sudo mount|grep -E 'ram|test'
|
||||
sudo cryptsetup status test-eph
|
||||
qvm-create -l red -P teph test-eph (should execute two pre_volume_create callbacks)
|
||||
qvm-volume | grep test-eph
|
||||
ls /mnt/test_eph/appvms/test-eph/ (should have private.img and volatile.img)
|
||||
ls /var/lib/qubes/appvms/test-eph (should only have the icon)
|
||||
qvm-start test-eph
|
||||
#reboot
|
||||
ls /mnt/ram (should be empty)
|
||||
ls /mnt/
|
||||
sudo mount|grep -E 'ram|test' (should be empty)
|
||||
qvm-ls | grep eph (should still have test-eph)
|
||||
grep eph /var/lib/qubes/qubes.xml (should still have test-eph)
|
||||
qvm-remove test-eph (should create a new encrypted pool backend)
|
||||
sudo cryptsetup status test-eph
|
||||
grep eph /var/lib/qubes/qubes.xml (only the pool should be left)
|
||||
ls /mnt/test_eph/ (should have the appvms directory etc.)
|
||||
qvm-create -l red -P teph test-eph2
|
||||
ls /mnt/test_eph/appvms/
|
||||
ls /mnt/ram
|
||||
qvm-start test-eph2
|
||||
md5sum /mnt/ram/teph.key ((2), different than in (1))
|
||||
qvm-shutdown --wait test-eph2
|
||||
systemctl restart qubesd
|
||||
qvm-start test-eph2 (trigger storage re-init)
|
||||
md5sum /mnt/ram/teph.key (same as in (2))
|
||||
qvm-shutdown --wait test-eph2
|
||||
sudo umount /mnt/test_eph
|
||||
qvm-create -l red -P teph test-eph-fail (must fail with error in journalctl)
|
||||
ls /mnt/test_eph/ (should be empty)
|
||||
systemctl restart qubesd
|
||||
qvm-remove test-eph2
|
||||
qvm-create -l red -P teph test-eph3
|
||||
md5sum /mnt/ram/teph.key (same as in (2))
|
||||
sudo mount|grep -E 'ram|test'
|
||||
ls /mnt/test_eph/appvms/test-eph3
|
||||
qvm-remove test-eph3
|
||||
qvm-ls | grep test-eph
|
||||
qvm-pool -r teph
|
||||
grep eph /var/lib/qubes/qubes.xml (nothing should be left)
|
||||
qvm-pool
|
||||
ls /mnt/
|
||||
ls /mnt/ram/ (should be empty)
|
||||
```
|
||||
'''
|
||||
|
||||
def __init__(self, *, name, conf_id):
|
||||
'''Constructor.
|
||||
:param conf_id: Identifier as found inside the user-controlled configuration at `/etc/qubes_callback.json`.
|
||||
Non-ASCII, non-alphanumeric characters may be disallowed.
|
||||
**Security Note**: Depending on your RPC policy (admin.pool.Add) this constructor and its parameters
|
||||
may be called from an untrusted VM (not by default though). In those cases it may be security-relevant
|
||||
not to pick easily guessable `conf_id` values for your configuration as untrusted VMs may otherwise
|
||||
execute callbacks meant for other pools.
|
||||
:raise StoragePoolException: For user configuration issues.
|
||||
'''
|
||||
#NOTE: attribute names **must** start with `_cb_` unless they are meant to be stored as self._cb_impl attributes
|
||||
self._cb_ctor_done = False #: Boolean to indicate whether or not `__init__` successfully ran through.
|
||||
self._cb_log = logging.getLogger('qubes.storage.callback') #: Logger instance.
|
||||
if not isinstance(conf_id, str):
|
||||
raise qubes.storage.StoragePoolException('conf_id is no String. VM attack?!')
|
||||
self._cb_conf_id = conf_id #: Configuration ID as passed to `__init__()`.
|
||||
|
||||
config_path = '/etc/qubes_callback.json'
|
||||
with open(config_path) as json_file:
|
||||
conf_all = json.load(json_file)
|
||||
if not isinstance(conf_all, dict):
|
||||
raise qubes.storage.StoragePoolException('The file %s is supposed to define a dict.' % config_path)
|
||||
|
||||
try:
|
||||
self._cb_conf = conf_all[self._cb_conf_id] #: Dictionary holding all configuration for the given _cb_conf_id.
|
||||
except KeyError:
|
||||
#we cannot throw KeyErrors as we'll otherwise generate incorrect error messages @qubes.app._get_pool()
|
||||
raise qubes.storage.StoragePoolException('The specified conf_id %s could not be found inside %s.' % (self._cb_conf_id, config_path))
|
||||
|
||||
try:
|
||||
bdriver = self._cb_conf['bdriver']
|
||||
except KeyError:
|
||||
raise qubes.storage.StoragePoolException('Missing bdriver for the conf_id %s inside %s.' % (self._cb_conf_id, config_path))
|
||||
|
||||
self._cb_cmd_arg = json.dumps(self._cb_conf, sort_keys=True, indent=2) #: Full configuration as string in the format required by _callback().
|
||||
|
||||
try:
|
||||
cls = qubes.utils.get_entry_point_one(qubes.storage.STORAGE_ENTRY_POINT, bdriver)
|
||||
except KeyError:
|
||||
raise qubes.storage.StoragePoolException('The driver %s was not found on your system.' % bdriver)
|
||||
|
||||
if not issubclass(cls, qubes.storage.Pool):
|
||||
raise qubes.storage.StoragePoolException('The class %s must be a subclass of qubes.storage.Pool.' % cls)
|
||||
|
||||
self._cb_requires_init = self._check_init() #: Boolean indicating whether late storage initialization yet has to be done or not.
|
||||
self._cb_init_lock = asyncio.Lock() #: Lock ensuring that late storage initialization is only run exactly once.
|
||||
bdriver_args = self._cb_conf.get('bdriver_args', {})
|
||||
self._cb_impl = cls(name=name, **bdriver_args) #: Instance of the backend pool driver.
|
||||
|
||||
super().__init__(name=name, revisions_to_keep=int(bdriver_args.get('revisions_to_keep', 1)))
|
||||
self._cb_ctor_done = True
|
||||
|
||||
def _check_init(self):
|
||||
''' Whether or not this object requires late storage initialization via callback. '''
|
||||
cmd = self._cb_conf.get('pre_sinit')
|
||||
if not cmd:
|
||||
cmd = self._cb_conf.get('cmd')
|
||||
return bool(cmd and cmd != '-')
|
||||
|
||||
@asyncio.coroutine
|
||||
def _init(self, callback=True):
|
||||
''' Late storage initialization on first use for e.g. decryption on first usage request.
|
||||
:param callback: Whether to trigger the `pre_sinit` callback or not.
|
||||
'''
|
||||
with (yield from self._cb_init_lock):
|
||||
if self._cb_requires_init:
|
||||
if callback:
|
||||
yield from self._callback('pre_sinit')
|
||||
self._cb_requires_init = False
|
||||
|
||||
@asyncio.coroutine
|
||||
def _assert_initialized(self, **kwargs):
|
||||
if self._cb_requires_init:
|
||||
yield from self._init(**kwargs)
|
||||
|
||||
@asyncio.coroutine
|
||||
def _callback(self, cb, cb_args=None):
|
||||
'''Run a callback.
|
||||
:param cb: Callback identifier string.
|
||||
:param cb_args: Optional list of arguments to pass to the command as last arguments.
|
||||
Only passed on for the generic command specified as `cmd`, not for `on_xyz` callbacks.
|
||||
:return: Nothing.
|
||||
'''
|
||||
if self._cb_ctor_done:
|
||||
cmd = self._cb_conf.get(cb)
|
||||
args = [] #on_xyz callbacks should never receive arguments
|
||||
if not cmd:
|
||||
if cb_args is None:
|
||||
cb_args = []
|
||||
cmd = self._cb_conf.get('cmd')
|
||||
args = [self.name, self._cb_conf['bdriver'], cb, self._cb_conf_id, self._cb_cmd_arg, *cb_args]
|
||||
if cmd and cmd != '-':
|
||||
args = ' '.join(quote(str(a)) for a in args)
|
||||
cmd = ' '.join(filter(None, [cmd, args]))
|
||||
self._cb_log.info('callback driver executing (%s, %s %s): %s', self._cb_conf_id, cb, cb_args, cmd)
|
||||
cmd_arr = ['/bin/bash', '-c', cmd]
|
||||
proc = yield from asyncio.create_subprocess_exec(*cmd_arr, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
stdout, stderr = yield from proc.communicate()
|
||||
encoding = locale.getpreferredencoding()
|
||||
stdout = stdout.decode(encoding)
|
||||
stderr = stderr.decode(encoding)
|
||||
if proc.returncode != 0:
|
||||
raise subprocess.CalledProcessError(returncode=proc.returncode, cmd=cmd, output=stdout, stderr=stderr)
|
||||
self._cb_log.debug('callback driver stdout (%s, %s %s): %s', self._cb_conf_id, cb, cb_args, stdout)
|
||||
self._cb_log.debug('callback driver stderr (%s, %s %s): %s', self._cb_conf_id, cb, cb_args, stderr)
|
||||
if self._cb_conf.get('signal_back', False) is True:
|
||||
yield from self._process_signals(stdout)
|
||||
|
||||
@asyncio.coroutine
|
||||
def _process_signals(self, out):
|
||||
'''Process any signals found inside a string.
|
||||
:param out: String to check for signals. Each signal must be on a dedicated line.
|
||||
They are executed in the order they are found. Callbacks are not triggered.
|
||||
'''
|
||||
for line in out.splitlines():
|
||||
if line == 'SIGNAL_setup':
|
||||
self._cb_log.info('callback driver processing SIGNAL_setup for %s', self._cb_conf_id)
|
||||
#NOTE: calling our own methods may lead to a deadlock / qubesd freeze due to `self._assert_initialized()` / `self._cb_init_lock`
|
||||
yield from coro_maybe(self._cb_impl.setup())
|
||||
|
||||
@property
|
||||
def backend_class(self):
|
||||
'''Class of the first non-CallbackPool backend Pool.'''
|
||||
if isinstance(self._cb_impl, CallbackPool):
|
||||
return self._cb_impl.backend_class
|
||||
return self._cb_impl.__class__
|
||||
|
||||
@property
|
||||
def config(self):
|
||||
return {
|
||||
'name': self.name,
|
||||
'driver': 'callback',
|
||||
'conf_id': self._cb_conf_id,
|
||||
}
|
||||
|
||||
@asyncio.coroutine
|
||||
def destroy(self):
|
||||
yield from self._assert_initialized()
|
||||
ret = yield from coro_maybe(self._cb_impl.destroy())
|
||||
yield from self._callback('post_destroy')
|
||||
return ret
|
||||
|
||||
def init_volume(self, vm, volume_config):
|
||||
ret = CallbackVolume(self, self._cb_impl.init_volume(vm, volume_config))
|
||||
volume_config['pool'] = self
|
||||
return ret
|
||||
|
||||
@asyncio.coroutine
|
||||
def setup(self):
|
||||
yield from self._assert_initialized(callback=False) #setup is assumed to include storage initialization
|
||||
yield from self._callback('pre_setup')
|
||||
return (yield from coro_maybe(self._cb_impl.setup()))
|
||||
|
||||
@property
|
||||
def volumes(self):
|
||||
for vol in self._cb_impl.volumes:
|
||||
yield CallbackVolume(self, vol)
|
||||
|
||||
def list_volumes(self):
|
||||
for vol in self._cb_impl.list_volumes():
|
||||
yield CallbackVolume(self, vol)
|
||||
|
||||
def get_volume(self, vid):
|
||||
return CallbackVolume(self, self._cb_impl.get_volume(vid))
|
||||
|
||||
def included_in(self, app):
|
||||
if self._cb_requires_init:
|
||||
return None
|
||||
return self._cb_impl.included_in(app)
|
||||
|
||||
@property
|
||||
def size(self):
|
||||
if self._cb_requires_init:
|
||||
return None
|
||||
return self._cb_impl.size
|
||||
|
||||
@property
|
||||
def usage(self):
|
||||
if self._cb_requires_init:
|
||||
return None
|
||||
return self._cb_impl.usage
|
||||
|
||||
@property
|
||||
def usage_details(self):
|
||||
if self._cb_requires_init:
|
||||
return {}
|
||||
return self._cb_impl.usage_details
|
||||
|
||||
#shadow all qubes.storage.Pool class attributes as instance properties
|
||||
#NOTE: this will cause a subtle difference to using an actual _cb_impl instance: CallbackPool.private_img_size will return a property object, Pool.private_img_size the actual value
|
||||
@property
|
||||
def private_img_size(self):
|
||||
return self._cb_impl.private_img_size
|
||||
|
||||
@private_img_size.setter
|
||||
def private_img_size(self, private_img_size):
|
||||
self._cb_impl.private_img_size = private_img_size
|
||||
|
||||
@property
|
||||
def root_img_size(self):
|
||||
return self._cb_impl.root_img_size
|
||||
|
||||
@root_img_size.setter
|
||||
def root_img_size(self, root_img_size):
|
||||
self._cb_impl.root_img_size = root_img_size
|
||||
|
||||
#remaining method & attribute delegation ("delegation pattern")
|
||||
#Convention: The methods of this object have priority over the delegated object's methods. All attributes are
|
||||
# passed to the delegated object unless their name starts with '_cb_'.
|
||||
def __getattr__(self, name):
|
||||
#NOTE: This method is only called when an attribute cannot be resolved locally (not part of the instance,
|
||||
# not part of the class tree). It is also called for methods that cannot be resolved.
|
||||
return getattr(self._cb_impl, name)
|
||||
|
||||
def __setattr__(self, name, value):
|
||||
#NOTE: This method is called on every attribute assignment.
|
||||
if name.startswith('_cb_'):
|
||||
super().__setattr__(name, value)
|
||||
else:
|
||||
setattr(self._cb_impl, name, value)
|
||||
|
||||
def __delattr__(self, name):
|
||||
if name.startswith('_cb_'):
|
||||
super().__delattr__(name)
|
||||
else:
|
||||
delattr(self._cb_impl, name)
|
||||
|
||||
class CallbackVolume(qubes.storage.Volume):
|
||||
''' Proxy volume adding callback functionality to other volumes.
|
||||
|
||||
Required to support the `pre_sinit` and other callbacks.
|
||||
'''
|
||||
|
||||
def __init__(self, pool, impl):
|
||||
'''Constructor.
|
||||
:param pool: `CallbackPool` of this volume
|
||||
:param impl: `qubes.storage.Volume` object to wrap
|
||||
'''
|
||||
# pylint: disable=super-init-not-called
|
||||
#NOTE: we must *not* call super().__init__() as it would prevent attribute delegation
|
||||
assert isinstance(impl, qubes.storage.Volume), 'impl must be a qubes.storage.Volume instance. Found a %s instance.' % impl.__class__
|
||||
assert isinstance(pool, CallbackPool), 'pool must use a qubes.storage.CallbackPool instance. Found a %s instance.' % pool.__class__
|
||||
impl.pool = pool #enforce the CallbackPool instance as the parent pool of the volume
|
||||
self._cb_pool = pool #: CallbackPool instance the Volume belongs to.
|
||||
self._cb_impl = impl #: Backend volume implementation instance.
|
||||
|
||||
@asyncio.coroutine
|
||||
def _assert_initialized(self, **kwargs):
|
||||
yield from self._cb_pool._assert_initialized(**kwargs) # pylint: disable=protected-access
|
||||
|
||||
@asyncio.coroutine
|
||||
def _callback(self, cb, cb_args=None, **kwargs):
|
||||
if cb_args is None:
|
||||
cb_args = []
|
||||
vol_args = [self.name, self.vid, self.source, *cb_args]
|
||||
yield from self._cb_pool._callback(cb, cb_args=vol_args, **kwargs) # pylint: disable=protected-access
|
||||
|
||||
@property
|
||||
def backend_class(self):
|
||||
'''Class of the first non-CallbackVolume backend Volume.'''
|
||||
if isinstance(self._cb_impl, CallbackVolume):
|
||||
return self._cb_impl.backend_class
|
||||
return self._cb_impl.__class__
|
||||
|
||||
@asyncio.coroutine
|
||||
def create(self):
|
||||
yield from self._assert_initialized()
|
||||
yield from self._callback('pre_volume_create')
|
||||
ret = yield from coro_maybe(self._cb_impl.create())
|
||||
yield from self._callback('post_volume_create')
|
||||
return ret
|
||||
|
||||
@asyncio.coroutine
|
||||
def remove(self):
|
||||
yield from self._assert_initialized()
|
||||
ret = yield from coro_maybe(self._cb_impl.remove())
|
||||
yield from self._callback('post_volume_remove')
|
||||
return ret
|
||||
|
||||
@asyncio.coroutine
|
||||
def resize(self, size):
|
||||
yield from self._assert_initialized()
|
||||
yield from self._callback('pre_volume_resize', cb_args=[size])
|
||||
return (yield from coro_maybe(self._cb_impl.resize(size)))
|
||||
|
||||
@asyncio.coroutine
|
||||
def start(self):
|
||||
yield from self._assert_initialized()
|
||||
yield from self._callback('pre_volume_start')
|
||||
ret = yield from coro_maybe(self._cb_impl.start())
|
||||
yield from self._callback('post_volume_start')
|
||||
return ret
|
||||
|
||||
@asyncio.coroutine
|
||||
def stop(self):
|
||||
yield from self._assert_initialized()
|
||||
ret = yield from coro_maybe(self._cb_impl.stop())
|
||||
yield from self._callback('post_volume_stop')
|
||||
return ret
|
||||
|
||||
@asyncio.coroutine
|
||||
def import_data(self, size):
|
||||
yield from self._assert_initialized()
|
||||
yield from self._callback('pre_volume_import_data', cb_args=[size])
|
||||
return (yield from coro_maybe(self._cb_impl.import_data(size)))
|
||||
|
||||
@asyncio.coroutine
|
||||
def import_data_end(self, success):
|
||||
yield from self._assert_initialized()
|
||||
ret = yield from coro_maybe(self._cb_impl.import_data_end(success))
|
||||
yield from self._callback('post_volume_import_data_end', cb_args=[success])
|
||||
return ret
|
||||
|
||||
@asyncio.coroutine
|
||||
def import_volume(self, src_volume):
|
||||
yield from self._assert_initialized()
|
||||
yield from self._callback('pre_volume_import', cb_args=[src_volume.vid])
|
||||
ret = yield from coro_maybe(self._cb_impl.import_volume(src_volume))
|
||||
yield from self._callback('post_volume_import', cb_args=[src_volume.vid])
|
||||
return ret
|
||||
|
||||
def is_dirty(self):
|
||||
# pylint: disable=protected-access
|
||||
if self._cb_pool._cb_requires_init:
|
||||
return False
|
||||
return self._cb_impl.is_dirty()
|
||||
|
||||
def is_outdated(self):
|
||||
# pylint: disable=protected-access
|
||||
if self._cb_pool._cb_requires_init:
|
||||
return False
|
||||
return self._cb_impl.is_outdated()
|
||||
|
||||
@property
|
||||
def revisions(self):
|
||||
return self._cb_impl.revisions
|
||||
|
||||
@property
|
||||
def size(self):
|
||||
return self._cb_impl.size
|
||||
|
||||
@size.setter
|
||||
def size(self, size):
|
||||
self._cb_impl.size = size
|
||||
|
||||
@property
|
||||
def config(self):
|
||||
return self._cb_impl.config
|
||||
|
||||
def block_device(self):
|
||||
# pylint: disable=protected-access
|
||||
if self._cb_pool._cb_requires_init:
|
||||
# usually Volume.start() is called beforehand
|
||||
# --> we should be initialized in 99% of cases
|
||||
return None
|
||||
return self._cb_impl.block_device()
|
||||
|
||||
@asyncio.coroutine
|
||||
def export(self):
|
||||
yield from self._assert_initialized()
|
||||
yield from self._callback('pre_volume_export')
|
||||
return (yield from coro_maybe(self._cb_impl.export()))
|
||||
|
||||
@asyncio.coroutine
|
||||
def export_end(self, path):
|
||||
yield from self._assert_initialized()
|
||||
ret = yield from coro_maybe(self._cb_impl.export_end(path))
|
||||
yield from self._callback('post_volume_export_end', cb_args=[path])
|
||||
return ret
|
||||
|
||||
@asyncio.coroutine
|
||||
def verify(self):
|
||||
yield from self._assert_initialized()
|
||||
return (yield from coro_maybe(self._cb_impl.verify()))
|
||||
|
||||
@asyncio.coroutine
|
||||
def revert(self, revision=None):
|
||||
yield from self._assert_initialized()
|
||||
return (yield from coro_maybe(self._cb_impl.revert(revision=revision)))
|
||||
|
||||
#shadow all qubes.storage.Volume class attributes as instance properties
|
||||
#NOTE: this will cause a subtle difference to using an actual _cb_impl instance: CallbackVolume.devtype will return a property object, Volume.devtype the actual value
|
||||
@property
|
||||
def devtype(self):
|
||||
return self._cb_impl.devtype
|
||||
|
||||
@devtype.setter
|
||||
def devtype(self, devtype):
|
||||
self._cb_impl.devtype = devtype
|
||||
|
||||
@property
|
||||
def domain(self):
|
||||
return self._cb_impl.domain
|
||||
|
||||
@domain.setter
|
||||
def domain(self, domain):
|
||||
self._cb_impl.domain = domain
|
||||
|
||||
@property
|
||||
def path(self):
|
||||
return self._cb_impl.path
|
||||
|
||||
@path.setter
|
||||
def path(self, path):
|
||||
self._cb_impl.path = path
|
||||
|
||||
@property
|
||||
def script(self):
|
||||
return self._cb_impl.script
|
||||
|
||||
@script.setter
|
||||
def script(self, script):
|
||||
self._cb_impl.script = script
|
||||
|
||||
@property
|
||||
def usage(self):
|
||||
return self._cb_impl.usage
|
||||
|
||||
@usage.setter
|
||||
def usage(self, usage):
|
||||
self._cb_impl.usage = usage
|
||||
|
||||
#remaining method & attribute delegation
|
||||
def __getattr__(self, name):
|
||||
return getattr(self._cb_impl, name)
|
||||
|
||||
def __setattr__(self, name, value):
|
||||
if name.startswith('_cb_'):
|
||||
super().__setattr__(name, value)
|
||||
else:
|
||||
setattr(self._cb_impl, name, value)
|
||||
|
||||
def __delattr__(self, name):
|
||||
if name.startswith('_cb_'):
|
||||
super().__delattr__(name)
|
||||
else:
|
||||
delattr(self._cb_impl, name)
|
||||
@ -509,7 +509,7 @@ class ThinVolume(qubes.storage.Volume):
|
||||
# HACK: neat trick to speed up testing if you have same physical thin
|
||||
# pool assigned to two qubes-pools i.e: qubes_dom0 and test-lvm
|
||||
# pylint: disable=line-too-long
|
||||
if isinstance(src_volume.pool, ThinPool) and \
|
||||
if hasattr(src_volume.pool, 'thin_pool') and \
|
||||
src_volume.pool.thin_pool == self.pool.thin_pool: # NOQA
|
||||
yield from self._commit(src_volume.path[len('/dev/'):], keep=True)
|
||||
else:
|
||||
|
||||
@ -1412,6 +1412,7 @@ def load_tests(loader, tests, pattern): # pylint: disable=unused-argument
|
||||
'qubes.tests.storage_file',
|
||||
'qubes.tests.storage_reflink',
|
||||
'qubes.tests.storage_lvm',
|
||||
'qubes.tests.storage_callback',
|
||||
'qubes.tests.storage_kernels',
|
||||
'qubes.tests.ext',
|
||||
'qubes.tests.vm.qubesvm',
|
||||
|
||||
@ -116,7 +116,7 @@ class TC_00_Pool(QubesTestCase):
|
||||
def test_001_all_pool_drivers(self):
|
||||
""" Expect all our pool drivers (and only them) """
|
||||
self.assertCountEqual(
|
||||
['linux-kernel', 'lvm_thin', 'file', 'file-reflink'],
|
||||
['linux-kernel', 'lvm_thin', 'file', 'file-reflink', 'callback'],
|
||||
pool_drivers())
|
||||
|
||||
def test_002_get_pool_klass(self):
|
||||
|
||||
378
qubes/tests/storage_callback.py
Normal file
378
qubes/tests/storage_callback.py
Normal file
@ -0,0 +1,378 @@
|
||||
#
|
||||
# The Qubes OS Project, http://www.qubes-os.org
|
||||
#
|
||||
# Copyright (C) 2020 David Hobach <david@hobach.de>
|
||||
#
|
||||
# This library is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU Lesser General Public
|
||||
# License as published by the Free Software Foundation; either
|
||||
# version 2.1 of the License, or (at your option) any later version.
|
||||
#
|
||||
# This library is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public
|
||||
# License along with this library; if not, see <https://www.gnu.org/licenses/>.
|
||||
#
|
||||
''' Tests for the callback storage driver.
|
||||
|
||||
They are mostly based upon the lvm storage driver tests.
|
||||
'''
|
||||
# pylint: disable=line-too-long
|
||||
|
||||
import os
|
||||
import json
|
||||
import subprocess
|
||||
import qubes.tests
|
||||
import qubes.tests.storage
|
||||
import qubes.tests.storage_lvm
|
||||
from qubes.tests.storage_lvm import skipUnlessLvmPoolExists
|
||||
from qubes.storage.callback import CallbackPool, CallbackVolume
|
||||
|
||||
CB_CONF = '/etc/qubes_callback.json'
|
||||
LOG_BIN = '/tmp/testCbLogArgs'
|
||||
|
||||
CB_DATA = {'utest-callback-01': {
|
||||
'bdriver': 'lvm_thin',
|
||||
'bdriver_args': {
|
||||
'volume_group': qubes.tests.storage_lvm.DEFAULT_LVM_POOL.split('/')[0],
|
||||
'thin_pool': qubes.tests.storage_lvm.DEFAULT_LVM_POOL.split('/')[1]
|
||||
},
|
||||
'description': 'For unit testing of the callback pool driver.'
|
||||
},
|
||||
'utest-callback-02': {
|
||||
'bdriver': 'lvm_thin',
|
||||
'bdriver_args': {
|
||||
'volume_group': qubes.tests.storage_lvm.DEFAULT_LVM_POOL.split('/')[0],
|
||||
'thin_pool': qubes.tests.storage_lvm.DEFAULT_LVM_POOL.split('/')[1]
|
||||
},
|
||||
'cmd': LOG_BIN,
|
||||
'description': 'For unit testing of the callback pool driver.'
|
||||
},
|
||||
'utest-callback-03': {
|
||||
'bdriver': 'lvm_thin',
|
||||
'bdriver_args': {
|
||||
'volume_group': qubes.tests.storage_lvm.DEFAULT_LVM_POOL.split('/')[0],
|
||||
'thin_pool': qubes.tests.storage_lvm.DEFAULT_LVM_POOL.split('/')[1]
|
||||
},
|
||||
'cmd': 'exit 1',
|
||||
'pre_sinit': LOG_BIN + ' pre_sinit',
|
||||
'pre_setup': LOG_BIN + ' pre_setup',
|
||||
'pre_volume_create': LOG_BIN + ' pre_volume_create',
|
||||
'post_volume_create': LOG_BIN + ' post_volume_create',
|
||||
'pre_volume_import_data': LOG_BIN + ' pre_volume_import_data',
|
||||
'post_volume_import_data_end': LOG_BIN + ' post_volume_import_data_end',
|
||||
'post_volume_remove': LOG_BIN + ' post_volume_remove',
|
||||
'post_destroy': '-',
|
||||
'description': 'For unit testing of the callback pool driver.'
|
||||
},
|
||||
'testing-fail-missing-all': {
|
||||
},
|
||||
'testing-fail-missing-bdriver-args': {
|
||||
'bdriver': 'file',
|
||||
'description': 'For unit testing of the callback pool driver.'
|
||||
},
|
||||
'testing-fail-incorrect-bdriver': {
|
||||
'bdriver': 'nonexisting-bdriver',
|
||||
'bdriver_args': {
|
||||
'foo': 'bar',
|
||||
'bla': 'blub'
|
||||
},
|
||||
'cmd': 'echo foo',
|
||||
'description': 'For unit testing of the callback pool driver.'
|
||||
},
|
||||
}
|
||||
|
||||
class CallbackBase:
|
||||
''' Mixin base class for callback tests. Has no base class. '''
|
||||
conf_id = None
|
||||
pool_name = 'test-callback'
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls, conf_id='utest-callback-01'):
|
||||
conf = {'name': cls.pool_name,
|
||||
'driver': 'callback',
|
||||
'conf_id': conf_id}
|
||||
cls.conf_id = conf_id
|
||||
|
||||
assert not(os.path.exists(CB_CONF)), '%s must NOT exist. Please delete it, if you do not need it.' % CB_CONF
|
||||
|
||||
sudo = [] if os.getuid() == 0 else ['sudo']
|
||||
subprocess.run(sudo + ['install', '-m', '666', '/dev/null', CB_CONF], check=True)
|
||||
|
||||
with open(CB_CONF, 'w') as outfile:
|
||||
json.dump(CB_DATA, outfile)
|
||||
super().setUpClass(pool_class=CallbackPool, volume_class=CallbackVolume, pool_conf=conf)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
super().tearDownClass()
|
||||
|
||||
sudo = [] if os.getuid() == 0 else ['sudo']
|
||||
subprocess.run(sudo + ['rm', '-f', CB_CONF], check=True)
|
||||
|
||||
def setUp(self, init_pool=True):
|
||||
super().setUp(init_pool=init_pool)
|
||||
if init_pool:
|
||||
#tests from other pools will assume that they're fully initialized after calling __init__()
|
||||
self.loop.run_until_complete(self.pool._assert_initialized())
|
||||
|
||||
def test_000_000_callback_test_init(self):
|
||||
''' Check whether the test init did work. '''
|
||||
if hasattr(self, 'pool'):
|
||||
self.assertIsInstance(self.pool, CallbackPool)
|
||||
self.assertEqual(self.pool.backend_class, qubes.storage.lvm.ThinPool)
|
||||
self.assertTrue(os.path.isfile(CB_CONF))
|
||||
|
||||
@skipUnlessLvmPoolExists
|
||||
class TC_00_CallbackPool(CallbackBase, qubes.tests.storage_lvm.TC_00_ThinPool):
|
||||
pass
|
||||
|
||||
@skipUnlessLvmPoolExists
|
||||
class TC_01_CallbackPool(CallbackBase, qubes.tests.storage_lvm.TC_01_ThinPool):
|
||||
pass
|
||||
|
||||
@skipUnlessLvmPoolExists
|
||||
class TC_02_cb_StorageHelpers(CallbackBase, qubes.tests.storage_lvm.TC_02_StorageHelpers):
|
||||
pass
|
||||
|
||||
class LoggingCallbackBase(CallbackBase):
|
||||
''' Mixin base class that sets up LOG_BIN and removes `LoggingCallbackBase.test_log`, if needed. '''
|
||||
test_log = '/tmp/cb_tests.log'
|
||||
test_log_expected = None #dict: class + test name --> test index (int, 0..x) --> expected _additional_ log content
|
||||
volume_name = 'volume_name'
|
||||
xml_path = '/tmp/qubes-test-callback.xml'
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls, conf_id=None, log_expected=None):
|
||||
script = """#!/bin/bash
|
||||
i=1
|
||||
for arg in "$@" ; do
|
||||
echo "$i: $arg" >> "LOG_OUT"
|
||||
(( i++))
|
||||
done
|
||||
exit 0
|
||||
"""
|
||||
script = script.replace('LOG_OUT', cls.test_log)
|
||||
with open(LOG_BIN, 'w') as f:
|
||||
f.write(script)
|
||||
os.chmod(LOG_BIN, 0o775)
|
||||
|
||||
cls.test_log_expected = log_expected
|
||||
super().setUpClass(conf_id=conf_id)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
super().tearDownClass()
|
||||
os.remove(LOG_BIN)
|
||||
|
||||
def setUp(self, init_pool=False):
|
||||
assert not(os.path.exists(self.test_log)), '%s must NOT exist. Please delete it, if you do not need it.' % self.test_log
|
||||
self.maxDiff = None
|
||||
|
||||
xml = """
|
||||
<qubes>
|
||||
<labels>
|
||||
<label color="0x000000" id="label-8">black</label>
|
||||
</labels>
|
||||
<pools>
|
||||
<pool dir_path="/var/lib/qubes" driver="file" name="varlibqubes" revisions_to_keep="1"/>
|
||||
<pool dir_path="/var/lib/qubes/vm-kernels" driver="linux-kernel" name="linux-kernel"/>
|
||||
<pool conf_id="CONF_ID" driver="callback" name="POOL_NAME"/>
|
||||
</pools>
|
||||
<properties>
|
||||
<property name="clockvm"></property>
|
||||
<property name="default_pool_kernel">linux-kernel</property>
|
||||
<property name="default_template"></property>
|
||||
<property name="updatevm"></property>
|
||||
</properties>
|
||||
<domains>
|
||||
<domain id="domain-0" class="AdminVM">
|
||||
<properties>
|
||||
<property name="label">black</property>
|
||||
</properties>
|
||||
<features/>
|
||||
<tags/>
|
||||
</domain>
|
||||
</domains>
|
||||
</qubes>
|
||||
"""
|
||||
xml = xml.replace('CONF_ID', self.conf_id)
|
||||
xml = xml.replace('POOL_NAME', self.pool_name)
|
||||
with open(self.xml_path, 'w') as f:
|
||||
f.write(xml)
|
||||
self.app = qubes.Qubes(self.xml_path,
|
||||
clockvm=None,
|
||||
updatevm=None,
|
||||
offline_mode=True,
|
||||
)
|
||||
os.environ['QUBES_XML_PATH'] = self.xml_path
|
||||
super().setUp(init_pool=init_pool)
|
||||
|
||||
def tearDown(self):
|
||||
super().tearDown()
|
||||
os.unlink(self.app.store)
|
||||
self.app.close()
|
||||
del self.app
|
||||
for attr in dir(self):
|
||||
if isinstance(getattr(self, attr), qubes.vm.BaseVM):
|
||||
delattr(self, attr)
|
||||
|
||||
if os.path.exists(self.test_log):
|
||||
os.remove(self.test_log)
|
||||
|
||||
if os.path.exists(self.xml_path):
|
||||
os.remove(self.xml_path)
|
||||
|
||||
def assertLogContent(self, expected):
|
||||
''' Assert that the log matches the given string.
|
||||
:param expected: Expected content of the log file (String).
|
||||
'''
|
||||
try:
|
||||
with open(self.test_log, 'r') as f:
|
||||
found = f.read()
|
||||
except FileNotFoundError:
|
||||
found = ''
|
||||
if expected != '':
|
||||
expected = expected + '\n'
|
||||
self.assertEqual(found, expected)
|
||||
|
||||
def assertLog(self, test_name, ind=0):
|
||||
''' Assert that the log matches the expected status.
|
||||
:param test_name: Name of the test.
|
||||
:param ind: Index inside `test_log_expected` to check against (Integer starting at 0).
|
||||
'''
|
||||
d = self.test_log_expected[str(self.__class__) + test_name]
|
||||
expected = []
|
||||
for i in range(ind+1):
|
||||
expected = expected + [d[i]]
|
||||
expected = filter(None, expected)
|
||||
self.assertLogContent('\n'.join(expected))
|
||||
|
||||
def test_001_callbacks(self):
|
||||
''' create a lvm pool with additional callbacks '''
|
||||
config = {
|
||||
'name': self.volume_name,
|
||||
'pool': self.pool_name,
|
||||
'save_on_stop': True,
|
||||
'rw': True,
|
||||
'revisions_to_keep': 2,
|
||||
'size': qubes.config.defaults['root_img_size'],
|
||||
}
|
||||
new_size = 2 * qubes.config.defaults['root_img_size']
|
||||
|
||||
test_name = 'test_001_callbacks'
|
||||
self.assertLog(test_name, 0)
|
||||
self.init_pool()
|
||||
self.assertFalse(self.created_pool)
|
||||
self.assertIsInstance(self.pool, CallbackPool)
|
||||
self.assertLog(test_name, 1)
|
||||
vm = qubes.tests.storage.TestVM(self)
|
||||
volume = self.app.get_pool(self.pool.name).init_volume(vm, config)
|
||||
self.assertLog(test_name, 2)
|
||||
self.loop.run_until_complete(volume.create())
|
||||
self.assertLog(test_name, 3)
|
||||
self.loop.run_until_complete(volume.import_data(new_size))
|
||||
self.assertLog(test_name, 4)
|
||||
self.loop.run_until_complete(volume.import_data_end(True))
|
||||
self.assertLog(test_name, 5)
|
||||
self.assertEqual(volume.size, new_size)
|
||||
self.loop.run_until_complete(volume.remove())
|
||||
self.assertLog(test_name, 6)
|
||||
|
||||
@skipUnlessLvmPoolExists
|
||||
class TC_91_CallbackPool(LoggingCallbackBase, qubes.tests.storage_lvm.ThinPoolBase):
|
||||
''' Tests for the actual callback functionality.
|
||||
conf_id = utest-callback-02
|
||||
'''
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
conf_id = 'utest-callback-02'
|
||||
name = cls.pool_name
|
||||
bdriver = (CB_DATA[conf_id])['bdriver']
|
||||
ctor_params = json.dumps(CB_DATA[conf_id], sort_keys=True, indent=2)
|
||||
vname = cls.volume_name
|
||||
vid = '{0}/vm-test-inst-appvm-{1}'.format(qubes.tests.storage_lvm.DEFAULT_LVM_POOL.split('/')[0], vname)
|
||||
vsize = 2 * qubes.config.defaults['root_img_size']
|
||||
log_expected = \
|
||||
{str(cls) + 'test_001_callbacks':
|
||||
{0: '',
|
||||
1: '',
|
||||
2: '',
|
||||
3: '1: {0}\n2: {1}\n3: pre_sinit\n4: {2}\n5: {3}\n1: {0}\n2: {1}\n3: pre_volume_create\n4: {2}\n5: {3}\n6: {4}\n7: {5}\n8: None\n1: {0}\n2: {1}\n3: post_volume_create\n4: {2}\n5: {3}\n6: {4}\n7: {5}\n8: None'.format(name, bdriver, conf_id, ctor_params, vname, vid),
|
||||
4: '1: {0}\n2: {1}\n3: pre_volume_import_data\n4: {2}\n5: {3}\n6: {4}\n7: {5}\n8: None\n9: {6}'.format(name, bdriver, conf_id, ctor_params, vname, vid, vsize),
|
||||
5: '1: {0}\n2: {1}\n3: post_volume_import_data_end\n4: {2}\n5: {3}\n6: {4}\n7: {5}\n8: None\n9: {6}'.format(name, bdriver, conf_id, ctor_params, vname, vid, True),
|
||||
6: '1: {0}\n2: {1}\n3: post_volume_remove\n4: {2}\n5: {3}\n6: {4}\n7: {5}\n8: None'.format(name, bdriver, conf_id, ctor_params, vname, vid),
|
||||
}
|
||||
}
|
||||
super().setUpClass(conf_id=conf_id, log_expected=log_expected)
|
||||
|
||||
@skipUnlessLvmPoolExists
|
||||
class TC_92_CallbackPool(LoggingCallbackBase, qubes.tests.storage_lvm.ThinPoolBase):
|
||||
''' Tests for the actual callback functionality.
|
||||
conf_id = utest-callback-03
|
||||
'''
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
log_expected = \
|
||||
{str(cls) + 'test_001_callbacks':
|
||||
{0: '',
|
||||
1: '',
|
||||
2: '',
|
||||
3: '1: pre_sinit\n1: pre_volume_create\n1: post_volume_create',
|
||||
4: '1: pre_volume_import_data',
|
||||
5: '1: post_volume_import_data_end',
|
||||
6: '1: post_volume_remove',
|
||||
}
|
||||
}
|
||||
super().setUpClass(conf_id='utest-callback-03', log_expected=log_expected)
|
||||
|
||||
def test_002_failing_callback(self):
|
||||
''' Make sure that we check the exit code of executed callbacks. '''
|
||||
config = {
|
||||
'name': self.volume_name,
|
||||
'pool': self.pool_name,
|
||||
'save_on_stop': True,
|
||||
'rw': True,
|
||||
'revisions_to_keep': 2,
|
||||
'size': qubes.config.defaults['root_img_size'],
|
||||
}
|
||||
self.init_pool()
|
||||
vm = qubes.tests.storage.TestVM(self)
|
||||
volume = self.app.get_pool(self.pool.name).init_volume(vm, config)
|
||||
with self.assertRaises(subprocess.CalledProcessError) as cm:
|
||||
#should trigger the `exit 1` of `cmd`
|
||||
self.loop.run_until_complete(volume.start())
|
||||
self.assertTrue('exit status 1' in str(cm.exception))
|
||||
|
||||
def test_003_errors(self):
|
||||
''' Make sure we error out on common user & dev mistakes. '''
|
||||
#missing conf_id
|
||||
with self.assertRaises(qubes.storage.StoragePoolException):
|
||||
cb = CallbackPool(name='some-name', conf_id='')
|
||||
|
||||
#invalid conf_id
|
||||
with self.assertRaises(qubes.storage.StoragePoolException):
|
||||
cb = CallbackPool(name='some-name', conf_id='nonexisting-id')
|
||||
|
||||
#incorrect backend driver
|
||||
with self.assertRaises(qubes.storage.StoragePoolException):
|
||||
cb = CallbackPool(name='some-name', conf_id='testing-fail-incorrect-bdriver')
|
||||
|
||||
#missing config entries
|
||||
with self.assertRaises(qubes.storage.StoragePoolException):
|
||||
cb = CallbackPool(name='some-name', conf_id='testing-fail-missing-all')
|
||||
|
||||
#missing bdriver args
|
||||
with self.assertRaises(TypeError):
|
||||
cb = CallbackPool(name='some-name', conf_id='testing-fail-missing-bdriver-args')
|
||||
|
||||
class TC_93_CallbackPool(qubes.tests.QubesTestCase):
|
||||
def test_001_missing_conf(self):
|
||||
''' A missing config file must cause errors. '''
|
||||
with self.assertRaises(FileNotFoundError):
|
||||
cb = CallbackPool(name='some-name', conf_id='nonexisting-id')
|
||||
@ -68,16 +68,26 @@ POOL_CONF = {'name': 'test-lvm',
|
||||
class ThinPoolBase(qubes.tests.QubesTestCase):
|
||||
''' Sanity tests for :py:class:`qubes.storage.lvm.ThinPool` '''
|
||||
|
||||
pool_class = None
|
||||
volume_class = None
|
||||
pool_conf = None
|
||||
created_pool = False
|
||||
|
||||
def setUp(self):
|
||||
@classmethod
|
||||
def setUpClass(cls, pool_class=ThinPool, volume_class=ThinVolume, pool_conf=None):
|
||||
''' Other test classes (e.g. callback) may use this to test their own config. '''
|
||||
conf = pool_conf
|
||||
if not conf:
|
||||
conf = POOL_CONF
|
||||
|
||||
cls.pool_class = pool_class
|
||||
cls.volume_class = volume_class
|
||||
cls.pool_conf = conf
|
||||
|
||||
def setUp(self, init_pool=True):
|
||||
super(ThinPoolBase, self).setUp()
|
||||
volume_group, thin_pool = DEFAULT_LVM_POOL.split('/', 1)
|
||||
self.pool = self._find_pool(volume_group, thin_pool)
|
||||
if not self.pool:
|
||||
self.pool = self.loop.run_until_complete(
|
||||
self.app.add_pool(**POOL_CONF))
|
||||
self.created_pool = True
|
||||
if init_pool:
|
||||
self.init_pool()
|
||||
|
||||
def cleanup_test_volumes(self):
|
||||
p = self.loop.run_until_complete(asyncio.create_subprocess_exec(
|
||||
@ -96,18 +106,26 @@ class ThinPoolBase(qubes.tests.QubesTestCase):
|
||||
|
||||
def tearDown(self):
|
||||
''' Remove the default lvm pool if it was created only for this test '''
|
||||
self.cleanup_test_volumes()
|
||||
if hasattr(self, 'pool'):
|
||||
self.cleanup_test_volumes()
|
||||
if self.created_pool:
|
||||
self.loop.run_until_complete(self.app.remove_pool(self.pool.name))
|
||||
super(ThinPoolBase, self).tearDown()
|
||||
|
||||
def init_pool(self):
|
||||
volume_group, thin_pool = DEFAULT_LVM_POOL.split('/', 1)
|
||||
self.pool = self._find_pool(volume_group, thin_pool)
|
||||
if not self.pool:
|
||||
self.pool = self.loop.run_until_complete(
|
||||
self.app.add_pool(**self.pool_conf))
|
||||
self.created_pool = True
|
||||
|
||||
def _find_pool(self, volume_group, thin_pool):
|
||||
''' Returns the pool matching the specified ``volume_group`` &
|
||||
``thin_pool``, or None.
|
||||
'''
|
||||
pools = [p for p in self.app.pools.values()
|
||||
if issubclass(p.__class__, ThinPool)]
|
||||
if issubclass(p.__class__, self.pool_class)]
|
||||
for pool in pools:
|
||||
if pool.volume_group == volume_group \
|
||||
and pool.thin_pool == thin_pool:
|
||||
@ -118,7 +136,7 @@ class ThinPoolBase(qubes.tests.QubesTestCase):
|
||||
class TC_00_ThinPool(ThinPoolBase):
|
||||
''' Sanity tests for :py:class:`qubes.storage.lvm.ThinPool` '''
|
||||
|
||||
def setUp(self):
|
||||
def setUp(self, **kwargs):
|
||||
xml_path = '/tmp/qubes-test.xml'
|
||||
self.app = qubes.Qubes.create_empty_store(store=xml_path,
|
||||
clockvm=None,
|
||||
@ -126,7 +144,7 @@ class TC_00_ThinPool(ThinPoolBase):
|
||||
offline_mode=True,
|
||||
)
|
||||
os.environ['QUBES_XML_PATH'] = xml_path
|
||||
super(TC_00_ThinPool, self).setUp()
|
||||
super().setUp(**kwargs)
|
||||
|
||||
def tearDown(self):
|
||||
super(TC_00_ThinPool, self).tearDown()
|
||||
@ -155,7 +173,7 @@ class TC_00_ThinPool(ThinPoolBase):
|
||||
}
|
||||
vm = qubes.tests.storage.TestVM(self)
|
||||
volume = self.app.get_pool(self.pool.name).init_volume(vm, config)
|
||||
self.assertIsInstance(volume, ThinVolume)
|
||||
self.assertIsInstance(volume, self.volume_class)
|
||||
self.assertEqual(volume.name, 'root')
|
||||
self.assertEqual(volume.pool, self.pool.name)
|
||||
self.assertEqual(volume.size, qubes.config.defaults['root_img_size'])
|
||||
@ -175,7 +193,7 @@ class TC_00_ThinPool(ThinPoolBase):
|
||||
}
|
||||
vm = qubes.tests.storage.TestVM(self)
|
||||
volume = self.app.get_pool(self.pool.name).init_volume(vm, config)
|
||||
self.assertIsInstance(volume, ThinVolume)
|
||||
self.assertIsInstance(volume, self.volume_class)
|
||||
self.assertEqual(volume.name, 'root')
|
||||
self.assertEqual(volume.pool, self.pool.name)
|
||||
self.assertEqual(volume.size, qubes.config.defaults['root_img_size'])
|
||||
@ -952,7 +970,7 @@ class TC_00_ThinPool(ThinPoolBase):
|
||||
}
|
||||
volume = self.app.get_pool(self.pool.name).init_volume(
|
||||
vm, config_snapshot)
|
||||
self.assertIsInstance(volume, ThinVolume)
|
||||
self.assertIsInstance(volume, self.volume_class)
|
||||
self.assertEqual(volume.name, 'root2')
|
||||
self.assertEqual(volume.pool, self.pool.name)
|
||||
self.assertEqual(volume.size, qubes.config.defaults['root_img_size'])
|
||||
@ -1059,8 +1077,8 @@ class TC_00_ThinPool(ThinPoolBase):
|
||||
class TC_01_ThinPool(ThinPoolBase, qubes.tests.SystemTestCase):
|
||||
''' Sanity tests for :py:class:`qubes.storage.lvm.ThinPool` '''
|
||||
|
||||
def setUp(self):
|
||||
super(TC_01_ThinPool, self).setUp()
|
||||
def setUp(self, **kwargs):
|
||||
super().setUp(**kwargs)
|
||||
self.init_default_template()
|
||||
|
||||
def test_004_import(self):
|
||||
@ -1107,7 +1125,7 @@ class TC_01_ThinPool(ThinPoolBase, qubes.tests.SystemTestCase):
|
||||
|
||||
@skipUnlessLvmPoolExists
|
||||
class TC_02_StorageHelpers(ThinPoolBase):
|
||||
def setUp(self):
|
||||
def setUp(self, **kwargs):
|
||||
xml_path = '/tmp/qubes-test.xml'
|
||||
self.app = qubes.Qubes.create_empty_store(store=xml_path,
|
||||
clockvm=None,
|
||||
@ -1115,7 +1133,7 @@ class TC_02_StorageHelpers(ThinPoolBase):
|
||||
offline_mode=True,
|
||||
)
|
||||
os.environ['QUBES_XML_PATH'] = xml_path
|
||||
super(TC_02_StorageHelpers, self).setUp()
|
||||
super().setUp(**kwargs)
|
||||
# reset cache
|
||||
qubes.storage.DirectoryThinPool._thin_pool = {}
|
||||
|
||||
|
||||
@ -402,6 +402,8 @@ done
|
||||
%{python3_sitelib}/qubes/storage/reflink.py
|
||||
%{python3_sitelib}/qubes/storage/kernels.py
|
||||
%{python3_sitelib}/qubes/storage/lvm.py
|
||||
%{python3_sitelib}/qubes/storage/callback.py
|
||||
%doc /usr/share/doc/qubes/qubes_callback.json.example
|
||||
|
||||
%dir %{python3_sitelib}/qubes/tools
|
||||
%dir %{python3_sitelib}/qubes/tools/__pycache__
|
||||
@ -452,6 +454,7 @@ done
|
||||
%{python3_sitelib}/qubes/tests/storage_reflink.py
|
||||
%{python3_sitelib}/qubes/tests/storage_kernels.py
|
||||
%{python3_sitelib}/qubes/tests/storage_lvm.py
|
||||
%{python3_sitelib}/qubes/tests/storage_callback.py
|
||||
%{python3_sitelib}/qubes/tests/tarwriter.py
|
||||
|
||||
%dir %{python3_sitelib}/qubes/tests/vm
|
||||
|
||||
2
setup.py
2
setup.py
@ -83,6 +83,7 @@ if __name__ == '__main__':
|
||||
'file-reflink = qubes.storage.reflink:ReflinkPool',
|
||||
'linux-kernel = qubes.storage.kernels:LinuxKernel',
|
||||
'lvm_thin = qubes.storage.lvm:ThinPool',
|
||||
'callback = qubes.storage.callback:CallbackPool',
|
||||
],
|
||||
'qubes.tests.storage': [
|
||||
'test = qubes.tests.storage:TestPool',
|
||||
@ -90,5 +91,6 @@ if __name__ == '__main__':
|
||||
'file-reflink = qubes.storage.reflink:ReflinkPool',
|
||||
'linux-kernel = qubes.storage.kernels:LinuxKernel',
|
||||
'lvm_thin = qubes.storage.lvm:ThinPool',
|
||||
'callback = qubes.storage.callback:CallbackPool',
|
||||
],
|
||||
})
|
||||
|
||||
Loading…
Reference in New Issue
Block a user