diff --git a/Makefile b/Makefile index eb951965..c8787c2e 100644 --- a/Makefile +++ b/Makefile @@ -141,7 +141,6 @@ rpms-dom0: all: $(PYTHON) setup.py build $(MAKE) -C qubes-rpc all -# make all -C tests # Currently supported only on xen install: @@ -158,7 +157,6 @@ endif ln -s qvm-device.1.gz $(DESTDIR)/usr/share/man/man1/qvm-block.1.gz ln -s qvm-device.1.gz $(DESTDIR)/usr/share/man/man1/qvm-pci.1.gz ln -s qvm-device.1.gz $(DESTDIR)/usr/share/man/man1/qvm-usb.1.gz -# $(MAKE) install -C tests $(MAKE) install -C relaxng mkdir -p $(DESTDIR)/etc/qubes ifeq ($(BACKEND_VMM),xen) diff --git a/ci/pylintrc b/ci/pylintrc index be73b0cc..88465989 100644 --- a/ci/pylintrc +++ b/ci/pylintrc @@ -6,6 +6,8 @@ ignore=tests # abstract-class-little-used: see http://www.logilab.org/ticket/111138 # deprecated-method: # enable again after disabling py-3.4.3 asyncio.ensure_future compat hack +# not-an-iterable: +# a lot of false possitives for asyncio (yield from (some coroutine)) disable= abstract-class-little-used, bad-continuation, @@ -18,6 +20,7 @@ disable= locally-enabled, logging-format-interpolation, missing-docstring, + not-an-iterable, star-args, wrong-import-order diff --git a/doc/index.rst b/doc/index.rst index 5d0e71d4..b375f67b 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -16,6 +16,7 @@ manpages and API documentation. For primary user documentation, see qubes qubes-vm/index qubes-events + qubes-features qubes-storage qubes-exc qubes-ext diff --git a/doc/libvirt.rst b/doc/libvirt.rst index bd35fe49..7add6a66 100644 --- a/doc/libvirt.rst +++ b/doc/libvirt.rst @@ -22,14 +22,14 @@ The distributor may put a file at :file:`/usr/share/qubes/template/xen-dist.xml`) to override this file. User may put a file at either :file:`/etc/qubes/templates/libvirt/xen-user.xml` or -:file:`/etc/qubes/templates/libvirt/by-name/.xml`, where ```` is +:file:`/etc/qubes/templates/libvirt/xen/by-name/.xml`, where ```` is full name of the domain. Wildcards are not supported but symlinks are. Jinja has a concept of template names, which basically is the path below some load point, which in Qubes' case is :file:`/etc/qubes/templates` and :file:`/usr/share/qubes/templates`. Thus names of those templates are respectively ``'libvirt/xen.xml'``, ``'libvirt/xen-dist.xml'``, -``'libvirt/xen-user.xml'`` and ``'libvirt/by-name/.xml'``. +``'libvirt/xen-user.xml'`` and ``'libvirt/xen/by-name/.xml'``. This will be important later. .. note:: @@ -95,6 +95,9 @@ basic Contains ````, ````, ````, ```` and ```` nodes. +cpu + ```` node. + os Contents of ```` node. diff --git a/doc/loading.svg b/doc/loading.svg new file mode 100644 index 00000000..fa6038a2 --- /dev/null +++ b/doc/loading.svg @@ -0,0 +1,3 @@ + + +
qubes.Qubes
qubes.Qubes
load()
load()
Labels
Pools
[Not supported by viewer]
VMs
VMs
.load() cont.
.load() cont.
load_properties(
load_stage=3)
load_properties(<br>load_stage=3)
.domains.add()
.domains.add()
domain-add
domain-add
qubes.vm.QubesVM
qubes.vm.QubesVM
__init__()
__init__()
load_properties(
load_stage=2)
load_properties(<br>load_stage=2)
load_properties(
load_stage=4)
load_properties(<br>load_stage=4)
.fire_event('domain-load')
.fire_event('domain-load')
domain-load
domain-load
admin.vm.Create
admin.vm.CreateInPool
admin.vm.CreateDisposable
[Not supported by viewer]
app.add_new_vm(cls)
app.add_new_vm(cls)
qubes.vm.QubesVM
qubes.vm.QubesVM
__init__()
__init__()
domain-init
domain-init
admin.vm.Remove
admin.vm.Remove
del app.domains[vm]
del app.domains[vm]
domain-pre-delete
domain-pre-delete
domain-delete
domain-delete
Current as of v4.0.27 (2b2cdf4)
Not included: storage
Current as of v4.0.27 (2b2cdf4)<br>Not included: storage
events
events
API
API
Key:
Key:
load_extras()
load_extras()
everything in app
everything in app
default_dispvm
kernelopts
netvm
template
[Not supported by viewer]
features
devices
tags
[Not supported by viewer]
anything by default,
if not in load_stage=4
anything by default,<br>if not in load_stage=4
all properties are set
or left as default
all properties are set<br>or left as default
the domain is
removed
from disk
[Not supported by viewer]
here the domain is actually
removed from the collection
here the domain is actually<br>removed from the collection \ No newline at end of file diff --git a/doc/qubes-features.rst b/doc/qubes-features.rst new file mode 100644 index 00000000..a334cff9 --- /dev/null +++ b/doc/qubes-features.rst @@ -0,0 +1,193 @@ +:py:class:`qubes.vm.Features` - Qubes VM features, services +============================================================ + +Features are generic mechanism for storing key-value pairs attached to a +VM. The primary use case for them is data storage for extensions (you can think +of them as more flexible properties, defined by extensions), but some are also +used in the qubes core itself. There is no definite list of supported features, +each extension can set their own and there is no requirement of registration, +but :program:`qvm-features` man page contains well known ones. +In addition, there is a mechanism for VM request setting a feature. This is +useful for extensions to discover if its VM part is present. + +Features can have three distinct values: no value (not present in mapping, +which is closest thing to :py:obj:`None`), empty string (which is +interpreted as :py:obj:`False`) and non-empty string, which is +:py:obj:`True`. Anything assigned to the mapping is coerced to strings, +however if you assign instances of :py:class:`bool`, they are converted as +described above. Be aware that assigning the number `0` (which is considered +false in Python) will result in string `'0'`, which is considered true. + +:py:class:`qubes.vm.Features` inherits from :py:class:`dict`, so provide all the +standard functions to get, list and set values. Additionally provide helper +functions to check if given feature is set on the VM and default to the value +on the VM's template or netvm. This is useful for features which nature is +inherited from other VMs, like "is package X is installed" or "is VM behind a +VPN". + +Example usage of features in extension: + +.. code-block:: python + + import qubes.exc + import qubes.ext + + class ExampleExtension(qubes.ext.Extension): + @qubes.ext.handler('domain-pre-start') + def on_domain_start(self, vm, event, **kwargs): + if vm.features.get('do-not-start', False): + raise qubes.exc.QubesVMError(vm, + 'Start prohibited because of do-not-start feature') + + if vm.features.check_with_template('something-installed', False): + # do something + +The above extension does two things: + + - prevent starting a qube with ``do-not-start`` feature set + - do something when ``something-installed`` feature is set on the qube, or its template + + +qvm-features-request, qubes.PostInstall service +------------------------------------------------ + +When some package in the VM want to request feature to be set (aka advertise +support for it), it should place a shell script in ``/etc/qubes/post-install.d``. +This script should call :program:`qvm-features-request` with ``FEATURE=VALUE`` pair(s) as +arguments to request those features. It is recommended to use very simple +values here (for example ``1``). The script should be named in form +``XX-package-name.sh`` where ``XX`` is two-digits number below 90 and +``package-name`` is unique name specific to this package (preferably actual +package name). The script needs executable bit set. + +``qubes.PostInstall`` service will call all those scripts after any package +installation and also after initial template installation. +This way package have a chance to report to dom0 if any feature is +added/removed. + +The features flow to dom0 according to the diagram below. Important part is +that qubes core :py:class:`qubes.ext.Extension` is responsible for handling such request in +``features-request`` event handler. If no extension handles given feature request, +it will be ignored. The extension should carefuly validate requested +features (ignoring those not recognized - may be for another extension) and +only then set appropriate value on VM object +(:py:attr:`qubes.vm.BaseVM.features`). It is recommended to make the +verification code as bulletproof as possible (for example allow only specific +simple values, instead of complex structures), because feature requests +come from untrusted sources. The features actually set on the VM in some cases +may not be necessary those requested. Similar for values. + +.. graphviz:: + + digraph { + + "qubes.PostInstall"; + "/etc/qubes/post-install.d/ scripts"; + "qvm-features-request"; + "qubes.FeaturesRequest"; + "qubes core extensions"; + "VM features"; + + "qubes.PostInstall" -> "/etc/qubes/post-install.d/ scripts"; + "/etc/qubes/post-install.d/ scripts" -> "qvm-features-request" + [xlabel="each script calls"]; + "qvm-features-request" -> "qubes.FeaturesRequest" + [xlabel="last script call the service to dom0"]; + "qubes.FeaturesRequest" -> "qubes core extensions" + [xlabel="features-request event"]; + "qubes core extensions" -> "VM features" + [xlabel="verification"]; + + } + +Example ``/etc/qubes/post-install.d/20-example.sh`` file: + +.. code-block:: shell + + #!/bin/sh + + qvm-features-request example-feature=1 + +Example extension handling the above: + +.. code-block:: python + + import qubes.ext + + class ExampleExtension(qubes.ext.Extension): + # the last argument must be named untrusted_features + @qubes.ext.handler('features-request') + def on_features_request(self, vm, event, untrusted_features): + # don't allow TemplateBasedVMs to request the feature - should be + # requested by the template instead + if hasattr(vm, 'template'): + return + + untrusted_value = untrusted_features.get('example-feature', None) + # check if feature is advertised and verify its value + if untrusted_value != '1': + return + value = untrusted_value + + # and finally set the value + vm.features['example-feature'] = value + +Services +--------- + +`Qubes services `_ are implemented +as features with ``service.`` prefix. The +:py:class:`qubes.ext.services.ServicesExtension` enumerate all the features +in form of ``service.`` prefix and write them to QubesDB as +``/qubes-service/`` and value either ``0`` or ``1``. +VM startup scripts list those entries for for each with value of ``1``, create +``/var/run/qubes-service/`` file. Then, it can be conveniently +used by other scripts to check whether dom0 wishes service to be enabled or +disabled. + +VM package can advertise what services are supported. For that, it needs to +request ``supported-service.`` feature with value ``1`` according +to description above. The :py:class:`qubes.ext.services.ServicesExtension` will +handle such request and set this feature on VM object. ``supported-service.`` +features that stop being advertised with ``qvm-features-request`` call are +removed. This way, it's enough to remove the file from +``/etc/qubes/post-install.d`` (for example by uninstalling package providing +the service) to tell dom0 the service is no longer supported. Services +advertised by TemplateBasedVMs are currently ignored (related +``supported-service.`` features are not set), but retrieving them may be added +in the future. Applications checking for specific service support should use +``vm.features.check_with_template('supported-service.', False)`` +call on desired VM object. When enumerating all supported services, application +should consider both the vm and its template (if any). + +Various tools will use this information to discover if given service is +supported. The API does not enforce service being first advertised before being +enabled (means: there can be service which is enabled, but without matching +``supported-service.`` feature). The list of well known services is in +:program:`qvm-service` man page. + +Example ``/etc/qubes/post-install.d/20-my-service.sh``: + +.. code-block:: shell + + #!/bin/sh + + qvm-features-request supported-service.my-service=1 + +Services and features can be then inspected from dom0 using +:program:`qvm-features` tool, for example: + +.. code-block:: shell + + $ qvm-features my-qube + supported-service.my-service 1 + +Module contents +--------------- + +.. autoclass:: qubes.vm.Features + :members: + :show-inheritance: + +.. vim: ts=3 sw=3 et + diff --git a/doc/qubes.rst b/doc/qubes.rst index 955870a8..5de9e78c 100644 --- a/doc/qubes.rst +++ b/doc/qubes.rst @@ -8,10 +8,20 @@ Because all objects in Qubes' world are interconnected, there is no possibility to instantiate them separately. They are all loaded together and contained in the one ``app`` object, an instance of :py:class:`qubes.Qubes` class. +Loading +^^^^^^^ + +The objects may come to existence in two ways: by explicit instantiation or by +loading from XML file. + The loading from XML is done in stages, because Qubes domains are dependent on each other in what can be even a circular dependency. Therefore some properties -(especcialy those that refer to another domains) are loaded later. Refer to -:py:class:`qubes.Qubes` class documentation to get description of every stage. +(especcialy those that refer to another domains) are loaded later. + +.. image:: loading.svg + +Refer to :py:class:`qubes.Qubes` class documentation to get description of every +stage. Properties diff --git a/linux/systemd/qubes-vm@.service b/linux/systemd/qubes-vm@.service index 0ff2255a..cb5adb1e 100644 --- a/linux/systemd/qubes-vm@.service +++ b/linux/systemd/qubes-vm@.service @@ -2,6 +2,7 @@ Description=Start Qubes VM %i Before=systemd-user-sessions.service After=qubesd.service qubes-meminfo-writer-dom0.service +ConditionKernelCommandLine=!qubes.skip_autostart [Service] Type=oneshot diff --git a/linux/systemd/qubesd.service b/linux/systemd/qubesd.service index 08791abf..44711761 100644 --- a/linux/systemd/qubesd.service +++ b/linux/systemd/qubesd.service @@ -1,5 +1,6 @@ [Unit] Description=Qubes OS daemon +Before=systemd-user-sessions.service [Service] Type=notify diff --git a/qubes-rpc-policy/qubes.GetDate.policy b/qubes-rpc-policy/qubes.GetDate.policy index 24c72be4..466d5396 100644 --- a/qubes-rpc-policy/qubes.GetDate.policy +++ b/qubes-rpc-policy/qubes.GetDate.policy @@ -3,4 +3,5 @@ ## Please use a single # to start your custom comments +$tag:anon-vm $anyvm deny $anyvm $anyvm allow,target=dom0 diff --git a/qubes-rpc-policy/qubes.OpenInVM.policy b/qubes-rpc-policy/qubes.OpenInVM.policy index 27303cc9..41217337 100644 --- a/qubes-rpc-policy/qubes.OpenInVM.policy +++ b/qubes-rpc-policy/qubes.OpenInVM.policy @@ -3,8 +3,5 @@ ## Please use a single # to start your custom comments -sys-whonix anon-whonix allow -whonix-gw anon-whonix allow -whonix-ws anon-whonix allow $anyvm $dispvm allow $anyvm $anyvm ask diff --git a/qubes-rpc-policy/qubes.OpenURL.policy b/qubes-rpc-policy/qubes.OpenURL.policy index 27303cc9..41217337 100644 --- a/qubes-rpc-policy/qubes.OpenURL.policy +++ b/qubes-rpc-policy/qubes.OpenURL.policy @@ -3,8 +3,5 @@ ## Please use a single # to start your custom comments -sys-whonix anon-whonix allow -whonix-gw anon-whonix allow -whonix-ws anon-whonix allow $anyvm $dispvm allow $anyvm $anyvm ask diff --git a/qubes-rpc-policy/qubes.UpdatesProxy.policy b/qubes-rpc-policy/qubes.UpdatesProxy.policy index 21c68c56..ff4f8434 100644 --- a/qubes-rpc-policy/qubes.UpdatesProxy.policy +++ b/qubes-rpc-policy/qubes.UpdatesProxy.policy @@ -3,6 +3,15 @@ ## Please use a single # to start your custom comments +# Upgrade all TemplateVMs through sys-whonix. +#$type:TemplateVM $default allow,target=sys-whonix + +# Upgrade Whonix TemplateVMs through sys-whonix. +$tag:whonix-updatevm $default allow,target=sys-whonix + +# Deny Whonix TemplateVMs using UpdatesProxy of any other VM. +$tag:whonix-updatevm $anyvm deny + # Default rule for all TemplateVMs - direct the connection to sys-net $type:TemplateVM $default allow,target=sys-net diff --git a/qubes/__init__.py b/qubes/__init__.py index 6919f67f..0dd6e038 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -42,7 +42,7 @@ __license__ = 'GPLv2 or later' __version__ = 'R3' -class Label(object): +class Label: '''Label definition for virtual machines Label specifies colour of the padlock displayed next to VM's name. @@ -134,7 +134,7 @@ class Label(object): self.icon_dispvm) + ".png" -class property(object): # pylint: disable=redefined-builtin,invalid-name +class property: # pylint: disable=redefined-builtin,invalid-name '''Qubes property. This class holds one property that can be saved to and loaded from @@ -350,11 +350,10 @@ class property(object): # pylint: disable=redefined-builtin,invalid-name raise qubes.exc.QubesValueError if self.type is bool: return self.bool(None, None, untrusted_newvalue) - else: - try: - return self.type(untrusted_newvalue) - except ValueError: - raise qubes.exc.QubesValueError + try: + return self.type(untrusted_newvalue) + except ValueError: + raise qubes.exc.QubesValueError else: # 'str' or not specified type try: diff --git a/qubes/api/__init__.py b/qubes/api/__init__.py index 20cde2a5..82c84d7e 100644 --- a/qubes/api/__init__.py +++ b/qubes/api/__init__.py @@ -97,7 +97,7 @@ def apply_filters(iterable, filters): return iterable -class AbstractQubesAPI(object): +class AbstractQubesAPI: '''Common code for Qubes Management Protocol handling Different interfaces can expose different API call sets, however they share diff --git a/qubes/api/admin.py b/qubes/api/admin.py index a207d7ee..753a32ea 100644 --- a/qubes/api/admin.py +++ b/qubes/api/admin.py @@ -44,7 +44,7 @@ import qubes.vm.adminvm import qubes.vm.qubesvm -class QubesMgmtEventsDispatcher(object): +class QubesMgmtEventsDispatcher: def __init__(self, filters, send_event): self.filters = filters self.send_event = send_event @@ -335,7 +335,7 @@ class QubesAdminAPI(qubes.api.AbstractQubesAPI): volume = self.dest.volumes[self.arg] # properties defined in API volume_properties = [ - 'pool', 'vid', 'size', 'usage', 'rw', 'source', + 'pool', 'vid', 'size', 'usage', 'rw', 'source', 'path', 'save_on_stop', 'snap_on_start', 'revisions_to_keep'] def _serialize(value): @@ -479,7 +479,7 @@ class QubesAdminAPI(qubes.api.AbstractQubesAPI): if not self.dest.is_halted(): raise qubes.exc.QubesVMNotHaltedError(self.dest) - path = self.dest.storage.import_data(self.arg) + path = yield from self.dest.storage.import_data(self.arg) self.enforce(' ' not in path) size = self.dest.volumes[self.arg].size @@ -1101,7 +1101,7 @@ class QubesAdminAPI(qubes.api.AbstractQubesAPI): try: yield from self.dest.remove_from_disk() except: # pylint: disable=bare-except - self.app.log.exception('Error wile removing VM \'%s\' files', + self.app.log.exception('Error while removing VM \'%s\' files', self.dest.name) self.app.save() diff --git a/qubes/api/internal.py b/qubes/api/internal.py index 2df550fe..f430254e 100644 --- a/qubes/api/internal.py +++ b/qubes/api/internal.py @@ -68,7 +68,8 @@ class QubesInternalAPI(qubes.api.AbstractQubesAPI): success = untrusted_payload == b'ok' try: - self.dest.storage.import_data_end(self.arg, success=success) + yield from self.dest.storage.import_data_end(self.arg, + success=success) except: self.dest.fire_event('domain-volume-import-end', volume=self.arg, success=False) diff --git a/qubes/api/misc.py b/qubes/api/misc.py index c6a9e677..f2ff3ca5 100644 --- a/qubes/api/misc.py +++ b/qubes/api/misc.py @@ -76,7 +76,8 @@ class QubesMiscAPI(qubes.api.AbstractQubesAPI): untrusted_features = {} safe_set = string.ascii_letters + string.digits - expected_features = ('qrexec', 'gui', 'default-user') + expected_features = ('qrexec', 'gui', 'gui-emulated', 'default-user', + 'os') for feature in expected_features: untrusted_value = self.src.untrusted_qdb.read( '/qubes-tools/' + feature) diff --git a/qubes/app.py b/qubes/app.py index 0b50f6a0..5e72159f 100644 --- a/qubes/app.py +++ b/qubes/app.py @@ -21,7 +21,7 @@ # import collections -import errno +import copy import functools import grp import itertools @@ -60,13 +60,14 @@ import qubes import qubes.ext import qubes.utils import qubes.storage +import qubes.storage.reflink import qubes.vm import qubes.vm.adminvm import qubes.vm.qubesvm import qubes.vm.templatevm # pylint: enable=wrong-import-position -class VirDomainWrapper(object): +class VirDomainWrapper: # pylint: disable=too-few-public-methods def __init__(self, connection, vm): @@ -97,7 +98,7 @@ class VirDomainWrapper(object): return wrapper -class VirConnectWrapper(object): +class VirConnectWrapper: # pylint: disable=too-few-public-methods def __init__(self, uri): @@ -134,7 +135,7 @@ class VirConnectWrapper(object): return wrapper -class VMMConnection(object): +class VMMConnection: '''Connection to Virtual Machine Manager (libvirt)''' def __init__(self, offline_mode=None): @@ -229,7 +230,7 @@ class VMMConnection(object): self._xc = None # and pray it will get garbage-collected -class QubesHost(object): +class QubesHost: '''Basic information about host machine :param qubes.Qubes app: Qubes application context (must have \ @@ -363,7 +364,7 @@ class QubesHost(object): return (current_time, current) -class VMCollection(object): +class VMCollection: '''A collection of Qubes VMs VMCollection supports ``in`` operator. You may test for ``qid``, ``name`` @@ -493,7 +494,7 @@ class VMCollection(object): self.app.fire_event('domain-delete', vm=vm) def __contains__(self, key): - return any((key == vm or key == vm.qid or key == vm.name) + return any((key in (vm, vm.qid, vm.name)) for vm in self) @@ -552,37 +553,37 @@ def _default_pool(app): 1. If there is one named 'default', use it. 2. Check if root fs is on LVM thin - use that - 3. Look for file-based pool pointing /var/lib/qubes + 3. Look for file(-reflink)-based pool pointing to /var/lib/qubes 4. Fail ''' if 'default' in app.pools: return app.pools['default'] - else: - if 'DEFAULT_LVM_POOL' in os.environ: - thin_pool = os.environ['DEFAULT_LVM_POOL'] - for pool in app.pools.values(): - if pool.config.get('driver', None) != 'lvm_thin': - continue - if pool.config['thin_pool'] == thin_pool: - return pool - # no DEFAULT_LVM_POOL, or pool not defined - root_volume_group, root_thin_pool = \ - qubes.storage.DirectoryThinPool.thin_pool('/') - if root_thin_pool: - for pool in app.pools.values(): - if pool.config.get('driver', None) != 'lvm_thin': - continue - if (pool.config['volume_group'] == root_volume_group and - pool.config['thin_pool'] == root_thin_pool): - return pool - # not a thin volume? look for file pools + if 'DEFAULT_LVM_POOL' in os.environ: + thin_pool = os.environ['DEFAULT_LVM_POOL'] for pool in app.pools.values(): - if pool.config.get('driver', None) not in ('file', 'file-reflink'): + if pool.config.get('driver', None) != 'lvm_thin': continue - if pool.config['dir_path'] == qubes.config.qubes_base_dir: + if pool.config['thin_pool'] == thin_pool: return pool - raise AttributeError('Cannot determine default storage pool') + # no DEFAULT_LVM_POOL, or pool not defined + root_volume_group, root_thin_pool = \ + qubes.storage.DirectoryThinPool.thin_pool('/') + if root_thin_pool: + for pool in app.pools.values(): + if pool.config.get('driver', None) != 'lvm_thin': + continue + if (pool.config['volume_group'] == root_volume_group and + pool.config['thin_pool'] == root_thin_pool): + return pool + + # not a thin volume? look for file pools + for pool in app.pools.values(): + if pool.config.get('driver', None) not in ('file', 'file-reflink'): + continue + if pool.config['dir_path'] == qubes.config.qubes_base_dir: + return pool + raise AttributeError('Cannot determine default storage pool') def _setter_pool(app, prop, value): if isinstance(value, qubes.storage.Pool): @@ -717,6 +718,19 @@ class Qubes(qubes.PropertyHolder): setter=_setter_pool, doc='Default storage pool for kernel volumes') + default_qrexec_timeout = qubes.property('default_qrexec_timeout', + load_stage=3, + default=60, + type=int, + doc='''Default time in seconds after which qrexec connection attempt is + deemed failed''') + + default_shutdown_timeout = qubes.property('default_shutdown_timeout', + load_stage=3, + default=60, + type=int, + doc='''Default time in seconds for VM shutdown to complete''') + stats_interval = qubes.property('stats_interval', default=3, type=int, @@ -1006,8 +1020,8 @@ class Qubes(qubes.PropertyHolder): try: fd = os.open(self._store, os.O_RDWR | (os.O_CREAT * int(for_save))) - except OSError as e: - if not for_save and e.errno == errno.ENOENT: + except FileNotFoundError: + if not for_save: raise qubes.exc.QubesException( 'Qubes XML store {!r} is missing; ' 'use qubes-create tool'.format(self._store)) @@ -1064,15 +1078,29 @@ class Qubes(qubes.PropertyHolder): } assert max(self.labels.keys()) == qubes.config.max_default_label + pool_configs = copy.deepcopy(qubes.config.defaults['pool_configs']) + root_volume_group, root_thin_pool = \ qubes.storage.DirectoryThinPool.thin_pool('/') - if root_thin_pool: - self.add_pool( - volume_group=root_volume_group, thin_pool=root_thin_pool, - name='lvm', driver='lvm_thin') - # pool based on /var/lib/qubes will be created here: - for name, config in qubes.config.defaults['pool_configs'].items(): + lvm_config = { + 'name': 'lvm', + 'driver': 'lvm_thin', + 'volume_group': root_volume_group, + 'thin_pool': root_thin_pool + } + pool_configs[lvm_config['name']] = lvm_config + + for name, config in pool_configs.items(): + if 'driver' not in config and 'dir_path' in config: + config['driver'] = 'file' + try: + os.makedirs(config['dir_path'], exist_ok=True) + if qubes.storage.reflink.is_supported(config['dir_path']): + config['driver'] = 'file-reflink' + config['setup_check'] = 'no' # don't check twice + except PermissionError: # looks like a testing environment + pass # stay with 'file' self.pools[name] = self._get_pool(**config) self.default_pool_kernel = 'linux-kernel' @@ -1170,6 +1198,11 @@ class Qubes(qubes.PropertyHolder): raise KeyError(label) + def setup_pools(self): + """ Run implementation specific setup for each storage pool. """ + for pool in self.pools.values(): + pool.setup() + def add_pool(self, name, **kwargs): """ Add a storage pool to config.""" @@ -1251,6 +1284,20 @@ class Qubes(qubes.PropertyHolder): if event == libvirt.VIR_DOMAIN_EVENT_STOPPED: vm.on_libvirt_domain_stopped() + elif event == libvirt.VIR_DOMAIN_EVENT_SUSPENDED: + try: + vm.fire_event('domain-paused') + except Exception: # pylint: disable=broad-except + self.log.exception( + 'Uncaught exception from domain-paused handler ' + 'for domain %s', vm.name) + elif event == libvirt.VIR_DOMAIN_EVENT_RESUMED: + try: + vm.fire_event('domain-unpaused') + except Exception: # pylint: disable=broad-except + self.log.exception( + 'Uncaught exception from domain-unpaused handler ' + 'for domain %s', vm.name) @qubes.events.handler('domain-pre-delete') def on_domain_pre_deleted(self, event, vm): @@ -1263,9 +1310,9 @@ class Qubes(qubes.PropertyHolder): self.log.error( 'Cannot remove %s, used by %s.%s', vm, obj, prop.__name__) - raise qubes.exc.QubesVMInUseError(vm, - 'Domain is in use: {!r}; details in system log' - .format(vm.name)) + raise qubes.exc.QubesVMInUseError(vm, 'Domain is in ' + 'use: {!r}; see /var/log/qubes/qubes.log in dom0 for ' + 'details'.format(vm.name)) except AttributeError: pass diff --git a/qubes/backup.py b/qubes/backup.py index a792692f..bfc6cb63 100644 --- a/qubes/backup.py +++ b/qubes/backup.py @@ -76,7 +76,7 @@ class BackupCanceledError(qubes.exc.QubesException): self.tmpdir = tmpdir -class BackupHeader(object): +class BackupHeader: '''Structure describing backup-header file included as the first file in backup archive ''' @@ -124,7 +124,7 @@ class BackupHeader(object): f_header.write("{!s}={!s}\n".format(key, getattr(self, attr))) -class SendWorker(object): +class SendWorker: # pylint: disable=too-few-public-methods def __init__(self, queue, base_dir, backup_stdout): super(SendWorker, self).__init__() @@ -148,6 +148,7 @@ class SendWorker(object): # verified before untaring. tar_final_cmd = ["tar", "-cO", "--posix", "-C", self.base_dir, filename] + # pylint: disable=not-an-iterable final_proc = yield from asyncio.create_subprocess_exec( *tar_final_cmd, stdout=self.backup_stdout) @@ -183,6 +184,7 @@ def launch_proc_with_pty(args, stdin=None, stdout=None, stderr=None, echo=True): termios_p[3] &= ~termios.ECHO termios.tcsetattr(ctty_fd, termios.TCSANOW, termios_p) (pty_master, pty_slave) = os.openpty() + # pylint: disable=not-an-iterable p = yield from asyncio.create_subprocess_exec(*args, stdin=stdin, stdout=stdout, @@ -228,7 +230,7 @@ def launch_scrypt(action, input_name, output_name, passphrase): return p -class Backup(object): +class Backup: '''Backup operation manager. Usage: >>> app = qubes.Qubes() @@ -251,7 +253,7 @@ class Backup(object): ''' # pylint: disable=too-many-instance-attributes - class FileToBackup(object): + class FileToBackup: # pylint: disable=too-few-public-methods def __init__(self, file_path, subdir=None, name=None, size=None): if size is None: @@ -278,7 +280,7 @@ class Backup(object): if name is not None: self.name = name - class VMToBackup(object): + class VMToBackup: # pylint: disable=too-few-public-methods def __init__(self, vm, files, subdir): self.vm = vm @@ -635,6 +637,7 @@ class Backup(object): # Pipe: tar-sparse | scrypt | tar | backup_target # TODO: log handle stderr + # pylint: disable=not-an-iterable tar_sparse = yield from asyncio.create_subprocess_exec( *tar_cmdline, stdout=subprocess.PIPE) diff --git a/qubes/config.py b/qubes/config.py index dfedfe23..da02dbb6 100644 --- a/qubes/config.py +++ b/qubes/config.py @@ -76,9 +76,8 @@ defaults = { 'root_img_size': 10*1024*1024*1024, 'pool_configs': { - # create file pool even when the default one is LVM + # create file(-reflink) pool even when the default one is LVM 'varlibqubes': {'dir_path': qubes_base_dir, - 'driver': 'file', 'name': 'varlibqubes'}, 'linux-kernel': { 'dir_path': os.path.join(qubes_base_dir, diff --git a/qubes/devices.py b/qubes/devices.py index 11849691..a535cc70 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -67,7 +67,7 @@ class DeviceAlreadyAttached(qubes.exc.QubesException, KeyError): '''Trying to attach already attached device''' pass -class DeviceInfo(object): +class DeviceInfo: ''' Holds all information about a device ''' # pylint: disable=too-few-public-methods def __init__(self, backend_domain, ident, description=None, @@ -117,7 +117,7 @@ class DeviceInfo(object): return '{!s}:{!s}'.format(self.backend_domain, self.ident) -class DeviceAssignment(object): # pylint: disable=too-few-public-methods +class DeviceAssignment: # pylint: disable=too-few-public-methods ''' Maps a device to a frontend_domain. ''' def __init__(self, backend_domain, ident, options=None, persistent=False, @@ -158,7 +158,7 @@ class DeviceAssignment(object): # pylint: disable=too-few-public-methods return self.backend_domain.devices[self.bus][self.ident] -class DeviceCollection(object): +class DeviceCollection: '''Bag for devices. Used as default value for :py:meth:`DeviceManager.__missing__` factory. @@ -168,13 +168,14 @@ class DeviceCollection(object): This class emits following events on VM object: - .. event:: device-attach: (device) + .. event:: device-attach: (device, options) Fired when device is attached to a VM. Handler for this event can be asynchronous (a coroutine). :param device: :py:class:`DeviceInfo` object to be attached + :param options: :py:class:`dict` of attachment options .. event:: device-pre-attach: (device) @@ -357,8 +358,7 @@ class DeviceCollection(object): if persistent is True: # don't break app.save() return self._set - else: - raise + raise result = set() for dev, options in devices: if dev in self._set and not persistent: @@ -433,7 +433,7 @@ class UnknownDevice(DeviceInfo): frontend_domain) -class PersistentCollection(object): +class PersistentCollection: ''' Helper object managing persistent `DeviceAssignment`s. ''' diff --git a/qubes/dochelpers.py b/qubes/dochelpers.py index 1fe20bd4..e55f9317 100644 --- a/qubes/dochelpers.py +++ b/qubes/dochelpers.py @@ -48,7 +48,7 @@ SUBCOMMANDS_TITLE = 'COMMANDS' OPTIONS_TITLE = 'OPTIONS' -class GithubTicket(object): +class GithubTicket: # pylint: disable=too-few-public-methods def __init__(self, data): self.number = data['number'] @@ -418,7 +418,7 @@ def parse_event(env, sig, signode): # -def break_to_pdb(app, *dummy): +def break_to_pdb(app, *_dummy): if not app.config.break_to_pdb: return import pdb diff --git a/qubes/events.py b/qubes/events.py index 34bcd9fd..2a0af524 100644 --- a/qubes/events.py +++ b/qubes/events.py @@ -94,7 +94,7 @@ class EmitterMeta(type): cls.__handlers__[event].add(attr) -class Emitter(object, metaclass=EmitterMeta): +class Emitter(metaclass=EmitterMeta): '''Subject that can emit events. By default all events are disabled not to interfere with loading from XML. diff --git a/qubes/exc.py b/qubes/exc.py index 427ae4fa..df31f60e 100644 --- a/qubes/exc.py +++ b/qubes/exc.py @@ -101,6 +101,14 @@ class QubesVMNotHaltedError(QubesVMError): super(QubesVMNotHaltedError, self).__init__(vm, msg or 'Domain is not powered off: {!r}'.format(vm.name)) +class QubesVMShutdownTimeoutError(QubesVMError): + '''Domain shutdown timed out. + + ''' + def __init__(self, vm, msg=None): + super(QubesVMShutdownTimeoutError, self).__init__(vm, + msg or 'Domain shutdown timed out: {!r}'.format(vm.name)) + class QubesNoTemplateError(QubesVMError): '''Cannot start domain, because there is no template''' @@ -151,7 +159,7 @@ class BackupCancelledError(QubesException): msg or 'Backup cancelled') -class QubesMemoryError(QubesException, MemoryError): +class QubesMemoryError(QubesVMError, MemoryError): '''Cannot start domain, because not enough memory is available''' def __init__(self, vm, msg=None): super(QubesMemoryError, self).__init__( diff --git a/qubes/ext/__init__.py b/qubes/ext/__init__.py index 2b4b668d..c10e29d5 100644 --- a/qubes/ext/__init__.py +++ b/qubes/ext/__init__.py @@ -29,7 +29,7 @@ import pkg_resources import qubes.events -class Extension(object): +class Extension: '''Base class for all extensions ''' # pylint: disable=too-few-public-methods diff --git a/qubes/ext/core_features.py b/qubes/ext/core_features.py index b2b77ea2..f07f5e93 100644 --- a/qubes/ext/core_features.py +++ b/qubes/ext/core_features.py @@ -32,7 +32,7 @@ class CoreFeatures(qubes.ext.Extension): return requested_features = {} - for feature in ('qrexec', 'gui', 'qubes-firewall'): + for feature in ('qrexec', 'gui', 'gui-emulated', 'qubes-firewall'): untrusted_value = untrusted_features.get(feature, None) if untrusted_value in ('1', '0'): requested_features[feature] = bool(int(untrusted_value)) @@ -44,7 +44,7 @@ class CoreFeatures(qubes.ext.Extension): # gui agent presence (0 or 1) qrexec_before = vm.features.get('qrexec', False) - for feature in ('qrexec', 'gui'): + for feature in ('qrexec', 'gui', 'gui-emulated'): # do not allow (Template)VM to override setting if already set # some other way if feature in requested_features and feature not in vm.features: diff --git a/qubes/ext/pci.py b/qubes/ext/pci.py index bd5f34cc..40e9ec2a 100644 --- a/qubes/ext/pci.py +++ b/qubes/ext/pci.py @@ -42,7 +42,8 @@ def load_pci_classes(): # subclass subclass_name <-- single tab # prog-if prog-if_name <-- two tabs result = {} - with open('/usr/share/hwdata/pci.ids') as pciids: + with open('/usr/share/hwdata/pci.ids', + encoding='utf-8', errors='ignore') as pciids: class_id = None subclass_id = None for line in pciids.readlines(): diff --git a/qubes/ext/r3compatibility.py b/qubes/ext/r3compatibility.py index acc1dd08..2fa8ec3a 100644 --- a/qubes/ext/r3compatibility.py +++ b/qubes/ext/r3compatibility.py @@ -80,6 +80,9 @@ class R3Compatibility(qubes.ext.Extension): def write_iptables_qubesdb_entry(self, firewallvm): # pylint: disable=no-self-use + # skip compatibility rules if new format support is advertised + if firewallvm.features.check_with_template('qubes-firewall', False): + return firewallvm.untrusted_qdb.rm("/qubes-iptables-domainrules/") iptables = "# Generated by Qubes Core on {0}\n".format( datetime.datetime.now().ctime()) diff --git a/qubes/ext/services.py b/qubes/ext/services.py index 77e94cdb..b09e2fda 100644 --- a/qubes/ext/services.py +++ b/qubes/ext/services.py @@ -62,3 +62,40 @@ class ServicesExtension(qubes.ext.Extension): return service = feature[len('service.'):] vm.untrusted_qdb.rm('/qubes-service/{}'.format(service)) + + @qubes.ext.handler('features-request') + def supported_services(self, vm, event, untrusted_features): + '''Handle advertisement of supported services''' + # pylint: disable=no-self-use,unused-argument + + if getattr(vm, 'template', None): + vm.log.warning( + 'Ignoring qubes.FeaturesRequest from template-based VM') + return + + new_supported_services = set() + for requested_service in untrusted_features: + if not requested_service.startswith('supported-service.'): + continue + if untrusted_features[requested_service] == '1': + # only allow to advertise service as supported, lack of entry + # means service is not supported + new_supported_services.add(requested_service) + del untrusted_features + + # if no service is supported, ignore the whole thing - do not clear + # all services in case of empty request (manual or such) + if not new_supported_services: + return + + old_supported_services = set( + feat for feat in vm.features + if feat.startswith('supported-service.') and vm.features[feat]) + + for feature in new_supported_services.difference( + old_supported_services): + vm.features[feature] = True + + for feature in old_supported_services.difference( + new_supported_services): + del vm.features[feature] diff --git a/qubes/ext/windows.py b/qubes/ext/windows.py new file mode 100644 index 00000000..0b417886 --- /dev/null +++ b/qubes/ext/windows.py @@ -0,0 +1,73 @@ +# -*- encoding: utf-8 -*- +# +# The Qubes OS Project, http://www.qubes-os.org +# +# Copyright (C) 2017 Marek Marczykowski-Górecki +# +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see . +import asyncio + +import qubes.ext + +class WindowsFeatures(qubes.ext.Extension): + # pylint: disable=too-few-public-methods + @qubes.ext.handler('features-request') + def qubes_features_request(self, vm, event, untrusted_features): + '''Handle features provided requested by Qubes Windows Tools''' + # pylint: disable=no-self-use,unused-argument + if getattr(vm, 'template', None): + vm.log.warning( + 'Ignoring qubes.NotifyTools for template-based VM') + return + + guest_os = None + if 'os' in untrusted_features: + if untrusted_features['os'] in ['Windows', 'Linux']: + guest_os = untrusted_features['os'] + + qrexec = None + if 'qrexec' in untrusted_features: + if untrusted_features['qrexec'] == '1': + # qrexec feature is set by CoreFeatures extension + qrexec = True + + del untrusted_features + + if guest_os: + vm.features['os'] = guest_os + if guest_os == 'Windows' and qrexec: + vm.features['rpc-clipboard'] = True + + @qubes.ext.handler('domain-create-on-disk') + @asyncio.coroutine + def on_domain_create_on_disk(self, vm, _event, **kwargs): + # pylint: disable=no-self-use,unused-argument + if getattr(vm, 'template', None) is None: + # handle only template-based vms + return + + template = vm.template + if template.features.check_with_template('os', None) != 'Windows': + # ignore non-windows templates + return + + if vm.volumes['private'].save_on_stop: + # until windows tools get ability to prepare private.img on its own, + # copy one from the template + vm.log.info('Windows template - cloning private volume') + import_op = vm.volumes['private'].import_volume( + template.volumes['private']) + if asyncio.iscoroutine(import_op): + yield from import_op diff --git a/qubes/firewall.py b/qubes/firewall.py index 36e2c3b0..0ca8f25c 100644 --- a/qubes/firewall.py +++ b/qubes/firewall.py @@ -34,7 +34,7 @@ import qubes import qubes.vm.qubesvm -class RuleOption(object): +class RuleOption: def __init__(self, untrusted_value): # subset of string.punctuation safe_set = string.ascii_letters + string.digits + \ @@ -209,7 +209,7 @@ class Expire(RuleOption): @property def rule(self): - return None + pass @property def api_rule(self): @@ -234,7 +234,7 @@ class Comment(RuleOption): @property def rule(self): - return None + pass @property def api_rule(self): @@ -451,7 +451,7 @@ class Rule(qubes.PropertyHolder): return hash(self.api_rule) -class Firewall(object): +class Firewall: def __init__(self, vm, load=True): assert hasattr(vm, 'firewall_conf') self.vm = vm diff --git a/qubes/rngdoc.py b/qubes/rngdoc.py index 82f41897..eeb11864 100755 --- a/qubes/rngdoc.py +++ b/qubes/rngdoc.py @@ -26,7 +26,7 @@ import textwrap import lxml.etree -class Element(object): +class Element: def __init__(self, schema, xml): self.schema = schema self.xml = xml @@ -157,7 +157,7 @@ class Element(object): write_rst_table(stream, childtable, ('element', 'number')) -class Schema(object): +class Schema: # pylint: disable=too-few-public-methods nsmap = { 'rng': 'http://relaxng.org/ns/structure/1.0', diff --git a/qubes/storage/__init__.py b/qubes/storage/__init__.py index fdcdceb5..2a6afaa3 100644 --- a/qubes/storage/__init__.py +++ b/qubes/storage/__init__.py @@ -47,7 +47,7 @@ class StoragePoolException(qubes.exc.QubesException): pass -class BlockDevice(object): +class BlockDevice: ''' Represents a storage block device. ''' # pylint: disable=too-few-public-methods def __init__(self, path, name, script=None, rw=True, domain=None, @@ -62,7 +62,7 @@ class BlockDevice(object): self.devtype = devtype -class Volume(object): +class Volume: ''' Encapsulates all data about a volume for serialization to qubes.xml and libvirt config. @@ -198,6 +198,8 @@ class Volume(object): volume data require something more than just writing to a file ( for example connecting to some other domain, or converting data on the fly), the returned path may be a pipe. + + This can be implemented as a coroutine. ''' raise self._not_implemented("import") @@ -207,6 +209,8 @@ class Volume(object): This method is called regardless the operation was successful or not. + This can be implemented as a coroutine. + :param success: True if data import was successful, otherwise False ''' # by default do nothing @@ -252,6 +256,8 @@ class Volume(object): def revert(self, revision=None): ''' Revert volume to previous revision + This can be implemented as a coroutine. + :param revision: revision to revert volume to, see :py:attr:`revisions` ''' # pylint: disable=unused-argument @@ -272,6 +278,7 @@ class Volume(object): This include committing data if :py:attr:`save_on_stop` is set. This can be implemented as a coroutine.''' + raise self._not_implemented("stop") def verify(self): ''' Verifies the volume. @@ -334,14 +341,14 @@ class Volume(object): msg = msg.format(str(self.__class__.__name__), method_name) return NotImplementedError(msg) -class Storage(object): +class Storage: ''' Class for handling VM virtual disks. This is base class for all other implementations, mostly with Xen on Linux in mind. ''' - AVAILABLE_FRONTENDS = set(['xvd' + c for c in string.ascii_lowercase]) + AVAILABLE_FRONTENDS = {'xvd' + c for c in string.ascii_lowercase} def __init__(self, vm): #: Domain for which we manage storage @@ -506,8 +513,7 @@ class Storage(object): ret = volume.create() if asyncio.iscoroutine(ret): coros.append(ret) - if coros: - yield from asyncio.wait(coros) + yield from _wait_and_reraise(coros) os.umask(old_umask) @@ -549,7 +555,7 @@ class Storage(object): self.vm.volumes = {} with VmCreationManager(self.vm): - yield from asyncio.wait([self.clone_volume(src_vm, vol_name) + yield from _wait_and_reraise([self.clone_volume(src_vm, vol_name) for vol_name in self.vm.volume_config.keys()]) @property @@ -581,11 +587,7 @@ class Storage(object): ret = volume.verify() if asyncio.iscoroutine(ret): futures.append(ret) - if futures: - done, _ = yield from asyncio.wait(futures) - for task in done: - # re-raise any exception from async task - task.result() + yield from _wait_and_reraise(futures) self.vm.fire_event('domain-verify-files') return True @@ -605,44 +607,32 @@ class Storage(object): except (IOError, OSError) as e: self.vm.log.exception("Failed to remove volume %s", name, e) - if futures: - try: - done, _ = yield from asyncio.wait(futures) - for task in done: - # re-raise any exception from async task - task.result() - except (IOError, OSError) as e: - self.vm.log.exception("Failed to remove some volume", e) + try: + yield from _wait_and_reraise(futures) + except (IOError, OSError) as e: + self.vm.log.exception("Failed to remove some volume", e) @asyncio.coroutine def start(self): - ''' Execute the start method on each pool ''' + ''' Execute the start method on each volume ''' futures = [] for volume in self.vm.volumes.values(): ret = volume.start() if asyncio.iscoroutine(ret): futures.append(ret) - if futures: - done, _ = yield from asyncio.wait(futures) - for task in done: - # re-raise any exception from async task - task.result() + yield from _wait_and_reraise(futures) @asyncio.coroutine def stop(self): - ''' Execute the start method on each pool ''' + ''' Execute the stop method on each volume ''' futures = [] for volume in self.vm.volumes.values(): ret = volume.stop() if asyncio.iscoroutine(ret): futures.append(ret) - if futures: - done, _ = yield from asyncio.wait(futures) - for task in done: - # re-raise any exception from async task - task.result() + yield from _wait_and_reraise(futures) def unused_frontend(self): ''' Find an unused device name ''' @@ -655,9 +645,9 @@ class Storage(object): ''' Used device names ''' xml = self.vm.libvirt_domain.XMLDesc() parsed_xml = lxml.etree.fromstring(xml) - return set([target.get('dev', None) + return {target.get('dev', None) for target in parsed_xml.xpath( - "//domain/devices/disk/target")]) + "//domain/devices/disk/target")} def export(self, volume): ''' Helper function to export volume (pool.export(volume))''' @@ -668,27 +658,37 @@ class Storage(object): return self.vm.volumes[volume].export() + @asyncio.coroutine def import_data(self, volume): ''' Helper function to import volume data (pool.import_data(volume))''' assert isinstance(volume, (Volume, str)), \ "You need to pass a Volume or pool name as str" if isinstance(volume, Volume): - return volume.import_data() + ret = volume.import_data() + else: + ret = self.vm.volumes[volume].import_data() - return self.vm.volumes[volume].import_data() + if asyncio.iscoroutine(ret): + ret = yield from ret + return ret + @asyncio.coroutine def import_data_end(self, volume, success): ''' Helper function to finish/cleanup data import (pool.import_data_end( volume))''' assert isinstance(volume, (Volume, str)), \ "You need to pass a Volume or pool name as str" if isinstance(volume, Volume): - return volume.import_data_end(success=success) + ret = volume.import_data_end(success=success) + else: + ret = self.vm.volumes[volume].import_data_end(success=success) - return self.vm.volumes[volume].import_data_end(success=success) + if asyncio.iscoroutine(ret): + ret = yield from ret + return ret -class VolumesCollection(object): +class VolumesCollection: '''Convenient collection wrapper for pool.get_volume and pool.list_volumes ''' @@ -706,8 +706,7 @@ class VolumesCollection(object): if isinstance(item, Volume): if item.pool == self._pool: return self[item.vid] - else: - raise KeyError(item) + raise KeyError(item) try: return self._pool.get_volume(item) except NotImplementedError: @@ -740,7 +739,7 @@ class VolumesCollection(object): return [vol for vol in self] -class Pool(object): +class Pool: ''' A Pool is used to manage different kind of volumes (File based/LVM/Btrfs/...). @@ -760,7 +759,7 @@ class Pool(object): def __eq__(self, other): if isinstance(other, Pool): return self.name == other.name - elif isinstance(other, str): + if isinstance(other, str): return self.name == other return NotImplemented @@ -829,12 +828,12 @@ class Pool(object): @property def size(self): ''' Storage pool size in bytes, or None if unknown ''' - return None + pass @property def usage(self): ''' Space used in the pool in bytes, or None if unknown ''' - return None + pass def _not_implemented(self, method_name): ''' Helper for emitting helpful `NotImplementedError` exceptions ''' @@ -843,6 +842,14 @@ class Pool(object): return NotImplementedError(msg) +@asyncio.coroutine +def _wait_and_reraise(futures): + if futures: + done, _ = yield from asyncio.wait(futures) + for task in done: # (re-)raise first exception in line + task.result() + + def _sanitize_config(config): ''' Helper function to convert types to appropriate strings ''' # FIXME: find another solution for serializing basic types @@ -872,7 +879,7 @@ def driver_parameters(name): return [p for p in params if p not in ignored_params] -def isodate(seconds=time.time()): +def isodate(seconds): ''' Helper method which returns an iso date ''' return datetime.utcfromtimestamp(seconds).isoformat("T") @@ -882,23 +889,27 @@ def search_pool_containing_dir(pools, dir_path): This is useful for implementing Pool.included_in method ''' + real_dir_path = os.path.realpath(dir_path) + # prefer filesystem pools for pool in pools: if hasattr(pool, 'dir_path'): - if dir_path.startswith(pool.dir_path): + pool_real_dir_path = os.path.realpath(pool.dir_path) + if os.path.commonpath([pool_real_dir_path, real_dir_path]) == \ + pool_real_dir_path: return pool # then look for lvm for pool in pools: if hasattr(pool, 'thin_pool') and hasattr(pool, 'volume_group'): if (pool.volume_group, pool.thin_pool) == \ - DirectoryThinPool.thin_pool(dir_path): + DirectoryThinPool.thin_pool(real_dir_path): return pool return None -class VmCreationManager(object): +class VmCreationManager: ''' A `ContextManager` which cleans up if volume creation fails. ''' # pylint: disable=too-few-public-methods def __init__(self, vm): diff --git a/qubes/storage/file.py b/qubes/storage/file.py index ead07f4f..46bc3608 100644 --- a/qubes/storage/file.py +++ b/qubes/storage/file.py @@ -356,9 +356,9 @@ class FileVolume(qubes.storage.Volume): def script(self): if not self.snap_on_start and not self.save_on_stop: return None - elif not self.snap_on_start and self.save_on_stop: + if not self.snap_on_start and self.save_on_stop: return 'block-origin' - elif self.snap_on_start: + if self.snap_on_start: return 'block-snapshot' return None diff --git a/qubes/storage/lvm.py b/qubes/storage/lvm.py index aa7c3c12..47324621 100644 --- a/qubes/storage/lvm.py +++ b/qubes/storage/lvm.py @@ -18,9 +18,8 @@ # ''' Driver for storing vm images in a LVM thin pool ''' - +import functools import logging -import operator import os import subprocess @@ -44,8 +43,37 @@ def check_lvm_version(): lvm_is_very_old = check_lvm_version() + class ThinPool(qubes.storage.Pool): ''' LVM Thin based pool implementation + + Volumes are stored as LVM thin volumes, in thin pool specified by + *volume_group*/*thin_pool* arguments. LVM volume naming scheme: + + vm-{vm_name}-{volume_name}[-suffix] + + Where suffix can be one of: + "-snap" - snapshot for currently running VM, at VM shutdown will be + either discarded (if save_on_stop=False), or committed + (if save_on_stop=True) + "-{revision_id}" - volume revision - new revision is automatically + created at each VM shutdown, *revisions_to_keep* control how many + old revisions (in addition to the current one) should be stored + "" (no suffix) - the most recent committed volume state; also volatile + volume (snap_on_start=False, save_on_stop=False) + + On VM startup, new volume is created, depending on volume type, + according to the table below: + + snap_on_start, save_on_stop + False, False, - no suffix, fresh empty volume + False, True, - "-snap", snapshot of last committed revision + True , False, - "-snap", snapshot of last committed revision + of source volume (from VM's template) + True, True, - unsupported configuration + + Volume's revision_id format is "{timestamp}-back", where timestamp is in + '%s' format (seconds since unix epoch) ''' # pylint: disable=protected-access size_cache = None @@ -62,13 +90,20 @@ class ThinPool(qubes.storage.Pool): self._volume_objects_cache = {} + def __repr__(self): + return '<{} at {:#x} name={!r} volume_group={!r} thin_pool={!r}>'.\ + format( + type(self).__name__, id(self), + self.name, self.volume_group, self.thin_pool) + @property def config(self): return { 'name': self.name, 'volume_group': self.volume_group, 'thin_pool': self.thin_pool, - 'driver': ThinPool.driver + 'driver': ThinPool.driver, + 'revisions_to_keep': self.revisions_to_keep, } def destroy(self): @@ -131,20 +166,16 @@ class ThinPool(qubes.storage.Pool): continue if vol_info['pool_lv'] != self.thin_pool: continue - if vid.endswith('-snap'): + if vid.endswith('-snap') or vid.endswith('-import'): # implementation detail volume continue if vid.endswith('-back'): # old revisions continue - config = { - 'pool': self, - 'vid': vid, - 'name': vid, - 'volume_group': self.volume_group, - 'rw': vol_info['attr'][1] == 'w', - } - volumes += [ThinVolume(**config)] + volume = self.get_volume(vid) + if volume in volumes: + continue + volumes.append(volume) return volumes @property @@ -164,26 +195,14 @@ class ThinPool(qubes.storage.Pool): return 0 -def init_cache(log=logging.getLogger('qubes.storage.lvm')): - cmd = ['lvs', '--noheadings', '-o', - 'vg_name,pool_lv,name,lv_size,data_percent,lv_attr,origin', - '--units', 'b', '--separator', ';'] - if os.getuid() != 0: - cmd.insert(0, 'sudo') - environ = os.environ.copy() - environ['LC_ALL'] = 'C.utf8' - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - close_fds=True, env=environ) - out, err = p.communicate() - return_code = p.returncode - if return_code == 0 and err: - log.warning(err) - elif return_code != 0: - raise qubes.storage.StoragePoolException(err) +_init_cache_cmd = ['lvs', '--noheadings', '-o', + 'vg_name,pool_lv,name,lv_size,data_percent,lv_attr,origin', + '--units', 'b', '--separator', ';'] +def _parse_lvm_cache(lvm_output): result = {} - for line in out.splitlines(): + for line in lvm_output.splitlines(): line = line.decode().strip() pool_name, pool_lv, name, size, usage_percent, attr, \ origin = line.split(';', 6) @@ -197,9 +216,73 @@ def init_cache(log=logging.getLogger('qubes.storage.lvm')): return result +def init_cache(log=logging.getLogger('qubes.storage.lvm')): + cmd = _init_cache_cmd + if os.getuid() != 0: + cmd = ['sudo'] + cmd + environ = os.environ.copy() + environ['LC_ALL'] = 'C.utf8' + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + close_fds=True, env=environ) + out, err = p.communicate() + return_code = p.returncode + if return_code == 0 and err: + log.warning(err) + elif return_code != 0: + raise qubes.storage.StoragePoolException(err) + + return _parse_lvm_cache(out) + +@asyncio.coroutine +def init_cache_coro(log=logging.getLogger('qubes.storage.lvm')): + cmd = _init_cache_cmd + if os.getuid() != 0: + cmd = ['sudo'] + cmd + environ = os.environ.copy() + environ['LC_ALL'] = 'C.utf8' + p = yield from asyncio.create_subprocess_exec(*cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=True, env=environ) + out, err = yield from p.communicate() + return_code = p.returncode + if return_code == 0 and err: + log.warning(err) + elif return_code != 0: + raise qubes.storage.StoragePoolException(err) + + return _parse_lvm_cache(out) size_cache = init_cache() + +def _revision_sort_key(revision): + '''Sort key for revisions. Sort them by time + + :returns timestamp + ''' + if isinstance(revision, tuple): + revision = revision[0] + if '-' in revision: + revision = revision.split('-')[0] + return int(revision) + +def locked(method): + '''Decorator running given Volume's coroutine under a lock. + Needs to be added after wrapping with @asyncio.coroutine, for example: + + >>>@locked + >>>@asyncio.coroutine + >>>def start(self): + >>> pass + ''' + @asyncio.coroutine + @functools.wraps(method) + def wrapper(self, *args, **kwargs): + with (yield from self._lock): # pylint: disable=protected-access + return (yield from method(self, *args, **kwargs)) + return wrapper + class ThinVolume(qubes.storage.Volume): ''' Default LVM thin volume implementation ''' # pylint: disable=too-few-public-methods @@ -212,12 +295,27 @@ class ThinVolume(qubes.storage.Volume): if self.snap_on_start or self.save_on_stop: self._vid_snap = self.vid + '-snap' + if self.save_on_stop: + self._vid_import = self.vid + '-import' self._size = size + self._lock = asyncio.Lock() @property def path(self): - return '/dev/' + self.vid + return '/dev/' + self._vid_current + + @property + def _vid_current(self): + if self.vid in size_cache: + return self.vid + vol_revisions = self.revisions + if vol_revisions: + last_revision = \ + max(vol_revisions.items(), key=_revision_sort_key)[0] + return self.vid + '-' + last_revision + # detached pool? return expected path + return self.vid @property def revisions(self): @@ -229,7 +327,8 @@ class ThinVolume(qubes.storage.Volume): if not revision_vid.endswith('-back'): continue revision_vid = revision_vid[len(name_prefix):] - seconds = int(revision_vid[:-len('-back')]) + # get revision without suffix + seconds = int(revision_vid.split('-')[0]) iso_date = qubes.storage.isodate(seconds).split('.', 1)[0] revisions[revision_vid] = iso_date return revisions @@ -239,7 +338,7 @@ class ThinVolume(qubes.storage.Volume): try: if self.is_dirty(): return qubes.storage.lvm.size_cache[self._vid_snap]['size'] - return qubes.storage.lvm.size_cache[self.vid]['size'] + return qubes.storage.lvm.size_cache[self._vid_current]['size'] except KeyError: return self._size @@ -248,6 +347,7 @@ class ThinVolume(qubes.storage.Volume): raise qubes.storage.StoragePoolException( "You shouldn't use lvm size setter") + @asyncio.coroutine def _reset(self): ''' Resets a volatile volume ''' assert not self.snap_on_start and not self.save_on_stop, \ @@ -255,14 +355,15 @@ class ThinVolume(qubes.storage.Volume): self.log.debug('Resetting volatile %s', self.vid) try: cmd = ['remove', self.vid] - qubes_lvm(cmd, self.log) + yield from qubes_lvm_coro(cmd, self.log) except qubes.storage.StoragePoolException: pass # pylint: disable=protected-access cmd = ['create', self.pool._pool_id, self.vid.split('/')[1], str(self.size)] - qubes_lvm(cmd, self.log) + yield from qubes_lvm_coro(cmd, self.log) + @asyncio.coroutine def _remove_revisions(self, revisions=None): '''Remove old volume revisions. @@ -273,19 +374,32 @@ class ThinVolume(qubes.storage.Volume): ''' if revisions is None: revisions = sorted(self.revisions.items(), - key=operator.itemgetter(1)) + key=_revision_sort_key) # pylint: disable=invalid-unary-operand-type revisions = revisions[:(-self.revisions_to_keep) or None] revisions = [rev_id for rev_id, _ in revisions] for rev_id in revisions: + # safety check + assert rev_id != self._vid_current try: cmd = ['remove', self.vid + '-' + rev_id] - qubes_lvm(cmd, self.log) + yield from qubes_lvm_coro(cmd, self.log) except qubes.storage.StoragePoolException: pass - def _commit(self): + @asyncio.coroutine + def _commit(self, vid_to_commit=None, keep=False): + ''' + Commit temporary volume into current one. By default + :py:attr:`_vid_snap` is used (which is created by :py:meth:`start()`), + but can be overriden by *vid_to_commit* argument. + + :param vid_to_commit: LVM volume ID to commit into this one + :param keep: whether to keep or not *vid_to_commit*. + IOW use 'clone' or 'rename' methods. + :return: None + ''' msg = "Trying to commit {!s}, but it has save_on_stop == False" msg = msg.format(self) assert self.save_on_stop, msg @@ -293,39 +407,41 @@ class ThinVolume(qubes.storage.Volume): msg = "Trying to commit {!s}, but it has rw == False" msg = msg.format(self) assert self.rw, msg - assert hasattr(self, '_vid_snap') + if vid_to_commit is None: + assert hasattr(self, '_vid_snap') + vid_to_commit = self._vid_snap - if self.revisions_to_keep > 0: - cmd = ['clone', self.vid, - '{}-{}-back'.format(self.vid, int(time.time()))] - qubes_lvm(cmd, self.log) - reset_cache() - self._remove_revisions() + assert self._lock.locked() + if not os.path.exists('/dev/' + vid_to_commit): + # nothing to commit + return - # TODO: when converting this function to coroutine, this _must_ be - # under a lock - # remove old volume only after _successful_ clone of the new one - cmd = ['rename', self.vid, self.vid + '-tmp'] - qubes_lvm(cmd, self.log) - try: - cmd = ['clone', self._vid_snap, self.vid] - qubes_lvm(cmd, self.log) - except: - # restore original volume - cmd = ['rename', self.vid + '-tmp', self.vid] - qubes_lvm(cmd, self.log) - raise - else: - cmd = ['remove', self.vid + '-tmp'] - qubes_lvm(cmd, self.log) + if self._vid_current == self.vid: + cmd = ['rename', self.vid, + '{}-{}-back'.format(self.vid, int(time.time()))] + yield from qubes_lvm_coro(cmd, self.log) + yield from reset_cache_coro() + cmd = ['clone' if keep else 'rename', + vid_to_commit, + self.vid] + yield from qubes_lvm_coro(cmd, self.log) + yield from reset_cache_coro() + # make sure the one we've committed right now is properly + # detected as the current one - before removing anything + assert self._vid_current == self.vid + # and remove old snapshots, if needed + yield from self._remove_revisions() + + @locked + @asyncio.coroutine def create(self): assert self.vid assert self.size if self.save_on_stop: if self.source: - cmd = ['clone', str(self.source), self.vid] + cmd = ['clone', self.source.path, self.vid] else: cmd = [ 'create', @@ -333,70 +449,126 @@ class ThinVolume(qubes.storage.Volume): self.vid.split('/', 1)[1], str(self.size) ] - qubes_lvm(cmd, self.log) - reset_cache() + yield from qubes_lvm_coro(cmd, self.log) + yield from reset_cache_coro() return self + @locked + @asyncio.coroutine def remove(self): assert self.vid try: if os.path.exists('/dev/' + self._vid_snap): cmd = ['remove', self._vid_snap] - qubes_lvm(cmd, self.log) + yield from qubes_lvm_coro(cmd, self.log) except AttributeError: pass - self._remove_revisions(self.revisions.keys()) + try: + if os.path.exists('/dev/' + self._vid_import): + cmd = ['remove', self._vid_import] + yield from qubes_lvm_coro(cmd, self.log) + except AttributeError: + pass + + yield from self._remove_revisions(self.revisions.keys()) if not os.path.exists(self.path): return - cmd = ['remove', self.vid] - qubes_lvm(cmd, self.log) - reset_cache() + cmd = ['remove', self.path] + yield from qubes_lvm_coro(cmd, self.log) + yield from reset_cache_coro() # pylint: disable=protected-access self.pool._volume_objects_cache.pop(self.vid, None) def export(self): ''' Returns an object that can be `open()`. ''' # make sure the device node is available - qubes_lvm(['activate', self.vid], self.log) - devpath = '/dev/' + self.vid + qubes_lvm(['activate', self.path], self.log) + devpath = self.path return devpath + @locked @asyncio.coroutine def import_volume(self, src_volume): if not src_volume.save_on_stop: return self + if self.is_dirty(): + raise qubes.storage.StoragePoolException( + 'Cannot import to dirty volume {} -' + ' start and stop a qube to cleanup'.format(self.vid)) + self.abort_if_import_in_progress() # HACK: neat trick to speed up testing if you have same physical thin # pool assigned to two qubes-pools i.e: qubes_dom0 and test-lvm # pylint: disable=line-too-long if isinstance(src_volume.pool, ThinPool) and \ src_volume.pool.thin_pool == self.pool.thin_pool: # NOQA - cmd = ['remove', self.vid] - qubes_lvm(cmd, self.log) - cmd = ['clone', str(src_volume), str(self)] - qubes_lvm(cmd, self.log) + yield from self._commit(src_volume.path[len('/dev/'):], keep=True) else: - if src_volume.size != self.size: - self.resize(src_volume.size) + cmd = ['create', + self.pool._pool_id, # pylint: disable=protected-access + self._vid_import.split('/')[1], + str(src_volume.size)] + yield from qubes_lvm_coro(cmd, self.log) src_path = src_volume.export() - cmd = ['dd', 'if=' + src_path, 'of=/dev/' + self.vid, - 'conv=sparse'] + cmd = ['dd', 'if=' + src_path, 'of=/dev/' + self._vid_import, + 'conv=sparse', 'status=none'] + if not os.access('/dev/' + self._vid_import, os.W_OK) or \ + not os.access(src_path, os.R_OK): + cmd.insert(0, 'sudo') + p = yield from asyncio.create_subprocess_exec(*cmd) yield from p.wait() if p.returncode != 0: + cmd = ['remove', self._vid_import] + yield from qubes_lvm_coro(cmd, self.log) raise qubes.storage.StoragePoolException( 'Failed to import volume {!r}, dd exit code: {}'.format( src_volume, p.returncode)) - reset_cache() + yield from self._commit(self._vid_import) return self + @locked + @asyncio.coroutine def import_data(self): ''' Returns an object that can be `open()`. ''' - devpath = '/dev/' + self.vid + if self.is_dirty(): + raise qubes.storage.StoragePoolException( + 'Cannot import data to dirty volume {}, stop the qube first'. + format(self.vid)) + self.abort_if_import_in_progress() + # pylint: disable=protected-access + cmd = ['create', self.pool._pool_id, self._vid_import.split('/')[1], + str(self.size)] + yield from qubes_lvm_coro(cmd, self.log) + yield from reset_cache_coro() + devpath = '/dev/' + self._vid_import return devpath + @locked + @asyncio.coroutine + def import_data_end(self, success): + '''Either commit imported data, or discard temporary volume''' + if not os.path.exists('/dev/' + self._vid_import): + raise qubes.storage.StoragePoolException( + 'No import operation in progress on {}'.format(self.vid)) + if success: + yield from self._commit(self._vid_import) + else: + cmd = ['remove', self._vid_import] + yield from qubes_lvm_coro(cmd, self.log) + + def abort_if_import_in_progress(self): + try: + devpath = '/dev/' + self._vid_import + if os.path.exists(devpath): + raise qubes.storage.StoragePoolException( + 'Import operation in progress on {}'.format(self.vid)) + except AttributeError: # self._vid_import + # no vid_import - import definitely not in progress + pass + def is_dirty(self): if self.save_on_stop: return os.path.exists('/dev/' + self._vid_snap) @@ -408,25 +580,34 @@ class ThinVolume(qubes.storage.Volume): if self._vid_snap not in size_cache: return False return (size_cache[self._vid_snap]['origin'] != - self.source.vid.split('/')[1]) - + self.source.path.split('/')[-1]) + @locked + @asyncio.coroutine def revert(self, revision=None): + if self.is_dirty(): + raise qubes.storage.StoragePoolException( + 'Cannot revert dirty volume {}, stop the qube first'.format( + self.vid)) + self.abort_if_import_in_progress() if revision is None: revision = \ - max(self.revisions.items(), key=operator.itemgetter(1))[0] - old_path = self.path + '-' + revision + max(self.revisions.items(), key=_revision_sort_key)[0] + old_path = '/dev/' + self.vid + '-' + revision if not os.path.exists(old_path): msg = "Volume {!s} has no {!s}".format(self, old_path) raise qubes.storage.StoragePoolException(msg) - cmd = ['remove', self.vid] - qubes_lvm(cmd, self.log) + if self.vid in size_cache: + cmd = ['remove', self.vid] + yield from qubes_lvm_coro(cmd, self.log) cmd = ['clone', self.vid + '-' + revision, self.vid] - qubes_lvm(cmd, self.log) - reset_cache() + yield from qubes_lvm_coro(cmd, self.log) + yield from reset_cache_coro() return self + @locked + @asyncio.coroutine def resize(self, size): ''' Expands volume, throws :py:class:`qubst.storage.qubes.storage.StoragePoolException` if @@ -448,49 +629,58 @@ class ThinVolume(qubes.storage.Volume): if self.is_dirty(): cmd = ['extend', self._vid_snap, str(size)] - qubes_lvm(cmd, self.log) + yield from qubes_lvm_coro(cmd, self.log) + elif hasattr(self, '_vid_import') and \ + os.path.exists('/dev/' + self._vid_import): + cmd = ['extend', self._vid_import, str(size)] + yield from qubes_lvm_coro(cmd, self.log) elif self.save_on_stop or not self.snap_on_start: - cmd = ['extend', self.vid, str(size)] - qubes_lvm(cmd, self.log) - reset_cache() + cmd = ['extend', self._vid_current, str(size)] + yield from qubes_lvm_coro(cmd, self.log) + yield from reset_cache_coro() + @asyncio.coroutine def _snapshot(self): try: cmd = ['remove', self._vid_snap] - qubes_lvm(cmd, self.log) + yield from qubes_lvm_coro(cmd, self.log) except: # pylint: disable=bare-except pass if self.source is None: - cmd = ['clone', self.vid, self._vid_snap] + cmd = ['clone', self._vid_current, self._vid_snap] else: - cmd = ['clone', str(self.source), self._vid_snap] - qubes_lvm(cmd, self.log) - + cmd = ['clone', self.source.path, self._vid_snap] + yield from qubes_lvm_coro(cmd, self.log) + @locked + @asyncio.coroutine def start(self): + self.abort_if_import_in_progress() try: if self.snap_on_start or self.save_on_stop: if not self.save_on_stop or not self.is_dirty(): - self._snapshot() + yield from self._snapshot() else: - self._reset() + yield from self._reset() finally: - reset_cache() + yield from reset_cache_coro() return self + @locked + @asyncio.coroutine def stop(self): try: if self.save_on_stop: - self._commit() - if self.snap_on_start or self.save_on_stop: + yield from self._commit() + if self.snap_on_start and not self.save_on_stop: cmd = ['remove', self._vid_snap] - qubes_lvm(cmd, self.log) - else: + yield from qubes_lvm_coro(cmd, self.log) + elif not self.snap_on_start and not self.save_on_stop: cmd = ['remove', self.vid] - qubes_lvm(cmd, self.log) + yield from qubes_lvm_coro(cmd, self.log) finally: - reset_cache() + yield from reset_cache_coro() return self def verify(self): @@ -499,9 +689,9 @@ class ThinVolume(qubes.storage.Volume): # volatile volumes don't need any files return True if self.source is not None: - vid = str(self.source) + vid = self.source.path[len('/dev/'):] else: - vid = self.vid + vid = self._vid_current try: vol_info = size_cache[vid] if vol_info['attr'][4] != 'a': @@ -528,7 +718,7 @@ class ThinVolume(qubes.storage.Volume): def usage(self): # lvm thin usage always returns at least the same usage as # the parent try: - return qubes.storage.lvm.size_cache[self.vid]['usage'] + return qubes.storage.lvm.size_cache[self._vid_current]['usage'] except KeyError: return 0 @@ -541,9 +731,14 @@ def pool_exists(pool_id): except KeyError: return False +def _get_lvm_cmdline(cmd): + ''' Build command line for :program:`lvm` call. + The purpose of this function is to keep all the detailed lvm options in + one place. -def qubes_lvm(cmd, log=logging.getLogger('qubes.storage.lvm')): - ''' Call :program:`lvm` to execute an LVM operation ''' + :param cmd: array of str, where cmd[0] is action and the rest are arguments + :return array of str appropriate for subprocess.Popen + ''' action = cmd[0] if action == 'remove': lvm_cmd = ['lvremove', '-f', cmd[1]] @@ -568,21 +763,57 @@ def qubes_lvm(cmd, log=logging.getLogger('qubes.storage.lvm')): cmd = ['sudo', 'lvm'] + lvm_cmd else: cmd = ['lvm'] + lvm_cmd + + return cmd + +def _process_lvm_output(returncode, stdout, stderr, log): + '''Process output of LVM, determine if the call was successful and + possibly log warnings.''' + # Filter out warning about intended over-provisioning. + # Upstream discussion about missing option to silence it: + # https://bugzilla.redhat.com/1347008 + err = '\n'.join(line for line in stderr.decode().splitlines() + if 'exceeds the size of thin pool' not in line) + if stdout: + log.debug(stdout) + if returncode == 0 and err: + log.warning(err) + elif returncode != 0: + assert err, "Command exited unsuccessful, but printed nothing to stderr" + err = err.replace('%', '%%') + raise qubes.storage.StoragePoolException(err) + return True + +def qubes_lvm(cmd, log=logging.getLogger('qubes.storage.lvm')): + ''' Call :program:`lvm` to execute an LVM operation ''' + # the only caller for this non-coroutine version is ThinVolume.export() + cmd = _get_lvm_cmdline(cmd) environ = os.environ.copy() environ['LC_ALL'] = 'C.utf8' p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True, env=environ) out, err = p.communicate() - return_code = p.returncode - if out: - log.debug(out) - if return_code == 0 and err: - log.warning(err) - elif return_code != 0: - assert err, "Command exited unsuccessful, but printed nothing to stderr" - raise qubes.storage.StoragePoolException(err) - return True + return _process_lvm_output(p.returncode, out, err, log) + +@asyncio.coroutine +def qubes_lvm_coro(cmd, log=logging.getLogger('qubes.storage.lvm')): + ''' Call :program:`lvm` to execute an LVM operation + + Coroutine version of :py:func:`qubes_lvm`''' + cmd = _get_lvm_cmdline(cmd) + environ = os.environ.copy() + environ['LC_ALL'] = 'C.utf8' + p = yield from asyncio.create_subprocess_exec(*cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=True, env=environ) + out, err = yield from p.communicate() + return _process_lvm_output(p.returncode, out, err, log) def reset_cache(): qubes.storage.lvm.size_cache = init_cache() + +@asyncio.coroutine +def reset_cache_coro(): + qubes.storage.lvm.size_cache = yield from init_cache_coro() diff --git a/qubes/storage/reflink.py b/qubes/storage/reflink.py index 574febb3..3f250011 100644 --- a/qubes/storage/reflink.py +++ b/qubes/storage/reflink.py @@ -22,13 +22,14 @@ but not required. ''' +import asyncio import collections import errno import fcntl +import functools import glob import logging import os -import re import subprocess import tempfile from contextlib import contextmanager, suppress @@ -36,7 +37,8 @@ from contextlib import contextmanager, suppress import qubes.storage BLKSIZE = 512 -FICLONE = 1074041865 # see ioctl_ficlone manpage +FICLONE = 1074041865 # defined in +LOOP_SET_CAPACITY = 0x4C07 # defined in LOGGER = logging.getLogger('qubes.storage.reflink') @@ -53,7 +55,7 @@ class ReflinkPool(qubes.storage.Pool): def setup(self): created = _make_dir(self.dir_path) - if self.setup_check and not is_reflink_supported(self.dir_path): + if self.setup_check and not is_supported(self.dir_path): if created: _remove_empty_dir(self.dir_path) raise qubes.storage.StoragePoolException( @@ -115,12 +117,37 @@ class ReflinkPool(qubes.storage.Pool): [pool for pool in app.pools.values() if pool is not self], self.dir_path) + +def _unblock(method): + ''' Decorator transforming a synchronous volume method into a + coroutine that runs the original method in the event loop's + thread-based default executor, under a per-volume lock. + ''' + @asyncio.coroutine + @functools.wraps(method) + def wrapper(self, *args, **kwargs): + with (yield from self._lock): # pylint: disable=protected-access + return (yield from asyncio.get_event_loop().run_in_executor( + None, functools.partial(method, self, *args, **kwargs))) + return wrapper + class ReflinkVolume(qubes.storage.Volume): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._lock = asyncio.Lock() + self._path_vid = os.path.join(self.pool.dir_path, self.vid) + self._path_clean = self._path_vid + '.img' + self._path_dirty = self._path_vid + '-dirty.img' + self._path_import = self._path_vid + '-import.img' + self.path = self._path_dirty + + @_unblock def create(self): if self.save_on_stop and not self.snap_on_start: _create_sparse_file(self._path_clean, self.size) return self + @_unblock def verify(self): if self.snap_on_start: img = self.source._path_clean # pylint: disable=protected-access @@ -132,19 +159,26 @@ class ReflinkVolume(qubes.storage.Volume): if img is None or os.path.exists(img): return True raise qubes.storage.StoragePoolException( - 'Missing image file {!r} for volume {!s}'.format(img, self.vid)) + 'Missing image file {!r} for volume {}'.format(img, self.vid)) + @_unblock def remove(self): ''' Drop volume object from pool; remove volume images from oldest to newest; remove empty VM directory. ''' self.pool._volumes.pop(self, None) # pylint: disable=protected-access + self._cleanup() self._prune_revisions(keep=0) _remove_file(self._path_clean) _remove_file(self._path_dirty) _remove_empty_dir(os.path.dirname(self._path_dirty)) return self + def _cleanup(self): + for tmp in glob.iglob(glob.escape(self._path_vid) + '*.img*~*'): + _remove_file(tmp) + _remove_file(self._path_import) + def is_outdated(self): if self.snap_on_start: with suppress(FileNotFoundError): @@ -156,7 +190,9 @@ class ReflinkVolume(qubes.storage.Volume): def is_dirty(self): return self.save_on_stop and os.path.exists(self._path_dirty) + @_unblock def start(self): + self._cleanup() if self.is_dirty(): # implies self.save_on_stop return self if self.snap_on_start: @@ -168,24 +204,23 @@ class ReflinkVolume(qubes.storage.Volume): _create_sparse_file(self._path_dirty, self.size) return self + @_unblock def stop(self): if self.save_on_stop: - self._commit() + self._commit(self._path_dirty) else: _remove_file(self._path_dirty) _remove_file(self._path_clean) return self - def _commit(self): + def _commit(self, path_from): self._add_revision() self._prune_revisions() - _rename_file(self._path_dirty, self._path_clean) + _rename_file(path_from, self._path_clean) def _add_revision(self): if self.revisions_to_keep == 0: return - if _get_file_disk_usage(self._path_clean) == 0: - return ctime = os.path.getctime(self._path_clean) timestamp = qubes.storage.isodate(int(ctime)) _copy_file(self._path_clean, @@ -198,7 +233,11 @@ class ReflinkVolume(qubes.storage.Volume): for number, timestamp in list(self.revisions.items())[:-keep or None]: _remove_file(self._path_revision(number, timestamp)) + @_unblock def revert(self, revision=None): + if self.is_dirty(): + raise qubes.storage.StoragePoolException( + 'Cannot revert: {} is not cleanly stopped'.format(self.vid)) if revision is None: number, timestamp = list(self.revisions.items())[-1] else: @@ -208,61 +247,58 @@ class ReflinkVolume(qubes.storage.Volume): _rename_file(path_revision, self._path_clean) return self + @_unblock def resize(self, size): ''' Expand a read-write volume image; notify any corresponding loop devices of the size change. ''' if not self.rw: raise qubes.storage.StoragePoolException( - 'Cannot resize: {!s} is read-only'.format(self.vid)) + 'Cannot resize: {} is read-only'.format(self.vid)) if size < self.size: raise qubes.storage.StoragePoolException( - 'For your own safety, shrinking of {!s} is disabled' - ' ({:d} < {:d}). If you really know what you are doing,' + 'For your own safety, shrinking of {} is disabled' + ' ({} < {}). If you really know what you are doing,' ' use "truncate" manually.'.format(self.vid, size, self.size)) try: # assume volume is not (cleanly) stopped ... _resize_file(self._path_dirty, size) + self.size = size except FileNotFoundError: # ... but it actually is. _resize_file(self._path_clean, size) + self.size = size + return self - self.size = size - - # resize any corresponding loop devices - out = _cmd('losetup', '--associated', self._path_dirty) - for match in re.finditer(br'^(/dev/loop[0-9]+): ', out, re.MULTILINE): - loop_dev = match.group(1).decode('ascii') - _cmd('losetup', '--set-capacity', loop_dev) - + _update_loopdev_sizes(self._path_dirty) return self - def _require_save_on_stop(self, method_name): + def export(self): if not self.save_on_stop: raise NotImplementedError( - 'Cannot {!s}: {!s} is not save_on_stop'.format( - method_name, self.vid)) - - def export(self): - self._require_save_on_stop('export') + 'Cannot export: {} is not save_on_stop'.format(self.vid)) return self._path_clean def import_data(self): - self._require_save_on_stop('import_data') - _create_sparse_file(self._path_dirty, self.size) - return self._path_dirty + if not self.save_on_stop: + raise NotImplementedError( + 'Cannot import_data: {} is not save_on_stop'.format(self.vid)) + _create_sparse_file(self._path_import, self.size) + return self._path_import def import_data_end(self, success): if success: - self._commit() + self._commit(self._path_import) else: - _remove_file(self._path_dirty) + _remove_file(self._path_import) return self + @_unblock def import_volume(self, src_volume): - self._require_save_on_stop('import_volume') + if not self.save_on_stop: + return self try: - _copy_file(src_volume.export(), self._path_dirty) + _copy_file(src_volume.export(), self._path_import) except: self.import_data_end(False) raise @@ -274,18 +310,6 @@ class ReflinkVolume(qubes.storage.Volume): timestamp = self.revisions[number] return self._path_clean + '.' + number + '@' + timestamp + 'Z' - @property - def _path_clean(self): - return os.path.join(self.pool.dir_path, self.vid + '.img') - - @property - def _path_dirty(self): - return os.path.join(self.pool.dir_path, self.vid + '-dirty.img') - - @property - def path(self): - return self._path_dirty - @property def _next_revision_number(self): numbers = self.revisions.keys() @@ -296,10 +320,10 @@ class ReflinkVolume(qubes.storage.Volume): @property def revisions(self): prefix = self._path_clean + '.' - paths = glob.glob(glob.escape(prefix) + '*@*Z') - items = sorted((path[len(prefix):-1].split('@') for path in paths), - key=lambda item: int(item[0])) - return collections.OrderedDict(items) + paths = glob.iglob(glob.escape(prefix) + '*@*Z') + items = (path[len(prefix):-1].split('@') for path in paths) + return collections.OrderedDict(sorted(items, + key=lambda item: int(item[0]))) @property def usage(self): @@ -391,39 +415,48 @@ def _create_sparse_file(path, size): tmp.truncate(size) LOGGER.info('Created sparse file: %s', tmp.name) +def _update_loopdev_sizes(img): + ''' Resolve img; update the size of loop devices backed by it. ''' + needle = os.fsencode(os.path.realpath(img)) + b'\n' + for sys_path in glob.iglob('/sys/block/loop[0-9]*/loop/backing_file'): + try: + with open(sys_path, 'rb') as sys_io: + if sys_io.read() != needle: + continue + except FileNotFoundError: + continue + with open('/dev/' + sys_path.split('/')[3]) as dev_io: + fcntl.ioctl(dev_io.fileno(), LOOP_SET_CAPACITY) + +def _attempt_ficlone(src, dst): + try: + fcntl.ioctl(dst.fileno(), FICLONE, src.fileno()) + return True + except OSError: + return False + def _copy_file(src, dst): ''' Copy src to dst as a reflink if possible, sparse if not. ''' - if not os.path.exists(src): - raise FileNotFoundError(src) - with _replace_file(dst) as tmp: - LOGGER.info('Copying file: %s -> %s', src, tmp.name) - _cmd('cp', '--sparse=always', '--reflink=auto', src, tmp.name) + with _replace_file(dst) as tmp_io: + with open(src, 'rb') as src_io: + if _attempt_ficlone(src_io, tmp_io): + LOGGER.info('Reflinked file: %s -> %s', src, tmp_io.name) + return True + LOGGER.info('Copying file: %s -> %s', src, tmp_io.name) + cmd = 'cp', '--sparse=always', src, tmp_io.name + p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if p.returncode != 0: + raise qubes.storage.StoragePoolException(str(p)) + return False -def _cmd(*args): - ''' Run command until finished; return stdout (as bytes) if it - exited 0. Otherwise, raise a detailed StoragePoolException. - ''' - try: - return subprocess.run(args, check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE).stdout - except subprocess.CalledProcessError as ex: - msg = '{!s} err={!r} out={!r}'.format(ex, ex.stderr, ex.stdout) - raise qubes.storage.StoragePoolException(msg) from ex - -def is_reflink_supported(dst_dir, src_dir=None): +def is_supported(dst_dir, src_dir=None): ''' Return whether destination directory supports reflink copies from source directory. (A temporary file is created in each directory, using O_TMPFILE if possible.) ''' if src_dir is None: src_dir = dst_dir - dst = tempfile.TemporaryFile(dir=dst_dir) - src = tempfile.TemporaryFile(dir=src_dir) - src.write(b'foo') # don't let any filesystem get clever with empty files - - try: - fcntl.ioctl(dst.fileno(), FICLONE, src.fileno()) - return True - except OSError: - return False + with tempfile.TemporaryFile(dir=src_dir) as src, \ + tempfile.TemporaryFile(dir=dst_dir) as dst: + src.write(b'foo') # don't let any fs get clever with empty files + return _attempt_ficlone(src, dst) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index 865534e2..cd44bef5 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -379,8 +379,27 @@ class QubesTestCase(unittest.TestCase): self.loop = asyncio.get_event_loop() self.addCleanup(self.cleanup_loop) + self.addCleanup(self.cleanup_traceback) self.addCleanup(qubes.ext.pci._cache_get.cache_clear) + def cleanup_traceback(self): + '''Remove local variables reference from tracebacks to allow garbage + collector to clean all Qubes*() objects, otherwise file descriptors + held by them will leak''' + exc_infos = [e for test_case, e in self._outcome.errors + if test_case is self] + if self._outcome.expectedFailure: + exc_infos.append(self._outcome.expectedFailure) + for exc_info in exc_infos: + if exc_info is None: + continue + ex = exc_info[1] + while ex is not None: + if isinstance(ex, qubes.exc.QubesVMError): + ex.vm = None + traceback.clear_frames(ex.__traceback__) + ex = ex.__context__ + def cleanup_gc(self): gc.collect() leaked = [obj for obj in gc.get_objects() + gc.garbage @@ -397,6 +416,8 @@ class QubesTestCase(unittest.TestCase): except ImportError: pass + # do not keep leaked object references in locals() + leaked = bool(leaked) assert not leaked def cleanup_loop(self): @@ -421,6 +442,13 @@ class QubesTestCase(unittest.TestCase): except asyncio.TimeoutError: raise AssertionError('libvirt event impl drain timeout') + # this is stupid, but apparently it requires two passes + # to cleanup SIGCHLD handlers + self.loop.stop() + self.loop.run_forever() + self.loop.stop() + self.loop.run_forever() + # Check there are no Tasks left. assert not self.loop._ready assert not self.loop._scheduled @@ -763,20 +791,6 @@ class SystemTestCase(QubesTestCase): vmname = vm.name app = vm.app - # avoid race with DispVM.auto_cleanup=True - try: - self.loop.run_until_complete( - asyncio.wait_for(vm.startup_lock.acquire(), 10)) - except asyncio.TimeoutError: - pass - - try: - # XXX .is_running() may throw libvirtError if undefined - if vm.is_running(): - self.loop.run_until_complete(vm.kill()) - except: # pylint: disable=bare-except - pass - try: self.loop.run_until_complete(vm.remove_from_disk()) except: # pylint: disable=bare-except @@ -842,7 +856,7 @@ class SystemTestCase(QubesTestCase): ''' try: volumes = subprocess.check_output( - ['sudo', 'lvs', '--noheadings', '-o', 'vg_name,name', + ['lvs', '--noheadings', '-o', 'vg_name,name', '--separator', '/']).decode() if ('/vm-' + prefix) not in volumes: return @@ -857,18 +871,36 @@ class SystemTestCase(QubesTestCase): vms = list(vms) if not vms: return + # first kill all the domains, to avoid side effects of changing netvm + for vm in vms: + try: + # XXX .is_running() may throw libvirtError if undefined + if vm.is_running(): + self.loop.run_until_complete(vm.kill()) + except: # pylint: disable=bare-except + pass # break dependencies for vm in vms: vm.default_dispvm = None - # then remove in reverse topological order (wrt netvm), using naive + vm.netvm = None + # take app instance from any VM to be removed + app = vms[0].app + if app.default_dispvm in vms: + app.default_dispvm = None + if app.default_netvm in vms: + app.default_netvm = None + del app + # then remove in reverse topological order (wrt template), using naive # algorithm - # this heavily depends on lack of netvm loops + # this heavily depends on lack of template loops, but those are + # impossible while vms: vm = vms.pop(0) # make sure that all connected VMs are going to be removed, # otherwise this will loop forever - assert all(x in vms for x in vm.connected_vms) - if list(vm.connected_vms): + child_vms = list(getattr(vm, 'appvms', [])) + assert all(x in vms for x in child_vms) + if child_vms: # if still something use this VM, put it at the end of queue # and try next one vms.append(vm) @@ -876,9 +908,16 @@ class SystemTestCase(QubesTestCase): self._remove_vm_qubes(vm) def remove_test_vms(self, xmlpath=XMLPATH, prefix=VMPREFIX): - '''Aggresively remove any domain that has name in testing namespace. + '''Aggressively remove any domain that has name in testing namespace. + + :param prefix: name prefix of VMs to remove, can be a list of prefixes ''' + if isinstance(prefix, str): + prefixes = [prefix] + else: + prefixes = prefix + del prefix # first, remove them Qubes-way if os.path.exists(xmlpath): try: @@ -891,7 +930,7 @@ class SystemTestCase(QubesTestCase): except AttributeError: host_app = qubes.Qubes() self.remove_vms([vm for vm in app.domains - if vm.name.startswith(prefix) or + if any(vm.name.startswith(prefix) for prefix in prefixes) or (isinstance(vm, qubes.vm.dispvm.DispVM) and vm.name not in host_app.domains)]) if not hasattr(self, 'host_app'): @@ -907,7 +946,7 @@ class SystemTestCase(QubesTestCase): # now remove what was only in libvirt conn = libvirt.open(qubes.config.defaults['libvirt_uri']) for dom in conn.listAllDomains(): - if dom.name().startswith(prefix): + if any(dom.name().startswith(prefix) for prefix in prefixes): self._remove_vm_libvirt(dom) conn.close() @@ -922,11 +961,12 @@ class SystemTestCase(QubesTestCase): if not os.path.exists(dirpath): continue for name in os.listdir(dirpath): - if name.startswith(prefix): + if any(name.startswith(prefix) for prefix in prefixes): vmnames.add(name) for vmname in vmnames: self._remove_vm_disk(vmname) - self._remove_vm_disk_lvm(prefix) + for prefix in prefixes: + self._remove_vm_disk_lvm(prefix) def qrexec_policy(self, service, source, destination, allow=True, action=None): @@ -943,7 +983,25 @@ class SystemTestCase(QubesTestCase): return _QrexecPolicyContext(service, source, destination, allow=allow, action=action) - def wait_for_window(self, title, timeout=30, show=True): + @asyncio.coroutine + def wait_for_window_hide_coro(self, title, winid, timeout=30): + """ + Wait for window do disappear + :param winid: window id + :return: + """ + wait_count = 0 + while subprocess.call(['xdotool', 'getwindowname', str(winid)], + stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) == 0: + wait_count += 1 + if wait_count > timeout * 10: + self.fail("Timeout while waiting for {}({}) window to " + "disappear".format(title, winid)) + yield from asyncio.sleep(0.1) + + @asyncio.coroutine + def wait_for_window_coro(self, title, search_class=False, timeout=30, + show=True): """ Wait for a window with a given title. Depending on show parameter, it will wait for either window to show or to disappear. @@ -952,19 +1010,59 @@ class SystemTestCase(QubesTestCase): :param timeout: timeout of the operation, in seconds :param show: if True - wait for the window to be visible, otherwise - to not be visible - :return: None + :param search_class: search based on window class instead of title + :return: window id of found window, if show=True """ - wait_count = 0 - while subprocess.call(['xdotool', 'search', '--name', title], - stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) \ - != int(not show): - wait_count += 1 - if wait_count > timeout*10: - self.fail("Timeout while waiting for {} window to {}".format( - title, "show" if show else "hide") - ) - self.loop.run_until_complete(asyncio.sleep(0.1)) + xdotool_search = ['xdotool', 'search', '--onlyvisible'] + if search_class: + xdotool_search.append('--class') + else: + xdotool_search.append('--name') + if show: + xdotool_search.append('--sync') + if not show: + try: + winid = subprocess.check_output(xdotool_search + [title], + stderr=subprocess.DEVNULL).decode() + except subprocess.CalledProcessError: + # already gone + return + yield from self.wait_for_window_hide_coro(winid, title, + timeout=timeout) + return + + winid = None + while not winid: + p = yield from asyncio.create_subprocess_exec( + *xdotool_search, title, + stderr=subprocess.DEVNULL, stdout=subprocess.PIPE) + try: + (winid, _) = yield from asyncio.wait_for( + p.communicate(), timeout) + # don't check exit code, getting winid on stdout is enough + # indicator of success; specifically ignore xdotool failing + # with BadWindow or such - when some window appears only for a + # moment by xdotool didn't manage to get its properties + except asyncio.TimeoutError: + self.fail( + "Timeout while waiting for {} window to show".format(title)) + return winid.decode().strip() + + def wait_for_window(self, *args, **kwargs): + """ + Wait for a window with a given title. Depending on show parameter, + it will wait for either window to show or to disappear. + + :param title: title of the window to wait for + :param timeout: timeout of the operation, in seconds + :param show: if True - wait for the window to be visible, + otherwise - to not be visible + :param search_class: search based on window class instead of title + :return: window id of found window, if show=True + """ + return self.loop.run_until_complete( + self.wait_for_window_coro(*args, **kwargs)) def enter_keys_in_window(self, title, keys): """ @@ -985,15 +1083,12 @@ class SystemTestCase(QubesTestCase): subprocess.check_call(command) def shutdown_and_wait(self, vm, timeout=60): - self.loop.run_until_complete(vm.shutdown()) - while timeout > 0: - if not vm.is_running(): - return - self.loop.run_until_complete(asyncio.sleep(1)) - timeout -= 1 - name = vm.name - del vm - self.fail("Timeout while waiting for VM {} shutdown".format(name)) + try: + self.loop.run_until_complete(vm.shutdown(wait=True, timeout=timeout)) + except qubes.exc.QubesException: + name = vm.name + del vm + self.fail("Timeout while waiting for VM {} shutdown".format(name)) def prepare_hvm_system_linux(self, vm, init_script, extra_files=None): if not os.path.exists('/usr/lib/grub/i386-pc'): @@ -1109,21 +1204,30 @@ class SystemTestCase(QubesTestCase): @asyncio.coroutine def wait_for_session(self, vm): + timeout = 30 + if getattr(vm, 'template', None) and 'whonix-ws' in vm.template.name: + # first boot of whonix-ws takes more time because of /home + # initialization, including Tor Browser copying + timeout = 120 yield from asyncio.wait_for( vm.run_service_for_stdio( 'qubes.WaitForSession', input=vm.default_user.encode()), - timeout=30) + timeout=timeout) _templates = None def list_templates(): '''Returns tuple of template names available in the system.''' global _templates + if _templates is None: + if 'QUBES_TEST_TEMPLATES' in os.environ: + _templates = os.environ['QUBES_TEST_TEMPLATES'].split() if _templates is None: try: app = qubes.Qubes() _templates = tuple(vm.name for vm in app.domains - if isinstance(vm, qubes.vm.templatevm.TemplateVM)) + if isinstance(vm, qubes.vm.templatevm.TemplateVM) and + vm.features.get('os', None) != 'Windows') app.close() del app except OSError: @@ -1165,12 +1269,29 @@ def create_testcases_for_templates(name, *bases, module, **kwds): for template in list_templates(): clsname = name + '_' + template + if hasattr(module, clsname): + continue cls = type(clsname, bases, {'template': template, **kwds}) cls.__module__ = module.__name__ # XXX I wonder what other __dunder__ attrs did I miss setattr(module, clsname, cls) yield '.'.join((module.__name__, clsname)) +def maybe_create_testcases_on_import(create_testcases_gen): + '''If certain conditions are met, call *create_testcases_gen* to create + testcases for templates tests. The purpose is to use it on integration + tests module(s) import, so the test runner could discover tests without + using load tests protocol. + + The conditions - any of: + - QUBES_TEST_TEMPLATES present in the environment (it's possible to + create test cases without opening qubes.xml) + - QUBES_TEST_LOAD_ALL present in the environment + ''' + if 'QUBES_TEST_TEMPLATES' in os.environ or \ + 'QUBES_TEST_LOAD_ALL' in os.environ: + list(create_testcases_gen()) + def extra_info(obj): '''Return short info identifying object. @@ -1203,6 +1324,7 @@ def load_tests(loader, tests, pattern): # pylint: disable=unused-argument 'qubes.tests.vm.init', 'qubes.tests.storage', 'qubes.tests.storage_file', + 'qubes.tests.storage_reflink', 'qubes.tests.storage_lvm', 'qubes.tests.storage_kernels', 'qubes.tests.ext', @@ -1240,11 +1362,13 @@ def load_tests(loader, tests, pattern): # pylint: disable=unused-argument 'qubes.tests.integ.basic', 'qubes.tests.integ.storage', 'qubes.tests.integ.pvgrub', + 'qubes.tests.integ.devices_block', 'qubes.tests.integ.devices_pci', 'qubes.tests.integ.dom0_update', 'qubes.tests.integ.network', 'qubes.tests.integ.dispvm', 'qubes.tests.integ.vm_qrexec_gui', + 'qubes.tests.integ.mime', 'qubes.tests.integ.salt', 'qubes.tests.integ.backup', 'qubes.tests.integ.backupcompatibility', diff --git a/qubes/tests/api_admin.py b/qubes/tests/api_admin.py index dc71991d..be19fa81 100644 --- a/qubes/tests/api_admin.py +++ b/qubes/tests/api_admin.py @@ -39,7 +39,7 @@ import qubes.storage # properties defined in API volume_properties = [ - 'pool', 'vid', 'size', 'usage', 'rw', 'source', + 'pool', 'vid', 'size', 'usage', 'rw', 'source', 'path', 'save_on_stop', 'snap_on_start', 'revisions_to_keep'] @@ -60,6 +60,7 @@ class AdminAPITestCase(qubes.tests.QubesTestCase): app = qubes.Qubes('/tmp/qubes-test.xml', load=False) app.vmm = unittest.mock.Mock(spec=qubes.app.VMMConnection) app.load_initial_values() + app.setup_pools() app.default_kernel = '1.0' app.default_netvm = None self.template = app.add_new_vm('TemplateVM', label='black', diff --git a/qubes/tests/api_misc.py b/qubes/tests/api_misc.py index 8d6efb35..d69c7b97 100644 --- a/qubes/tests/api_misc.py +++ b/qubes/tests/api_misc.py @@ -131,11 +131,14 @@ class TC_00_API_Misc(qubes.tests.QubesTestCase): self.assertEqual(self.src.mock_calls, [ mock.call.untrusted_qdb.read('/qubes-tools/qrexec'), mock.call.untrusted_qdb.read('/qubes-tools/gui'), + mock.call.untrusted_qdb.read('/qubes-tools/gui-emulated'), mock.call.untrusted_qdb.read('/qubes-tools/default-user'), + mock.call.untrusted_qdb.read('/qubes-tools/os'), mock.call.fire_event_async('features-request', untrusted_features={ 'gui': '1', 'default-user': 'user', - 'qrexec': '1'}), + 'qrexec': '1', + 'os': 'Linux'}), ('fire_event_async().__iter__', (), {}), ]) self.assertEqual(self.app.mock_calls, [mock.call.save()]) @@ -153,11 +156,14 @@ class TC_00_API_Misc(qubes.tests.QubesTestCase): self.assertEqual(self.src.mock_calls, [ mock.call.untrusted_qdb.read('/qubes-tools/qrexec'), mock.call.untrusted_qdb.read('/qubes-tools/gui'), + mock.call.untrusted_qdb.read('/qubes-tools/gui-emulated'), mock.call.untrusted_qdb.read('/qubes-tools/default-user'), + mock.call.untrusted_qdb.read('/qubes-tools/os'), mock.call.fire_event_async('features-request', untrusted_features={ 'gui': '1', 'default-user': 'user', - 'qrexec': '1'}), + 'qrexec': '1', + 'os': 'Linux'}), ('fire_event_async().__iter__', (), {}), ]) self.assertEqual(self.app.mock_calls, [mock.call.save()]) diff --git a/qubes/tests/app.py b/qubes/tests/app.py index 04ad2171..63b630c9 100644 --- a/qubes/tests/app.py +++ b/qubes/tests/app.py @@ -30,6 +30,7 @@ import qubes.events import qubes.tests import qubes.tests.init +import qubes.tests.storage_reflink class TestApp(qubes.tests.TestEmitter): pass @@ -264,6 +265,44 @@ class TC_30_VMCollection(qubes.tests.QubesTestCase): # pass +class TC_80_QubesInitialPools(qubes.tests.QubesTestCase): + def setUp(self): + super().setUp() + self.app = qubes.Qubes('/tmp/qubestest.xml', load=False, + offline_mode=True) + self.test_dir = '/var/tmp/test-varlibqubes' + self.test_patch = mock.patch.dict( + qubes.config.defaults['pool_configs']['varlibqubes'], + {'dir_path': self.test_dir}) + self.test_patch.start() + + def tearDown(self): + self.test_patch.stop() + self.app.close() + del self.app + + def get_driver(self, fs_type, accessible): + qubes.tests.storage_reflink.mkdir_fs(self.test_dir, fs_type, + accessible=accessible, cleanup_via=self.addCleanup) + self.app.load_initial_values() + + varlibqubes = self.app.pools['varlibqubes'] + self.assertEqual(varlibqubes.dir_path, self.test_dir) + return varlibqubes.driver + + def test_100_varlibqubes_btrfs_accessible(self): + self.assertEqual(self.get_driver('btrfs', True), 'file-reflink') + + def test_101_varlibqubes_btrfs_inaccessible(self): + self.assertEqual(self.get_driver('btrfs', False), 'file') + + def test_102_varlibqubes_ext4_accessible(self): + self.assertEqual(self.get_driver('ext4', True), 'file') + + def test_103_varlibqubes_ext4_inaccessible(self): + self.assertEqual(self.get_driver('ext4', False), 'file') + + class TC_89_QubesEmpty(qubes.tests.QubesTestCase): def tearDown(self): try: diff --git a/qubes/tests/ext.py b/qubes/tests/ext.py index 1197875c..6f59132f 100644 --- a/qubes/tests/ext.py +++ b/qubes/tests/ext.py @@ -21,6 +21,8 @@ from unittest import mock import qubes.ext.core_features +import qubes.ext.services +import qubes.ext.windows import qubes.tests @@ -163,3 +165,143 @@ class TC_00_CoreFeatures(qubes.tests.QubesTestCase): ('features.__contains__', ('qrexec',), {}), ('features.__contains__', ('gui',), {}), ]) + +class TC_10_WindowsFeatures(qubes.tests.QubesTestCase): + def setUp(self): + super().setUp() + self.ext = qubes.ext.windows.WindowsFeatures() + self.vm = mock.MagicMock() + self.features = {} + self.vm.configure_mock(**{ + 'features.get.side_effect': self.features.get, + 'features.__contains__.side_effect': self.features.__contains__, + 'features.__setitem__.side_effect': self.features.__setitem__, + }) + + def test_000_notify_tools_full(self): + del self.vm.template + self.ext.qubes_features_request(self.vm, 'features-request', + untrusted_features={ + 'gui': '1', + 'version': '1', + 'default-user': 'user', + 'qrexec': '1', + 'os': 'Windows'}) + self.assertEqual(self.vm.mock_calls, [ + ('features.__setitem__', ('os', 'Windows'), {}), + ('features.__setitem__', ('rpc-clipboard', True), {}), + ]) + + def test_001_notify_tools_no_qrexec(self): + del self.vm.template + self.ext.qubes_features_request(self.vm, 'features-request', + untrusted_features={ + 'gui': '1', + 'version': '1', + 'default-user': 'user', + 'qrexec': '0', + 'os': 'Windows'}) + self.assertEqual(self.vm.mock_calls, [ + ('features.__setitem__', ('os', 'Windows'), {}), + ]) + + def test_002_notify_tools_other_os(self): + del self.vm.template + self.ext.qubes_features_request(self.vm, 'features-request', + untrusted_features={ + 'gui': '1', + 'version': '1', + 'default-user': 'user', + 'qrexec': '1', + 'os': 'other'}) + self.assertEqual(self.vm.mock_calls, []) + +class TC_20_Services(qubes.tests.QubesTestCase): + def setUp(self): + super().setUp() + self.ext = qubes.ext.services.ServicesExtension() + self.vm = mock.MagicMock() + self.features = {} + self.vm.configure_mock(**{ + 'template': None, + 'is_running.return_value': True, + 'features.get.side_effect': self.features.get, + 'features.items.side_effect': self.features.items, + 'features.__iter__.side_effect': self.features.__iter__, + 'features.__contains__.side_effect': self.features.__contains__, + 'features.__setitem__.side_effect': self.features.__setitem__, + 'features.__delitem__.side_effect': self.features.__delitem__, + }) + + def test_000_write_to_qdb(self): + self.features['service.test1'] = '1' + self.features['service.test2'] = '' + + self.ext.on_domain_qdb_create(self.vm, 'domain-qdb-create') + self.assertEqual(sorted(self.vm.untrusted_qdb.mock_calls), [ + ('write', ('/qubes-service/test1', '1'), {}), + ('write', ('/qubes-service/test2', '0'), {}), + ]) + + def test_001_feature_set(self): + self.ext.on_domain_feature_set(self.vm, + 'feature-set:service.test_no_oldvalue', + 'service.test_no_oldvalue', '1') + self.ext.on_domain_feature_set(self.vm, + 'feature-set:service.test_oldvalue', + 'service.test_oldvalue', '1', '') + self.ext.on_domain_feature_set(self.vm, + 'feature-set:service.test_disable', + 'service.test_disable', '', '1') + self.ext.on_domain_feature_set(self.vm, + 'feature-set:service.test_disable_no_oldvalue', + 'service.test_disable_no_oldvalue', '') + + self.assertEqual(sorted(self.vm.untrusted_qdb.mock_calls), sorted([ + ('write', ('/qubes-service/test_no_oldvalue', '1'), {}), + ('write', ('/qubes-service/test_oldvalue', '1'), {}), + ('write', ('/qubes-service/test_disable', '0'), {}), + ('write', ('/qubes-service/test_disable_no_oldvalue', '0'), {}), + ])) + + def test_002_feature_delete(self): + self.ext.on_domain_feature_delete(self.vm, + 'feature-delete:service.test3', 'service.test3') + self.assertEqual(sorted(self.vm.untrusted_qdb.mock_calls), [ + ('rm', ('/qubes-service/test3',), {}), + ]) + + def test_010_supported_services(self): + self.ext.supported_services(self.vm, 'features-request', + untrusted_features={ + 'supported-service.test1': '1', # ok + 'supported-service.test2': '0', # ignored + 'supported-service.test3': 'some text', # ignored + 'no-service': '1', # ignored + }) + self.assertEqual(self.features, { + 'supported-service.test1': True, + }) + + def test_011_supported_services_add(self): + self.features['supported-service.test1'] = '1' + self.ext.supported_services(self.vm, 'features-request', + untrusted_features={ + 'supported-service.test1': '1', # ok + 'supported-service.test2': '1', # ok + }) + # also check if existing one is untouched + self.assertEqual(self.features, { + 'supported-service.test1': '1', + 'supported-service.test2': True, + }) + + def test_012_supported_services_remove(self): + self.features['supported-service.test1'] = '1' + self.ext.supported_services(self.vm, 'features-request', + untrusted_features={ + 'supported-service.test2': '1', # ok + }) + self.assertEqual(self.features, { + 'supported-service.test2': True, + }) diff --git a/qubes/tests/extra.py b/qubes/tests/extra.py index 4ee6534e..d4ee2f34 100644 --- a/qubes/tests/extra.py +++ b/qubes/tests/extra.py @@ -66,8 +66,9 @@ class VMWrapper(object): def __hash__(self): return hash(self._vm) - def start(self): - return self._loop.run_until_complete(self._vm.start()) + def start(self, start_guid=True): + return self._loop.run_until_complete( + self._vm.start(start_guid=start_guid)) def shutdown(self): return self._loop.run_until_complete(self._vm.shutdown()) @@ -194,7 +195,8 @@ def load_tests(loader, tests, pattern): for entry in pkg_resources.iter_entry_points('qubes.tests.extra'): try: for test_case in entry.load()(): - tests.addTests(loader.loadTestsFromTestCase(test_case)) + tests.addTests(loader.loadTestsFromNames([ + '{}.{}'.format(test_case.__module__, test_case.__name__)])) except Exception as err: # pylint: disable=broad-except def runTest(self): raise err @@ -207,10 +209,10 @@ def load_tests(loader, tests, pattern): 'qubes.tests.extra.for_template'): try: for test_case in entry.load()(): - test.addTests(loader.loadTestsFromNames( + tests.addTests(loader.loadTestsFromNames( qubes.tests.create_testcases_for_templates( test_case.__name__, test_case, - globals=sys.modules[test_case.__module__].__dict__))) + module=sys.modules[test_case.__module__]))) except Exception as err: # pylint: disable=broad-except def runTest(self): raise err diff --git a/qubes/tests/integ/backup.py b/qubes/tests/integ/backup.py index 2954e945..73ad8db7 100644 --- a/qubes/tests/integ/backup.py +++ b/qubes/tests/integ/backup.py @@ -480,16 +480,7 @@ class TC_00_Backup(BackupTestsMixin, qubes.tests.SystemTestCase): os.mkdir(test_dir) with open(os.path.join(test_dir, 'some-file.txt'), 'w') as f: f.write('test file\n') - self.restore_backup(expect_errors=[ - 'Error restoring VM test-inst-test-net, skipping: Got empty ' - 'response from qubesd. See journalctl in dom0 for details.', - 'Error setting test-inst-test1.netvm to test-inst-test-net: ' - '\'"No such domain: \\\'test-inst-test-net\\\'"\'', - ]) - del vms_info['test-inst-test-net'] - vms_info['test-inst-test1']['properties']['netvm'] = \ - str(self.app.default_netvm) - vms_info['test-inst-test1']['default']['netvm'] = True + self.restore_backup() self.assertCorrectlyRestored(vms_info, orig_hashes) finally: del vms @@ -650,9 +641,14 @@ class TC_10_BackupVMMixin(BackupTestsMixin): del vms +def create_testcases_for_templates(): + return qubes.tests.create_testcases_for_templates('TC_10_BackupVM', + TC_10_BackupVMMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('TC_10_BackupVM', - TC_10_BackupVMMixin, qubes.tests.SystemTestCase, - globals=globals()))) + create_testcases_for_templates())) return tests + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) diff --git a/qubes/tests/integ/backupcompatibility.py b/qubes/tests/integ/backupcompatibility.py index c52e0e62..88c3708a 100644 --- a/qubes/tests/integ/backupcompatibility.py +++ b/qubes/tests/integ/backupcompatibility.py @@ -19,6 +19,7 @@ from multiprocessing import Queue +import asyncio import os import shutil import subprocess @@ -122,7 +123,10 @@ class TC_00_BackupCompatibility( qubes.tests.integ.backup.BackupTestsMixin, qubes.tests.SystemTestCase): def tearDown(self): - self.remove_test_vms(prefix="test-") + prefixes = ["test-", "disp-test-"] + if 'disp-no-netvm' not in self.host_app.domains: + prefixes.append('disp-no-netvm') + self.remove_test_vms(prefix=prefixes) super(TC_00_BackupCompatibility, self).tearDown() def create_whitelisted_appmenus(self, filename): @@ -233,7 +237,7 @@ class TC_00_BackupCompatibility( self.create_sparse(self.fullpath( "vm-templates/test-template-clone/root.img"), 10*2**30) self.fill_image(self.fullpath( - "vm-templates/test-template-clone/root.img"), 1*2**30, True) + "vm-templates/test-template-clone/root.img"), 100*2**20, True) self.create_volatile_img(self.fullpath( "vm-templates/test-template-clone/volatile.img")) subprocess.check_call([ @@ -382,7 +386,7 @@ class TC_00_BackupCompatibility( def assertRestored(self, name, **kwargs): with self.assertNotRaises((KeyError, qubes.exc.QubesException)): vm = self.app.domains[name] - vm.storage.verify() + asyncio.get_event_loop().run_until_complete(vm.storage.verify()) for prop, value in kwargs.items(): if prop == 'klass': self.assertIsInstance(vm, value) diff --git a/qubes/tests/integ/basic.py b/qubes/tests/integ/basic.py index 5b0eb089..e11c63ee 100644 --- a/qubes/tests/integ/basic.py +++ b/qubes/tests/integ/basic.py @@ -35,6 +35,8 @@ import collections import pkg_resources import shutil +import sys + import qubes import qubes.firewall import qubes.tests @@ -106,10 +108,33 @@ class TC_00_Basic(qubes.tests.SystemTestCase): self.assertTrue(self.vm.is_running()) # Type 'poweroff' subprocess.check_call(['xdotool', 'search', '--name', self.vm.name, - 'type', 'poweroff\r']) - self.loop.run_until_complete(asyncio.sleep(1)) + 'type', '--window', '%1', 'poweroff\r']) + for _ in range(10): + if not self.vm.is_running(): + break + self.loop.run_until_complete(asyncio.sleep(1)) self.assertFalse(self.vm.is_running()) + def test_130_autostart_disable_on_remove(self): + vm = self.app.add_new_vm(qubes.vm.appvm.AppVM, + name=self.make_vm_name('vm'), + template=self.app.default_template, + label='red') + + self.assertIsNotNone(vm) + self.loop.run_until_complete(vm.create_on_disk()) + vm.autostart = True + self.assertTrue(os.path.exists( + '/etc/systemd/system/multi-user.target.wants/' + 'qubes-vm@{}.service'.format(vm.name)), + "systemd service not enabled by autostart=True") + del self.app.domains[vm] + self.loop.run_until_complete(vm.remove_from_disk()) + self.assertFalse(os.path.exists( + '/etc/systemd/system/multi-user.target.wants/' + 'qubes-vm@{}.service'.format(vm.name)), + "systemd service not disabled on domain remove") + def _test_200_on_domain_start(self, vm, event, **_kwargs): '''Simulate domain crash just after startup''' vm.libvirt_domain.destroy() @@ -203,8 +228,10 @@ class TC_00_Basic(qubes.tests.SystemTestCase): if self.test_failure_reason: self.fail(self.test_failure_reason) + while self.vm.get_power_state() != 'Halted': + self.loop.run_until_complete(asyncio.sleep(1)) # and give a chance for both domain-shutdown handlers to execute - self.loop.run_until_complete(asyncio.sleep(1)) + self.loop.run_until_complete(asyncio.sleep(3)) if self.test_failure_reason: self.fail(self.test_failure_reason) @@ -227,6 +254,7 @@ class TC_00_Basic(qubes.tests.SystemTestCase): try: # first boot, mkfs private volume self.loop.run_until_complete(vm.start()) + self.loop.run_until_complete(self.wait_for_session(vm)) # get private volume UUID private_uuid, _ = self.loop.run_until_complete( vm.run_for_stdio('blkid -o value /dev/xvdb', user='root')) @@ -422,127 +450,6 @@ class TC_01_Properties(qubes.tests.SystemTestCase): self.loop.run_until_complete(self.vm2.create_on_disk()) -class TC_02_QvmPrefs(qubes.tests.SystemTestCase): - # pylint: disable=attribute-defined-outside-init - - def setUp(self): - super(TC_02_QvmPrefs, self).setUp() - self.init_default_template() - self.sharedopts = ['--qubesxml', qubes.tests.XMLPATH] - - def setup_appvm(self): - self.testvm = self.app.add_new_vm( - qubes.vm.appvm.AppVM, - name=self.make_vm_name("vm"), - label='red') - self.loop.run_until_complete(self.testvm.create_on_disk()) - self.app.save() - - def setup_hvm(self): - self.testvm = self.app.add_new_vm( - qubes.vm.appvm.AppVM, - name=self.make_vm_name("hvm"), - label='red') - self.testvm.virt_mode = 'hvm' - self.loop.run_until_complete(self.testvm.create_on_disk()) - self.app.save() - - def pref_set(self, name, value, valid=True): - self.loop.run_until_complete(self._pref_set(name, value, valid)) - - @asyncio.coroutine - def _pref_set(self, name, value, valid=True): - cmd = ['qvm-prefs'] - if value != '-D': - cmd.append('--') - cmd.extend((self.testvm.name, name, value)) - p = yield from asyncio.create_subprocess_exec(*cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - (stdout, stderr) = yield from p.communicate() - if valid: - self.assertEqual(p.returncode, 0, - "qvm-prefs .. '{}' '{}' failed: {}{}".format( - name, value, stdout, stderr - )) - else: - self.assertNotEquals(p.returncode, 0, - "qvm-prefs should reject value '{}' for " - "property '{}'".format(value, name)) - - def pref_get(self, name): - self.loop.run_until_complete(self._pref_get(name)) - - @asyncio.coroutine - def _pref_get(self, name): - p = yield from asyncio.create_subprocess_exec( - 'qvm-prefs', *self.sharedopts, '--', self.testvm.name, name, - stdout=subprocess.PIPE) - (stdout, _) = yield from p.communicate() - self.assertEqual(p.returncode, 0) - return stdout.strip() - - bool_test_values = [ - ('true', 'True', True), - ('False', 'False', True), - ('0', 'False', True), - ('1', 'True', True), - ('invalid', '', False) - ] - - def execute_tests(self, name, values): - """ - Helper function, which executes tests for given property. - :param values: list of tuples (value, expected, valid), - where 'value' is what should be set and 'expected' is what should - qvm-prefs returns as a property value and 'valid' marks valid and - invalid values - if it's False, qvm-prefs should reject the value - :return: None - """ - for (value, expected, valid) in values: - self.pref_set(name, value, valid) - if valid: - self.assertEqual(self.pref_get(name), expected) - - @unittest.skip('test not converted to core3 API') - def test_006_template(self): - templates = [tpl for tpl in self.app.domains.values() if - isinstance(tpl, qubes.vm.templatevm.TemplateVM)] - if not templates: - self.skipTest("No templates installed") - some_template = templates[0].name - self.setup_appvm() - self.execute_tests('template', [ - (some_template, some_template, True), - ('invalid', '', False), - ]) - - @unittest.skip('test not converted to core3 API') - def test_014_pcidevs(self): - self.setup_appvm() - self.execute_tests('pcidevs', [ - ('[]', '[]', True), - ('[ "00:00.0" ]', "['00:00.0']", True), - ('invalid', '', False), - ('[invalid]', '', False), - # TODO: - # ('["12:12.0"]', '', False) - ]) - - @unittest.skip('test not converted to core3 API') - def test_024_pv_reject_hvm_props(self): - self.setup_appvm() - self.execute_tests('guiagent_installed', [('False', '', False)]) - self.execute_tests('qrexec_installed', [('False', '', False)]) - self.execute_tests('drive', [('/tmp/drive.img', '', False)]) - self.execute_tests('timezone', [('localtime', '', False)]) - - @unittest.skip('test not converted to core3 API') - def test_025_hvm_reject_pv_props(self): - self.setup_hvm() - self.execute_tests('kernel', [('default', '', False)]) - self.execute_tests('kernelopts', [('default', '', False)]) - class TC_03_QvmRevertTemplateChanges(qubes.tests.SystemTestCase): # pylint: disable=attribute-defined-outside-init @@ -570,11 +477,13 @@ class TC_03_QvmRevertTemplateChanges(qubes.tests.SystemTestCase): def get_rootimg_checksum(self): return subprocess.check_output( - ['sha1sum', self.test_template.volumes['root'].path]) + ['sha1sum', self.test_template.volumes['root'].export()]).\ + decode().split(' ')[0] def _do_test(self): checksum_before = self.get_rootimg_checksum() self.loop.run_until_complete(self.test_template.start()) + self.loop.run_until_complete(self.wait_for_session(self.test_template)) self.shutdown_and_wait(self.test_template) checksum_changed = self.get_rootimg_checksum() if checksum_before == checksum_changed: @@ -787,21 +696,27 @@ class TC_06_AppVMMixin(object): self.assertTrue(self.vm.is_running()) # Type 'poweroff' subprocess.check_call(['xdotool', 'search', '--name', self.vm.name, - 'type', 'poweroff\r']) - self.loop.run_until_complete(asyncio.sleep(1)) + 'type', '--window', '%1', 'poweroff\r']) + for _ in range(10): + if not self.vm.is_running(): + break + self.loop.run_until_complete(asyncio.sleep(1)) self.assertFalse(self.vm.is_running()) +def create_testcases_for_templates(): + yield from qubes.tests.create_testcases_for_templates('TC_05_StandaloneVM', + TC_05_StandaloneVMMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + yield from qubes.tests.create_testcases_for_templates('TC_06_AppVM', + TC_06_AppVMMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('TC_05_StandaloneVM', - TC_05_StandaloneVMMixin, qubes.tests.SystemTestCase, - globals=globals()))) - tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('TC_06_AppVM', - TC_06_AppVMMixin, qubes.tests.SystemTestCase, - globals=globals()))) + create_testcases_for_templates())) return tests +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) + # vim: ts=4 sw=4 et diff --git a/tests/block.py b/qubes/tests/integ/devices_block.py similarity index 63% rename from tests/block.py rename to qubes/tests/integ/devices_block.py index 6220a977..1ad55e42 100644 --- a/tests/block.py +++ b/qubes/tests/integ/devices_block.py @@ -2,7 +2,7 @@ # # The Qubes OS Project, https://www.qubes-os.org/ # -# Copyright (C) 2016 +# Copyright (C) 2018 # Marek Marczykowski-Górecki # # This library is free software; you can redistribute it and/or @@ -20,42 +20,44 @@ # import os +import sys + +import qubes import qubes.tests -import qubes.qubesutils import subprocess # the same class for both dom0 and VMs -class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): +class TC_00_List(qubes.tests.SystemTestCase): template = None def setUp(self): - super(TC_00_List, self).setUp() + super().setUp() self.img_path = '/tmp/test.img' self.mount_point = '/tmp/test-dir' if self.template is not None: - self.vm = self.qc.add_new_vm( - "QubesAppVm", - name=self.make_vm_name("vm"), - template=self.qc.get_vm_by_name(self.template)) - self.vm.create_on_disk(verbose=False) + self.vm = self.app.add_new_vm( + "AppVM", + label='red', + name=self.make_vm_name("vm")) + self.loop.run_until_complete( + self.vm.create_on_disk()) self.app.save() - self.qc.unlock_db() - self.vm.start() + self.loop.run_until_complete(self.vm.start()) else: - self.qc.unlock_db() - self.vm = self.qc[0] + self.vm = self.app.domains[0] def tearDown(self): - super(TC_00_List, self).tearDown() + super().tearDown() if self.template is None: if os.path.exists(self.mount_point): subprocess.call(['sudo', 'umount', self.mount_point]) subprocess.call(['sudo', 'rmdir', self.mount_point]) - subprocess.call(['sudo', 'dmsetup', 'remove', 'test-dm']) + if os.path.exists('/dev/mapper/test-dm'): + subprocess.call(['sudo', 'dmsetup', 'remove', 'test-dm']) if os.path.exists(self.img_path): loopdev = subprocess.check_output(['losetup', '-j', self.img_path]) - for dev in loopdev.splitlines(): + for dev in loopdev.decode().splitlines(): subprocess.call( ['sudo', 'losetup', '-d', dev.split(':')[0]]) subprocess.call(['sudo', 'rm', '-f', self.img_path]) @@ -67,9 +69,8 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): elif user == "root": subprocess.check_call(['sudo', 'sh', '-c', script]) else: - retcode = self.vm.run(script, user=user, wait=True) - if retcode != 0: - raise subprocess.CalledProcessError + self.loop.run_until_complete( + self.vm.run_for_stdio(script, user=user)) def test_000_list_loop(self): if self.template is None: @@ -80,19 +81,18 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): "losetup -f {path}; " "udevadm settle".format(path=self.img_path), user="root") - dev_list = qubes.qubesutils.block_list_vm(self.vm) + dev_list = list(self.vm.devices['block']) found = False - for dev in dev_list.keys(): - if dev_list[dev]['desc'] == self.img_path: - self.assertTrue(dev.startswith(self.vm.name + ':loop')) - self.assertEquals(dev_list[dev]['mode'], 'w') - self.assertEquals(dev_list[dev]['size'], 1024 * 1024 * 128) - self.assertEquals( - dev_list[dev]['device'], '/dev/' + dev.split(':')[1]) + for dev in dev_list: + if dev.description == self.img_path: + self.assertTrue(dev.ident.startswith('loop')) + self.assertEquals(dev.mode, 'w') + self.assertEquals(dev.size, 1024 * 1024 * 128) found = True if not found: - self.fail("Device {} not found in {!r}".format(self.img_path, dev_list)) + self.fail("Device {} not found in {!r}".format( + self.img_path, dev_list)) def test_001_list_loop_mounted(self): if self.template is None: @@ -108,9 +108,9 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): mntdir=self.mount_point), user="root") - dev_list = qubes.qubesutils.block_list_vm(self.vm) - for dev in dev_list.keys(): - if dev_list[dev]['desc'] == self.img_path: + dev_list = list(self.vm.devices['block']) + for dev in dev_list: + if dev.description == self.img_path: self.fail( 'Device {} ({}) should not be listed because is mounted' .format(dev, self.img_path)) @@ -125,19 +125,17 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): "/sys/block/$(basename $loopdev)/dev) 0\";" "udevadm settle".format(path=self.img_path), user="root") - dev_list = qubes.qubesutils.block_list_vm(self.vm) + dev_list = list(self.vm.devices['block']) found = False - for dev in dev_list.keys(): - if dev.startswith(self.vm.name + ':loop'): - self.assertNotEquals(dev_list[dev]['desc'], self.img_path, + for dev in dev_list: + if dev.ident.startswith('loop'): + self.assertNotEquals(dev.description, self.img_path, "Device {} ({}) should not be listed as it is used in " "device-mapper".format(dev, self.img_path) ) - elif dev_list[dev]['desc'] == 'test-dm': - self.assertEquals(dev_list[dev]['mode'], 'w') - self.assertEquals(dev_list[dev]['size'], 1024 * 1024 * 128) - self.assertEquals( - dev_list[dev]['device'], '/dev/' + dev.split(':')[1]) + elif dev.description == 'test-dm': + self.assertEquals(dev.mode, 'w') + self.assertEquals(dev.size, 1024 * 1024 * 128) found = True if not found: @@ -159,15 +157,15 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): mntdir=self.mount_point), user="root") - dev_list = qubes.qubesutils.block_list_vm(self.vm) - for dev in dev_list.keys(): - if dev.startswith(self.vm.name + ':loop'): - self.assertNotEquals(dev_list[dev]['desc'], self.img_path, + dev_list = list(self.vm.devices['block']) + for dev in dev_list: + if dev.ident.startswith('loop'): + self.assertNotEquals(dev.description, self.img_path, "Device {} ({}) should not be listed as it is used in " "device-mapper".format(dev, self.img_path) ) else: - self.assertNotEquals(dev_list[dev]['desc'], 'test-dm', + self.assertNotEquals(dev.description, 'test-dm', "Device {} ({}) should not be listed as it is " "mounted".format(dev, 'test-dm') ) @@ -183,19 +181,17 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): "/sys/block/$(basename $loopdev)/dev) 0\";" "udevadm settle".format(path=self.img_path), user="root") - dev_list = qubes.qubesutils.block_list_vm(self.vm) + dev_list = list(self.vm.devices['block']) found = False - for dev in dev_list.keys(): - if dev.startswith(self.vm.name + ':loop'): - self.assertNotEquals(dev_list[dev]['desc'], self.img_path, + for dev in dev_list: + if dev.ident.startswith('loop'): + self.assertNotEquals(dev.description, self.img_path, "Device {} ({}) should not be listed as it is used in " "device-mapper".format(dev, self.img_path) ) - elif dev_list[dev]['desc'] == 'test-dm': - self.assertEquals(dev_list[dev]['mode'], 'w') - self.assertEquals(dev_list[dev]['size'], 1024 * 1024 * 128) - self.assertEquals( - dev_list[dev]['device'], '/dev/' + dev.split(':')[1]) + elif dev.description == 'test-dm': + self.assertEquals(dev.mode, 'w') + self.assertEquals(dev.size, 1024 * 1024 * 128) found = True if not found: @@ -216,15 +212,13 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): "dmsetup remove test-dm;" "udevadm settle".format(path=self.img_path), user="root") - dev_list = qubes.qubesutils.block_list_vm(self.vm) + dev_list = list(self.vm.devices['block']) found = False - for dev in dev_list.keys(): - if dev_list[dev]['desc'] == self.img_path: - self.assertTrue(dev.startswith(self.vm.name + ':loop')) - self.assertEquals(dev_list[dev]['mode'], 'w') - self.assertEquals(dev_list[dev]['size'], 1024 * 1024 * 128) - self.assertEquals( - dev_list[dev]['device'], '/dev/' + dev.split(':')[1]) + for dev in dev_list: + if dev.description == self.img_path: + self.assertTrue(dev.ident.startswith('loop')) + self.assertEquals(dev.mode, 'w') + self.assertEquals(dev.size, 1024 * 1024 * 128) found = True if not found: @@ -242,16 +236,14 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): "blockdev --rereadpt $loopdev; " "udevadm settle".format(path=self.img_path), user="root") - dev_list = qubes.qubesutils.block_list_vm(self.vm) + dev_list = list(self.vm.devices['block']) found = False - for dev in dev_list.keys(): - if dev_list[dev]['desc'] == self.img_path: - self.assertTrue(dev.startswith(self.vm.name + ':loop')) - self.assertEquals(dev_list[dev]['mode'], 'w') - self.assertEquals(dev_list[dev]['size'], 1024 * 1024 * 128) - self.assertEquals( - dev_list[dev]['device'], '/dev/' + dev.split(':')[1]) - self.assertIn(dev + 'p1', dev_list) + for dev in dev_list: + if dev.description == self.img_path: + self.assertTrue(dev.ident.startswith('loop')) + self.assertEquals(dev.mode, 'w') + self.assertEquals(dev.size, 1024 * 1024 * 128) + self.assertIn(dev.ident + 'p1', [d.ident for d in dev_list]) found = True if not found: @@ -274,14 +266,14 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): path=self.img_path, mntdir=self.mount_point), user="root") - dev_list = qubes.qubesutils.block_list_vm(self.vm) - for dev in dev_list.keys(): - if dev_list[dev]['desc'] == self.img_path: + dev_list = list(self.vm.devices['block']) + for dev in dev_list: + if dev.description == self.img_path: self.fail( 'Device {} ({}) should not be listed because its ' 'partition is mounted' .format(dev, self.img_path)) - elif dev.startswith(self.vm.name + ':loop') and dev.endswith('p1'): + elif dev.ident.startswith('loop') and dev.ident.endswith('p1'): # FIXME: risky assumption that only tests create partitioned # loop devices self.fail( @@ -289,21 +281,14 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): .format(dev, self.img_path)) -def load_tests(loader, tests, pattern): - try: - qc = qubes.qubes.QubesVmCollection() - qc.lock_db_for_reading() - qc.load() - qc.unlock_db() - templates = [vm.name for vm in qc.values() if - isinstance(vm, qubes.qubes.QubesTemplateVm)] - except OSError: - templates = [] - for template in templates: - tests.addTests(loader.loadTestsFromTestCase( - type( - 'TC_00_List_' + template, - (TC_00_List, qubes.tests.QubesTestCase), - {'template': template}))) +def create_testcases_for_templates(): + return qubes.tests.create_testcases_for_templates('TC_00_List', + TC_00_List, + module=sys.modules[__name__]) +def load_tests(loader, tests, pattern): + tests.addTests(loader.loadTestsFromNames( + create_testcases_for_templates())) return tests + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) diff --git a/qubes/tests/integ/dispvm.py b/qubes/tests/integ/dispvm.py index 7b1abd71..c9e2f697 100644 --- a/qubes/tests/integ/dispvm.py +++ b/qubes/tests/integ/dispvm.py @@ -23,11 +23,14 @@ import subprocess import tempfile import time import unittest +from contextlib import suppress from distutils import spawn import asyncio +import sys + import qubes.tests class TC_04_DispVM(qubes.tests.SystemTestCase): @@ -67,7 +70,7 @@ class TC_04_DispVM(qubes.tests.SystemTestCase): self.assertEqual(lines[0], "test") dispvm_name = lines[1] # wait for actual DispVM destruction - self.loop.run_until_complete(asyncio.sleep(1)) + self.loop.run_until_complete(asyncio.sleep(5)) self.assertNotIn(dispvm_name, self.app.domains) def test_003_cleanup_destroyed(self): @@ -86,7 +89,7 @@ class TC_04_DispVM(qubes.tests.SystemTestCase): p.stdin.write(b"sudo poweroff\n") # do not close p.stdin on purpose - wait to automatic disconnect when # domain is destroyed - timeout = 30 + timeout = 70 lines_task = asyncio.ensure_future(p.stdout.read()) self.loop.run_until_complete(asyncio.wait_for(p.wait(), timeout)) self.loop.run_until_complete(lines_task) @@ -160,8 +163,15 @@ class TC_20_DispVMMixin(object): self.enter_keys_in_window(window_title, ['Return']) # Wait for window to close self.wait_for_window(window_title, show=False) - finally: p.stdin.close() + self.loop.run_until_complete( + asyncio.wait_for(p.wait(), 30)) + except: + with suppress(ProcessLookupError): + p.terminate() + self.loop.run_until_complete(p.wait()) + raise + finally: del p finally: self.loop.run_until_complete(dispvm.cleanup()) @@ -169,7 +179,7 @@ class TC_20_DispVMMixin(object): del dispvm # give it a time for shutdown + cleanup - self.loop.run_until_complete(asyncio.sleep(2)) + self.loop.run_until_complete(asyncio.sleep(5)) self.assertNotIn(dispvm_name, self.app.domains, "DispVM not removed from qubes.xml") @@ -181,7 +191,7 @@ class TC_20_DispVMMixin(object): window_title = window_title.decode().strip().\ replace('(', '\(').replace(')', '\)') time.sleep(1) - if "gedit" in window_title: + if "gedit" in window_title or 'KWrite' in window_title: subprocess.check_call(['xdotool', 'windowactivate', '--sync', winid, 'type', 'Test test 2']) subprocess.check_call(['xdotool', 'key', '--window', winid, @@ -243,37 +253,44 @@ class TC_20_DispVMMixin(object): self.testvm1.run_for_stdio("echo test1 > /home/user/test.txt")) p = self.loop.run_until_complete( - self.testvm1.run("qvm-open-in-dvm /home/user/test.txt")) + self.testvm1.run("qvm-open-in-dvm /home/user/test.txt", + stdout=subprocess.PIPE, stderr=subprocess.STDOUT)) - wait_count = 0 + # if first 5 windows isn't expected editor, there is no hope winid = None - while True: - search = self.loop.run_until_complete( - asyncio.create_subprocess_exec( - 'xdotool', 'search', '--onlyvisible', '--class', 'disp*', - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL)) - stdout, _ = self.loop.run_until_complete(search.communicate()) - if search.returncode == 0: - winid = stdout.strip() - # get window title - (window_title, _) = subprocess.Popen( - ['xdotool', 'getwindowname', winid], stdout=subprocess.PIPE). \ - communicate() - window_title = window_title.decode().strip() - # ignore LibreOffice splash screen and window with no title - # set yet - if window_title and not window_title.startswith("LibreOffice")\ - and not window_title == 'VMapp command': - break - wait_count += 1 - if wait_count > 100: - self.fail("Timeout while waiting for editor window") - self.loop.run_until_complete(asyncio.sleep(0.3)) + for _ in range(5): + try: + winid = self.wait_for_window('disp[0-9]*', search_class=True) + except Exception as e: + try: + self.loop.run_until_complete(asyncio.wait_for(p.wait(), 1)) + except asyncio.TimeoutError: + raise e + else: + stdout = self.loop.run_until_complete(p.stdout.read()) + self.fail( + 'qvm-open-in-dvm exited prematurely with {}: {}'.format( + p.returncode, stdout)) + # get window title + (window_title, _) = subprocess.Popen( + ['xdotool', 'getwindowname', winid], stdout=subprocess.PIPE). \ + communicate() + window_title = window_title.decode().strip() + # ignore LibreOffice splash screen and window with no title + # set yet + if window_title and not window_title.startswith("LibreOffice")\ + and not window_title == 'VMapp command' \ + and 'whonixcheck' not in window_title \ + and not window_title == 'NetworkManager Applet': + break + self.loop.run_until_complete(asyncio.sleep(1)) + winid = None + if winid is None: + self.fail('Timeout waiting for editor window') time.sleep(0.5) self._handle_editor(winid) - self.loop.run_until_complete(p.wait()) + self.loop.run_until_complete(p.communicate()) (test_txt_content, _) = self.loop.run_until_complete( self.testvm1.run_for_stdio("cat /home/user/test.txt")) # Drop BOM if added by editor @@ -281,9 +298,15 @@ class TC_20_DispVMMixin(object): test_txt_content = test_txt_content[3:] self.assertEqual(test_txt_content, b"Test test 2\ntest1\n") + +def create_testcases_for_templates(): + return qubes.tests.create_testcases_for_templates('TC_20_DispVM', + TC_20_DispVMMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('TC_20_DispVM', - TC_20_DispVMMixin, qubes.tests.SystemTestCase, - globals=globals()))) + create_testcases_for_templates())) return tests + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) diff --git a/qubes/tests/integ/dom0_update.py b/qubes/tests/integ/dom0_update.py index a289396f..14ec7f1a 100644 --- a/qubes/tests/integ/dom0_update.py +++ b/qubes/tests/integ/dom0_update.py @@ -78,23 +78,18 @@ Expire-Date: 0 cls.keyid = cls.generate_key(cls.tmpdir) - p = subprocess.Popen(['sudo', 'dd', - 'status=none', 'of=/etc/yum.repos.d/test.repo'], - stdin=subprocess.PIPE) - p.stdin.write(b''' + with open('/etc/yum.repos.d/test.repo', 'w') as repo_file: + repo_file.write(''' [test] name = Test baseurl = http://localhost:8080/ enabled = 1 ''') - p.stdin.close() - p.wait() @classmethod def tearDownClass(cls): - subprocess.check_call(['sudo', 'rm', '-f', - '/etc/yum.repos.d/test.repo']) + os.unlink('/etc/yum.repos.d/test.repo') shutil.rmtree(cls.tmpdir) @@ -113,9 +108,9 @@ enabled = 1 self.loop.run_until_complete(self.updatevm.create_on_disk()) self.app.updatevm = self.updatevm self.app.save() - subprocess.call(['sudo', 'rpm', '-e', self.pkg_name], + subprocess.call(['rpm', '-e', self.pkg_name], stderr=subprocess.DEVNULL) - subprocess.check_call(['sudo', 'rpm', '--import', + subprocess.check_call(['rpm', '--import', os.path.join(self.tmpdir, 'pubkey.asc')]) self.loop.run_until_complete(self.updatevm.start()) self.repo_running = False @@ -126,11 +121,12 @@ enabled = 1 self.repo_proc.terminate() self.loop.run_until_complete(self.repo_proc.wait()) del self.repo_proc + self.app.updatevm = None super(TC_00_Dom0UpgradeMixin, self).tearDown() - subprocess.call(['sudo', 'rpm', '-e', self.pkg_name], + subprocess.call(['rpm', '-e', self.pkg_name], stderr=subprocess.DEVNULL) - subprocess.call(['sudo', 'rpm', '-e', 'gpg-pubkey-{}'.format( + subprocess.call(['rpm', '-e', 'gpg-pubkey-{}'.format( self.keyid)], stderr=subprocess.DEVNULL) for pkg in os.listdir(self.tmpdir): @@ -165,7 +161,7 @@ Test package spec_path]) pkg_path = os.path.join(dir, 'x86_64', '{}-{}-1.x86_64.rpm'.format(name, version)) - subprocess.check_call(['sudo', 'chmod', 'go-rw', '/dev/tty']) + subprocess.check_call(['chmod', 'go-rw', '/dev/tty']) subprocess.check_call( ['rpm', '--quiet', '--define=_gpg_path {}'.format(dir), '--define=_gpg_name {}'.format("Qubes test"), @@ -173,7 +169,7 @@ Test package stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) - subprocess.check_call(['sudo', 'chmod', 'go+rw', '/dev/tty']) + subprocess.check_call(['chmod', 'go+rw', '/dev/tty']) return pkg_path def send_pkg(self, filename): @@ -212,7 +208,7 @@ Test package - "updates pending" flag is cleared """ filename = self.create_pkg(self.tmpdir, self.pkg_name, '1.0') - subprocess.check_call(['sudo', 'rpm', '-i', filename]) + subprocess.check_call(['rpm', '-i', filename]) filename = self.create_pkg(self.tmpdir, self.pkg_name, '2.0') self.send_pkg(filename) open(self.update_flag_path, 'a').close() @@ -331,7 +327,7 @@ Test package self.pkg_name)) def test_020_install_wrong_sign(self): - subprocess.call(['sudo', 'rpm', '-e', 'gpg-pubkey-{}'.format( + subprocess.call(['rpm', '-e', 'gpg-pubkey-{}'.format( self.keyid)]) filename = self.create_pkg(self.tmpdir, self.pkg_name, '1.0') self.send_pkg(filename) @@ -385,9 +381,14 @@ Test package 'UNSIGNED package {}-1.0 installed'.format(self.pkg_name)) +def create_testcases_for_templates(): + return qubes.tests.create_testcases_for_templates('TC_00_Dom0Upgrade', + TC_00_Dom0UpgradeMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('TC_00_Dom0Upgrade', - TC_00_Dom0UpgradeMixin, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) + create_testcases_for_templates())) return tests + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) diff --git a/tests/mime.py b/qubes/tests/integ/mime.py similarity index 54% rename from tests/mime.py rename to qubes/tests/integ/mime.py index 5128585d..9a2105d7 100644 --- a/tests/mime.py +++ b/qubes/tests/integ/mime.py @@ -20,113 +20,93 @@ # License along with this library; if not, see . # # -from distutils import spawn import os +from distutils import spawn import re import subprocess import time import unittest +import itertools + +import asyncio + +import sys + import qubes.tests -import qubes.qubes -from qubes.qubes import QubesVmCollection +import qubes @unittest.skipUnless( spawn.find_executable('xprop') and spawn.find_executable('xdotool') and spawn.find_executable('wmctrl'), "xprop or xdotool or wmctrl not installed") -class TC_50_MimeHandlers(qubes.tests.SystemTestsMixin): - @classmethod - def setUpClass(cls): - if cls.template == 'whonix-gw' or 'minimal' in cls.template: - raise unittest.SkipTest( - 'Template {} not supported by this test'.format(cls.template)) - - if cls.template == 'whonix-ws': - # TODO remove when Whonix-based DispVMs will work (Whonix 13?) - raise unittest.SkipTest( - 'Template {} not supported by this test'.format(cls.template)) - - qc = QubesVmCollection() - - cls._kill_test_vms(qc, prefix=qubes.tests.CLSVMPREFIX) - - qc.lock_db_for_writing() - qc.load() - - cls._remove_test_vms(qc, qubes.qubes.vmm.libvirt_conn, - prefix=qubes.tests.CLSVMPREFIX) - - cls.source_vmname = cls.make_vm_name('source', True) - source_vm = qc.add_new_vm("QubesAppVm", - template=qc.get_vm_by_name(cls.template), - name=cls.source_vmname) - source_vm.create_on_disk(verbose=False) - - cls.target_vmname = cls.make_vm_name('target', True) - target_vm = qc.add_new_vm("QubesAppVm", - template=qc.get_vm_by_name(cls.template), - name=cls.target_vmname) - target_vm.create_on_disk(verbose=False) - - qc.save() - qc.unlock_db() - source_vm.start() - target_vm.start() - - # make sure that DispVMs will be started of the same template - retcode = subprocess.call(['/usr/bin/qvm-create-default-dvm', - cls.template], - stderr=open(os.devnull, 'w')) - assert retcode == 0, "Error preparing DispVM" - +class TC_50_MimeHandlers: def setUp(self): super(TC_50_MimeHandlers, self).setUp() - self.source_vm = self.qc.get_vm_by_name(self.source_vmname) - self.target_vm = self.qc.get_vm_by_name(self.target_vmname) + if self.template.startswith('whonix-gw') or 'minimal' in self.template: + raise unittest.SkipTest( + 'Template {} not supported by this test'.format(self.template)) + + self.source_vmname = self.make_vm_name('source') + self.source_vm = self.app.add_new_vm("AppVM", + template=self.template, + name=self.source_vmname, + label='red') + self.loop.run_until_complete(self.source_vm.create_on_disk()) + + self.target_vmname = self.make_vm_name('target') + self.target_vm = self.app.add_new_vm("AppVM", + template=self.template, + name=self.target_vmname, + label='red') + self.loop.run_until_complete(self.target_vm.create_on_disk()) + + self.target_vm.template_for_dispvms = True + self.source_vm.default_dispvm = self.target_vm + + done, not_done = self.loop.run_until_complete(asyncio.wait([ + self.source_vm.start(), + self.target_vm.start()])) + for result in itertools.chain(done, not_done): + # catch any exceptions + result.result() + def get_window_class(self, winid, dispvm=False): (vm_winid, _) = subprocess.Popen( ['xprop', '-id', winid, '_QUBES_VMWINDOWID'], stdout=subprocess.PIPE ).communicate() - vm_winid = vm_winid.split("#")[1].strip('\n" ') + vm_winid = vm_winid.decode().split("#")[1].strip('\n" ') if dispvm: (vmname, _) = subprocess.Popen( ['xprop', '-id', winid, '_QUBES_VMNAME'], stdout=subprocess.PIPE ).communicate() - vmname = vmname.split("=")[1].strip('\n" ') - window_class = None - while window_class is None: - # XXX to use self.qc.get_vm_by_name would require reloading - # qubes.xml, so use qvm-run instead - xprop = subprocess.Popen( - ['qvm-run', '-p', vmname, 'xprop -id {} WM_CLASS'.format( - vm_winid)], stdout=subprocess.PIPE) - (window_class, _) = xprop.communicate() - if xprop.returncode != 0: - self.skipTest("xprop failed, not installed?") - if 'not found' in window_class: - # WM_CLASS not set yet, wait a little - time.sleep(0.1) - window_class = None + vmname = vmname.decode().split("=")[1].strip('\n" ') + vm = self.app.domains[vmname] else: - window_class = None - while window_class is None: - xprop = self.target_vm.run( - 'xprop -id {} WM_CLASS'.format(vm_winid), - passio_popen=True) - (window_class, _) = xprop.communicate() - if xprop.returncode != 0: - self.skipTest("xprop failed, not installed?") - if 'not found' in window_class: - # WM_CLASS not set yet, wait a little - time.sleep(0.1) - window_class = None + vm = self.target_vm + window_class = None + while window_class is None: + try: + window_class, _ = self.loop.run_until_complete( + vm.run_for_stdio('xprop -id {} WM_CLASS'.format(vm_winid))) + except subprocess.CalledProcessError as e: + if e.returncode == 127: + self.skipTest('xprop not installed') + self.fail( + "xprop -id {} WM_CLASS failed: {}".format( + vm_winid, e.stderr.decode())) + if b'not found' in window_class: + # WM_CLASS not set yet, wait a little + time.sleep(0.1) + window_class = None + # output: WM_CLASS(STRING) = "gnome-terminal-server", "Gnome-terminal" try: + window_class = window_class.decode() window_class = window_class.split("=")[1].split(",")[0].strip('\n" ') except IndexError: raise Exception( @@ -136,44 +116,45 @@ class TC_50_MimeHandlers(qubes.tests.SystemTestsMixin): def open_file_and_check_viewer(self, filename, expected_app_titles, expected_app_classes, dispvm=False): - self.qc.unlock_db() if dispvm: - p = self.source_vm.run("qvm-open-in-dvm {}".format(filename), - passio_popen=True) - vmpattern = "disp*" + p = self.loop.run_until_complete(self.source_vm.run( + "qvm-open-in-dvm {}".format(filename), stdout=subprocess.PIPE)) + vmpattern = "disp[0-9]*" else: - self.qrexec_policy('qubes.OpenInVM', self.source_vm.name, - self.target_vmname) - self.qrexec_policy('qubes.OpenURL', self.source_vm.name, - self.target_vmname) - p = self.source_vm.run("qvm-open-in-vm {} {}".format( - self.target_vmname, filename), passio_popen=True) + p = self.loop.run_until_complete(self.source_vm.run( + "qvm-open-in-vm {} {}".format(self.target_vmname, filename), + stdout=subprocess.PIPE)) vmpattern = self.target_vmname wait_count = 0 winid = None - window_title = None - while True: - search = subprocess.Popen(['xdotool', 'search', - '--onlyvisible', '--class', vmpattern], - stdout=subprocess.PIPE, - stderr=open(os.path.devnull, 'w')) - retcode = search.wait() - if retcode == 0: - winid = search.stdout.read().strip() - # get window title - (window_title, _) = subprocess.Popen( - ['xdotool', 'getwindowname', winid], stdout=subprocess.PIPE). \ - communicate() - window_title = window_title.strip() - # ignore LibreOffice splash screen and window with no title - # set yet - if window_title and not window_title.startswith("LibreOffice")\ - and not window_title == 'VMapp command': - break - wait_count += 1 - if wait_count > 100: - self.fail("Timeout while waiting for editor window") - time.sleep(0.3) + with self.qrexec_policy('qubes.OpenInVM', self.source_vm.name, + self.target_vmname): + with self.qrexec_policy('qubes.OpenURL', self.source_vm.name, + self.target_vmname): + while True: + search = subprocess.Popen(['xdotool', 'search', + '--onlyvisible', '--class', vmpattern], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL) + retcode = search.wait() + if retcode == 0: + winid = search.stdout.read().strip() + # get window title + (window_title, _) = subprocess.Popen( + ['xdotool', 'getwindowname', winid], stdout=subprocess.PIPE). \ + communicate() + window_title = window_title.decode('utf8').strip() + # ignore LibreOffice splash screen and window with no title + # set yet + if window_title and \ + not window_title.startswith("LibreOffice") and\ + not window_title.startswith("NetworkManager") and\ + not window_title == 'VMapp command': + break + wait_count += 1 + if wait_count > 100: + self.fail("Timeout while waiting for editor window") + self.loop.run_until_complete(asyncio.sleep(0.3)) # get window class window_class = self.get_window_class(winid, dispvm) @@ -194,45 +175,66 @@ class TC_50_MimeHandlers(qubes.tests.SystemTestsMixin): expected_app_titles, expected_app_classes)) def prepare_txt(self, filename): - p = self.source_vm.run("cat > {}".format(filename), passio_popen=True) - p.stdin.write("This is test\n") - p.stdin.close() - retcode = p.wait() - assert retcode == 0, "Failed to write {} file".format(filename) + self.loop.run_until_complete( + self.source_vm.run_for_stdio("cat > {}".format(filename), + input=b'This is test\n')) def prepare_pdf(self, filename): self.prepare_txt("/tmp/source.txt") - cmd = "convert /tmp/source.txt {}".format(filename) - retcode = self.source_vm.run(cmd, wait=True) - assert retcode == 0, "Failed to run '{}'".format(cmd) + cmd = "convert text:/tmp/source.txt {}".format(filename) + try: + self.loop.run_until_complete( + self.source_vm.run_for_stdio(cmd)) + except subprocess.CalledProcessError as e: + self.fail('{} failed: {}'.format(cmd, e.stderr.decode())) def prepare_doc(self, filename): self.prepare_txt("/tmp/source.txt") cmd = "unoconv -f doc -o {} /tmp/source.txt".format(filename) - retcode = self.source_vm.run(cmd, wait=True) - if retcode != 0: - self.skipTest("Failed to run '{}', not installed?".format(cmd)) + try: + self.loop.run_until_complete( + self.source_vm.run_for_stdio(cmd)) + except subprocess.CalledProcessError as e: + if e.returncode == 127: + self.skipTest("unoconv not installed".format(cmd)) + self.skipTest("Failed to run '{}': {}".format(cmd, + e.stderr.decode())) def prepare_pptx(self, filename): self.prepare_txt("/tmp/source.txt") cmd = "unoconv -f pptx -o {} /tmp/source.txt".format(filename) - retcode = self.source_vm.run(cmd, wait=True) - if retcode != 0: - self.skipTest("Failed to run '{}', not installed?".format(cmd)) + try: + self.loop.run_until_complete( + self.source_vm.run_for_stdio(cmd)) + except subprocess.CalledProcessError as e: + if e.returncode == 127: + self.skipTest("unoconv not installed".format(cmd)) + self.skipTest("Failed to run '{}': {}".format(cmd, + e.stderr.decode())) def prepare_png(self, filename): self.prepare_txt("/tmp/source.txt") - cmd = "convert /tmp/source.txt {}".format(filename) - retcode = self.source_vm.run(cmd, wait=True) - if retcode != 0: - self.skipTest("Failed to run '{}', not installed?".format(cmd)) + cmd = "convert text:/tmp/source.txt {}".format(filename) + try: + self.loop.run_until_complete( + self.source_vm.run_for_stdio(cmd)) + except subprocess.CalledProcessError as e: + if e.returncode == 127: + self.skipTest("convert not installed".format(cmd)) + self.skipTest("Failed to run '{}': {}".format(cmd, + e.stderr.decode())) def prepare_jpg(self, filename): self.prepare_txt("/tmp/source.txt") - cmd = "convert /tmp/source.txt {}".format(filename) - retcode = self.source_vm.run(cmd, wait=True) - if retcode != 0: - self.skipTest("Failed to run '{}', not installed?".format(cmd)) + cmd = "convert text:/tmp/source.txt {}".format(filename) + try: + self.loop.run_until_complete( + self.source_vm.run_for_stdio(cmd)) + except subprocess.CalledProcessError as e: + if e.returncode == 127: + self.skipTest("convert not installed".format(cmd)) + self.skipTest("Failed to run '{}': {}".format(cmd, + e.stderr.decode())) def test_000_txt(self): filename = "/home/user/test_file.txt" @@ -334,20 +336,14 @@ class TC_50_MimeHandlers(qubes.tests.SystemTestsMixin): ["Firefox", "Iceweasel", "Navigator"], dispvm=True) +def create_testcases_for_templates(): + return qubes.tests.create_testcases_for_templates('TC_50_MimeHandlers', + TC_50_MimeHandlers, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + def load_tests(loader, tests, pattern): - try: - qc = qubes.qubes.QubesVmCollection() - qc.lock_db_for_reading() - qc.load() - qc.unlock_db() - templates = [vm.name for vm in qc.values() if - isinstance(vm, qubes.qubes.QubesTemplateVm)] - except OSError: - templates = [] - for template in templates: - tests.addTests(loader.loadTestsFromTestCase( - type( - 'TC_50_MimeHandlers_' + template, - (TC_50_MimeHandlers, qubes.tests.QubesTestCase), - {'template': template}))) - return tests \ No newline at end of file + tests.addTests(loader.loadTestsFromNames( + create_testcases_for_templates())) + return tests + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) diff --git a/qubes/tests/integ/network.py b/qubes/tests/integ/network.py index 68801820..8869c28b 100644 --- a/qubes/tests/integ/network.py +++ b/qubes/tests/integ/network.py @@ -22,8 +22,6 @@ from distutils import spawn import asyncio -import multiprocessing -import os import subprocess import sys import time @@ -31,13 +29,11 @@ import unittest import qubes.tests import qubes.firewall +import qubes.vm.qubesvm import qubes.vm.appvm -class NcVersion: - Trad = 1 - Nmap = 2 -# noinspection PyAttributeOutsideInit +# noinspection PyAttributeOutsideInit,PyPep8Naming class VmNetworkingMixin(object): test_ip = '192.168.123.45' test_name = 'test.example.com' @@ -50,21 +46,23 @@ class VmNetworkingMixin(object): template = None def run_cmd(self, vm, cmd, user="root"): + '''Run a command *cmd* in a *vm* as *user*. Return its exit code. + :type self: qubes.tests.SystemTestCase | VmNetworkingMixin + :param qubes.vm.qubesvm.QubesVM vm: VM object to run command in + :param str cmd: command to execute + :param std user: user to execute command as + :return int: command exit code + ''' try: self.loop.run_until_complete(vm.run_for_stdio(cmd, user=user)) except subprocess.CalledProcessError as e: return e.returncode return 0 - def check_nc_version(self, vm): - if self.run_cmd(vm, 'nc -h >/dev/null 2>&1') != 0: - self.skipTest('nc not installed') - if self.run_cmd(vm, 'nc -h 2>&1|grep -q nmap.org') == 0: - return NcVersion.Nmap - else: - return NcVersion.Trad - def setUp(self): + ''' + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' super(VmNetworkingMixin, self).setUp() if self.template.startswith('whonix-'): self.skipTest("Test not supported here - Whonix uses its own " @@ -75,6 +73,7 @@ class VmNetworkingMixin(object): label='red') self.loop.run_until_complete(self.testnetvm.create_on_disk()) self.testnetvm.provides_network = True + self.testnetvm.netvm = None self.testvm1 = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=self.make_vm_name('vm1'), label='red') @@ -86,6 +85,9 @@ class VmNetworkingMixin(object): def configure_netvm(self): + ''' + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' def run_netvm_cmd(cmd): if self.run_cmd(self.testnetvm, cmd) != 0: self.fail("Command '%s' failed" % cmd) @@ -102,7 +104,8 @@ class VmNetworkingMixin(object): run_netvm_cmd("ip link add test0 type dummy") run_netvm_cmd("ip link set test0 up") run_netvm_cmd("ip addr add {}/24 dev test0".format(self.test_ip)) - run_netvm_cmd("iptables -I INPUT -d {} -j ACCEPT".format(self.test_ip)) + run_netvm_cmd("iptables -I INPUT -d {} -j ACCEPT --wait".format( + self.test_ip)) # ignore failure self.run_cmd(self.testnetvm, "killall --wait dnsmasq") run_netvm_cmd("dnsmasq -a {ip} -A /{name}/{ip} -i test0 -z".format( @@ -113,12 +116,18 @@ class VmNetworkingMixin(object): def test_000_simple_networking(self): + ''' + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.loop.run_until_complete(self.testvm1.start()) self.assertEqual(self.run_cmd(self.testvm1, self.ping_ip), 0) self.assertEqual(self.run_cmd(self.testvm1, self.ping_name), 0) def test_010_simple_proxyvm(self): + ''' + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=self.make_vm_name('proxy'), label='red') @@ -144,6 +153,9 @@ class VmNetworkingMixin(object): @unittest.skipUnless(spawn.find_executable('xdotool'), "xdotool not installed") def test_020_simple_proxyvm_nm(self): + ''' + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=self.make_vm_name('proxy'), label='red') @@ -189,6 +201,9 @@ class VmNetworkingMixin(object): def test_030_firewallvm_firewall(self): + ''' + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=self.make_vm_name('proxy'), label='red') @@ -198,8 +213,6 @@ class VmNetworkingMixin(object): self.testvm1.netvm = self.proxy self.app.save() - nc_version = self.check_nc_version(self.testnetvm) - # block all for first self.testvm1.firewall.rules = [qubes.firewall.Rule(action='drop')] @@ -207,10 +220,8 @@ class VmNetworkingMixin(object): self.loop.run_until_complete(self.testvm1.start()) self.assertTrue(self.proxy.is_running()) - nc = self.loop.run_until_complete(self.testnetvm.run( - 'nc -l --send-only -e /bin/hostname -k 1234' - if nc_version == NcVersion.Nmap - else 'while nc -l -e /bin/hostname -p 1234; do true; done')) + server = self.loop.run_until_complete(self.testnetvm.run( + 'socat TCP-LISTEN:1234,fork EXEC:/bin/uname')) try: self.assertEqual(self.run_cmd(self.proxy, self.ping_ip), 0, @@ -220,11 +231,8 @@ class VmNetworkingMixin(object): self.assertNotEqual(self.run_cmd(self.testvm1, self.ping_ip), 0, "Ping by IP should be blocked") - if nc_version == NcVersion.Nmap: - nc_cmd = "nc -w 1 --recv-only {} 1234".format(self.test_ip) - else: - nc_cmd = "nc -w 1 {} 1234".format(self.test_ip) - self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0, + client_cmd = "socat TCP:{}:1234 -".format(self.test_ip) + self.assertNotEqual(self.run_cmd(self.testvm1, client_cmd), 0, "TCP connection should be blocked") # block all except ICMP @@ -253,7 +261,7 @@ class VmNetworkingMixin(object): time.sleep(3) self.assertEqual(self.run_cmd(self.testvm1, self.ping_name), 0, "Ping by name failed (should be allowed now)") - self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0, + self.assertNotEqual(self.run_cmd(self.testvm1, client_cmd), 0, "TCP connection should be blocked") # block all except target @@ -267,7 +275,7 @@ class VmNetworkingMixin(object): # Ugly hack b/c there is no feedback when the rules are actually # applied time.sleep(3) - self.assertEqual(self.run_cmd(self.testvm1, nc_cmd), 0, + self.assertEqual(self.run_cmd(self.testvm1, client_cmd), 0, "TCP connection failed (should be allowed now)") # allow all except target @@ -282,14 +290,17 @@ class VmNetworkingMixin(object): # Ugly hack b/c there is no feedback when the rules are actually # applied time.sleep(3) - self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0, + self.assertNotEqual(self.run_cmd(self.testvm1, client_cmd), 0, "TCP connection should be blocked") finally: - nc.terminate() - self.loop.run_until_complete(nc.wait()) + server.terminate() + self.loop.run_until_complete(server.wait()) def test_040_inter_vm(self): + ''' + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=self.make_vm_name('proxy'), label='red') @@ -327,7 +338,10 @@ class VmNetworkingMixin(object): self.ping_cmd.format(target=self.testvm1.ip)), 0) def test_050_spoof_ip(self): - """Test if VM IP spoofing is blocked""" + '''Test if VM IP spoofing is blocked + + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.loop.run_until_complete(self.testvm1.start()) self.assertEqual(self.run_cmd(self.testvm1, self.ping_ip), 0) @@ -353,7 +367,10 @@ class VmNetworkingMixin(object): self.assertEquals(packets, '0', 'Some packet hit the INPUT rule') def test_100_late_xldevd_startup(self): - """Regression test for #1990""" + '''Regression test for #1990 + + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' # Simulater late xl devd startup cmd = "systemctl stop xendriverdomain" if self.run_cmd(self.testnetvm, cmd) != 0: @@ -367,7 +384,10 @@ class VmNetworkingMixin(object): self.assertEqual(self.run_cmd(self.testvm1, self.ping_ip), 0) def test_200_fake_ip_simple(self): - '''Test hiding VM real IP''' + '''Test hiding VM real IP + + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.testvm1.features['net.fake-ip'] = '192.168.1.128' self.testvm1.features['net.fake-gateway'] = '192.168.1.1' self.testvm1.features['net.fake-netmask'] = '255.255.255.0' @@ -398,7 +418,10 @@ class VmNetworkingMixin(object): self.assertNotIn(str(self.testvm1.netvm.ip), output) def test_201_fake_ip_without_gw(self): - '''Test hiding VM real IP''' + '''Test hiding VM real IP + + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.testvm1.features['net.fake-ip'] = '192.168.1.128' self.app.save() self.loop.run_until_complete(self.testvm1.start()) @@ -417,7 +440,10 @@ class VmNetworkingMixin(object): self.assertNotIn(str(self.testvm1.ip), output) def test_202_fake_ip_firewall(self): - '''Test hiding VM real IP, firewall''' + '''Test hiding VM real IP, firewall + + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.testvm1.features['net.fake-ip'] = '192.168.1.128' self.testvm1.features['net.fake-gateway'] = '192.168.1.1' self.testvm1.features['net.fake-netmask'] = '255.255.255.0' @@ -431,8 +457,6 @@ class VmNetworkingMixin(object): self.testvm1.netvm = self.proxy self.app.save() - nc_version = self.check_nc_version(self.testnetvm) - # block all but ICMP and DNS self.testvm1.firewall.rules = [ @@ -443,10 +467,8 @@ class VmNetworkingMixin(object): self.loop.run_until_complete(self.testvm1.start()) self.assertTrue(self.proxy.is_running()) - nc = self.loop.run_until_complete(self.testnetvm.run( - 'nc -l --send-only -e /bin/hostname -k 1234' - if nc_version == NcVersion.Nmap - else 'while nc -l -e /bin/hostname -p 1234; do true; done')) + server = self.loop.run_until_complete(self.testnetvm.run( + 'socat TCP-LISTEN:1234,fork EXEC:/bin/uname')) try: self.assertEqual(self.run_cmd(self.proxy, self.ping_ip), 0, @@ -457,18 +479,18 @@ class VmNetworkingMixin(object): "Ping by IP should be allowed") self.assertEqual(self.run_cmd(self.testvm1, self.ping_name), 0, "Ping by name should be allowed") - if nc_version == NcVersion.Nmap: - nc_cmd = "nc -w 1 --recv-only {} 1234".format(self.test_ip) - else: - nc_cmd = "nc -w 1 {} 1234".format(self.test_ip) - self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0, + client_cmd = "socat TCP:{}:1234 -".format(self.test_ip) + self.assertNotEqual(self.run_cmd(self.testvm1, client_cmd), 0, "TCP connection should be blocked") finally: - nc.terminate() - self.loop.run_until_complete(nc.wait()) + server.terminate() + self.loop.run_until_complete(server.wait()) def test_203_fake_ip_inter_vm_allow(self): - '''Access VM with "fake IP" from other VM (when firewall allows)''' + '''Access VM with "fake IP" from other VM (when firewall allows) + + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=self.make_vm_name('proxy'), label='red') @@ -490,9 +512,9 @@ class VmNetworkingMixin(object): self.loop.run_until_complete(self.testvm1.start()) self.loop.run_until_complete(self.testvm2.start()) + cmd = 'iptables -I FORWARD -s {} -d {} -j ACCEPT'.format( + self.testvm2.ip, self.testvm1.ip) try: - cmd = 'iptables -I FORWARD -s {} -d {} -j ACCEPT'.format( - self.testvm2.ip, self.testvm1.ip) self.loop.run_until_complete(self.proxy.run_for_stdio( cmd, user='root')) except subprocess.CalledProcessError as e: @@ -521,7 +543,10 @@ class VmNetworkingMixin(object): 'Packets didn\'t managed to the VM') def test_204_fake_ip_proxy(self): - '''Test hiding VM real IP''' + '''Test hiding VM real IP + + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=self.make_vm_name('proxy'), label='red') @@ -545,7 +570,7 @@ class VmNetworkingMixin(object): (output, _) = self.loop.run_until_complete( self.proxy.run_for_stdio( 'ip addr show dev eth0', user='root')) - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError: self.fail('ip addr show dev eth0 failed') output = output.decode() self.assertIn('192.168.1.128', output) @@ -555,7 +580,7 @@ class VmNetworkingMixin(object): (output, _) = self.loop.run_until_complete( self.proxy.run_for_stdio( 'ip route show', user='root')) - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError: self.fail('ip route show failed') output = output.decode() self.assertIn('192.168.1.1', output) @@ -565,7 +590,7 @@ class VmNetworkingMixin(object): (output, _) = self.loop.run_until_complete( self.testvm1.run_for_stdio( 'ip addr show dev eth0', user='root')) - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError: self.fail('ip addr show dev eth0 failed') output = output.decode() self.assertNotIn('192.168.1.128', output) @@ -575,14 +600,17 @@ class VmNetworkingMixin(object): (output, _) = self.loop.run_until_complete( self.testvm1.run_for_stdio( 'ip route show', user='root')) - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError: self.fail('ip route show failed') output = output.decode() self.assertIn('192.168.1.128', output) self.assertNotIn(str(self.proxy.ip), output) def test_210_custom_ip_simple(self): - '''Custom AppVM IP''' + '''Custom AppVM IP + + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.testvm1.ip = '192.168.1.1' self.app.save() self.loop.run_until_complete(self.testvm1.start()) @@ -590,7 +618,10 @@ class VmNetworkingMixin(object): self.assertEqual(self.run_cmd(self.testvm1, self.ping_name), 0) def test_211_custom_ip_proxy(self): - '''Custom ProxyVM IP''' + '''Custom ProxyVM IP + + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=self.make_vm_name('proxy'), label='red') @@ -607,7 +638,10 @@ class VmNetworkingMixin(object): self.assertEqual(self.run_cmd(self.testvm1, self.ping_name), 0) def test_212_custom_ip_firewall(self): - '''Custom VM IP and firewall''' + '''Custom VM IP and firewall + + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.testvm1.ip = '192.168.1.1' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, @@ -619,8 +653,6 @@ class VmNetworkingMixin(object): self.testvm1.netvm = self.proxy self.app.save() - nc_version = self.check_nc_version(self.testnetvm) - # block all but ICMP and DNS self.testvm1.firewall.rules = [ @@ -631,10 +663,8 @@ class VmNetworkingMixin(object): self.loop.run_until_complete(self.testvm1.start()) self.assertTrue(self.proxy.is_running()) - nc = self.loop.run_until_complete(self.testnetvm.run( - 'nc -l --send-only -e /bin/hostname -k 1234' - if nc_version == NcVersion.Nmap - else 'while nc -l -e /bin/hostname -p 1234; do true; done')) + server = self.loop.run_until_complete(self.testnetvm.run( + 'socat TCP-LISTEN:1234,fork EXEC:/bin/uname')) try: self.assertEqual(self.run_cmd(self.proxy, self.ping_ip), 0, @@ -645,16 +675,14 @@ class VmNetworkingMixin(object): "Ping by IP should be allowed") self.assertEqual(self.run_cmd(self.testvm1, self.ping_name), 0, "Ping by name should be allowed") - if nc_version == NcVersion.Nmap: - nc_cmd = "nc -w 1 --recv-only {} 1234".format(self.test_ip) - else: - nc_cmd = "nc -w 1 {} 1234".format(self.test_ip) - self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0, + client_cmd = "socat TCP:{}:1234 -".format(self.test_ip) + self.assertNotEqual(self.run_cmd(self.testvm1, client_cmd), 0, "TCP connection should be blocked") finally: - nc.terminate() - self.loop.run_until_complete(nc.wait()) + server.terminate() + self.loop.run_until_complete(server.wait()) +# noinspection PyAttributeOutsideInit,PyPep8Naming class VmIPv6NetworkingMixin(VmNetworkingMixin): test_ip6 = '2000:abcd::1' @@ -666,6 +694,9 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): self.ping6_name = self.ping6_cmd.format(target=self.test_name) def configure_netvm(self): + ''' + :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin + ''' self.testnetvm.features['ipv6'] = True super(VmIPv6NetworkingMixin, self).configure_netvm() @@ -683,12 +714,18 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): format(ip=self.test_ip, ip6=self.test_ip6, name=self.test_name)) def test_500_ipv6_simple_networking(self): + ''' + :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin + ''' self.loop.run_until_complete(self.testvm1.start()) self.assertEqual(self.run_cmd(self.testvm1, self.ping6_ip), 0) self.assertEqual(self.run_cmd(self.testvm1, self.ping6_name), 0) def test_510_ipv6_simple_proxyvm(self): + ''' + :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin + ''' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=self.make_vm_name('proxy'), label='red') @@ -714,6 +751,9 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): @unittest.skipUnless(spawn.find_executable('xdotool'), "xdotool not installed") def test_520_ipv6_simple_proxyvm_nm(self): + ''' + :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin + ''' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=self.make_vm_name('proxy'), label='red') @@ -764,6 +804,9 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): def test_530_ipv6_firewallvm_firewall(self): + ''' + :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin + ''' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=self.make_vm_name('proxy'), label='red') @@ -773,9 +816,6 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): self.testvm1.netvm = self.proxy self.app.save() - if self.run_cmd(self.testnetvm, 'ncat -h') != 0: - self.skipTest('nmap ncat not installed') - # block all for first self.testvm1.firewall.rules = [qubes.firewall.Rule(action='drop')] @@ -783,8 +823,8 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): self.loop.run_until_complete(self.testvm1.start()) self.assertTrue(self.proxy.is_running()) - nc = self.loop.run_until_complete(self.testnetvm.run( - 'ncat -l --send-only -e /bin/hostname -k 1234')) + server = self.loop.run_until_complete(self.testnetvm.run( + 'socat TCP6-LISTEN:1234,fork EXEC:/bin/uname')) try: self.assertEqual(self.run_cmd(self.proxy, self.ping6_ip), 0, @@ -794,8 +834,9 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): self.assertNotEqual(self.run_cmd(self.testvm1, self.ping6_ip), 0, "Ping by IP should be blocked") - nc_cmd = "ncat -w 1 --recv-only {} 1234".format(self.test_ip6) - self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0, + client6_cmd = "socat TCP:[{}]:1234 -".format(self.test_ip6) + client4_cmd = "socat TCP:{}:1234 -".format(self.test_ip) + self.assertNotEqual(self.run_cmd(self.testvm1, client6_cmd), 0, "TCP connection should be blocked") # block all except ICMP @@ -825,13 +866,14 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): time.sleep(3) self.assertEqual(self.run_cmd(self.testvm1, self.ping6_name), 0, "Ping by name failed (should be allowed now)") - self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0, + self.assertNotEqual(self.run_cmd(self.testvm1, client6_cmd), 0, "TCP connection should be blocked") # block all except target self.testvm1.firewall.rules = [ - qubes.firewall.Rule(None, action='accept', dsthost=self.test_ip6, + qubes.firewall.Rule(None, action='accept', + dsthost=self.test_ip6, proto='tcp', dstports=1234), ] self.testvm1.firewall.save() @@ -839,7 +881,7 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): # Ugly hack b/c there is no feedback when the rules are actually # applied time.sleep(3) - self.assertEqual(self.run_cmd(self.testvm1, nc_cmd), 0, + self.assertEqual(self.run_cmd(self.testvm1, client6_cmd), 0, "TCP connection failed (should be allowed now)") # block all except target - by name @@ -854,10 +896,9 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): # Ugly hack b/c there is no feedback when the rules are actually # applied time.sleep(3) - self.assertEqual(self.run_cmd(self.testvm1, nc_cmd), 0, + self.assertEqual(self.run_cmd(self.testvm1, client6_cmd), 0, "TCP (IPv6) connection failed (should be allowed now)") - self.assertEqual(self.run_cmd(self.testvm1, - nc_cmd.replace(self.test_ip6, self.test_ip)), + self.assertEqual(self.run_cmd(self.testvm1, client4_cmd), 0, "TCP (IPv4) connection failed (should be allowed now)") @@ -873,14 +914,17 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): # Ugly hack b/c there is no feedback when the rules are actually # applied time.sleep(3) - self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0, + self.assertNotEqual(self.run_cmd(self.testvm1, client6_cmd), 0, "TCP connection should be blocked") finally: - nc.terminate() - self.loop.run_until_complete(nc.wait()) + server.terminate() + self.loop.run_until_complete(server.wait()) def test_540_ipv6_inter_vm(self): + ''' + :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin + ''' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=self.make_vm_name('proxy'), label='red') @@ -920,7 +964,10 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): def test_550_ipv6_spoof_ip(self): - """Test if VM IP spoofing is blocked""" + '''Test if VM IP spoofing is blocked + + :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin + ''' self.loop.run_until_complete(self.testvm1.start()) self.assertEqual(self.run_cmd(self.testvm1, self.ping6_ip), 0) @@ -949,7 +996,10 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): self.assertEquals(packets, '0', 'Some packet hit the INPUT rule') def test_710_ipv6_custom_ip_simple(self): - '''Custom AppVM IP''' + '''Custom AppVM IP + + :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin + ''' self.testvm1.ip6 = '2000:aaaa:bbbb::1' self.app.save() self.loop.run_until_complete(self.testvm1.start()) @@ -957,7 +1007,10 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): self.assertEqual(self.run_cmd(self.testvm1, self.ping6_name), 0) def test_711_ipv6_custom_ip_proxy(self): - '''Custom ProxyVM IP''' + '''Custom ProxyVM IP + + :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin + ''' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=self.make_vm_name('proxy'), label='red') @@ -974,7 +1027,10 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): self.assertEqual(self.run_cmd(self.testvm1, self.ping6_name), 0) def test_712_ipv6_custom_ip_firewall(self): - '''Custom VM IP and firewall''' + '''Custom VM IP and firewall + + :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin + ''' self.testvm1.ip6 = '2000:aaaa:bbbb::1' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, @@ -986,8 +1042,6 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): self.testvm1.netvm = self.proxy self.app.save() - nc_version = self.check_nc_version(self.testnetvm) - # block all but ICMP and DNS self.testvm1.firewall.rules = [ @@ -998,10 +1052,8 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): self.loop.run_until_complete(self.testvm1.start()) self.assertTrue(self.proxy.is_running()) - nc = self.loop.run_until_complete(self.testnetvm.run( - 'nc -l --send-only -e /bin/hostname -k 1234' - if nc_version == NcVersion.Nmap - else 'while nc -l -e /bin/hostname -p 1234; do true; done')) + server = self.loop.run_until_complete(self.testnetvm.run( + 'socat TCP6-LISTEN:1234,fork EXEC:/bin/uname')) try: self.assertEqual(self.run_cmd(self.proxy, self.ping6_ip), 0, @@ -1012,17 +1064,14 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): "Ping by IP should be allowed") self.assertEqual(self.run_cmd(self.testvm1, self.ping6_name), 0, "Ping by name should be allowed") - if nc_version == NcVersion.Nmap: - nc_cmd = "nc -w 1 --recv-only {} 1234".format(self.test_ip6) - else: - nc_cmd = "nc -w 1 {} 1234".format(self.test_ip6) - self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0, + client_cmd = "socat TCP:[{}]:1234 -".format(self.test_ip6) + self.assertNotEqual(self.run_cmd(self.testvm1, client_cmd), 0, "TCP connection should be blocked") finally: - nc.terminate() - self.loop.run_until_complete(nc.wait()) + server.terminate() + self.loop.run_until_complete(server.wait()) -# noinspection PyAttributeOutsideInit +# noinspection PyAttributeOutsideInit,PyPep8Naming class VmUpdatesMixin(object): """ Tests for VM updates @@ -1099,6 +1148,14 @@ class VmUpdatesMixin(object): ) def run_cmd(self, vm, cmd, user="root"): + '''Run a command *cmd* in a *vm* as *user*. Return its exit code. + + :type self: qubes.tests.SystemTestCase | VmUpdatesMixin + :param qubes.vm.qubesvm.QubesVM vm: VM object to run command in + :param str cmd: command to execute + :param std user: user to execute command as + :return int: command exit code + ''' try: self.loop.run_until_complete(vm.run_for_stdio(cmd)) except subprocess.CalledProcessError as e: @@ -1106,6 +1163,9 @@ class VmUpdatesMixin(object): return 0 def setUp(self): + ''' + :type self: qubes.tests.SystemTestCase | VmUpdatesMixin + ''' if not self.template.count('debian') and \ not self.template.count('fedora'): self.skipTest("Template {} not supported by this test".format( @@ -1142,6 +1202,9 @@ class VmUpdatesMixin(object): self.loop.run_until_complete(self.testvm1.create_on_disk()) def test_000_simple_update(self): + ''' + :type self: qubes.tests.SystemTestCase | VmUpdatesMixin + ''' self.app.save() # reload the VM to have all the properties properly set (especially # default netvm) @@ -1155,6 +1218,9 @@ class VmUpdatesMixin(object): '{}: {}\n{}'.format(self.update_cmd, stdout, stderr)) def create_repo_apt(self): + ''' + :type self: qubes.tests.SystemTestCase | VmUpdatesMixin + ''' pkg_file_name = "test-pkg_1.0-1_amd64.deb" self.loop.run_until_complete(self.netvm_repo.run_for_stdio(''' mkdir /tmp/apt-repo \ @@ -1209,6 +1275,9 @@ SHA256: ''')) def create_repo_yum(self): + ''' + :type self: qubes.tests.SystemTestCase | VmUpdatesMixin + ''' pkg_file_name = "test-pkg-1.0-1.fc21.x86_64.rpm" self.loop.run_until_complete(self.netvm_repo.run_for_stdio(''' mkdir /tmp/yum-repo \ @@ -1221,6 +1290,9 @@ SHA256: 'createrepo /tmp/yum-repo')) def create_repo_and_serve(self): + ''' + :type self: qubes.tests.SystemTestCase | VmUpdatesMixin + ''' if self.template.count("debian") or self.template.count("whonix"): self.create_repo_apt() self.loop.run_until_complete(self.netvm_repo.run( @@ -1242,6 +1314,8 @@ SHA256: The critical part is to use "localhost" - this will work only when accessed through update proxy and this is exactly what we want to test here. + + :type self: qubes.tests.SystemTestCase | VmUpdatesMixin """ if self.template.count("debian") or self.template.count("whonix"): @@ -1266,9 +1340,12 @@ SHA256: self.template)) def test_010_update_via_proxy(self): - """ - Test both whether updates proxy works and whether is actually used by the VM - """ + ''' + Test both whether updates proxy works and whether is actually used + by the VM + + :type self: qubes.tests.SystemTestCase | VmUpdatesMixin + ''' if self.template.count("minimal"): self.skipTest("Template {} not supported by this test".format( self.template)) @@ -1319,17 +1396,20 @@ SHA256: self.assertIn(self.loop.run_until_complete(p.wait()), self.exit_code_ok, '{}: {}\n{}'.format(self.update_cmd, stdout, stderr)) +def create_testcases_for_templates(): + yield from qubes.tests.create_testcases_for_templates('VmNetworking', + VmNetworkingMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + yield from qubes.tests.create_testcases_for_templates('VmIPv6Networking', + VmIPv6NetworkingMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + yield from qubes.tests.create_testcases_for_templates('VmUpdates', + VmUpdatesMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('VmNetworking', - VmNetworkingMixin, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) - tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('VmIPv6Networking', - VmIPv6NetworkingMixin, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) - tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('VmUpdates', - VmUpdates, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) + create_testcases_for_templates())) return tests + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) diff --git a/qubes/tests/integ/pvgrub.py b/qubes/tests/integ/pvgrub.py index fa78f03b..5c1e7cb1 100644 --- a/qubes/tests/integ/pvgrub.py +++ b/qubes/tests/integ/pvgrub.py @@ -70,7 +70,7 @@ class TC_40_PVGrub(object): def get_kernel_version(self, vm): if self.template.startswith('fedora-'): - cmd_get_kernel_version = 'rpm -q kernel-core|sort -n|tail -1|' \ + cmd_get_kernel_version = 'rpm -q kernel-core|sort -V|tail -1|' \ 'cut -d - -f 3-' elif self.template.startswith('debian-'): cmd_get_kernel_version = \ @@ -137,10 +137,14 @@ class TC_40_PVGrub(object): self.test_template.run_for_stdio('uname -r')) self.assertEquals(actual_kver.strip(), kver) +def create_testcases_for_templates(): + return qubes.tests.create_testcases_for_templates('TC_40_PVGrub', + TC_40_PVGrub, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('TC_40_PVGrub', - TC_40_PVGrub, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) + create_testcases_for_templates())) return tests + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) diff --git a/qubes/tests/integ/salt.py b/qubes/tests/integ/salt.py index f598c7e7..bfbbd2b2 100644 --- a/qubes/tests/integ/salt.py +++ b/qubes/tests/integ/salt.py @@ -391,9 +391,15 @@ class SaltVMTestMixin(SaltTestMixin): self.assertEqual(stderr, b'') +def create_testcases_for_templates(): + return qubes.tests.create_testcases_for_templates('TC_10_VMSalt', + SaltVMTestMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + + def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('TC_10_VMSalt', - SaltVMTestMixin, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) + create_testcases_for_templates())) return tests + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) diff --git a/qubes/tests/integ/storage.py b/qubes/tests/integ/storage.py index 4d8f1569..a4766e18 100644 --- a/qubes/tests/integ/storage.py +++ b/qubes/tests/integ/storage.py @@ -26,6 +26,7 @@ import subprocess import qubes.storage.lvm import qubes.tests import qubes.tests.storage_lvm +import qubes.tests.storage_reflink import qubes.vm.appvm @@ -76,6 +77,7 @@ class StorageTestMixin(object): del coro_maybe self.app.save() yield from (self.vm1.start()) + yield from self.wait_for_session(self.vm1) # volatile image not clean yield from (self.vm1.run_for_stdio( @@ -111,6 +113,7 @@ class StorageTestMixin(object): del coro_maybe self.app.save() yield from self.vm1.start() + yield from self.wait_for_session(self.vm1) # non-volatile image not clean yield from self.vm1.run_for_stdio( 'head -c {} /dev/zero 2>&1 | diff -q /dev/xvde - 2>&1'.format(size), @@ -196,6 +199,9 @@ class StorageTestMixin(object): self.app.save() yield from self.vm1.start() yield from self.vm2.start() + yield from asyncio.wait( + [self.wait_for_session(self.vm1), self.wait_for_session(self.vm2)]) + try: yield from self.vm1.run_for_stdio( @@ -284,6 +290,7 @@ class StorageTestMixin(object): del coro_maybe self.app.save() yield from self.vm2.start() + yield from self.wait_for_session(self.vm2) # snapshot image not clean yield from self.vm2.run_for_stdio( @@ -318,6 +325,28 @@ class StorageFile(StorageTestMixin, qubes.tests.SystemTestCase): super(StorageFile, self).tearDown() +class StorageReflinkMixin(StorageTestMixin): + def tearDown(self): + self.app.remove_pool(self.pool.name) + super().tearDown() + + def init_pool(self, fs_type, **kwargs): + name = 'test-reflink-integration-on-' + fs_type + dir_path = os.path.join('/var/tmp', name) + qubes.tests.storage_reflink.mkdir_fs(dir_path, fs_type, + cleanup_via=self.addCleanup) + self.pool = self.app.add_pool(name=name, dir_path=dir_path, + driver='file-reflink', **kwargs) + +class StorageReflinkOnBtrfs(StorageReflinkMixin, qubes.tests.SystemTestCase): + def init_pool(self): + super().init_pool('btrfs') + +class StorageReflinkOnExt4(StorageReflinkMixin, qubes.tests.SystemTestCase): + def init_pool(self): + super().init_pool('ext4', setup_check='no') + + @qubes.tests.storage_lvm.skipUnlessLvmPoolExists class StorageLVM(StorageTestMixin, qubes.tests.SystemTestCase): def init_pool(self): diff --git a/qubes/tests/integ/vm_qrexec_gui.py b/qubes/tests/integ/vm_qrexec_gui.py index 0ed3ee7f..a6e532a1 100644 --- a/qubes/tests/integ/vm_qrexec_gui.py +++ b/qubes/tests/integ/vm_qrexec_gui.py @@ -24,11 +24,15 @@ import multiprocessing import os import subprocess import sys +import tempfile import unittest from distutils import spawn +import grp + import qubes.config +import qubes.devices import qubes.tests import qubes.vm.appvm import qubes.vm.templatevm @@ -60,6 +64,7 @@ class TC_00_AppVMMixin(object): # TODO: wait_for, timeout self.loop.run_until_complete(self.testvm1.start()) self.assertEqual(self.testvm1.get_power_state(), "Running") + self.loop.run_until_complete(self.wait_for_session(self.testvm1)) self.loop.run_until_complete(self.testvm1.shutdown(wait=True)) self.assertEqual(self.testvm1.get_power_state(), "Halted") @@ -72,32 +77,17 @@ class TC_00_AppVMMixin(object): self.loop.run_until_complete(self.wait_for_session(self.testvm1)) p = self.loop.run_until_complete(self.testvm1.run('xterm')) try: - wait_count = 0 title = 'user@{}'.format(self.testvm1.name) if self.template.count("whonix"): title = 'user@host' - while subprocess.call( - ['xdotool', 'search', '--name', title], - stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) > 0: - wait_count += 1 - if wait_count > 100: - self.fail("Timeout while waiting for xterm window") - self.loop.run_until_complete(asyncio.sleep(0.1)) + self.wait_for_window(title) self.loop.run_until_complete(asyncio.sleep(0.5)) subprocess.check_call( ['xdotool', 'search', '--name', title, 'windowactivate', 'type', 'exit\n']) - wait_count = 0 - while subprocess.call(['xdotool', 'search', '--name', title], - stdout=open(os.path.devnull, 'w'), - stderr=subprocess.STDOUT) == 0: - wait_count += 1 - if wait_count > 100: - self.fail("Timeout while waiting for xterm " - "termination") - self.loop.run_until_complete(asyncio.sleep(0.1)) + self.wait_for_window(title, show=False) finally: try: p.terminate() @@ -110,6 +100,8 @@ class TC_00_AppVMMixin(object): def test_011_run_gnome_terminal(self): if "minimal" in self.template: self.skipTest("Minimal template doesn't have 'gnome-terminal'") + if 'whonix' in self.template: + self.skipTest("Whonix template doesn't have 'gnome-terminal'") self.loop.run_until_complete(self.testvm1.start()) self.assertEqual(self.testvm1.get_power_state(), "Running") self.loop.run_until_complete(self.wait_for_session(self.testvm1)) @@ -118,15 +110,7 @@ class TC_00_AppVMMixin(object): title = 'user@{}'.format(self.testvm1.name) if self.template.count("whonix"): title = 'user@host' - wait_count = 0 - while subprocess.call( - ['xdotool', 'search', '--name', title], - stdout=open(os.path.devnull, 'w'), - stderr=subprocess.STDOUT) > 0: - wait_count += 1 - if wait_count > 100: - self.fail("Timeout while waiting for gnome-terminal window") - self.loop.run_until_complete(asyncio.sleep(0.1)) + self.wait_for_window(title) self.loop.run_until_complete(asyncio.sleep(0.5)) subprocess.check_call( @@ -172,30 +156,14 @@ class TC_00_AppVMMixin(object): title = 'user@{}'.format(self.testvm1.name) if self.template.count("whonix"): title = 'user@host' - wait_count = 0 - while subprocess.call( - ['xdotool', 'search', '--name', title], - stdout=open(os.path.devnull, 'w'), - stderr=subprocess.STDOUT) > 0: - wait_count += 1 - if wait_count > 100: - self.fail("Timeout while waiting for xterm window") - self.loop.run_until_complete(asyncio.sleep(0.1)) + self.wait_for_window(title) self.loop.run_until_complete(asyncio.sleep(0.5)) subprocess.check_call( ['xdotool', 'search', '--name', title, 'windowactivate', '--sync', 'type', 'exit\n']) - wait_count = 0 - while subprocess.call(['xdotool', 'search', '--name', title], - stdout=open(os.path.devnull, 'w'), - stderr=subprocess.STDOUT) == 0: - wait_count += 1 - if wait_count > 100: - self.fail("Timeout while waiting for xterm " - "termination") - self.loop.run_until_complete(asyncio.sleep(0.1)) + self.wait_for_window(title, show=False) def test_050_qrexec_simple_eof(self): """Test for data and EOF transmission dom0->VM""" @@ -217,7 +185,6 @@ class TC_00_AppVMMixin(object): self.assertFalse(stderr, 'Some data was printed to stderr') - @unittest.skip('#2851, because there is no GUI in vm') def test_051_qrexec_simple_eof_reverse(self): """Test for EOF transmission VM->dom0""" @@ -235,7 +202,7 @@ class TC_00_AppVMMixin(object): p.stdin.write(TEST_DATA) yield from p.stdin.drain() p.stdin.close() - self.assertEqual(stdout.strip(), 'test', + self.assertEqual(stdout.strip(), b'test', 'Received data differs from what was expected') # this may hang in some buggy cases self.assertFalse((yield from p.stderr.read()), @@ -248,15 +215,18 @@ class TC_00_AppVMMixin(object): "probably EOF wasn't transferred from the VM process") self.loop.run_until_complete(self.testvm1.start()) + self.loop.run_until_complete(self.wait_for_session(self.testvm1)) self.loop.run_until_complete(run(self)) - @unittest.skip('#2851, because there is no GUI in vm') def test_052_qrexec_vm_service_eof(self): """Test for EOF transmission VM(src)->VM(dst)""" self.loop.run_until_complete(asyncio.wait([ self.testvm1.start(), self.testvm2.start()])) + self.loop.run_until_complete(asyncio.wait([ + self.wait_for_session(self.testvm1), + self.wait_for_session(self.testvm2)])) self.loop.run_until_complete(self.testvm2.run_for_stdio( 'cat > /etc/qubes-rpc/test.EOF', user='root', @@ -273,7 +243,7 @@ class TC_00_AppVMMixin(object): except asyncio.TimeoutError: self.fail("Timeout, probably EOF wasn't transferred") - self.assertEqual(stdout, b'test', + self.assertEqual(stdout, b'test\n', 'Received data differs from what was expected') @unittest.expectedFailure @@ -394,23 +364,14 @@ class TC_00_AppVMMixin(object): except asyncio.TimeoutError: self.fail('Timeout, probably deadlock') - @unittest.skip('localcmd= argument went away') def test_071_qrexec_dom0_simultaneous_write(self): """Test for simultaneous write in dom0(src)->VM(dst) connection Similar to test_070_qrexec_vm_simultaneous_write, but with dom0 as a source. """ - def run(self): - result.value = self.testvm2.run_service( - "test.write", localcmd="/bin/sh -c '" - # first write a lot of data to fill all the buffers - "dd if=/dev/zero bs=993 count=10000 iflag=fullblock & " - # then after some time start reading - "sleep 1; " - "dd of=/dev/null bs=993 count=10000 iflag=fullblock; " - "wait" - "'") + + self.loop.run_until_complete(self.testvm2.start()) self.create_remote_file(self.testvm2, '/etc/qubes-rpc/test.write', '''\ # first write a lot of data @@ -418,58 +379,93 @@ class TC_00_AppVMMixin(object): # and only then read something dd of=/dev/null bs=993 count=10000 iflag=fullblock ''') - self.create_local_file('/etc/qubes-rpc/policy/test.write', - '{} {} allow'.format(self.testvm1.name, self.testvm2.name)) - t = multiprocessing.Process(target=run, args=(self,)) - t.start() - t.join(timeout=10) - if t.is_alive(): - t.terminate() + # can't use subprocess.PIPE, because asyncio will claim those FDs + pipe1_r, pipe1_w = os.pipe() + pipe2_r, pipe2_w = os.pipe() + try: + local_proc = self.loop.run_until_complete( + asyncio.create_subprocess_shell( + # first write a lot of data to fill all the buffers + "dd if=/dev/zero bs=993 count=10000 iflag=fullblock & " + # then after some time start reading + "sleep 1; " + "dd of=/dev/null bs=993 count=10000 iflag=fullblock; " + "wait", stdin=pipe1_r, stdout=pipe2_w)) + + service_proc = self.loop.run_until_complete(self.testvm2.run_service( + "test.write", stdin=pipe2_r, stdout=pipe1_w)) + finally: + os.close(pipe1_r) + os.close(pipe1_w) + os.close(pipe2_r) + os.close(pipe2_w) + + try: + self.loop.run_until_complete( + asyncio.wait_for(service_proc.wait(), timeout=10)) + except asyncio.TimeoutError: self.fail("Timeout, probably deadlock") - self.assertEqual(result.value, 0, "Service call failed") + else: + self.assertEqual(service_proc.returncode, 0, + "Service call failed") + finally: + try: + service_proc.terminate() + except ProcessLookupError: + pass - @unittest.skip('localcmd= argument went away') def test_072_qrexec_to_dom0_simultaneous_write(self): """Test for simultaneous write in dom0(src)<-VM(dst) connection Similar to test_071_qrexec_dom0_simultaneous_write, but with dom0 as a "hanging" side. """ - result = multiprocessing.Value('i', -1) - - def run(self): - result.value = self.testvm2.run_service( - "test.write", localcmd="/bin/sh -c '" - # first write a lot of data to fill all the buffers - "dd if=/dev/zero bs=993 count=10000 iflag=fullblock " - # then, only when all written, read something - "dd of=/dev/null bs=993 count=10000 iflag=fullblock; " - "'") self.loop.run_until_complete(self.testvm2.start()) - p = self.testvm2.run("cat > /etc/qubes-rpc/test.write", user="root", - passio_popen=True) - # first write a lot of data - p.stdin.write(b"dd if=/dev/zero bs=993 count=10000 iflag=fullblock &\n") - # and only then read something - p.stdin.write(b"dd of=/dev/null bs=993 count=10000 iflag=fullblock\n") - p.stdin.write(b"sleep 1; \n") - p.stdin.write(b"wait\n") - p.stdin.close() - p.wait() - policy = open("/etc/qubes-rpc/policy/test.write", "w") - policy.write("%s %s allow" % (self.testvm1.name, self.testvm2.name)) - policy.close() - self.addCleanup(os.unlink, "/etc/qubes-rpc/policy/test.write") - t = multiprocessing.Process(target=run, args=(self,)) - t.start() - t.join(timeout=10) - if t.is_alive(): - t.terminate() + self.create_remote_file(self.testvm2, '/etc/qubes-rpc/test.write', '''\ + # first write a lot of data + dd if=/dev/zero bs=993 count=10000 iflag=fullblock & + # and only then read something + dd of=/dev/null bs=993 count=10000 iflag=fullblock + sleep 1; + wait + ''') + + # can't use subprocess.PIPE, because asyncio will claim those FDs + pipe1_r, pipe1_w = os.pipe() + pipe2_r, pipe2_w = os.pipe() + try: + local_proc = self.loop.run_until_complete( + asyncio.create_subprocess_shell( + # first write a lot of data to fill all the buffers + "dd if=/dev/zero bs=993 count=10000 iflag=fullblock & " + # then, only when all written, read something + "dd of=/dev/null bs=993 count=10000 iflag=fullblock; ", + stdin=pipe1_r, stdout=pipe2_w)) + + service_proc = self.loop.run_until_complete(self.testvm2.run_service( + "test.write", stdin=pipe2_r, stdout=pipe1_w)) + finally: + os.close(pipe1_r) + os.close(pipe1_w) + os.close(pipe2_r) + os.close(pipe2_w) + + try: + self.loop.run_until_complete( + asyncio.wait_for(service_proc.wait(), timeout=10)) + except asyncio.TimeoutError: self.fail("Timeout, probably deadlock") - self.assertEqual(result.value, 0, "Service call failed") + else: + self.assertEqual(service_proc.returncode, 0, + "Service call failed") + finally: + try: + service_proc.terminate() + except ProcessLookupError: + pass def test_080_qrexec_service_argument_allow_default(self): """Qrexec service call with argument""" @@ -742,7 +738,8 @@ class TC_00_AppVMMixin(object): if self.template.startswith('whonix-'): self.skipTest('qvm-sync-clock disabled for Whonix VMs') self.loop.run_until_complete(asyncio.wait([ - self.testvm1.start()])) + self.testvm1.start(), + self.testvm2.start(),])) start_time = subprocess.check_output(['date', '-u', '+%s']) try: @@ -752,11 +749,11 @@ class TC_00_AppVMMixin(object): subprocess.check_call(['sudo', 'date', '-s', '2001-01-01T12:34:56'], stdout=subprocess.DEVNULL) self.loop.run_until_complete( - self.testvm1.run_for_stdio('date -s 2001-01-01T12:34:56', + self.testvm2.run_for_stdio('date -s 2001-01-01T12:34:56', user='root')) self.loop.run_until_complete( - self.testvm1.run_for_stdio('qvm-sync-clock', + self.testvm2.run_for_stdio('qvm-sync-clock', user='root')) p = self.loop.run_until_complete( @@ -765,7 +762,7 @@ class TC_00_AppVMMixin(object): self.loop.run_until_complete(p.wait()) self.assertEqual(p.returncode, 0) vm_time, _ = self.loop.run_until_complete( - self.testvm1.run_for_stdio('date -u +%s')) + self.testvm2.run_for_stdio('date -u +%s')) self.assertAlmostEquals(int(vm_time), int(start_time), delta=30) dom0_time = subprocess.check_output(['date', '-u', '+%s']) @@ -779,6 +776,154 @@ class TC_00_AppVMMixin(object): finally: self.app.clockvm = None + @unittest.skipUnless(spawn.find_executable('parecord'), + "pulseaudio-utils not installed in dom0") + def test_220_audio_playback(self): + if 'whonix-gw' in self.template: + self.skipTest('whonix-gw have no audio') + self.loop.run_until_complete(self.testvm1.start()) + try: + self.loop.run_until_complete( + self.testvm1.run_for_stdio('which parecord')) + except subprocess.CalledProcessError: + self.skipTest('pulseaudio-utils not installed in VM') + + self.loop.run_until_complete( + self.wait_for_session(self.testvm1)) + # and some more... + self.loop.run_until_complete(asyncio.sleep(1)) + # generate some "audio" data + audio_in = b'\x20' * 44100 + self.loop.run_until_complete( + self.testvm1.run_for_stdio('cat > audio_in.raw', input=audio_in)) + local_user = grp.getgrnam('qubes').gr_mem[0] + with tempfile.NamedTemporaryFile() as recorded_audio: + os.chmod(recorded_audio.name, 0o666) + # FIXME: -d 0 assumes only one audio device + p = subprocess.Popen(['sudo', '-E', '-u', local_user, + 'parecord', '-d', '0', '--raw', recorded_audio.name], + stdout=subprocess.PIPE) + self.loop.run_until_complete( + self.testvm1.run_for_stdio('paplay --raw audio_in.raw')) + # wait for possible parecord buffering + self.loop.run_until_complete(asyncio.sleep(1)) + p.terminate() + # for some reason sudo do not relay SIGTERM sent above + subprocess.check_call(['pkill', 'parecord']) + p.wait() + # allow few bytes missing, don't use assertIn, to avoid printing + # the whole data in error message + if audio_in[:-8] not in recorded_audio.file.read(): + self.fail('played sound not found in dom0') + + def _configure_audio_recording(self, vm): + '''Connect VM's output-source to sink monitor instead of mic''' + local_user = grp.getgrnam('qubes').gr_mem[0] + sudo = ['sudo', '-E', '-u', local_user] + source_outputs = subprocess.check_output( + sudo + ['pacmd', 'list-source-outputs']).decode() + + last_index = None + found = False + for line in source_outputs.splitlines(): + if line.startswith(' index: '): + last_index = line.split(':')[1].strip() + elif line.startswith('\t\tapplication.name = '): + app_name = line.split('=')[1].strip('" ') + if vm.name == app_name: + found = True + break + if not found: + self.fail('source-output for VM {} not found'.format(vm.name)) + + subprocess.check_call(sudo + + ['pacmd', 'move-source-output', last_index, '0']) + + @unittest.skipUnless(spawn.find_executable('parecord'), + "pulseaudio-utils not installed in dom0") + def test_221_audio_record_muted(self): + if 'whonix-gw' in self.template: + self.skipTest('whonix-gw have no audio') + self.loop.run_until_complete(self.testvm1.start()) + try: + self.loop.run_until_complete( + self.testvm1.run_for_stdio('which parecord')) + except subprocess.CalledProcessError: + self.skipTest('pulseaudio-utils not installed in VM') + + self.loop.run_until_complete( + self.wait_for_session(self.testvm1)) + # and some more... + self.loop.run_until_complete(asyncio.sleep(1)) + # connect VM's recording source output monitor (instead of mic) + self._configure_audio_recording(self.testvm1) + + # generate some "audio" data + audio_in = b'\x20' * 44100 + local_user = grp.getgrnam('qubes').gr_mem[0] + record = self.loop.run_until_complete( + self.testvm1.run('parecord --raw audio_rec.raw')) + # give it time to start recording + self.loop.run_until_complete(asyncio.sleep(0.5)) + p = subprocess.Popen(['sudo', '-E', '-u', local_user, + 'paplay', '--raw'], + stdin=subprocess.PIPE) + p.communicate(audio_in) + # wait for possible parecord buffering + self.loop.run_until_complete(asyncio.sleep(1)) + self.loop.run_until_complete( + self.testvm1.run_for_stdio('pkill parecord')) + record.wait() + recorded_audio, _ = self.loop.run_until_complete( + self.testvm1.run_for_stdio('cat audio_rec.raw')) + # should be empty or silence, so check just a little fragment + if audio_in[:32] in recorded_audio: + self.fail('VM recorded something, even though mic disabled') + + @unittest.skipUnless(spawn.find_executable('parecord'), + "pulseaudio-utils not installed in dom0") + def test_222_audio_record_unmuted(self): + if 'whonix-gw' in self.template: + self.skipTest('whonix-gw have no audio') + self.loop.run_until_complete(self.testvm1.start()) + try: + self.loop.run_until_complete( + self.testvm1.run_for_stdio('which parecord')) + except subprocess.CalledProcessError: + self.skipTest('pulseaudio-utils not installed in VM') + + self.loop.run_until_complete( + self.wait_for_session(self.testvm1)) + # and some more... + self.loop.run_until_complete(asyncio.sleep(1)) + da = qubes.devices.DeviceAssignment(self.app.domains[0], 'mic') + self.loop.run_until_complete( + self.testvm1.devices['mic'].attach(da)) + # connect VM's recording source output monitor (instead of mic) + self._configure_audio_recording(self.testvm1) + + # generate some "audio" data + audio_in = b'\x20' * 44100 + local_user = grp.getgrnam('qubes').gr_mem[0] + record = self.loop.run_until_complete( + self.testvm1.run('parecord --raw audio_rec.raw')) + # give it time to start recording + self.loop.run_until_complete(asyncio.sleep(0.5)) + p = subprocess.Popen(['sudo', '-E', '-u', local_user, + 'paplay', '--raw'], + stdin=subprocess.PIPE) + p.communicate(audio_in) + # wait for possible parecord buffering + self.loop.run_until_complete(asyncio.sleep(1)) + self.loop.run_until_complete( + self.testvm1.run_for_stdio('pkill parecord')) + record.wait() + recorded_audio, _ = self.loop.run_until_complete( + self.testvm1.run_for_stdio('cat audio_rec.raw')) + # allow few bytes to be missing + if audio_in[:-8] not in recorded_audio: + self.fail('VM not recorded expected data') + def test_250_resize_private_img(self): """ Test private.img resize, both offline and online @@ -882,10 +1027,10 @@ int main(int argc, char **argv) { input=allocator_c.encode()) try: - stdout, stderr = yield from self.testvm1.run_for_stdio( + yield from self.testvm1.run_for_stdio( 'gcc allocator.c -o allocator') - except subprocess.CalledProcessError: - self.skipTest('allocator compile failed: {}'.format(stderr)) + except subprocess.CalledProcessError as e: + self.skipTest('allocator compile failed: {}'.format(e.stderr)) # drop caches to have even more memory pressure yield from self.testvm1.run_for_stdio( @@ -928,15 +1073,12 @@ int main(int argc, char **argv) { proc = yield from self.testvm1.run( 'xterm -maximized -e top') - # help xdotool a little... - yield from asyncio.sleep(2) if proc.returncode is not None: self.fail('xterm failed to start') # get window ID - winid = (yield from asyncio.get_event_loop().run_in_executor(None, - subprocess.check_output, - ['xdotool', 'search', '--sync', '--onlyvisible', '--class', - self.testvm1.name + ':xterm'])).decode() + winid = yield from self.wait_for_window_coro( + self.testvm1.name + ':xterm', + search_class=True) xprop = yield from asyncio.get_event_loop().run_in_executor(None, subprocess.check_output, ['xprop', '-notype', '-id', winid, '_QUBES_VMWINDOWID']) @@ -959,6 +1101,9 @@ int main(int argc, char **argv) { # wait for damage notify - top updates every 3 sec by default yield from asyncio.sleep(6) + # stop changing the window content + subprocess.check_call(['xdotool', 'key', '--window', winid, 'd']) + # now take screenshot of the window, from dom0 and VM # choose pnm format, as it doesn't have any useless metadata - easy # to compare @@ -1011,10 +1156,14 @@ class TC_10_Generic(qubes.tests.SystemTestCase): 'Flag file created (service was run) even though should be denied,' ' qrexec-client-vm output: {} {}'.format(stdout, stderr)) +def create_testcases_for_templates(): + return qubes.tests.create_testcases_for_templates('TC_00_AppVM', + TC_00_AppVMMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('TC_00_AppVM', - TC_00_AppVMMixin, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) + create_testcases_for_templates())) return tests + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) diff --git a/qubes/tests/storage.py b/qubes/tests/storage.py index 1af72a08..97d5932f 100644 --- a/qubes/tests/storage.py +++ b/qubes/tests/storage.py @@ -22,6 +22,7 @@ import qubes.storage from qubes.exc import QubesException from qubes.storage import pool_drivers from qubes.storage.file import FilePool +from qubes.storage.reflink import ReflinkPool from qubes.tests import SystemTestCase # :pylint: disable=invalid-name @@ -107,10 +108,11 @@ class TC_00_Pool(SystemTestCase): pool_drivers()) def test_002_get_pool_klass(self): - """ Expect the default pool to be `FilePool` """ + """ Expect the default pool to be `FilePool` or `ReflinkPool` """ # :pylint: disable=protected-access result = self.app.get_pool('varlibqubes') - self.assertIsInstance(result, FilePool) + self.assertTrue(isinstance(result, FilePool) + or isinstance(result, ReflinkPool)) def test_003_pool_exists_default(self): """ Expect the default pool to exists """ diff --git a/qubes/tests/storage_lvm.py b/qubes/tests/storage_lvm.py index f7082cad..3f320790 100644 --- a/qubes/tests/storage_lvm.py +++ b/qubes/tests/storage_lvm.py @@ -24,15 +24,15 @@ 'volume_group/thin_pool' combination. Pool variables without a prefix represent a :py:class:`qubes.storage.lvm.ThinPool`. ''' - import os import subprocess import tempfile import unittest +import unittest.mock import qubes.tests import qubes.storage -from qubes.storage.lvm import ThinPool, ThinVolume +from qubes.storage.lvm import ThinPool, ThinVolume, qubes_lvm if 'DEFAULT_LVM_POOL' in os.environ.keys(): DEFAULT_LVM_POOL = os.environ['DEFAULT_LVM_POOL'] @@ -84,7 +84,7 @@ class ThinPoolBase(qubes.tests.QubesTestCase): ''' Returns the pool matching the specified ``volume_group`` & ``thin_pool``, or None. ''' - pools = [p for p in self.app.pools + pools = [p for p in self.app.pools.values() if issubclass(p.__class__, ThinPool)] for pool in pools: if pool.volume_group == volume_group \ @@ -136,10 +136,10 @@ class TC_00_ThinPool(ThinPoolBase): self.assertEqual(volume.name, 'root') self.assertEqual(volume.pool, self.pool.name) self.assertEqual(volume.size, qubes.config.defaults['root_img_size']) - volume.create() + self.loop.run_until_complete(volume.create()) path = "/dev/%s" % volume.vid - self.assertTrue(os.path.exists(path)) - volume.remove() + self.assertTrue(os.path.exists(path), path) + self.loop.run_until_complete(volume.remove()) def test_003_read_write_volume(self): ''' Test read-write volume creation ''' @@ -156,10 +156,10 @@ class TC_00_ThinPool(ThinPoolBase): self.assertEqual(volume.name, 'root') self.assertEqual(volume.pool, self.pool.name) self.assertEqual(volume.size, qubes.config.defaults['root_img_size']) - volume.create() + self.loop.run_until_complete(volume.create()) path = "/dev/%s" % volume.vid - self.assertTrue(os.path.exists(path)) - volume.remove() + self.assertTrue(os.path.exists(path), path) + self.loop.run_until_complete(volume.remove()) def test_004_size(self): with self.assertNotRaises(NotImplementedError): @@ -207,11 +207,11 @@ class TC_00_ThinPool(ThinPoolBase): } vm = qubes.tests.storage.TestVM(self) volume = self.app.get_pool(self.pool.name).init_volume(vm, config) - volume.create() - self.addCleanup(volume.remove) + self.loop.run_until_complete(volume.create()) + self.addCleanup(self.loop.run_until_complete, volume.remove()) path = "/dev/%s" % volume.vid new_size = 64 * 1024 ** 2 - volume.resize(new_size) + self.loop.run_until_complete(volume.resize(new_size)) self.assertEqual(self._get_size(path), new_size) self.assertEqual(volume.size, new_size) @@ -226,20 +226,711 @@ class TC_00_ThinPool(ThinPoolBase): } vm = qubes.tests.storage.TestVM(self) volume = self.app.get_pool(self.pool.name).init_volume(vm, config) - volume.create() - self.addCleanup(volume.remove) - volume.start() + self.loop.run_until_complete(volume.create()) + self.addCleanup(self.loop.run_until_complete, volume.remove()) + self.loop.run_until_complete(volume.start()) path = "/dev/%s" % volume.vid path2 = "/dev/%s" % volume._vid_snap new_size = 64 * 1024 ** 2 - volume.resize(new_size) + self.loop.run_until_complete(volume.resize(new_size)) self.assertEqual(self._get_size(path), old_size) self.assertEqual(self._get_size(path2), new_size) self.assertEqual(volume.size, new_size) - volume.stop() + self.loop.run_until_complete(volume.stop()) self.assertEqual(self._get_size(path), new_size) self.assertEqual(volume.size, new_size) + def _get_lv_uuid(self, lv): + sudo = [] if os.getuid() == 0 else ['sudo'] + lvs_output = subprocess.check_output( + sudo + ['lvs', '--noheadings', '-o', 'lv_uuid', lv]) + return lvs_output.strip() + + def _get_lv_origin_uuid(self, lv): + sudo = [] if os.getuid() == 0 else ['sudo'] + if qubes.storage.lvm.lvm_is_very_old: + # no support for origin_uuid directly + lvs_output = subprocess.check_output( + sudo + ['lvs', '--noheadings', '-o', 'origin', lv]) + lvs_output = subprocess.check_output( + sudo + ['lvs', '--noheadings', '-o', 'lv_uuid', + lv.rsplit('/', 1)[0] + '/' + lvs_output.strip().decode()]) + else: + lvs_output = subprocess.check_output( + sudo + ['lvs', '--noheadings', '-o', 'origin_uuid', lv]) + return lvs_output.strip() + + def test_008_commit(self): + ''' Test volume changes commit''' + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + self.loop.run_until_complete(volume.create()) + path_snap = '/dev/' + volume._vid_snap + self.assertFalse(os.path.exists(path_snap), path_snap) + origin_uuid = self._get_lv_uuid(volume.path) + self.loop.run_until_complete(volume.start()) + snap_uuid = self._get_lv_uuid(path_snap) + self.assertNotEqual(origin_uuid, snap_uuid) + path = volume.path + self.assertTrue(path.startswith('/dev/' + volume.vid), + '{} does not start with /dev/{}'.format(path, volume.vid)) + self.assertTrue(os.path.exists(path), path) + self.loop.run_until_complete(volume.remove()) + + def test_009_interrupted_commit(self): + ''' Test volume changes commit''' + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 2, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + # mock logging, to not interfere with time.time() mock + volume.log = unittest.mock.Mock() + # do not call volume.create(), do it manually to simulate + # interrupted commit + revisions = ['-1521065904-back', '-1521065905-back', '-snap'] + orig_uuids = {} + for rev in revisions: + cmd = ['create', self.pool._pool_id, + volume.vid.split('/')[1] + rev, str(config['size'])] + qubes_lvm(cmd) + orig_uuids[rev] = self._get_lv_uuid(volume.vid + rev) + qubes.storage.lvm.reset_cache() + path_snap = '/dev/' + volume._vid_snap + self.assertTrue(volume.is_dirty()) + self.assertEqual(volume.path, + '/dev/' + volume.vid + revisions[1]) + expected_revisions = { + revisions[0].lstrip('-'): '2018-03-14T22:18:24', + revisions[1].lstrip('-'): '2018-03-14T22:18:25', + } + self.assertEqual(volume.revisions, expected_revisions) + self.loop.run_until_complete(volume.start()) + self.assertEqual(volume.revisions, expected_revisions) + snap_uuid = self._get_lv_uuid(path_snap) + self.assertEqual(orig_uuids['-snap'], snap_uuid) + self.assertTrue(volume.is_dirty()) + self.assertEqual(volume.path, + '/dev/' + volume.vid + revisions[1]) + with unittest.mock.patch('time.time') as mock_time: + mock_time.side_effect = [521065906] + self.loop.run_until_complete(volume.stop()) + expected_revisions = { + revisions[0].lstrip('-'): '2018-03-14T22:18:24', + revisions[1].lstrip('-'): '2018-03-14T22:18:25', + } + self.assertFalse(volume.is_dirty()) + self.assertEqual(volume.revisions, expected_revisions) + self.assertEqual(volume.path, '/dev/' + volume.vid) + self.assertEqual(snap_uuid, self._get_lv_uuid(volume.path)) + self.assertFalse(os.path.exists(path_snap), path_snap) + + self.loop.run_until_complete(volume.remove()) + + def test_010_migration1(self): + '''Start with old revisions, then start interacting using new code''' + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 2, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + # mock logging, to not interfere with time.time() mock + volume.log = unittest.mock.Mock() + # do not call volume.create(), do it manually to have old LV naming + revisions = ['', '-1521065904-back', '-1521065905-back'] + orig_uuids = {} + for rev in revisions: + cmd = ['create', self.pool._pool_id, + volume.vid.split('/')[1] + rev, str(config['size'])] + qubes_lvm(cmd) + orig_uuids[rev] = self._get_lv_uuid(volume.vid + rev) + qubes.storage.lvm.reset_cache() + path_snap = '/dev/' + volume._vid_snap + self.assertFalse(os.path.exists(path_snap), path_snap) + expected_revisions = { + revisions[1].lstrip('-'): '2018-03-14T22:18:24', + revisions[2].lstrip('-'): '2018-03-14T22:18:25', + } + self.assertEqual(volume.revisions, expected_revisions) + self.assertEqual(volume.path, '/dev/' + volume.vid) + + self.loop.run_until_complete(volume.start()) + snap_uuid = self._get_lv_uuid(path_snap) + self.assertNotEqual(orig_uuids[''], snap_uuid) + snap_origin_uuid = self._get_lv_origin_uuid(path_snap) + self.assertEqual(orig_uuids[''], snap_origin_uuid) + path = volume.path + self.assertEqual(path, '/dev/' + volume.vid) + self.assertTrue(os.path.exists(path), path) + + with unittest.mock.patch('time.time') as mock_time: + mock_time.side_effect = ('1521065906', '1521065907') + self.loop.run_until_complete(volume.stop()) + revisions.extend(['-1521065906-back']) + expected_revisions = { + revisions[2].lstrip('-'): '2018-03-14T22:18:25', + revisions[3].lstrip('-'): '2018-03-14T22:18:26', + } + self.assertEqual(volume.revisions, expected_revisions) + self.assertEqual(volume.path, '/dev/' + volume.vid) + path_snap = '/dev/' + volume._vid_snap + self.assertFalse(os.path.exists(path_snap), path_snap) + self.assertTrue(os.path.exists('/dev/' + volume.vid)) + self.assertEqual(self._get_lv_uuid(volume.path), snap_uuid) + prev_path = '/dev/' + volume.vid + revisions[3] + self.assertEqual(self._get_lv_uuid(prev_path), orig_uuids['']) + + self.loop.run_until_complete(volume.remove()) + for rev in revisions: + path = '/dev/' + volume.vid + rev + self.assertFalse(os.path.exists(path), path) + + def test_011_migration2(self): + '''VM started with old code, stopped with new''' + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 1, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + # mock logging, to not interfere with time.time() mock + volume.log = unittest.mock.Mock() + # do not call volume.create(), do it manually to have old LV naming + revisions = ['', '-snap'] + orig_uuids = {} + for rev in revisions: + cmd = ['create', self.pool._pool_id, + volume.vid.split('/')[1] + rev, str(config['size'])] + qubes_lvm(cmd) + orig_uuids[rev] = self._get_lv_uuid(volume.vid + rev) + qubes.storage.lvm.reset_cache() + path_snap = '/dev/' + volume._vid_snap + self.assertTrue(os.path.exists(path_snap), path_snap) + expected_revisions = {} + self.assertEqual(volume.revisions, expected_revisions) + self.assertEqual(volume.path, '/dev/' + volume.vid) + self.assertTrue(volume.is_dirty()) + + path = volume.path + self.assertEqual(path, '/dev/' + volume.vid) + self.assertTrue(os.path.exists(path), path) + + with unittest.mock.patch('time.time') as mock_time: + mock_time.side_effect = ('1521065906', '1521065907') + self.loop.run_until_complete(volume.stop()) + revisions.extend(['-1521065906-back']) + expected_revisions = { + revisions[2].lstrip('-'): '2018-03-14T22:18:26', + } + self.assertEqual(volume.revisions, expected_revisions) + self.assertEqual(volume.path, '/dev/' + volume.vid) + path_snap = '/dev/' + volume._vid_snap + self.assertFalse(os.path.exists(path_snap), path_snap) + self.assertTrue(os.path.exists('/dev/' + volume.vid)) + self.assertEqual(self._get_lv_uuid(volume.path), orig_uuids['-snap']) + prev_path = '/dev/' + volume.vid + revisions[2] + self.assertEqual(self._get_lv_uuid(prev_path), orig_uuids['']) + + self.loop.run_until_complete(volume.remove()) + for rev in revisions: + path = '/dev/' + volume.vid + rev + self.assertFalse(os.path.exists(path), path) + + def test_012_migration3(self): + '''VM started with old code, started again with new, stopped with new''' + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 1, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + # mock logging, to not interfere with time.time() mock + volume.log = unittest.mock.Mock() + # do not call volume.create(), do it manually to have old LV naming + revisions = ['', '-snap'] + orig_uuids = {} + for rev in revisions: + cmd = ['create', self.pool._pool_id, + volume.vid.split('/')[1] + rev, str(config['size'])] + qubes_lvm(cmd) + orig_uuids[rev] = self._get_lv_uuid(volume.vid + rev) + qubes.storage.lvm.reset_cache() + path_snap = '/dev/' + volume._vid_snap + self.assertTrue(os.path.exists(path_snap), path_snap) + expected_revisions = {} + self.assertEqual(volume.revisions, expected_revisions) + self.assertTrue(volume.path, '/dev/' + volume.vid) + self.assertTrue(volume.is_dirty()) + + self.loop.run_until_complete(volume.start()) + self.assertEqual(volume.revisions, expected_revisions) + self.assertEqual(volume.path, '/dev/' + volume.vid) + # -snap LV should be unchanged + self.assertEqual(self._get_lv_uuid(volume._vid_snap), + orig_uuids['-snap']) + + self.loop.run_until_complete(volume.remove()) + for rev in revisions: + path = '/dev/' + volume.vid + rev + self.assertFalse(os.path.exists(path), path) + + def test_013_migration4(self): + '''revisions_to_keep=0, VM started with old code, stopped with new''' + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 0, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + # mock logging, to not interfere with time.time() mock + volume.log = unittest.mock.Mock() + # do not call volume.create(), do it manually to have old LV naming + revisions = ['', '-snap'] + orig_uuids = {} + for rev in revisions: + cmd = ['create', self.pool._pool_id, + volume.vid.split('/')[1] + rev, str(config['size'])] + qubes_lvm(cmd) + orig_uuids[rev] = self._get_lv_uuid(volume.vid + rev) + qubes.storage.lvm.reset_cache() + path_snap = '/dev/' + volume._vid_snap + self.assertTrue(os.path.exists(path_snap), path_snap) + expected_revisions = {} + self.assertEqual(volume.revisions, expected_revisions) + self.assertEqual(volume.path, '/dev/' + volume.vid) + self.assertTrue(volume.is_dirty()) + + with unittest.mock.patch('time.time') as mock_time: + mock_time.side_effect = ('1521065906', '1521065907') + self.loop.run_until_complete(volume.stop()) + expected_revisions = {} + self.assertEqual(volume.revisions, expected_revisions) + self.assertEqual(volume.path, '/dev/' + volume.vid) + + self.loop.run_until_complete(volume.remove()) + for rev in revisions: + path = '/dev/' + volume.vid + rev + self.assertFalse(os.path.exists(path), path) + + def test_014_commit_keep_0(self): + ''' Test volume changes commit, with revisions_to_keep=0''' + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 0, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + # mock logging, to not interfere with time.time() mock + volume.log = unittest.mock.Mock() + self.loop.run_until_complete(volume.create()) + self.assertFalse(volume.is_dirty()) + path = volume.path + expected_revisions = {} + self.assertEqual(volume.revisions, expected_revisions) + + self.loop.run_until_complete(volume.start()) + self.assertEqual(volume.revisions, expected_revisions) + path_snap = '/dev/' + volume._vid_snap + snap_uuid = self._get_lv_uuid(path_snap) + self.assertTrue(volume.is_dirty()) + self.assertEqual(volume.path, path) + + with unittest.mock.patch('time.time') as mock_time: + mock_time.side_effect = [521065906] + self.loop.run_until_complete(volume.stop()) + self.assertFalse(volume.is_dirty()) + self.assertEqual(volume.revisions, {}) + self.assertEqual(volume.path, '/dev/' + volume.vid) + self.assertEqual(snap_uuid, self._get_lv_uuid(volume.path)) + self.assertFalse(os.path.exists(path_snap), path_snap) + + self.loop.run_until_complete(volume.remove()) + + def test_020_revert_last(self): + ''' Test volume revert''' + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 2, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + self.loop.run_until_complete(volume.create()) + self.loop.run_until_complete(volume.start()) + self.loop.run_until_complete(volume.stop()) + self.loop.run_until_complete(volume.start()) + self.loop.run_until_complete(volume.stop()) + self.assertEqual(len(volume.revisions), 2) + revisions = volume.revisions + revision_id = max(revisions.keys()) + current_path = volume.path + current_uuid = self._get_lv_uuid(volume.path) + rev_uuid = self._get_lv_uuid(volume.vid + '-' + revision_id) + self.assertFalse(volume.is_dirty()) + self.assertNotEqual(current_uuid, rev_uuid) + self.loop.run_until_complete(volume.revert()) + path_snap = '/dev/' + volume._vid_snap + self.assertFalse(os.path.exists(path_snap), path_snap) + self.assertEqual(current_path, volume.path) + new_uuid = self._get_lv_origin_uuid(volume.path) + self.assertEqual(new_uuid, rev_uuid) + self.assertEqual(volume.revisions, revisions) + + self.loop.run_until_complete(volume.remove()) + + def test_021_revert_earlier(self): + ''' Test volume revert''' + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 2, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + self.loop.run_until_complete(volume.create()) + self.loop.run_until_complete(volume.start()) + self.loop.run_until_complete(volume.stop()) + self.loop.run_until_complete(volume.start()) + self.loop.run_until_complete(volume.stop()) + self.assertEqual(len(volume.revisions), 2) + revisions = volume.revisions + revision_id = min(revisions.keys()) + current_path = volume.path + current_uuid = self._get_lv_uuid(volume.path) + rev_uuid = self._get_lv_uuid(volume.vid + '-' + revision_id) + self.assertFalse(volume.is_dirty()) + self.assertNotEqual(current_uuid, rev_uuid) + self.loop.run_until_complete(volume.revert(revision_id)) + path_snap = '/dev/' + volume._vid_snap + self.assertFalse(os.path.exists(path_snap), path_snap) + self.assertEqual(current_path, volume.path) + new_uuid = self._get_lv_origin_uuid(volume.path) + self.assertEqual(new_uuid, rev_uuid) + self.assertEqual(volume.revisions, revisions) + + self.loop.run_until_complete(volume.remove()) + + def test_030_import_data(self): + ''' Test volume import''' + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 2, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + self.loop.run_until_complete(volume.create()) + current_uuid = self._get_lv_uuid(volume.path) + self.assertFalse(volume.is_dirty()) + import_path = self.loop.run_until_complete(volume.import_data()) + import_uuid = self._get_lv_uuid(import_path) + self.assertNotEqual(current_uuid, import_uuid) + # success - commit data + self.loop.run_until_complete(volume.import_data_end(True)) + new_current_uuid = self._get_lv_uuid(volume.path) + self.assertEqual(new_current_uuid, import_uuid) + revisions = volume.revisions + self.assertEqual(len(revisions), 1) + revision = revisions.popitem()[0] + self.assertEqual(current_uuid, + self._get_lv_uuid(volume.vid + '-' + revision)) + self.assertFalse(os.path.exists(import_path), import_path) + + self.loop.run_until_complete(volume.remove()) + + def test_031_import_data_fail(self): + ''' Test volume import''' + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 2, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + self.loop.run_until_complete(volume.create()) + current_uuid = self._get_lv_uuid(volume.path) + self.assertFalse(volume.is_dirty()) + import_path = self.loop.run_until_complete(volume.import_data()) + import_uuid = self._get_lv_uuid(import_path) + self.assertNotEqual(current_uuid, import_uuid) + # fail - discard data + self.loop.run_until_complete(volume.import_data_end(False)) + new_current_uuid = self._get_lv_uuid(volume.path) + self.assertEqual(new_current_uuid, current_uuid) + revisions = volume.revisions + self.assertEqual(len(revisions), 0) + self.assertFalse(os.path.exists(import_path), import_path) + + self.loop.run_until_complete(volume.remove()) + + def test_032_import_volume_same_pool(self): + '''Import volume from the same pool''' + # source volume + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 2, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + source_volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + self.loop.run_until_complete(source_volume.create()) + + source_uuid = self._get_lv_uuid(source_volume.path) + + # destination volume + config = { + 'name': 'root2', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 2, + 'size': qubes.config.defaults['root_img_size'], + } + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + volume.log = unittest.mock.Mock() + with unittest.mock.patch('time.time') as mock_time: + mock_time.side_effect = [1521065905] + self.loop.run_until_complete(volume.create()) + + self.assertEqual(volume.revisions, {}) + uuid_before = self._get_lv_uuid(volume.path) + + with unittest.mock.patch('time.time') as mock_time: + mock_time.side_effect = [1521065906] + self.loop.run_until_complete( + volume.import_volume(source_volume)) + + uuid_after = self._get_lv_uuid(volume.path) + self.assertNotEqual(uuid_after, uuid_before) + + # also should be different than source volume (clone, not the same LV) + self.assertNotEqual(uuid_after, source_uuid) + self.assertEqual(self._get_lv_origin_uuid(volume.path), source_uuid) + + expected_revisions = { + '1521065906-back': '2018-03-14T22:18:26', + } + self.assertEqual(volume.revisions, expected_revisions) + + self.loop.run_until_complete(volume.remove()) + self.loop.run_until_complete(source_volume.remove()) + + def test_033_import_volume_different_pool(self): + '''Import volume from a different pool''' + source_volume = unittest.mock.Mock() + # destination volume + config = { + 'name': 'root2', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 2, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + volume.log = unittest.mock.Mock() + with unittest.mock.patch('time.time') as mock_time: + mock_time.side_effect = [1521065905] + self.loop.run_until_complete(volume.create()) + + self.assertEqual(volume.revisions, {}) + uuid_before = self._get_lv_uuid(volume.path) + + with tempfile.NamedTemporaryFile() as source_volume_file: + source_volume_file.write(b'test-content') + source_volume_file.flush() + source_volume.size = 16 * 1024 * 1024 # 16MiB + source_volume.export.return_value = source_volume_file.name + with unittest.mock.patch('time.time') as mock_time: + mock_time.side_effect = [1521065906] + self.loop.run_until_complete( + volume.import_volume(source_volume)) + + uuid_after = self._get_lv_uuid(volume.path) + self.assertNotEqual(uuid_after, uuid_before) + self.assertEqual(volume.size, 16 * 1024 * 1024) + + volume_content = subprocess.check_output(['sudo', 'cat', volume.path]) + self.assertEqual(volume_content.rstrip(b'\0'), b'test-content') + + expected_revisions = { + '1521065906-back': '2018-03-14T22:18:26', + } + self.assertEqual(volume.revisions, expected_revisions) + + self.loop.run_until_complete(volume.remove()) + + def test_040_volatile(self): + '''Volatile volume test''' + config = { + 'name': 'volatile', + 'pool': self.pool.name, + 'rw': True, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + # volatile volume don't need any file, verify should succeed + self.assertTrue(volume.verify()) + self.loop.run_until_complete(volume.create()) + self.assertTrue(volume.verify()) + self.assertFalse(volume.save_on_stop) + self.assertFalse(volume.snap_on_start) + path = volume.path + self.assertEqual(path, '/dev/' + volume.vid) + self.assertFalse(os.path.exists(path)) + self.loop.run_until_complete(volume.start()) + self.assertTrue(os.path.exists(path)) + vol_uuid = self._get_lv_uuid(path) + self.loop.run_until_complete(volume.start()) + self.assertTrue(os.path.exists(path)) + vol_uuid2 = self._get_lv_uuid(path) + self.assertNotEqual(vol_uuid, vol_uuid2) + self.loop.run_until_complete(volume.stop()) + self.assertFalse(os.path.exists(path)) + + def test_050_snapshot_volume(self): + ''' Test snapshot volume creation ''' + config_origin = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume_origin = self.app.get_pool(self.pool.name).init_volume( + vm, config_origin) + self.loop.run_until_complete(volume_origin.create()) + config_snapshot = { + 'name': 'root2', + 'pool': self.pool.name, + 'snap_on_start': True, + 'source': volume_origin, + 'rw': True, + 'size': qubes.config.defaults['root_img_size'], + } + volume = self.app.get_pool(self.pool.name).init_volume( + vm, config_snapshot) + self.assertIsInstance(volume, ThinVolume) + self.assertEqual(volume.name, 'root2') + self.assertEqual(volume.pool, self.pool.name) + self.assertEqual(volume.size, qubes.config.defaults['root_img_size']) + # only origin volume really needs to exist, verify should succeed + # even before create + self.assertTrue(volume.verify()) + self.loop.run_until_complete(volume.create()) + path = volume.path + self.assertEqual(path, '/dev/' + volume.vid) + self.assertFalse(os.path.exists(path), path) + self.loop.run_until_complete(volume.start()) + # snapshot volume isn't considered dirty at any time + self.assertFalse(volume.is_dirty()) + # not outdated yet + self.assertFalse(volume.is_outdated()) + origin_uuid = self._get_lv_uuid(volume_origin.path) + snap_origin_uuid = self._get_lv_origin_uuid(volume._vid_snap) + self.assertEqual(origin_uuid, snap_origin_uuid) + + # now make it outdated + self.loop.run_until_complete(volume_origin.start()) + self.loop.run_until_complete(volume_origin.stop()) + self.assertTrue(volume.is_outdated()) + origin_uuid = self._get_lv_uuid(volume_origin.path) + self.assertNotEqual(origin_uuid, snap_origin_uuid) + + self.loop.run_until_complete(volume.stop()) + # stopped volume is never outdated + self.assertFalse(volume.is_outdated()) + path = volume.path + self.assertFalse(os.path.exists(path), path) + path = '/dev/' + volume._vid_snap + self.assertFalse(os.path.exists(path), path) + + self.loop.run_until_complete(volume.remove()) + self.loop.run_until_complete(volume_origin.remove()) + + def test_100_pool_list_volumes(self): + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 2, + 'size': qubes.config.defaults['root_img_size'], + } + config2 = config.copy() + vm = qubes.tests.storage.TestVM(self) + volume1 = self.app.get_pool(self.pool.name).init_volume(vm, config) + self.loop.run_until_complete(volume1.create()) + config2['name'] = 'private' + volume2 = self.app.get_pool(self.pool.name).init_volume(vm, config2) + self.loop.run_until_complete(volume2.create()) + + # create some revisions + self.loop.run_until_complete(volume1.start()) + self.loop.run_until_complete(volume1.stop()) + + # and have one in dirty state + self.loop.run_until_complete(volume2.start()) + + self.assertIn(volume1, list(self.pool.volumes)) + self.assertIn(volume2, list(self.pool.volumes)) + self.loop.run_until_complete(volume1.remove()) + self.assertNotIn(volume1, list(self.pool.volumes)) + self.assertIn(volume2, list(self.pool.volumes)) + self.loop.run_until_complete(volume2.remove()) + self.assertNotIn(volume1, list(self.pool.volumes)) + self.assertNotIn(volume1, list(self.pool.volumes)) @skipUnlessLvmPoolExists class TC_01_ThinPool(ThinPoolBase, qubes.tests.SystemTestCase): @@ -255,26 +946,27 @@ class TC_01_ThinPool(ThinPoolBase, qubes.tests.SystemTestCase): vm = self.app.add_new_vm(qubes.vm.templatevm.TemplateVM, name=name, label='red') vm.clone_properties(template_vm) - vm.clone_disk_files(template_vm, pool='test-lvm') + self.loop.run_until_complete( + vm.clone_disk_files(template_vm, pool=self.pool.name)) for v_name, volume in vm.volumes.items(): if volume.save_on_stop: expected = "/dev/{!s}/vm-{!s}-{!s}".format( DEFAULT_LVM_POOL.split('/')[0], vm.name, v_name) self.assertEqual(volume.path, expected) with self.assertNotRaises(qubes.exc.QubesException): - vm.start() + self.loop.run_until_complete(vm.start()) def test_005_create_appvm(self): vm = self.app.add_new_vm(cls=qubes.vm.appvm.AppVM, name=self.make_vm_name('appvm'), label='red') - vm.create_on_disk(pool='test-lvm') + self.loop.run_until_complete(vm.create_on_disk(pool=self.pool.name)) for v_name, volume in vm.volumes.items(): if volume.save_on_stop: expected = "/dev/{!s}/vm-{!s}-{!s}".format( DEFAULT_LVM_POOL.split('/')[0], vm.name, v_name) self.assertEqual(volume.path, expected) with self.assertNotRaises(qubes.exc.QubesException): - vm.start() + self.loop.run_until_complete(vm.start()) @skipUnlessLvmPoolExists class TC_02_StorageHelpers(ThinPoolBase): diff --git a/qubes/tests/storage_reflink.py b/qubes/tests/storage_reflink.py new file mode 100644 index 00000000..fff26144 --- /dev/null +++ b/qubes/tests/storage_reflink.py @@ -0,0 +1,154 @@ +# +# The Qubes OS Project, https://www.qubes-os.org +# +# Copyright (C) 2018 Rusty Bird +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, see . +# + +''' Tests for the file-reflink storage driver ''' + +# pylint: disable=protected-access +# pylint: disable=invalid-name + +import os +import shutil +import subprocess +import sys + +import qubes.tests +from qubes.storage import reflink + + +class ReflinkMixin: + def setUp(self, fs_type='btrfs'): # pylint: disable=arguments-differ + super().setUp() + self.test_dir = '/var/tmp/test-reflink-units-on-' + fs_type + mkdir_fs(self.test_dir, fs_type, cleanup_via=self.addCleanup) + + def test_000_copy_file(self): + source = os.path.join(self.test_dir, 'source-file') + dest = os.path.join(self.test_dir, 'new-directory', 'dest-file') + content = os.urandom(1024**2) + + with open(source, 'wb') as source_io: + source_io.write(content) + + ficlone_succeeded = reflink._copy_file(source, dest) + self.assertEqual(ficlone_succeeded, self.ficlone_supported) + + self.assertNotEqual(os.stat(source).st_ino, os.stat(dest).st_ino) + with open(source, 'rb') as source_io: + self.assertEqual(source_io.read(), content) + with open(dest, 'rb') as dest_io: + self.assertEqual(dest_io.read(), content) + + def test_001_create_and_resize_files_and_update_loopdevs(self): + img_real = os.path.join(self.test_dir, 'img-real') + img_sym = os.path.join(self.test_dir, 'img-sym') + size_initial = 111 * 1024**2 + size_resized = 222 * 1024**2 + + os.symlink(img_real, img_sym) + reflink._create_sparse_file(img_real, size_initial) + self.assertEqual(reflink._get_file_disk_usage(img_real), 0) + self.assertEqual(os.stat(img_real).st_size, size_initial) + + dev_from_real = setup_loopdev(img_real, cleanup_via=self.addCleanup) + dev_from_sym = setup_loopdev(img_sym, cleanup_via=self.addCleanup) + + reflink._resize_file(img_real, size_resized) + self.assertEqual(reflink._get_file_disk_usage(img_real), 0) + self.assertEqual(os.stat(img_real).st_size, size_resized) + + reflink_update_loopdev_sizes(os.path.join(self.test_dir, 'unrelated')) + + for dev in (dev_from_real, dev_from_sym): + self.assertEqual(get_blockdev_size(dev), size_initial) + + reflink_update_loopdev_sizes(img_sym) + + for dev in (dev_from_real, dev_from_sym): + self.assertEqual(get_blockdev_size(dev), size_resized) + +class TC_00_ReflinkOnBtrfs(ReflinkMixin, qubes.tests.QubesTestCase): + def setUp(self): # pylint: disable=arguments-differ + super().setUp('btrfs') + self.ficlone_supported = True + +class TC_01_ReflinkOnExt4(ReflinkMixin, qubes.tests.QubesTestCase): + def setUp(self): # pylint: disable=arguments-differ + super().setUp('ext4') + self.ficlone_supported = False + + +def setup_loopdev(img, cleanup_via=None): + dev = str.strip(cmd('sudo', 'losetup', '-f', '--show', img).decode()) + if cleanup_via is not None: + cleanup_via(detach_loopdev, dev) + return dev + +def detach_loopdev(dev): + cmd('sudo', 'losetup', '-d', dev) + +def get_fs_type(directory): + # 'stat -f -c %T' would identify ext4 as 'ext2/ext3' + return cmd('df', '--output=fstype', directory).decode().splitlines()[1] + +def mkdir_fs(directory, fs_type, + accessible=True, max_size=100*1024**3, cleanup_via=None): + os.mkdir(directory) + + if get_fs_type(directory) != fs_type: + img = os.path.join(directory, 'img') + with open(img, 'xb') as img_io: + img_io.truncate(max_size) + cmd('mkfs.' + fs_type, img) + dev = setup_loopdev(img) + os.remove(img) + cmd('sudo', 'mount', dev, directory) + detach_loopdev(dev) + + if accessible: + cmd('sudo', 'chmod', '777', directory) + else: + cmd('sudo', 'chmod', '000', directory) + cmd('sudo', 'chattr', '+i', directory) # cause EPERM on write as root + + if cleanup_via is not None: + cleanup_via(rmtree_fs, directory) + +def rmtree_fs(directory): + if os.path.ismount(directory): + cmd('sudo', 'umount', '-l', directory) + # loop device and backing file are garbage collected automatically + cmd('sudo', 'chattr', '-i', directory) + cmd('sudo', 'chmod', '777', directory) + shutil.rmtree(directory) + +def get_blockdev_size(dev): + return int(cmd('sudo', 'blockdev', '--getsize64', dev)) + +def reflink_update_loopdev_sizes(img): + env = [k + '=' + v for k, v in os.environ.items() # 'sudo -E' alone would + if k.startswith('PYTHON')] # drop some of these + code = ('from qubes.storage import reflink\n' + 'reflink._update_loopdev_sizes(%r)' % img) + cmd('sudo', '-E', 'env', *env, sys.executable, '-c', code) + +def cmd(*argv): + p = subprocess.run(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if p.returncode != 0: + raise Exception(str(p)) # this will show stdout and stderr + return p.stdout diff --git a/qubes/tests/vm/__init__.py b/qubes/tests/vm/__init__.py index d42148ec..54564b75 100644 --- a/qubes/tests/vm/__init__.py +++ b/qubes/tests/vm/__init__.py @@ -88,6 +88,7 @@ class TestApp(qubes.tests.TestEmitter): self.default_pool_root = 'default' self.default_pool_private = 'default' self.default_pool_kernel = 'linux-kernel' + self.default_qrexec_timeout = 60 self.default_netvm = None self.domains = TestVMsCollection() #: jinja2 environment for libvirt XML templates diff --git a/qubes/tests/vm/qubesvm.py b/qubes/tests/vm/qubesvm.py index 168bd579..308fd782 100644 --- a/qubes/tests/vm/qubesvm.py +++ b/qubes/tests/vm/qubesvm.py @@ -141,6 +141,50 @@ class TC_00_setters(qubes.tests.QubesTestCase): with self.assertRaises(ValueError): qubes.vm.qubesvm._setter_virt_mode(self.vm, self.prop, 'True') +class TC_10_default(qubes.tests.QubesTestCase): + def setUp(self): + super().setUp() + self.app = TestApp() + self.vm = TestVM(app=self.app) + self.prop = TestProp() + + def test_000_default_with_template_simple(self): + default_getter = qubes.vm.qubesvm._default_with_template('kernel', + 'dfl-kernel') + self.assertEqual(default_getter(self.vm), 'dfl-kernel') + self.vm.template = None + self.assertEqual(default_getter(self.vm), 'dfl-kernel') + self.vm.template = unittest.mock.Mock() + self.vm.template.kernel = 'template-kernel' + self.assertEqual(default_getter(self.vm), 'template-kernel') + + def test_001_default_with_template_callable(self): + default_getter = qubes.vm.qubesvm._default_with_template('kernel', + lambda x: x.app.default_kernel) + self.app.default_kernel = 'global-dfl-kernel' + self.assertEqual(default_getter(self.vm), 'global-dfl-kernel') + self.vm.template = None + self.assertEqual(default_getter(self.vm), 'global-dfl-kernel') + self.vm.template = unittest.mock.Mock() + self.vm.template.kernel = 'template-kernel' + self.assertEqual(default_getter(self.vm), 'template-kernel') + + def test_010_default_virt_mode(self): + default_getter = qubes.vm.qubesvm._default_with_template('kernel', + lambda x: x.app.default_kernel) + self.assertEqual(qubes.vm.qubesvm._default_virt_mode(self.vm), + 'pvh') + self.vm.template = unittest.mock.Mock() + self.vm.template.virt_mode = 'hvm' + self.assertEqual(qubes.vm.qubesvm._default_virt_mode(self.vm), + 'hvm') + self.vm.template = None + self.assertEqual(qubes.vm.qubesvm._default_virt_mode(self.vm), + 'pvh') + self.vm.devices['pci'].persistent().append('some-dev') + self.assertEqual(qubes.vm.qubesvm._default_virt_mode(self.vm), + 'hvm') + class QubesVMTestsMixin(object): property_no_default = object() @@ -415,6 +459,15 @@ class TC_90_QubesVM(QubesVMTestsMixin, qubes.tests.QubesTestCase): self.assertPropertyInvalidValue(vm, 'qrexec_timeout', '-2') self.assertPropertyInvalidValue(vm, 'qrexec_timeout', '') + def test_272_qrexec_timeout_global_changed(self): + self.app.default_qrexec_timeout = 123 + vm = self.get_vm() + self.assertPropertyDefaultValue(vm, 'qrexec_timeout', 123) + self.assertPropertyValue(vm, 'qrexec_timeout', 3, 3, '3') + del vm.qrexec_timeout + self.assertPropertyDefaultValue(vm, 'qrexec_timeout', 123) + self.assertPropertyValue(vm, 'qrexec_timeout', '3', 3, '3') + def test_280_autostart(self): vm = self.get_vm() # FIXME any better idea to not involve systemctl call at this stage? diff --git a/qubes/tools/qubes_create.py b/qubes/tools/qubes_create.py index 28059a09..56e66c7f 100644 --- a/qubes/tools/qubes_create.py +++ b/qubes/tools/qubes_create.py @@ -18,7 +18,7 @@ # License along with this library; if not, see . # -'''qvm-create - Create new Qubes OS store''' +'''qubes-create - Create new Qubes OS store''' import sys import qubes @@ -38,7 +38,7 @@ def main(args=None): args = parser.parse_args(args) qubes.Qubes.create_empty_store(args.app, - offline_mode=args.offline_mode) + offline_mode=args.offline_mode).setup_pools() return 0 diff --git a/qubes/utils.py b/qubes/utils.py index 4fefe7df..63c49d90 100644 --- a/qubes/utils.py +++ b/qubes/utils.py @@ -41,7 +41,7 @@ def get_timezone(): if os.path.islink('/etc/localtime'): return '/'.join(os.readlink('/etc/localtime').split('/')[-2:]) # <=fc17 - elif os.path.exists('/etc/sysconfig/clock'): + if os.path.exists('/etc/sysconfig/clock'): clock_config = open('/etc/sysconfig/clock', "r") clock_config_lines = clock_config.readlines() clock_config.close() @@ -50,18 +50,17 @@ def get_timezone(): line_match = zone_re.match(line) if line_match: return line_match.group(1) - else: - # last resort way, some applications makes /etc/localtime - # hardlink instead of symlink... - tz_info = os.stat('/etc/localtime') - if not tz_info: - return None - if tz_info.st_nlink > 1: - p = subprocess.Popen(['find', '/usr/share/zoneinfo', - '-inum', str(tz_info.st_ino), '-print', '-quit'], - stdout=subprocess.PIPE) - tz_path = p.communicate()[0].strip() - return tz_path.replace('/usr/share/zoneinfo/', '') + # last resort way, some applications makes /etc/localtime + # hardlink instead of symlink... + tz_info = os.stat('/etc/localtime') + if not tz_info: + return None + if tz_info.st_nlink > 1: + p = subprocess.Popen(['find', '/usr/share/zoneinfo', + '-inum', str(tz_info.st_ino), '-print', '-quit'], + stdout=subprocess.PIPE) + tz_path = p.communicate()[0].strip() + return tz_path.replace(b'/usr/share/zoneinfo/', b'') return None @@ -132,9 +131,9 @@ def size_to_human(size): """Humane readable size, with 1/10 precision""" if size < 1024: return str(size) - elif size < 1024 * 1024: + if size < 1024 * 1024: return str(round(size / 1024.0, 1)) + ' KiB' - elif size < 1024 * 1024 * 1024: + if size < 1024 * 1024 * 1024: return str(round(size / (1024.0 * 1024), 1)) + ' MiB' return str(round(size / (1024.0 * 1024 * 1024), 1)) + ' GiB' @@ -181,6 +180,6 @@ def match_vm_name_with_special(vm, name): or @type:...''' if name.startswith('@tag:'): return name[len('@tag:'):] in vm.tags - elif name.startswith('@type:'): + if name.startswith('@type:'): return name[len('@type:'):] == vm.__class__.__name__ return name == vm.name diff --git a/qubes/vm/__init__.py b/qubes/vm/__init__.py index 517e08ec..ce7d7979 100644 --- a/qubes/vm/__init__.py +++ b/qubes/vm/__init__.py @@ -569,10 +569,9 @@ class VMProperty(qubes.property): if self.allow_none: super(VMProperty, self).__set__(instance, value) return - else: - raise ValueError( - 'Property {!r} does not allow setting to {!r}'.format( - self.__name__, value)) + raise ValueError( + 'Property {!r} does not allow setting to {!r}'.format( + self.__name__, value)) app = instance if isinstance(instance, qubes.Qubes) else instance.app diff --git a/qubes/vm/adminvm.py b/qubes/vm/adminvm.py index 2d044f67..e83f0ab9 100644 --- a/qubes/vm/adminvm.py +++ b/qubes/vm/adminvm.py @@ -147,12 +147,11 @@ class AdminVM(qubes.vm.BaseVM): if self.app.vmm.offline_mode: # default value passed on xen cmdline return 4096 - else: - try: - return self.app.vmm.libvirt_conn.getInfo()[1] - except libvirt.libvirtError as e: - self.log.warning('Failed to get memory limit for dom0: %s', e) - return 4096 + try: + return self.app.vmm.libvirt_conn.getInfo()[1] + except libvirt.libvirtError as e: + self.log.warning('Failed to get memory limit for dom0: %s', e) + return 4096 def verify_files(self): '''Always :py:obj:`True` @@ -181,7 +180,7 @@ class AdminVM(qubes.vm.BaseVM): @property def icon_path(self): - return None + pass @property def untrusted_qdb(self): diff --git a/qubes/vm/dispvm.py b/qubes/vm/dispvm.py index 2179dd9f..0c6389f3 100644 --- a/qubes/vm/dispvm.py +++ b/qubes/vm/dispvm.py @@ -142,8 +142,8 @@ class DispVM(qubes.vm.qubesvm.QubesVM): def _auto_cleanup(self): '''Do auto cleanup if enabled''' if self.auto_cleanup and self in self.app.domains: - yield from self.remove_from_disk() del self.app.domains[self] + yield from self.remove_from_disk() self.app.save() @classmethod @@ -193,8 +193,8 @@ class DispVM(qubes.vm.qubesvm.QubesVM): pass # if auto_cleanup is set, this will be done automatically if not self.auto_cleanup: - yield from self.remove_from_disk() del self.app.domains[self] + yield from self.remove_from_disk() self.app.save() @asyncio.coroutine diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 4730a2aa..bb4c1e03 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -24,7 +24,6 @@ from __future__ import absolute_import import asyncio import base64 -import errno import grp import os import os.path @@ -102,7 +101,25 @@ def _setter_virt_mode(self, prop, value): def _default_virt_mode(self): if self.devices['pci'].persistent(): return 'hvm' - return 'pvh' + try: + return self.template.virt_mode + except AttributeError: + return 'pvh' + +def _default_with_template(prop, default): + '''Return a callable for 'default' argument of a property. Use a value + from a template (if any), otherwise *default* + ''' + + def _func(self): + try: + return getattr(self.template, prop) + except AttributeError: + if callable(default): + return default(self) + return default + + return _func class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): @@ -158,6 +175,32 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): *other arguments are as in :py:meth:`start`* + .. event:: domain-start-failed (subject, event, reason) + + Fired when :py:meth:`start` method fails. + *reason* argument is a textual error message. + + Handler for this event can be asynchronous (a coroutine). + + :param subject: Event emitter (the qube object) + :param event: Event name (``'domain-start'``) + + *other arguments are as in :py:meth:`start`* + + .. event:: domain-paused (subject, event) + + Fired when the domain has been paused. + + :param subject: Event emitter (the qube object) + :param event: Event name (``'domain-paused'``) + + .. event:: domain-unpaused (subject, event) + + Fired when the domain has been unpaused. + + :param subject: Event emitter (the qube object) + :param event: Event name (``'domain-unpaused'``) + .. event:: domain-stopped (subject, event) Fired when domain has been stopped. @@ -387,7 +430,8 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): type=str, setter=_setter_virt_mode, default=_default_virt_mode, doc='''Virtualisation mode: full virtualisation ("HVM"), - or paravirtualisation ("PV"), or hybrid ("PVH")''') + or paravirtualisation ("PV"), or hybrid ("PVH"). TemplateBasedVMs use its ' + 'template\'s value by default.''') installed_by_rpm = qubes.property('installed_by_rpm', type=bool, setter=qubes.property.bool, @@ -397,17 +441,19 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): memory = qubes.property('memory', type=int, setter=_setter_positive_int, - default=(lambda self: + default=_default_with_template('memory', lambda self: qubes.config.defaults[ 'hvm_memory' if self.virt_mode == 'hvm' else 'memory']), - doc='Memory currently available for this VM.') + doc='Memory currently available for this VM. TemplateBasedVMs use its ' + 'template\'s value by default.') maxmem = qubes.property('maxmem', type=int, setter=_setter_positive_int, - default=(lambda self: - int(min(self.app.host.memory_total / 1024 / 2, 4000))), + default=_default_with_template('maxmem', (lambda self: + int(min(self.app.host.memory_total / 1024 / 2, 4000)))), doc='''Maximum amount of memory available for this VM (for the purpose - of the memory balancer).''') + of the memory balancer). TemplateBasedVMs use its ' + 'template\'s value by default.''') stubdom_mem = qubes.property('stubdom_mem', type=int, setter=_setter_positive_int, @@ -417,14 +463,17 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): vcpus = qubes.property('vcpus', type=int, setter=_setter_positive_int, - default=2, - doc='Number of virtual CPUs for a qube') + default=_default_with_template('vcpus', 2), + doc='Number of virtual CPUs for a qube. TemplateBasedVMs use its ' + 'template\'s value by default.') # CORE2: swallowed uses_default_kernel kernel = qubes.property('kernel', type=str, setter=_setter_kernel, - default=(lambda self: self.app.default_kernel), - doc='Kernel used by this domain.') + default=_default_with_template('kernel', + lambda self: self.app.default_kernel), + doc='Kernel used by this domain. TemplateBasedVMs use its ' + 'template\'s value by default.') # CORE2: swallowed uses_default_kernelopts # pylint: disable=no-member @@ -434,7 +483,8 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): if list(self.devices['pci'].persistent()) else self.template.kernelopts if hasattr(self, 'template') else qubes.config.defaults['kernelopts']), - doc='Kernel command line passed to domain.') + doc='Kernel command line passed to domain. TemplateBasedVMs use its ' + 'template\'s value by default.') debug = qubes.property('debug', type=bool, default=False, setter=qubes.property.bool, @@ -445,10 +495,10 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): # only plain property? default_user = qubes.property('default_user', type=str, # pylint: disable=no-member - default=(lambda self: self.template.default_user - if hasattr(self, 'template') else 'user'), + default=_default_with_template('default_user', 'user'), setter=_setter_default_user, - doc='FIXME') + doc='Default user to start applications as. TemplateBasedVMs use its ' + 'template\'s value by default.') # pylint: enable=no-member @@ -459,12 +509,22 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): # else: # return self._default_user - qrexec_timeout = qubes.property('qrexec_timeout', type=int, default=60, + qrexec_timeout = qubes.property('qrexec_timeout', type=int, + default=_default_with_template('qrexec_timeout', + lambda self: self.app.default_qrexec_timeout), setter=_setter_positive_int, doc='''Time in seconds after which qrexec connection attempt is deemed failed. Operating system inside VM should be able to boot in this time.''') + shutdown_timeout = qubes.property('shutdown_timeout', type=int, + default=_default_with_template('shutdown_timeout', + lambda self: self.app.default_shutdown_timeout), + setter=_setter_positive_int, + doc='''Time in seconds for shutdown of the VM, after which VM may be + forcefully powered off. Operating system inside VM should be + able to fully shutdown in this time.''') + autostart = qubes.property('autostart', default=False, type=bool, setter=qubes.property.bool, doc='''Setting this to `True` means that VM should be autostarted on @@ -518,10 +578,9 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): except libvirt.libvirtError as e: if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN: return -1 - else: - self.log.exception('libvirt error code: {!r}'.format( - e.get_error_code())) - raise + self.log.exception('libvirt error code: {!r}'.format( + e.get_error_code())) + raise @qubes.stateless_property def stubdom_xid(self): @@ -815,10 +874,56 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): raise qubes.exc.QubesException( 'Failed to reset autostart for VM in systemd') + @qubes.events.handler('domain-remove-from-disk') + def on_remove_from_disk(self, event, **kwargs): + # pylint: disable=unused-argument + if self.autostart: + subprocess.call( + ['sudo', 'systemctl', 'disable', + 'qubes-vm@{}.service'.format(self.name)]) + + @qubes.events.handler('domain-create-on-disk') + def on_create_on_disk(self, event, **kwargs): + # pylint: disable=unused-argument + if self.autostart: + subprocess.call( + ['sudo', 'systemctl', 'enable', + 'qubes-vm@{}.service'.format(self.name)]) + # # methods for changing domain state # + @asyncio.coroutine + def _ensure_shutdown_handled(self): + '''Make sure previous shutdown is fully handled. + MUST NOT be called when domain is running. + ''' + with (yield from self._domain_stopped_lock): + # Don't accept any new stopped event's till a new VM has been + # created. If we didn't received any stopped event or it wasn't + # handled yet we will handle this in the next lines. + self._domain_stopped_event_received = True + + if self._domain_stopped_future is not None: + # Libvirt stopped event was already received, so cancel the + # future. If it didn't generate the Qubes events yet we + # will do it below. + self._domain_stopped_future.cancel() + self._domain_stopped_future = None + + if not self._domain_stopped_event_handled: + # No Qubes domain-stopped events have been generated yet. + # So do this now. + + # Set this immediately such that we don't generate events + # twice if an exception gets thrown. + self._domain_stopped_event_handled = True + + yield from self.fire_event_async('domain-stopped') + yield from self.fire_event_async('domain-shutdown') + + @asyncio.coroutine def start(self, start_guid=True, notify_function=None, mem_required=None): @@ -835,39 +940,28 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): if self.get_power_state() != 'Halted': return self - with (yield from self._domain_stopped_lock): - # Don't accept any new stopped event's till a new VM has been - # created. If we didn't received any stopped event or it wasn't - # handled yet we will handle this in the next lines. - self._domain_stopped_event_received = True - - if self._domain_stopped_future is not None: - # Libvirt stopped event was already received, so cancel the - # future. If it didn't generate the Qubes events yet we - # will do it below. - self._domain_stopped_future.cancel() - self._domain_stopped_future = None - - if not self._domain_stopped_event_handled: - # No Qubes domain-stopped events have been generated yet. - # So do this now. - - # Set this immediately such that we don't generate events - # twice if an exception gets thrown. - self._domain_stopped_event_handled = True - - yield from self.fire_event_async('domain-stopped') - yield from self.fire_event_async('domain-shutdown') + yield from self._ensure_shutdown_handled() self.log.info('Starting {}'.format(self.name)) - yield from self.fire_event_async('domain-pre-start', - pre_event=True, - start_guid=start_guid, mem_required=mem_required) + try: + yield from self.fire_event_async('domain-pre-start', + pre_event=True, + start_guid=start_guid, mem_required=mem_required) + except Exception as exc: + yield from self.fire_event_async('domain-start-failed', + reason=str(exc)) + raise + + for devclass in self.devices: + for dev in self.devices[devclass].persistent(): + if isinstance(dev, qubes.devices.UnknownDevice): + raise qubes.exc.QubesException( + '{} device {} not available'.format(devclass, dev)) qmemman_client = None try: - if self.virt_mode == 'pvh' and self.kernel is None: + if self.virt_mode == 'pvh' and not self.kernel: raise qubes.exc.QubesException( 'virt_mode PVH require kernel to be set') yield from self.storage.verify() @@ -960,8 +1054,8 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): return if self._domain_stopped_event_received: - self.log.warning('Duplicated stopped event from libvirt received!') - # ignore this unexpected event + # ignore this event - already triggered by shutdown(), kill(), + # or subsequent start() return self._domain_stopped_event_received = True @@ -993,9 +1087,13 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): self.name) @asyncio.coroutine - def shutdown(self, force=False, wait=False): + def shutdown(self, force=False, wait=False, timeout=None): '''Shutdown domain. + :param force: ignored + :param wait: wait for shutdown to complete + :param timeout: shutdown wait timeout (for *wait*=True), defaults to + :py:attr:`shutdown_timeout` :raises qubes.exc.QubesVMNotStartedError: \ when domain is already shut down. ''' @@ -1008,8 +1106,18 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): self.libvirt_domain.shutdown() - while wait and not self.is_halted(): - yield from asyncio.sleep(0.25) + if wait: + if timeout is None: + timeout = self.shutdown_timeout + while timeout > 0 and not self.is_halted(): + yield from asyncio.sleep(0.25) + timeout -= 0.25 + with (yield from self.startup_lock): + if self.is_halted(): + # make sure all shutdown tasks are completed + yield from self._ensure_shutdown_handled() + else: + raise qubes.exc.QubesVMShutdownTimeoutError(self) return self @@ -1024,13 +1132,17 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): if not self.is_running() and not self.is_paused(): raise qubes.exc.QubesVMNotStartedError(self) - try: - self.libvirt_domain.destroy() - except libvirt.libvirtError as e: - if e.get_error_code() == libvirt.VIR_ERR_OPERATION_INVALID: - raise qubes.exc.QubesVMNotStartedError(self) - else: - raise + with (yield from self.startup_lock): + try: + self.libvirt_domain.destroy() + except libvirt.libvirtError as e: + if e.get_error_code() == libvirt.VIR_ERR_OPERATION_INVALID: + raise qubes.exc.QubesVMNotStartedError(self) + else: + raise + + # make sure all shutdown tasks are completed + yield from self._ensure_shutdown_handled() return self @@ -1377,6 +1489,8 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): 'creation'.format(self.dir_path)) raise + if os.path.exists(self.icon_path): + os.unlink(self.icon_path) self.log.info('Creating icon symlink: {} -> {}'.format( self.icon_path, self.label.icon_path)) if hasattr(os, "symlink"): @@ -1395,15 +1509,18 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): "Can't remove VM {!s}, beacuse it's in state {!r}.".format( self, self.get_power_state())) + # make sure shutdown is handled before removing anything, but only if + # handling is pending; if not, we may be called from within + # domain-shutdown event (DispVM._auto_cleanup), which would deadlock + if not self._domain_stopped_event_handled: + yield from self._ensure_shutdown_handled() + yield from self.fire_event_async('domain-remove-from-disk') try: # TODO: make it async? shutil.rmtree(self.dir_path) - except OSError as e: - if e.errno == errno.ENOENT: - pass - else: - raise + except FileNotFoundError: + pass yield from self.storage.remove() @asyncio.coroutine @@ -1575,8 +1692,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): except libvirt.libvirtError as e: if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN: return 'Halted' - else: - raise + raise libvirt_domain = self.libvirt_domain if libvirt_domain is None: @@ -1587,19 +1703,17 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): # pylint: disable=line-too-long if libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_PAUSED: return "Paused" - elif libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_CRASHED: + if libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_CRASHED: return "Crashed" - elif libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_SHUTDOWN: + if libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_SHUTDOWN: return "Halting" - elif libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_SHUTOFF: + if libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_SHUTOFF: return "Dying" - elif libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_PMSUSPENDED: # nopep8 + if libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_PMSUSPENDED: # nopep8 return "Suspended" - else: - if not self.is_fully_usable(): - return "Transient" - - return "Running" + if not self.is_fully_usable(): + return "Transient" + return "Running" return 'Halted' except libvirt.libvirtError as e: @@ -1639,8 +1753,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): except libvirt.libvirtError as e: if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN: return False - else: - raise + raise return bool(self.libvirt_domain.isActive()) @@ -1662,7 +1775,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): :py:obj:`False` otherwise. :rtype: bool ''' - if self.xid < 0: + if self.xid < 0: # pylint: disable=comparison-with-callable return False return os.path.exists('/var/run/qubes/qrexec.%s' % self.name) @@ -1706,10 +1819,9 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): libvirt.VIR_ERR_INTERNAL_ERROR): return 0 - else: - self.log.exception( - 'libvirt error code: {!r}'.format(e.get_error_code())) - raise + self.log.exception( + 'libvirt error code: {!r}'.format(e.get_error_code())) + raise def get_mem_static_max(self): '''Get maximum memory available to VM. @@ -1733,10 +1845,9 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): libvirt.VIR_ERR_INTERNAL_ERROR): return 0 - else: - self.log.exception( - 'libvirt error code: {!r}'.format(e.get_error_code())) - raise + self.log.exception( + 'libvirt error code: {!r}'.format(e.get_error_code())) + raise def get_cputime(self): '''Get total CPU time burned by this domain since start. @@ -1772,10 +1883,9 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): libvirt.VIR_ERR_INTERNAL_ERROR): return 0 - else: - self.log.exception( - 'libvirt error code: {!r}'.format(e.get_error_code())) - raise + self.log.exception( + 'libvirt error code: {!r}'.format(e.get_error_code())) + raise # miscellanous @@ -1936,7 +2046,9 @@ def _patch_pool_config(config, pool=None, pools=None): if not is_snapshot: config['pool'] = str(pools[name]) else: - msg = "Can't clone a snapshot volume {!s} to pool {!s} " \ + msg = "Snapshot volume {0!s} must be in the same pool as its " \ + "origin ({0!s} volume of template)," \ + "cannot move to pool {1!s} " \ .format(name, pools[name]) raise qubes.exc.QubesException(msg) return config diff --git a/qubespolicy/__init__.py b/qubespolicy/__init__.py index 7e1b9c61..ff23842f 100755 --- a/qubespolicy/__init__.py +++ b/qubespolicy/__init__.py @@ -18,6 +18,8 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, see . +# pylint: disable=no-else-return,useless-object-inheritance,try-except-raise + ''' Qrexec policy parser and evaluator ''' import enum import itertools diff --git a/qubespolicy/agent.py b/qubespolicy/agent.py index 4ae465eb..661ec8a1 100644 --- a/qubespolicy/agent.py +++ b/qubespolicy/agent.py @@ -33,7 +33,7 @@ import qubespolicy.rpcconfirmation import qubespolicy.policycreateconfirmation # pylint: enable=wrong-import-position -class PolicyAgent(object): +class PolicyAgent: # pylint: disable=too-few-public-methods dbus = """ diff --git a/qubespolicy/cli.py b/qubespolicy/cli.py index da1d02a7..0be0a4e7 100644 --- a/qubespolicy/cli.py +++ b/qubespolicy/cli.py @@ -72,12 +72,12 @@ def main(args=None): if not log.handlers: handler = logging.handlers.SysLogHandler(address='/dev/log') log.addHandler(handler) - log_prefix = 'qrexec: {}: {} -> {}: '.format( + log_prefix = 'qrexec: {}: {} -> {}:'.format( args.service_name, args.domain, args.target) try: system_info = qubespolicy.get_system_info() except qubespolicy.QubesMgmtException as e: - log.error(log_prefix + 'error getting system info: ' + str(e)) + log.error('%s error getting system info: %s', log_prefix, str(e)) return 1 try: try: @@ -130,13 +130,13 @@ def main(args=None): action.handle_user_response(True, response) else: action.handle_user_response(False) - log.info(log_prefix + 'allowed to {}'.format(action.target)) + log.info('%s allowed to %s', log_prefix, str(action.target)) action.execute(caller_ident) except qubespolicy.PolicySyntaxError as e: - log.error(log_prefix + 'error loading policy: ' + str(e)) + log.error('%s error loading policy: %s', log_prefix, str(e)) return 1 except qubespolicy.AccessDenied as e: - log.info(log_prefix + 'denied: ' + str(e)) + log.info('%s denied: %s', log_prefix, str(e)) return 1 return 0 diff --git a/qubespolicy/policycreateconfirmation.py b/qubespolicy/policycreateconfirmation.py index 248bad15..3aeec822 100644 --- a/qubespolicy/policycreateconfirmation.py +++ b/qubespolicy/policycreateconfirmation.py @@ -28,7 +28,7 @@ gi.require_version('Gtk', '3.0') from gi.repository import Gtk # pylint: enable=import-error -class PolicyCreateConfirmationWindow(object): +class PolicyCreateConfirmationWindow: # pylint: disable=too-few-public-methods _source_file = pkg_resources.resource_filename('qubespolicy', os.path.join('glade', "PolicyCreateConfirmationWindow.glade")) diff --git a/qvm-tools/qvm-sync-clock b/qvm-tools/qvm-sync-clock index 0e607f19..f2c6ecb3 100755 --- a/qvm-tools/qvm-sync-clock +++ b/qvm-tools/qvm-sync-clock @@ -32,6 +32,11 @@ def main(): app = Qubes() clockvm = app.clockvm + if not clockvm.is_running(): + sys.stderr.write('ClockVM {} is not running, aborting.\n'.format( + clockvm.name)) + sys.exit(0) + p = clockvm.run_service('qubes.GetDate') untrusted_date_out = p.stdout.read(25).decode('ascii', errors='strict') untrusted_date_out = untrusted_date_out.strip() @@ -42,7 +47,7 @@ def main(): date_out = untrusted_date_out subprocess.check_call(['date', '-u', '-Iseconds', '-s', date_out], stdout=subprocess.DEVNULL) - subprocess.check_call(['hwclock', '--systohc'], + subprocess.check_call(['/sbin/hwclock', '--systohc'], stdout=subprocess.DEVNULL) if __name__ == '__main__': diff --git a/rpm_spec/core-dom0.spec.in b/rpm_spec/core-dom0.spec.in index 2e3a5f54..34705ca8 100644 --- a/rpm_spec/core-dom0.spec.in +++ b/rpm_spec/core-dom0.spec.in @@ -122,6 +122,7 @@ make -C doc PYTHON=%{__python3} SPHINXBUILD=sphinx-build-%{python3_version} man make install \ DESTDIR=$RPM_BUILD_ROOT \ + BACKEND_VMM=%{backend_vmm} \ UNITDIR=%{_unitdir} \ PYTHON_SITEPATH=%{python3_sitelib} \ SYSCONFDIR=%{_sysconfdir} @@ -284,6 +285,7 @@ fi %{python3_sitelib}/qubes/ext/qubesmanager.py %{python3_sitelib}/qubes/ext/r3compatibility.py %{python3_sitelib}/qubes/ext/services.py +%{python3_sitelib}/qubes/ext/windows.py %dir %{python3_sitelib}/qubes/tests %dir %{python3_sitelib}/qubes/tests/__pycache__ @@ -304,6 +306,7 @@ fi %{python3_sitelib}/qubes/tests/init.py %{python3_sitelib}/qubes/tests/storage.py %{python3_sitelib}/qubes/tests/storage_file.py +%{python3_sitelib}/qubes/tests/storage_reflink.py %{python3_sitelib}/qubes/tests/storage_kernels.py %{python3_sitelib}/qubes/tests/storage_lvm.py %{python3_sitelib}/qubes/tests/tarwriter.py @@ -336,9 +339,11 @@ fi %{python3_sitelib}/qubes/tests/integ/backup.py %{python3_sitelib}/qubes/tests/integ/backupcompatibility.py %{python3_sitelib}/qubes/tests/integ/basic.py +%{python3_sitelib}/qubes/tests/integ/devices_block.py %{python3_sitelib}/qubes/tests/integ/devices_pci.py %{python3_sitelib}/qubes/tests/integ/dispvm.py %{python3_sitelib}/qubes/tests/integ/dom0_update.py +%{python3_sitelib}/qubes/tests/integ/mime.py %{python3_sitelib}/qubes/tests/integ/network.py %{python3_sitelib}/qubes/tests/integ/pvgrub.py %{python3_sitelib}/qubes/tests/integ/salt.py diff --git a/setup.py b/setup.py index 1f3ae9ce..d2110ac2 100644 --- a/setup.py +++ b/setup.py @@ -74,6 +74,7 @@ if __name__ == '__main__': 'qubes.ext.pci = qubes.ext.pci:PCIDeviceExtension', 'qubes.ext.block = qubes.ext.block:BlockDeviceExtension', 'qubes.ext.services = qubes.ext.services:ServicesExtension', + 'qubes.ext.windows = qubes.ext.windows:WindowsFeatures', ], 'qubes.devices': [ 'pci = qubes.ext.pci:PCIDevice', diff --git a/tests/Makefile b/tests/Makefile deleted file mode 100644 index 8b68f0b2..00000000 --- a/tests/Makefile +++ /dev/null @@ -1,49 +0,0 @@ -PYTHON_TESTSPATH = $(PYTHON_SITEPATH)/qubes/tests - -all: - python -m compileall . - python -O -m compileall . - -install: -ifndef PYTHON_SITEPATH - $(error PYTHON_SITEPATH not defined) -endif - mkdir -p $(DESTDIR)$(PYTHON_TESTSPATH) - cp __init__.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp __init__.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp backup.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp backup.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp backupcompatibility.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp backupcompatibility.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp basic.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp basic.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp block.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp block.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp dispvm.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp dispvm.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp dom0_update.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp dom0_update.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp extra.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp extra.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp hardware.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp hardware.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp hvm.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp hvm.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp mime.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp mime.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp network.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp network.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp pvgrub.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp pvgrub.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp regressions.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp regressions.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp run.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp run.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp storage.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp storage.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp storage_file.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp storage_file.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp storage_xen.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp storage_xen.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp vm_qrexec_gui.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp vm_qrexec_gui.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) diff --git a/tests/hardware.py b/tests/hardware.py deleted file mode 100644 index 619267f5..00000000 --- a/tests/hardware.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/python2 -# -*- coding: utf-8 -*- -# -# The Qubes OS Project, http://www.qubes-os.org -# -# Copyright (C) 2016 Marek Marczykowski-Górecki -# -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, see . -# -# -import os - -import qubes.tests -import time -import subprocess -from unittest import expectedFailure - - -class TC_00_HVM(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): - def setUp(self): - super(TC_00_HVM, self).setUp() - self.vm = self.qc.add_new_vm("QubesHVm", - name=self.make_vm_name('vm1')) - self.vm.create_on_disk(verbose=False) - - @expectedFailure - def test_000_pci_passthrough_presence(self): - pcidev = os.environ.get('QUBES_TEST_PCIDEV', None) - if pcidev is None: - self.skipTest('Specify PCI device with QUBES_TEST_PCIDEV ' - 'environment variable') - self.vm.pcidevs = [pcidev] - self.vm.pci_strictreset = False - self.qc.save() - self.qc.unlock_db() - - init_script = ( - "#!/bin/sh\n" - "set -e\n" - "lspci -n > /dev/xvdb\n" - "poweroff\n" - ) - - self.prepare_hvm_system_linux(self.vm, init_script, - ['/usr/sbin/lspci']) - self.vm.start() - timeout = 60 - while timeout > 0: - if not self.vm.is_running(): - break - time.sleep(1) - timeout -= 1 - if self.vm.is_running(): - self.fail("Timeout while waiting for VM shutdown") - - with open(self.vm.storage.private_img, 'r') as f: - lspci_vm = f.read(512).strip('\0') - p = subprocess.Popen(['lspci', '-ns', pcidev], stdout=subprocess.PIPE) - (lspci_host, _) = p.communicate() - # strip BDF, as it is different in VM - pcidev_desc = ' '.join(lspci_host.strip().split(' ')[1:]) - self.assertIn(pcidev_desc, lspci_vm) diff --git a/tests/hvm.py b/tests/hvm.py deleted file mode 100644 index f4c42b7b..00000000 --- a/tests/hvm.py +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# -# The Qubes OS Project, http://www.qubes-os.org -# -# Copyright (C) 2016 Marek Marczykowski-Górecki -# -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, see . -# -# - -import qubes.tests -from qubes.qubes import QubesException - -class TC_10_HVM(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): - # TODO: test with some OS inside - # TODO: windows tools tests - - def test_000_create_start(self): - testvm1 = self.qc.add_new_vm("QubesHVm", - name=self.make_vm_name('vm1')) - testvm1.create_on_disk(verbose=False) - self.qc.save() - self.qc.unlock_db() - testvm1.start() - self.assertEquals(testvm1.get_power_state(), "Running") - - def test_010_create_start_template(self): - templatevm = self.qc.add_new_vm("QubesTemplateHVm", - name=self.make_vm_name('template')) - templatevm.create_on_disk(verbose=False) - self.qc.save() - self.qc.unlock_db() - - templatevm.start() - self.assertEquals(templatevm.get_power_state(), "Running") - - def test_020_create_start_template_vm(self): - templatevm = self.qc.add_new_vm("QubesTemplateHVm", - name=self.make_vm_name('template')) - templatevm.create_on_disk(verbose=False) - testvm2 = self.qc.add_new_vm("QubesHVm", - name=self.make_vm_name('vm2'), - template=templatevm) - testvm2.create_on_disk(verbose=False) - self.qc.save() - self.qc.unlock_db() - - testvm2.start() - self.assertEquals(testvm2.get_power_state(), "Running") - - def test_030_prevent_simultaneus_start(self): - templatevm = self.qc.add_new_vm("QubesTemplateHVm", - name=self.make_vm_name('template')) - templatevm.create_on_disk(verbose=False) - testvm2 = self.qc.add_new_vm("QubesHVm", - name=self.make_vm_name('vm2'), - template=templatevm) - testvm2.create_on_disk(verbose=False) - self.qc.save() - self.qc.unlock_db() - - templatevm.start() - self.assertEquals(templatevm.get_power_state(), "Running") - self.assertRaises(QubesException, testvm2.start) - templatevm.force_shutdown() - testvm2.start() - self.assertEquals(testvm2.get_power_state(), "Running") - self.assertRaises(QubesException, templatevm.start) - - def test_100_resize_root_img(self): - testvm1 = self.qc.add_new_vm("QubesHVm", - name=self.make_vm_name('vm1')) - testvm1.create_on_disk(verbose=False) - self.qc.save() - self.qc.unlock_db() - testvm1.resize_root_img(30*1024**3) - self.assertEquals(testvm1.get_root_img_sz(), 30*1024**3) - testvm1.start() - self.assertEquals(testvm1.get_power_state(), "Running") - # TODO: launch some OS there and check the size - - def test_200_start_invalid_drive(self): - """Regression test for #1619""" - testvm1 = self.qc.add_new_vm("QubesHVm", - name=self.make_vm_name('vm1')) - testvm1.create_on_disk(verbose=False) - testvm1.drive = 'hd:dom0:/invalid' - self.qc.save() - self.qc.unlock_db() - try: - testvm1.start() - except Exception as e: - self.assertIsInstance(e, QubesException) - else: - self.fail('No exception raised') - - def test_201_start_invalid_drive_cdrom(self): - """Regression test for #1619""" - testvm1 = self.qc.add_new_vm("QubesHVm", - name=self.make_vm_name('vm1')) - testvm1.create_on_disk(verbose=False) - testvm1.drive = 'cdrom:dom0:/invalid' - self.qc.save() - self.qc.unlock_db() - try: - testvm1.start() - except Exception as e: - self.assertIsInstance(e, QubesException) - else: - self.fail('No exception raised') - diff --git a/version b/version index f5cd8d2e..a6e85764 100644 --- a/version +++ b/version @@ -1 +1 @@ -4.0.27 +4.0.33