Merge branch 'master' into devel-no-assert

Makefile

@@ -141,7 +141,6 @@ rpms-dom0:
 all:
 	$(PYTHON) setup.py build
 	$(MAKE) -C qubes-rpc all
-#	make all -C tests
 	# Currently supported only on xen
 
 install:
@@ -158,7 +157,6 @@ endif
 	ln -s qvm-device.1.gz $(DESTDIR)/usr/share/man/man1/qvm-block.1.gz
 	ln -s qvm-device.1.gz $(DESTDIR)/usr/share/man/man1/qvm-pci.1.gz
 	ln -s qvm-device.1.gz $(DESTDIR)/usr/share/man/man1/qvm-usb.1.gz
-#	$(MAKE) install -C tests
 	$(MAKE) install -C relaxng
 	mkdir -p $(DESTDIR)/etc/qubes
 ifeq ($(BACKEND_VMM),xen)

ci/pylintrc

@@ -6,6 +6,8 @@ ignore=tests
 # abstract-class-little-used: see http://www.logilab.org/ticket/111138
 # deprecated-method:
 #   enable again after disabling py-3.4.3 asyncio.ensure_future compat hack
+# not-an-iterable:
+#   a lot of false possitives for asyncio (yield from (some coroutine))
 disable=
   abstract-class-little-used,
   bad-continuation,
@@ -18,6 +20,7 @@ disable=
   locally-enabled,
   logging-format-interpolation,
   missing-docstring,
+  not-an-iterable,
   star-args,
   wrong-import-order
 

doc/index.rst

@@ -16,6 +16,7 @@ manpages and API documentation. For primary user documentation, see
    qubes
    qubes-vm/index
    qubes-events
+   qubes-features
    qubes-storage
    qubes-exc
    qubes-ext

doc/libvirt.rst

@@ -22,14 +22,14 @@ The distributor may put a file at
 :file:`/usr/share/qubes/template/xen-dist.xml`) to override this file. 
 User may put a file at either
 :file:`/etc/qubes/templates/libvirt/xen-user.xml` or
-:file:`/etc/qubes/templates/libvirt/by-name/<name>.xml`, where ``<name>`` is
+:file:`/etc/qubes/templates/libvirt/xen/by-name/<name>.xml`, where ``<name>`` is
 full name of the domain. Wildcards are not supported but symlinks are.
 
 Jinja has a concept of template names, which basically is the path below some
 load point, which in Qubes' case is :file:`/etc/qubes/templates` and
 :file:`/usr/share/qubes/templates`. Thus names of those templates are
 respectively ``'libvirt/xen.xml'``, ``'libvirt/xen-dist.xml'``,
-``'libvirt/xen-user.xml'`` and ``'libvirt/by-name/<name>.xml'``.
+``'libvirt/xen-user.xml'`` and ``'libvirt/xen/by-name/<name>.xml'``.
 This will be important later.
 
 .. note::
@@ -95,6 +95,9 @@ basic
    Contains ``<name>``, ``<uuid>``, ``<memory>``, ``<currentMemory>`` and
    ``<vcpu>`` nodes.
 
+cpu
+   ``<cpu>`` node.
+
 os
    Contents of ``<os>`` node.
 

doc/qubes-features.rst

@@ -0,0 +1,193 @@
+:py:class:`qubes.vm.Features` - Qubes VM features, services
+============================================================
+
+Features are generic mechanism for storing key-value pairs attached to a
+VM. The primary use case for them is data storage for extensions (you can think
+of them as more flexible properties, defined by extensions), but some are also
+used in the qubes core itself. There is no definite list of supported features,
+each extension can set their own and there is no requirement of registration,
+but :program:`qvm-features` man page contains well known ones.
+In addition, there is a mechanism for VM request setting a feature. This is
+useful for extensions to discover if its VM part is present. 
+
+Features can have three distinct values: no value (not present in mapping,
+which is closest thing to :py:obj:`None`), empty string (which is
+interpreted as :py:obj:`False`) and non-empty string, which is
+:py:obj:`True`. Anything assigned to the mapping is coerced to strings,
+however if you assign instances of :py:class:`bool`, they are converted as
+described above. Be aware that assigning the number `0` (which is considered
+false in Python) will result in string `'0'`, which is considered true.
+
+:py:class:`qubes.vm.Features` inherits from :py:class:`dict`, so provide all the
+standard functions to get, list and set values.  Additionally provide helper
+functions to check if given feature is set on the VM and default to the value
+on the VM's template or netvm. This is useful for features which nature is
+inherited from other VMs, like "is package X is installed" or "is VM behind a
+VPN".
+
+Example usage of features in extension:
+
+.. code-block:: python
+
+   import qubes.exc
+   import qubes.ext
+   
+   class ExampleExtension(qubes.ext.Extension):
+      @qubes.ext.handler('domain-pre-start')
+      def on_domain_start(self, vm, event, **kwargs):
+         if vm.features.get('do-not-start', False):
+            raise qubes.exc.QubesVMError(vm, 
+               'Start prohibited because of do-not-start feature')
+
+         if vm.features.check_with_template('something-installed', False):
+            # do something
+
+The above extension does two things:
+
+ - prevent starting a qube with ``do-not-start`` feature set
+ - do something when ``something-installed`` feature is set on the qube, or its template
+
+
+qvm-features-request, qubes.PostInstall service
+------------------------------------------------
+
+When some package in the VM want to request feature to be set (aka advertise
+support for it), it should place a shell script in ``/etc/qubes/post-install.d``.
+This script should call :program:`qvm-features-request` with ``FEATURE=VALUE`` pair(s) as
+arguments to request those features. It is recommended to use very simple
+values here (for example ``1``). The script should be named in form
+``XX-package-name.sh`` where ``XX`` is two-digits number below 90 and
+``package-name`` is unique name specific to this package (preferably actual
+package name). The script needs executable bit set.
+
+``qubes.PostInstall`` service will call all those scripts after any package
+installation and also after initial template installation.
+This way package have a chance to report to dom0 if any feature is
+added/removed.
+
+The features flow to dom0 according to the diagram below. Important part is
+that qubes core :py:class:`qubes.ext.Extension` is responsible for handling such request in
+``features-request`` event handler. If no extension handles given feature request,
+it will be ignored. The extension should carefuly validate requested
+features (ignoring those not recognized - may be for another extension) and
+only then set appropriate value on VM object
+(:py:attr:`qubes.vm.BaseVM.features`). It is recommended to make the
+verification code as bulletproof  as possible (for example allow only specific
+simple values, instead of complex structures), because feature requests
+come from untrusted sources. The features actually set on the VM in some cases
+may not be necessary those requested. Similar for values.
+
+.. graphviz::
+
+   digraph {
+
+      "qubes.PostInstall";
+      "/etc/qubes/post-install.d/ scripts";
+      "qvm-features-request";
+      "qubes.FeaturesRequest";
+      "qubes core extensions";
+      "VM features";
+
+      "qubes.PostInstall" -> "/etc/qubes/post-install.d/ scripts";
+      "/etc/qubes/post-install.d/ scripts" -> "qvm-features-request" 
+         [xlabel="each script calls"];
+      "qvm-features-request" -> "qubes.FeaturesRequest" 
+         [xlabel="last script call the service to dom0"];
+      "qubes.FeaturesRequest" -> "qubes core extensions" 
+         [xlabel="features-request event"];
+      "qubes core extensions" -> "VM features" 
+         [xlabel="verification"];
+
+   }
+
+Example ``/etc/qubes/post-install.d/20-example.sh`` file:
+
+.. code-block:: shell
+
+   #!/bin/sh
+
+   qvm-features-request example-feature=1
+
+Example extension handling the above:
+
+.. code-block:: python
+
+   import qubes.ext
+
+   class ExampleExtension(qubes.ext.Extension):
+      # the last argument must be named untrusted_features
+      @qubes.ext.handler('features-request')
+      def on_features_request(self, vm, event, untrusted_features):
+         # don't allow TemplateBasedVMs to request the feature - should be
+         # requested by the template instead
+         if hasattr(vm, 'template'):
+            return
+
+         untrusted_value = untrusted_features.get('example-feature', None)
+         # check if feature is advertised and verify its value
+         if untrusted_value != '1':
+            return
+         value = untrusted_value
+
+         # and finally set the value
+         vm.features['example-feature'] = value
+
+Services
+---------
+
+`Qubes services <https://www.qubes-os.org/doc/qubes-service/>`_ are implemented
+as features with ``service.`` prefix. The
+:py:class:`qubes.ext.services.ServicesExtension` enumerate all the features
+in form of ``service.<service-name>`` prefix and write them to QubesDB as
+``/qubes-service/<service-name>`` and value either ``0`` or ``1``.
+VM startup scripts list those entries for for each with value of ``1``, create
+``/var/run/qubes-service/<service-name>`` file. Then, it can be conveniently
+used by other scripts to check whether dom0 wishes service to be enabled or
+disabled.
+
+VM package can advertise what services are supported. For that, it needs to
+request ``supported-service.<service-name>`` feature with value ``1`` according
+to description above. The :py:class:`qubes.ext.services.ServicesExtension` will
+handle such request and set this feature on VM object. ``supported-service.``
+features that stop being advertised with ``qvm-features-request`` call are
+removed. This way, it's enough to remove the file from
+``/etc/qubes/post-install.d`` (for example by uninstalling package providing
+the service) to tell dom0 the service is no longer supported. Services
+advertised by TemplateBasedVMs are currently ignored (related
+``supported-service.`` features are not set), but retrieving them may be added
+in the future. Applications checking for specific service support should use
+``vm.features.check_with_template('supported-service.<service-name>', False)``
+call on desired VM object. When enumerating all supported services, application
+should consider both the vm and its template (if any).
+
+Various tools will use this information to discover if given service is
+supported. The API does not enforce service being first advertised before being
+enabled (means: there can be service which is enabled, but without matching
+``supported-service.`` feature). The list of well known services is in
+:program:`qvm-service` man page.
+
+Example ``/etc/qubes/post-install.d/20-my-service.sh``:
+
+.. code-block:: shell
+
+   #!/bin/sh
+
+   qvm-features-request supported-service.my-service=1
+
+Services and features can be then inspected from dom0 using
+:program:`qvm-features` tool, for example:
+
+.. code-block:: shell
+
+   $ qvm-features my-qube
+   supported-service.my-service  1
+
+Module contents
+---------------
+
+.. autoclass:: qubes.vm.Features
+   :members:
+   :show-inheritance:
+
+.. vim: ts=3 sw=3 et
+

doc/qubes.rst

@@ -8,10 +8,20 @@ Because all objects in Qubes' world are interconnected, there is no possibility
 to instantiate them separately. They are all loaded together and contained in
 the one ``app`` object, an instance of :py:class:`qubes.Qubes` class.
 
+Loading
+^^^^^^^
+
+The objects may come to existence in two ways: by explicit instantiation or by
+loading from XML file.
+
 The loading from XML is done in stages, because Qubes domains are dependent on
 each other in what can be even a circular dependency. Therefore some properties
-(especcialy those that refer to another domains) are loaded later. Refer to
-:py:class:`qubes.Qubes` class documentation to get description of every stage.
+(especcialy those that refer to another domains) are loaded later.
+
+.. image:: loading.svg
+
+Refer to :py:class:`qubes.Qubes` class documentation to get description of every
+stage.
 
 
 Properties

linux/systemd/qubes-vm@.service

@@ -2,6 +2,7 @@
 Description=Start Qubes VM %i
 Before=systemd-user-sessions.service
 After=qubesd.service qubes-meminfo-writer-dom0.service
+ConditionKernelCommandLine=!qubes.skip_autostart
 
 [Service]
 Type=oneshot

linux/systemd/qubesd.service

@@ -1,5 +1,6 @@
 [Unit]
 Description=Qubes OS daemon
+Before=systemd-user-sessions.service
 
 [Service]
 Type=notify

qubes-rpc-policy/qubes.GetDate.policy

@@ -3,4 +3,5 @@
 
 ## Please use a single # to start your custom comments
 
+$tag:anon-vm	$anyvm	deny
 $anyvm	$anyvm	allow,target=dom0

qubes-rpc-policy/qubes.OpenInVM.policy

@@ -3,8 +3,5 @@
 
 ## Please use a single # to start your custom comments
 
-sys-whonix anon-whonix allow
-whonix-gw anon-whonix allow
-whonix-ws anon-whonix allow
 $anyvm	$dispvm	allow
 $anyvm	$anyvm	ask

qubes-rpc-policy/qubes.OpenURL.policy

@@ -3,8 +3,5 @@
 
 ## Please use a single # to start your custom comments
 
-sys-whonix anon-whonix allow
-whonix-gw anon-whonix allow
-whonix-ws anon-whonix allow
 $anyvm	$dispvm	allow
 $anyvm	$anyvm	ask

qubes-rpc-policy/qubes.UpdatesProxy.policy

@@ -3,6 +3,15 @@
 
 ## Please use a single # to start your custom comments
 
+# Upgrade all TemplateVMs through sys-whonix.
+#$type:TemplateVM $default allow,target=sys-whonix
+
+# Upgrade Whonix TemplateVMs through sys-whonix.
+$tag:whonix-updatevm $default allow,target=sys-whonix
+
+# Deny Whonix TemplateVMs using UpdatesProxy of any other VM.
+$tag:whonix-updatevm $anyvm deny
+
 # Default rule for all TemplateVMs - direct the connection to sys-net
 $type:TemplateVM $default allow,target=sys-net
 

qubes/__init__.py

@@ -42,7 +42,7 @@ __license__ = 'GPLv2 or later'
 __version__ = 'R3'
 
 
-class Label(object):
+class Label:
     '''Label definition for virtual machines
 
     Label specifies colour of the padlock displayed next to VM's name.
@@ -134,7 +134,7 @@ class Label(object):
             self.icon_dispvm) + ".png"
 
 
-class property(object):  # pylint: disable=redefined-builtin,invalid-name
+class property:  # pylint: disable=redefined-builtin,invalid-name
     '''Qubes property.
 
     This class holds one property that can be saved to and loaded from
@@ -350,11 +350,10 @@ class property(object):  # pylint: disable=redefined-builtin,invalid-name
                 raise qubes.exc.QubesValueError
             if self.type is bool:
                 return self.bool(None, None, untrusted_newvalue)
-            else:
-                try:
-                    return self.type(untrusted_newvalue)
-                except ValueError:
-                    raise qubes.exc.QubesValueError
+            try:
+                return self.type(untrusted_newvalue)
+            except ValueError:
+                raise qubes.exc.QubesValueError
         else:
             # 'str' or not specified type
             try:

qubes/api/__init__.py

@@ -97,7 +97,7 @@ def apply_filters(iterable, filters):
     return iterable
 
 
-class AbstractQubesAPI(object):
+class AbstractQubesAPI:
     '''Common code for Qubes Management Protocol handling
 
     Different interfaces can expose different API call sets, however they share

qubes/api/admin.py

@@ -44,7 +44,7 @@ import qubes.vm.adminvm
 import qubes.vm.qubesvm
 
 
-class QubesMgmtEventsDispatcher(object):
+class QubesMgmtEventsDispatcher:
     def __init__(self, filters, send_event):
         self.filters = filters
         self.send_event = send_event
@@ -335,7 +335,7 @@ class QubesAdminAPI(qubes.api.AbstractQubesAPI):
         volume = self.dest.volumes[self.arg]
         # properties defined in API
         volume_properties = [
-            'pool', 'vid', 'size', 'usage', 'rw', 'source',
+            'pool', 'vid', 'size', 'usage', 'rw', 'source', 'path',
             'save_on_stop', 'snap_on_start', 'revisions_to_keep']
 
         def _serialize(value):
@@ -479,7 +479,7 @@ class QubesAdminAPI(qubes.api.AbstractQubesAPI):
         if not self.dest.is_halted():
             raise qubes.exc.QubesVMNotHaltedError(self.dest)
 
-        path = self.dest.storage.import_data(self.arg)
+        path = yield from self.dest.storage.import_data(self.arg)
         self.enforce(' ' not in path)
         size = self.dest.volumes[self.arg].size
 
@@ -1101,7 +1101,7 @@ class QubesAdminAPI(qubes.api.AbstractQubesAPI):
         try:
             yield from self.dest.remove_from_disk()
         except:  # pylint: disable=bare-except
-            self.app.log.exception('Error wile removing VM \'%s\' files',
+            self.app.log.exception('Error while removing VM \'%s\' files',
                 self.dest.name)
 
         self.app.save()

qubes/api/internal.py

@@ -68,7 +68,8 @@ class QubesInternalAPI(qubes.api.AbstractQubesAPI):
         success = untrusted_payload == b'ok'
 
         try:
-            self.dest.storage.import_data_end(self.arg, success=success)
+            yield from self.dest.storage.import_data_end(self.arg,
+                success=success)
         except:
             self.dest.fire_event('domain-volume-import-end', volume=self.arg,
                 success=False)

qubes/api/misc.py

@@ -76,7 +76,8 @@ class QubesMiscAPI(qubes.api.AbstractQubesAPI):
 
         untrusted_features = {}
         safe_set = string.ascii_letters + string.digits
-        expected_features = ('qrexec', 'gui', 'default-user')
+        expected_features = ('qrexec', 'gui', 'gui-emulated', 'default-user',
+            'os')
         for feature in expected_features:
             untrusted_value = self.src.untrusted_qdb.read(
                 '/qubes-tools/' + feature)

qubes/app.py

@@ -21,7 +21,7 @@
 #
 
 import collections
-import errno
+import copy
 import functools
 import grp
 import itertools
@@ -60,13 +60,14 @@ import qubes
 import qubes.ext
 import qubes.utils
 import qubes.storage
+import qubes.storage.reflink
 import qubes.vm
 import qubes.vm.adminvm
 import qubes.vm.qubesvm
 import qubes.vm.templatevm
 # pylint: enable=wrong-import-position
 
-class VirDomainWrapper(object):
+class VirDomainWrapper:
     # pylint: disable=too-few-public-methods
 
     def __init__(self, connection, vm):
@@ -97,7 +98,7 @@ class VirDomainWrapper(object):
         return wrapper
 
 
-class VirConnectWrapper(object):
+class VirConnectWrapper:
     # pylint: disable=too-few-public-methods
 
     def __init__(self, uri):
@@ -134,7 +135,7 @@ class VirConnectWrapper(object):
         return wrapper
 
 
-class VMMConnection(object):
+class VMMConnection:
     '''Connection to Virtual Machine Manager (libvirt)'''
 
     def __init__(self, offline_mode=None):
@@ -229,7 +230,7 @@ class VMMConnection(object):
         self._xc = None  # and pray it will get garbage-collected
 
 
-class QubesHost(object):
+class QubesHost:
     '''Basic information about host machine
 
     :param qubes.Qubes app: Qubes application context (must have \
@@ -363,7 +364,7 @@ class QubesHost(object):
         return (current_time, current)
 
 
-class VMCollection(object):
+class VMCollection:
     '''A collection of Qubes VMs
 
     VMCollection supports ``in`` operator. You may test for ``qid``, ``name``
@@ -493,7 +494,7 @@ class VMCollection(object):
         self.app.fire_event('domain-delete', vm=vm)
 
     def __contains__(self, key):
-        return any((key == vm or key == vm.qid or key == vm.name)
+        return any((key in (vm, vm.qid, vm.name))
                    for vm in self)
 
 
@@ -552,37 +553,37 @@ def _default_pool(app):
 
     1. If there is one named 'default', use it.
     2. Check if root fs is on LVM thin - use that
-    3. Look for file-based pool pointing /var/lib/qubes
+    3. Look for file(-reflink)-based pool pointing to /var/lib/qubes
     4. Fail
     '''
     if 'default' in app.pools:
         return app.pools['default']
-    else:
-        if 'DEFAULT_LVM_POOL' in os.environ:
-            thin_pool = os.environ['DEFAULT_LVM_POOL']
-            for pool in app.pools.values():
-                if pool.config.get('driver', None) != 'lvm_thin':
-                    continue
-                if pool.config['thin_pool'] == thin_pool:
-                    return pool
-        # no DEFAULT_LVM_POOL, or pool not defined
-        root_volume_group, root_thin_pool = \
-            qubes.storage.DirectoryThinPool.thin_pool('/')
-        if root_thin_pool:
-            for pool in app.pools.values():
-                if pool.config.get('driver', None) != 'lvm_thin':
-                    continue
-                if (pool.config['volume_group'] == root_volume_group and
-                    pool.config['thin_pool'] == root_thin_pool):
-                    return pool
 
-        # not a thin volume? look for file pools
+    if 'DEFAULT_LVM_POOL' in os.environ:
+        thin_pool = os.environ['DEFAULT_LVM_POOL']
+        for pool in app.pools.values():
+            if pool.config.get('driver', None) != 'lvm_thin':
+                continue
+            if pool.config['thin_pool'] == thin_pool:
+                return pool
+    # no DEFAULT_LVM_POOL, or pool not defined
+    root_volume_group, root_thin_pool = \
+        qubes.storage.DirectoryThinPool.thin_pool('/')
+    if root_thin_pool:
         for pool in app.pools.values():
-            if pool.config.get('driver', None) not in ('file', 'file-reflink'):
+            if pool.config.get('driver', None) != 'lvm_thin':
                 continue
-            if pool.config['dir_path'] == qubes.config.qubes_base_dir:
+            if (pool.config['volume_group'] == root_volume_group and
+                pool.config['thin_pool'] == root_thin_pool):
                 return pool
-        raise AttributeError('Cannot determine default storage pool')
+
+    # not a thin volume? look for file pools
+    for pool in app.pools.values():
+        if pool.config.get('driver', None) not in ('file', 'file-reflink'):
+            continue
+        if pool.config['dir_path'] == qubes.config.qubes_base_dir:
+            return pool
+    raise AttributeError('Cannot determine default storage pool')
 
 def _setter_pool(app, prop, value):
     if isinstance(value, qubes.storage.Pool):
@@ -717,6 +718,19 @@ class Qubes(qubes.PropertyHolder):
         setter=_setter_pool,
         doc='Default storage pool for kernel volumes')
 
+    default_qrexec_timeout = qubes.property('default_qrexec_timeout',
+        load_stage=3,
+        default=60,
+        type=int,
+        doc='''Default time in seconds after which qrexec connection attempt is
+            deemed failed''')
+
+    default_shutdown_timeout = qubes.property('default_shutdown_timeout',
+        load_stage=3,
+        default=60,
+        type=int,
+        doc='''Default time in seconds for VM shutdown to complete''')
+
     stats_interval = qubes.property('stats_interval',
         default=3,
         type=int,
@@ -1006,8 +1020,8 @@ class Qubes(qubes.PropertyHolder):
             try:
                 fd = os.open(self._store,
                     os.O_RDWR | (os.O_CREAT * int(for_save)))
-            except OSError as e:
-                if not for_save and e.errno == errno.ENOENT:
+            except FileNotFoundError:
+                if not for_save:
                     raise qubes.exc.QubesException(
                         'Qubes XML store {!r} is missing; '
                         'use qubes-create tool'.format(self._store))
@@ -1064,15 +1078,29 @@ class Qubes(qubes.PropertyHolder):
         }
         assert max(self.labels.keys()) == qubes.config.max_default_label
 
+        pool_configs = copy.deepcopy(qubes.config.defaults['pool_configs'])
+
         root_volume_group, root_thin_pool = \
             qubes.storage.DirectoryThinPool.thin_pool('/')
-
         if root_thin_pool:
-            self.add_pool(
-                volume_group=root_volume_group, thin_pool=root_thin_pool,
-                name='lvm', driver='lvm_thin')
-        # pool based on /var/lib/qubes will be created here:
-        for name, config in qubes.config.defaults['pool_configs'].items():
+            lvm_config = {
+                'name': 'lvm',
+                'driver': 'lvm_thin',
+                'volume_group': root_volume_group,
+                'thin_pool': root_thin_pool
+            }
+            pool_configs[lvm_config['name']] = lvm_config
+
+        for name, config in pool_configs.items():
+            if 'driver' not in config and 'dir_path' in config:
+                config['driver'] = 'file'
+                try:
+                    os.makedirs(config['dir_path'], exist_ok=True)
+                    if qubes.storage.reflink.is_supported(config['dir_path']):
+                        config['driver'] = 'file-reflink'
+                        config['setup_check'] = 'no'  # don't check twice
+                except PermissionError:  # looks like a testing environment
+                    pass  # stay with 'file'
             self.pools[name] = self._get_pool(**config)
 
         self.default_pool_kernel = 'linux-kernel'
@@ -1170,6 +1198,11 @@ class Qubes(qubes.PropertyHolder):
 
         raise KeyError(label)
 
+    def setup_pools(self):
+        """ Run implementation specific setup for each storage pool. """
+        for pool in self.pools.values():
+            pool.setup()
+
     def add_pool(self, name, **kwargs):
         """ Add a storage pool to config."""
 
@@ -1251,6 +1284,20 @@ class Qubes(qubes.PropertyHolder):
 
         if event == libvirt.VIR_DOMAIN_EVENT_STOPPED:
             vm.on_libvirt_domain_stopped()
+        elif event == libvirt.VIR_DOMAIN_EVENT_SUSPENDED:
+            try:
+                vm.fire_event('domain-paused')
+            except Exception:  # pylint: disable=broad-except
+                self.log.exception(
+                    'Uncaught exception from domain-paused handler '
+                    'for domain %s', vm.name)
+        elif event == libvirt.VIR_DOMAIN_EVENT_RESUMED:
+            try:
+                vm.fire_event('domain-unpaused')
+            except Exception:  # pylint: disable=broad-except
+                self.log.exception(
+                    'Uncaught exception from domain-unpaused handler '
+                    'for domain %s', vm.name)
 
     @qubes.events.handler('domain-pre-delete')
     def on_domain_pre_deleted(self, event, vm):
@@ -1263,9 +1310,9 @@ class Qubes(qubes.PropertyHolder):
                         self.log.error(
                             'Cannot remove %s, used by %s.%s',
                             vm, obj, prop.__name__)
-                        raise qubes.exc.QubesVMInUseError(vm,
-                            'Domain is in use: {!r}; details in system log'
-                                .format(vm.name))
+                        raise qubes.exc.QubesVMInUseError(vm, 'Domain is in '
+                        'use: {!r}; see /var/log/qubes/qubes.log in dom0 for '
+                        'details'.format(vm.name))
                 except AttributeError:
                     pass
 

qubes/backup.py

@@ -76,7 +76,7 @@ class BackupCanceledError(qubes.exc.QubesException):
         self.tmpdir = tmpdir
 
 
-class BackupHeader(object):
+class BackupHeader:
     '''Structure describing backup-header file included as the first file in
     backup archive
     '''
@@ -124,7 +124,7 @@ class BackupHeader(object):
                 f_header.write("{!s}={!s}\n".format(key, getattr(self, attr)))
 
 
-class SendWorker(object):
+class SendWorker:
     # pylint: disable=too-few-public-methods
     def __init__(self, queue, base_dir, backup_stdout):
         super(SendWorker, self).__init__()
@@ -148,6 +148,7 @@ class SendWorker(object):
             # verified before untaring.
             tar_final_cmd = ["tar", "-cO", "--posix",
                              "-C", self.base_dir, filename]
+            # pylint: disable=not-an-iterable
             final_proc = yield from asyncio.create_subprocess_exec(
                 *tar_final_cmd,
                 stdout=self.backup_stdout)
@@ -183,6 +184,7 @@ def launch_proc_with_pty(args, stdin=None, stdout=None, stderr=None, echo=True):
             termios_p[3] &= ~termios.ECHO
             termios.tcsetattr(ctty_fd, termios.TCSANOW, termios_p)
     (pty_master, pty_slave) = os.openpty()
+    # pylint: disable=not-an-iterable
     p = yield from asyncio.create_subprocess_exec(*args,
         stdin=stdin,
         stdout=stdout,
@@ -228,7 +230,7 @@ def launch_scrypt(action, input_name, output_name, passphrase):
     return p
 
 
-class Backup(object):
+class Backup:
     '''Backup operation manager. Usage:
 
     >>> app = qubes.Qubes()
@@ -251,7 +253,7 @@ class Backup(object):
 
     '''
     # pylint: disable=too-many-instance-attributes
-    class FileToBackup(object):
+    class FileToBackup:
         # pylint: disable=too-few-public-methods
         def __init__(self, file_path, subdir=None, name=None, size=None):
             if size is None:
@@ -278,7 +280,7 @@ class Backup(object):
             if name is not None:
                 self.name = name
 
-    class VMToBackup(object):
+    class VMToBackup:
         # pylint: disable=too-few-public-methods
         def __init__(self, vm, files, subdir):
             self.vm = vm
@@ -635,6 +637,7 @@ class Backup(object):
 
                 # Pipe: tar-sparse | scrypt | tar | backup_target
                 # TODO: log handle stderr
+                # pylint: disable=not-an-iterable
                 tar_sparse = yield from asyncio.create_subprocess_exec(
                     *tar_cmdline, stdout=subprocess.PIPE)
 

qubes/config.py

@@ -76,9 +76,8 @@ defaults = {
     'root_img_size': 10*1024*1024*1024,
 
     'pool_configs': {
-        # create file pool even when the default one is LVM
+        # create file(-reflink) pool even when the default one is LVM
         'varlibqubes': {'dir_path': qubes_base_dir,
-                    'driver': 'file',
                     'name': 'varlibqubes'},
         'linux-kernel': {
             'dir_path': os.path.join(qubes_base_dir,

qubes/devices.py

@@ -67,7 +67,7 @@ class DeviceAlreadyAttached(qubes.exc.QubesException, KeyError):
     '''Trying to attach already attached device'''
     pass
 
-class DeviceInfo(object):
+class DeviceInfo:
     ''' Holds all information about a device '''
     # pylint: disable=too-few-public-methods
     def __init__(self, backend_domain, ident, description=None,
@@ -117,7 +117,7 @@ class DeviceInfo(object):
         return '{!s}:{!s}'.format(self.backend_domain, self.ident)
 
 
-class DeviceAssignment(object): # pylint: disable=too-few-public-methods
+class DeviceAssignment: # pylint: disable=too-few-public-methods
     ''' Maps a device to a frontend_domain. '''
 
     def __init__(self, backend_domain, ident, options=None, persistent=False,
@@ -158,7 +158,7 @@ class DeviceAssignment(object): # pylint: disable=too-few-public-methods
         return self.backend_domain.devices[self.bus][self.ident]
 
 
-class DeviceCollection(object):
+class DeviceCollection:
     '''Bag for devices.
 
     Used as default value for :py:meth:`DeviceManager.__missing__` factory.
@@ -168,13 +168,14 @@ class DeviceCollection(object):
 
     This class emits following events on VM object:
 
-        .. event:: device-attach:<class> (device)
+        .. event:: device-attach:<class> (device, options)
 
             Fired when device is attached to a VM.
 
             Handler for this event can be asynchronous (a coroutine).
 
             :param device: :py:class:`DeviceInfo` object to be attached
+            :param options: :py:class:`dict` of attachment options
 
         .. event:: device-pre-attach:<class> (device)
 
@@ -357,8 +358,7 @@ class DeviceCollection(object):
             if persistent is True:
                 # don't break app.save()
                 return self._set
-            else:
-                raise
+            raise
         result = set()
         for dev, options in devices:
             if dev in self._set and not persistent:
@@ -433,7 +433,7 @@ class UnknownDevice(DeviceInfo):
             frontend_domain)
 
 
-class PersistentCollection(object):
+class PersistentCollection:
 
     ''' Helper object managing persistent `DeviceAssignment`s.
     '''

qubes/dochelpers.py

@@ -48,7 +48,7 @@ SUBCOMMANDS_TITLE = 'COMMANDS'
 OPTIONS_TITLE = 'OPTIONS'
 
 
-class GithubTicket(object):
+class GithubTicket:
     # pylint: disable=too-few-public-methods
     def __init__(self, data):
         self.number = data['number']
@@ -418,7 +418,7 @@ def parse_event(env, sig, signode):
 #
 
 
-def break_to_pdb(app, *dummy):
+def break_to_pdb(app, *_dummy):
     if not app.config.break_to_pdb:
         return
     import pdb

qubes/events.py

@@ -94,7 +94,7 @@ class EmitterMeta(type):
                 cls.__handlers__[event].add(attr)
 
 
-class Emitter(object, metaclass=EmitterMeta):
+class Emitter(metaclass=EmitterMeta):
     '''Subject that can emit events.
 
     By default all events are disabled not to interfere with loading from XML.

qubes/exc.py

@@ -101,6 +101,14 @@ class QubesVMNotHaltedError(QubesVMError):
         super(QubesVMNotHaltedError, self).__init__(vm,
             msg or 'Domain is not powered off: {!r}'.format(vm.name))
 
+class QubesVMShutdownTimeoutError(QubesVMError):
+    '''Domain shutdown timed out.
+
+    '''
+    def __init__(self, vm, msg=None):
+        super(QubesVMShutdownTimeoutError, self).__init__(vm,
+            msg or 'Domain shutdown timed out: {!r}'.format(vm.name))
+
 
 class QubesNoTemplateError(QubesVMError):
     '''Cannot start domain, because there is no template'''
@@ -151,7 +159,7 @@ class BackupCancelledError(QubesException):
             msg or 'Backup cancelled')
 
 
-class QubesMemoryError(QubesException, MemoryError):
+class QubesMemoryError(QubesVMError, MemoryError):
     '''Cannot start domain, because not enough memory is available'''
     def __init__(self, vm, msg=None):
         super(QubesMemoryError, self).__init__(

qubes/ext/__init__.py

@@ -29,7 +29,7 @@ import pkg_resources
 import qubes.events
 
 
-class Extension(object):
+class Extension:
     '''Base class for all extensions
     '''  # pylint: disable=too-few-public-methods
 

qubes/ext/core_features.py

@@ -32,7 +32,7 @@ class CoreFeatures(qubes.ext.Extension):
             return
 
         requested_features = {}
-        for feature in ('qrexec', 'gui', 'qubes-firewall'):
+        for feature in ('qrexec', 'gui', 'gui-emulated', 'qubes-firewall'):
             untrusted_value = untrusted_features.get(feature, None)
             if untrusted_value in ('1', '0'):
                 requested_features[feature] = bool(int(untrusted_value))
@@ -44,7 +44,7 @@ class CoreFeatures(qubes.ext.Extension):
         # gui agent presence (0 or 1)
 
         qrexec_before = vm.features.get('qrexec', False)
-        for feature in ('qrexec', 'gui'):
+        for feature in ('qrexec', 'gui', 'gui-emulated'):
             # do not allow (Template)VM to override setting if already set
             # some other way
             if feature in requested_features and feature not in vm.features:

qubes/ext/pci.py

@@ -42,7 +42,8 @@ def load_pci_classes():
     #       subclass        subclass_name           <-- single tab
     #               prog-if  prog-if_name   <-- two tabs
     result = {}
-    with open('/usr/share/hwdata/pci.ids') as pciids:
+    with open('/usr/share/hwdata/pci.ids',
+            encoding='utf-8', errors='ignore') as pciids:
         class_id = None
         subclass_id = None
         for line in pciids.readlines():

qubes/ext/r3compatibility.py

@@ -80,6 +80,9 @@ class R3Compatibility(qubes.ext.Extension):
 
     def write_iptables_qubesdb_entry(self, firewallvm):
         # pylint: disable=no-self-use
+        # skip compatibility rules if new format support is advertised
+        if firewallvm.features.check_with_template('qubes-firewall', False):
+            return
         firewallvm.untrusted_qdb.rm("/qubes-iptables-domainrules/")
         iptables = "# Generated by Qubes Core on {0}\n".format(
             datetime.datetime.now().ctime())

qubes/ext/services.py

@@ -62,3 +62,40 @@ class ServicesExtension(qubes.ext.Extension):
             return
         service = feature[len('service.'):]
         vm.untrusted_qdb.rm('/qubes-service/{}'.format(service))
+
+    @qubes.ext.handler('features-request')
+    def supported_services(self, vm, event, untrusted_features):
+        '''Handle advertisement of supported services'''
+        # pylint: disable=no-self-use,unused-argument
+
+        if getattr(vm, 'template', None):
+            vm.log.warning(
+                'Ignoring qubes.FeaturesRequest from template-based VM')
+            return
+
+        new_supported_services = set()
+        for requested_service in untrusted_features:
+            if not requested_service.startswith('supported-service.'):
+                continue
+            if untrusted_features[requested_service] == '1':
+                # only allow to advertise service as supported, lack of entry
+                #  means service is not supported
+                new_supported_services.add(requested_service)
+        del untrusted_features
+
+        # if no service is supported, ignore the whole thing - do not clear
+        # all services in case of empty request (manual or such)
+        if not new_supported_services:
+            return
+
+        old_supported_services = set(
+            feat for feat in vm.features
+            if feat.startswith('supported-service.') and vm.features[feat])
+
+        for feature in new_supported_services.difference(
+                old_supported_services):
+            vm.features[feature] = True
+
+        for feature in old_supported_services.difference(
+                new_supported_services):
+            del vm.features[feature]

qubes/ext/windows.py

@@ -0,0 +1,73 @@
+# -*- encoding: utf-8 -*-
+#
+# The Qubes OS Project, http://www.qubes-os.org
+#
+# Copyright (C) 2017 Marek Marczykowski-Górecki
+#                               <marmarek@invisiblethingslab.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, see <http://www.gnu.org/licenses/>.
+import asyncio
+
+import qubes.ext
+
+class WindowsFeatures(qubes.ext.Extension):
+    # pylint: disable=too-few-public-methods
+    @qubes.ext.handler('features-request')
+    def qubes_features_request(self, vm, event, untrusted_features):
+        '''Handle features provided requested by Qubes Windows Tools'''
+        # pylint: disable=no-self-use,unused-argument
+        if getattr(vm, 'template', None):
+            vm.log.warning(
+                'Ignoring qubes.NotifyTools for template-based VM')
+            return
+
+        guest_os = None
+        if 'os' in untrusted_features:
+            if untrusted_features['os'] in ['Windows', 'Linux']:
+                guest_os = untrusted_features['os']
+
+        qrexec = None
+        if 'qrexec' in untrusted_features:
+            if untrusted_features['qrexec'] == '1':
+                # qrexec feature is set by CoreFeatures extension
+                qrexec = True
+
+        del untrusted_features
+
+        if guest_os:
+            vm.features['os'] = guest_os
+        if guest_os == 'Windows' and qrexec:
+            vm.features['rpc-clipboard'] = True
+
+    @qubes.ext.handler('domain-create-on-disk')
+    @asyncio.coroutine
+    def on_domain_create_on_disk(self, vm, _event, **kwargs):
+        # pylint: disable=no-self-use,unused-argument
+        if getattr(vm, 'template', None) is None:
+            # handle only template-based vms
+            return
+
+        template = vm.template
+        if template.features.check_with_template('os', None) != 'Windows':
+            # ignore non-windows templates
+            return
+
+        if vm.volumes['private'].save_on_stop:
+            # until windows tools get ability to prepare private.img on its own,
+            # copy one from the template
+            vm.log.info('Windows template - cloning private volume')
+            import_op = vm.volumes['private'].import_volume(
+                template.volumes['private'])
+            if asyncio.iscoroutine(import_op):
+                yield from import_op

qubes/firewall.py

@@ -34,7 +34,7 @@ import qubes
 import qubes.vm.qubesvm
 
 
-class RuleOption(object):
+class RuleOption:
     def __init__(self, untrusted_value):
         # subset of string.punctuation
         safe_set = string.ascii_letters + string.digits + \
@@ -209,7 +209,7 @@ class Expire(RuleOption):
 
     @property
     def rule(self):
-        return None
+        pass
 
     @property
     def api_rule(self):
@@ -234,7 +234,7 @@ class Comment(RuleOption):
 
     @property
     def rule(self):
-        return None
+        pass
 
     @property
     def api_rule(self):
@@ -451,7 +451,7 @@ class Rule(qubes.PropertyHolder):
         return hash(self.api_rule)
 
 
-class Firewall(object):
+class Firewall:
     def __init__(self, vm, load=True):
         assert hasattr(vm, 'firewall_conf')
         self.vm = vm

qubes/rngdoc.py

@@ -26,7 +26,7 @@ import textwrap
 
 import lxml.etree
 
-class Element(object):
+class Element:
     def __init__(self, schema, xml):
         self.schema = schema
         self.xml = xml
@@ -157,7 +157,7 @@ class Element(object):
             write_rst_table(stream, childtable, ('element', 'number'))
 
 
-class Schema(object):
+class Schema:
     # pylint: disable=too-few-public-methods
     nsmap = {
         'rng': 'http://relaxng.org/ns/structure/1.0',

qubes/storage/__init__.py

@@ -47,7 +47,7 @@ class StoragePoolException(qubes.exc.QubesException):
     pass
 
 
-class BlockDevice(object):
+class BlockDevice:
     ''' Represents a storage block device. '''
     # pylint: disable=too-few-public-methods
     def __init__(self, path, name, script=None, rw=True, domain=None,
@@ -62,7 +62,7 @@ class BlockDevice(object):
         self.devtype = devtype
 
 
-class Volume(object):
+class Volume:
     ''' Encapsulates all data about a volume for serialization to qubes.xml and
         libvirt config.
 
@@ -198,6 +198,8 @@ class Volume(object):
             volume data require something more than just writing to a file (
             for example connecting to some other domain, or converting data
             on the fly), the returned path may be a pipe.
+
+            This can be implemented as a coroutine.
         '''
         raise self._not_implemented("import")
 
@@ -207,6 +209,8 @@ class Volume(object):
 
         This method is called regardless the operation was successful or not.
 
+        This can be implemented as a coroutine.
+
         :param success: True if data import was successful, otherwise False
         '''
         # by default do nothing
@@ -252,6 +256,8 @@ class Volume(object):
     def revert(self, revision=None):
         ''' Revert volume to previous revision
 
+        This can be implemented as a coroutine.
+
         :param revision: revision to revert volume to, see :py:attr:`revisions`
         '''
         # pylint: disable=unused-argument
@@ -272,6 +278,7 @@ class Volume(object):
         This include committing data if :py:attr:`save_on_stop` is set.
 
         This can be implemented as a coroutine.'''
+        raise self._not_implemented("stop")
 
     def verify(self):
         ''' Verifies the volume.
@@ -334,14 +341,14 @@ class Volume(object):
         msg = msg.format(str(self.__class__.__name__), method_name)
         return NotImplementedError(msg)
 
-class Storage(object):
+class Storage:
     ''' Class for handling VM virtual disks.
 
     This is base class for all other implementations, mostly with Xen on Linux
     in mind.
     '''
 
-    AVAILABLE_FRONTENDS = set(['xvd' + c for c in string.ascii_lowercase])
+    AVAILABLE_FRONTENDS = {'xvd' + c for c in string.ascii_lowercase}
 
     def __init__(self, vm):
         #: Domain for which we manage storage
@@ -506,8 +513,7 @@ class Storage(object):
             ret = volume.create()
             if asyncio.iscoroutine(ret):
                 coros.append(ret)
-        if coros:
-            yield from asyncio.wait(coros)
+        yield from _wait_and_reraise(coros)
 
         os.umask(old_umask)
 
@@ -549,7 +555,7 @@ class Storage(object):
 
         self.vm.volumes = {}
         with VmCreationManager(self.vm):
-            yield from asyncio.wait([self.clone_volume(src_vm, vol_name)
+            yield from _wait_and_reraise([self.clone_volume(src_vm, vol_name)
                 for vol_name in self.vm.volume_config.keys()])
 
     @property
@@ -581,11 +587,7 @@ class Storage(object):
             ret = volume.verify()
             if asyncio.iscoroutine(ret):
                 futures.append(ret)
-        if futures:
-            done, _ = yield from asyncio.wait(futures)
-            for task in done:
-                # re-raise any exception from async task
-                task.result()
+        yield from _wait_and_reraise(futures)
         self.vm.fire_event('domain-verify-files')
         return True
 
@@ -605,44 +607,32 @@ class Storage(object):
             except (IOError, OSError) as e:
                 self.vm.log.exception("Failed to remove volume %s", name, e)
 
-        if futures:
-            try:
-                done, _ = yield from asyncio.wait(futures)
-                for task in done:
-                    # re-raise any exception from async task
-                    task.result()
-            except (IOError, OSError) as e:
-                self.vm.log.exception("Failed to remove some volume", e)
+        try:
+            yield from _wait_and_reraise(futures)
+        except (IOError, OSError) as e:
+            self.vm.log.exception("Failed to remove some volume", e)
 
     @asyncio.coroutine
     def start(self):
-        ''' Execute the start method on each pool '''
+        ''' Execute the start method on each volume '''
         futures = []
         for volume in self.vm.volumes.values():
             ret = volume.start()
             if asyncio.iscoroutine(ret):
                 futures.append(ret)
 
-        if futures:
-            done, _ = yield from asyncio.wait(futures)
-            for task in done:
-                # re-raise any exception from async task
-                task.result()
+        yield from _wait_and_reraise(futures)
 
     @asyncio.coroutine
     def stop(self):
-        ''' Execute the start method on each pool '''
+        ''' Execute the stop method on each volume '''
         futures = []
         for volume in self.vm.volumes.values():
             ret = volume.stop()
             if asyncio.iscoroutine(ret):
                 futures.append(ret)
 
-        if futures:
-            done, _ = yield from asyncio.wait(futures)
-            for task in done:
-                # re-raise any exception from async task
-                task.result()
+        yield from _wait_and_reraise(futures)
 
     def unused_frontend(self):
         ''' Find an unused device name '''
@@ -655,9 +645,9 @@ class Storage(object):
         ''' Used device names '''
         xml = self.vm.libvirt_domain.XMLDesc()
         parsed_xml = lxml.etree.fromstring(xml)
-        return set([target.get('dev', None)
+        return {target.get('dev', None)
                     for target in parsed_xml.xpath(
-                        "//domain/devices/disk/target")])
+                        "//domain/devices/disk/target")}
 
     def export(self, volume):
         ''' Helper function to export volume (pool.export(volume))'''
@@ -668,27 +658,37 @@ class Storage(object):
 
         return self.vm.volumes[volume].export()
 
+    @asyncio.coroutine
     def import_data(self, volume):
         ''' Helper function to import volume data (pool.import_data(volume))'''
         assert isinstance(volume, (Volume, str)), \
             "You need to pass a Volume or pool name as str"
         if isinstance(volume, Volume):
-            return volume.import_data()
+            ret = volume.import_data()
+        else:
+            ret = self.vm.volumes[volume].import_data()
 
-        return self.vm.volumes[volume].import_data()
+        if asyncio.iscoroutine(ret):
+            ret = yield from ret
+        return ret
 
+    @asyncio.coroutine
     def import_data_end(self, volume, success):
         ''' Helper function to finish/cleanup data import
         (pool.import_data_end( volume))'''
         assert isinstance(volume, (Volume, str)), \
             "You need to pass a Volume or pool name as str"
         if isinstance(volume, Volume):
-            return volume.import_data_end(success=success)
+            ret = volume.import_data_end(success=success)
+        else:
+            ret = self.vm.volumes[volume].import_data_end(success=success)
 
-        return self.vm.volumes[volume].import_data_end(success=success)
+        if asyncio.iscoroutine(ret):
+            ret = yield from ret
+        return ret
 
 
-class VolumesCollection(object):
+class VolumesCollection:
     '''Convenient collection wrapper for pool.get_volume and
     pool.list_volumes
     '''
@@ -706,8 +706,7 @@ class VolumesCollection(object):
         if isinstance(item, Volume):
             if item.pool == self._pool:
                 return self[item.vid]
-            else:
-                raise KeyError(item)
+            raise KeyError(item)
         try:
             return self._pool.get_volume(item)
         except NotImplementedError:
@@ -740,7 +739,7 @@ class VolumesCollection(object):
         return [vol for vol in self]
 
 
-class Pool(object):
+class Pool:
     ''' A Pool is used to manage different kind of volumes (File
         based/LVM/Btrfs/...).
 
@@ -760,7 +759,7 @@ class Pool(object):
     def __eq__(self, other):
         if isinstance(other, Pool):
             return self.name == other.name
-        elif isinstance(other, str):
+        if isinstance(other, str):
             return self.name == other
         return NotImplemented
 
@@ -829,12 +828,12 @@ class Pool(object):
     @property
     def size(self):
         ''' Storage pool size in bytes, or None if unknown '''
-        return None
+        pass
 
     @property
     def usage(self):
         ''' Space used in the pool in bytes, or None if unknown '''
-        return None
+        pass
 
     def _not_implemented(self, method_name):
         ''' Helper for emitting helpful `NotImplementedError` exceptions '''
@@ -843,6 +842,14 @@ class Pool(object):
         return NotImplementedError(msg)
 
 
+@asyncio.coroutine
+def _wait_and_reraise(futures):
+    if futures:
+        done, _ = yield from asyncio.wait(futures)
+        for task in done:  # (re-)raise first exception in line
+            task.result()
+
+
 def _sanitize_config(config):
     ''' Helper function to convert types to appropriate strings
     '''  # FIXME: find another solution for serializing basic types
@@ -872,7 +879,7 @@ def driver_parameters(name):
     return [p for p in params if p not in ignored_params]
 
 
-def isodate(seconds=time.time()):
+def isodate(seconds):
     ''' Helper method which returns an iso date '''
     return datetime.utcfromtimestamp(seconds).isoformat("T")
 
@@ -882,23 +889,27 @@ def search_pool_containing_dir(pools, dir_path):
     This is useful for implementing Pool.included_in method
     '''
 
+    real_dir_path = os.path.realpath(dir_path)
+
     # prefer filesystem pools
     for pool in pools:
         if hasattr(pool, 'dir_path'):
-            if dir_path.startswith(pool.dir_path):
+            pool_real_dir_path = os.path.realpath(pool.dir_path)
+            if os.path.commonpath([pool_real_dir_path, real_dir_path]) == \
+               pool_real_dir_path:
                 return pool
 
     # then look for lvm
     for pool in pools:
         if hasattr(pool, 'thin_pool') and hasattr(pool, 'volume_group'):
             if (pool.volume_group, pool.thin_pool) == \
-                    DirectoryThinPool.thin_pool(dir_path):
+                    DirectoryThinPool.thin_pool(real_dir_path):
                 return pool
 
     return None
 
 
-class VmCreationManager(object):
+class VmCreationManager:
     ''' A `ContextManager` which cleans up if volume creation fails.
     '''  # pylint: disable=too-few-public-methods
     def __init__(self, vm):

qubes/storage/file.py

@@ -356,9 +356,9 @@ class FileVolume(qubes.storage.Volume):
     def script(self):
         if not self.snap_on_start and not self.save_on_stop:
             return None
-        elif not self.snap_on_start and self.save_on_stop:
+        if not self.snap_on_start and self.save_on_stop:
             return 'block-origin'
-        elif self.snap_on_start:
+        if self.snap_on_start:
             return 'block-snapshot'
         return None
 

qubes/storage/lvm.py

@@ -18,9 +18,8 @@
 #
 
 ''' Driver for storing vm images in a LVM thin pool '''
-
+import functools
 import logging
-import operator
 import os
 import subprocess
 
@@ -44,8 +43,37 @@ def check_lvm_version():
 
 lvm_is_very_old = check_lvm_version()
 
+
 class ThinPool(qubes.storage.Pool):
     ''' LVM Thin based pool implementation
+
+    Volumes are stored as LVM thin volumes, in thin pool specified by
+    *volume_group*/*thin_pool* arguments. LVM volume naming scheme:
+
+        vm-{vm_name}-{volume_name}[-suffix]
+
+    Where suffix can be one of:
+        "-snap" - snapshot for currently running VM, at VM shutdown will be
+        either discarded (if save_on_stop=False), or committed
+        (if save_on_stop=True)
+        "-{revision_id}" - volume revision - new revision is automatically
+        created at each VM shutdown, *revisions_to_keep* control how many
+        old revisions (in addition to the current one) should be stored
+        "" (no suffix) - the most recent committed volume state; also volatile
+        volume (snap_on_start=False, save_on_stop=False)
+
+    On VM startup, new volume is created, depending on volume type,
+    according to the table below:
+
+    snap_on_start, save_on_stop
+    False,         False,        - no suffix, fresh empty volume
+    False,         True,         - "-snap", snapshot of last committed revision
+    True ,         False,        - "-snap", snapshot of last committed revision
+                                   of source volume (from VM's template)
+    True,          True,         - unsupported configuration
+
+    Volume's revision_id format is "{timestamp}-back", where timestamp is in
+    '%s' format (seconds since unix epoch)
     '''  # pylint: disable=protected-access
 
     size_cache = None
@@ -62,13 +90,20 @@ class ThinPool(qubes.storage.Pool):
 
         self._volume_objects_cache = {}
 
+    def __repr__(self):
+        return '<{} at {:#x} name={!r} volume_group={!r} thin_pool={!r}>'.\
+            format(
+                type(self).__name__, id(self),
+                self.name, self.volume_group, self.thin_pool)
+
     @property
     def config(self):
         return {
             'name': self.name,
             'volume_group': self.volume_group,
             'thin_pool': self.thin_pool,
-            'driver': ThinPool.driver
+            'driver': ThinPool.driver,
+            'revisions_to_keep': self.revisions_to_keep,
         }
 
     def destroy(self):
@@ -131,20 +166,16 @@ class ThinPool(qubes.storage.Pool):
                 continue
             if vol_info['pool_lv'] != self.thin_pool:
                 continue
-            if vid.endswith('-snap'):
+            if vid.endswith('-snap') or vid.endswith('-import'):
                 # implementation detail volume
                 continue
             if vid.endswith('-back'):
                 # old revisions
                 continue
-            config = {
-                'pool': self,
-                'vid': vid,
-                'name': vid,
-                'volume_group': self.volume_group,
-                'rw': vol_info['attr'][1] == 'w',
-            }
-            volumes += [ThinVolume(**config)]
+            volume = self.get_volume(vid)
+            if volume in volumes:
+                continue
+            volumes.append(volume)
         return volumes
 
     @property
@@ -164,26 +195,14 @@ class ThinPool(qubes.storage.Pool):
             return 0
 
 
-def init_cache(log=logging.getLogger('qubes.storage.lvm')):
-    cmd = ['lvs', '--noheadings', '-o',
-           'vg_name,pool_lv,name,lv_size,data_percent,lv_attr,origin',
-           '--units', 'b', '--separator', ';']
-    if os.getuid() != 0:
-        cmd.insert(0, 'sudo')
-    environ = os.environ.copy()
-    environ['LC_ALL'] = 'C.utf8'
-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-        close_fds=True, env=environ)
-    out, err = p.communicate()
-    return_code = p.returncode
-    if return_code == 0 and err:
-        log.warning(err)
-    elif return_code != 0:
-        raise qubes.storage.StoragePoolException(err)
+_init_cache_cmd = ['lvs', '--noheadings', '-o',
+   'vg_name,pool_lv,name,lv_size,data_percent,lv_attr,origin',
+   '--units', 'b', '--separator', ';']
 
+def _parse_lvm_cache(lvm_output):
     result = {}
 
-    for line in out.splitlines():
+    for line in lvm_output.splitlines():
         line = line.decode().strip()
         pool_name, pool_lv, name, size, usage_percent, attr, \
             origin = line.split(';', 6)
@@ -197,9 +216,73 @@ def init_cache(log=logging.getLogger('qubes.storage.lvm')):
 
     return result
 
+def init_cache(log=logging.getLogger('qubes.storage.lvm')):
+    cmd = _init_cache_cmd
+    if os.getuid() != 0:
+        cmd = ['sudo'] + cmd
+    environ = os.environ.copy()
+    environ['LC_ALL'] = 'C.utf8'
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+        close_fds=True, env=environ)
+    out, err = p.communicate()
+    return_code = p.returncode
+    if return_code == 0 and err:
+        log.warning(err)
+    elif return_code != 0:
+        raise qubes.storage.StoragePoolException(err)
+
+    return _parse_lvm_cache(out)
+
+@asyncio.coroutine
+def init_cache_coro(log=logging.getLogger('qubes.storage.lvm')):
+    cmd = _init_cache_cmd
+    if os.getuid() != 0:
+        cmd = ['sudo'] + cmd
+    environ = os.environ.copy()
+    environ['LC_ALL'] = 'C.utf8'
+    p = yield from asyncio.create_subprocess_exec(*cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        close_fds=True, env=environ)
+    out, err = yield from p.communicate()
+    return_code = p.returncode
+    if return_code == 0 and err:
+        log.warning(err)
+    elif return_code != 0:
+        raise qubes.storage.StoragePoolException(err)
+
+    return _parse_lvm_cache(out)
 
 size_cache = init_cache()
 
+
+def _revision_sort_key(revision):
+    '''Sort key for revisions. Sort them by time
+
+    :returns timestamp
+    '''
+    if isinstance(revision, tuple):
+        revision = revision[0]
+    if '-' in revision:
+        revision = revision.split('-')[0]
+    return int(revision)
+
+def locked(method):
+    '''Decorator running given Volume's coroutine under a lock.
+    Needs to be added after wrapping with @asyncio.coroutine, for example:
+
+    >>>@locked
+    >>>@asyncio.coroutine
+    >>>def start(self):
+    >>>    pass
+    '''
+    @asyncio.coroutine
+    @functools.wraps(method)
+    def wrapper(self, *args, **kwargs):
+        with (yield from self._lock):  # pylint: disable=protected-access
+            return (yield from method(self, *args, **kwargs))
+    return wrapper
+
 class ThinVolume(qubes.storage.Volume):
     ''' Default LVM thin volume implementation
     '''  # pylint: disable=too-few-public-methods
@@ -212,12 +295,27 @@ class ThinVolume(qubes.storage.Volume):
 
         if self.snap_on_start or self.save_on_stop:
             self._vid_snap = self.vid + '-snap'
+        if self.save_on_stop:
+            self._vid_import = self.vid + '-import'
 
         self._size = size
+        self._lock = asyncio.Lock()
 
     @property
     def path(self):
-        return '/dev/' + self.vid
+        return '/dev/' + self._vid_current
+
+    @property
+    def _vid_current(self):
+        if self.vid in size_cache:
+            return self.vid
+        vol_revisions = self.revisions
+        if vol_revisions:
+            last_revision = \
+                max(vol_revisions.items(), key=_revision_sort_key)[0]
+            return self.vid + '-' + last_revision
+        # detached pool? return expected path
+        return self.vid
 
     @property
     def revisions(self):
@@ -229,7 +327,8 @@ class ThinVolume(qubes.storage.Volume):
             if not revision_vid.endswith('-back'):
                 continue
             revision_vid = revision_vid[len(name_prefix):]
-            seconds = int(revision_vid[:-len('-back')])
+            # get revision without suffix
+            seconds = int(revision_vid.split('-')[0])
             iso_date = qubes.storage.isodate(seconds).split('.', 1)[0]
             revisions[revision_vid] = iso_date
         return revisions
@@ -239,7 +338,7 @@ class ThinVolume(qubes.storage.Volume):
         try:
             if self.is_dirty():
                 return qubes.storage.lvm.size_cache[self._vid_snap]['size']
-            return qubes.storage.lvm.size_cache[self.vid]['size']
+            return qubes.storage.lvm.size_cache[self._vid_current]['size']
         except KeyError:
             return self._size
 
@@ -248,6 +347,7 @@ class ThinVolume(qubes.storage.Volume):
         raise qubes.storage.StoragePoolException(
             "You shouldn't use lvm size setter")
 
+    @asyncio.coroutine
     def _reset(self):
         ''' Resets a volatile volume '''
         assert not self.snap_on_start and not self.save_on_stop, \
@@ -255,14 +355,15 @@ class ThinVolume(qubes.storage.Volume):
         self.log.debug('Resetting volatile %s', self.vid)
         try:
             cmd = ['remove', self.vid]
-            qubes_lvm(cmd, self.log)
+            yield from qubes_lvm_coro(cmd, self.log)
         except qubes.storage.StoragePoolException:
             pass
         # pylint: disable=protected-access
         cmd = ['create', self.pool._pool_id, self.vid.split('/')[1],
                str(self.size)]
-        qubes_lvm(cmd, self.log)
+        yield from qubes_lvm_coro(cmd, self.log)
 
+    @asyncio.coroutine
     def _remove_revisions(self, revisions=None):
         '''Remove old volume revisions.
 
@@ -273,19 +374,32 @@ class ThinVolume(qubes.storage.Volume):
         '''
         if revisions is None:
             revisions = sorted(self.revisions.items(),
-                key=operator.itemgetter(1))
+                key=_revision_sort_key)
             # pylint: disable=invalid-unary-operand-type
             revisions = revisions[:(-self.revisions_to_keep) or None]
             revisions = [rev_id for rev_id, _ in revisions]
 
         for rev_id in revisions:
+            # safety check
+            assert rev_id != self._vid_current
             try:
                 cmd = ['remove', self.vid + '-' + rev_id]
-                qubes_lvm(cmd, self.log)
+                yield from qubes_lvm_coro(cmd, self.log)
             except qubes.storage.StoragePoolException:
                 pass
 
-    def _commit(self):
+    @asyncio.coroutine
+    def _commit(self, vid_to_commit=None, keep=False):
+        '''
+        Commit temporary volume into current one. By default
+        :py:attr:`_vid_snap` is used (which is created by :py:meth:`start()`),
+        but can be overriden by *vid_to_commit* argument.
+
+        :param vid_to_commit: LVM volume ID to commit into this one
+        :param keep: whether to keep or not *vid_to_commit*.
+          IOW use 'clone' or 'rename' methods.
+        :return: None
+        '''
         msg = "Trying to commit {!s}, but it has save_on_stop == False"
         msg = msg.format(self)
         assert self.save_on_stop, msg
@@ -293,39 +407,41 @@ class ThinVolume(qubes.storage.Volume):
         msg = "Trying to commit {!s}, but it has rw == False"
         msg = msg.format(self)
         assert self.rw, msg
-        assert hasattr(self, '_vid_snap')
-
-        if self.revisions_to_keep > 0:
-            cmd = ['clone', self.vid,
-                '{}-{}-back'.format(self.vid, int(time.time()))]
-            qubes_lvm(cmd, self.log)
-            reset_cache()
-        self._remove_revisions()
-
-        # TODO: when converting this function to coroutine, this _must_ be
-        # under a lock
-        # remove old volume only after _successful_ clone of the new one
-        cmd = ['rename', self.vid, self.vid + '-tmp']
-        qubes_lvm(cmd, self.log)
-        try:
-            cmd = ['clone', self._vid_snap, self.vid]
-            qubes_lvm(cmd, self.log)
-        except:
-            # restore original volume
-            cmd = ['rename', self.vid + '-tmp', self.vid]
-            qubes_lvm(cmd, self.log)
-            raise
-        else:
-            cmd = ['remove', self.vid + '-tmp']
-            qubes_lvm(cmd, self.log)
+        if vid_to_commit is None:
+            assert hasattr(self, '_vid_snap')
+            vid_to_commit = self._vid_snap
 
+        assert self._lock.locked()
+        if not os.path.exists('/dev/' + vid_to_commit):
+            # nothing to commit
+            return
 
+        if self._vid_current == self.vid:
+            cmd = ['rename', self.vid,
+                   '{}-{}-back'.format(self.vid, int(time.time()))]
+            yield from qubes_lvm_coro(cmd, self.log)
+            yield from reset_cache_coro()
+
+        cmd = ['clone' if keep else 'rename',
+               vid_to_commit,
+               self.vid]
+        yield from qubes_lvm_coro(cmd, self.log)
+        yield from reset_cache_coro()
+        # make sure the one we've committed right now is properly
+        # detected as the current one - before removing anything
+        assert self._vid_current == self.vid
+
+        # and remove old snapshots, if needed
+        yield from self._remove_revisions()
+
+    @locked
+    @asyncio.coroutine
     def create(self):
         assert self.vid
         assert self.size
         if self.save_on_stop:
             if self.source:
-                cmd = ['clone', str(self.source), self.vid]
+                cmd = ['clone', self.source.path, self.vid]
             else:
                 cmd = [
                     'create',
@@ -333,70 +449,126 @@ class ThinVolume(qubes.storage.Volume):
                     self.vid.split('/', 1)[1],
                     str(self.size)
                 ]
-            qubes_lvm(cmd, self.log)
-            reset_cache()
+            yield from qubes_lvm_coro(cmd, self.log)
+            yield from reset_cache_coro()
         return self
 
+    @locked
+    @asyncio.coroutine
     def remove(self):
         assert self.vid
         try:
             if os.path.exists('/dev/' + self._vid_snap):
                 cmd = ['remove', self._vid_snap]
-                qubes_lvm(cmd, self.log)
+                yield from qubes_lvm_coro(cmd, self.log)
+        except AttributeError:
+            pass
+
+        try:
+            if os.path.exists('/dev/' + self._vid_import):
+                cmd = ['remove', self._vid_import]
+                yield from qubes_lvm_coro(cmd, self.log)
         except AttributeError:
             pass
 
-        self._remove_revisions(self.revisions.keys())
+        yield from self._remove_revisions(self.revisions.keys())
         if not os.path.exists(self.path):
             return
-        cmd = ['remove', self.vid]
-        qubes_lvm(cmd, self.log)
-        reset_cache()
+        cmd = ['remove', self.path]
+        yield from qubes_lvm_coro(cmd, self.log)
+        yield from reset_cache_coro()
         # pylint: disable=protected-access
         self.pool._volume_objects_cache.pop(self.vid, None)
 
     def export(self):
         ''' Returns an object that can be `open()`. '''
         # make sure the device node is available
-        qubes_lvm(['activate', self.vid], self.log)
-        devpath = '/dev/' + self.vid
+        qubes_lvm(['activate', self.path], self.log)
+        devpath = self.path
         return devpath
 
+    @locked
     @asyncio.coroutine
     def import_volume(self, src_volume):
         if not src_volume.save_on_stop:
             return self
 
+        if self.is_dirty():
+            raise qubes.storage.StoragePoolException(
+                'Cannot import to dirty volume {} -'
+                ' start and stop a qube to cleanup'.format(self.vid))
+        self.abort_if_import_in_progress()
         # HACK: neat trick to speed up testing if you have same physical thin
         # pool assigned to two qubes-pools i.e: qubes_dom0 and test-lvm
         # pylint: disable=line-too-long
         if isinstance(src_volume.pool, ThinPool) and \
                 src_volume.pool.thin_pool == self.pool.thin_pool:  # NOQA
-            cmd = ['remove', self.vid]
-            qubes_lvm(cmd, self.log)
-            cmd = ['clone', str(src_volume), str(self)]
-            qubes_lvm(cmd, self.log)
+            yield from self._commit(src_volume.path[len('/dev/'):], keep=True)
         else:
-            if src_volume.size != self.size:
-                self.resize(src_volume.size)
+            cmd = ['create',
+                   self.pool._pool_id,  # pylint: disable=protected-access
+                   self._vid_import.split('/')[1],
+                   str(src_volume.size)]
+            yield from qubes_lvm_coro(cmd, self.log)
             src_path = src_volume.export()
-            cmd = ['dd', 'if=' + src_path, 'of=/dev/' + self.vid,
-                'conv=sparse']
+            cmd = ['dd', 'if=' + src_path, 'of=/dev/' + self._vid_import,
+                'conv=sparse', 'status=none']
+            if not os.access('/dev/' + self._vid_import, os.W_OK) or \
+                    not os.access(src_path, os.R_OK):
+                cmd.insert(0, 'sudo')
+
             p = yield from asyncio.create_subprocess_exec(*cmd)
             yield from p.wait()
             if p.returncode != 0:
+                cmd = ['remove', self._vid_import]
+                yield from qubes_lvm_coro(cmd, self.log)
                 raise qubes.storage.StoragePoolException(
                     'Failed to import volume {!r}, dd exit code: {}'.format(
                         src_volume, p.returncode))
-            reset_cache()
+            yield from self._commit(self._vid_import)
 
         return self
 
+    @locked
+    @asyncio.coroutine
     def import_data(self):
         ''' Returns an object that can be `open()`. '''
-        devpath = '/dev/' + self.vid
+        if self.is_dirty():
+            raise qubes.storage.StoragePoolException(
+                'Cannot import data to dirty volume {}, stop the qube first'.
+                format(self.vid))
+        self.abort_if_import_in_progress()
+        # pylint: disable=protected-access
+        cmd = ['create', self.pool._pool_id, self._vid_import.split('/')[1],
+               str(self.size)]
+        yield from qubes_lvm_coro(cmd, self.log)
+        yield from reset_cache_coro()
+        devpath = '/dev/' + self._vid_import
         return devpath
 
+    @locked
+    @asyncio.coroutine
+    def import_data_end(self, success):
+        '''Either commit imported data, or discard temporary volume'''
+        if not os.path.exists('/dev/' + self._vid_import):
+            raise qubes.storage.StoragePoolException(
+                'No import operation in progress on {}'.format(self.vid))
+        if success:
+            yield from self._commit(self._vid_import)
+        else:
+            cmd = ['remove', self._vid_import]
+            yield from qubes_lvm_coro(cmd, self.log)
+
+    def abort_if_import_in_progress(self):
+        try:
+            devpath = '/dev/' + self._vid_import
+            if os.path.exists(devpath):
+                raise qubes.storage.StoragePoolException(
+                    'Import operation in progress on {}'.format(self.vid))
+        except AttributeError:  # self._vid_import
+            # no vid_import - import definitely not in progress
+            pass
+
     def is_dirty(self):
         if self.save_on_stop:
             return os.path.exists('/dev/' + self._vid_snap)
@@ -408,25 +580,34 @@ class ThinVolume(qubes.storage.Volume):
         if self._vid_snap not in size_cache:
             return False
         return (size_cache[self._vid_snap]['origin'] !=
-               self.source.vid.split('/')[1])
-
+               self.source.path.split('/')[-1])
 
+    @locked
+    @asyncio.coroutine
     def revert(self, revision=None):
+        if self.is_dirty():
+            raise qubes.storage.StoragePoolException(
+                'Cannot revert dirty volume {}, stop the qube first'.format(
+                    self.vid))
+        self.abort_if_import_in_progress()
         if revision is None:
             revision = \
-                max(self.revisions.items(), key=operator.itemgetter(1))[0]
-        old_path = self.path + '-' + revision
+                max(self.revisions.items(), key=_revision_sort_key)[0]
+        old_path = '/dev/' + self.vid + '-' + revision
         if not os.path.exists(old_path):
             msg = "Volume {!s} has no {!s}".format(self, old_path)
             raise qubes.storage.StoragePoolException(msg)
 
-        cmd = ['remove', self.vid]
-        qubes_lvm(cmd, self.log)
+        if self.vid in size_cache:
+            cmd = ['remove', self.vid]
+            yield from qubes_lvm_coro(cmd, self.log)
         cmd = ['clone', self.vid + '-' + revision, self.vid]
-        qubes_lvm(cmd, self.log)
-        reset_cache()
+        yield from qubes_lvm_coro(cmd, self.log)
+        yield from reset_cache_coro()
         return self
 
+    @locked
+    @asyncio.coroutine
     def resize(self, size):
         ''' Expands volume, throws
             :py:class:`qubst.storage.qubes.storage.StoragePoolException` if
@@ -448,49 +629,58 @@ class ThinVolume(qubes.storage.Volume):
 
         if self.is_dirty():
             cmd = ['extend', self._vid_snap, str(size)]
-            qubes_lvm(cmd, self.log)
+            yield from qubes_lvm_coro(cmd, self.log)
+        elif hasattr(self, '_vid_import') and \
+                os.path.exists('/dev/' + self._vid_import):
+            cmd = ['extend', self._vid_import, str(size)]
+            yield from qubes_lvm_coro(cmd, self.log)
         elif self.save_on_stop or not self.snap_on_start:
-            cmd = ['extend', self.vid, str(size)]
-            qubes_lvm(cmd, self.log)
-        reset_cache()
+            cmd = ['extend', self._vid_current, str(size)]
+            yield from qubes_lvm_coro(cmd, self.log)
+        yield from reset_cache_coro()
 
+    @asyncio.coroutine
     def _snapshot(self):
         try:
             cmd = ['remove', self._vid_snap]
-            qubes_lvm(cmd, self.log)
+            yield from qubes_lvm_coro(cmd, self.log)
         except:  # pylint: disable=bare-except
             pass
 
         if self.source is None:
-            cmd = ['clone', self.vid, self._vid_snap]
+            cmd = ['clone', self._vid_current, self._vid_snap]
         else:
-            cmd = ['clone', str(self.source), self._vid_snap]
-        qubes_lvm(cmd, self.log)
-
+            cmd = ['clone', self.source.path, self._vid_snap]
+        yield from qubes_lvm_coro(cmd, self.log)
 
+    @locked
+    @asyncio.coroutine
     def start(self):
+        self.abort_if_import_in_progress()
         try:
             if self.snap_on_start or self.save_on_stop:
                 if not self.save_on_stop or not self.is_dirty():
-                    self._snapshot()
+                    yield from self._snapshot()
             else:
-                self._reset()
+                yield from self._reset()
         finally:
-            reset_cache()
+            yield from reset_cache_coro()
         return self
 
+    @locked
+    @asyncio.coroutine
     def stop(self):
         try:
             if self.save_on_stop:
-                self._commit()
-            if self.snap_on_start or self.save_on_stop:
+                yield from self._commit()
+            if self.snap_on_start and not self.save_on_stop:
                 cmd = ['remove', self._vid_snap]
-                qubes_lvm(cmd, self.log)
-            else:
+                yield from qubes_lvm_coro(cmd, self.log)
+            elif not self.snap_on_start and not self.save_on_stop:
                 cmd = ['remove', self.vid]
-                qubes_lvm(cmd, self.log)
+                yield from qubes_lvm_coro(cmd, self.log)
         finally:
-            reset_cache()
+            yield from reset_cache_coro()
         return self
 
     def verify(self):
@@ -499,9 +689,9 @@ class ThinVolume(qubes.storage.Volume):
             # volatile volumes don't need any files
             return True
         if self.source is not None:
-            vid = str(self.source)
+            vid = self.source.path[len('/dev/'):]
         else:
-            vid = self.vid
+            vid = self._vid_current
         try:
             vol_info = size_cache[vid]
             if vol_info['attr'][4] != 'a':
@@ -528,7 +718,7 @@ class ThinVolume(qubes.storage.Volume):
     def usage(self):  # lvm thin usage always returns at least the same usage as
                       # the parent
         try:
-            return qubes.storage.lvm.size_cache[self.vid]['usage']
+            return qubes.storage.lvm.size_cache[self._vid_current]['usage']
         except KeyError:
             return 0
 
@@ -541,9 +731,14 @@ def pool_exists(pool_id):
     except KeyError:
         return False
 
+def _get_lvm_cmdline(cmd):
+    ''' Build command line for :program:`lvm` call.
+    The purpose of this function is to keep all the detailed lvm options in
+    one place.
 
-def qubes_lvm(cmd, log=logging.getLogger('qubes.storage.lvm')):
-    ''' Call :program:`lvm` to execute an LVM operation '''
+    :param cmd: array of str, where cmd[0] is action and the rest are arguments
+    :return array of str appropriate for subprocess.Popen
+    '''
     action = cmd[0]
     if action == 'remove':
         lvm_cmd = ['lvremove', '-f', cmd[1]]
@@ -568,21 +763,57 @@ def qubes_lvm(cmd, log=logging.getLogger('qubes.storage.lvm')):
         cmd = ['sudo', 'lvm'] + lvm_cmd
     else:
         cmd = ['lvm'] + lvm_cmd
+
+    return cmd
+
+def _process_lvm_output(returncode, stdout, stderr, log):
+    '''Process output of LVM, determine if the call was successful and
+    possibly log warnings.'''
+    # Filter out warning about intended over-provisioning.
+    # Upstream discussion about missing option to silence it:
+    # https://bugzilla.redhat.com/1347008
+    err = '\n'.join(line for line in stderr.decode().splitlines()
+        if 'exceeds the size of thin pool' not in line)
+    if stdout:
+        log.debug(stdout)
+    if returncode == 0 and err:
+        log.warning(err)
+    elif returncode != 0:
+        assert err, "Command exited unsuccessful, but printed nothing to stderr"
+        err = err.replace('%', '%%')
+        raise qubes.storage.StoragePoolException(err)
+    return True
+
+def qubes_lvm(cmd, log=logging.getLogger('qubes.storage.lvm')):
+    ''' Call :program:`lvm` to execute an LVM operation '''
+    # the only caller for this non-coroutine version is ThinVolume.export()
+    cmd = _get_lvm_cmdline(cmd)
     environ = os.environ.copy()
     environ['LC_ALL'] = 'C.utf8'
     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
         close_fds=True, env=environ)
     out, err = p.communicate()
-    return_code = p.returncode
-    if out:
-        log.debug(out)
-    if return_code == 0 and err:
-        log.warning(err)
-    elif return_code != 0:
-        assert err, "Command exited unsuccessful, but printed nothing to stderr"
-        raise qubes.storage.StoragePoolException(err)
-    return True
+    return _process_lvm_output(p.returncode, out, err, log)
+
+@asyncio.coroutine
+def qubes_lvm_coro(cmd, log=logging.getLogger('qubes.storage.lvm')):
+    ''' Call :program:`lvm` to execute an LVM operation
+
+    Coroutine version of :py:func:`qubes_lvm`'''
+    cmd = _get_lvm_cmdline(cmd)
+    environ = os.environ.copy()
+    environ['LC_ALL'] = 'C.utf8'
+    p = yield from asyncio.create_subprocess_exec(*cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        close_fds=True, env=environ)
+    out, err = yield from p.communicate()
+    return _process_lvm_output(p.returncode, out, err, log)
 
 
 def reset_cache():
     qubes.storage.lvm.size_cache = init_cache()
+
+@asyncio.coroutine
+def reset_cache_coro():
+    qubes.storage.lvm.size_cache = yield from init_cache_coro()

qubes/storage/reflink.py

@@ -22,13 +22,14 @@
     but not required.
 '''
 
+import asyncio
 import collections
 import errno
 import fcntl
+import functools
 import glob
 import logging
 import os
-import re
 import subprocess
 import tempfile
 from contextlib import contextmanager, suppress
@@ -36,7 +37,8 @@ from contextlib import contextmanager, suppress
 import qubes.storage
 
 BLKSIZE = 512
-FICLONE = 1074041865  # see ioctl_ficlone manpage
+FICLONE = 1074041865        # defined in <linux/fs.h>
+LOOP_SET_CAPACITY = 0x4C07  # defined in <linux/loop.h>
 LOGGER = logging.getLogger('qubes.storage.reflink')
 
 
@@ -53,7 +55,7 @@ class ReflinkPool(qubes.storage.Pool):
 
     def setup(self):
         created = _make_dir(self.dir_path)
-        if self.setup_check and not is_reflink_supported(self.dir_path):
+        if self.setup_check and not is_supported(self.dir_path):
             if created:
                 _remove_empty_dir(self.dir_path)
             raise qubes.storage.StoragePoolException(
@@ -115,12 +117,37 @@ class ReflinkPool(qubes.storage.Pool):
             [pool for pool in app.pools.values() if pool is not self],
             self.dir_path)
 
+
+def _unblock(method):
+    ''' Decorator transforming a synchronous volume method into a
+        coroutine that runs the original method in the event loop's
+        thread-based default executor, under a per-volume lock.
+    '''
+    @asyncio.coroutine
+    @functools.wraps(method)
+    def wrapper(self, *args, **kwargs):
+        with (yield from self._lock):  # pylint: disable=protected-access
+            return (yield from asyncio.get_event_loop().run_in_executor(
+                None, functools.partial(method, self, *args, **kwargs)))
+    return wrapper
+
 class ReflinkVolume(qubes.storage.Volume):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._lock = asyncio.Lock()
+        self._path_vid = os.path.join(self.pool.dir_path, self.vid)
+        self._path_clean = self._path_vid + '.img'
+        self._path_dirty = self._path_vid + '-dirty.img'
+        self._path_import = self._path_vid + '-import.img'
+        self.path = self._path_dirty
+
+    @_unblock
     def create(self):
         if self.save_on_stop and not self.snap_on_start:
             _create_sparse_file(self._path_clean, self.size)
         return self
 
+    @_unblock
     def verify(self):
         if self.snap_on_start:
             img = self.source._path_clean  # pylint: disable=protected-access
@@ -132,19 +159,26 @@ class ReflinkVolume(qubes.storage.Volume):
         if img is None or os.path.exists(img):
             return True
         raise qubes.storage.StoragePoolException(
-            'Missing image file {!r} for volume {!s}'.format(img, self.vid))
+            'Missing image file {!r} for volume {}'.format(img, self.vid))
 
+    @_unblock
     def remove(self):
         ''' Drop volume object from pool; remove volume images from
             oldest to newest; remove empty VM directory.
         '''
         self.pool._volumes.pop(self, None)  # pylint: disable=protected-access
+        self._cleanup()
         self._prune_revisions(keep=0)
         _remove_file(self._path_clean)
         _remove_file(self._path_dirty)
         _remove_empty_dir(os.path.dirname(self._path_dirty))
         return self
 
+    def _cleanup(self):
+        for tmp in glob.iglob(glob.escape(self._path_vid) + '*.img*~*'):
+            _remove_file(tmp)
+        _remove_file(self._path_import)
+
     def is_outdated(self):
         if self.snap_on_start:
             with suppress(FileNotFoundError):
@@ -156,7 +190,9 @@ class ReflinkVolume(qubes.storage.Volume):
     def is_dirty(self):
         return self.save_on_stop and os.path.exists(self._path_dirty)
 
+    @_unblock
     def start(self):
+        self._cleanup()
         if self.is_dirty():  # implies self.save_on_stop
             return self
         if self.snap_on_start:
@@ -168,24 +204,23 @@ class ReflinkVolume(qubes.storage.Volume):
             _create_sparse_file(self._path_dirty, self.size)
         return self
 
+    @_unblock
     def stop(self):
         if self.save_on_stop:
-            self._commit()
+            self._commit(self._path_dirty)
         else:
             _remove_file(self._path_dirty)
             _remove_file(self._path_clean)
         return self
 
-    def _commit(self):
+    def _commit(self, path_from):
         self._add_revision()
         self._prune_revisions()
-        _rename_file(self._path_dirty, self._path_clean)
+        _rename_file(path_from, self._path_clean)
 
     def _add_revision(self):
         if self.revisions_to_keep == 0:
             return
-        if _get_file_disk_usage(self._path_clean) == 0:
-            return
         ctime = os.path.getctime(self._path_clean)
         timestamp = qubes.storage.isodate(int(ctime))
         _copy_file(self._path_clean,
@@ -198,7 +233,11 @@ class ReflinkVolume(qubes.storage.Volume):
         for number, timestamp in list(self.revisions.items())[:-keep or None]:
             _remove_file(self._path_revision(number, timestamp))
 
+    @_unblock
     def revert(self, revision=None):
+        if self.is_dirty():
+            raise qubes.storage.StoragePoolException(
+                'Cannot revert: {} is not cleanly stopped'.format(self.vid))
         if revision is None:
             number, timestamp = list(self.revisions.items())[-1]
         else:
@@ -208,61 +247,58 @@ class ReflinkVolume(qubes.storage.Volume):
         _rename_file(path_revision, self._path_clean)
         return self
 
+    @_unblock
     def resize(self, size):
         ''' Expand a read-write volume image; notify any corresponding
             loop devices of the size change.
         '''
         if not self.rw:
             raise qubes.storage.StoragePoolException(
-                'Cannot resize: {!s} is read-only'.format(self.vid))
+                'Cannot resize: {} is read-only'.format(self.vid))
 
         if size < self.size:
             raise qubes.storage.StoragePoolException(
-                'For your own safety, shrinking of {!s} is disabled'
-                ' ({:d} < {:d}). If you really know what you are doing,'
+                'For your own safety, shrinking of {} is disabled'
+                ' ({} < {}). If you really know what you are doing,'
                 ' use "truncate" manually.'.format(self.vid, size, self.size))
 
         try:  # assume volume is not (cleanly) stopped ...
             _resize_file(self._path_dirty, size)
+            self.size = size
         except FileNotFoundError:  # ... but it actually is.
             _resize_file(self._path_clean, size)
+            self.size = size
+            return self
 
-        self.size = size
-
-        # resize any corresponding loop devices
-        out = _cmd('losetup', '--associated', self._path_dirty)
-        for match in re.finditer(br'^(/dev/loop[0-9]+): ', out, re.MULTILINE):
-            loop_dev = match.group(1).decode('ascii')
-            _cmd('losetup', '--set-capacity', loop_dev)
-
+        _update_loopdev_sizes(self._path_dirty)
         return self
 
-    def _require_save_on_stop(self, method_name):
+    def export(self):
         if not self.save_on_stop:
             raise NotImplementedError(
-                'Cannot {!s}: {!s} is not save_on_stop'.format(
-                    method_name, self.vid))
-
-    def export(self):
-        self._require_save_on_stop('export')
+                'Cannot export: {} is not save_on_stop'.format(self.vid))
         return self._path_clean
 
     def import_data(self):
-        self._require_save_on_stop('import_data')
-        _create_sparse_file(self._path_dirty, self.size)
-        return self._path_dirty
+        if not self.save_on_stop:
+            raise NotImplementedError(
+                'Cannot import_data: {} is not save_on_stop'.format(self.vid))
+        _create_sparse_file(self._path_import, self.size)
+        return self._path_import
 
     def import_data_end(self, success):
         if success:
-            self._commit()
+            self._commit(self._path_import)
         else:
-            _remove_file(self._path_dirty)
+            _remove_file(self._path_import)
         return self
 
+    @_unblock
     def import_volume(self, src_volume):
-        self._require_save_on_stop('import_volume')
+        if not self.save_on_stop:
+            return self
         try:
-            _copy_file(src_volume.export(), self._path_dirty)
+            _copy_file(src_volume.export(), self._path_import)
         except:
             self.import_data_end(False)
             raise
@@ -274,18 +310,6 @@ class ReflinkVolume(qubes.storage.Volume):
             timestamp = self.revisions[number]
         return self._path_clean + '.' + number + '@' + timestamp + 'Z'
 
-    @property
-    def _path_clean(self):
-        return os.path.join(self.pool.dir_path, self.vid + '.img')
-
-    @property
-    def _path_dirty(self):
-        return os.path.join(self.pool.dir_path, self.vid + '-dirty.img')
-
-    @property
-    def path(self):
-        return self._path_dirty
-
     @property
     def _next_revision_number(self):
         numbers = self.revisions.keys()
@@ -296,10 +320,10 @@ class ReflinkVolume(qubes.storage.Volume):
     @property
     def revisions(self):
         prefix = self._path_clean + '.'
-        paths = glob.glob(glob.escape(prefix) + '*@*Z')
-        items = sorted((path[len(prefix):-1].split('@') for path in paths),
-                       key=lambda item: int(item[0]))
-        return collections.OrderedDict(items)
+        paths = glob.iglob(glob.escape(prefix) + '*@*Z')
+        items = (path[len(prefix):-1].split('@') for path in paths)
+        return collections.OrderedDict(sorted(items,
+                                              key=lambda item: int(item[0])))
 
     @property
     def usage(self):
@@ -391,39 +415,48 @@ def _create_sparse_file(path, size):
         tmp.truncate(size)
         LOGGER.info('Created sparse file: %s', tmp.name)
 
+def _update_loopdev_sizes(img):
+    ''' Resolve img; update the size of loop devices backed by it. '''
+    needle = os.fsencode(os.path.realpath(img)) + b'\n'
+    for sys_path in glob.iglob('/sys/block/loop[0-9]*/loop/backing_file'):
+        try:
+            with open(sys_path, 'rb') as sys_io:
+                if sys_io.read() != needle:
+                    continue
+        except FileNotFoundError:
+            continue
+        with open('/dev/' + sys_path.split('/')[3]) as dev_io:
+            fcntl.ioctl(dev_io.fileno(), LOOP_SET_CAPACITY)
+
+def _attempt_ficlone(src, dst):
+    try:
+        fcntl.ioctl(dst.fileno(), FICLONE, src.fileno())
+        return True
+    except OSError:
+        return False
+
 def _copy_file(src, dst):
     ''' Copy src to dst as a reflink if possible, sparse if not. '''
-    if not os.path.exists(src):
-        raise FileNotFoundError(src)
-    with _replace_file(dst) as tmp:
-        LOGGER.info('Copying file: %s -> %s', src, tmp.name)
-        _cmd('cp', '--sparse=always', '--reflink=auto', src, tmp.name)
-
-def _cmd(*args):
-    ''' Run command until finished; return stdout (as bytes) if it
-        exited 0. Otherwise, raise a detailed StoragePoolException.
-    '''
-    try:
-        return subprocess.run(args, check=True,
-                              stdout=subprocess.PIPE,
-                              stderr=subprocess.PIPE).stdout
-    except subprocess.CalledProcessError as ex:
-        msg = '{!s} err={!r} out={!r}'.format(ex, ex.stderr, ex.stdout)
-        raise qubes.storage.StoragePoolException(msg) from ex
-
-def is_reflink_supported(dst_dir, src_dir=None):
+    with _replace_file(dst) as tmp_io:
+        with open(src, 'rb') as src_io:
+            if _attempt_ficlone(src_io, tmp_io):
+                LOGGER.info('Reflinked file: %s -> %s', src, tmp_io.name)
+                return True
+        LOGGER.info('Copying file: %s -> %s', src, tmp_io.name)
+        cmd = 'cp', '--sparse=always', src, tmp_io.name
+        p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        if p.returncode != 0:
+            raise qubes.storage.StoragePoolException(str(p))
+        return False
+
+def is_supported(dst_dir, src_dir=None):
     ''' Return whether destination directory supports reflink copies
         from source directory. (A temporary file is created in each
         directory, using O_TMPFILE if possible.)
     '''
     if src_dir is None:
         src_dir = dst_dir
-    dst = tempfile.TemporaryFile(dir=dst_dir)
-    src = tempfile.TemporaryFile(dir=src_dir)
-    src.write(b'foo')  # don't let any filesystem get clever with empty files
-
-    try:
-        fcntl.ioctl(dst.fileno(), FICLONE, src.fileno())
-        return True
-    except OSError:
-        return False
+    with tempfile.TemporaryFile(dir=src_dir) as src, \
+         tempfile.TemporaryFile(dir=dst_dir) as dst:
+        src.write(b'foo')  # don't let any fs get clever with empty files
+        return _attempt_ficlone(src, dst)

qubes/tests/__init__.py

@@ -379,8 +379,27 @@ class QubesTestCase(unittest.TestCase):
 
         self.loop = asyncio.get_event_loop()
         self.addCleanup(self.cleanup_loop)
+        self.addCleanup(self.cleanup_traceback)
         self.addCleanup(qubes.ext.pci._cache_get.cache_clear)
 
+    def cleanup_traceback(self):
+        '''Remove local variables reference from tracebacks to allow garbage
+        collector to clean all Qubes*() objects, otherwise file descriptors
+        held by them will leak'''
+        exc_infos = [e for test_case, e in self._outcome.errors
+            if test_case is self]
+        if self._outcome.expectedFailure:
+            exc_infos.append(self._outcome.expectedFailure)
+        for exc_info in exc_infos:
+            if exc_info is None:
+                continue
+            ex = exc_info[1]
+            while ex is not None:
+                if isinstance(ex, qubes.exc.QubesVMError):
+                    ex.vm = None
+                traceback.clear_frames(ex.__traceback__)
+                ex = ex.__context__
+
     def cleanup_gc(self):
         gc.collect()
         leaked = [obj for obj in gc.get_objects() + gc.garbage
@@ -397,6 +416,8 @@ class QubesTestCase(unittest.TestCase):
             except ImportError:
                 pass
 
+        # do not keep leaked object references in locals()
+        leaked = bool(leaked)
         assert not leaked
 
     def cleanup_loop(self):
@@ -421,6 +442,13 @@ class QubesTestCase(unittest.TestCase):
             except asyncio.TimeoutError:
                 raise AssertionError('libvirt event impl drain timeout')
 
+        # this is stupid, but apparently it requires two passes
+        # to cleanup SIGCHLD handlers
+        self.loop.stop()
+        self.loop.run_forever()
+        self.loop.stop()
+        self.loop.run_forever()
+
         # Check there are no Tasks left.
         assert not self.loop._ready
         assert not self.loop._scheduled
@@ -763,20 +791,6 @@ class SystemTestCase(QubesTestCase):
         vmname = vm.name
         app = vm.app
 
-        # avoid race with DispVM.auto_cleanup=True
-        try:
-            self.loop.run_until_complete(
-                asyncio.wait_for(vm.startup_lock.acquire(), 10))
-        except asyncio.TimeoutError:
-            pass
-
-        try:
-            # XXX .is_running() may throw libvirtError if undefined
-            if vm.is_running():
-                self.loop.run_until_complete(vm.kill())
-        except:  # pylint: disable=bare-except
-            pass
-
         try:
             self.loop.run_until_complete(vm.remove_from_disk())
         except:  # pylint: disable=bare-except
@@ -842,7 +856,7 @@ class SystemTestCase(QubesTestCase):
         '''
         try:
             volumes = subprocess.check_output(
-                ['sudo', 'lvs', '--noheadings', '-o', 'vg_name,name',
+                ['lvs', '--noheadings', '-o', 'vg_name,name',
                     '--separator', '/']).decode()
             if ('/vm-' + prefix) not in volumes:
                 return
@@ -857,18 +871,36 @@ class SystemTestCase(QubesTestCase):
         vms = list(vms)
         if not vms:
             return
+        # first kill all the domains, to avoid side effects of changing netvm
+        for vm in vms:
+            try:
+                # XXX .is_running() may throw libvirtError if undefined
+                if vm.is_running():
+                    self.loop.run_until_complete(vm.kill())
+            except:  # pylint: disable=bare-except
+                pass
         # break dependencies
         for vm in vms:
             vm.default_dispvm = None
-        # then remove in reverse topological order (wrt netvm), using naive
+            vm.netvm = None
+        # take app instance from any VM to be removed
+        app = vms[0].app
+        if app.default_dispvm in vms:
+            app.default_dispvm = None
+        if app.default_netvm in vms:
+            app.default_netvm = None
+        del app
+        # then remove in reverse topological order (wrt template), using naive
         # algorithm
-        # this heavily depends on lack of netvm loops
+        # this heavily depends on lack of template loops, but those are
+        # impossible
         while vms:
             vm = vms.pop(0)
             # make sure that all connected VMs are going to be removed,
             # otherwise this will loop forever
-            assert all(x in vms for x in vm.connected_vms)
-            if list(vm.connected_vms):
+            child_vms = list(getattr(vm, 'appvms', []))
+            assert all(x in vms for x in child_vms)
+            if child_vms:
                 # if still something use this VM, put it at the end of queue
                 # and try next one
                 vms.append(vm)
@@ -876,9 +908,16 @@ class SystemTestCase(QubesTestCase):
             self._remove_vm_qubes(vm)
 
     def remove_test_vms(self, xmlpath=XMLPATH, prefix=VMPREFIX):
-        '''Aggresively remove any domain that has name in testing namespace.
+        '''Aggressively remove any domain that has name in testing namespace.
+
+        :param prefix: name prefix of VMs to remove, can be a list of prefixes
         '''
 
+        if isinstance(prefix, str):
+            prefixes = [prefix]
+        else:
+            prefixes = prefix
+        del prefix
         # first, remove them Qubes-way
         if os.path.exists(xmlpath):
             try:
@@ -891,7 +930,7 @@ class SystemTestCase(QubesTestCase):
                 except AttributeError:
                     host_app = qubes.Qubes()
                 self.remove_vms([vm for vm in app.domains
-                    if vm.name.startswith(prefix) or
+                    if any(vm.name.startswith(prefix) for prefix in prefixes) or
                        (isinstance(vm, qubes.vm.dispvm.DispVM) and vm.name
                         not in host_app.domains)])
                 if not hasattr(self, 'host_app'):
@@ -907,7 +946,7 @@ class SystemTestCase(QubesTestCase):
         # now remove what was only in libvirt
         conn = libvirt.open(qubes.config.defaults['libvirt_uri'])
         for dom in conn.listAllDomains():
-            if dom.name().startswith(prefix):
+            if any(dom.name().startswith(prefix) for prefix in prefixes):
                 self._remove_vm_libvirt(dom)
         conn.close()
 
@@ -922,11 +961,12 @@ class SystemTestCase(QubesTestCase):
             if not os.path.exists(dirpath):
                 continue
             for name in os.listdir(dirpath):
-                if name.startswith(prefix):
+                if any(name.startswith(prefix) for prefix in prefixes):
                     vmnames.add(name)
         for vmname in vmnames:
             self._remove_vm_disk(vmname)
-        self._remove_vm_disk_lvm(prefix)
+        for prefix in prefixes:
+            self._remove_vm_disk_lvm(prefix)
 
     def qrexec_policy(self, service, source, destination, allow=True,
             action=None):
@@ -943,7 +983,25 @@ class SystemTestCase(QubesTestCase):
         return _QrexecPolicyContext(service, source, destination,
             allow=allow, action=action)
 
-    def wait_for_window(self, title, timeout=30, show=True):
+    @asyncio.coroutine
+    def wait_for_window_hide_coro(self, title, winid, timeout=30):
+        """
+        Wait for window do disappear
+        :param winid: window id
+        :return:
+        """
+        wait_count = 0
+        while subprocess.call(['xdotool', 'getwindowname', str(winid)],
+                stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) == 0:
+            wait_count += 1
+            if wait_count > timeout * 10:
+                self.fail("Timeout while waiting for {}({}) window to "
+                          "disappear".format(title, winid))
+            yield from asyncio.sleep(0.1)
+
+    @asyncio.coroutine
+    def wait_for_window_coro(self, title, search_class=False, timeout=30,
+            show=True):
         """
         Wait for a window with a given title. Depending on show parameter,
         it will wait for either window to show or to disappear.
@@ -952,19 +1010,59 @@ class SystemTestCase(QubesTestCase):
         :param timeout: timeout of the operation, in seconds
         :param show: if True - wait for the window to be visible,
             otherwise - to not be visible
-        :return: None
+        :param search_class: search based on window class instead of title
+        :return: window id of found window, if show=True
         """
 
-        wait_count = 0
-        while subprocess.call(['xdotool', 'search', '--name', title],
-                stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) \
-                != int(not show):
-            wait_count += 1
-            if wait_count > timeout*10:
-                self.fail("Timeout while waiting for {} window to {}".format(
-                    title, "show" if show else "hide")
-                )
-            self.loop.run_until_complete(asyncio.sleep(0.1))
+        xdotool_search = ['xdotool', 'search', '--onlyvisible']
+        if search_class:
+            xdotool_search.append('--class')
+        else:
+            xdotool_search.append('--name')
+        if show:
+            xdotool_search.append('--sync')
+        if not show:
+            try:
+                winid = subprocess.check_output(xdotool_search + [title],
+                    stderr=subprocess.DEVNULL).decode()
+            except subprocess.CalledProcessError:
+                # already gone
+                return
+            yield from self.wait_for_window_hide_coro(winid, title,
+                timeout=timeout)
+            return
+
+        winid = None
+        while not winid:
+            p = yield from asyncio.create_subprocess_exec(
+                *xdotool_search, title,
+                stderr=subprocess.DEVNULL, stdout=subprocess.PIPE)
+            try:
+                (winid, _) = yield from asyncio.wait_for(
+                    p.communicate(), timeout)
+                # don't check exit code, getting winid on stdout is enough
+                # indicator of success; specifically ignore xdotool failing
+                # with BadWindow or such - when some window appears only for a
+                # moment by xdotool didn't manage to get its properties
+            except asyncio.TimeoutError:
+                self.fail(
+                    "Timeout while waiting for {} window to show".format(title))
+        return winid.decode().strip()
+
+    def wait_for_window(self, *args, **kwargs):
+        """
+        Wait for a window with a given title. Depending on show parameter,
+        it will wait for either window to show or to disappear.
+
+        :param title: title of the window to wait for
+        :param timeout: timeout of the operation, in seconds
+        :param show: if True - wait for the window to be visible,
+            otherwise - to not be visible
+        :param search_class: search based on window class instead of title
+        :return: window id of found window, if show=True
+        """
+        return self.loop.run_until_complete(
+            self.wait_for_window_coro(*args, **kwargs))
 
     def enter_keys_in_window(self, title, keys):
         """
@@ -985,15 +1083,12 @@ class SystemTestCase(QubesTestCase):
         subprocess.check_call(command)
 
     def shutdown_and_wait(self, vm, timeout=60):
-        self.loop.run_until_complete(vm.shutdown())
-        while timeout > 0:
-            if not vm.is_running():
-                return
-            self.loop.run_until_complete(asyncio.sleep(1))
-            timeout -= 1
-        name = vm.name
-        del vm
-        self.fail("Timeout while waiting for VM {} shutdown".format(name))
+        try:
+            self.loop.run_until_complete(vm.shutdown(wait=True, timeout=timeout))
+        except qubes.exc.QubesException:
+            name = vm.name
+            del vm
+            self.fail("Timeout while waiting for VM {} shutdown".format(name))
 
     def prepare_hvm_system_linux(self, vm, init_script, extra_files=None):
         if not os.path.exists('/usr/lib/grub/i386-pc'):
@@ -1109,21 +1204,30 @@ class SystemTestCase(QubesTestCase):
 
     @asyncio.coroutine
     def wait_for_session(self, vm):
+        timeout = 30
+        if getattr(vm, 'template', None) and 'whonix-ws' in vm.template.name:
+            # first boot of whonix-ws takes more time because of /home
+            # initialization, including Tor Browser copying
+            timeout = 120
         yield from asyncio.wait_for(
             vm.run_service_for_stdio(
                 'qubes.WaitForSession', input=vm.default_user.encode()),
-            timeout=30)
+            timeout=timeout)
 
 
 _templates = None
 def list_templates():
     '''Returns tuple of template names available in the system.'''
     global _templates
+    if _templates is None:
+        if 'QUBES_TEST_TEMPLATES' in os.environ:
+            _templates = os.environ['QUBES_TEST_TEMPLATES'].split()
     if _templates is None:
         try:
             app = qubes.Qubes()
             _templates = tuple(vm.name for vm in app.domains
-                if isinstance(vm, qubes.vm.templatevm.TemplateVM))
+                if isinstance(vm, qubes.vm.templatevm.TemplateVM) and
+                    vm.features.get('os', None) != 'Windows')
             app.close()
             del app
         except OSError:
@@ -1165,12 +1269,29 @@ def create_testcases_for_templates(name, *bases, module, **kwds):
 
     for template in list_templates():
         clsname = name + '_' + template
+        if hasattr(module, clsname):
+            continue
         cls = type(clsname, bases, {'template': template, **kwds})
         cls.__module__ = module.__name__
         # XXX I wonder what other __dunder__ attrs did I miss
         setattr(module, clsname, cls)
         yield '.'.join((module.__name__, clsname))
 
+def maybe_create_testcases_on_import(create_testcases_gen):
+    '''If certain conditions are met, call *create_testcases_gen* to create
+    testcases for templates tests. The purpose is to use it on integration
+    tests module(s) import, so the test runner could discover tests without
+    using load tests protocol.
+
+    The conditions - any of:
+     - QUBES_TEST_TEMPLATES present in the environment (it's possible to
+     create test cases without opening qubes.xml)
+     - QUBES_TEST_LOAD_ALL present in the environment
+    '''
+    if 'QUBES_TEST_TEMPLATES' in os.environ or \
+            'QUBES_TEST_LOAD_ALL' in os.environ:
+        list(create_testcases_gen())
+
 def extra_info(obj):
     '''Return short info identifying object.
 
@@ -1203,6 +1324,7 @@ def load_tests(loader, tests, pattern): # pylint: disable=unused-argument
             'qubes.tests.vm.init',
             'qubes.tests.storage',
             'qubes.tests.storage_file',
+            'qubes.tests.storage_reflink',
             'qubes.tests.storage_lvm',
             'qubes.tests.storage_kernels',
             'qubes.tests.ext',
@@ -1240,11 +1362,13 @@ def load_tests(loader, tests, pattern): # pylint: disable=unused-argument
             'qubes.tests.integ.basic',
             'qubes.tests.integ.storage',
             'qubes.tests.integ.pvgrub',
+            'qubes.tests.integ.devices_block',
             'qubes.tests.integ.devices_pci',
             'qubes.tests.integ.dom0_update',
             'qubes.tests.integ.network',
             'qubes.tests.integ.dispvm',
             'qubes.tests.integ.vm_qrexec_gui',
+            'qubes.tests.integ.mime',
             'qubes.tests.integ.salt',
             'qubes.tests.integ.backup',
             'qubes.tests.integ.backupcompatibility',

qubes/tests/api_admin.py

@@ -39,7 +39,7 @@ import qubes.storage
 
 # properties defined in API
 volume_properties = [
-    'pool', 'vid', 'size', 'usage', 'rw', 'source',
+    'pool', 'vid', 'size', 'usage', 'rw', 'source', 'path',
     'save_on_stop', 'snap_on_start', 'revisions_to_keep']
 
 
@@ -60,6 +60,7 @@ class AdminAPITestCase(qubes.tests.QubesTestCase):
         app = qubes.Qubes('/tmp/qubes-test.xml', load=False)
         app.vmm = unittest.mock.Mock(spec=qubes.app.VMMConnection)
         app.load_initial_values()
+        app.setup_pools()
         app.default_kernel = '1.0'
         app.default_netvm = None
         self.template = app.add_new_vm('TemplateVM', label='black',

qubes/tests/api_misc.py

@@ -131,11 +131,14 @@ class TC_00_API_Misc(qubes.tests.QubesTestCase):
         self.assertEqual(self.src.mock_calls, [
             mock.call.untrusted_qdb.read('/qubes-tools/qrexec'),
             mock.call.untrusted_qdb.read('/qubes-tools/gui'),
+            mock.call.untrusted_qdb.read('/qubes-tools/gui-emulated'),
             mock.call.untrusted_qdb.read('/qubes-tools/default-user'),
+            mock.call.untrusted_qdb.read('/qubes-tools/os'),
             mock.call.fire_event_async('features-request', untrusted_features={
                 'gui': '1',
                 'default-user': 'user',
-                'qrexec': '1'}),
+                'qrexec': '1',
+                'os': 'Linux'}),
             ('fire_event_async().__iter__', (), {}),
         ])
         self.assertEqual(self.app.mock_calls, [mock.call.save()])
@@ -153,11 +156,14 @@ class TC_00_API_Misc(qubes.tests.QubesTestCase):
         self.assertEqual(self.src.mock_calls, [
             mock.call.untrusted_qdb.read('/qubes-tools/qrexec'),
             mock.call.untrusted_qdb.read('/qubes-tools/gui'),
+            mock.call.untrusted_qdb.read('/qubes-tools/gui-emulated'),
             mock.call.untrusted_qdb.read('/qubes-tools/default-user'),
+            mock.call.untrusted_qdb.read('/qubes-tools/os'),
             mock.call.fire_event_async('features-request', untrusted_features={
                 'gui': '1',
                 'default-user': 'user',
-                'qrexec': '1'}),
+                'qrexec': '1',
+                'os': 'Linux'}),
             ('fire_event_async().__iter__', (), {}),
         ])
         self.assertEqual(self.app.mock_calls, [mock.call.save()])

qubes/tests/app.py

@@ -30,6 +30,7 @@ import qubes.events
 
 import qubes.tests
 import qubes.tests.init
+import qubes.tests.storage_reflink
 
 class TestApp(qubes.tests.TestEmitter):
     pass
@@ -264,6 +265,44 @@ class TC_30_VMCollection(qubes.tests.QubesTestCase):
 #       pass
 
 
+class TC_80_QubesInitialPools(qubes.tests.QubesTestCase):
+    def setUp(self):
+        super().setUp()
+        self.app = qubes.Qubes('/tmp/qubestest.xml', load=False,
+            offline_mode=True)
+        self.test_dir = '/var/tmp/test-varlibqubes'
+        self.test_patch = mock.patch.dict(
+            qubes.config.defaults['pool_configs']['varlibqubes'],
+            {'dir_path': self.test_dir})
+        self.test_patch.start()
+
+    def tearDown(self):
+        self.test_patch.stop()
+        self.app.close()
+        del self.app
+
+    def get_driver(self, fs_type, accessible):
+        qubes.tests.storage_reflink.mkdir_fs(self.test_dir, fs_type,
+            accessible=accessible, cleanup_via=self.addCleanup)
+        self.app.load_initial_values()
+
+        varlibqubes = self.app.pools['varlibqubes']
+        self.assertEqual(varlibqubes.dir_path, self.test_dir)
+        return varlibqubes.driver
+
+    def test_100_varlibqubes_btrfs_accessible(self):
+        self.assertEqual(self.get_driver('btrfs', True), 'file-reflink')
+
+    def test_101_varlibqubes_btrfs_inaccessible(self):
+        self.assertEqual(self.get_driver('btrfs', False), 'file')
+
+    def test_102_varlibqubes_ext4_accessible(self):
+        self.assertEqual(self.get_driver('ext4', True), 'file')
+
+    def test_103_varlibqubes_ext4_inaccessible(self):
+        self.assertEqual(self.get_driver('ext4', False), 'file')
+
+
 class TC_89_QubesEmpty(qubes.tests.QubesTestCase):
     def tearDown(self):
         try:

qubes/tests/ext.py

@@ -21,6 +21,8 @@
 from unittest import mock
 
 import qubes.ext.core_features
+import qubes.ext.services
+import qubes.ext.windows
 import qubes.tests
 
 
@@ -163,3 +165,143 @@ class TC_00_CoreFeatures(qubes.tests.QubesTestCase):
             ('features.__contains__', ('qrexec',), {}),
             ('features.__contains__', ('gui',), {}),
         ])
+
+class TC_10_WindowsFeatures(qubes.tests.QubesTestCase):
+    def setUp(self):
+        super().setUp()
+        self.ext = qubes.ext.windows.WindowsFeatures()
+        self.vm = mock.MagicMock()
+        self.features = {}
+        self.vm.configure_mock(**{
+            'features.get.side_effect': self.features.get,
+            'features.__contains__.side_effect': self.features.__contains__,
+            'features.__setitem__.side_effect': self.features.__setitem__,
+            })
+
+    def test_000_notify_tools_full(self):
+        del self.vm.template
+        self.ext.qubes_features_request(self.vm, 'features-request',
+            untrusted_features={
+                'gui': '1',
+                'version': '1',
+                'default-user': 'user',
+                'qrexec': '1',
+                'os': 'Windows'})
+        self.assertEqual(self.vm.mock_calls, [
+            ('features.__setitem__', ('os', 'Windows'), {}),
+            ('features.__setitem__', ('rpc-clipboard', True), {}),
+        ])
+
+    def test_001_notify_tools_no_qrexec(self):
+        del self.vm.template
+        self.ext.qubes_features_request(self.vm, 'features-request',
+            untrusted_features={
+                'gui': '1',
+                'version': '1',
+                'default-user': 'user',
+                'qrexec': '0',
+                'os': 'Windows'})
+        self.assertEqual(self.vm.mock_calls, [
+            ('features.__setitem__', ('os', 'Windows'), {}),
+        ])
+
+    def test_002_notify_tools_other_os(self):
+        del self.vm.template
+        self.ext.qubes_features_request(self.vm, 'features-request',
+            untrusted_features={
+                'gui': '1',
+                'version': '1',
+                'default-user': 'user',
+                'qrexec': '1',
+                'os': 'other'})
+        self.assertEqual(self.vm.mock_calls, [])
+
+class TC_20_Services(qubes.tests.QubesTestCase):
+    def setUp(self):
+        super().setUp()
+        self.ext = qubes.ext.services.ServicesExtension()
+        self.vm = mock.MagicMock()
+        self.features = {}
+        self.vm.configure_mock(**{
+            'template': None,
+            'is_running.return_value': True,
+            'features.get.side_effect': self.features.get,
+            'features.items.side_effect': self.features.items,
+            'features.__iter__.side_effect': self.features.__iter__,
+            'features.__contains__.side_effect': self.features.__contains__,
+            'features.__setitem__.side_effect': self.features.__setitem__,
+            'features.__delitem__.side_effect': self.features.__delitem__,
+            })
+
+    def test_000_write_to_qdb(self):
+        self.features['service.test1'] = '1'
+        self.features['service.test2'] = ''
+
+        self.ext.on_domain_qdb_create(self.vm, 'domain-qdb-create')
+        self.assertEqual(sorted(self.vm.untrusted_qdb.mock_calls), [
+            ('write', ('/qubes-service/test1', '1'), {}),
+            ('write', ('/qubes-service/test2', '0'), {}),
+        ])
+
+    def test_001_feature_set(self):
+        self.ext.on_domain_feature_set(self.vm,
+            'feature-set:service.test_no_oldvalue',
+            'service.test_no_oldvalue', '1')
+        self.ext.on_domain_feature_set(self.vm,
+            'feature-set:service.test_oldvalue',
+            'service.test_oldvalue', '1', '')
+        self.ext.on_domain_feature_set(self.vm,
+            'feature-set:service.test_disable',
+            'service.test_disable', '', '1')
+        self.ext.on_domain_feature_set(self.vm,
+            'feature-set:service.test_disable_no_oldvalue',
+            'service.test_disable_no_oldvalue', '')
+
+        self.assertEqual(sorted(self.vm.untrusted_qdb.mock_calls), sorted([
+            ('write', ('/qubes-service/test_no_oldvalue', '1'), {}),
+            ('write', ('/qubes-service/test_oldvalue', '1'), {}),
+            ('write', ('/qubes-service/test_disable', '0'), {}),
+            ('write', ('/qubes-service/test_disable_no_oldvalue', '0'), {}),
+        ]))
+
+    def test_002_feature_delete(self):
+        self.ext.on_domain_feature_delete(self.vm,
+            'feature-delete:service.test3', 'service.test3')
+        self.assertEqual(sorted(self.vm.untrusted_qdb.mock_calls), [
+            ('rm', ('/qubes-service/test3',), {}),
+        ])
+
+    def test_010_supported_services(self):
+        self.ext.supported_services(self.vm, 'features-request',
+            untrusted_features={
+                'supported-service.test1': '1',  # ok
+                'supported-service.test2': '0',  # ignored
+                'supported-service.test3': 'some text',  # ignored
+                'no-service': '1',  # ignored
+            })
+        self.assertEqual(self.features, {
+            'supported-service.test1': True,
+        })
+
+    def test_011_supported_services_add(self):
+        self.features['supported-service.test1'] = '1'
+        self.ext.supported_services(self.vm, 'features-request',
+            untrusted_features={
+                'supported-service.test1': '1',  # ok
+                'supported-service.test2': '1',  # ok
+            })
+        # also check if existing one is untouched
+        self.assertEqual(self.features, {
+            'supported-service.test1': '1',
+            'supported-service.test2': True,
+        })
+
+    def test_012_supported_services_remove(self):
+        self.features['supported-service.test1'] = '1'
+        self.ext.supported_services(self.vm, 'features-request',
+            untrusted_features={
+                'supported-service.test2': '1',  # ok
+            })
+        self.assertEqual(self.features, {
+            'supported-service.test2': True,
+        })

qubes/tests/extra.py

@@ -66,8 +66,9 @@ class VMWrapper(object):
     def __hash__(self):
         return hash(self._vm)
 
-    def start(self):
-        return self._loop.run_until_complete(self._vm.start())
+    def start(self, start_guid=True):
+        return self._loop.run_until_complete(
+            self._vm.start(start_guid=start_guid))
 
     def shutdown(self):
         return self._loop.run_until_complete(self._vm.shutdown())
@@ -194,7 +195,8 @@ def load_tests(loader, tests, pattern):
     for entry in pkg_resources.iter_entry_points('qubes.tests.extra'):
         try:
             for test_case in entry.load()():
-                tests.addTests(loader.loadTestsFromTestCase(test_case))
+                tests.addTests(loader.loadTestsFromNames([
+                    '{}.{}'.format(test_case.__module__, test_case.__name__)]))
         except Exception as err:  # pylint: disable=broad-except
             def runTest(self):
                 raise err
@@ -207,10 +209,10 @@ def load_tests(loader, tests, pattern):
             'qubes.tests.extra.for_template'):
         try:
             for test_case in entry.load()():
-                test.addTests(loader.loadTestsFromNames(
+                tests.addTests(loader.loadTestsFromNames(
                     qubes.tests.create_testcases_for_templates(
                         test_case.__name__, test_case,
-                        globals=sys.modules[test_case.__module__].__dict__)))
+                        module=sys.modules[test_case.__module__])))
         except Exception as err:  # pylint: disable=broad-except
             def runTest(self):
                 raise err

qubes/tests/integ/backup.py

@@ -480,16 +480,7 @@ class TC_00_Backup(BackupTestsMixin, qubes.tests.SystemTestCase):
             os.mkdir(test_dir)
             with open(os.path.join(test_dir, 'some-file.txt'), 'w') as f:
                 f.write('test file\n')
-            self.restore_backup(expect_errors=[
-                'Error restoring VM test-inst-test-net, skipping: Got empty '
-                'response from qubesd. See journalctl in dom0 for details.',
-                'Error setting test-inst-test1.netvm to test-inst-test-net: '
-                '\'"No such domain: \\\'test-inst-test-net\\\'"\'',
-            ])
-            del vms_info['test-inst-test-net']
-            vms_info['test-inst-test1']['properties']['netvm'] = \
-                str(self.app.default_netvm)
-            vms_info['test-inst-test1']['default']['netvm'] = True
+            self.restore_backup()
             self.assertCorrectlyRestored(vms_info, orig_hashes)
         finally:
             del vms
@@ -650,9 +641,14 @@ class TC_10_BackupVMMixin(BackupTestsMixin):
             del vms
 
 
+def create_testcases_for_templates():
+    return qubes.tests.create_testcases_for_templates('TC_10_BackupVM',
+        TC_10_BackupVMMixin, qubes.tests.SystemTestCase,
+        module=sys.modules[__name__])
+
 def load_tests(loader, tests, pattern):
     tests.addTests(loader.loadTestsFromNames(
-        qubes.tests.create_testcases_for_templates('TC_10_BackupVM',
-            TC_10_BackupVMMixin, qubes.tests.SystemTestCase,
-            globals=globals())))
+        create_testcases_for_templates()))
     return tests
+
+qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)

qubes/tests/integ/backupcompatibility.py

@@ -19,6 +19,7 @@
 
 from multiprocessing import Queue
 
+import asyncio
 import os
 import shutil
 import subprocess
@@ -122,7 +123,10 @@ class TC_00_BackupCompatibility(
         qubes.tests.integ.backup.BackupTestsMixin, qubes.tests.SystemTestCase):
 
     def tearDown(self):
-        self.remove_test_vms(prefix="test-")
+        prefixes = ["test-", "disp-test-"]
+        if 'disp-no-netvm' not in self.host_app.domains:
+            prefixes.append('disp-no-netvm')
+        self.remove_test_vms(prefix=prefixes)
         super(TC_00_BackupCompatibility, self).tearDown()
 
     def create_whitelisted_appmenus(self, filename):
@@ -233,7 +237,7 @@ class TC_00_BackupCompatibility(
         self.create_sparse(self.fullpath(
             "vm-templates/test-template-clone/root.img"), 10*2**30)
         self.fill_image(self.fullpath(
-            "vm-templates/test-template-clone/root.img"), 1*2**30, True)
+            "vm-templates/test-template-clone/root.img"), 100*2**20, True)
         self.create_volatile_img(self.fullpath(
             "vm-templates/test-template-clone/volatile.img"))
         subprocess.check_call([
@@ -382,7 +386,7 @@ class TC_00_BackupCompatibility(
     def assertRestored(self, name, **kwargs):
         with self.assertNotRaises((KeyError, qubes.exc.QubesException)):
             vm = self.app.domains[name]
-            vm.storage.verify()
+            asyncio.get_event_loop().run_until_complete(vm.storage.verify())
             for prop, value in kwargs.items():
                 if prop == 'klass':
                     self.assertIsInstance(vm, value)

qubes/tests/integ/basic.py

@@ -35,6 +35,8 @@ import collections
 import pkg_resources
 import shutil
 
+import sys
+
 import qubes
 import qubes.firewall
 import qubes.tests
@@ -106,10 +108,33 @@ class TC_00_Basic(qubes.tests.SystemTestCase):
         self.assertTrue(self.vm.is_running())
         # Type 'poweroff'
         subprocess.check_call(['xdotool', 'search', '--name', self.vm.name,
-                               'type', 'poweroff\r'])
-        self.loop.run_until_complete(asyncio.sleep(1))
+                               'type', '--window', '%1', 'poweroff\r'])
+        for _ in range(10):
+            if not self.vm.is_running():
+                break
+            self.loop.run_until_complete(asyncio.sleep(1))
         self.assertFalse(self.vm.is_running())
 
+    def test_130_autostart_disable_on_remove(self):
+        vm = self.app.add_new_vm(qubes.vm.appvm.AppVM,
+            name=self.make_vm_name('vm'),
+            template=self.app.default_template,
+            label='red')
+
+        self.assertIsNotNone(vm)
+        self.loop.run_until_complete(vm.create_on_disk())
+        vm.autostart = True
+        self.assertTrue(os.path.exists(
+            '/etc/systemd/system/multi-user.target.wants/'
+            'qubes-vm@{}.service'.format(vm.name)),
+            "systemd service not enabled by autostart=True")
+        del self.app.domains[vm]
+        self.loop.run_until_complete(vm.remove_from_disk())
+        self.assertFalse(os.path.exists(
+            '/etc/systemd/system/multi-user.target.wants/'
+            'qubes-vm@{}.service'.format(vm.name)),
+            "systemd service not disabled on domain remove")
+
     def _test_200_on_domain_start(self, vm, event, **_kwargs):
         '''Simulate domain crash just after startup'''
         vm.libvirt_domain.destroy()
@@ -203,8 +228,10 @@ class TC_00_Basic(qubes.tests.SystemTestCase):
         if self.test_failure_reason:
             self.fail(self.test_failure_reason)
 
+        while self.vm.get_power_state() != 'Halted':
+            self.loop.run_until_complete(asyncio.sleep(1))
         # and give a chance for both domain-shutdown handlers to execute
-        self.loop.run_until_complete(asyncio.sleep(1))
+        self.loop.run_until_complete(asyncio.sleep(3))
 
         if self.test_failure_reason:
             self.fail(self.test_failure_reason)
@@ -227,6 +254,7 @@ class TC_00_Basic(qubes.tests.SystemTestCase):
         try:
             # first boot, mkfs private volume
             self.loop.run_until_complete(vm.start())
+            self.loop.run_until_complete(self.wait_for_session(vm))
             # get private volume UUID
             private_uuid, _ = self.loop.run_until_complete(
                 vm.run_for_stdio('blkid -o value /dev/xvdb', user='root'))
@@ -422,127 +450,6 @@ class TC_01_Properties(qubes.tests.SystemTestCase):
             self.loop.run_until_complete(self.vm2.create_on_disk())
 
 
-class TC_02_QvmPrefs(qubes.tests.SystemTestCase):
-    # pylint: disable=attribute-defined-outside-init
-
-    def setUp(self):
-        super(TC_02_QvmPrefs, self).setUp()
-        self.init_default_template()
-        self.sharedopts = ['--qubesxml', qubes.tests.XMLPATH]
-
-    def setup_appvm(self):
-        self.testvm = self.app.add_new_vm(
-            qubes.vm.appvm.AppVM,
-            name=self.make_vm_name("vm"),
-            label='red')
-        self.loop.run_until_complete(self.testvm.create_on_disk())
-        self.app.save()
-
-    def setup_hvm(self):
-        self.testvm = self.app.add_new_vm(
-            qubes.vm.appvm.AppVM,
-            name=self.make_vm_name("hvm"),
-            label='red')
-        self.testvm.virt_mode = 'hvm'
-        self.loop.run_until_complete(self.testvm.create_on_disk())
-        self.app.save()
-
-    def pref_set(self, name, value, valid=True):
-        self.loop.run_until_complete(self._pref_set(name, value, valid))
-
-    @asyncio.coroutine
-    def _pref_set(self, name, value, valid=True):
-        cmd = ['qvm-prefs']
-        if value != '-D':
-            cmd.append('--')
-        cmd.extend((self.testvm.name, name, value))
-        p = yield from asyncio.create_subprocess_exec(*cmd,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE)
-        (stdout, stderr) = yield from p.communicate()
-        if valid:
-            self.assertEqual(p.returncode, 0,
-                              "qvm-prefs .. '{}' '{}' failed: {}{}".format(
-                                  name, value, stdout, stderr
-                              ))
-        else:
-            self.assertNotEquals(p.returncode, 0,
-                                 "qvm-prefs should reject value '{}' for "
-                                 "property '{}'".format(value, name))
-
-    def pref_get(self, name):
-        self.loop.run_until_complete(self._pref_get(name))
-
-    @asyncio.coroutine
-    def _pref_get(self, name):
-        p = yield from asyncio.create_subprocess_exec(
-            'qvm-prefs', *self.sharedopts, '--', self.testvm.name, name,
-            stdout=subprocess.PIPE)
-        (stdout, _) = yield from p.communicate()
-        self.assertEqual(p.returncode, 0)
-        return stdout.strip()
-
-    bool_test_values = [
-        ('true', 'True', True),
-        ('False', 'False', True),
-        ('0', 'False', True),
-        ('1', 'True', True),
-        ('invalid', '', False)
-    ]
-
-    def execute_tests(self, name, values):
-        """
-        Helper function, which executes tests for given property.
-        :param values: list of tuples (value, expected, valid),
-        where 'value' is what should be set and 'expected' is what should
-        qvm-prefs returns as a property value and 'valid' marks valid and
-        invalid values - if it's False, qvm-prefs should reject the value
-        :return: None
-        """
-        for (value, expected, valid) in values:
-            self.pref_set(name, value, valid)
-            if valid:
-                self.assertEqual(self.pref_get(name), expected)
-
-    @unittest.skip('test not converted to core3 API')
-    def test_006_template(self):
-        templates = [tpl for tpl in self.app.domains.values() if
-            isinstance(tpl, qubes.vm.templatevm.TemplateVM)]
-        if not templates:
-            self.skipTest("No templates installed")
-        some_template = templates[0].name
-        self.setup_appvm()
-        self.execute_tests('template', [
-            (some_template, some_template, True),
-            ('invalid', '', False),
-        ])
-
-    @unittest.skip('test not converted to core3 API')
-    def test_014_pcidevs(self):
-        self.setup_appvm()
-        self.execute_tests('pcidevs', [
-            ('[]', '[]', True),
-            ('[ "00:00.0" ]', "['00:00.0']", True),
-            ('invalid', '', False),
-            ('[invalid]', '', False),
-            # TODO:
-            # ('["12:12.0"]', '', False)
-        ])
-
-    @unittest.skip('test not converted to core3 API')
-    def test_024_pv_reject_hvm_props(self):
-        self.setup_appvm()
-        self.execute_tests('guiagent_installed', [('False', '', False)])
-        self.execute_tests('qrexec_installed', [('False', '', False)])
-        self.execute_tests('drive', [('/tmp/drive.img', '', False)])
-        self.execute_tests('timezone', [('localtime', '', False)])
-
-    @unittest.skip('test not converted to core3 API')
-    def test_025_hvm_reject_pv_props(self):
-        self.setup_hvm()
-        self.execute_tests('kernel', [('default', '', False)])
-        self.execute_tests('kernelopts', [('default', '', False)])
-
 class TC_03_QvmRevertTemplateChanges(qubes.tests.SystemTestCase):
     # pylint: disable=attribute-defined-outside-init
 
@@ -570,11 +477,13 @@ class TC_03_QvmRevertTemplateChanges(qubes.tests.SystemTestCase):
 
     def get_rootimg_checksum(self):
         return subprocess.check_output(
-            ['sha1sum', self.test_template.volumes['root'].path])
+            ['sha1sum', self.test_template.volumes['root'].export()]).\
+            decode().split(' ')[0]
 
     def _do_test(self):
         checksum_before = self.get_rootimg_checksum()
         self.loop.run_until_complete(self.test_template.start())
+        self.loop.run_until_complete(self.wait_for_session(self.test_template))
         self.shutdown_and_wait(self.test_template)
         checksum_changed = self.get_rootimg_checksum()
         if checksum_before == checksum_changed:
@@ -787,21 +696,27 @@ class TC_06_AppVMMixin(object):
         self.assertTrue(self.vm.is_running())
         # Type 'poweroff'
         subprocess.check_call(['xdotool', 'search', '--name', self.vm.name,
-                               'type', 'poweroff\r'])
-        self.loop.run_until_complete(asyncio.sleep(1))
+                               'type', '--window', '%1', 'poweroff\r'])
+        for _ in range(10):
+            if not self.vm.is_running():
+                break
+            self.loop.run_until_complete(asyncio.sleep(1))
         self.assertFalse(self.vm.is_running())
 
+def create_testcases_for_templates():
+    yield from qubes.tests.create_testcases_for_templates('TC_05_StandaloneVM',
+            TC_05_StandaloneVMMixin, qubes.tests.SystemTestCase,
+            module=sys.modules[__name__])
+    yield from qubes.tests.create_testcases_for_templates('TC_06_AppVM',
+            TC_06_AppVMMixin, qubes.tests.SystemTestCase,
+            module=sys.modules[__name__])
 
 def load_tests(loader, tests, pattern):
     tests.addTests(loader.loadTestsFromNames(
-        qubes.tests.create_testcases_for_templates('TC_05_StandaloneVM',
-            TC_05_StandaloneVMMixin, qubes.tests.SystemTestCase,
-            globals=globals())))
-    tests.addTests(loader.loadTestsFromNames(
-        qubes.tests.create_testcases_for_templates('TC_06_AppVM',
-            TC_06_AppVMMixin, qubes.tests.SystemTestCase,
-            globals=globals())))
+        create_testcases_for_templates()))
 
     return tests
 
+qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)
+
 # vim: ts=4 sw=4 et

tests/block.py → qubes/tests/integ/devices_block.py

@@ -2,7 +2,7 @@
 #
 # The Qubes OS Project, https://www.qubes-os.org/
 #
-# Copyright (C) 2016
+# Copyright (C) 2018
 #                   Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
 #
 # This library is free software; you can redistribute it and/or
@@ -20,42 +20,44 @@
 #
 import os
 
+import sys
+
+import qubes
 import qubes.tests
-import qubes.qubesutils
 import subprocess
 
 # the same class for both dom0 and VMs
-class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
+class TC_00_List(qubes.tests.SystemTestCase):
     template = None
 
     def setUp(self):
-        super(TC_00_List, self).setUp()
+        super().setUp()
         self.img_path = '/tmp/test.img'
         self.mount_point = '/tmp/test-dir'
         if self.template is not None:
-            self.vm = self.qc.add_new_vm(
-                "QubesAppVm",
-                name=self.make_vm_name("vm"),
-                template=self.qc.get_vm_by_name(self.template))
-            self.vm.create_on_disk(verbose=False)
+            self.vm = self.app.add_new_vm(
+                "AppVM",
+                label='red',
+                name=self.make_vm_name("vm"))
+            self.loop.run_until_complete(
+                self.vm.create_on_disk())
             self.app.save()
-            self.qc.unlock_db()
-            self.vm.start()
+            self.loop.run_until_complete(self.vm.start())
         else:
-            self.qc.unlock_db()
-            self.vm = self.qc[0]
+            self.vm = self.app.domains[0]
 
     def tearDown(self):
-        super(TC_00_List, self).tearDown()
+        super().tearDown()
         if self.template is None:
             if os.path.exists(self.mount_point):
                 subprocess.call(['sudo', 'umount', self.mount_point])
                 subprocess.call(['sudo', 'rmdir', self.mount_point])
-            subprocess.call(['sudo', 'dmsetup', 'remove', 'test-dm'])
+            if os.path.exists('/dev/mapper/test-dm'):
+                subprocess.call(['sudo', 'dmsetup', 'remove', 'test-dm'])
             if os.path.exists(self.img_path):
                 loopdev = subprocess.check_output(['losetup', '-j',
                     self.img_path])
-                for dev in loopdev.splitlines():
+                for dev in loopdev.decode().splitlines():
                     subprocess.call(
                         ['sudo', 'losetup', '-d', dev.split(':')[0]])
                 subprocess.call(['sudo', 'rm', '-f', self.img_path])
@@ -67,9 +69,8 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
             elif user == "root":
                 subprocess.check_call(['sudo', 'sh', '-c', script])
         else:
-            retcode = self.vm.run(script, user=user, wait=True)
-            if retcode != 0:
-                raise subprocess.CalledProcessError
+            self.loop.run_until_complete(
+                self.vm.run_for_stdio(script, user=user))
 
     def test_000_list_loop(self):
         if self.template is None:
@@ -80,19 +81,18 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
             "losetup -f {path}; "
             "udevadm settle".format(path=self.img_path), user="root")
 
-        dev_list = qubes.qubesutils.block_list_vm(self.vm)
+        dev_list = list(self.vm.devices['block'])
         found = False
-        for dev in dev_list.keys():
-            if dev_list[dev]['desc'] == self.img_path:
-                self.assertTrue(dev.startswith(self.vm.name + ':loop'))
-                self.assertEquals(dev_list[dev]['mode'], 'w')
-                self.assertEquals(dev_list[dev]['size'], 1024 * 1024 * 128)
-                self.assertEquals(
-                    dev_list[dev]['device'], '/dev/' + dev.split(':')[1])
+        for dev in dev_list:
+            if dev.description == self.img_path:
+                self.assertTrue(dev.ident.startswith('loop'))
+                self.assertEquals(dev.mode, 'w')
+                self.assertEquals(dev.size, 1024 * 1024 * 128)
                 found = True
 
         if not found:
-            self.fail("Device {} not found in {!r}".format(self.img_path, dev_list))
+            self.fail("Device {} not found in {!r}".format(
+                self.img_path, dev_list))
 
     def test_001_list_loop_mounted(self):
         if self.template is None:
@@ -108,9 +108,9 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
                 mntdir=self.mount_point),
             user="root")
 
-        dev_list = qubes.qubesutils.block_list_vm(self.vm)
-        for dev in dev_list.keys():
-            if dev_list[dev]['desc'] == self.img_path:
+        dev_list = list(self.vm.devices['block'])
+        for dev in dev_list:
+            if dev.description == self.img_path:
                 self.fail(
                     'Device {} ({}) should not be listed because is mounted'
                     .format(dev, self.img_path))
@@ -125,19 +125,17 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
             "/sys/block/$(basename $loopdev)/dev) 0\";"
             "udevadm settle".format(path=self.img_path), user="root")
 
-        dev_list = qubes.qubesutils.block_list_vm(self.vm)
+        dev_list = list(self.vm.devices['block'])
         found = False
-        for dev in dev_list.keys():
-            if dev.startswith(self.vm.name + ':loop'):
-                self.assertNotEquals(dev_list[dev]['desc'], self.img_path,
+        for dev in dev_list:
+            if dev.ident.startswith('loop'):
+                self.assertNotEquals(dev.description, self.img_path,
                     "Device {} ({}) should not be listed as it is used in "
                     "device-mapper".format(dev, self.img_path)
                 )
-            elif dev_list[dev]['desc'] == 'test-dm':
-                self.assertEquals(dev_list[dev]['mode'], 'w')
-                self.assertEquals(dev_list[dev]['size'], 1024 * 1024 * 128)
-                self.assertEquals(
-                    dev_list[dev]['device'], '/dev/' + dev.split(':')[1])
+            elif dev.description == 'test-dm':
+                self.assertEquals(dev.mode, 'w')
+                self.assertEquals(dev.size, 1024 * 1024 * 128)
                 found = True
 
         if not found:
@@ -159,15 +157,15 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
                 mntdir=self.mount_point),
             user="root")
 
-        dev_list = qubes.qubesutils.block_list_vm(self.vm)
-        for dev in dev_list.keys():
-            if dev.startswith(self.vm.name + ':loop'):
-                self.assertNotEquals(dev_list[dev]['desc'], self.img_path,
+        dev_list = list(self.vm.devices['block'])
+        for dev in dev_list:
+            if dev.ident.startswith('loop'):
+                self.assertNotEquals(dev.description, self.img_path,
                     "Device {} ({}) should not be listed as it is used in "
                     "device-mapper".format(dev, self.img_path)
                 )
             else:
-                self.assertNotEquals(dev_list[dev]['desc'], 'test-dm',
+                self.assertNotEquals(dev.description, 'test-dm',
                     "Device {} ({}) should not be listed as it is "
                     "mounted".format(dev, 'test-dm')
                 )
@@ -183,19 +181,17 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
             "/sys/block/$(basename $loopdev)/dev) 0\";"
             "udevadm settle".format(path=self.img_path), user="root")
 
-        dev_list = qubes.qubesutils.block_list_vm(self.vm)
+        dev_list = list(self.vm.devices['block'])
         found = False
-        for dev in dev_list.keys():
-            if dev.startswith(self.vm.name + ':loop'):
-                self.assertNotEquals(dev_list[dev]['desc'], self.img_path,
+        for dev in dev_list:
+            if dev.ident.startswith('loop'):
+                self.assertNotEquals(dev.description, self.img_path,
                     "Device {} ({}) should not be listed as it is used in "
                     "device-mapper".format(dev, self.img_path)
                 )
-            elif dev_list[dev]['desc'] == 'test-dm':
-                self.assertEquals(dev_list[dev]['mode'], 'w')
-                self.assertEquals(dev_list[dev]['size'], 1024 * 1024 * 128)
-                self.assertEquals(
-                    dev_list[dev]['device'], '/dev/' + dev.split(':')[1])
+            elif dev.description == 'test-dm':
+                self.assertEquals(dev.mode, 'w')
+                self.assertEquals(dev.size, 1024 * 1024 * 128)
                 found = True
 
         if not found:
@@ -216,15 +212,13 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
             "dmsetup remove test-dm;"
             "udevadm settle".format(path=self.img_path), user="root")
 
-        dev_list = qubes.qubesutils.block_list_vm(self.vm)
+        dev_list = list(self.vm.devices['block'])
         found = False
-        for dev in dev_list.keys():
-            if dev_list[dev]['desc'] == self.img_path:
-                self.assertTrue(dev.startswith(self.vm.name + ':loop'))
-                self.assertEquals(dev_list[dev]['mode'], 'w')
-                self.assertEquals(dev_list[dev]['size'], 1024 * 1024 * 128)
-                self.assertEquals(
-                    dev_list[dev]['device'], '/dev/' + dev.split(':')[1])
+        for dev in dev_list:
+            if dev.description == self.img_path:
+                self.assertTrue(dev.ident.startswith('loop'))
+                self.assertEquals(dev.mode, 'w')
+                self.assertEquals(dev.size, 1024 * 1024 * 128)
                 found = True
 
         if not found:
@@ -242,16 +236,14 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
             "blockdev --rereadpt $loopdev; "
             "udevadm settle".format(path=self.img_path), user="root")
 
-        dev_list = qubes.qubesutils.block_list_vm(self.vm)
+        dev_list = list(self.vm.devices['block'])
         found = False
-        for dev in dev_list.keys():
-            if dev_list[dev]['desc'] == self.img_path:
-                self.assertTrue(dev.startswith(self.vm.name + ':loop'))
-                self.assertEquals(dev_list[dev]['mode'], 'w')
-                self.assertEquals(dev_list[dev]['size'], 1024 * 1024 * 128)
-                self.assertEquals(
-                    dev_list[dev]['device'], '/dev/' + dev.split(':')[1])
-                self.assertIn(dev + 'p1', dev_list)
+        for dev in dev_list:
+            if dev.description == self.img_path:
+                self.assertTrue(dev.ident.startswith('loop'))
+                self.assertEquals(dev.mode, 'w')
+                self.assertEquals(dev.size, 1024 * 1024 * 128)
+                self.assertIn(dev.ident + 'p1', [d.ident for d in dev_list])
                 found = True
 
         if not found:
@@ -274,14 +266,14 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
                 path=self.img_path, mntdir=self.mount_point),
             user="root")
 
-        dev_list = qubes.qubesutils.block_list_vm(self.vm)
-        for dev in dev_list.keys():
-            if dev_list[dev]['desc'] == self.img_path:
+        dev_list = list(self.vm.devices['block'])
+        for dev in dev_list:
+            if dev.description == self.img_path:
                 self.fail(
                     'Device {} ({}) should not be listed because its '
                     'partition is mounted'
                     .format(dev, self.img_path))
-            elif dev.startswith(self.vm.name + ':loop') and dev.endswith('p1'):
+            elif dev.ident.startswith('loop') and dev.ident.endswith('p1'):
                 # FIXME: risky assumption that only tests create partitioned
                 # loop devices
                 self.fail(
@@ -289,21 +281,14 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
                     .format(dev, self.img_path))
 
 
-def load_tests(loader, tests, pattern):
-    try:
-        qc = qubes.qubes.QubesVmCollection()
-        qc.lock_db_for_reading()
-        qc.load()
-        qc.unlock_db()
-        templates = [vm.name for vm in qc.values() if
-                     isinstance(vm, qubes.qubes.QubesTemplateVm)]
-    except OSError:
-        templates = []
-    for template in templates:
-        tests.addTests(loader.loadTestsFromTestCase(
-            type(
-                'TC_00_List_' + template,
-                (TC_00_List, qubes.tests.QubesTestCase),
-                {'template': template})))
+def create_testcases_for_templates():
+    return qubes.tests.create_testcases_for_templates('TC_00_List',
+        TC_00_List,
+        module=sys.modules[__name__])
 
+def load_tests(loader, tests, pattern):
+    tests.addTests(loader.loadTestsFromNames(
+        create_testcases_for_templates()))
     return tests
+
+qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)

qubes/tests/integ/dispvm.py

@@ -23,11 +23,14 @@ import subprocess
 import tempfile
 import time
 import unittest
+from contextlib import suppress
 
 from distutils import spawn
 
 import asyncio
 
+import sys
+
 import qubes.tests
 
 class TC_04_DispVM(qubes.tests.SystemTestCase):
@@ -67,7 +70,7 @@ class TC_04_DispVM(qubes.tests.SystemTestCase):
         self.assertEqual(lines[0], "test")
         dispvm_name = lines[1]
         # wait for actual DispVM destruction
-        self.loop.run_until_complete(asyncio.sleep(1))
+        self.loop.run_until_complete(asyncio.sleep(5))
         self.assertNotIn(dispvm_name, self.app.domains)
 
     def test_003_cleanup_destroyed(self):
@@ -86,7 +89,7 @@ class TC_04_DispVM(qubes.tests.SystemTestCase):
         p.stdin.write(b"sudo poweroff\n")
         # do not close p.stdin on purpose - wait to automatic disconnect when
         #  domain is destroyed
-        timeout = 30
+        timeout = 70
         lines_task = asyncio.ensure_future(p.stdout.read())
         self.loop.run_until_complete(asyncio.wait_for(p.wait(), timeout))
         self.loop.run_until_complete(lines_task)
@@ -160,8 +163,15 @@ class TC_20_DispVMMixin(object):
                 self.enter_keys_in_window(window_title, ['Return'])
                 # Wait for window to close
                 self.wait_for_window(window_title, show=False)
-            finally:
                 p.stdin.close()
+                self.loop.run_until_complete(
+                    asyncio.wait_for(p.wait(), 30))
+            except:
+                with suppress(ProcessLookupError):
+                    p.terminate()
+                self.loop.run_until_complete(p.wait())
+                raise
+            finally:
                 del p
         finally:
             self.loop.run_until_complete(dispvm.cleanup())
@@ -169,7 +179,7 @@ class TC_20_DispVMMixin(object):
             del dispvm
 
         # give it a time for shutdown + cleanup
-        self.loop.run_until_complete(asyncio.sleep(2))
+        self.loop.run_until_complete(asyncio.sleep(5))
 
         self.assertNotIn(dispvm_name, self.app.domains,
                           "DispVM not removed from qubes.xml")
@@ -181,7 +191,7 @@ class TC_20_DispVMMixin(object):
         window_title = window_title.decode().strip().\
             replace('(', '\(').replace(')', '\)')
         time.sleep(1)
-        if "gedit" in window_title:
+        if "gedit" in window_title or 'KWrite' in window_title:
             subprocess.check_call(['xdotool', 'windowactivate', '--sync', winid,
                                    'type', 'Test test 2'])
             subprocess.check_call(['xdotool', 'key', '--window', winid,
@@ -243,37 +253,44 @@ class TC_20_DispVMMixin(object):
             self.testvm1.run_for_stdio("echo test1 > /home/user/test.txt"))
 
         p = self.loop.run_until_complete(
-            self.testvm1.run("qvm-open-in-dvm /home/user/test.txt"))
+            self.testvm1.run("qvm-open-in-dvm /home/user/test.txt",
+            stdout=subprocess.PIPE, stderr=subprocess.STDOUT))
 
-        wait_count = 0
+        # if first 5 windows isn't expected editor, there is no hope
         winid = None
-        while True:
-            search = self.loop.run_until_complete(
-                asyncio.create_subprocess_exec(
-                    'xdotool', 'search', '--onlyvisible', '--class', 'disp*',
-                    stdout=subprocess.PIPE,
-                    stderr=subprocess.DEVNULL))
-            stdout, _ = self.loop.run_until_complete(search.communicate())
-            if search.returncode == 0:
-                winid = stdout.strip()
-                # get window title
-                (window_title, _) = subprocess.Popen(
-                    ['xdotool', 'getwindowname', winid], stdout=subprocess.PIPE). \
-                    communicate()
-                window_title = window_title.decode().strip()
-                # ignore LibreOffice splash screen and window with no title
-                # set yet
-                if window_title and not window_title.startswith("LibreOffice")\
-                        and not window_title == 'VMapp command':
-                    break
-            wait_count += 1
-            if wait_count > 100:
-                self.fail("Timeout while waiting for editor window")
-            self.loop.run_until_complete(asyncio.sleep(0.3))
+        for _ in range(5):
+            try:
+                winid = self.wait_for_window('disp[0-9]*', search_class=True)
+            except Exception as e:
+                try:
+                    self.loop.run_until_complete(asyncio.wait_for(p.wait(), 1))
+                except asyncio.TimeoutError:
+                    raise e
+                else:
+                    stdout = self.loop.run_until_complete(p.stdout.read())
+                    self.fail(
+                        'qvm-open-in-dvm exited prematurely with {}: {}'.format(
+                            p.returncode, stdout))
+            # get window title
+            (window_title, _) = subprocess.Popen(
+                ['xdotool', 'getwindowname', winid], stdout=subprocess.PIPE). \
+                communicate()
+            window_title = window_title.decode().strip()
+            # ignore LibreOffice splash screen and window with no title
+            # set yet
+            if window_title and not window_title.startswith("LibreOffice")\
+                    and not window_title == 'VMapp command' \
+                    and 'whonixcheck' not in window_title \
+                    and not window_title == 'NetworkManager Applet':
+                break
+            self.loop.run_until_complete(asyncio.sleep(1))
+            winid = None
+        if winid is None:
+            self.fail('Timeout waiting for editor window')
 
         time.sleep(0.5)
         self._handle_editor(winid)
-        self.loop.run_until_complete(p.wait())
+        self.loop.run_until_complete(p.communicate())
         (test_txt_content, _) = self.loop.run_until_complete(
             self.testvm1.run_for_stdio("cat /home/user/test.txt"))
         # Drop BOM if added by editor
@@ -281,9 +298,15 @@ class TC_20_DispVMMixin(object):
             test_txt_content = test_txt_content[3:]
         self.assertEqual(test_txt_content, b"Test test 2\ntest1\n")
 
+
+def create_testcases_for_templates():
+    return qubes.tests.create_testcases_for_templates('TC_20_DispVM',
+        TC_20_DispVMMixin, qubes.tests.SystemTestCase,
+        module=sys.modules[__name__])
+
 def load_tests(loader, tests, pattern):
     tests.addTests(loader.loadTestsFromNames(
-        qubes.tests.create_testcases_for_templates('TC_20_DispVM',
-            TC_20_DispVMMixin, qubes.tests.SystemTestCase,
-            globals=globals())))
+        create_testcases_for_templates()))
     return tests
+
+qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)

qubes/tests/integ/dom0_update.py

@@ -78,23 +78,18 @@ Expire-Date: 0
 
         cls.keyid = cls.generate_key(cls.tmpdir)
 
-        p = subprocess.Popen(['sudo', 'dd',
-                              'status=none', 'of=/etc/yum.repos.d/test.repo'],
-                             stdin=subprocess.PIPE)
-        p.stdin.write(b'''
+        with open('/etc/yum.repos.d/test.repo', 'w') as repo_file:
+            repo_file.write('''
 [test]
 name = Test
 baseurl = http://localhost:8080/
 enabled = 1
 ''')
-        p.stdin.close()
-        p.wait()
 
 
     @classmethod
     def tearDownClass(cls):
-        subprocess.check_call(['sudo', 'rm', '-f',
-                               '/etc/yum.repos.d/test.repo'])
+        os.unlink('/etc/yum.repos.d/test.repo')
 
         shutil.rmtree(cls.tmpdir)
 
@@ -113,9 +108,9 @@ enabled = 1
         self.loop.run_until_complete(self.updatevm.create_on_disk())
         self.app.updatevm = self.updatevm
         self.app.save()
-        subprocess.call(['sudo', 'rpm', '-e', self.pkg_name],
+        subprocess.call(['rpm', '-e', self.pkg_name],
                         stderr=subprocess.DEVNULL)
-        subprocess.check_call(['sudo', 'rpm', '--import',
+        subprocess.check_call(['rpm', '--import',
                                os.path.join(self.tmpdir, 'pubkey.asc')])
         self.loop.run_until_complete(self.updatevm.start())
         self.repo_running = False
@@ -126,11 +121,12 @@ enabled = 1
             self.repo_proc.terminate()
             self.loop.run_until_complete(self.repo_proc.wait())
             del self.repo_proc
+        self.app.updatevm = None
         super(TC_00_Dom0UpgradeMixin, self).tearDown()
 
-        subprocess.call(['sudo', 'rpm', '-e', self.pkg_name],
+        subprocess.call(['rpm', '-e', self.pkg_name],
             stderr=subprocess.DEVNULL)
-        subprocess.call(['sudo', 'rpm', '-e', 'gpg-pubkey-{}'.format(
+        subprocess.call(['rpm', '-e', 'gpg-pubkey-{}'.format(
             self.keyid)], stderr=subprocess.DEVNULL)
 
         for pkg in os.listdir(self.tmpdir):
@@ -165,7 +161,7 @@ Test package
              spec_path])
         pkg_path = os.path.join(dir, 'x86_64',
                                 '{}-{}-1.x86_64.rpm'.format(name, version))
-        subprocess.check_call(['sudo', 'chmod', 'go-rw', '/dev/tty'])
+        subprocess.check_call(['chmod', 'go-rw', '/dev/tty'])
         subprocess.check_call(
             ['rpm', '--quiet', '--define=_gpg_path {}'.format(dir),
              '--define=_gpg_name {}'.format("Qubes test"),
@@ -173,7 +169,7 @@ Test package
             stdin=subprocess.DEVNULL,
             stdout=subprocess.DEVNULL,
             stderr=subprocess.STDOUT)
-        subprocess.check_call(['sudo', 'chmod', 'go+rw', '/dev/tty'])
+        subprocess.check_call(['chmod', 'go+rw', '/dev/tty'])
         return pkg_path
 
     def send_pkg(self, filename):
@@ -212,7 +208,7 @@ Test package
          - "updates pending" flag is cleared
         """
         filename = self.create_pkg(self.tmpdir, self.pkg_name, '1.0')
-        subprocess.check_call(['sudo', 'rpm', '-i', filename])
+        subprocess.check_call(['rpm', '-i', filename])
         filename = self.create_pkg(self.tmpdir, self.pkg_name, '2.0')
         self.send_pkg(filename)
         open(self.update_flag_path, 'a').close()
@@ -331,7 +327,7 @@ Test package
             self.pkg_name))
 
     def test_020_install_wrong_sign(self):
-        subprocess.call(['sudo', 'rpm', '-e', 'gpg-pubkey-{}'.format(
+        subprocess.call(['rpm', '-e', 'gpg-pubkey-{}'.format(
             self.keyid)])
         filename = self.create_pkg(self.tmpdir, self.pkg_name, '1.0')
         self.send_pkg(filename)
@@ -385,9 +381,14 @@ Test package
                          'UNSIGNED package {}-1.0 installed'.format(self.pkg_name))
 
 
+def create_testcases_for_templates():
+    return qubes.tests.create_testcases_for_templates('TC_00_Dom0Upgrade',
+        TC_00_Dom0UpgradeMixin, qubes.tests.SystemTestCase,
+        module=sys.modules[__name__])
+
 def load_tests(loader, tests, pattern):
     tests.addTests(loader.loadTestsFromNames(
-        qubes.tests.create_testcases_for_templates('TC_00_Dom0Upgrade',
-            TC_00_Dom0UpgradeMixin, qubes.tests.SystemTestCase,
-            module=sys.modules[__name__])))
+        create_testcases_for_templates()))
     return tests
+
+qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)

tests/mime.py → qubes/tests/integ/mime.py

@@ -20,113 +20,93 @@
 # License along with this library; if not, see <https://www.gnu.org/licenses/>.
 #
 #
-from distutils import spawn
 import os
+from distutils import spawn
 import re
 import subprocess
 import time
 import unittest
 
+import itertools
+
+import asyncio
+
+import sys
+
 import qubes.tests
-import qubes.qubes
-from qubes.qubes import QubesVmCollection
+import qubes
 
 @unittest.skipUnless(
     spawn.find_executable('xprop') and
     spawn.find_executable('xdotool') and
     spawn.find_executable('wmctrl'),
     "xprop or xdotool or wmctrl not installed")
-class TC_50_MimeHandlers(qubes.tests.SystemTestsMixin):
-    @classmethod
-    def setUpClass(cls):
-        if cls.template == 'whonix-gw' or 'minimal' in cls.template:
-            raise unittest.SkipTest(
-                'Template {} not supported by this test'.format(cls.template))
-
-        if cls.template == 'whonix-ws':
-            # TODO remove when Whonix-based DispVMs will work (Whonix 13?)
+class TC_50_MimeHandlers:
+    def setUp(self):
+        super(TC_50_MimeHandlers, self).setUp()
+        if self.template.startswith('whonix-gw') or 'minimal' in self.template:
             raise unittest.SkipTest(
-                'Template {} not supported by this test'.format(cls.template))
-
-        qc = QubesVmCollection()
-
-        cls._kill_test_vms(qc, prefix=qubes.tests.CLSVMPREFIX)
-
-        qc.lock_db_for_writing()
-        qc.load()
+                'Template {} not supported by this test'.format(self.template))
 
-        cls._remove_test_vms(qc, qubes.qubes.vmm.libvirt_conn,
-                            prefix=qubes.tests.CLSVMPREFIX)
+        self.source_vmname = self.make_vm_name('source')
+        self.source_vm = self.app.add_new_vm("AppVM",
+            template=self.template,
+            name=self.source_vmname,
+            label='red')
+        self.loop.run_until_complete(self.source_vm.create_on_disk())
 
-        cls.source_vmname = cls.make_vm_name('source', True)
-        source_vm = qc.add_new_vm("QubesAppVm",
-                                  template=qc.get_vm_by_name(cls.template),
-                                  name=cls.source_vmname)
-        source_vm.create_on_disk(verbose=False)
+        self.target_vmname = self.make_vm_name('target')
+        self.target_vm = self.app.add_new_vm("AppVM",
+            template=self.template,
+            name=self.target_vmname,
+            label='red')
+        self.loop.run_until_complete(self.target_vm.create_on_disk())
 
-        cls.target_vmname = cls.make_vm_name('target', True)
-        target_vm = qc.add_new_vm("QubesAppVm",
-                                  template=qc.get_vm_by_name(cls.template),
-                                  name=cls.target_vmname)
-        target_vm.create_on_disk(verbose=False)
+        self.target_vm.template_for_dispvms = True
+        self.source_vm.default_dispvm = self.target_vm
 
-        qc.save()
-        qc.unlock_db()
-        source_vm.start()
-        target_vm.start()
+        done, not_done = self.loop.run_until_complete(asyncio.wait([
+            self.source_vm.start(),
+            self.target_vm.start()]))
+        for result in itertools.chain(done, not_done):
+            # catch any exceptions
+            result.result()
 
-        # make sure that DispVMs will be started of the same template
-        retcode = subprocess.call(['/usr/bin/qvm-create-default-dvm',
-                                   cls.template],
-                                  stderr=open(os.devnull, 'w'))
-        assert retcode == 0, "Error preparing DispVM"
-
-    def setUp(self):
-        super(TC_50_MimeHandlers, self).setUp()
-        self.source_vm = self.qc.get_vm_by_name(self.source_vmname)
-        self.target_vm = self.qc.get_vm_by_name(self.target_vmname)
 
     def get_window_class(self, winid, dispvm=False):
         (vm_winid, _) = subprocess.Popen(
             ['xprop', '-id', winid, '_QUBES_VMWINDOWID'],
             stdout=subprocess.PIPE
         ).communicate()
-        vm_winid = vm_winid.split("#")[1].strip('\n" ')
+        vm_winid = vm_winid.decode().split("#")[1].strip('\n" ')
         if dispvm:
             (vmname, _) = subprocess.Popen(
                 ['xprop', '-id', winid, '_QUBES_VMNAME'],
                 stdout=subprocess.PIPE
             ).communicate()
-            vmname = vmname.split("=")[1].strip('\n" ')
-            window_class = None
-            while window_class is None:
-                # XXX to use self.qc.get_vm_by_name would require reloading
-                # qubes.xml, so use qvm-run instead
-                xprop = subprocess.Popen(
-                    ['qvm-run', '-p', vmname, 'xprop -id {} WM_CLASS'.format(
-                        vm_winid)], stdout=subprocess.PIPE)
-                (window_class, _) = xprop.communicate()
-                if xprop.returncode != 0:
-                    self.skipTest("xprop failed, not installed?")
-                if 'not found' in window_class:
-                    # WM_CLASS not set yet, wait a little
-                    time.sleep(0.1)
-                    window_class = None
+            vmname = vmname.decode().split("=")[1].strip('\n" ')
+            vm = self.app.domains[vmname]
         else:
-            window_class = None
-            while window_class is None:
-                xprop = self.target_vm.run(
-                    'xprop -id {} WM_CLASS'.format(vm_winid),
-                    passio_popen=True)
-                (window_class, _) = xprop.communicate()
-                if xprop.returncode != 0:
-                    self.skipTest("xprop failed, not installed?")
-                if 'not found' in window_class:
-                    # WM_CLASS not set yet, wait a little
-                    time.sleep(0.1)
-                    window_class = None
+            vm = self.target_vm
+        window_class = None
+        while window_class is None:
+            try:
+                window_class, _ = self.loop.run_until_complete(
+                    vm.run_for_stdio('xprop -id {} WM_CLASS'.format(vm_winid)))
+            except subprocess.CalledProcessError as e:
+                if e.returncode == 127:
+                    self.skipTest('xprop not installed')
+                self.fail(
+                    "xprop -id {} WM_CLASS failed: {}".format(
+                        vm_winid, e.stderr.decode()))
+            if b'not found' in window_class:
+                # WM_CLASS not set yet, wait a little
+                time.sleep(0.1)
+                window_class = None
+
         # output: WM_CLASS(STRING) = "gnome-terminal-server", "Gnome-terminal"
         try:
+            window_class = window_class.decode()
             window_class = window_class.split("=")[1].split(",")[0].strip('\n" ')
         except IndexError:
             raise Exception(
@@ -136,44 +116,45 @@ class TC_50_MimeHandlers(qubes.tests.SystemTestsMixin):
 
     def open_file_and_check_viewer(self, filename, expected_app_titles,
                                    expected_app_classes, dispvm=False):
-        self.qc.unlock_db()
         if dispvm:
-            p = self.source_vm.run("qvm-open-in-dvm {}".format(filename),
-                                   passio_popen=True)
-            vmpattern = "disp*"
+            p = self.loop.run_until_complete(self.source_vm.run(
+                "qvm-open-in-dvm {}".format(filename), stdout=subprocess.PIPE))
+            vmpattern = "disp[0-9]*"
         else:
-            self.qrexec_policy('qubes.OpenInVM', self.source_vm.name,
-                self.target_vmname)
-            self.qrexec_policy('qubes.OpenURL', self.source_vm.name,
-                self.target_vmname)
-            p = self.source_vm.run("qvm-open-in-vm {} {}".format(
-                self.target_vmname, filename), passio_popen=True)
+            p = self.loop.run_until_complete(self.source_vm.run(
+                "qvm-open-in-vm {} {}".format(self.target_vmname, filename),
+                stdout=subprocess.PIPE))
             vmpattern = self.target_vmname
         wait_count = 0
         winid = None
-        window_title = None
-        while True:
-            search = subprocess.Popen(['xdotool', 'search',
-                                       '--onlyvisible', '--class', vmpattern],
-                                      stdout=subprocess.PIPE,
-                                      stderr=open(os.path.devnull, 'w'))
-            retcode = search.wait()
-            if retcode == 0:
-                winid = search.stdout.read().strip()
-                # get window title
-                (window_title, _) = subprocess.Popen(
-                    ['xdotool', 'getwindowname', winid], stdout=subprocess.PIPE). \
-                    communicate()
-                window_title = window_title.strip()
-                # ignore LibreOffice splash screen and window with no title
-                # set yet
-                if window_title and not window_title.startswith("LibreOffice")\
-                        and not window_title == 'VMapp command':
-                    break
-            wait_count += 1
-            if wait_count > 100:
-                self.fail("Timeout while waiting for editor window")
-            time.sleep(0.3)
+        with self.qrexec_policy('qubes.OpenInVM', self.source_vm.name,
+                self.target_vmname):
+            with self.qrexec_policy('qubes.OpenURL', self.source_vm.name,
+                    self.target_vmname):
+                while True:
+                    search = subprocess.Popen(['xdotool', 'search',
+                                               '--onlyvisible', '--class', vmpattern],
+                                              stdout=subprocess.PIPE,
+                                              stderr=subprocess.DEVNULL)
+                    retcode = search.wait()
+                    if retcode == 0:
+                        winid = search.stdout.read().strip()
+                        # get window title
+                        (window_title, _) = subprocess.Popen(
+                            ['xdotool', 'getwindowname', winid], stdout=subprocess.PIPE). \
+                            communicate()
+                        window_title = window_title.decode('utf8').strip()
+                        # ignore LibreOffice splash screen and window with no title
+                        # set yet
+                        if window_title and \
+                                not window_title.startswith("LibreOffice") and\
+                                not window_title.startswith("NetworkManager") and\
+                                not window_title == 'VMapp command':
+                            break
+                    wait_count += 1
+                    if wait_count > 100:
+                        self.fail("Timeout while waiting for editor window")
+                    self.loop.run_until_complete(asyncio.sleep(0.3))
 
         # get window class
         window_class = self.get_window_class(winid, dispvm)
@@ -194,45 +175,66 @@ class TC_50_MimeHandlers(qubes.tests.SystemTestsMixin):
                           expected_app_titles, expected_app_classes))
 
     def prepare_txt(self, filename):
-        p = self.source_vm.run("cat > {}".format(filename), passio_popen=True)
-        p.stdin.write("This is test\n")
-        p.stdin.close()
-        retcode = p.wait()
-        assert retcode == 0, "Failed to write {} file".format(filename)
+        self.loop.run_until_complete(
+            self.source_vm.run_for_stdio("cat > {}".format(filename),
+            input=b'This is test\n'))
 
     def prepare_pdf(self, filename):
         self.prepare_txt("/tmp/source.txt")
-        cmd = "convert /tmp/source.txt {}".format(filename)
-        retcode = self.source_vm.run(cmd, wait=True)
-        assert retcode == 0, "Failed to run '{}'".format(cmd)
+        cmd = "convert text:/tmp/source.txt {}".format(filename)
+        try:
+            self.loop.run_until_complete(
+                self.source_vm.run_for_stdio(cmd))
+        except subprocess.CalledProcessError as e:
+            self.fail('{} failed: {}'.format(cmd, e.stderr.decode()))
 
     def prepare_doc(self, filename):
         self.prepare_txt("/tmp/source.txt")
         cmd = "unoconv -f doc -o {} /tmp/source.txt".format(filename)
-        retcode = self.source_vm.run(cmd, wait=True)
-        if retcode != 0:
-            self.skipTest("Failed to run '{}', not installed?".format(cmd))
+        try:
+            self.loop.run_until_complete(
+                self.source_vm.run_for_stdio(cmd))
+        except subprocess.CalledProcessError as e:
+            if e.returncode == 127:
+                self.skipTest("unoconv not installed".format(cmd))
+            self.skipTest("Failed to run '{}': {}".format(cmd,
+                e.stderr.decode()))
 
     def prepare_pptx(self, filename):
         self.prepare_txt("/tmp/source.txt")
         cmd = "unoconv -f pptx -o {} /tmp/source.txt".format(filename)
-        retcode = self.source_vm.run(cmd, wait=True)
-        if retcode != 0:
-            self.skipTest("Failed to run '{}', not installed?".format(cmd))
+        try:
+            self.loop.run_until_complete(
+                self.source_vm.run_for_stdio(cmd))
+        except subprocess.CalledProcessError as e:
+            if e.returncode == 127:
+                self.skipTest("unoconv not installed".format(cmd))
+            self.skipTest("Failed to run '{}': {}".format(cmd,
+                e.stderr.decode()))
 
     def prepare_png(self, filename):
         self.prepare_txt("/tmp/source.txt")
-        cmd = "convert /tmp/source.txt {}".format(filename)
-        retcode = self.source_vm.run(cmd, wait=True)
-        if retcode != 0:
-            self.skipTest("Failed to run '{}', not installed?".format(cmd))
+        cmd = "convert text:/tmp/source.txt {}".format(filename)
+        try:
+            self.loop.run_until_complete(
+                self.source_vm.run_for_stdio(cmd))
+        except subprocess.CalledProcessError as e:
+            if e.returncode == 127:
+                self.skipTest("convert not installed".format(cmd))
+            self.skipTest("Failed to run '{}': {}".format(cmd,
+                e.stderr.decode()))
 
     def prepare_jpg(self, filename):
         self.prepare_txt("/tmp/source.txt")
-        cmd = "convert /tmp/source.txt {}".format(filename)
-        retcode = self.source_vm.run(cmd, wait=True)
-        if retcode != 0:
-            self.skipTest("Failed to run '{}', not installed?".format(cmd))
+        cmd = "convert text:/tmp/source.txt {}".format(filename)
+        try:
+            self.loop.run_until_complete(
+                self.source_vm.run_for_stdio(cmd))
+        except subprocess.CalledProcessError as e:
+            if e.returncode == 127:
+                self.skipTest("convert not installed".format(cmd))
+            self.skipTest("Failed to run '{}': {}".format(cmd,
+                e.stderr.decode()))
 
     def test_000_txt(self):
         filename = "/home/user/test_file.txt"
@@ -334,20 +336,14 @@ class TC_50_MimeHandlers(qubes.tests.SystemTestsMixin):
                                         ["Firefox", "Iceweasel", "Navigator"],
                                         dispvm=True)
 
+def create_testcases_for_templates():
+    return qubes.tests.create_testcases_for_templates('TC_50_MimeHandlers',
+        TC_50_MimeHandlers, qubes.tests.SystemTestCase,
+        module=sys.modules[__name__])
+
 def load_tests(loader, tests, pattern):
-    try:
-        qc = qubes.qubes.QubesVmCollection()
-        qc.lock_db_for_reading()
-        qc.load()
-        qc.unlock_db()
-        templates = [vm.name for vm in qc.values() if
-                     isinstance(vm, qubes.qubes.QubesTemplateVm)]
-    except OSError:
-        templates = []
-    for template in templates:
-        tests.addTests(loader.loadTestsFromTestCase(
-            type(
-                'TC_50_MimeHandlers_' + template,
-                (TC_50_MimeHandlers, qubes.tests.QubesTestCase),
-                {'template': template})))
-    return tests
+    tests.addTests(loader.loadTestsFromNames(
+        create_testcases_for_templates()))
+    return tests
+
+qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)

qubes/tests/integ/network.py

@@ -22,8 +22,6 @@
 from distutils import spawn
 
 import asyncio
-import multiprocessing
-import os
 import subprocess
 import sys
 import time
@@ -31,13 +29,11 @@ import unittest
 
 import qubes.tests
 import qubes.firewall
+import qubes.vm.qubesvm
 import qubes.vm.appvm
 
-class NcVersion:
-    Trad = 1
-    Nmap = 2
 
-# noinspection PyAttributeOutsideInit
+# noinspection PyAttributeOutsideInit,PyPep8Naming
 class VmNetworkingMixin(object):
     test_ip = '192.168.123.45'
     test_name = 'test.example.com'
@@ -50,21 +46,23 @@ class VmNetworkingMixin(object):
     template = None
 
     def run_cmd(self, vm, cmd, user="root"):
+        '''Run a command *cmd* in a *vm* as *user*. Return its exit code.
+        :type self: qubes.tests.SystemTestCase | VmNetworkingMixin
+        :param qubes.vm.qubesvm.QubesVM vm: VM object to run command in
+        :param str cmd: command to execute
+        :param std user: user to execute command as
+        :return int: command exit code
+        '''
         try:
             self.loop.run_until_complete(vm.run_for_stdio(cmd, user=user))
         except subprocess.CalledProcessError as e:
             return e.returncode
         return 0
 
-    def check_nc_version(self, vm):
-        if self.run_cmd(vm, 'nc -h >/dev/null 2>&1') != 0:
-            self.skipTest('nc not installed')
-        if self.run_cmd(vm, 'nc -h 2>&1|grep -q nmap.org') == 0:
-            return NcVersion.Nmap
-        else:
-            return NcVersion.Trad
-
     def setUp(self):
+        '''
+        :type self: qubes.tests.SystemTestCase | VMNetworkingMixin
+        '''
         super(VmNetworkingMixin, self).setUp()
         if self.template.startswith('whonix-'):
             self.skipTest("Test not supported here - Whonix uses its own "
@@ -75,6 +73,7 @@ class VmNetworkingMixin(object):
             label='red')
         self.loop.run_until_complete(self.testnetvm.create_on_disk())
         self.testnetvm.provides_network = True
+        self.testnetvm.netvm = None
         self.testvm1 = self.app.add_new_vm(qubes.vm.appvm.AppVM,
             name=self.make_vm_name('vm1'),
             label='red')
@@ -86,6 +85,9 @@ class VmNetworkingMixin(object):
 
 
     def configure_netvm(self):
+        '''
+        :type self: qubes.tests.SystemTestCase | VMNetworkingMixin
+        '''
         def run_netvm_cmd(cmd):
             if self.run_cmd(self.testnetvm, cmd) != 0:
                 self.fail("Command '%s' failed" % cmd)
@@ -102,7 +104,8 @@ class VmNetworkingMixin(object):
         run_netvm_cmd("ip link add test0 type dummy")
         run_netvm_cmd("ip link set test0 up")
         run_netvm_cmd("ip addr add {}/24 dev test0".format(self.test_ip))
-        run_netvm_cmd("iptables -I INPUT -d {} -j ACCEPT".format(self.test_ip))
+        run_netvm_cmd("iptables -I INPUT -d {} -j ACCEPT --wait".format(
+            self.test_ip))
         # ignore failure
         self.run_cmd(self.testnetvm, "killall --wait dnsmasq")
         run_netvm_cmd("dnsmasq -a {ip} -A /{name}/{ip} -i test0 -z".format(
@@ -113,12 +116,18 @@ class VmNetworkingMixin(object):
 
 
     def test_000_simple_networking(self):
+        '''
+        :type self: qubes.tests.SystemTestCase | VMNetworkingMixin
+        '''
         self.loop.run_until_complete(self.testvm1.start())
         self.assertEqual(self.run_cmd(self.testvm1, self.ping_ip), 0)
         self.assertEqual(self.run_cmd(self.testvm1, self.ping_name), 0)
 
 
     def test_010_simple_proxyvm(self):
+        '''
+        :type self: qubes.tests.SystemTestCase | VMNetworkingMixin
+        '''
         self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM,
             name=self.make_vm_name('proxy'),
             label='red')
@@ -144,6 +153,9 @@ class VmNetworkingMixin(object):
     @unittest.skipUnless(spawn.find_executable('xdotool'),
                          "xdotool not installed")
     def test_020_simple_proxyvm_nm(self):
+        '''
+        :type self: qubes.tests.SystemTestCase | VMNetworkingMixin
+        '''
         self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM,
             name=self.make_vm_name('proxy'),
             label='red')
@@ -189,6 +201,9 @@ class VmNetworkingMixin(object):
 
 
     def test_030_firewallvm_firewall(self):
+        '''
+        :type self: qubes.tests.SystemTestCase | VMNetworkingMixin
+        '''
         self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM,
             name=self.make_vm_name('proxy'),
             label='red')
@@ -198,8 +213,6 @@ class VmNetworkingMixin(object):
         self.testvm1.netvm = self.proxy
         self.app.save()
 
-        nc_version = self.check_nc_version(self.testnetvm)
-
         # block all for first
 
         self.testvm1.firewall.rules = [qubes.firewall.Rule(action='drop')]
@@ -207,10 +220,8 @@ class VmNetworkingMixin(object):
         self.loop.run_until_complete(self.testvm1.start())
         self.assertTrue(self.proxy.is_running())
 
-        nc = self.loop.run_until_complete(self.testnetvm.run(
-            'nc -l --send-only -e /bin/hostname -k 1234'
-            if nc_version == NcVersion.Nmap
-            else 'while nc -l -e /bin/hostname -p 1234; do true; done'))
+        server = self.loop.run_until_complete(self.testnetvm.run(
+            'socat TCP-LISTEN:1234,fork EXEC:/bin/uname'))
 
         try:
             self.assertEqual(self.run_cmd(self.proxy, self.ping_ip), 0,
@@ -220,11 +231,8 @@ class VmNetworkingMixin(object):
             self.assertNotEqual(self.run_cmd(self.testvm1, self.ping_ip), 0,
                             "Ping by IP should be blocked")
 
-            if nc_version == NcVersion.Nmap:
-                nc_cmd = "nc -w 1 --recv-only {} 1234".format(self.test_ip)
-            else:
-                nc_cmd = "nc -w 1 {} 1234".format(self.test_ip)
-            self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0,
+            client_cmd = "socat TCP:{}:1234 -".format(self.test_ip)
+            self.assertNotEqual(self.run_cmd(self.testvm1, client_cmd), 0,
                             "TCP connection should be blocked")
 
             # block all except ICMP
@@ -253,7 +261,7 @@ class VmNetworkingMixin(object):
             time.sleep(3)
             self.assertEqual(self.run_cmd(self.testvm1, self.ping_name), 0,
                             "Ping by name failed (should be allowed now)")
-            self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0,
+            self.assertNotEqual(self.run_cmd(self.testvm1, client_cmd), 0,
                             "TCP connection should be blocked")
 
             # block all except target
@@ -267,7 +275,7 @@ class VmNetworkingMixin(object):
             # Ugly hack b/c there is no feedback when the rules are actually
             # applied
             time.sleep(3)
-            self.assertEqual(self.run_cmd(self.testvm1, nc_cmd), 0,
+            self.assertEqual(self.run_cmd(self.testvm1, client_cmd), 0,
                             "TCP connection failed (should be allowed now)")
 
             # allow all except target
@@ -282,14 +290,17 @@ class VmNetworkingMixin(object):
             # Ugly hack b/c there is no feedback when the rules are actually
             # applied
             time.sleep(3)
-            self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0,
+            self.assertNotEqual(self.run_cmd(self.testvm1, client_cmd), 0,
                             "TCP connection should be blocked")
         finally:
-            nc.terminate()
-            self.loop.run_until_complete(nc.wait())
+            server.terminate()
+            self.loop.run_until_complete(server.wait())
 
 
     def test_040_inter_vm(self):
+        '''
+        :type self: qubes.tests.SystemTestCase | VMNetworkingMixin
+        '''
         self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM,
             name=self.make_vm_name('proxy'),
             label='red')
@@ -327,7 +338,10 @@ class VmNetworkingMixin(object):
             self.ping_cmd.format(target=self.testvm1.ip)), 0)
 
     def test_050_spoof_ip(self):
-        """Test if VM IP spoofing is blocked"""
+        '''Test if VM IP spoofing is blocked
+
+        :type self: qubes.tests.SystemTestCase | VMNetworkingMixin
+        '''
         self.loop.run_until_complete(self.testvm1.start())
 
         self.assertEqual(self.run_cmd(self.testvm1, self.ping_ip), 0)
@@ -353,7 +367,10 @@ class VmNetworkingMixin(object):
         self.assertEquals(packets, '0', 'Some packet hit the INPUT rule')
 
     def test_100_late_xldevd_startup(self):
-        """Regression test for #1990"""
+        '''Regression test for #1990
+
+        :type self: qubes.tests.SystemTestCase | VMNetworkingMixin
+        '''
         # Simulater late xl devd startup
         cmd = "systemctl stop xendriverdomain"
         if self.run_cmd(self.testnetvm, cmd) != 0:
@@ -367,7 +384,10 @@ class VmNetworkingMixin(object):
         self.assertEqual(self.run_cmd(self.testvm1, self.ping_ip), 0)
 
     def test_200_fake_ip_simple(self):
-        '''Test hiding VM real IP'''
+        '''Test hiding VM real IP
+
+        :type self: qubes.tests.SystemTestCase | VMNetworkingMixin
+        '''
         self.testvm1.features['net.fake-ip'] = '192.168.1.128'
         self.testvm1.features['net.fake-gateway'] = '192.168.1.1'
         self.testvm1.features['net.fake-netmask'] = '255.255.255.0'
@@ -398,7 +418,10 @@ class VmNetworkingMixin(object):
         self.assertNotIn(str(self.testvm1.netvm.ip), output)
 
     def test_201_fake_ip_without_gw(self):
-        '''Test hiding VM real IP'''
+        '''Test hiding VM real IP
+
+        :type self: qubes.tests.SystemTestCase | VMNetworkingMixin
+        '''
         self.testvm1.features['net.fake-ip'] = '192.168.1.128'
         self.app.save()
         self.loop.run_until_complete(self.testvm1.start())
@@ -417,7 +440,10 @@ class VmNetworkingMixin(object):
         self.assertNotIn(str(self.testvm1.ip), output)
 
     def test_202_fake_ip_firewall(self):
-        '''Test hiding VM real IP, firewall'''
+        '''Test hiding VM real IP, firewall
+
+        :type self: qubes.tests.SystemTestCase | VMNetworkingMixin
+        '''
         self.testvm1.features['net.fake-ip'] = '192.168.1.128'
         self.testvm1.features['net.fake-gateway'] = '192.168.1.1'
         self.testvm1.features['net.fake-netmask'] = '255.255.255.0'
@@ -431,8 +457,6 @@ class VmNetworkingMixin(object):
         self.testvm1.netvm = self.proxy
         self.app.save()
 
-        nc_version = self.check_nc_version(self.testnetvm)
-
         # block all but ICMP and DNS
 
         self.testvm1.firewall.rules = [
@@ -443,10 +467,8 @@ class VmNetworkingMixin(object):
         self.loop.run_until_complete(self.testvm1.start())
         self.assertTrue(self.proxy.is_running())
 
-        nc = self.loop.run_until_complete(self.testnetvm.run(
-            'nc -l --send-only -e /bin/hostname -k 1234'
-            if nc_version == NcVersion.Nmap
-            else 'while nc -l -e /bin/hostname -p 1234; do true; done'))
+        server = self.loop.run_until_complete(self.testnetvm.run(
+            'socat TCP-LISTEN:1234,fork EXEC:/bin/uname'))
 
         try:
             self.assertEqual(self.run_cmd(self.proxy, self.ping_ip), 0,
@@ -457,18 +479,18 @@ class VmNetworkingMixin(object):
                             "Ping by IP should be allowed")
             self.assertEqual(self.run_cmd(self.testvm1, self.ping_name), 0,
                             "Ping by name should be allowed")
-            if nc_version == NcVersion.Nmap:
-                nc_cmd = "nc -w 1 --recv-only {} 1234".format(self.test_ip)
-            else:
-                nc_cmd = "nc -w 1 {} 1234".format(self.test_ip)
-            self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0,
+            client_cmd = "socat TCP:{}:1234 -".format(self.test_ip)
+            self.assertNotEqual(self.run_cmd(self.testvm1, client_cmd), 0,
                             "TCP connection should be blocked")
         finally:
-            nc.terminate()
-            self.loop.run_until_complete(nc.wait())
+            server.terminate()
+            self.loop.run_until_complete(server.wait())
 
     def test_203_fake_ip_inter_vm_allow(self):
-        '''Access VM with "fake IP" from other VM (when firewall allows)'''
+        '''Access VM with "fake IP" from other VM (when firewall allows)
+
+        :type self: qubes.tests.SystemTestCase | VMNetworkingMixin
+        '''
         self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM,
             name=self.make_vm_name('proxy'),
             label='red')
@@ -490,9 +512,9 @@ class VmNetworkingMixin(object):
         self.loop.run_until_complete(self.testvm1.start())
         self.loop.run_until_complete(self.testvm2.start())
 
+        cmd = 'iptables -I FORWARD -s {} -d {} -j ACCEPT'.format(
+            self.testvm2.ip, self.testvm1.ip)
         try:
-            cmd = 'iptables -I FORWARD -s {} -d {} -j ACCEPT'.format(
-                self.testvm2.ip, self.testvm1.ip)
             self.loop.run_until_complete(self.proxy.run_for_stdio(
                 cmd, user='root'))
         except subprocess.CalledProcessError as e:
@@ -521,7 +543,10 @@ class VmNetworkingMixin(object):
             'Packets didn\'t managed to the VM')
 
     def test_204_fake_ip_proxy(self):
-        '''Test hiding VM real IP'''
+        '''Test hiding VM real IP
+
+        :type self: qubes.tests.SystemTestCase | VMNetworkingMixin
+        '''
         self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM,
             name=self.make_vm_name('proxy'),
             label='red')
@@ -545,7 +570,7 @@ class VmNetworkingMixin(object):
             (output, _) = self.loop.run_until_complete(
                 self.proxy.run_for_stdio(
                     'ip addr show dev eth0', user='root'))
-        except subprocess.CalledProcessError as e:
+        except subprocess.CalledProcessError:
             self.fail('ip addr show dev eth0 failed')
         output = output.decode()
         self.assertIn('192.168.1.128', output)
@@ -555,7 +580,7 @@ class VmNetworkingMixin(object):
             (output, _) = self.loop.run_until_complete(
                 self.proxy.run_for_stdio(
                     'ip route show', user='root'))
-        except subprocess.CalledProcessError as e:
+        except subprocess.CalledProcessError:
             self.fail('ip route show failed')
         output = output.decode()
         self.assertIn('192.168.1.1', output)
@@ -565,7 +590,7 @@ class VmNetworkingMixin(object):
             (output, _) = self.loop.run_until_complete(
                 self.testvm1.run_for_stdio(
                     'ip addr show dev eth0', user='root'))
-        except subprocess.CalledProcessError as e:
+        except subprocess.CalledProcessError:
             self.fail('ip addr show dev eth0 failed')
         output = output.decode()
         self.assertNotIn('192.168.1.128', output)
@@ -575,14 +600,17 @@ class VmNetworkingMixin(object):
             (output, _) = self.loop.run_until_complete(
                 self.testvm1.run_for_stdio(
                     'ip route show', user='root'))
-        except subprocess.CalledProcessError as e:
+        except subprocess.CalledProcessError:
             self.fail('ip route show failed')
         output = output.decode()
         self.assertIn('192.168.1.128', output)
         self.assertNotIn(str(self.proxy.ip), output)
 
     def test_210_custom_ip_simple(self):
-        '''Custom AppVM IP'''
+        '''Custom AppVM IP
+
+        :type self: qubes.tests.SystemTestCase | VMNetworkingMixin
+        '''
         self.testvm1.ip = '192.168.1.1'
         self.app.save()
         self.loop.run_until_complete(self.testvm1.start())
@@ -590,7 +618,10 @@ class VmNetworkingMixin(object):
         self.assertEqual(self.run_cmd(self.testvm1, self.ping_name), 0)
 
     def test_211_custom_ip_proxy(self):
-        '''Custom ProxyVM IP'''
+        '''Custom ProxyVM IP
+
+        :type self: qubes.tests.SystemTestCase | VMNetworkingMixin
+        '''
         self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM,
             name=self.make_vm_name('proxy'),
             label='red')
@@ -607,7 +638,10 @@ class VmNetworkingMixin(object):
         self.assertEqual(self.run_cmd(self.testvm1, self.ping_name), 0)
 
     def test_212_custom_ip_firewall(self):
-        '''Custom VM IP and firewall'''
+        '''Custom VM IP and firewall
+
+        :type self: qubes.tests.SystemTestCase | VMNetworkingMixin
+        '''
         self.testvm1.ip = '192.168.1.1'
 
         self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM,
@@ -619,8 +653,6 @@ class VmNetworkingMixin(object):
         self.testvm1.netvm = self.proxy
         self.app.save()
 
-        nc_version = self.check_nc_version(self.testnetvm)
-
         # block all but ICMP and DNS
 
         self.testvm1.firewall.rules = [
@@ -631,10 +663,8 @@ class VmNetworkingMixin(object):
         self.loop.run_until_complete(self.testvm1.start())
         self.assertTrue(self.proxy.is_running())
 
-        nc = self.loop.run_until_complete(self.testnetvm.run(
-            'nc -l --send-only -e /bin/hostname -k 1234'
-            if nc_version == NcVersion.Nmap
-            else 'while nc -l -e /bin/hostname -p 1234; do true; done'))
+        server = self.loop.run_until_complete(self.testnetvm.run(
+            'socat TCP-LISTEN:1234,fork EXEC:/bin/uname'))
 
         try:
             self.assertEqual(self.run_cmd(self.proxy, self.ping_ip), 0,
@@ -645,16 +675,14 @@ class VmNetworkingMixin(object):
                             "Ping by IP should be allowed")
             self.assertEqual(self.run_cmd(self.testvm1, self.ping_name), 0,
                             "Ping by name should be allowed")
-            if nc_version == NcVersion.Nmap:
-                nc_cmd = "nc -w 1 --recv-only {} 1234".format(self.test_ip)
-            else:
-                nc_cmd = "nc -w 1 {} 1234".format(self.test_ip)
-            self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0,
+            client_cmd = "socat TCP:{}:1234 -".format(self.test_ip)
+            self.assertNotEqual(self.run_cmd(self.testvm1, client_cmd), 0,
                             "TCP connection should be blocked")
         finally:
-            nc.terminate()
-            self.loop.run_until_complete(nc.wait())
+            server.terminate()
+            self.loop.run_until_complete(server.wait())
 
+# noinspection PyAttributeOutsideInit,PyPep8Naming
 class VmIPv6NetworkingMixin(VmNetworkingMixin):
     test_ip6 = '2000:abcd::1'
 
@@ -666,6 +694,9 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin):
         self.ping6_name = self.ping6_cmd.format(target=self.test_name)
 
     def configure_netvm(self):
+        '''
+        :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin
+        '''
         self.testnetvm.features['ipv6'] = True
         super(VmIPv6NetworkingMixin, self).configure_netvm()
 
@@ -683,12 +714,18 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin):
             format(ip=self.test_ip, ip6=self.test_ip6, name=self.test_name))
 
     def test_500_ipv6_simple_networking(self):
+        '''
+        :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin
+        '''
         self.loop.run_until_complete(self.testvm1.start())
         self.assertEqual(self.run_cmd(self.testvm1, self.ping6_ip), 0)
         self.assertEqual(self.run_cmd(self.testvm1, self.ping6_name), 0)
 
 
     def test_510_ipv6_simple_proxyvm(self):
+        '''
+        :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin
+        '''
         self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM,
             name=self.make_vm_name('proxy'),
             label='red')
@@ -714,6 +751,9 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin):
     @unittest.skipUnless(spawn.find_executable('xdotool'),
                          "xdotool not installed")
     def test_520_ipv6_simple_proxyvm_nm(self):
+        '''
+        :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin
+        '''
         self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM,
             name=self.make_vm_name('proxy'),
             label='red')
@@ -764,6 +804,9 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin):
 
 
     def test_530_ipv6_firewallvm_firewall(self):
+        '''
+        :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin
+        '''
         self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM,
             name=self.make_vm_name('proxy'),
             label='red')
@@ -773,9 +816,6 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin):
         self.testvm1.netvm = self.proxy
         self.app.save()
 
-        if self.run_cmd(self.testnetvm, 'ncat -h') != 0:
-            self.skipTest('nmap ncat not installed')
-
         # block all for first
 
         self.testvm1.firewall.rules = [qubes.firewall.Rule(action='drop')]
@@ -783,8 +823,8 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin):
         self.loop.run_until_complete(self.testvm1.start())
         self.assertTrue(self.proxy.is_running())
 
-        nc = self.loop.run_until_complete(self.testnetvm.run(
-            'ncat -l --send-only -e /bin/hostname -k 1234'))
+        server = self.loop.run_until_complete(self.testnetvm.run(
+            'socat TCP6-LISTEN:1234,fork EXEC:/bin/uname'))
 
         try:
             self.assertEqual(self.run_cmd(self.proxy, self.ping6_ip), 0,
@@ -794,8 +834,9 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin):
             self.assertNotEqual(self.run_cmd(self.testvm1, self.ping6_ip), 0,
                             "Ping by IP should be blocked")
 
-            nc_cmd = "ncat -w 1 --recv-only {} 1234".format(self.test_ip6)
-            self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0,
+            client6_cmd = "socat TCP:[{}]:1234 -".format(self.test_ip6)
+            client4_cmd = "socat TCP:{}:1234 -".format(self.test_ip)
+            self.assertNotEqual(self.run_cmd(self.testvm1, client6_cmd), 0,
                             "TCP connection should be blocked")
 
             # block all except ICMP
@@ -825,13 +866,14 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin):
             time.sleep(3)
             self.assertEqual(self.run_cmd(self.testvm1, self.ping6_name), 0,
                             "Ping by name failed (should be allowed now)")
-            self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0,
+            self.assertNotEqual(self.run_cmd(self.testvm1, client6_cmd), 0,
                             "TCP connection should be blocked")
 
             # block all except target
 
             self.testvm1.firewall.rules = [
-                qubes.firewall.Rule(None, action='accept', dsthost=self.test_ip6,
+                qubes.firewall.Rule(None, action='accept',
+                    dsthost=self.test_ip6,
                     proto='tcp', dstports=1234),
             ]
             self.testvm1.firewall.save()
@@ -839,7 +881,7 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin):
             # Ugly hack b/c there is no feedback when the rules are actually
             # applied
             time.sleep(3)
-            self.assertEqual(self.run_cmd(self.testvm1, nc_cmd), 0,
+            self.assertEqual(self.run_cmd(self.testvm1, client6_cmd), 0,
                             "TCP connection failed (should be allowed now)")
 
             # block all except target - by name
@@ -854,10 +896,9 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin):
             # Ugly hack b/c there is no feedback when the rules are actually
             # applied
             time.sleep(3)
-            self.assertEqual(self.run_cmd(self.testvm1, nc_cmd), 0,
+            self.assertEqual(self.run_cmd(self.testvm1, client6_cmd), 0,
                 "TCP (IPv6) connection failed (should be allowed now)")
-            self.assertEqual(self.run_cmd(self.testvm1,
-                nc_cmd.replace(self.test_ip6, self.test_ip)),
+            self.assertEqual(self.run_cmd(self.testvm1, client4_cmd),
                 0,
                 "TCP (IPv4) connection failed (should be allowed now)")
 
@@ -873,14 +914,17 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin):
             # Ugly hack b/c there is no feedback when the rules are actually
             # applied
             time.sleep(3)
-            self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0,
+            self.assertNotEqual(self.run_cmd(self.testvm1, client6_cmd), 0,
                             "TCP connection should be blocked")
         finally:
-            nc.terminate()
-            self.loop.run_until_complete(nc.wait())
+            server.terminate()
+            self.loop.run_until_complete(server.wait())
 
 
     def test_540_ipv6_inter_vm(self):
+        '''
+        :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin
+        '''
         self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM,
             name=self.make_vm_name('proxy'),
             label='red')
@@ -920,7 +964,10 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin):
 
 
     def test_550_ipv6_spoof_ip(self):
-        """Test if VM IP spoofing is blocked"""
+        '''Test if VM IP spoofing is blocked
+
+        :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin
+        '''
         self.loop.run_until_complete(self.testvm1.start())
 
         self.assertEqual(self.run_cmd(self.testvm1, self.ping6_ip), 0)
@@ -949,7 +996,10 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin):
         self.assertEquals(packets, '0', 'Some packet hit the INPUT rule')
 
     def test_710_ipv6_custom_ip_simple(self):
-        '''Custom AppVM IP'''
+        '''Custom AppVM IP
+
+        :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin
+        '''
         self.testvm1.ip6 = '2000:aaaa:bbbb::1'
         self.app.save()
         self.loop.run_until_complete(self.testvm1.start())
@@ -957,7 +1007,10 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin):
         self.assertEqual(self.run_cmd(self.testvm1, self.ping6_name), 0)
 
     def test_711_ipv6_custom_ip_proxy(self):
-        '''Custom ProxyVM IP'''
+        '''Custom ProxyVM IP
+
+        :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin
+        '''
         self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM,
             name=self.make_vm_name('proxy'),
             label='red')
@@ -974,7 +1027,10 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin):
         self.assertEqual(self.run_cmd(self.testvm1, self.ping6_name), 0)
 
     def test_712_ipv6_custom_ip_firewall(self):
-        '''Custom VM IP and firewall'''
+        '''Custom VM IP and firewall
+
+        :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin
+        '''
         self.testvm1.ip6 = '2000:aaaa:bbbb::1'
 
         self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM,
@@ -986,8 +1042,6 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin):
         self.testvm1.netvm = self.proxy
         self.app.save()
 
-        nc_version = self.check_nc_version(self.testnetvm)
-
         # block all but ICMP and DNS
 
         self.testvm1.firewall.rules = [
@@ -998,10 +1052,8 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin):
         self.loop.run_until_complete(self.testvm1.start())
         self.assertTrue(self.proxy.is_running())
 
-        nc = self.loop.run_until_complete(self.testnetvm.run(
-            'nc -l --send-only -e /bin/hostname -k 1234'
-            if nc_version == NcVersion.Nmap
-            else 'while nc -l -e /bin/hostname -p 1234; do true; done'))
+        server = self.loop.run_until_complete(self.testnetvm.run(
+            'socat TCP6-LISTEN:1234,fork EXEC:/bin/uname'))
 
         try:
             self.assertEqual(self.run_cmd(self.proxy, self.ping6_ip), 0,
@@ -1012,17 +1064,14 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin):
                             "Ping by IP should be allowed")
             self.assertEqual(self.run_cmd(self.testvm1, self.ping6_name), 0,
                             "Ping by name should be allowed")
-            if nc_version == NcVersion.Nmap:
-                nc_cmd = "nc -w 1 --recv-only {} 1234".format(self.test_ip6)
-            else:
-                nc_cmd = "nc -w 1 {} 1234".format(self.test_ip6)
-            self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0,
+            client_cmd = "socat TCP:[{}]:1234 -".format(self.test_ip6)
+            self.assertNotEqual(self.run_cmd(self.testvm1, client_cmd), 0,
                             "TCP connection should be blocked")
         finally:
-            nc.terminate()
-            self.loop.run_until_complete(nc.wait())
+            server.terminate()
+            self.loop.run_until_complete(server.wait())
 
-# noinspection PyAttributeOutsideInit
+# noinspection PyAttributeOutsideInit,PyPep8Naming
 class VmUpdatesMixin(object):
     """
     Tests for VM updates
@@ -1099,6 +1148,14 @@ class VmUpdatesMixin(object):
     )
 
     def run_cmd(self, vm, cmd, user="root"):
+        '''Run a command *cmd* in a *vm* as *user*. Return its exit code.
+
+        :type self: qubes.tests.SystemTestCase | VmUpdatesMixin
+        :param qubes.vm.qubesvm.QubesVM vm: VM object to run command in
+        :param str cmd: command to execute
+        :param std user: user to execute command as
+        :return int: command exit code
+        '''
         try:
             self.loop.run_until_complete(vm.run_for_stdio(cmd))
         except subprocess.CalledProcessError as e:
@@ -1106,6 +1163,9 @@ class VmUpdatesMixin(object):
         return 0
 
     def setUp(self):
+        '''
+        :type self: qubes.tests.SystemTestCase | VmUpdatesMixin
+        '''
         if not self.template.count('debian') and \
                 not self.template.count('fedora'):
             self.skipTest("Template {} not supported by this test".format(
@@ -1142,6 +1202,9 @@ class VmUpdatesMixin(object):
         self.loop.run_until_complete(self.testvm1.create_on_disk())
 
     def test_000_simple_update(self):
+        '''
+        :type self: qubes.tests.SystemTestCase | VmUpdatesMixin
+        '''
         self.app.save()
         # reload the VM to have all the properties properly set (especially
         # default netvm)
@@ -1155,6 +1218,9 @@ class VmUpdatesMixin(object):
             '{}: {}\n{}'.format(self.update_cmd, stdout, stderr))
 
     def create_repo_apt(self):
+        '''
+        :type self: qubes.tests.SystemTestCase | VmUpdatesMixin
+        '''
         pkg_file_name = "test-pkg_1.0-1_amd64.deb"
         self.loop.run_until_complete(self.netvm_repo.run_for_stdio('''
             mkdir /tmp/apt-repo \
@@ -1209,6 +1275,9 @@ SHA256:
 '''))
 
     def create_repo_yum(self):
+        '''
+        :type self: qubes.tests.SystemTestCase | VmUpdatesMixin
+        '''
         pkg_file_name = "test-pkg-1.0-1.fc21.x86_64.rpm"
         self.loop.run_until_complete(self.netvm_repo.run_for_stdio('''
             mkdir /tmp/yum-repo \
@@ -1221,6 +1290,9 @@ SHA256:
             'createrepo /tmp/yum-repo'))
 
     def create_repo_and_serve(self):
+        '''
+        :type self: qubes.tests.SystemTestCase | VmUpdatesMixin
+        '''
         if self.template.count("debian") or self.template.count("whonix"):
             self.create_repo_apt()
             self.loop.run_until_complete(self.netvm_repo.run(
@@ -1242,6 +1314,8 @@ SHA256:
         The critical part is to use "localhost" - this will work only when
         accessed through update proxy and this is exactly what we want to
         test here.
+
+        :type self: qubes.tests.SystemTestCase | VmUpdatesMixin
         """
 
         if self.template.count("debian") or self.template.count("whonix"):
@@ -1266,9 +1340,12 @@ SHA256:
                 self.template))
 
     def test_010_update_via_proxy(self):
-        """
-        Test both whether updates proxy works and whether is actually used by the VM
-        """
+        '''
+        Test both whether updates proxy works and whether is actually used
+        by the VM
+
+        :type self: qubes.tests.SystemTestCase | VmUpdatesMixin
+        '''
         if self.template.count("minimal"):
             self.skipTest("Template {} not supported by this test".format(
                 self.template))
@@ -1319,17 +1396,20 @@ SHA256:
             self.assertIn(self.loop.run_until_complete(p.wait()), self.exit_code_ok,
                 '{}: {}\n{}'.format(self.update_cmd, stdout, stderr))
 
+def create_testcases_for_templates():
+    yield from qubes.tests.create_testcases_for_templates('VmNetworking',
+        VmNetworkingMixin, qubes.tests.SystemTestCase,
+        module=sys.modules[__name__])
+    yield from qubes.tests.create_testcases_for_templates('VmIPv6Networking',
+        VmIPv6NetworkingMixin, qubes.tests.SystemTestCase,
+        module=sys.modules[__name__])
+    yield from qubes.tests.create_testcases_for_templates('VmUpdates',
+        VmUpdatesMixin, qubes.tests.SystemTestCase,
+        module=sys.modules[__name__])
+
 def load_tests(loader, tests, pattern):
     tests.addTests(loader.loadTestsFromNames(
-        qubes.tests.create_testcases_for_templates('VmNetworking',
-            VmNetworkingMixin, qubes.tests.SystemTestCase,
-            module=sys.modules[__name__])))
-    tests.addTests(loader.loadTestsFromNames(
-        qubes.tests.create_testcases_for_templates('VmIPv6Networking',
-            VmIPv6NetworkingMixin, qubes.tests.SystemTestCase,
-            module=sys.modules[__name__])))
-    tests.addTests(loader.loadTestsFromNames(
-        qubes.tests.create_testcases_for_templates('VmUpdates',
-            VmUpdates, qubes.tests.SystemTestCase,
-            module=sys.modules[__name__])))
+        create_testcases_for_templates()))
     return tests
+
+qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)

qubes/tests/integ/pvgrub.py

@@ -70,7 +70,7 @@ class TC_40_PVGrub(object):
 
     def get_kernel_version(self, vm):
         if self.template.startswith('fedora-'):
-            cmd_get_kernel_version = 'rpm -q kernel-core|sort -n|tail -1|' \
+            cmd_get_kernel_version = 'rpm -q kernel-core|sort -V|tail -1|' \
                                      'cut -d - -f 3-'
         elif self.template.startswith('debian-'):
             cmd_get_kernel_version = \
@@ -137,10 +137,14 @@ class TC_40_PVGrub(object):
             self.test_template.run_for_stdio('uname -r'))
         self.assertEquals(actual_kver.strip(), kver)
 
+def create_testcases_for_templates():
+    return qubes.tests.create_testcases_for_templates('TC_40_PVGrub',
+        TC_40_PVGrub, qubes.tests.SystemTestCase,
+        module=sys.modules[__name__])
 
 def load_tests(loader, tests, pattern):
     tests.addTests(loader.loadTestsFromNames(
-        qubes.tests.create_testcases_for_templates('TC_40_PVGrub',
-            TC_40_PVGrub, qubes.tests.SystemTestCase,
-            module=sys.modules[__name__])))
+        create_testcases_for_templates()))
     return tests
+
+qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)

qubes/tests/integ/salt.py

@@ -391,9 +391,15 @@ class SaltVMTestMixin(SaltTestMixin):
             self.assertEqual(stderr, b'')
 
 
+def create_testcases_for_templates():
+    return qubes.tests.create_testcases_for_templates('TC_10_VMSalt',
+        SaltVMTestMixin, qubes.tests.SystemTestCase,
+        module=sys.modules[__name__])
+
+
 def load_tests(loader, tests, pattern):
     tests.addTests(loader.loadTestsFromNames(
-        qubes.tests.create_testcases_for_templates('TC_10_VMSalt',
-            SaltVMTestMixin, qubes.tests.SystemTestCase,
-            module=sys.modules[__name__])))
+        create_testcases_for_templates()))
     return tests
+
+qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)

qubes/tests/integ/storage.py

@@ -26,6 +26,7 @@ import subprocess
 import qubes.storage.lvm
 import qubes.tests
 import qubes.tests.storage_lvm
+import qubes.tests.storage_reflink
 import qubes.vm.appvm
 
 
@@ -76,6 +77,7 @@ class StorageTestMixin(object):
         del coro_maybe
         self.app.save()
         yield from (self.vm1.start())
+        yield from self.wait_for_session(self.vm1)
 
         # volatile image not clean
         yield from (self.vm1.run_for_stdio(
@@ -111,6 +113,7 @@ class StorageTestMixin(object):
         del coro_maybe
         self.app.save()
         yield from self.vm1.start()
+        yield from self.wait_for_session(self.vm1)
         # non-volatile image not clean
         yield from self.vm1.run_for_stdio(
             'head -c {} /dev/zero 2>&1 | diff -q /dev/xvde - 2>&1'.format(size),
@@ -196,6 +199,9 @@ class StorageTestMixin(object):
         self.app.save()
         yield from self.vm1.start()
         yield from self.vm2.start()
+        yield from asyncio.wait(
+            [self.wait_for_session(self.vm1), self.wait_for_session(self.vm2)])
+
 
         try:
             yield from self.vm1.run_for_stdio(
@@ -284,6 +290,7 @@ class StorageTestMixin(object):
         del coro_maybe
         self.app.save()
         yield from self.vm2.start()
+        yield from self.wait_for_session(self.vm2)
 
         # snapshot image not clean
         yield from self.vm2.run_for_stdio(
@@ -318,6 +325,28 @@ class StorageFile(StorageTestMixin, qubes.tests.SystemTestCase):
         super(StorageFile, self).tearDown()
 
 
+class StorageReflinkMixin(StorageTestMixin):
+    def tearDown(self):
+        self.app.remove_pool(self.pool.name)
+        super().tearDown()
+
+    def init_pool(self, fs_type, **kwargs):
+        name = 'test-reflink-integration-on-' + fs_type
+        dir_path = os.path.join('/var/tmp', name)
+        qubes.tests.storage_reflink.mkdir_fs(dir_path, fs_type,
+                                             cleanup_via=self.addCleanup)
+        self.pool = self.app.add_pool(name=name, dir_path=dir_path,
+                                      driver='file-reflink', **kwargs)
+
+class StorageReflinkOnBtrfs(StorageReflinkMixin, qubes.tests.SystemTestCase):
+    def init_pool(self):
+        super().init_pool('btrfs')
+
+class StorageReflinkOnExt4(StorageReflinkMixin, qubes.tests.SystemTestCase):
+    def init_pool(self):
+        super().init_pool('ext4', setup_check='no')
+
+
 @qubes.tests.storage_lvm.skipUnlessLvmPoolExists
 class StorageLVM(StorageTestMixin, qubes.tests.SystemTestCase):
     def init_pool(self):

qubes/tests/integ/vm_qrexec_gui.py

@@ -24,11 +24,15 @@ import multiprocessing
 import os
 import subprocess
 import sys
+import tempfile
 import unittest
 
 from distutils import spawn
 
+import grp
+
 import qubes.config
+import qubes.devices
 import qubes.tests
 import qubes.vm.appvm
 import qubes.vm.templatevm
@@ -60,6 +64,7 @@ class TC_00_AppVMMixin(object):
         # TODO: wait_for, timeout
         self.loop.run_until_complete(self.testvm1.start())
         self.assertEqual(self.testvm1.get_power_state(), "Running")
+        self.loop.run_until_complete(self.wait_for_session(self.testvm1))
         self.loop.run_until_complete(self.testvm1.shutdown(wait=True))
         self.assertEqual(self.testvm1.get_power_state(), "Halted")
 
@@ -72,32 +77,17 @@ class TC_00_AppVMMixin(object):
         self.loop.run_until_complete(self.wait_for_session(self.testvm1))
         p = self.loop.run_until_complete(self.testvm1.run('xterm'))
         try:
-            wait_count = 0
             title = 'user@{}'.format(self.testvm1.name)
             if self.template.count("whonix"):
                 title = 'user@host'
-            while subprocess.call(
-                    ['xdotool', 'search', '--name', title],
-                    stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) > 0:
-                wait_count += 1
-                if wait_count > 100:
-                    self.fail("Timeout while waiting for xterm window")
-                self.loop.run_until_complete(asyncio.sleep(0.1))
+            self.wait_for_window(title)
 
             self.loop.run_until_complete(asyncio.sleep(0.5))
             subprocess.check_call(
                 ['xdotool', 'search', '--name', title,
                 'windowactivate', 'type', 'exit\n'])
 
-            wait_count = 0
-            while subprocess.call(['xdotool', 'search', '--name', title],
-                                stdout=open(os.path.devnull, 'w'),
-                                stderr=subprocess.STDOUT) == 0:
-                wait_count += 1
-                if wait_count > 100:
-                    self.fail("Timeout while waiting for xterm "
-                            "termination")
-                self.loop.run_until_complete(asyncio.sleep(0.1))
+            self.wait_for_window(title, show=False)
         finally:
             try:
                 p.terminate()
@@ -110,6 +100,8 @@ class TC_00_AppVMMixin(object):
     def test_011_run_gnome_terminal(self):
         if "minimal" in self.template:
             self.skipTest("Minimal template doesn't have 'gnome-terminal'")
+        if 'whonix' in self.template:
+            self.skipTest("Whonix template doesn't have 'gnome-terminal'")
         self.loop.run_until_complete(self.testvm1.start())
         self.assertEqual(self.testvm1.get_power_state(), "Running")
         self.loop.run_until_complete(self.wait_for_session(self.testvm1))
@@ -118,15 +110,7 @@ class TC_00_AppVMMixin(object):
             title = 'user@{}'.format(self.testvm1.name)
             if self.template.count("whonix"):
                 title = 'user@host'
-            wait_count = 0
-            while subprocess.call(
-                    ['xdotool', 'search', '--name', title],
-                    stdout=open(os.path.devnull, 'w'),
-                    stderr=subprocess.STDOUT) > 0:
-                wait_count += 1
-                if wait_count > 100:
-                    self.fail("Timeout while waiting for gnome-terminal window")
-                self.loop.run_until_complete(asyncio.sleep(0.1))
+            self.wait_for_window(title)
 
             self.loop.run_until_complete(asyncio.sleep(0.5))
             subprocess.check_call(
@@ -172,30 +156,14 @@ class TC_00_AppVMMixin(object):
         title = 'user@{}'.format(self.testvm1.name)
         if self.template.count("whonix"):
             title = 'user@host'
-        wait_count = 0
-        while subprocess.call(
-                ['xdotool', 'search', '--name', title],
-                stdout=open(os.path.devnull, 'w'),
-                stderr=subprocess.STDOUT) > 0:
-            wait_count += 1
-            if wait_count > 100:
-                self.fail("Timeout while waiting for xterm window")
-            self.loop.run_until_complete(asyncio.sleep(0.1))
+        self.wait_for_window(title)
 
         self.loop.run_until_complete(asyncio.sleep(0.5))
         subprocess.check_call(
             ['xdotool', 'search', '--name', title,
              'windowactivate', '--sync', 'type', 'exit\n'])
 
-        wait_count = 0
-        while subprocess.call(['xdotool', 'search', '--name', title],
-                              stdout=open(os.path.devnull, 'w'),
-                              stderr=subprocess.STDOUT) == 0:
-            wait_count += 1
-            if wait_count > 100:
-                self.fail("Timeout while waiting for xterm "
-                          "termination")
-            self.loop.run_until_complete(asyncio.sleep(0.1))
+        self.wait_for_window(title, show=False)
 
     def test_050_qrexec_simple_eof(self):
         """Test for data and EOF transmission dom0->VM"""
@@ -217,7 +185,6 @@ class TC_00_AppVMMixin(object):
         self.assertFalse(stderr,
             'Some data was printed to stderr')
 
-    @unittest.skip('#2851, because there is no GUI in vm')
     def test_051_qrexec_simple_eof_reverse(self):
         """Test for EOF transmission VM->dom0"""
 
@@ -235,7 +202,7 @@ class TC_00_AppVMMixin(object):
             p.stdin.write(TEST_DATA)
             yield from p.stdin.drain()
             p.stdin.close()
-            self.assertEqual(stdout.strip(), 'test',
+            self.assertEqual(stdout.strip(), b'test',
                 'Received data differs from what was expected')
             # this may hang in some buggy cases
             self.assertFalse((yield from p.stderr.read()),
@@ -248,15 +215,18 @@ class TC_00_AppVMMixin(object):
                     "probably EOF wasn't transferred from the VM process")
 
         self.loop.run_until_complete(self.testvm1.start())
+        self.loop.run_until_complete(self.wait_for_session(self.testvm1))
         self.loop.run_until_complete(run(self))
 
-    @unittest.skip('#2851, because there is no GUI in vm')
     def test_052_qrexec_vm_service_eof(self):
         """Test for EOF transmission VM(src)->VM(dst)"""
 
         self.loop.run_until_complete(asyncio.wait([
             self.testvm1.start(),
             self.testvm2.start()]))
+        self.loop.run_until_complete(asyncio.wait([
+            self.wait_for_session(self.testvm1),
+            self.wait_for_session(self.testvm2)]))
         self.loop.run_until_complete(self.testvm2.run_for_stdio(
             'cat > /etc/qubes-rpc/test.EOF',
             user='root',
@@ -273,7 +243,7 @@ class TC_00_AppVMMixin(object):
             except asyncio.TimeoutError:
                 self.fail("Timeout, probably EOF wasn't transferred")
 
-        self.assertEqual(stdout, b'test',
+        self.assertEqual(stdout, b'test\n',
             'Received data differs from what was expected')
 
     @unittest.expectedFailure
@@ -394,23 +364,14 @@ class TC_00_AppVMMixin(object):
             except asyncio.TimeoutError:
                 self.fail('Timeout, probably deadlock')
 
-    @unittest.skip('localcmd= argument went away')
     def test_071_qrexec_dom0_simultaneous_write(self):
         """Test for simultaneous write in dom0(src)->VM(dst) connection
 
             Similar to test_070_qrexec_vm_simultaneous_write, but with dom0
             as a source.
         """
-        def run(self):
-            result.value = self.testvm2.run_service(
-                "test.write", localcmd="/bin/sh -c '"
-                # first write a lot of data to fill all the buffers
-                "dd if=/dev/zero bs=993 count=10000 iflag=fullblock & "
-                # then after some time start reading
-                "sleep 1; "
-                "dd of=/dev/null bs=993 count=10000 iflag=fullblock; "
-                "wait"
-                "'")
+
+        self.loop.run_until_complete(self.testvm2.start())
 
         self.create_remote_file(self.testvm2, '/etc/qubes-rpc/test.write', '''\
             # first write a lot of data
@@ -418,58 +379,93 @@ class TC_00_AppVMMixin(object):
             # and only then read something
             dd of=/dev/null bs=993 count=10000 iflag=fullblock
             ''')
-        self.create_local_file('/etc/qubes-rpc/policy/test.write',
-            '{} {} allow'.format(self.testvm1.name, self.testvm2.name))
-
-        t = multiprocessing.Process(target=run, args=(self,))
-        t.start()
-        t.join(timeout=10)
-        if t.is_alive():
-            t.terminate()
+
+        # can't use subprocess.PIPE, because asyncio will claim those FDs
+        pipe1_r, pipe1_w = os.pipe()
+        pipe2_r, pipe2_w = os.pipe()
+        try:
+            local_proc = self.loop.run_until_complete(
+                asyncio.create_subprocess_shell(
+                    # first write a lot of data to fill all the buffers
+                    "dd if=/dev/zero bs=993 count=10000 iflag=fullblock & "
+                    # then after some time start reading
+                    "sleep 1; "
+                    "dd of=/dev/null bs=993 count=10000 iflag=fullblock; "
+                    "wait", stdin=pipe1_r, stdout=pipe2_w))
+
+            service_proc = self.loop.run_until_complete(self.testvm2.run_service(
+                "test.write", stdin=pipe2_r, stdout=pipe1_w))
+        finally:
+            os.close(pipe1_r)
+            os.close(pipe1_w)
+            os.close(pipe2_r)
+            os.close(pipe2_w)
+
+        try:
+            self.loop.run_until_complete(
+                asyncio.wait_for(service_proc.wait(), timeout=10))
+        except asyncio.TimeoutError:
             self.fail("Timeout, probably deadlock")
-        self.assertEqual(result.value, 0, "Service call failed")
+        else:
+            self.assertEqual(service_proc.returncode, 0,
+                "Service call failed")
+        finally:
+            try:
+                service_proc.terminate()
+            except ProcessLookupError:
+                pass
 
-    @unittest.skip('localcmd= argument went away')
     def test_072_qrexec_to_dom0_simultaneous_write(self):
         """Test for simultaneous write in dom0(src)<-VM(dst) connection
 
             Similar to test_071_qrexec_dom0_simultaneous_write, but with dom0
             as a "hanging" side.
         """
-        result = multiprocessing.Value('i', -1)
-
-        def run(self):
-            result.value = self.testvm2.run_service(
-                "test.write", localcmd="/bin/sh -c '"
-                # first write a lot of data to fill all the buffers
-                "dd if=/dev/zero bs=993 count=10000 iflag=fullblock "
-                # then, only when all written, read something
-                "dd of=/dev/null bs=993 count=10000 iflag=fullblock; "
-                "'")
 
         self.loop.run_until_complete(self.testvm2.start())
-        p = self.testvm2.run("cat > /etc/qubes-rpc/test.write", user="root",
-                             passio_popen=True)
-        # first write a lot of data
-        p.stdin.write(b"dd if=/dev/zero bs=993 count=10000 iflag=fullblock &\n")
-        # and only then read something
-        p.stdin.write(b"dd of=/dev/null bs=993 count=10000 iflag=fullblock\n")
-        p.stdin.write(b"sleep 1; \n")
-        p.stdin.write(b"wait\n")
-        p.stdin.close()
-        p.wait()
-        policy = open("/etc/qubes-rpc/policy/test.write", "w")
-        policy.write("%s %s allow" % (self.testvm1.name, self.testvm2.name))
-        policy.close()
-        self.addCleanup(os.unlink, "/etc/qubes-rpc/policy/test.write")
 
-        t = multiprocessing.Process(target=run, args=(self,))
-        t.start()
-        t.join(timeout=10)
-        if t.is_alive():
-            t.terminate()
+        self.create_remote_file(self.testvm2, '/etc/qubes-rpc/test.write', '''\
+            # first write a lot of data
+            dd if=/dev/zero bs=993 count=10000 iflag=fullblock &
+            # and only then read something
+            dd of=/dev/null bs=993 count=10000 iflag=fullblock
+            sleep 1;
+            wait
+            ''')
+
+        # can't use subprocess.PIPE, because asyncio will claim those FDs
+        pipe1_r, pipe1_w = os.pipe()
+        pipe2_r, pipe2_w = os.pipe()
+        try:
+            local_proc = self.loop.run_until_complete(
+                asyncio.create_subprocess_shell(
+                    # first write a lot of data to fill all the buffers
+                    "dd if=/dev/zero bs=993 count=10000 iflag=fullblock & "
+                    # then, only when all written, read something
+                    "dd of=/dev/null bs=993 count=10000 iflag=fullblock; ",
+                    stdin=pipe1_r, stdout=pipe2_w))
+
+            service_proc = self.loop.run_until_complete(self.testvm2.run_service(
+                "test.write", stdin=pipe2_r, stdout=pipe1_w))
+        finally:
+            os.close(pipe1_r)
+            os.close(pipe1_w)
+            os.close(pipe2_r)
+            os.close(pipe2_w)
+
+        try:
+            self.loop.run_until_complete(
+                asyncio.wait_for(service_proc.wait(), timeout=10))
+        except asyncio.TimeoutError:
             self.fail("Timeout, probably deadlock")
-        self.assertEqual(result.value, 0, "Service call failed")
+        else:
+            self.assertEqual(service_proc.returncode, 0,
+                "Service call failed")
+        finally:
+            try:
+                service_proc.terminate()
+            except ProcessLookupError:
+                pass
 
     def test_080_qrexec_service_argument_allow_default(self):
         """Qrexec service call with argument"""
@@ -742,7 +738,8 @@ class TC_00_AppVMMixin(object):
         if self.template.startswith('whonix-'):
             self.skipTest('qvm-sync-clock disabled for Whonix VMs')
         self.loop.run_until_complete(asyncio.wait([
-            self.testvm1.start()]))
+            self.testvm1.start(),
+            self.testvm2.start(),]))
         start_time = subprocess.check_output(['date', '-u', '+%s'])
 
         try:
@@ -752,11 +749,11 @@ class TC_00_AppVMMixin(object):
             subprocess.check_call(['sudo', 'date', '-s', '2001-01-01T12:34:56'],
                 stdout=subprocess.DEVNULL)
             self.loop.run_until_complete(
-                self.testvm1.run_for_stdio('date -s 2001-01-01T12:34:56',
+                self.testvm2.run_for_stdio('date -s 2001-01-01T12:34:56',
                     user='root'))
 
             self.loop.run_until_complete(
-                self.testvm1.run_for_stdio('qvm-sync-clock',
+                self.testvm2.run_for_stdio('qvm-sync-clock',
                     user='root'))
 
             p = self.loop.run_until_complete(
@@ -765,7 +762,7 @@ class TC_00_AppVMMixin(object):
             self.loop.run_until_complete(p.wait())
             self.assertEqual(p.returncode, 0)
             vm_time, _ = self.loop.run_until_complete(
-                self.testvm1.run_for_stdio('date -u +%s'))
+                self.testvm2.run_for_stdio('date -u +%s'))
             self.assertAlmostEquals(int(vm_time), int(start_time), delta=30)
 
             dom0_time = subprocess.check_output(['date', '-u', '+%s'])
@@ -779,6 +776,154 @@ class TC_00_AppVMMixin(object):
         finally:
             self.app.clockvm = None
 
+    @unittest.skipUnless(spawn.find_executable('parecord'),
+                         "pulseaudio-utils not installed in dom0")
+    def test_220_audio_playback(self):
+        if 'whonix-gw' in self.template:
+            self.skipTest('whonix-gw have no audio')
+        self.loop.run_until_complete(self.testvm1.start())
+        try:
+            self.loop.run_until_complete(
+                self.testvm1.run_for_stdio('which parecord'))
+        except subprocess.CalledProcessError:
+            self.skipTest('pulseaudio-utils not installed in VM')
+
+        self.loop.run_until_complete(
+            self.wait_for_session(self.testvm1))
+        # and some more...
+        self.loop.run_until_complete(asyncio.sleep(1))
+        # generate some "audio" data
+        audio_in = b'\x20' * 44100
+        self.loop.run_until_complete(
+            self.testvm1.run_for_stdio('cat > audio_in.raw', input=audio_in))
+        local_user = grp.getgrnam('qubes').gr_mem[0]
+        with tempfile.NamedTemporaryFile() as recorded_audio:
+            os.chmod(recorded_audio.name, 0o666)
+            # FIXME: -d 0 assumes only one audio device
+            p = subprocess.Popen(['sudo', '-E', '-u', local_user,
+                'parecord', '-d', '0', '--raw', recorded_audio.name],
+                stdout=subprocess.PIPE)
+            self.loop.run_until_complete(
+                self.testvm1.run_for_stdio('paplay --raw audio_in.raw'))
+            # wait for possible parecord buffering
+            self.loop.run_until_complete(asyncio.sleep(1))
+            p.terminate()
+            # for some reason sudo do not relay SIGTERM sent above
+            subprocess.check_call(['pkill', 'parecord'])
+            p.wait()
+            # allow few bytes missing, don't use assertIn, to avoid printing
+            # the whole data in error message
+            if audio_in[:-8] not in recorded_audio.file.read():
+                self.fail('played sound not found in dom0')
+
+    def _configure_audio_recording(self, vm):
+        '''Connect VM's output-source to sink monitor instead of mic'''
+        local_user = grp.getgrnam('qubes').gr_mem[0]
+        sudo = ['sudo', '-E', '-u', local_user]
+        source_outputs = subprocess.check_output(
+            sudo + ['pacmd', 'list-source-outputs']).decode()
+
+        last_index = None
+        found = False
+        for line in source_outputs.splitlines():
+            if line.startswith('    index: '):
+                last_index = line.split(':')[1].strip()
+            elif line.startswith('\t\tapplication.name = '):
+                app_name = line.split('=')[1].strip('" ')
+                if vm.name == app_name:
+                    found = True
+                    break
+        if not found:
+            self.fail('source-output for VM {} not found'.format(vm.name))
+
+        subprocess.check_call(sudo +
+            ['pacmd', 'move-source-output', last_index, '0'])
+
+    @unittest.skipUnless(spawn.find_executable('parecord'),
+                         "pulseaudio-utils not installed in dom0")
+    def test_221_audio_record_muted(self):
+        if 'whonix-gw' in self.template:
+            self.skipTest('whonix-gw have no audio')
+        self.loop.run_until_complete(self.testvm1.start())
+        try:
+            self.loop.run_until_complete(
+                self.testvm1.run_for_stdio('which parecord'))
+        except subprocess.CalledProcessError:
+            self.skipTest('pulseaudio-utils not installed in VM')
+
+        self.loop.run_until_complete(
+            self.wait_for_session(self.testvm1))
+        # and some more...
+        self.loop.run_until_complete(asyncio.sleep(1))
+        # connect VM's recording source output monitor (instead of mic)
+        self._configure_audio_recording(self.testvm1)
+
+        # generate some "audio" data
+        audio_in = b'\x20' * 44100
+        local_user = grp.getgrnam('qubes').gr_mem[0]
+        record = self.loop.run_until_complete(
+            self.testvm1.run('parecord --raw audio_rec.raw'))
+        # give it time to start recording
+        self.loop.run_until_complete(asyncio.sleep(0.5))
+        p = subprocess.Popen(['sudo', '-E', '-u', local_user,
+            'paplay', '--raw'],
+            stdin=subprocess.PIPE)
+        p.communicate(audio_in)
+        # wait for possible parecord buffering
+        self.loop.run_until_complete(asyncio.sleep(1))
+        self.loop.run_until_complete(
+            self.testvm1.run_for_stdio('pkill parecord'))
+        record.wait()
+        recorded_audio, _ = self.loop.run_until_complete(
+            self.testvm1.run_for_stdio('cat audio_rec.raw'))
+        # should be empty or silence, so check just a little fragment
+        if audio_in[:32] in recorded_audio:
+            self.fail('VM recorded something, even though mic disabled')
+
+    @unittest.skipUnless(spawn.find_executable('parecord'),
+                         "pulseaudio-utils not installed in dom0")
+    def test_222_audio_record_unmuted(self):
+        if 'whonix-gw' in self.template:
+            self.skipTest('whonix-gw have no audio')
+        self.loop.run_until_complete(self.testvm1.start())
+        try:
+            self.loop.run_until_complete(
+                self.testvm1.run_for_stdio('which parecord'))
+        except subprocess.CalledProcessError:
+            self.skipTest('pulseaudio-utils not installed in VM')
+
+        self.loop.run_until_complete(
+            self.wait_for_session(self.testvm1))
+        # and some more...
+        self.loop.run_until_complete(asyncio.sleep(1))
+        da = qubes.devices.DeviceAssignment(self.app.domains[0], 'mic')
+        self.loop.run_until_complete(
+            self.testvm1.devices['mic'].attach(da))
+        # connect VM's recording source output monitor (instead of mic)
+        self._configure_audio_recording(self.testvm1)
+
+        # generate some "audio" data
+        audio_in = b'\x20' * 44100
+        local_user = grp.getgrnam('qubes').gr_mem[0]
+        record = self.loop.run_until_complete(
+            self.testvm1.run('parecord --raw audio_rec.raw'))
+        # give it time to start recording
+        self.loop.run_until_complete(asyncio.sleep(0.5))
+        p = subprocess.Popen(['sudo', '-E', '-u', local_user,
+            'paplay', '--raw'],
+            stdin=subprocess.PIPE)
+        p.communicate(audio_in)
+        # wait for possible parecord buffering
+        self.loop.run_until_complete(asyncio.sleep(1))
+        self.loop.run_until_complete(
+            self.testvm1.run_for_stdio('pkill parecord'))
+        record.wait()
+        recorded_audio, _ = self.loop.run_until_complete(
+            self.testvm1.run_for_stdio('cat audio_rec.raw'))
+        # allow few bytes to be missing
+        if audio_in[:-8] not in recorded_audio:
+            self.fail('VM not recorded expected data')
+
     def test_250_resize_private_img(self):
         """
         Test private.img resize, both offline and online
@@ -882,10 +1027,10 @@ int main(int argc, char **argv) {
             input=allocator_c.encode())
 
         try:
-            stdout, stderr = yield from self.testvm1.run_for_stdio(
+            yield from self.testvm1.run_for_stdio(
                 'gcc allocator.c -o allocator')
-        except subprocess.CalledProcessError:
-            self.skipTest('allocator compile failed: {}'.format(stderr))
+        except subprocess.CalledProcessError as e:
+            self.skipTest('allocator compile failed: {}'.format(e.stderr))
 
         # drop caches to have even more memory pressure
         yield from self.testvm1.run_for_stdio(
@@ -928,15 +1073,12 @@ int main(int argc, char **argv) {
         proc = yield from self.testvm1.run(
             'xterm -maximized -e top')
 
-        # help xdotool a little...
-        yield from asyncio.sleep(2)
         if proc.returncode is not None:
             self.fail('xterm failed to start')
         # get window ID
-        winid = (yield from asyncio.get_event_loop().run_in_executor(None,
-            subprocess.check_output,
-            ['xdotool', 'search', '--sync', '--onlyvisible', '--class',
-                self.testvm1.name + ':xterm'])).decode()
+        winid = yield from self.wait_for_window_coro(
+            self.testvm1.name + ':xterm',
+            search_class=True)
         xprop = yield from asyncio.get_event_loop().run_in_executor(None,
             subprocess.check_output,
             ['xprop', '-notype', '-id', winid, '_QUBES_VMWINDOWID'])
@@ -959,6 +1101,9 @@ int main(int argc, char **argv) {
         # wait for damage notify - top updates every 3 sec by default
         yield from asyncio.sleep(6)
 
+        # stop changing the window content
+        subprocess.check_call(['xdotool', 'key', '--window', winid, 'd'])
+
         # now take screenshot of the window, from dom0 and VM
         # choose pnm format, as it doesn't have any useless metadata - easy
         # to compare
@@ -1011,10 +1156,14 @@ class TC_10_Generic(qubes.tests.SystemTestCase):
             'Flag file created (service was run) even though should be denied,'
             ' qrexec-client-vm output: {} {}'.format(stdout, stderr))
 
+def create_testcases_for_templates():
+    return qubes.tests.create_testcases_for_templates('TC_00_AppVM',
+        TC_00_AppVMMixin, qubes.tests.SystemTestCase,
+        module=sys.modules[__name__])
 
 def load_tests(loader, tests, pattern):
     tests.addTests(loader.loadTestsFromNames(
-        qubes.tests.create_testcases_for_templates('TC_00_AppVM',
-            TC_00_AppVMMixin, qubes.tests.SystemTestCase,
-            module=sys.modules[__name__])))
+        create_testcases_for_templates()))
     return tests
+
+qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates)

qubes/tests/storage.py

@@ -22,6 +22,7 @@ import qubes.storage
 from qubes.exc import QubesException
 from qubes.storage import pool_drivers
 from qubes.storage.file import FilePool
+from qubes.storage.reflink import ReflinkPool
 from qubes.tests import SystemTestCase
 
 # :pylint: disable=invalid-name
@@ -107,10 +108,11 @@ class TC_00_Pool(SystemTestCase):
             pool_drivers())
 
     def test_002_get_pool_klass(self):
-        """ Expect the default pool to be `FilePool` """
+        """ Expect the default pool to be `FilePool` or `ReflinkPool` """
         # :pylint: disable=protected-access
         result = self.app.get_pool('varlibqubes')
-        self.assertIsInstance(result, FilePool)
+        self.assertTrue(isinstance(result, FilePool)
+                        or isinstance(result, ReflinkPool))
 
     def test_003_pool_exists_default(self):
         """ Expect the default pool to exists """

qubes/tests/storage_lvm.py

@@ -24,15 +24,15 @@
     'volume_group/thin_pool' combination. Pool variables without a prefix
     represent a :py:class:`qubes.storage.lvm.ThinPool`.
 '''
-
 import os
 import subprocess
 import tempfile
 import unittest
+import unittest.mock
 
 import qubes.tests
 import qubes.storage
-from qubes.storage.lvm import ThinPool, ThinVolume
+from qubes.storage.lvm import ThinPool, ThinVolume, qubes_lvm
 
 if 'DEFAULT_LVM_POOL' in os.environ.keys():
     DEFAULT_LVM_POOL = os.environ['DEFAULT_LVM_POOL']
@@ -84,7 +84,7 @@ class ThinPoolBase(qubes.tests.QubesTestCase):
         ''' Returns the pool matching the specified ``volume_group`` &
             ``thin_pool``, or None.
         '''
-        pools = [p for p in self.app.pools
+        pools = [p for p in self.app.pools.values()
             if issubclass(p.__class__, ThinPool)]
         for pool in pools:
             if pool.volume_group == volume_group \
@@ -136,10 +136,10 @@ class TC_00_ThinPool(ThinPoolBase):
         self.assertEqual(volume.name, 'root')
         self.assertEqual(volume.pool, self.pool.name)
         self.assertEqual(volume.size, qubes.config.defaults['root_img_size'])
-        volume.create()
+        self.loop.run_until_complete(volume.create())
         path = "/dev/%s" % volume.vid
-        self.assertTrue(os.path.exists(path))
-        volume.remove()
+        self.assertTrue(os.path.exists(path), path)
+        self.loop.run_until_complete(volume.remove())
 
     def test_003_read_write_volume(self):
         ''' Test read-write volume creation '''
@@ -156,10 +156,10 @@ class TC_00_ThinPool(ThinPoolBase):
         self.assertEqual(volume.name, 'root')
         self.assertEqual(volume.pool, self.pool.name)
         self.assertEqual(volume.size, qubes.config.defaults['root_img_size'])
-        volume.create()
+        self.loop.run_until_complete(volume.create())
         path = "/dev/%s" % volume.vid
-        self.assertTrue(os.path.exists(path))
-        volume.remove()
+        self.assertTrue(os.path.exists(path), path)
+        self.loop.run_until_complete(volume.remove())
 
     def test_004_size(self):
         with self.assertNotRaises(NotImplementedError):
@@ -207,11 +207,11 @@ class TC_00_ThinPool(ThinPoolBase):
         }
         vm = qubes.tests.storage.TestVM(self)
         volume = self.app.get_pool(self.pool.name).init_volume(vm, config)
-        volume.create()
-        self.addCleanup(volume.remove)
+        self.loop.run_until_complete(volume.create())
+        self.addCleanup(self.loop.run_until_complete, volume.remove())
         path = "/dev/%s" % volume.vid
         new_size = 64 * 1024 ** 2
-        volume.resize(new_size)
+        self.loop.run_until_complete(volume.resize(new_size))
         self.assertEqual(self._get_size(path), new_size)
         self.assertEqual(volume.size, new_size)
 
@@ -226,20 +226,711 @@ class TC_00_ThinPool(ThinPoolBase):
         }
         vm = qubes.tests.storage.TestVM(self)
         volume = self.app.get_pool(self.pool.name).init_volume(vm, config)
-        volume.create()
-        self.addCleanup(volume.remove)
-        volume.start()
+        self.loop.run_until_complete(volume.create())
+        self.addCleanup(self.loop.run_until_complete, volume.remove())
+        self.loop.run_until_complete(volume.start())
         path = "/dev/%s" % volume.vid
         path2 = "/dev/%s" % volume._vid_snap
         new_size = 64 * 1024 ** 2
-        volume.resize(new_size)
+        self.loop.run_until_complete(volume.resize(new_size))
         self.assertEqual(self._get_size(path), old_size)
         self.assertEqual(self._get_size(path2), new_size)
         self.assertEqual(volume.size, new_size)
-        volume.stop()
+        self.loop.run_until_complete(volume.stop())
         self.assertEqual(self._get_size(path), new_size)
         self.assertEqual(volume.size, new_size)
 
+    def _get_lv_uuid(self, lv):
+        sudo = [] if os.getuid() == 0 else ['sudo']
+        lvs_output = subprocess.check_output(
+            sudo + ['lvs', '--noheadings', '-o', 'lv_uuid', lv])
+        return lvs_output.strip()
+
+    def _get_lv_origin_uuid(self, lv):
+        sudo = [] if os.getuid() == 0 else ['sudo']
+        if qubes.storage.lvm.lvm_is_very_old:
+            # no support for origin_uuid directly
+            lvs_output = subprocess.check_output(
+                sudo + ['lvs', '--noheadings', '-o', 'origin', lv])
+            lvs_output = subprocess.check_output(
+                sudo + ['lvs', '--noheadings', '-o', 'lv_uuid',
+                    lv.rsplit('/', 1)[0] + '/' + lvs_output.strip().decode()])
+        else:
+            lvs_output = subprocess.check_output(
+                sudo + ['lvs', '--noheadings', '-o', 'origin_uuid', lv])
+        return lvs_output.strip()
+
+    def test_008_commit(self):
+        ''' Test volume changes commit'''
+        config = {
+            'name': 'root',
+            'pool': self.pool.name,
+            'save_on_stop': True,
+            'rw': True,
+            'size': qubes.config.defaults['root_img_size'],
+        }
+        vm = qubes.tests.storage.TestVM(self)
+        volume = self.app.get_pool(self.pool.name).init_volume(vm, config)
+        self.loop.run_until_complete(volume.create())
+        path_snap = '/dev/' + volume._vid_snap
+        self.assertFalse(os.path.exists(path_snap), path_snap)
+        origin_uuid = self._get_lv_uuid(volume.path)
+        self.loop.run_until_complete(volume.start())
+        snap_uuid = self._get_lv_uuid(path_snap)
+        self.assertNotEqual(origin_uuid, snap_uuid)
+        path = volume.path
+        self.assertTrue(path.startswith('/dev/' + volume.vid),
+                        '{} does not start with /dev/{}'.format(path, volume.vid))
+        self.assertTrue(os.path.exists(path), path)
+        self.loop.run_until_complete(volume.remove())
+
+    def test_009_interrupted_commit(self):
+        ''' Test volume changes commit'''
+        config = {
+            'name': 'root',
+            'pool': self.pool.name,
+            'save_on_stop': True,
+            'rw': True,
+            'revisions_to_keep': 2,
+            'size': qubes.config.defaults['root_img_size'],
+        }
+        vm = qubes.tests.storage.TestVM(self)
+        volume = self.app.get_pool(self.pool.name).init_volume(vm, config)
+        # mock logging, to not interfere with time.time() mock
+        volume.log = unittest.mock.Mock()
+        # do not call volume.create(), do it manually to simulate
+        # interrupted commit
+        revisions = ['-1521065904-back', '-1521065905-back', '-snap']
+        orig_uuids = {}
+        for rev in revisions:
+            cmd = ['create', self.pool._pool_id,
+                   volume.vid.split('/')[1] + rev, str(config['size'])]
+            qubes_lvm(cmd)
+            orig_uuids[rev] = self._get_lv_uuid(volume.vid + rev)
+        qubes.storage.lvm.reset_cache()
+        path_snap = '/dev/' + volume._vid_snap
+        self.assertTrue(volume.is_dirty())
+        self.assertEqual(volume.path,
+                         '/dev/' + volume.vid + revisions[1])
+        expected_revisions = {
+            revisions[0].lstrip('-'): '2018-03-14T22:18:24',
+            revisions[1].lstrip('-'): '2018-03-14T22:18:25',
+        }
+        self.assertEqual(volume.revisions, expected_revisions)
+        self.loop.run_until_complete(volume.start())
+        self.assertEqual(volume.revisions, expected_revisions)
+        snap_uuid = self._get_lv_uuid(path_snap)
+        self.assertEqual(orig_uuids['-snap'], snap_uuid)
+        self.assertTrue(volume.is_dirty())
+        self.assertEqual(volume.path,
+                         '/dev/' + volume.vid + revisions[1])
+        with unittest.mock.patch('time.time') as mock_time:
+            mock_time.side_effect = [521065906]
+            self.loop.run_until_complete(volume.stop())
+        expected_revisions = {
+            revisions[0].lstrip('-'): '2018-03-14T22:18:24',
+            revisions[1].lstrip('-'): '2018-03-14T22:18:25',
+        }
+        self.assertFalse(volume.is_dirty())
+        self.assertEqual(volume.revisions, expected_revisions)
+        self.assertEqual(volume.path, '/dev/' + volume.vid)
+        self.assertEqual(snap_uuid, self._get_lv_uuid(volume.path))
+        self.assertFalse(os.path.exists(path_snap), path_snap)
+
+        self.loop.run_until_complete(volume.remove())
+
+    def test_010_migration1(self):
+        '''Start with old revisions, then start interacting using new code'''
+        config = {
+            'name': 'root',
+            'pool': self.pool.name,
+            'save_on_stop': True,
+            'rw': True,
+            'revisions_to_keep': 2,
+            'size': qubes.config.defaults['root_img_size'],
+        }
+        vm = qubes.tests.storage.TestVM(self)
+        volume = self.app.get_pool(self.pool.name).init_volume(vm, config)
+        # mock logging, to not interfere with time.time() mock
+        volume.log = unittest.mock.Mock()
+        # do not call volume.create(), do it manually to have old LV naming
+        revisions = ['', '-1521065904-back', '-1521065905-back']
+        orig_uuids = {}
+        for rev in revisions:
+            cmd = ['create', self.pool._pool_id,
+                   volume.vid.split('/')[1] + rev, str(config['size'])]
+            qubes_lvm(cmd)
+            orig_uuids[rev] = self._get_lv_uuid(volume.vid + rev)
+        qubes.storage.lvm.reset_cache()
+        path_snap = '/dev/' + volume._vid_snap
+        self.assertFalse(os.path.exists(path_snap), path_snap)
+        expected_revisions = {
+            revisions[1].lstrip('-'): '2018-03-14T22:18:24',
+            revisions[2].lstrip('-'): '2018-03-14T22:18:25',
+        }
+        self.assertEqual(volume.revisions, expected_revisions)
+        self.assertEqual(volume.path, '/dev/' + volume.vid)
+
+        self.loop.run_until_complete(volume.start())
+        snap_uuid = self._get_lv_uuid(path_snap)
+        self.assertNotEqual(orig_uuids[''], snap_uuid)
+        snap_origin_uuid = self._get_lv_origin_uuid(path_snap)
+        self.assertEqual(orig_uuids[''], snap_origin_uuid)
+        path = volume.path
+        self.assertEqual(path, '/dev/' + volume.vid)
+        self.assertTrue(os.path.exists(path), path)
+
+        with unittest.mock.patch('time.time') as mock_time:
+            mock_time.side_effect = ('1521065906', '1521065907')
+            self.loop.run_until_complete(volume.stop())
+        revisions.extend(['-1521065906-back'])
+        expected_revisions = {
+            revisions[2].lstrip('-'): '2018-03-14T22:18:25',
+            revisions[3].lstrip('-'): '2018-03-14T22:18:26',
+        }
+        self.assertEqual(volume.revisions, expected_revisions)
+        self.assertEqual(volume.path, '/dev/' + volume.vid)
+        path_snap = '/dev/' + volume._vid_snap
+        self.assertFalse(os.path.exists(path_snap), path_snap)
+        self.assertTrue(os.path.exists('/dev/' + volume.vid))
+        self.assertEqual(self._get_lv_uuid(volume.path), snap_uuid)
+        prev_path = '/dev/' + volume.vid + revisions[3]
+        self.assertEqual(self._get_lv_uuid(prev_path), orig_uuids[''])
+
+        self.loop.run_until_complete(volume.remove())
+        for rev in revisions:
+            path = '/dev/' + volume.vid + rev
+            self.assertFalse(os.path.exists(path), path)
+
+    def test_011_migration2(self):
+        '''VM started with old code, stopped with new'''
+        config = {
+            'name': 'root',
+            'pool': self.pool.name,
+            'save_on_stop': True,
+            'rw': True,
+            'revisions_to_keep': 1,
+            'size': qubes.config.defaults['root_img_size'],
+        }
+        vm = qubes.tests.storage.TestVM(self)
+        volume = self.app.get_pool(self.pool.name).init_volume(vm, config)
+        # mock logging, to not interfere with time.time() mock
+        volume.log = unittest.mock.Mock()
+        # do not call volume.create(), do it manually to have old LV naming
+        revisions = ['', '-snap']
+        orig_uuids = {}
+        for rev in revisions:
+            cmd = ['create', self.pool._pool_id,
+                   volume.vid.split('/')[1] + rev, str(config['size'])]
+            qubes_lvm(cmd)
+            orig_uuids[rev] = self._get_lv_uuid(volume.vid + rev)
+        qubes.storage.lvm.reset_cache()
+        path_snap = '/dev/' + volume._vid_snap
+        self.assertTrue(os.path.exists(path_snap), path_snap)
+        expected_revisions = {}
+        self.assertEqual(volume.revisions, expected_revisions)
+        self.assertEqual(volume.path, '/dev/' + volume.vid)
+        self.assertTrue(volume.is_dirty())
+
+        path = volume.path
+        self.assertEqual(path, '/dev/' + volume.vid)
+        self.assertTrue(os.path.exists(path), path)
+
+        with unittest.mock.patch('time.time') as mock_time:
+            mock_time.side_effect = ('1521065906', '1521065907')
+            self.loop.run_until_complete(volume.stop())
+        revisions.extend(['-1521065906-back'])
+        expected_revisions = {
+            revisions[2].lstrip('-'): '2018-03-14T22:18:26',
+        }
+        self.assertEqual(volume.revisions, expected_revisions)
+        self.assertEqual(volume.path, '/dev/' + volume.vid)
+        path_snap = '/dev/' + volume._vid_snap
+        self.assertFalse(os.path.exists(path_snap), path_snap)
+        self.assertTrue(os.path.exists('/dev/' + volume.vid))
+        self.assertEqual(self._get_lv_uuid(volume.path), orig_uuids['-snap'])
+        prev_path = '/dev/' + volume.vid + revisions[2]
+        self.assertEqual(self._get_lv_uuid(prev_path), orig_uuids[''])
+
+        self.loop.run_until_complete(volume.remove())
+        for rev in revisions:
+            path = '/dev/' + volume.vid + rev
+            self.assertFalse(os.path.exists(path), path)
+
+    def test_012_migration3(self):
+        '''VM started with old code, started again with new, stopped with new'''
+        config = {
+            'name': 'root',
+            'pool': self.pool.name,
+            'save_on_stop': True,
+            'rw': True,
+            'revisions_to_keep': 1,
+            'size': qubes.config.defaults['root_img_size'],
+        }
+        vm = qubes.tests.storage.TestVM(self)
+        volume = self.app.get_pool(self.pool.name).init_volume(vm, config)
+        # mock logging, to not interfere with time.time() mock
+        volume.log = unittest.mock.Mock()
+        # do not call volume.create(), do it manually to have old LV naming
+        revisions = ['', '-snap']
+        orig_uuids = {}
+        for rev in revisions:
+            cmd = ['create', self.pool._pool_id,
+                   volume.vid.split('/')[1] + rev, str(config['size'])]
+            qubes_lvm(cmd)
+            orig_uuids[rev] = self._get_lv_uuid(volume.vid + rev)
+        qubes.storage.lvm.reset_cache()
+        path_snap = '/dev/' + volume._vid_snap
+        self.assertTrue(os.path.exists(path_snap), path_snap)
+        expected_revisions = {}
+        self.assertEqual(volume.revisions, expected_revisions)
+        self.assertTrue(volume.path, '/dev/' + volume.vid)
+        self.assertTrue(volume.is_dirty())
+
+        self.loop.run_until_complete(volume.start())
+        self.assertEqual(volume.revisions, expected_revisions)
+        self.assertEqual(volume.path, '/dev/' + volume.vid)
+        # -snap LV should be unchanged
+        self.assertEqual(self._get_lv_uuid(volume._vid_snap),
+                         orig_uuids['-snap'])
+
+        self.loop.run_until_complete(volume.remove())
+        for rev in revisions:
+            path = '/dev/' + volume.vid + rev
+            self.assertFalse(os.path.exists(path), path)
+
+    def test_013_migration4(self):
+        '''revisions_to_keep=0, VM started with old code, stopped with new'''
+        config = {
+            'name': 'root',
+            'pool': self.pool.name,
+            'save_on_stop': True,
+            'rw': True,
+            'revisions_to_keep': 0,
+            'size': qubes.config.defaults['root_img_size'],
+        }
+        vm = qubes.tests.storage.TestVM(self)
+        volume = self.app.get_pool(self.pool.name).init_volume(vm, config)
+        # mock logging, to not interfere with time.time() mock
+        volume.log = unittest.mock.Mock()
+        # do not call volume.create(), do it manually to have old LV naming
+        revisions = ['', '-snap']
+        orig_uuids = {}
+        for rev in revisions:
+            cmd = ['create', self.pool._pool_id,
+                   volume.vid.split('/')[1] + rev, str(config['size'])]
+            qubes_lvm(cmd)
+            orig_uuids[rev] = self._get_lv_uuid(volume.vid + rev)
+        qubes.storage.lvm.reset_cache()
+        path_snap = '/dev/' + volume._vid_snap
+        self.assertTrue(os.path.exists(path_snap), path_snap)
+        expected_revisions = {}
+        self.assertEqual(volume.revisions, expected_revisions)
+        self.assertEqual(volume.path, '/dev/' + volume.vid)
+        self.assertTrue(volume.is_dirty())
+
+        with unittest.mock.patch('time.time') as mock_time:
+            mock_time.side_effect = ('1521065906', '1521065907')
+            self.loop.run_until_complete(volume.stop())
+        expected_revisions = {}
+        self.assertEqual(volume.revisions, expected_revisions)
+        self.assertEqual(volume.path, '/dev/' + volume.vid)
+
+        self.loop.run_until_complete(volume.remove())
+        for rev in revisions:
+            path = '/dev/' + volume.vid + rev
+            self.assertFalse(os.path.exists(path), path)
+
+    def test_014_commit_keep_0(self):
+        ''' Test volume changes commit, with revisions_to_keep=0'''
+        config = {
+            'name': 'root',
+            'pool': self.pool.name,
+            'save_on_stop': True,
+            'rw': True,
+            'revisions_to_keep': 0,
+            'size': qubes.config.defaults['root_img_size'],
+        }
+        vm = qubes.tests.storage.TestVM(self)
+        volume = self.app.get_pool(self.pool.name).init_volume(vm, config)
+        # mock logging, to not interfere with time.time() mock
+        volume.log = unittest.mock.Mock()
+        self.loop.run_until_complete(volume.create())
+        self.assertFalse(volume.is_dirty())
+        path = volume.path
+        expected_revisions = {}
+        self.assertEqual(volume.revisions, expected_revisions)
+
+        self.loop.run_until_complete(volume.start())
+        self.assertEqual(volume.revisions, expected_revisions)
+        path_snap = '/dev/' + volume._vid_snap
+        snap_uuid = self._get_lv_uuid(path_snap)
+        self.assertTrue(volume.is_dirty())
+        self.assertEqual(volume.path, path)
+
+        with unittest.mock.patch('time.time') as mock_time:
+            mock_time.side_effect = [521065906]
+            self.loop.run_until_complete(volume.stop())
+        self.assertFalse(volume.is_dirty())
+        self.assertEqual(volume.revisions, {})
+        self.assertEqual(volume.path, '/dev/' + volume.vid)
+        self.assertEqual(snap_uuid, self._get_lv_uuid(volume.path))
+        self.assertFalse(os.path.exists(path_snap), path_snap)
+
+        self.loop.run_until_complete(volume.remove())
+
+    def test_020_revert_last(self):
+        ''' Test volume revert'''
+        config = {
+            'name': 'root',
+            'pool': self.pool.name,
+            'save_on_stop': True,
+            'rw': True,
+            'revisions_to_keep': 2,
+            'size': qubes.config.defaults['root_img_size'],
+        }
+        vm = qubes.tests.storage.TestVM(self)
+        volume = self.app.get_pool(self.pool.name).init_volume(vm, config)
+        self.loop.run_until_complete(volume.create())
+        self.loop.run_until_complete(volume.start())
+        self.loop.run_until_complete(volume.stop())
+        self.loop.run_until_complete(volume.start())
+        self.loop.run_until_complete(volume.stop())
+        self.assertEqual(len(volume.revisions), 2)
+        revisions = volume.revisions
+        revision_id = max(revisions.keys())
+        current_path = volume.path
+        current_uuid = self._get_lv_uuid(volume.path)
+        rev_uuid = self._get_lv_uuid(volume.vid + '-' + revision_id)
+        self.assertFalse(volume.is_dirty())
+        self.assertNotEqual(current_uuid, rev_uuid)
+        self.loop.run_until_complete(volume.revert())
+        path_snap = '/dev/' + volume._vid_snap
+        self.assertFalse(os.path.exists(path_snap), path_snap)
+        self.assertEqual(current_path, volume.path)
+        new_uuid = self._get_lv_origin_uuid(volume.path)
+        self.assertEqual(new_uuid, rev_uuid)
+        self.assertEqual(volume.revisions, revisions)
+
+        self.loop.run_until_complete(volume.remove())
+
+    def test_021_revert_earlier(self):
+        ''' Test volume revert'''
+        config = {
+            'name': 'root',
+            'pool': self.pool.name,
+            'save_on_stop': True,
+            'rw': True,
+            'revisions_to_keep': 2,
+            'size': qubes.config.defaults['root_img_size'],
+        }
+        vm = qubes.tests.storage.TestVM(self)
+        volume = self.app.get_pool(self.pool.name).init_volume(vm, config)
+        self.loop.run_until_complete(volume.create())
+        self.loop.run_until_complete(volume.start())
+        self.loop.run_until_complete(volume.stop())
+        self.loop.run_until_complete(volume.start())
+        self.loop.run_until_complete(volume.stop())
+        self.assertEqual(len(volume.revisions), 2)
+        revisions = volume.revisions
+        revision_id = min(revisions.keys())
+        current_path = volume.path
+        current_uuid = self._get_lv_uuid(volume.path)
+        rev_uuid = self._get_lv_uuid(volume.vid + '-' + revision_id)
+        self.assertFalse(volume.is_dirty())
+        self.assertNotEqual(current_uuid, rev_uuid)
+        self.loop.run_until_complete(volume.revert(revision_id))
+        path_snap = '/dev/' + volume._vid_snap
+        self.assertFalse(os.path.exists(path_snap), path_snap)
+        self.assertEqual(current_path, volume.path)
+        new_uuid = self._get_lv_origin_uuid(volume.path)
+        self.assertEqual(new_uuid, rev_uuid)
+        self.assertEqual(volume.revisions, revisions)
+
+        self.loop.run_until_complete(volume.remove())
+
+    def test_030_import_data(self):
+        ''' Test volume import'''
+        config = {
+            'name': 'root',
+            'pool': self.pool.name,
+            'save_on_stop': True,
+            'rw': True,
+            'revisions_to_keep': 2,
+            'size': qubes.config.defaults['root_img_size'],
+        }
+        vm = qubes.tests.storage.TestVM(self)
+        volume = self.app.get_pool(self.pool.name).init_volume(vm, config)
+        self.loop.run_until_complete(volume.create())
+        current_uuid = self._get_lv_uuid(volume.path)
+        self.assertFalse(volume.is_dirty())
+        import_path = self.loop.run_until_complete(volume.import_data())
+        import_uuid = self._get_lv_uuid(import_path)
+        self.assertNotEqual(current_uuid, import_uuid)
+        # success - commit data
+        self.loop.run_until_complete(volume.import_data_end(True))
+        new_current_uuid = self._get_lv_uuid(volume.path)
+        self.assertEqual(new_current_uuid, import_uuid)
+        revisions = volume.revisions
+        self.assertEqual(len(revisions), 1)
+        revision = revisions.popitem()[0]
+        self.assertEqual(current_uuid,
+                         self._get_lv_uuid(volume.vid + '-' + revision))
+        self.assertFalse(os.path.exists(import_path), import_path)
+
+        self.loop.run_until_complete(volume.remove())
+
+    def test_031_import_data_fail(self):
+        ''' Test volume import'''
+        config = {
+            'name': 'root',
+            'pool': self.pool.name,
+            'save_on_stop': True,
+            'rw': True,
+            'revisions_to_keep': 2,
+            'size': qubes.config.defaults['root_img_size'],
+        }
+        vm = qubes.tests.storage.TestVM(self)
+        volume = self.app.get_pool(self.pool.name).init_volume(vm, config)
+        self.loop.run_until_complete(volume.create())
+        current_uuid = self._get_lv_uuid(volume.path)
+        self.assertFalse(volume.is_dirty())
+        import_path = self.loop.run_until_complete(volume.import_data())
+        import_uuid = self._get_lv_uuid(import_path)
+        self.assertNotEqual(current_uuid, import_uuid)
+        # fail - discard data
+        self.loop.run_until_complete(volume.import_data_end(False))
+        new_current_uuid = self._get_lv_uuid(volume.path)
+        self.assertEqual(new_current_uuid, current_uuid)
+        revisions = volume.revisions
+        self.assertEqual(len(revisions), 0)
+        self.assertFalse(os.path.exists(import_path), import_path)
+
+        self.loop.run_until_complete(volume.remove())
+
+    def test_032_import_volume_same_pool(self):
+        '''Import volume from the same pool'''
+        # source volume
+        config = {
+            'name': 'root',
+            'pool': self.pool.name,
+            'save_on_stop': True,
+            'rw': True,
+            'revisions_to_keep': 2,
+            'size': qubes.config.defaults['root_img_size'],
+        }
+        vm = qubes.tests.storage.TestVM(self)
+        source_volume = self.app.get_pool(self.pool.name).init_volume(vm, config)
+        self.loop.run_until_complete(source_volume.create())
+
+        source_uuid = self._get_lv_uuid(source_volume.path)
+
+        # destination volume
+        config = {
+            'name': 'root2',
+            'pool': self.pool.name,
+            'save_on_stop': True,
+            'rw': True,
+            'revisions_to_keep': 2,
+            'size': qubes.config.defaults['root_img_size'],
+        }
+        volume = self.app.get_pool(self.pool.name).init_volume(vm, config)
+        volume.log = unittest.mock.Mock()
+        with unittest.mock.patch('time.time') as mock_time:
+            mock_time.side_effect = [1521065905]
+            self.loop.run_until_complete(volume.create())
+
+        self.assertEqual(volume.revisions, {})
+        uuid_before = self._get_lv_uuid(volume.path)
+
+        with unittest.mock.patch('time.time') as mock_time:
+            mock_time.side_effect = [1521065906]
+            self.loop.run_until_complete(
+                volume.import_volume(source_volume))
+
+        uuid_after = self._get_lv_uuid(volume.path)
+        self.assertNotEqual(uuid_after, uuid_before)
+
+        # also should be different than source volume (clone, not the same LV)
+        self.assertNotEqual(uuid_after, source_uuid)
+        self.assertEqual(self._get_lv_origin_uuid(volume.path), source_uuid)
+
+        expected_revisions = {
+            '1521065906-back': '2018-03-14T22:18:26',
+        }
+        self.assertEqual(volume.revisions, expected_revisions)
+
+        self.loop.run_until_complete(volume.remove())
+        self.loop.run_until_complete(source_volume.remove())
+
+    def test_033_import_volume_different_pool(self):
+        '''Import volume from a different pool'''
+        source_volume = unittest.mock.Mock()
+        # destination volume
+        config = {
+            'name': 'root2',
+            'pool': self.pool.name,
+            'save_on_stop': True,
+            'rw': True,
+            'revisions_to_keep': 2,
+            'size': qubes.config.defaults['root_img_size'],
+        }
+        vm = qubes.tests.storage.TestVM(self)
+        volume = self.app.get_pool(self.pool.name).init_volume(vm, config)
+        volume.log = unittest.mock.Mock()
+        with unittest.mock.patch('time.time') as mock_time:
+            mock_time.side_effect = [1521065905]
+            self.loop.run_until_complete(volume.create())
+
+        self.assertEqual(volume.revisions, {})
+        uuid_before = self._get_lv_uuid(volume.path)
+
+        with tempfile.NamedTemporaryFile() as source_volume_file:
+            source_volume_file.write(b'test-content')
+            source_volume_file.flush()
+            source_volume.size = 16 * 1024 * 1024  # 16MiB
+            source_volume.export.return_value = source_volume_file.name
+            with unittest.mock.patch('time.time') as mock_time:
+                mock_time.side_effect = [1521065906]
+                self.loop.run_until_complete(
+                    volume.import_volume(source_volume))
+
+        uuid_after = self._get_lv_uuid(volume.path)
+        self.assertNotEqual(uuid_after, uuid_before)
+        self.assertEqual(volume.size, 16 * 1024 * 1024)
+
+        volume_content = subprocess.check_output(['sudo', 'cat', volume.path])
+        self.assertEqual(volume_content.rstrip(b'\0'), b'test-content')
+
+        expected_revisions = {
+            '1521065906-back': '2018-03-14T22:18:26',
+        }
+        self.assertEqual(volume.revisions, expected_revisions)
+
+        self.loop.run_until_complete(volume.remove())
+
+    def test_040_volatile(self):
+        '''Volatile volume test'''
+        config = {
+            'name': 'volatile',
+            'pool': self.pool.name,
+            'rw': True,
+            'size': qubes.config.defaults['root_img_size'],
+        }
+        vm = qubes.tests.storage.TestVM(self)
+        volume = self.app.get_pool(self.pool.name).init_volume(vm, config)
+        # volatile volume don't need any file, verify should succeed
+        self.assertTrue(volume.verify())
+        self.loop.run_until_complete(volume.create())
+        self.assertTrue(volume.verify())
+        self.assertFalse(volume.save_on_stop)
+        self.assertFalse(volume.snap_on_start)
+        path = volume.path
+        self.assertEqual(path, '/dev/' + volume.vid)
+        self.assertFalse(os.path.exists(path))
+        self.loop.run_until_complete(volume.start())
+        self.assertTrue(os.path.exists(path))
+        vol_uuid = self._get_lv_uuid(path)
+        self.loop.run_until_complete(volume.start())
+        self.assertTrue(os.path.exists(path))
+        vol_uuid2 = self._get_lv_uuid(path)
+        self.assertNotEqual(vol_uuid, vol_uuid2)
+        self.loop.run_until_complete(volume.stop())
+        self.assertFalse(os.path.exists(path))
+
+    def test_050_snapshot_volume(self):
+        ''' Test snapshot volume creation '''
+        config_origin = {
+            'name': 'root',
+            'pool': self.pool.name,
+            'save_on_stop': True,
+            'rw': True,
+            'size': qubes.config.defaults['root_img_size'],
+        }
+        vm = qubes.tests.storage.TestVM(self)
+        volume_origin = self.app.get_pool(self.pool.name).init_volume(
+            vm, config_origin)
+        self.loop.run_until_complete(volume_origin.create())
+        config_snapshot = {
+            'name': 'root2',
+            'pool': self.pool.name,
+            'snap_on_start': True,
+            'source': volume_origin,
+            'rw': True,
+            'size': qubes.config.defaults['root_img_size'],
+        }
+        volume = self.app.get_pool(self.pool.name).init_volume(
+            vm, config_snapshot)
+        self.assertIsInstance(volume, ThinVolume)
+        self.assertEqual(volume.name, 'root2')
+        self.assertEqual(volume.pool, self.pool.name)
+        self.assertEqual(volume.size, qubes.config.defaults['root_img_size'])
+        # only origin volume really needs to exist, verify should succeed
+        # even before create
+        self.assertTrue(volume.verify())
+        self.loop.run_until_complete(volume.create())
+        path = volume.path
+        self.assertEqual(path, '/dev/' + volume.vid)
+        self.assertFalse(os.path.exists(path), path)
+        self.loop.run_until_complete(volume.start())
+        # snapshot volume isn't considered dirty at any time
+        self.assertFalse(volume.is_dirty())
+        # not outdated yet
+        self.assertFalse(volume.is_outdated())
+        origin_uuid = self._get_lv_uuid(volume_origin.path)
+        snap_origin_uuid = self._get_lv_origin_uuid(volume._vid_snap)
+        self.assertEqual(origin_uuid, snap_origin_uuid)
+
+        # now make it outdated
+        self.loop.run_until_complete(volume_origin.start())
+        self.loop.run_until_complete(volume_origin.stop())
+        self.assertTrue(volume.is_outdated())
+        origin_uuid = self._get_lv_uuid(volume_origin.path)
+        self.assertNotEqual(origin_uuid, snap_origin_uuid)
+
+        self.loop.run_until_complete(volume.stop())
+        # stopped volume is never outdated
+        self.assertFalse(volume.is_outdated())
+        path = volume.path
+        self.assertFalse(os.path.exists(path), path)
+        path = '/dev/' + volume._vid_snap
+        self.assertFalse(os.path.exists(path), path)
+
+        self.loop.run_until_complete(volume.remove())
+        self.loop.run_until_complete(volume_origin.remove())
+
+    def test_100_pool_list_volumes(self):
+        config = {
+            'name': 'root',
+            'pool': self.pool.name,
+            'save_on_stop': True,
+            'rw': True,
+            'revisions_to_keep': 2,
+            'size': qubes.config.defaults['root_img_size'],
+        }
+        config2 = config.copy()
+        vm = qubes.tests.storage.TestVM(self)
+        volume1 = self.app.get_pool(self.pool.name).init_volume(vm, config)
+        self.loop.run_until_complete(volume1.create())
+        config2['name'] = 'private'
+        volume2 = self.app.get_pool(self.pool.name).init_volume(vm, config2)
+        self.loop.run_until_complete(volume2.create())
+
+        # create some revisions
+        self.loop.run_until_complete(volume1.start())
+        self.loop.run_until_complete(volume1.stop())
+
+        # and have one in dirty state
+        self.loop.run_until_complete(volume2.start())
+
+        self.assertIn(volume1, list(self.pool.volumes))
+        self.assertIn(volume2, list(self.pool.volumes))
+        self.loop.run_until_complete(volume1.remove())
+        self.assertNotIn(volume1, list(self.pool.volumes))
+        self.assertIn(volume2, list(self.pool.volumes))
+        self.loop.run_until_complete(volume2.remove())
+        self.assertNotIn(volume1, list(self.pool.volumes))
+        self.assertNotIn(volume1, list(self.pool.volumes))
 
 @skipUnlessLvmPoolExists
 class TC_01_ThinPool(ThinPoolBase, qubes.tests.SystemTestCase):
@@ -255,26 +946,27 @@ class TC_01_ThinPool(ThinPoolBase, qubes.tests.SystemTestCase):
         vm = self.app.add_new_vm(qubes.vm.templatevm.TemplateVM, name=name,
                             label='red')
         vm.clone_properties(template_vm)
-        vm.clone_disk_files(template_vm, pool='test-lvm')
+        self.loop.run_until_complete(
+            vm.clone_disk_files(template_vm, pool=self.pool.name))
         for v_name, volume in vm.volumes.items():
             if volume.save_on_stop:
                 expected = "/dev/{!s}/vm-{!s}-{!s}".format(
                     DEFAULT_LVM_POOL.split('/')[0], vm.name, v_name)
                 self.assertEqual(volume.path, expected)
         with self.assertNotRaises(qubes.exc.QubesException):
-            vm.start()
+            self.loop.run_until_complete(vm.start())
 
     def test_005_create_appvm(self):
         vm = self.app.add_new_vm(cls=qubes.vm.appvm.AppVM,
                                  name=self.make_vm_name('appvm'), label='red')
-        vm.create_on_disk(pool='test-lvm')
+        self.loop.run_until_complete(vm.create_on_disk(pool=self.pool.name))
         for v_name, volume in vm.volumes.items():
             if volume.save_on_stop:
                 expected = "/dev/{!s}/vm-{!s}-{!s}".format(
                     DEFAULT_LVM_POOL.split('/')[0], vm.name, v_name)
                 self.assertEqual(volume.path, expected)
         with self.assertNotRaises(qubes.exc.QubesException):
-            vm.start()
+            self.loop.run_until_complete(vm.start())
 
 @skipUnlessLvmPoolExists
 class TC_02_StorageHelpers(ThinPoolBase):

qubes/tests/storage_reflink.py

@@ -0,0 +1,154 @@
+#
+# The Qubes OS Project, https://www.qubes-os.org
+#
+# Copyright (C) 2018 Rusty Bird <rustybird@net-c.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, see <https://www.gnu.org/licenses/>.
+#
+
+''' Tests for the file-reflink storage driver '''
+
+# pylint: disable=protected-access
+# pylint: disable=invalid-name
+
+import os
+import shutil
+import subprocess
+import sys
+
+import qubes.tests
+from qubes.storage import reflink
+
+
+class ReflinkMixin:
+    def setUp(self, fs_type='btrfs'):  # pylint: disable=arguments-differ
+        super().setUp()
+        self.test_dir = '/var/tmp/test-reflink-units-on-' + fs_type
+        mkdir_fs(self.test_dir, fs_type, cleanup_via=self.addCleanup)
+
+    def test_000_copy_file(self):
+        source = os.path.join(self.test_dir, 'source-file')
+        dest = os.path.join(self.test_dir, 'new-directory', 'dest-file')
+        content = os.urandom(1024**2)
+
+        with open(source, 'wb') as source_io:
+            source_io.write(content)
+
+        ficlone_succeeded = reflink._copy_file(source, dest)
+        self.assertEqual(ficlone_succeeded, self.ficlone_supported)
+
+        self.assertNotEqual(os.stat(source).st_ino, os.stat(dest).st_ino)
+        with open(source, 'rb') as source_io:
+            self.assertEqual(source_io.read(), content)
+        with open(dest, 'rb') as dest_io:
+            self.assertEqual(dest_io.read(), content)
+
+    def test_001_create_and_resize_files_and_update_loopdevs(self):
+        img_real = os.path.join(self.test_dir, 'img-real')
+        img_sym = os.path.join(self.test_dir, 'img-sym')
+        size_initial = 111 * 1024**2
+        size_resized = 222 * 1024**2
+
+        os.symlink(img_real, img_sym)
+        reflink._create_sparse_file(img_real, size_initial)
+        self.assertEqual(reflink._get_file_disk_usage(img_real), 0)
+        self.assertEqual(os.stat(img_real).st_size, size_initial)
+
+        dev_from_real = setup_loopdev(img_real, cleanup_via=self.addCleanup)
+        dev_from_sym = setup_loopdev(img_sym, cleanup_via=self.addCleanup)
+
+        reflink._resize_file(img_real, size_resized)
+        self.assertEqual(reflink._get_file_disk_usage(img_real), 0)
+        self.assertEqual(os.stat(img_real).st_size, size_resized)
+
+        reflink_update_loopdev_sizes(os.path.join(self.test_dir, 'unrelated'))
+
+        for dev in (dev_from_real, dev_from_sym):
+            self.assertEqual(get_blockdev_size(dev), size_initial)
+
+        reflink_update_loopdev_sizes(img_sym)
+
+        for dev in (dev_from_real, dev_from_sym):
+            self.assertEqual(get_blockdev_size(dev), size_resized)
+
+class TC_00_ReflinkOnBtrfs(ReflinkMixin, qubes.tests.QubesTestCase):
+    def setUp(self):  # pylint: disable=arguments-differ
+        super().setUp('btrfs')
+        self.ficlone_supported = True
+
+class TC_01_ReflinkOnExt4(ReflinkMixin, qubes.tests.QubesTestCase):
+    def setUp(self):  # pylint: disable=arguments-differ
+        super().setUp('ext4')
+        self.ficlone_supported = False
+
+
+def setup_loopdev(img, cleanup_via=None):
+    dev = str.strip(cmd('sudo', 'losetup', '-f', '--show', img).decode())
+    if cleanup_via is not None:
+        cleanup_via(detach_loopdev, dev)
+    return dev
+
+def detach_loopdev(dev):
+    cmd('sudo', 'losetup', '-d', dev)
+
+def get_fs_type(directory):
+    # 'stat -f -c %T' would identify ext4 as 'ext2/ext3'
+    return cmd('df', '--output=fstype', directory).decode().splitlines()[1]
+
+def mkdir_fs(directory, fs_type,
+             accessible=True, max_size=100*1024**3, cleanup_via=None):
+    os.mkdir(directory)
+
+    if get_fs_type(directory) != fs_type:
+        img = os.path.join(directory, 'img')
+        with open(img, 'xb') as img_io:
+            img_io.truncate(max_size)
+        cmd('mkfs.' + fs_type, img)
+        dev = setup_loopdev(img)
+        os.remove(img)
+        cmd('sudo', 'mount', dev, directory)
+        detach_loopdev(dev)
+
+    if accessible:
+        cmd('sudo', 'chmod', '777', directory)
+    else:
+        cmd('sudo', 'chmod', '000', directory)
+        cmd('sudo', 'chattr', '+i', directory)  # cause EPERM on write as root
+
+    if cleanup_via is not None:
+        cleanup_via(rmtree_fs, directory)
+
+def rmtree_fs(directory):
+    if os.path.ismount(directory):
+        cmd('sudo', 'umount', '-l', directory)
+        # loop device and backing file are garbage collected automatically
+    cmd('sudo', 'chattr', '-i', directory)
+    cmd('sudo', 'chmod', '777', directory)
+    shutil.rmtree(directory)
+
+def get_blockdev_size(dev):
+    return int(cmd('sudo', 'blockdev', '--getsize64', dev))
+
+def reflink_update_loopdev_sizes(img):
+    env = [k + '=' + v for k, v in os.environ.items()  # 'sudo -E' alone would
+           if k.startswith('PYTHON')]                  # drop some of these
+    code = ('from qubes.storage import reflink\n'
+            'reflink._update_loopdev_sizes(%r)' % img)
+    cmd('sudo', '-E', 'env', *env, sys.executable, '-c', code)
+
+def cmd(*argv):
+    p = subprocess.run(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    if p.returncode != 0:
+        raise Exception(str(p))  # this will show stdout and stderr
+    return p.stdout

qubes/tests/vm/__init__.py

@@ -88,6 +88,7 @@ class TestApp(qubes.tests.TestEmitter):
         self.default_pool_root = 'default'
         self.default_pool_private = 'default'
         self.default_pool_kernel = 'linux-kernel'
+        self.default_qrexec_timeout = 60
         self.default_netvm = None
         self.domains = TestVMsCollection()
         #: jinja2 environment for libvirt XML templates

qubes/tests/vm/qubesvm.py

@@ -141,6 +141,50 @@ class TC_00_setters(qubes.tests.QubesTestCase):
         with self.assertRaises(ValueError):
             qubes.vm.qubesvm._setter_virt_mode(self.vm, self.prop, 'True')
 
+class TC_10_default(qubes.tests.QubesTestCase):
+    def setUp(self):
+        super().setUp()
+        self.app = TestApp()
+        self.vm = TestVM(app=self.app)
+        self.prop = TestProp()
+
+    def test_000_default_with_template_simple(self):
+        default_getter = qubes.vm.qubesvm._default_with_template('kernel',
+            'dfl-kernel')
+        self.assertEqual(default_getter(self.vm), 'dfl-kernel')
+        self.vm.template = None
+        self.assertEqual(default_getter(self.vm), 'dfl-kernel')
+        self.vm.template = unittest.mock.Mock()
+        self.vm.template.kernel = 'template-kernel'
+        self.assertEqual(default_getter(self.vm), 'template-kernel')
+
+    def test_001_default_with_template_callable(self):
+        default_getter = qubes.vm.qubesvm._default_with_template('kernel',
+            lambda x: x.app.default_kernel)
+        self.app.default_kernel = 'global-dfl-kernel'
+        self.assertEqual(default_getter(self.vm), 'global-dfl-kernel')
+        self.vm.template = None
+        self.assertEqual(default_getter(self.vm), 'global-dfl-kernel')
+        self.vm.template = unittest.mock.Mock()
+        self.vm.template.kernel = 'template-kernel'
+        self.assertEqual(default_getter(self.vm), 'template-kernel')
+
+    def test_010_default_virt_mode(self):
+        default_getter = qubes.vm.qubesvm._default_with_template('kernel',
+            lambda x: x.app.default_kernel)
+        self.assertEqual(qubes.vm.qubesvm._default_virt_mode(self.vm),
+            'pvh')
+        self.vm.template = unittest.mock.Mock()
+        self.vm.template.virt_mode = 'hvm'
+        self.assertEqual(qubes.vm.qubesvm._default_virt_mode(self.vm),
+            'hvm')
+        self.vm.template = None
+        self.assertEqual(qubes.vm.qubesvm._default_virt_mode(self.vm),
+            'pvh')
+        self.vm.devices['pci'].persistent().append('some-dev')
+        self.assertEqual(qubes.vm.qubesvm._default_virt_mode(self.vm),
+            'hvm')
+
 
 class QubesVMTestsMixin(object):
     property_no_default = object()
@@ -415,6 +459,15 @@ class TC_90_QubesVM(QubesVMTestsMixin, qubes.tests.QubesTestCase):
         self.assertPropertyInvalidValue(vm, 'qrexec_timeout', '-2')
         self.assertPropertyInvalidValue(vm, 'qrexec_timeout', '')
 
+    def test_272_qrexec_timeout_global_changed(self):
+        self.app.default_qrexec_timeout = 123
+        vm = self.get_vm()
+        self.assertPropertyDefaultValue(vm, 'qrexec_timeout', 123)
+        self.assertPropertyValue(vm, 'qrexec_timeout', 3, 3, '3')
+        del vm.qrexec_timeout
+        self.assertPropertyDefaultValue(vm, 'qrexec_timeout', 123)
+        self.assertPropertyValue(vm, 'qrexec_timeout', '3', 3, '3')
+
     def test_280_autostart(self):
         vm = self.get_vm()
         # FIXME any better idea to not involve systemctl call at this stage?

qubes/tools/qubes_create.py

@@ -18,7 +18,7 @@
 # License along with this library; if not, see <https://www.gnu.org/licenses/>.
 #
 
-'''qvm-create - Create new Qubes OS store'''
+'''qubes-create - Create new Qubes OS store'''
 
 import sys
 import qubes
@@ -38,7 +38,7 @@ def main(args=None):
 
     args = parser.parse_args(args)
     qubes.Qubes.create_empty_store(args.app,
-        offline_mode=args.offline_mode)
+        offline_mode=args.offline_mode).setup_pools()
     return 0
 
 

qubes/utils.py

@@ -41,7 +41,7 @@ def get_timezone():
     if os.path.islink('/etc/localtime'):
         return '/'.join(os.readlink('/etc/localtime').split('/')[-2:])
     # <=fc17
-    elif os.path.exists('/etc/sysconfig/clock'):
+    if os.path.exists('/etc/sysconfig/clock'):
         clock_config = open('/etc/sysconfig/clock', "r")
         clock_config_lines = clock_config.readlines()
         clock_config.close()
@@ -50,18 +50,17 @@ def get_timezone():
             line_match = zone_re.match(line)
             if line_match:
                 return line_match.group(1)
-    else:
-        # last resort way, some applications makes /etc/localtime
-        # hardlink instead of symlink...
-        tz_info = os.stat('/etc/localtime')
-        if not tz_info:
-            return None
-        if tz_info.st_nlink > 1:
-            p = subprocess.Popen(['find', '/usr/share/zoneinfo',
-                '-inum', str(tz_info.st_ino), '-print', '-quit'],
-                stdout=subprocess.PIPE)
-            tz_path = p.communicate()[0].strip()
-            return tz_path.replace('/usr/share/zoneinfo/', '')
+    # last resort way, some applications makes /etc/localtime
+    # hardlink instead of symlink...
+    tz_info = os.stat('/etc/localtime')
+    if not tz_info:
+        return None
+    if tz_info.st_nlink > 1:
+        p = subprocess.Popen(['find', '/usr/share/zoneinfo',
+            '-inum', str(tz_info.st_ino), '-print', '-quit'],
+            stdout=subprocess.PIPE)
+        tz_path = p.communicate()[0].strip()
+        return tz_path.replace(b'/usr/share/zoneinfo/', b'')
     return None
 
 
@@ -132,9 +131,9 @@ def size_to_human(size):
     """Humane readable size, with 1/10 precision"""
     if size < 1024:
         return str(size)
-    elif size < 1024 * 1024:
+    if size < 1024 * 1024:
         return str(round(size / 1024.0, 1)) + ' KiB'
-    elif size < 1024 * 1024 * 1024:
+    if size < 1024 * 1024 * 1024:
         return str(round(size / (1024.0 * 1024), 1)) + ' MiB'
 
     return str(round(size / (1024.0 * 1024 * 1024), 1)) + ' GiB'
@@ -181,6 +180,6 @@ def match_vm_name_with_special(vm, name):
     or @type:...'''
     if name.startswith('@tag:'):
         return name[len('@tag:'):] in vm.tags
-    elif name.startswith('@type:'):
+    if name.startswith('@type:'):
         return name[len('@type:'):] == vm.__class__.__name__
     return name == vm.name

qubes/vm/__init__.py

@@ -569,10 +569,9 @@ class VMProperty(qubes.property):
             if self.allow_none:
                 super(VMProperty, self).__set__(instance, value)
                 return
-            else:
-                raise ValueError(
-                    'Property {!r} does not allow setting to {!r}'.format(
-                        self.__name__, value))
+            raise ValueError(
+                'Property {!r} does not allow setting to {!r}'.format(
+                    self.__name__, value))
 
         app = instance if isinstance(instance, qubes.Qubes) else instance.app
 

qubes/vm/adminvm.py

@@ -147,12 +147,11 @@ class AdminVM(qubes.vm.BaseVM):
         if self.app.vmm.offline_mode:
             # default value passed on xen cmdline
             return 4096
-        else:
-            try:
-                return self.app.vmm.libvirt_conn.getInfo()[1]
-            except libvirt.libvirtError as e:
-                self.log.warning('Failed to get memory limit for dom0: %s', e)
-                return 4096
+        try:
+            return self.app.vmm.libvirt_conn.getInfo()[1]
+        except libvirt.libvirtError as e:
+            self.log.warning('Failed to get memory limit for dom0: %s', e)
+            return 4096
 
     def verify_files(self):
         '''Always :py:obj:`True`
@@ -181,7 +180,7 @@ class AdminVM(qubes.vm.BaseVM):
 
     @property
     def icon_path(self):
-        return None
+        pass
 
     @property
     def untrusted_qdb(self):

qubes/vm/dispvm.py

@@ -142,8 +142,8 @@ class DispVM(qubes.vm.qubesvm.QubesVM):
     def _auto_cleanup(self):
         '''Do auto cleanup if enabled'''
         if self.auto_cleanup and self in self.app.domains:
-            yield from self.remove_from_disk()
             del self.app.domains[self]
+            yield from self.remove_from_disk()
             self.app.save()
 
     @classmethod
@@ -193,8 +193,8 @@ class DispVM(qubes.vm.qubesvm.QubesVM):
             pass
         # if auto_cleanup is set, this will be done automatically
         if not self.auto_cleanup:
-            yield from self.remove_from_disk()
             del self.app.domains[self]
+            yield from self.remove_from_disk()
             self.app.save()
 
     @asyncio.coroutine

qubes/vm/qubesvm.py

@@ -24,7 +24,6 @@ from __future__ import absolute_import
 
 import asyncio
 import base64
-import errno
 import grp
 import os
 import os.path
@@ -102,7 +101,25 @@ def _setter_virt_mode(self, prop, value):
 def _default_virt_mode(self):
     if self.devices['pci'].persistent():
         return 'hvm'
-    return 'pvh'
+    try:
+        return self.template.virt_mode
+    except AttributeError:
+        return 'pvh'
+
+def _default_with_template(prop, default):
+    '''Return a callable for 'default' argument of a property. Use a value
+    from a template (if any), otherwise *default*
+    '''
+
+    def _func(self):
+        try:
+            return getattr(self.template, prop)
+        except AttributeError:
+            if callable(default):
+                return default(self)
+            return default
+
+    return _func
 
 
 class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
@@ -158,6 +175,32 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
 
             *other arguments are as in :py:meth:`start`*
 
+        .. event:: domain-start-failed (subject, event, reason)
+
+            Fired when :py:meth:`start` method fails.
+            *reason* argument is a textual error message.
+
+            Handler for this event can be asynchronous (a coroutine).
+
+            :param subject: Event emitter (the qube object)
+            :param event: Event name (``'domain-start'``)
+
+            *other arguments are as in :py:meth:`start`*
+
+        .. event:: domain-paused (subject, event)
+
+            Fired when the domain has been paused.
+
+            :param subject: Event emitter (the qube object)
+            :param event: Event name (``'domain-paused'``)
+
+        .. event:: domain-unpaused (subject, event)
+
+            Fired when the domain has been unpaused.
+
+            :param subject: Event emitter (the qube object)
+            :param event: Event name (``'domain-unpaused'``)
+
         .. event:: domain-stopped (subject, event)
 
             Fired when domain has been stopped.
@@ -387,7 +430,8 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
         type=str, setter=_setter_virt_mode,
         default=_default_virt_mode,
         doc='''Virtualisation mode: full virtualisation ("HVM"),
-            or paravirtualisation ("PV"), or hybrid ("PVH")''')
+            or paravirtualisation ("PV"), or hybrid ("PVH"). TemplateBasedVMs use its '
+            'template\'s value by default.''')
 
     installed_by_rpm = qubes.property('installed_by_rpm',
         type=bool, setter=qubes.property.bool,
@@ -397,17 +441,19 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
 
     memory = qubes.property('memory', type=int,
         setter=_setter_positive_int,
-        default=(lambda self:
+        default=_default_with_template('memory', lambda self:
             qubes.config.defaults[
                 'hvm_memory' if self.virt_mode == 'hvm' else 'memory']),
-        doc='Memory currently available for this VM.')
+        doc='Memory currently available for this VM. TemplateBasedVMs use its '
+            'template\'s value by default.')
 
     maxmem = qubes.property('maxmem', type=int,
         setter=_setter_positive_int,
-        default=(lambda self:
-            int(min(self.app.host.memory_total / 1024 / 2, 4000))),
+        default=_default_with_template('maxmem', (lambda self:
+            int(min(self.app.host.memory_total / 1024 / 2, 4000)))),
         doc='''Maximum amount of memory available for this VM (for the purpose
-            of the memory balancer).''')
+            of the memory balancer). TemplateBasedVMs use its '
+            'template\'s value by default.''')
 
     stubdom_mem = qubes.property('stubdom_mem', type=int,
         setter=_setter_positive_int,
@@ -417,14 +463,17 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
     vcpus = qubes.property('vcpus',
         type=int,
         setter=_setter_positive_int,
-        default=2,
-        doc='Number of virtual CPUs for a qube')
+        default=_default_with_template('vcpus', 2),
+        doc='Number of virtual CPUs for a qube. TemplateBasedVMs use its '
+            'template\'s value by default.')
 
     # CORE2: swallowed uses_default_kernel
     kernel = qubes.property('kernel', type=str,
         setter=_setter_kernel,
-        default=(lambda self: self.app.default_kernel),
-        doc='Kernel used by this domain.')
+        default=_default_with_template('kernel',
+            lambda self: self.app.default_kernel),
+        doc='Kernel used by this domain. TemplateBasedVMs use its '
+            'template\'s value by default.')
 
     # CORE2: swallowed uses_default_kernelopts
     # pylint: disable=no-member
@@ -434,7 +483,8 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
             if list(self.devices['pci'].persistent())
             else self.template.kernelopts if hasattr(self, 'template')
             else qubes.config.defaults['kernelopts']),
-        doc='Kernel command line passed to domain.')
+        doc='Kernel command line passed to domain. TemplateBasedVMs use its '
+            'template\'s value by default.')
 
     debug = qubes.property('debug', type=bool, default=False,
         setter=qubes.property.bool,
@@ -445,10 +495,10 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
     #     only plain property?
     default_user = qubes.property('default_user', type=str,
         # pylint: disable=no-member
-        default=(lambda self: self.template.default_user
-            if hasattr(self, 'template') else 'user'),
+        default=_default_with_template('default_user', 'user'),
         setter=_setter_default_user,
-        doc='FIXME')
+        doc='Default user to start applications as. TemplateBasedVMs use its '
+            'template\'s value by default.')
 
     # pylint: enable=no-member
 
@@ -459,12 +509,22 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
 #       else:
 #           return self._default_user
 
-    qrexec_timeout = qubes.property('qrexec_timeout', type=int, default=60,
+    qrexec_timeout = qubes.property('qrexec_timeout', type=int,
+        default=_default_with_template('qrexec_timeout',
+            lambda self: self.app.default_qrexec_timeout),
         setter=_setter_positive_int,
         doc='''Time in seconds after which qrexec connection attempt is deemed
             failed. Operating system inside VM should be able to boot in this
             time.''')
 
+    shutdown_timeout = qubes.property('shutdown_timeout', type=int,
+        default=_default_with_template('shutdown_timeout',
+            lambda self: self.app.default_shutdown_timeout),
+        setter=_setter_positive_int,
+        doc='''Time in seconds for shutdown of the VM, after which VM may be
+            forcefully powered off. Operating system inside VM should be
+            able to fully shutdown in this time.''')
+
     autostart = qubes.property('autostart', default=False,
         type=bool, setter=qubes.property.bool,
         doc='''Setting this to `True` means that VM should be autostarted on
@@ -518,10 +578,9 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
         except libvirt.libvirtError as e:
             if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN:
                 return -1
-            else:
-                self.log.exception('libvirt error code: {!r}'.format(
-                    e.get_error_code()))
-                raise
+            self.log.exception('libvirt error code: {!r}'.format(
+                e.get_error_code()))
+            raise
 
     @qubes.stateless_property
     def stubdom_xid(self):
@@ -815,10 +874,56 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
                 raise qubes.exc.QubesException(
                     'Failed to reset autostart for VM in systemd')
 
+    @qubes.events.handler('domain-remove-from-disk')
+    def on_remove_from_disk(self, event, **kwargs):
+        # pylint: disable=unused-argument
+        if self.autostart:
+            subprocess.call(
+                ['sudo', 'systemctl', 'disable',
+                    'qubes-vm@{}.service'.format(self.name)])
+
+    @qubes.events.handler('domain-create-on-disk')
+    def on_create_on_disk(self, event, **kwargs):
+        # pylint: disable=unused-argument
+        if self.autostart:
+            subprocess.call(
+                ['sudo', 'systemctl', 'enable',
+                    'qubes-vm@{}.service'.format(self.name)])
+
     #
     # methods for changing domain state
     #
 
+    @asyncio.coroutine
+    def _ensure_shutdown_handled(self):
+        '''Make sure previous shutdown is fully handled.
+        MUST NOT be called when domain is running.
+        '''
+        with (yield from self._domain_stopped_lock):
+            # Don't accept any new stopped event's till a new VM has been
+            # created. If we didn't received any stopped event or it wasn't
+            # handled yet we will handle this in the next lines.
+            self._domain_stopped_event_received = True
+
+            if self._domain_stopped_future is not None:
+                # Libvirt stopped event was already received, so cancel the
+                # future. If it didn't generate the Qubes events yet we
+                # will do it below.
+                self._domain_stopped_future.cancel()
+                self._domain_stopped_future = None
+
+            if not self._domain_stopped_event_handled:
+                # No Qubes domain-stopped events have been generated yet.
+                # So do this now.
+
+                # Set this immediately such that we don't generate events
+                # twice if an exception gets thrown.
+                self._domain_stopped_event_handled = True
+
+                yield from self.fire_event_async('domain-stopped')
+                yield from self.fire_event_async('domain-shutdown')
+
+
     @asyncio.coroutine
     def start(self, start_guid=True, notify_function=None,
             mem_required=None):
@@ -835,39 +940,28 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
             if self.get_power_state() != 'Halted':
                 return self
 
-            with (yield from self._domain_stopped_lock):
-                # Don't accept any new stopped event's till a new VM has been
-                # created. If we didn't received any stopped event or it wasn't
-                # handled yet we will handle this in the next lines.
-                self._domain_stopped_event_received = True
-
-                if self._domain_stopped_future is not None:
-                    # Libvirt stopped event was already received, so cancel the
-                    # future. If it didn't generate the Qubes events yet we
-                    # will do it below.
-                    self._domain_stopped_future.cancel()
-                    self._domain_stopped_future = None
-
-                if not self._domain_stopped_event_handled:
-                    # No Qubes domain-stopped events have been generated yet.
-                    # So do this now.
-
-                    # Set this immediately such that we don't generate events
-                    # twice if an exception gets thrown.
-                    self._domain_stopped_event_handled = True
-
-                    yield from self.fire_event_async('domain-stopped')
-                    yield from self.fire_event_async('domain-shutdown')
+            yield from self._ensure_shutdown_handled()
 
             self.log.info('Starting {}'.format(self.name))
 
-            yield from self.fire_event_async('domain-pre-start',
-                pre_event=True,
-                start_guid=start_guid, mem_required=mem_required)
+            try:
+                yield from self.fire_event_async('domain-pre-start',
+                    pre_event=True,
+                    start_guid=start_guid, mem_required=mem_required)
+            except Exception as exc:
+                yield from self.fire_event_async('domain-start-failed',
+                    reason=str(exc))
+                raise
+
+            for devclass in self.devices:
+                for dev in self.devices[devclass].persistent():
+                    if isinstance(dev, qubes.devices.UnknownDevice):
+                        raise qubes.exc.QubesException(
+                            '{} device {} not available'.format(devclass, dev))
 
             qmemman_client = None
             try:
-                if self.virt_mode == 'pvh' and self.kernel is None:
+                if self.virt_mode == 'pvh' and not self.kernel:
                     raise qubes.exc.QubesException(
                         'virt_mode PVH require kernel to be set')
                 yield from self.storage.verify()
@@ -960,8 +1054,8 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
             return
 
         if self._domain_stopped_event_received:
-            self.log.warning('Duplicated stopped event from libvirt received!')
-            # ignore this unexpected event
+            # ignore this event - already triggered by shutdown(), kill(),
+            # or subsequent start()
             return
 
         self._domain_stopped_event_received = True
@@ -993,9 +1087,13 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
                 self.name)
 
     @asyncio.coroutine
-    def shutdown(self, force=False, wait=False):
+    def shutdown(self, force=False, wait=False, timeout=None):
         '''Shutdown domain.
 
+        :param force: ignored
+        :param wait: wait for shutdown to complete
+        :param timeout: shutdown wait timeout (for *wait*=True), defaults to
+        :py:attr:`shutdown_timeout`
         :raises qubes.exc.QubesVMNotStartedError: \
             when domain is already shut down.
         '''
@@ -1008,8 +1106,18 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
 
         self.libvirt_domain.shutdown()
 
-        while wait and not self.is_halted():
-            yield from asyncio.sleep(0.25)
+        if wait:
+            if timeout is None:
+                timeout = self.shutdown_timeout
+            while timeout > 0 and not self.is_halted():
+                yield from asyncio.sleep(0.25)
+                timeout -= 0.25
+            with (yield from self.startup_lock):
+                if self.is_halted():
+                    # make sure all shutdown tasks are completed
+                    yield from self._ensure_shutdown_handled()
+                else:
+                    raise qubes.exc.QubesVMShutdownTimeoutError(self)
 
         return self
 
@@ -1024,13 +1132,17 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
         if not self.is_running() and not self.is_paused():
             raise qubes.exc.QubesVMNotStartedError(self)
 
-        try:
-            self.libvirt_domain.destroy()
-        except libvirt.libvirtError as e:
-            if e.get_error_code() == libvirt.VIR_ERR_OPERATION_INVALID:
-                raise qubes.exc.QubesVMNotStartedError(self)
-            else:
-                raise
+        with (yield from self.startup_lock):
+            try:
+                self.libvirt_domain.destroy()
+            except libvirt.libvirtError as e:
+                if e.get_error_code() == libvirt.VIR_ERR_OPERATION_INVALID:
+                    raise qubes.exc.QubesVMNotStartedError(self)
+                else:
+                    raise
+
+            # make sure all shutdown tasks are completed
+            yield from self._ensure_shutdown_handled()
 
         return self
 
@@ -1377,6 +1489,8 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
                                    'creation'.format(self.dir_path))
             raise
 
+        if os.path.exists(self.icon_path):
+            os.unlink(self.icon_path)
         self.log.info('Creating icon symlink: {} -> {}'.format(
             self.icon_path, self.label.icon_path))
         if hasattr(os, "symlink"):
@@ -1395,15 +1509,18 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
                 "Can't remove VM {!s}, beacuse it's in state {!r}.".format(
                     self, self.get_power_state()))
 
+        # make sure shutdown is handled before removing anything, but only if
+        # handling is pending; if not, we may be called from within
+        # domain-shutdown event (DispVM._auto_cleanup), which would deadlock
+        if not self._domain_stopped_event_handled:
+            yield from self._ensure_shutdown_handled()
+
         yield from self.fire_event_async('domain-remove-from-disk')
         try:
             # TODO: make it async?
             shutil.rmtree(self.dir_path)
-        except OSError as e:
-            if e.errno == errno.ENOENT:
-                pass
-            else:
-                raise
+        except FileNotFoundError:
+            pass
         yield from self.storage.remove()
 
     @asyncio.coroutine
@@ -1575,8 +1692,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
             except libvirt.libvirtError as e:
                 if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN:
                     return 'Halted'
-                else:
-                    raise
+                raise
 
         libvirt_domain = self.libvirt_domain
         if libvirt_domain is None:
@@ -1587,19 +1703,17 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
                 # pylint: disable=line-too-long
                 if libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_PAUSED:
                     return "Paused"
-                elif libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_CRASHED:
+                if libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_CRASHED:
                     return "Crashed"
-                elif libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_SHUTDOWN:
+                if libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_SHUTDOWN:
                     return "Halting"
-                elif libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_SHUTOFF:
+                if libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_SHUTOFF:
                     return "Dying"
-                elif libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_PMSUSPENDED:  # nopep8
+                if libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_PMSUSPENDED:  # nopep8
                     return "Suspended"
-                else:
-                    if not self.is_fully_usable():
-                        return "Transient"
-
-                    return "Running"
+                if not self.is_fully_usable():
+                    return "Transient"
+                return "Running"
 
             return 'Halted'
         except libvirt.libvirtError as e:
@@ -1639,8 +1753,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
             except libvirt.libvirtError as e:
                 if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN:
                     return False
-                else:
-                    raise
+                raise
 
         return bool(self.libvirt_domain.isActive())
 
@@ -1662,7 +1775,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
             :py:obj:`False` otherwise.
         :rtype: bool
         '''
-        if self.xid < 0:
+        if self.xid < 0:  # pylint: disable=comparison-with-callable
             return False
         return os.path.exists('/var/run/qubes/qrexec.%s' % self.name)
 
@@ -1706,10 +1819,9 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
                     libvirt.VIR_ERR_INTERNAL_ERROR):
                 return 0
 
-            else:
-                self.log.exception(
-                    'libvirt error code: {!r}'.format(e.get_error_code()))
-                raise
+            self.log.exception(
+                'libvirt error code: {!r}'.format(e.get_error_code()))
+            raise
 
     def get_mem_static_max(self):
         '''Get maximum memory available to VM.
@@ -1733,10 +1845,9 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
                     libvirt.VIR_ERR_INTERNAL_ERROR):
                 return 0
 
-            else:
-                self.log.exception(
-                    'libvirt error code: {!r}'.format(e.get_error_code()))
-                raise
+            self.log.exception(
+                'libvirt error code: {!r}'.format(e.get_error_code()))
+            raise
 
     def get_cputime(self):
         '''Get total CPU time burned by this domain since start.
@@ -1772,10 +1883,9 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
                     libvirt.VIR_ERR_INTERNAL_ERROR):
                 return 0
 
-            else:
-                self.log.exception(
-                    'libvirt error code: {!r}'.format(e.get_error_code()))
-                raise
+            self.log.exception(
+                'libvirt error code: {!r}'.format(e.get_error_code()))
+            raise
 
     # miscellanous
 
@@ -1936,7 +2046,9 @@ def _patch_pool_config(config, pool=None, pools=None):
         if not is_snapshot:
             config['pool'] = str(pools[name])
         else:
-            msg = "Can't clone a snapshot volume {!s} to pool {!s} " \
+            msg = "Snapshot volume {0!s} must be in the same pool as its " \
+                  "origin ({0!s} volume of template)," \
+                  "cannot move to pool {1!s} " \
                 .format(name, pools[name])
             raise qubes.exc.QubesException(msg)
     return config

qubespolicy/__init__.py

@@ -18,6 +18,8 @@
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, see <https://www.gnu.org/licenses/>.
 
+# pylint: disable=no-else-return,useless-object-inheritance,try-except-raise
+
 ''' Qrexec policy parser and evaluator '''
 import enum
 import itertools

qubespolicy/agent.py

@@ -33,7 +33,7 @@ import qubespolicy.rpcconfirmation
 import qubespolicy.policycreateconfirmation
 # pylint: enable=wrong-import-position
 
-class PolicyAgent(object):
+class PolicyAgent:
     # pylint: disable=too-few-public-methods
     dbus = """
     <node>

qubespolicy/cli.py

@@ -72,12 +72,12 @@ def main(args=None):
     if not log.handlers:
         handler = logging.handlers.SysLogHandler(address='/dev/log')
         log.addHandler(handler)
-    log_prefix = 'qrexec: {}: {} -> {}: '.format(
+    log_prefix = 'qrexec: {}: {} -> {}:'.format(
         args.service_name, args.domain, args.target)
     try:
         system_info = qubespolicy.get_system_info()
     except qubespolicy.QubesMgmtException as e:
-        log.error(log_prefix + 'error getting system info: ' + str(e))
+        log.error('%s error getting system info: %s', log_prefix, str(e))
         return 1
     try:
         try:
@@ -130,13 +130,13 @@ def main(args=None):
                 action.handle_user_response(True, response)
             else:
                 action.handle_user_response(False)
-        log.info(log_prefix + 'allowed to {}'.format(action.target))
+        log.info('%s allowed to %s', log_prefix, str(action.target))
         action.execute(caller_ident)
     except qubespolicy.PolicySyntaxError as e:
-        log.error(log_prefix + 'error loading policy: ' + str(e))
+        log.error('%s error loading policy: %s', log_prefix, str(e))
         return 1
     except qubespolicy.AccessDenied as e:
-        log.info(log_prefix + 'denied: ' + str(e))
+        log.info('%s denied: %s', log_prefix, str(e))
         return 1
     return 0
 

qubespolicy/policycreateconfirmation.py

@@ -28,7 +28,7 @@ gi.require_version('Gtk', '3.0')
 from gi.repository import Gtk
 # pylint: enable=import-error
 
-class PolicyCreateConfirmationWindow(object):
+class PolicyCreateConfirmationWindow:
     # pylint: disable=too-few-public-methods
     _source_file = pkg_resources.resource_filename('qubespolicy',
         os.path.join('glade', "PolicyCreateConfirmationWindow.glade"))

qvm-tools/qvm-sync-clock

@@ -32,6 +32,11 @@ def main():
     app = Qubes()
     clockvm = app.clockvm
 
+    if not clockvm.is_running():
+        sys.stderr.write('ClockVM {} is not running, aborting.\n'.format(
+            clockvm.name))
+        sys.exit(0)
+
     p = clockvm.run_service('qubes.GetDate')
     untrusted_date_out = p.stdout.read(25).decode('ascii', errors='strict')
     untrusted_date_out = untrusted_date_out.strip()
@@ -42,7 +47,7 @@ def main():
     date_out = untrusted_date_out
     subprocess.check_call(['date', '-u', '-Iseconds', '-s', date_out],
         stdout=subprocess.DEVNULL)
-    subprocess.check_call(['hwclock', '--systohc'],
+    subprocess.check_call(['/sbin/hwclock', '--systohc'],
         stdout=subprocess.DEVNULL)
 
 if __name__ == '__main__':

rpm_spec/core-dom0.spec.in

@@ -122,6 +122,7 @@ make -C doc PYTHON=%{__python3} SPHINXBUILD=sphinx-build-%{python3_version} man
 
 make install \
     DESTDIR=$RPM_BUILD_ROOT \
+    BACKEND_VMM=%{backend_vmm} \
     UNITDIR=%{_unitdir} \
     PYTHON_SITEPATH=%{python3_sitelib} \
     SYSCONFDIR=%{_sysconfdir}
@@ -284,6 +285,7 @@ fi
 %{python3_sitelib}/qubes/ext/qubesmanager.py
 %{python3_sitelib}/qubes/ext/r3compatibility.py
 %{python3_sitelib}/qubes/ext/services.py
+%{python3_sitelib}/qubes/ext/windows.py
 
 %dir %{python3_sitelib}/qubes/tests
 %dir %{python3_sitelib}/qubes/tests/__pycache__
@@ -304,6 +306,7 @@ fi
 %{python3_sitelib}/qubes/tests/init.py
 %{python3_sitelib}/qubes/tests/storage.py
 %{python3_sitelib}/qubes/tests/storage_file.py
+%{python3_sitelib}/qubes/tests/storage_reflink.py
 %{python3_sitelib}/qubes/tests/storage_kernels.py
 %{python3_sitelib}/qubes/tests/storage_lvm.py
 %{python3_sitelib}/qubes/tests/tarwriter.py
@@ -336,9 +339,11 @@ fi
 %{python3_sitelib}/qubes/tests/integ/backup.py
 %{python3_sitelib}/qubes/tests/integ/backupcompatibility.py
 %{python3_sitelib}/qubes/tests/integ/basic.py
+%{python3_sitelib}/qubes/tests/integ/devices_block.py
 %{python3_sitelib}/qubes/tests/integ/devices_pci.py
 %{python3_sitelib}/qubes/tests/integ/dispvm.py
 %{python3_sitelib}/qubes/tests/integ/dom0_update.py
+%{python3_sitelib}/qubes/tests/integ/mime.py
 %{python3_sitelib}/qubes/tests/integ/network.py
 %{python3_sitelib}/qubes/tests/integ/pvgrub.py
 %{python3_sitelib}/qubes/tests/integ/salt.py

setup.py

@@ -74,6 +74,7 @@ if __name__ == '__main__':
                 'qubes.ext.pci = qubes.ext.pci:PCIDeviceExtension',
                 'qubes.ext.block = qubes.ext.block:BlockDeviceExtension',
                 'qubes.ext.services = qubes.ext.services:ServicesExtension',
+                'qubes.ext.windows = qubes.ext.windows:WindowsFeatures',
             ],
             'qubes.devices': [
                 'pci = qubes.ext.pci:PCIDevice',

tests/Makefile

@@ -1,49 +0,0 @@
-PYTHON_TESTSPATH = $(PYTHON_SITEPATH)/qubes/tests
-
-all:
-	python -m compileall .
-	python -O -m compileall .
-
-install:
-ifndef PYTHON_SITEPATH
-	$(error PYTHON_SITEPATH not defined)
-endif
-	mkdir -p $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp __init__.py $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp __init__.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp backup.py $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp backup.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp backupcompatibility.py $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp backupcompatibility.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp basic.py $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp basic.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp block.py $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp block.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp dispvm.py $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp dispvm.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp dom0_update.py $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp dom0_update.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp extra.py $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp extra.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp hardware.py $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp hardware.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp hvm.py $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp hvm.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp mime.py $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp mime.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp network.py $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp network.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp pvgrub.py $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp pvgrub.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp regressions.py $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp regressions.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp run.py $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp run.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp storage.py $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp storage.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp storage_file.py $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp storage_file.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp storage_xen.py $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp storage_xen.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp vm_qrexec_gui.py $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp vm_qrexec_gui.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)

tests/hardware.py

@@ -1,74 +0,0 @@
-#!/usr/bin/python2
-# -*- coding: utf-8 -*-
-#
-# The Qubes OS Project, http://www.qubes-os.org
-#
-# Copyright (C) 2016  Marek Marczykowski-Górecki
-#                                        <marmarek@invisiblethingslab.com>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, see <https://www.gnu.org/licenses/>.
-#
-#
-import os
-
-import qubes.tests
-import time
-import subprocess
-from unittest import expectedFailure
-
-
-class TC_00_HVM(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
-    def setUp(self):
-        super(TC_00_HVM, self).setUp()
-        self.vm = self.qc.add_new_vm("QubesHVm",
-            name=self.make_vm_name('vm1'))
-        self.vm.create_on_disk(verbose=False)
-
-    @expectedFailure
-    def test_000_pci_passthrough_presence(self):
-        pcidev = os.environ.get('QUBES_TEST_PCIDEV', None)
-        if pcidev is None:
-            self.skipTest('Specify PCI device with QUBES_TEST_PCIDEV '
-                          'environment variable')
-        self.vm.pcidevs = [pcidev]
-        self.vm.pci_strictreset = False
-        self.qc.save()
-        self.qc.unlock_db()
-
-        init_script = (
-            "#!/bin/sh\n"
-            "set -e\n"
-            "lspci -n > /dev/xvdb\n"
-            "poweroff\n"
-        )
-
-        self.prepare_hvm_system_linux(self.vm, init_script,
-            ['/usr/sbin/lspci'])
-        self.vm.start()
-        timeout = 60
-        while timeout > 0:
-            if not self.vm.is_running():
-                break
-            time.sleep(1)
-            timeout -= 1
-        if self.vm.is_running():
-            self.fail("Timeout while waiting for VM shutdown")
-
-        with open(self.vm.storage.private_img, 'r') as f:
-            lspci_vm = f.read(512).strip('\0')
-        p = subprocess.Popen(['lspci', '-ns', pcidev], stdout=subprocess.PIPE)
-        (lspci_host, _) = p.communicate()
-        # strip BDF, as it is different in VM
-        pcidev_desc = ' '.join(lspci_host.strip().split(' ')[1:])
-        self.assertIn(pcidev_desc, lspci_vm)

tests/hvm.py

@@ -1,124 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-#
-# The Qubes OS Project, http://www.qubes-os.org
-#
-# Copyright (C) 2016 Marek Marczykowski-Górecki
-#                                        <marmarek@invisiblethingslab.com>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, see <https://www.gnu.org/licenses/>.
-#
-#
-
-import qubes.tests
-from qubes.qubes import QubesException
-
-class TC_10_HVM(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
-    # TODO: test with some OS inside
-    # TODO: windows tools tests
-
-    def test_000_create_start(self):
-        testvm1 = self.qc.add_new_vm("QubesHVm",
-                                     name=self.make_vm_name('vm1'))
-        testvm1.create_on_disk(verbose=False)
-        self.qc.save()
-        self.qc.unlock_db()
-        testvm1.start()
-        self.assertEquals(testvm1.get_power_state(), "Running")
-
-    def test_010_create_start_template(self):
-        templatevm = self.qc.add_new_vm("QubesTemplateHVm",
-                                        name=self.make_vm_name('template'))
-        templatevm.create_on_disk(verbose=False)
-        self.qc.save()
-        self.qc.unlock_db()
-
-        templatevm.start()
-        self.assertEquals(templatevm.get_power_state(), "Running")
-
-    def test_020_create_start_template_vm(self):
-        templatevm = self.qc.add_new_vm("QubesTemplateHVm",
-                                        name=self.make_vm_name('template'))
-        templatevm.create_on_disk(verbose=False)
-        testvm2 = self.qc.add_new_vm("QubesHVm",
-                                     name=self.make_vm_name('vm2'),
-                                     template=templatevm)
-        testvm2.create_on_disk(verbose=False)
-        self.qc.save()
-        self.qc.unlock_db()
-
-        testvm2.start()
-        self.assertEquals(testvm2.get_power_state(), "Running")
-
-    def test_030_prevent_simultaneus_start(self):
-        templatevm = self.qc.add_new_vm("QubesTemplateHVm",
-                                        name=self.make_vm_name('template'))
-        templatevm.create_on_disk(verbose=False)
-        testvm2 = self.qc.add_new_vm("QubesHVm",
-                                     name=self.make_vm_name('vm2'),
-                                     template=templatevm)
-        testvm2.create_on_disk(verbose=False)
-        self.qc.save()
-        self.qc.unlock_db()
-
-        templatevm.start()
-        self.assertEquals(templatevm.get_power_state(), "Running")
-        self.assertRaises(QubesException, testvm2.start)
-        templatevm.force_shutdown()
-        testvm2.start()
-        self.assertEquals(testvm2.get_power_state(), "Running")
-        self.assertRaises(QubesException, templatevm.start)
-
-    def test_100_resize_root_img(self):
-        testvm1 = self.qc.add_new_vm("QubesHVm",
-                                     name=self.make_vm_name('vm1'))
-        testvm1.create_on_disk(verbose=False)
-        self.qc.save()
-        self.qc.unlock_db()
-        testvm1.resize_root_img(30*1024**3)
-        self.assertEquals(testvm1.get_root_img_sz(), 30*1024**3)
-        testvm1.start()
-        self.assertEquals(testvm1.get_power_state(), "Running")
-        # TODO: launch some OS there and check the size
-
-    def test_200_start_invalid_drive(self):
-        """Regression test for #1619"""
-        testvm1 = self.qc.add_new_vm("QubesHVm",
-                                     name=self.make_vm_name('vm1'))
-        testvm1.create_on_disk(verbose=False)
-        testvm1.drive = 'hd:dom0:/invalid'
-        self.qc.save()
-        self.qc.unlock_db()
-        try:
-            testvm1.start()
-        except Exception as e:
-            self.assertIsInstance(e, QubesException)
-        else:
-            self.fail('No exception raised')
-
-    def test_201_start_invalid_drive_cdrom(self):
-        """Regression test for #1619"""
-        testvm1 = self.qc.add_new_vm("QubesHVm",
-                                     name=self.make_vm_name('vm1'))
-        testvm1.create_on_disk(verbose=False)
-        testvm1.drive = 'cdrom:dom0:/invalid'
-        self.qc.save()
-        self.qc.unlock_db()
-        try:
-            testvm1.start()
-        except Exception as e:
-            self.assertIsInstance(e, QubesException)
-        else:
-            self.fail('No exception raised')
-

version

@@ -1 +1 @@
-4.0.27
+4.0.33