From 94c8e25d3c3e3a9e3cb02577798542f05a1d620b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 13 Apr 2018 15:56:23 +0200 Subject: [PATCH 001/145] storage/lvm: save pool's revision_to_keep property And also report it as part of admin.pool.Info Admin API. QubesOS/qubes-issues#3256 --- qubes/storage/lvm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qubes/storage/lvm.py b/qubes/storage/lvm.py index aa7c3c12..76e5cdae 100644 --- a/qubes/storage/lvm.py +++ b/qubes/storage/lvm.py @@ -68,7 +68,8 @@ class ThinPool(qubes.storage.Pool): 'name': self.name, 'volume_group': self.volume_group, 'thin_pool': self.thin_pool, - 'driver': ThinPool.driver + 'driver': ThinPool.driver, + 'revisions_to_keep': self.revisions_to_keep, } def destroy(self): From 2aa14623bf7ddb0a057b1304de5be414b0729712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 13 Apr 2018 15:57:09 +0200 Subject: [PATCH 002/145] storage/lvm: fix reporting lvm command error Escape '%' in error message, as required by Admin API. Fixes QubesOS/qubes-issues#3809 --- qubes/storage/lvm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qubes/storage/lvm.py b/qubes/storage/lvm.py index 76e5cdae..c2460249 100644 --- a/qubes/storage/lvm.py +++ b/qubes/storage/lvm.py @@ -574,6 +574,7 @@ def qubes_lvm(cmd, log=logging.getLogger('qubes.storage.lvm')): p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True, env=environ) out, err = p.communicate() + err = err.decode() return_code = p.returncode if out: log.debug(out) @@ -581,6 +582,7 @@ def qubes_lvm(cmd, log=logging.getLogger('qubes.storage.lvm')): log.warning(err) elif return_code != 0: assert err, "Command exited unsuccessful, but printed nothing to stderr" + err = err.replace('%', '%%') raise qubes.storage.StoragePoolException(err) return True From ba82d9dc21c0c8b63d417a4c0640e2f7f2a00c70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 13 Apr 2018 16:03:42 +0200 Subject: [PATCH 003/145] vm/qubesvm: check if all required devices are available before start Fail the VM start early if some persistently-assigned device is missing. This will both save time and provide clearer error message. Fixes QubesOS/qubes-issues#3810 --- qubes/vm/qubesvm.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 4730a2aa..20d2da9c 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -865,6 +865,12 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): pre_event=True, start_guid=start_guid, mem_required=mem_required) + for devclass in self.devices: + for dev in self.devices[devclass].persistent(): + if isinstance(dev, qubes.devices.UnknownDevice): + raise qubes.exc.QubesException( + '{} device {} not available'.format(devclass, dev)) + qmemman_client = None try: if self.virt_mode == 'pvh' and self.kernel is None: From 6a191febc3611acb50c960434abcf899fd39327f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 13 Apr 2018 16:07:32 +0200 Subject: [PATCH 004/145] vm/qubesvm: fire 'domain-start-failed' event even if fail was early Fire 'domain-start-failed' even even if failure occurred during 'domain-pre-start' event. This will make sure if _anyone_ have seen 'domain-pre-start' event, will also see 'domain-start-failed'. In some cases it will look like spurious 'domain-start-failed', but it is safer option than the alternative. --- qubes/vm/qubesvm.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 20d2da9c..53d5e2ee 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -861,9 +861,14 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): self.log.info('Starting {}'.format(self.name)) - yield from self.fire_event_async('domain-pre-start', - pre_event=True, - start_guid=start_guid, mem_required=mem_required) + try: + yield from self.fire_event_async('domain-pre-start', + pre_event=True, + start_guid=start_guid, mem_required=mem_required) + except Exception as exc: + yield from self.fire_event_async('domain-start-failed', + reason=str(exc)) + raise for devclass in self.devices: for dev in self.devices[devclass].persistent(): From 69f19bb7bb6b464c4d19b226ca6b3a07a8841b6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 13 Apr 2018 21:43:15 +0200 Subject: [PATCH 005/145] tests/extra: add start_guid option to VMWrapper Pass start_guid option to vm.start(), when using core2 compatibility layer. --- qubes/tests/extra.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qubes/tests/extra.py b/qubes/tests/extra.py index 215200e7..c779794e 100644 --- a/qubes/tests/extra.py +++ b/qubes/tests/extra.py @@ -65,8 +65,9 @@ class VMWrapper(object): def __hash__(self): return hash(self._vm) - def start(self): - return self._loop.run_until_complete(self._vm.start()) + def start(self, start_guid=True): + return self._loop.run_until_complete( + self._vm.start(start_guid=start_guid)) def shutdown(self): return self._loop.run_until_complete(self._vm.shutdown()) From 4794232745b2e61888289186e96259263e2dfacd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 13 Apr 2018 21:44:44 +0200 Subject: [PATCH 006/145] tests: fix getting kernel package version inside VM Use `sort -V` instead of `sort -n`. --- qubes/tests/integ/pvgrub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes/tests/integ/pvgrub.py b/qubes/tests/integ/pvgrub.py index 92a12f29..fcc67672 100644 --- a/qubes/tests/integ/pvgrub.py +++ b/qubes/tests/integ/pvgrub.py @@ -67,7 +67,7 @@ class TC_40_PVGrub(object): def get_kernel_version(self, vm): if self.template.startswith('fedora-'): - cmd_get_kernel_version = 'rpm -q kernel-core|sort -n|tail -1|' \ + cmd_get_kernel_version = 'rpm -q kernel-core|sort -V|tail -1|' \ 'cut -d - -f 3-' elif self.template.startswith('debian-'): cmd_get_kernel_version = \ From bb40d61af993c150af86350e97da40d521a58e5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 14 Apr 2018 21:36:03 +0200 Subject: [PATCH 007/145] storage/lvm: filter out warning about intended over-provisioning Over-provisioning on LVM is intended. Since LVM do not have any option to disable it (see [1] and discussion linked from there), filter the warning in post-processing. [1] https://bugzilla.redhat.com/1347008 Fixes QubesOS/qubes-issues#3744 --- qubes/storage/lvm.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/qubes/storage/lvm.py b/qubes/storage/lvm.py index c2460249..168af3c4 100644 --- a/qubes/storage/lvm.py +++ b/qubes/storage/lvm.py @@ -575,6 +575,11 @@ def qubes_lvm(cmd, log=logging.getLogger('qubes.storage.lvm')): close_fds=True, env=environ) out, err = p.communicate() err = err.decode() + # Filter out warning about intended over-provisioning. + # Upstream discussion about missing option to silence it: + # https://bugzilla.redhat.com/1347008 + err = '\n'.join(line for line in err.splitlines() + if 'exceeds the size of thin pool' not in line) return_code = p.returncode if out: log.debug(out) From 0862ce8a1f69416210511f005f858054ca5acd63 Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Tue, 12 Jun 2018 02:28:58 +0200 Subject: [PATCH 008/145] doc: loading graph fixes QubesOS/qubes-issues#1560 --- doc/loading.svg | 3 +++ doc/qubes.rst | 14 ++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 doc/loading.svg diff --git a/doc/loading.svg b/doc/loading.svg new file mode 100644 index 00000000..b0552d4e --- /dev/null +++ b/doc/loading.svg @@ -0,0 +1,3 @@ + + +
qubes.Qubes
qubes.Qubes
load()
load()
Labels
Pools
Labels<br>Pools
VMs
VMs
.load() cont.
.load() cont.
load_properties(
load_stage=3)
load_properties(<br>load_stage=3)
.domains.add()
.domains.add()
domain-add
domain-add
qubes.vm.QubesVM
qubes.vm.QubesVM
__init__()
__init__()
load_properties(
load_stage=2)
load_properties(<br>load_stage=2)
load_properties(
load_stage=4)
load_properties(<br>load_stage=4)
.fire_event('domain-load')
.fire_event('domain-load')
domain-load
domain-load
default_dispvm
kernelopts
netvm
template
[Not supported by viewer]
anything by default,
if not in load_stage=4
anything by default,<br>if not in load_stage=4
admin.vm.Create
admin.vm.CreateInPool
admin.vm.CreateDisposable
[Not supported by viewer]
app.add_new_vm(cls)
app.add_new_vm(cls)
qubes.vm.QubesVM
qubes.vm.QubesVM
__init__()
__init__()
here the domain is actually removed from the collection
here the domain is actually removed from the collection
domain-init
domain-init
admin.vm.Remove
admin.vm.Remove
del app.domains[vm]
del app.domains[vm]
domain-pre-delete
domain-pre-delete
domain-delete
domain-delete
Current as of v4.0.27 (2b2cdf4)
Not included: storage
Current as of v4.0.27 (2b2cdf4)<br>Not included: storage
all properties are set
or left as default
all properties are set<br>or left as default
everything in app
everything in app
events
events
API
API
Key:
Key:
the domain is removed
from disk
the domain is removed<br>from disk \ No newline at end of file diff --git a/doc/qubes.rst b/doc/qubes.rst index 955870a8..5de9e78c 100644 --- a/doc/qubes.rst +++ b/doc/qubes.rst @@ -8,10 +8,20 @@ Because all objects in Qubes' world are interconnected, there is no possibility to instantiate them separately. They are all loaded together and contained in the one ``app`` object, an instance of :py:class:`qubes.Qubes` class. +Loading +^^^^^^^ + +The objects may come to existence in two ways: by explicit instantiation or by +loading from XML file. + The loading from XML is done in stages, because Qubes domains are dependent on each other in what can be even a circular dependency. Therefore some properties -(especcialy those that refer to another domains) are loaded later. Refer to -:py:class:`qubes.Qubes` class documentation to get description of every stage. +(especcialy those that refer to another domains) are loaded later. + +.. image:: loading.svg + +Refer to :py:class:`qubes.Qubes` class documentation to get description of every +stage. Properties From 11c7b4bb512023b76cc9a70c987abceea8e0e785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 14 Mar 2018 22:54:00 +0100 Subject: [PATCH 009/145] storage/lvm: improve handling interrupted commit First rename volume to backup revision, regardless of revisions_to_keep, then rename -snap to current volume. And only then remove backup revision (if exceed revisions_to_keep). This way even if commit operation is interrupted, there is still a volume with the data. This requires also adjusting few functions to actually fallback to most recent backup revision if the current volume isn't found - create _vid_current property for this purpose. Also, use -snap volume for clone operation and commit it normally later. This makes it safer to interrupt or even revert. QubesOS/qubes-issues#2256 --- qubes/storage/lvm.py | 195 +++++++++++++++++++++++++------------ qubes/tests/storage_lvm.py | 4 +- 2 files changed, 135 insertions(+), 64 deletions(-) diff --git a/qubes/storage/lvm.py b/qubes/storage/lvm.py index aa7c3c12..83d2c3bb 100644 --- a/qubes/storage/lvm.py +++ b/qubes/storage/lvm.py @@ -20,7 +20,6 @@ ''' Driver for storing vm images in a LVM thin pool ''' import logging -import operator import os import subprocess @@ -44,8 +43,37 @@ def check_lvm_version(): lvm_is_very_old = check_lvm_version() + class ThinPool(qubes.storage.Pool): ''' LVM Thin based pool implementation + + Volumes are stored as LVM thin volumes, in thin pool specified by + *volume_group*/*thin_pool* arguments. LVM volume naming scheme: + + vm-{vm_name}-{volume_name}[-suffix] + + Where suffix can be one of: + "-snap" - snapshot for currently running VM, at VM shutdown will be + either discarded (if save_on_stop=False), or committed + (if save_on_stop=True) + "-{revision_id}" - volume revision - new revision is automatically + created at each VM shutdown, *revisions_to_keep* control how many + old revisions (in addition to the current one) should be stored + "" (no suffix) - the most recent committed volume state; also volatile + volume (snap_on_start=False, save_on_stop=False) + + On VM startup, new volume is created, depending on volume type, + according to the table below: + + snap_on_start, save_on_stop + False, False, - no suffix, fresh empty volume + False, True, - "-snap", snapshot of last committed revision + True , False, - "-snap", snapshot of last committed revision + of source volume (from VM's template) + True, True, - unsupported configuration + + Volume's revision_id format is "{timestamp}-back", where timestamp is in + '%s' format (seconds since unix epoch) ''' # pylint: disable=protected-access size_cache = None @@ -137,14 +165,10 @@ class ThinPool(qubes.storage.Pool): if vid.endswith('-back'): # old revisions continue - config = { - 'pool': self, - 'vid': vid, - 'name': vid, - 'volume_group': self.volume_group, - 'rw': vol_info['attr'][1] == 'w', - } - volumes += [ThinVolume(**config)] + volume = self.get_volume(vid) + if volume in volumes: + continue + volumes.append(volume) return volumes @property @@ -200,6 +224,19 @@ def init_cache(log=logging.getLogger('qubes.storage.lvm')): size_cache = init_cache() + +def _revision_sort_key(revision): + '''Sort key for revisions. Sort them by time + + :returns timestamp + ''' + if isinstance(revision, tuple): + revision = revision[0] + if '-' in revision: + revision = revision.split('-')[0] + return int(revision) + + class ThinVolume(qubes.storage.Volume): ''' Default LVM thin volume implementation ''' # pylint: disable=too-few-public-methods @@ -217,7 +254,19 @@ class ThinVolume(qubes.storage.Volume): @property def path(self): - return '/dev/' + self.vid + return '/dev/' + self._vid_current + + @property + def _vid_current(self): + if self.vid in size_cache: + return self.vid + vol_revisions = self.revisions + if vol_revisions: + last_revision = \ + max(vol_revisions.items(), key=_revision_sort_key)[0] + return self.vid + '-' + last_revision + # detached pool? return expected path + return self.vid @property def revisions(self): @@ -229,7 +278,8 @@ class ThinVolume(qubes.storage.Volume): if not revision_vid.endswith('-back'): continue revision_vid = revision_vid[len(name_prefix):] - seconds = int(revision_vid[:-len('-back')]) + # get revision without suffix + seconds = int(revision_vid.split('-')[0]) iso_date = qubes.storage.isodate(seconds).split('.', 1)[0] revisions[revision_vid] = iso_date return revisions @@ -239,7 +289,7 @@ class ThinVolume(qubes.storage.Volume): try: if self.is_dirty(): return qubes.storage.lvm.size_cache[self._vid_snap]['size'] - return qubes.storage.lvm.size_cache[self.vid]['size'] + return qubes.storage.lvm.size_cache[self._vid_current]['size'] except KeyError: return self._size @@ -273,19 +323,31 @@ class ThinVolume(qubes.storage.Volume): ''' if revisions is None: revisions = sorted(self.revisions.items(), - key=operator.itemgetter(1)) + key=_revision_sort_key) # pylint: disable=invalid-unary-operand-type revisions = revisions[:(-self.revisions_to_keep) or None] revisions = [rev_id for rev_id, _ in revisions] for rev_id in revisions: + # safety check + assert rev_id != self._vid_current try: cmd = ['remove', self.vid + '-' + rev_id] qubes_lvm(cmd, self.log) except qubes.storage.StoragePoolException: pass - def _commit(self): + def _commit(self, vid_to_commit=None, keep=False): + ''' + Commit temporary volume into current one. By default + :py:attr:`_vid_snap` is used (which is created by :py:meth:`start()`), + but can be overriden by *vid_to_commit* argument. + + :param vid_to_commit: LVM volume ID to commit into this one + :param keep: whether to keep or not *vid_to_commit*. + IOW use 'clone' or 'rename' methods. + :return: None + ''' msg = "Trying to commit {!s}, but it has save_on_stop == False" msg = msg.format(self) assert self.save_on_stop, msg @@ -293,39 +355,40 @@ class ThinVolume(qubes.storage.Volume): msg = "Trying to commit {!s}, but it has rw == False" msg = msg.format(self) assert self.rw, msg - assert hasattr(self, '_vid_snap') - - if self.revisions_to_keep > 0: - cmd = ['clone', self.vid, - '{}-{}-back'.format(self.vid, int(time.time()))] - qubes_lvm(cmd, self.log) - reset_cache() - self._remove_revisions() + if vid_to_commit is None: + assert hasattr(self, '_vid_snap') + vid_to_commit = self._vid_snap # TODO: when converting this function to coroutine, this _must_ be # under a lock - # remove old volume only after _successful_ clone of the new one - cmd = ['rename', self.vid, self.vid + '-tmp'] - qubes_lvm(cmd, self.log) - try: - cmd = ['clone', self._vid_snap, self.vid] - qubes_lvm(cmd, self.log) - except: - # restore original volume - cmd = ['rename', self.vid + '-tmp', self.vid] - qubes_lvm(cmd, self.log) - raise - else: - cmd = ['remove', self.vid + '-tmp'] - qubes_lvm(cmd, self.log) + if not os.path.exists('/dev/' + vid_to_commit): + # nothing to commit + return + if self._vid_current == self.vid: + cmd = ['rename', self.vid, + '{}-{}-back'.format(self.vid, int(time.time()))] + qubes_lvm(cmd, self.log) + reset_cache() + + cmd = ['clone' if keep else 'rename', + vid_to_commit, + self.vid] + qubes_lvm(cmd, self.log) + reset_cache() + # make sure the one we've committed right now is properly + # detected as the current one - before removing anything + assert self._vid_current == self.vid + + # and remove old snapshots, if needed + self._remove_revisions() def create(self): assert self.vid assert self.size if self.save_on_stop: if self.source: - cmd = ['clone', str(self.source), self.vid] + cmd = ['clone', self.source.path, self.vid] else: cmd = [ 'create', @@ -349,7 +412,7 @@ class ThinVolume(qubes.storage.Volume): self._remove_revisions(self.revisions.keys()) if not os.path.exists(self.path): return - cmd = ['remove', self.vid] + cmd = ['remove', self.path] qubes_lvm(cmd, self.log) reset_cache() # pylint: disable=protected-access @@ -358,8 +421,8 @@ class ThinVolume(qubes.storage.Volume): def export(self): ''' Returns an object that can be `open()`. ''' # make sure the device node is available - qubes_lvm(['activate', self.vid], self.log) - devpath = '/dev/' + self.vid + qubes_lvm(['activate', self.path], self.log) + devpath = self.path return devpath @asyncio.coroutine @@ -367,28 +430,32 @@ class ThinVolume(qubes.storage.Volume): if not src_volume.save_on_stop: return self + if self.is_dirty(): + raise qubes.storage.StoragePoolException( + 'Cannot import to dirty volume {} -' + ' start and stop a qube to cleanup'.format(self.vid)) # HACK: neat trick to speed up testing if you have same physical thin # pool assigned to two qubes-pools i.e: qubes_dom0 and test-lvm # pylint: disable=line-too-long if isinstance(src_volume.pool, ThinPool) and \ src_volume.pool.thin_pool == self.pool.thin_pool: # NOQA - cmd = ['remove', self.vid] - qubes_lvm(cmd, self.log) - cmd = ['clone', str(src_volume), str(self)] - qubes_lvm(cmd, self.log) + self._commit(src_volume.path, keep=True) else: - if src_volume.size != self.size: - self.resize(src_volume.size) + cmd = ['create', self.pool._pool_id, self._vid_snap.split('/')[1], + str(src_volume.size)] + qubes_lvm(cmd) src_path = src_volume.export() - cmd = ['dd', 'if=' + src_path, 'of=/dev/' + self.vid, + cmd = ['dd', 'if=' + src_path, 'of=/dev/' + self._vid_snap, 'conv=sparse'] p = yield from asyncio.create_subprocess_exec(*cmd) yield from p.wait() if p.returncode != 0: + cmd = ['remove', self._vid_snap] + qubes_lvm(cmd) raise qubes.storage.StoragePoolException( 'Failed to import volume {!r}, dd exit code: {}'.format( src_volume, p.returncode)) - reset_cache() + self._commit() return self @@ -408,20 +475,24 @@ class ThinVolume(qubes.storage.Volume): if self._vid_snap not in size_cache: return False return (size_cache[self._vid_snap]['origin'] != - self.source.vid.split('/')[1]) - + self.source.path.split('/')[-1]) def revert(self, revision=None): + if self.is_dirty(): + raise qubes.storage.StoragePoolException( + 'Cannot revert dirty volume {}, stop the qube first'.format( + self.vid)) if revision is None: revision = \ - max(self.revisions.items(), key=operator.itemgetter(1))[0] - old_path = self.path + '-' + revision + max(self.revisions.items(), key=_revision_sort_key)[0] + old_path = '/dev/' + self.vid + '-' + revision if not os.path.exists(old_path): msg = "Volume {!s} has no {!s}".format(self, old_path) raise qubes.storage.StoragePoolException(msg) - cmd = ['remove', self.vid] - qubes_lvm(cmd, self.log) + if self.vid in size_cache: + cmd = ['remove', self.vid] + qubes_lvm(cmd, self.log) cmd = ['clone', self.vid + '-' + revision, self.vid] qubes_lvm(cmd, self.log) reset_cache() @@ -450,7 +521,7 @@ class ThinVolume(qubes.storage.Volume): cmd = ['extend', self._vid_snap, str(size)] qubes_lvm(cmd, self.log) elif self.save_on_stop or not self.snap_on_start: - cmd = ['extend', self.vid, str(size)] + cmd = ['extend', self._vid_current, str(size)] qubes_lvm(cmd, self.log) reset_cache() @@ -462,9 +533,9 @@ class ThinVolume(qubes.storage.Volume): pass if self.source is None: - cmd = ['clone', self.vid, self._vid_snap] + cmd = ['clone', self._vid_current, self._vid_snap] else: - cmd = ['clone', str(self.source), self._vid_snap] + cmd = ['clone', self.source.path, self._vid_snap] qubes_lvm(cmd, self.log) @@ -483,10 +554,10 @@ class ThinVolume(qubes.storage.Volume): try: if self.save_on_stop: self._commit() - if self.snap_on_start or self.save_on_stop: + if self.snap_on_start and not self.save_on_stop: cmd = ['remove', self._vid_snap] qubes_lvm(cmd, self.log) - else: + elif not self.snap_on_start and not self.save_on_stop: cmd = ['remove', self.vid] qubes_lvm(cmd, self.log) finally: @@ -499,9 +570,9 @@ class ThinVolume(qubes.storage.Volume): # volatile volumes don't need any files return True if self.source is not None: - vid = str(self.source) + vid = self.source.path[len('/dev/'):] else: - vid = self.vid + vid = self._vid_current try: vol_info = size_cache[vid] if vol_info['attr'][4] != 'a': @@ -528,7 +599,7 @@ class ThinVolume(qubes.storage.Volume): def usage(self): # lvm thin usage always returns at least the same usage as # the parent try: - return qubes.storage.lvm.size_cache[self.vid]['usage'] + return qubes.storage.lvm.size_cache[self._vid_current]['usage'] except KeyError: return 0 diff --git a/qubes/tests/storage_lvm.py b/qubes/tests/storage_lvm.py index f7082cad..462d6cee 100644 --- a/qubes/tests/storage_lvm.py +++ b/qubes/tests/storage_lvm.py @@ -138,7 +138,7 @@ class TC_00_ThinPool(ThinPoolBase): self.assertEqual(volume.size, qubes.config.defaults['root_img_size']) volume.create() path = "/dev/%s" % volume.vid - self.assertTrue(os.path.exists(path)) + self.assertTrue(os.path.exists(path), path) volume.remove() def test_003_read_write_volume(self): @@ -158,7 +158,7 @@ class TC_00_ThinPool(ThinPoolBase): self.assertEqual(volume.size, qubes.config.defaults['root_img_size']) volume.create() path = "/dev/%s" % volume.vid - self.assertTrue(os.path.exists(path)) + self.assertTrue(os.path.exists(path), path) volume.remove() def test_004_size(self): From 8cf92642836a9ef6aefba6032e2fc6d6ccd52f91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 15 Mar 2018 02:14:29 +0100 Subject: [PATCH 010/145] tests: LVM volume naming migration, and new naming in general --- qubes/tests/storage_lvm.py | 342 ++++++++++++++++++++++++++++++++++++- 1 file changed, 341 insertions(+), 1 deletion(-) diff --git a/qubes/tests/storage_lvm.py b/qubes/tests/storage_lvm.py index 462d6cee..bb61907e 100644 --- a/qubes/tests/storage_lvm.py +++ b/qubes/tests/storage_lvm.py @@ -29,10 +29,11 @@ import os import subprocess import tempfile import unittest +import unittest.mock import qubes.tests import qubes.storage -from qubes.storage.lvm import ThinPool, ThinVolume +from qubes.storage.lvm import ThinPool, ThinVolume, qubes_lvm if 'DEFAULT_LVM_POOL' in os.environ.keys(): DEFAULT_LVM_POOL = os.environ['DEFAULT_LVM_POOL'] @@ -240,6 +241,345 @@ class TC_00_ThinPool(ThinPoolBase): self.assertEqual(self._get_size(path), new_size) self.assertEqual(volume.size, new_size) + def _get_lv_uuid(self, lv): + sudo = [] if os.getuid() == 0 else ['sudo'] + lvs_output = subprocess.check_output( + sudo + ['lvs', '--noheadings', '-o', 'lv_uuid', lv]) + return lvs_output.strip() + + def _get_lv_origin_uuid(self, lv): + sudo = [] if os.getuid() == 0 else ['sudo'] + if qubes.storage.lvm.lvm_is_very_old: + # no support for origin_uuid directly + lvs_output = subprocess.check_output( + sudo + ['lvs', '--noheadings', '-o', 'origin', lv]) + lvs_output = subprocess.check_output( + sudo + ['lvs', '--noheadings', '-o', 'lv_uuid', + lv.rsplit('/', 1)[0] + '/' + lvs_output.strip().decode()]) + else: + lvs_output = subprocess.check_output( + sudo + ['lvs', '--noheadings', '-o', 'origin_uuid', lv]) + return lvs_output.strip() + + def test_008_commit(self): + ''' Test volume changes commit''' + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + volume.create() + path_snap = '/dev/' + volume._vid_snap + self.assertFalse(os.path.exists(path_snap), path_snap) + origin_uuid = self._get_lv_uuid(volume.path) + volume.start() + snap_uuid = self._get_lv_uuid(path_snap) + self.assertNotEqual(origin_uuid, snap_uuid) + path = volume.path + self.assertTrue(path.startswith('/dev/' + volume.vid), + '{} does not start with /dev/{}'.format(path, volume.vid)) + self.assertTrue(os.path.exists(path), path) + volume.remove() + + def test_009_interrupted_commit(self): + ''' Test volume changes commit''' + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 2, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + # mock logging, to not interfere with time.time() mock + volume.log = unittest.mock.Mock() + # do not call volume.create(), do it manually to simulate + # interrupted commit + revisions = ['-1521065904-back', '-1521065905-back', '-snap'] + orig_uuids = {} + for rev in revisions: + cmd = ['create', self.pool._pool_id, + volume.vid.split('/')[1] + rev, str(config['size'])] + qubes_lvm(cmd) + orig_uuids[rev] = self._get_lv_uuid(volume.vid + rev) + qubes.storage.lvm.reset_cache() + path_snap = '/dev/' + volume._vid_snap + self.assertTrue(volume.is_dirty()) + self.assertEqual(volume.path, + '/dev/' + volume.vid + revisions[1]) + expected_revisions = { + revisions[0].lstrip('-'): '2018-03-14T22:18:24', + revisions[1].lstrip('-'): '2018-03-14T22:18:25', + } + self.assertEqual(volume.revisions, expected_revisions) + volume.start() + self.assertEqual(volume.revisions, expected_revisions) + snap_uuid = self._get_lv_uuid(path_snap) + self.assertEqual(orig_uuids['-snap'], snap_uuid) + self.assertTrue(volume.is_dirty()) + self.assertEqual(volume.path, + '/dev/' + volume.vid + revisions[1]) + with unittest.mock.patch('time.time') as mock_time: + mock_time.side_effect = [521065906] + volume.stop() + expected_revisions = { + revisions[0].lstrip('-'): '2018-03-14T22:18:24', + revisions[1].lstrip('-'): '2018-03-14T22:18:25', + } + self.assertFalse(volume.is_dirty()) + self.assertEqual(volume.revisions, expected_revisions) + self.assertEqual(volume.path, '/dev/' + volume.vid) + self.assertEqual(snap_uuid, self._get_lv_uuid(volume.path)) + self.assertFalse(os.path.exists(path_snap), path_snap) + + volume.remove() + + def test_010_migration1(self): + '''Start with old revisions, then start interacting using new code''' + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 2, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + # mock logging, to not interfere with time.time() mock + volume.log = unittest.mock.Mock() + # do not call volume.create(), do it manually to have old LV naming + revisions = ['', '-1521065904-back', '-1521065905-back'] + orig_uuids = {} + for rev in revisions: + cmd = ['create', self.pool._pool_id, + volume.vid.split('/')[1] + rev, str(config['size'])] + qubes_lvm(cmd) + orig_uuids[rev] = self._get_lv_uuid(volume.vid + rev) + qubes.storage.lvm.reset_cache() + path_snap = '/dev/' + volume._vid_snap + self.assertFalse(os.path.exists(path_snap), path_snap) + expected_revisions = { + revisions[1].lstrip('-'): '2018-03-14T22:18:24', + revisions[2].lstrip('-'): '2018-03-14T22:18:25', + } + self.assertEqual(volume.revisions, expected_revisions) + self.assertEqual(volume.path, '/dev/' + volume.vid) + + volume.start() + snap_uuid = self._get_lv_uuid(path_snap) + self.assertNotEqual(orig_uuids[''], snap_uuid) + snap_origin_uuid = self._get_lv_origin_uuid(path_snap) + self.assertEqual(orig_uuids[''], snap_origin_uuid) + path = volume.path + self.assertEqual(path, '/dev/' + volume.vid) + self.assertTrue(os.path.exists(path), path) + + with unittest.mock.patch('time.time') as mock_time: + mock_time.side_effect = ('1521065906', '1521065907') + volume.stop() + revisions.extend(['-1521065906-back']) + expected_revisions = { + revisions[2].lstrip('-'): '2018-03-14T22:18:25', + revisions[3].lstrip('-'): '2018-03-14T22:18:26', + } + self.assertEqual(volume.revisions, expected_revisions) + self.assertEqual(volume.path, '/dev/' + volume.vid) + path_snap = '/dev/' + volume._vid_snap + self.assertFalse(os.path.exists(path_snap), path_snap) + self.assertTrue(os.path.exists('/dev/' + volume.vid)) + self.assertEqual(self._get_lv_uuid(volume.path), snap_uuid) + prev_path = '/dev/' + volume.vid + revisions[3] + self.assertEqual(self._get_lv_uuid(prev_path), orig_uuids['']) + + volume.remove() + for rev in revisions: + path = '/dev/' + volume.vid + rev + self.assertFalse(os.path.exists(path), path) + + def test_011_migration2(self): + '''VM started with old code, stopped with new''' + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 1, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + # mock logging, to not interfere with time.time() mock + volume.log = unittest.mock.Mock() + # do not call volume.create(), do it manually to have old LV naming + revisions = ['', '-snap'] + orig_uuids = {} + for rev in revisions: + cmd = ['create', self.pool._pool_id, + volume.vid.split('/')[1] + rev, str(config['size'])] + qubes_lvm(cmd) + orig_uuids[rev] = self._get_lv_uuid(volume.vid + rev) + qubes.storage.lvm.reset_cache() + path_snap = '/dev/' + volume._vid_snap + self.assertTrue(os.path.exists(path_snap), path_snap) + expected_revisions = {} + self.assertEqual(volume.revisions, expected_revisions) + self.assertEqual(volume.path, '/dev/' + volume.vid) + self.assertTrue(volume.is_dirty()) + + path = volume.path + self.assertEqual(path, '/dev/' + volume.vid) + self.assertTrue(os.path.exists(path), path) + + with unittest.mock.patch('time.time') as mock_time: + mock_time.side_effect = ('1521065906', '1521065907') + volume.stop() + revisions.extend(['-1521065906-back']) + expected_revisions = { + revisions[2].lstrip('-'): '2018-03-14T22:18:26', + } + self.assertEqual(volume.revisions, expected_revisions) + self.assertEqual(volume.path, '/dev/' + volume.vid) + path_snap = '/dev/' + volume._vid_snap + self.assertFalse(os.path.exists(path_snap), path_snap) + self.assertTrue(os.path.exists('/dev/' + volume.vid)) + self.assertEqual(self._get_lv_uuid(volume.path), orig_uuids['-snap']) + prev_path = '/dev/' + volume.vid + revisions[2] + self.assertEqual(self._get_lv_uuid(prev_path), orig_uuids['']) + + volume.remove() + for rev in revisions: + path = '/dev/' + volume.vid + rev + self.assertFalse(os.path.exists(path), path) + + def test_012_migration3(self): + '''VM started with old code, started again with new, stopped with new''' + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 1, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + # mock logging, to not interfere with time.time() mock + volume.log = unittest.mock.Mock() + # do not call volume.create(), do it manually to have old LV naming + revisions = ['', '-snap'] + orig_uuids = {} + for rev in revisions: + cmd = ['create', self.pool._pool_id, + volume.vid.split('/')[1] + rev, str(config['size'])] + qubes_lvm(cmd) + orig_uuids[rev] = self._get_lv_uuid(volume.vid + rev) + qubes.storage.lvm.reset_cache() + path_snap = '/dev/' + volume._vid_snap + self.assertTrue(os.path.exists(path_snap), path_snap) + expected_revisions = {} + self.assertEqual(volume.revisions, expected_revisions) + self.assertTrue(volume.path, '/dev/' + volume.vid) + self.assertTrue(volume.is_dirty()) + + volume.start() + self.assertEqual(volume.revisions, expected_revisions) + self.assertEqual(volume.path, '/dev/' + volume.vid) + # -snap LV should be unchanged + self.assertEqual(self._get_lv_uuid(volume._vid_snap), + orig_uuids['-snap']) + + volume.remove() + for rev in revisions: + path = '/dev/' + volume.vid + rev + self.assertFalse(os.path.exists(path), path) + + def test_013_migration4(self): + '''revisions_to_keep=0, VM started with old code, stopped with new''' + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 0, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + # mock logging, to not interfere with time.time() mock + volume.log = unittest.mock.Mock() + # do not call volume.create(), do it manually to have old LV naming + revisions = ['', '-snap'] + orig_uuids = {} + for rev in revisions: + cmd = ['create', self.pool._pool_id, + volume.vid.split('/')[1] + rev, str(config['size'])] + qubes_lvm(cmd) + orig_uuids[rev] = self._get_lv_uuid(volume.vid + rev) + qubes.storage.lvm.reset_cache() + path_snap = '/dev/' + volume._vid_snap + self.assertTrue(os.path.exists(path_snap), path_snap) + expected_revisions = {} + self.assertEqual(volume.revisions, expected_revisions) + self.assertEqual(volume.path, '/dev/' + volume.vid) + self.assertTrue(volume.is_dirty()) + + with unittest.mock.patch('time.time') as mock_time: + mock_time.side_effect = ('1521065906', '1521065907') + volume.stop() + expected_revisions = {} + self.assertEqual(volume.revisions, expected_revisions) + self.assertEqual(volume.path, '/dev/' + volume.vid) + + volume.remove() + for rev in revisions: + path = '/dev/' + volume.vid + rev + self.assertFalse(os.path.exists(path), path) + + def test_014_commit_keep_0(self): + ''' Test volume changes commit, with revisions_to_keep=0''' + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 0, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + # mock logging, to not interfere with time.time() mock + volume.log = unittest.mock.Mock() + volume.create() + self.assertFalse(volume.is_dirty()) + path = volume.path + expected_revisions = {} + self.assertEqual(volume.revisions, expected_revisions) + + volume.start() + self.assertEqual(volume.revisions, expected_revisions) + path_snap = '/dev/' + volume._vid_snap + snap_uuid = self._get_lv_uuid(path_snap) + self.assertTrue(volume.is_dirty()) + self.assertEqual(volume.path, path) + + with unittest.mock.patch('time.time') as mock_time: + mock_time.side_effect = [521065906] + volume.stop() + self.assertFalse(volume.is_dirty()) + self.assertEqual(volume.revisions, {}) + self.assertEqual(volume.path, '/dev/' + volume.vid) + self.assertEqual(snap_uuid, self._get_lv_uuid(volume.path)) + self.assertFalse(os.path.exists(path_snap), path_snap) + + volume.remove() + @skipUnlessLvmPoolExists class TC_01_ThinPool(ThinPoolBase, qubes.tests.SystemTestCase): From aea0de35ad0bfb574b34b5ecb06a24544eb11c4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 15 Mar 2018 02:59:35 +0100 Subject: [PATCH 011/145] tests: ThinVolume.revert() --- qubes/tests/storage_lvm.py | 71 +++++++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/qubes/tests/storage_lvm.py b/qubes/tests/storage_lvm.py index bb61907e..55612feb 100644 --- a/qubes/tests/storage_lvm.py +++ b/qubes/tests/storage_lvm.py @@ -24,7 +24,6 @@ 'volume_group/thin_pool' combination. Pool variables without a prefix represent a :py:class:`qubes.storage.lvm.ThinPool`. ''' - import os import subprocess import tempfile @@ -580,6 +579,76 @@ class TC_00_ThinPool(ThinPoolBase): volume.remove() + def test_020_revert_last(self): + ''' Test volume revert''' + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 2, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + volume.create() + volume.start() + volume.stop() + volume.start() + volume.stop() + self.assertEqual(len(volume.revisions), 2) + revisions = volume.revisions + revision_id = max(revisions.keys()) + current_path = volume.path + current_uuid = self._get_lv_uuid(volume.path) + rev_uuid = self._get_lv_uuid(volume.vid + '-' + revision_id) + self.assertFalse(volume.is_dirty()) + self.assertNotEqual(current_uuid, rev_uuid) + volume.revert() + path_snap = '/dev/' + volume._vid_snap + self.assertFalse(os.path.exists(path_snap), path_snap) + self.assertEqual(current_path, volume.path) + new_uuid = self._get_lv_origin_uuid(volume.path) + self.assertEqual(new_uuid, rev_uuid) + self.assertEqual(volume.revisions, revisions) + + volume.remove() + + def test_021_revert_earlier(self): + ''' Test volume revert''' + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 2, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + volume.create() + volume.start() + volume.stop() + volume.start() + volume.stop() + self.assertEqual(len(volume.revisions), 2) + revisions = volume.revisions + revision_id = min(revisions.keys()) + current_path = volume.path + current_uuid = self._get_lv_uuid(volume.path) + rev_uuid = self._get_lv_uuid(volume.vid + '-' + revision_id) + self.assertFalse(volume.is_dirty()) + self.assertNotEqual(current_uuid, rev_uuid) + volume.revert(revision_id) + path_snap = '/dev/' + volume._vid_snap + self.assertFalse(os.path.exists(path_snap), path_snap) + self.assertEqual(current_path, volume.path) + new_uuid = self._get_lv_origin_uuid(volume.path) + self.assertEqual(new_uuid, rev_uuid) + self.assertEqual(volume.revisions, revisions) + + volume.remove() + @skipUnlessLvmPoolExists class TC_01_ThinPool(ThinPoolBase, qubes.tests.SystemTestCase): From 2b80f0c044aae2a393feef496f77f3a61c297fd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 15 Mar 2018 22:11:05 +0100 Subject: [PATCH 012/145] storage/lvm: use temporary volume for data import Do not write directly to main volume, instead create temporary volume and only commit it to the main one when operation is finished. This solve multiple problems: - import operation can be aborted, without data loss - importing new data over existing volume will not leave traces of previous content - especially when importing smaller volume to bigger one - import operation can be reverted - it create separate revision, similar to start/stop - easier to prevent qube from starting during import operation - template still can be used when importing new version QubesOS/qubes-issues#2256 --- qubes/storage/lvm.py | 74 ++++++++++++++++++++++++++++++++------ qubes/tests/storage_lvm.py | 59 ++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 11 deletions(-) diff --git a/qubes/storage/lvm.py b/qubes/storage/lvm.py index 83d2c3bb..01953647 100644 --- a/qubes/storage/lvm.py +++ b/qubes/storage/lvm.py @@ -159,7 +159,7 @@ class ThinPool(qubes.storage.Pool): continue if vol_info['pool_lv'] != self.thin_pool: continue - if vid.endswith('-snap'): + if vid.endswith('-snap') or vid.endswith('-import'): # implementation detail volume continue if vid.endswith('-back'): @@ -249,6 +249,8 @@ class ThinVolume(qubes.storage.Volume): if self.snap_on_start or self.save_on_stop: self._vid_snap = self.vid + '-snap' + if self.save_on_stop: + self._vid_import = self.vid + '-import' self._size = size @@ -409,6 +411,13 @@ class ThinVolume(qubes.storage.Volume): except AttributeError: pass + try: + if os.path.exists('/dev/' + self._vid_import): + cmd = ['remove', self._vid_import] + qubes_lvm(cmd, self.log) + except AttributeError: + pass + self._remove_revisions(self.revisions.keys()) if not os.path.exists(self.path): return @@ -434,36 +443,74 @@ class ThinVolume(qubes.storage.Volume): raise qubes.storage.StoragePoolException( 'Cannot import to dirty volume {} -' ' start and stop a qube to cleanup'.format(self.vid)) + self.abort_if_import_in_progress() # HACK: neat trick to speed up testing if you have same physical thin # pool assigned to two qubes-pools i.e: qubes_dom0 and test-lvm # pylint: disable=line-too-long if isinstance(src_volume.pool, ThinPool) and \ src_volume.pool.thin_pool == self.pool.thin_pool: # NOQA - self._commit(src_volume.path, keep=True) + self._commit(src_volume.path[len('/dev/'):], keep=True) else: - cmd = ['create', self.pool._pool_id, self._vid_snap.split('/')[1], + cmd = ['create', + self.pool._pool_id, # pylint: disable=protected-access + self._vid_import.split('/')[1], str(src_volume.size)] - qubes_lvm(cmd) + qubes_lvm(cmd, self.log) src_path = src_volume.export() - cmd = ['dd', 'if=' + src_path, 'of=/dev/' + self._vid_snap, - 'conv=sparse'] + cmd = ['dd', 'if=' + src_path, 'of=/dev/' + self._vid_import, + 'conv=sparse', 'status=none'] + if not os.access('/dev/' + self._vid_import, os.W_OK) or \ + not os.access(src_path, os.R_OK): + cmd.insert(0, 'sudo') + p = yield from asyncio.create_subprocess_exec(*cmd) yield from p.wait() if p.returncode != 0: - cmd = ['remove', self._vid_snap] - qubes_lvm(cmd) + cmd = ['remove', self._vid_import] + qubes_lvm(cmd, self.log) raise qubes.storage.StoragePoolException( 'Failed to import volume {!r}, dd exit code: {}'.format( src_volume, p.returncode)) - self._commit() + self._commit(self._vid_import) return self def import_data(self): ''' Returns an object that can be `open()`. ''' - devpath = '/dev/' + self.vid + if self.is_dirty(): + raise qubes.storage.StoragePoolException( + 'Cannot import data to dirty volume {}, stop the qube first'. + format(self.vid)) + self.abort_if_import_in_progress() + # pylint: disable=protected-access + cmd = ['create', self.pool._pool_id, self._vid_import.split('/')[1], + str(self.size)] + qubes_lvm(cmd, self.log) + reset_cache() + devpath = '/dev/' + self._vid_import return devpath + def import_data_end(self, success): + '''Either commit imported data, or discard temporary volume''' + if not os.path.exists('/dev/' + self._vid_import): + raise qubes.storage.StoragePoolException( + 'No import operation in progress on {}'.format(self.vid)) + if success: + self._commit(self._vid_import) + else: + cmd = ['remove', self._vid_import] + qubes_lvm(cmd, self.log) + + def abort_if_import_in_progress(self): + try: + devpath = '/dev/' + self._vid_import + if os.path.exists(devpath): + raise qubes.storage.StoragePoolException( + 'Import operation in progress on {}'.format(self.vid)) + except AttributeError: # self._vid_import + # no vid_import - import definitely not in progress + pass + def is_dirty(self): if self.save_on_stop: return os.path.exists('/dev/' + self._vid_snap) @@ -482,6 +529,7 @@ class ThinVolume(qubes.storage.Volume): raise qubes.storage.StoragePoolException( 'Cannot revert dirty volume {}, stop the qube first'.format( self.vid)) + self.abort_if_import_in_progress() if revision is None: revision = \ max(self.revisions.items(), key=_revision_sort_key)[0] @@ -520,6 +568,10 @@ class ThinVolume(qubes.storage.Volume): if self.is_dirty(): cmd = ['extend', self._vid_snap, str(size)] qubes_lvm(cmd, self.log) + elif hasattr(self, '_vid_import') and \ + os.path.exists('/dev/' + self._vid_import): + cmd = ['extend', self._vid_import, str(size)] + qubes_lvm(cmd, self.log) elif self.save_on_stop or not self.snap_on_start: cmd = ['extend', self._vid_current, str(size)] qubes_lvm(cmd, self.log) @@ -538,8 +590,8 @@ class ThinVolume(qubes.storage.Volume): cmd = ['clone', self.source.path, self._vid_snap] qubes_lvm(cmd, self.log) - def start(self): + self.abort_if_import_in_progress() try: if self.snap_on_start or self.save_on_stop: if not self.save_on_stop or not self.is_dirty(): diff --git a/qubes/tests/storage_lvm.py b/qubes/tests/storage_lvm.py index 55612feb..a693670e 100644 --- a/qubes/tests/storage_lvm.py +++ b/qubes/tests/storage_lvm.py @@ -649,6 +649,65 @@ class TC_00_ThinPool(ThinPoolBase): volume.remove() + def test_030_import_data(self): + ''' Test volume import''' + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 2, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + volume.create() + current_uuid = self._get_lv_uuid(volume.path) + self.assertFalse(volume.is_dirty()) + import_path = volume.import_data() + import_uuid = self._get_lv_uuid(import_path) + self.assertNotEqual(current_uuid, import_uuid) + # success - commit data + volume.import_data_end(True) + new_current_uuid = self._get_lv_uuid(volume.path) + self.assertEqual(new_current_uuid, import_uuid) + revisions = volume.revisions + self.assertEqual(len(revisions), 1) + revision = revisions.popitem()[0] + self.assertEqual(current_uuid, + self._get_lv_uuid(volume.vid + '-' + revision)) + self.assertFalse(os.path.exists(import_path), import_path) + + volume.remove() + + def test_031_import_data_fail(self): + ''' Test volume import''' + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 2, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + volume.create() + current_uuid = self._get_lv_uuid(volume.path) + self.assertFalse(volume.is_dirty()) + import_path = volume.import_data() + import_uuid = self._get_lv_uuid(import_path) + self.assertNotEqual(current_uuid, import_uuid) + # fail - discard data + volume.import_data_end(False) + new_current_uuid = self._get_lv_uuid(volume.path) + self.assertEqual(new_current_uuid, current_uuid) + revisions = volume.revisions + self.assertEqual(len(revisions), 0) + self.assertFalse(os.path.exists(import_path), import_path) + + volume.remove() + @skipUnlessLvmPoolExists class TC_01_ThinPool(ThinPoolBase, qubes.tests.SystemTestCase): From 76c872a43aa688eed1a266e843e702325e2b6b2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 17 Mar 2018 01:10:43 +0100 Subject: [PATCH 013/145] tests: collect all SIGCHLD before cleaning event loop On python 3.6.4 apparently it requires two callbacks runs to cleanup stale SIGCHLD handlers. --- qubes/tests/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index 865534e2..7965c7fe 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -421,6 +421,13 @@ class QubesTestCase(unittest.TestCase): except asyncio.TimeoutError: raise AssertionError('libvirt event impl drain timeout') + # this is stupid, but apparently it requires two passes + # to cleanup SIGCHLD handlers + self.loop.stop() + self.loop.run_forever() + self.loop.stop() + self.loop.run_forever() + # Check there are no Tasks left. assert not self.loop._ready assert not self.loop._scheduled From 4282a41fcb8bb770ac47ea32f093350b14c63554 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 17 Mar 2018 01:12:33 +0100 Subject: [PATCH 014/145] tests: LVM: import, list_volumes, volatile volume, snapshot volume --- qubes/tests/storage_lvm.py | 223 +++++++++++++++++++++++++++++++++++++ 1 file changed, 223 insertions(+) diff --git a/qubes/tests/storage_lvm.py b/qubes/tests/storage_lvm.py index a693670e..b9f5ccba 100644 --- a/qubes/tests/storage_lvm.py +++ b/qubes/tests/storage_lvm.py @@ -708,6 +708,229 @@ class TC_00_ThinPool(ThinPoolBase): volume.remove() + def test_032_import_volume_same_pool(self): + '''Import volume from the same pool''' + # source volume + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 2, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + source_volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + source_volume.create() + + source_uuid = self._get_lv_uuid(source_volume.path) + + # destination volume + config = { + 'name': 'root2', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 2, + 'size': qubes.config.defaults['root_img_size'], + } + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + volume.log = unittest.mock.Mock() + with unittest.mock.patch('time.time') as mock_time: + mock_time.side_effect = [1521065905] + volume.create() + + self.assertEqual(volume.revisions, {}) + uuid_before = self._get_lv_uuid(volume.path) + + with unittest.mock.patch('time.time') as mock_time: + mock_time.side_effect = [1521065906] + self.loop.run_until_complete( + volume.import_volume(source_volume)) + + uuid_after = self._get_lv_uuid(volume.path) + self.assertNotEqual(uuid_after, uuid_before) + + # also should be different than source volume (clone, not the same LV) + self.assertNotEqual(uuid_after, source_uuid) + self.assertEqual(self._get_lv_origin_uuid(volume.path), source_uuid) + + expected_revisions = { + '1521065906-back': '2018-03-14T22:18:26', + } + self.assertEqual(volume.revisions, expected_revisions) + + volume.remove() + source_volume.remove() + + def test_033_import_volume_different_pool(self): + '''Import volume from a different pool''' + source_volume = unittest.mock.Mock() + # destination volume + config = { + 'name': 'root2', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 2, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + volume.log = unittest.mock.Mock() + with unittest.mock.patch('time.time') as mock_time: + mock_time.side_effect = [1521065905] + volume.create() + + self.assertEqual(volume.revisions, {}) + uuid_before = self._get_lv_uuid(volume.path) + + with tempfile.NamedTemporaryFile() as source_volume_file: + source_volume_file.write(b'test-content') + source_volume_file.flush() + source_volume.size = 16 * 1024 * 1024 # 16MiB + source_volume.export.return_value = source_volume_file.name + with unittest.mock.patch('time.time') as mock_time: + mock_time.side_effect = [1521065906] + self.loop.run_until_complete( + volume.import_volume(source_volume)) + + uuid_after = self._get_lv_uuid(volume.path) + self.assertNotEqual(uuid_after, uuid_before) + self.assertEqual(volume.size, 16 * 1024 * 1024) + + volume_content = subprocess.check_output(['sudo', 'cat', volume.path]) + self.assertEqual(volume_content.rstrip(b'\0'), b'test-content') + + expected_revisions = { + '1521065906-back': '2018-03-14T22:18:26', + } + self.assertEqual(volume.revisions, expected_revisions) + + volume.remove() + + def test_040_volatile(self): + '''Volatile volume test''' + config = { + 'name': 'volatile', + 'pool': self.pool.name, + 'rw': True, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + # volatile volume don't need any file, verify should succeed + self.assertTrue(volume.verify()) + volume.create() + self.assertTrue(volume.verify()) + self.assertFalse(volume.save_on_stop) + self.assertFalse(volume.snap_on_start) + path = volume.path + self.assertEqual(path, '/dev/' + volume.vid) + self.assertFalse(os.path.exists(path)) + volume.start() + self.assertTrue(os.path.exists(path)) + vol_uuid = self._get_lv_uuid(path) + volume.start() + self.assertTrue(os.path.exists(path)) + vol_uuid2 = self._get_lv_uuid(path) + self.assertNotEqual(vol_uuid, vol_uuid2) + volume.stop() + self.assertFalse(os.path.exists(path)) + + def test_050_snapshot_volume(self): + ''' Test snapshot volume creation ''' + config_origin = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'size': qubes.config.defaults['root_img_size'], + } + vm = qubes.tests.storage.TestVM(self) + volume_origin = self.app.get_pool(self.pool.name).init_volume( + vm, config_origin) + volume_origin.create() + config_snapshot = { + 'name': 'root2', + 'pool': self.pool.name, + 'snap_on_start': True, + 'source': volume_origin, + 'rw': True, + 'size': qubes.config.defaults['root_img_size'], + } + volume = self.app.get_pool(self.pool.name).init_volume( + vm, config_snapshot) + self.assertIsInstance(volume, ThinVolume) + self.assertEqual(volume.name, 'root2') + self.assertEqual(volume.pool, self.pool.name) + self.assertEqual(volume.size, qubes.config.defaults['root_img_size']) + # only origin volume really needs to exist, verify should succeed + # even before create + self.assertTrue(volume.verify()) + volume.create() + path = volume.path + self.assertEqual(path, '/dev/' + volume.vid) + self.assertFalse(os.path.exists(path), path) + volume.start() + # snapshot volume isn't considered dirty at any time + self.assertFalse(volume.is_dirty()) + # not outdated yet + self.assertFalse(volume.is_outdated()) + origin_uuid = self._get_lv_uuid(volume_origin.path) + snap_origin_uuid = self._get_lv_origin_uuid(volume._vid_snap) + self.assertEqual(origin_uuid, snap_origin_uuid) + + # now make it outdated + volume_origin.start() + volume_origin.stop() + self.assertTrue(volume.is_outdated()) + origin_uuid = self._get_lv_uuid(volume_origin.path) + self.assertNotEqual(origin_uuid, snap_origin_uuid) + + volume.stop() + # stopped volume is never outdated + self.assertFalse(volume.is_outdated()) + path = volume.path + self.assertFalse(os.path.exists(path), path) + path = '/dev/' + volume._vid_snap + self.assertFalse(os.path.exists(path), path) + + volume.remove() + volume_origin.remove() + + def test_100_pool_list_volumes(self): + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 2, + 'size': qubes.config.defaults['root_img_size'], + } + config2 = config.copy() + vm = qubes.tests.storage.TestVM(self) + volume1 = self.app.get_pool(self.pool.name).init_volume(vm, config) + volume1.create() + config2['name'] = 'private' + volume2 = self.app.get_pool(self.pool.name).init_volume(vm, config2) + volume2.create() + + # create some revisions + volume1.start() + volume1.stop() + + # and have one in dirty state + volume2.start() + + self.assertIn(volume1, list(self.pool.volumes)) + self.assertIn(volume2, list(self.pool.volumes)) + volume1.remove() + self.assertNotIn(volume1, list(self.pool.volumes)) + self.assertIn(volume2, list(self.pool.volumes)) + volume2.remove() + self.assertNotIn(volume1, list(self.pool.volumes)) + self.assertNotIn(volume1, list(self.pool.volumes)) @skipUnlessLvmPoolExists class TC_01_ThinPool(ThinPoolBase, qubes.tests.SystemTestCase): From d211a2771a6a50215ad0ad3185778086b8969960 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 22 Mar 2018 01:33:51 +0100 Subject: [PATCH 015/145] api/admin: expose volume path in admin.vm.volume.Info Since (for LVM at least) path is dynamic now, add information about it to volume info. This is not very useful outside of dom0, but in dom0 it can be very useful for various scripts. This will disclose current volume revision id, but it is already possible to deduce it from snapshots list. --- qubes/api/admin.py | 2 +- qubes/tests/api_admin.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qubes/api/admin.py b/qubes/api/admin.py index a4a803c5..d0b9e32d 100644 --- a/qubes/api/admin.py +++ b/qubes/api/admin.py @@ -335,7 +335,7 @@ class QubesAdminAPI(qubes.api.AbstractQubesAPI): volume = self.dest.volumes[self.arg] # properties defined in API volume_properties = [ - 'pool', 'vid', 'size', 'usage', 'rw', 'source', + 'pool', 'vid', 'size', 'usage', 'rw', 'source', 'path', 'save_on_stop', 'snap_on_start', 'revisions_to_keep'] def _serialize(value): diff --git a/qubes/tests/api_admin.py b/qubes/tests/api_admin.py index 3d785ac4..4a425d4b 100644 --- a/qubes/tests/api_admin.py +++ b/qubes/tests/api_admin.py @@ -39,7 +39,7 @@ import qubes.storage # properties defined in API volume_properties = [ - 'pool', 'vid', 'size', 'usage', 'rw', 'source', + 'pool', 'vid', 'size', 'usage', 'rw', 'source', 'path', 'save_on_stop', 'snap_on_start', 'revisions_to_keep'] From e644378f18f028a2216f2cc2d4f877ac346acc68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 14 Apr 2018 21:33:38 +0200 Subject: [PATCH 016/145] tests: adjust for variable volume path LVM volumes now have variable volume path. Compare strip path before comparing content's hash in tests. --- qubes/tests/integ/basic.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qubes/tests/integ/basic.py b/qubes/tests/integ/basic.py index 5b0eb089..b8799a0f 100644 --- a/qubes/tests/integ/basic.py +++ b/qubes/tests/integ/basic.py @@ -570,7 +570,8 @@ class TC_03_QvmRevertTemplateChanges(qubes.tests.SystemTestCase): def get_rootimg_checksum(self): return subprocess.check_output( - ['sha1sum', self.test_template.volumes['root'].path]) + ['sha1sum', self.test_template.volumes['root'].path]).\ + decode().split(' ')[0] def _do_test(self): checksum_before = self.get_rootimg_checksum() From 2af1815ab784ad30a033df97484f58f0dbd919f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 15 Jul 2018 21:30:04 +0200 Subject: [PATCH 017/145] storage/lvm: add repr(ThinPool) for more meaningful test reports --- qubes/storage/lvm.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/qubes/storage/lvm.py b/qubes/storage/lvm.py index 01953647..18756acf 100644 --- a/qubes/storage/lvm.py +++ b/qubes/storage/lvm.py @@ -90,6 +90,12 @@ class ThinPool(qubes.storage.Pool): self._volume_objects_cache = {} + def __repr__(self): + return '<{} at {:#x} name={!r} volume_group={!r} thin_pool={!r}>'.\ + format( + type(self).__name__, id(self), + self.name, self.volume_group, self.thin_pool) + @property def config(self): return { From 69e3018b94288a965c928558f149504de057775d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 15 Jul 2018 21:31:48 +0200 Subject: [PATCH 018/145] tests: fix handling app.pools iteration --- qubes/tests/storage_lvm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes/tests/storage_lvm.py b/qubes/tests/storage_lvm.py index b9f5ccba..f6b67d5b 100644 --- a/qubes/tests/storage_lvm.py +++ b/qubes/tests/storage_lvm.py @@ -84,7 +84,7 @@ class ThinPoolBase(qubes.tests.QubesTestCase): ''' Returns the pool matching the specified ``volume_group`` & ``thin_pool``, or None. ''' - pools = [p for p in self.app.pools + pools = [p for p in self.app.pools.values() if issubclass(p.__class__, ThinPool)] for pool in pools: if pool.volume_group == volume_group \ From f8d17012c3ff263fc83472dacbf8e705b7261111 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 15 Jul 2018 21:57:34 +0200 Subject: [PATCH 019/145] tests: fix loading tests for templates Fix #209 --- qubes/tests/extra.py | 4 ++-- qubes/tests/integ/backup.py | 2 +- qubes/tests/integ/basic.py | 6 ++++-- qubes/tests/integ/dispvm.py | 4 +++- qubes/tests/integ/network.py | 2 +- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/qubes/tests/extra.py b/qubes/tests/extra.py index 4ee6534e..d93b8cc2 100644 --- a/qubes/tests/extra.py +++ b/qubes/tests/extra.py @@ -207,10 +207,10 @@ def load_tests(loader, tests, pattern): 'qubes.tests.extra.for_template'): try: for test_case in entry.load()(): - test.addTests(loader.loadTestsFromNames( + tests.addTests(loader.loadTestsFromNames( qubes.tests.create_testcases_for_templates( test_case.__name__, test_case, - globals=sys.modules[test_case.__module__].__dict__))) + module=sys.modules[test_case.__module__]))) except Exception as err: # pylint: disable=broad-except def runTest(self): raise err diff --git a/qubes/tests/integ/backup.py b/qubes/tests/integ/backup.py index 2954e945..36b4b98b 100644 --- a/qubes/tests/integ/backup.py +++ b/qubes/tests/integ/backup.py @@ -654,5 +654,5 @@ def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( qubes.tests.create_testcases_for_templates('TC_10_BackupVM', TC_10_BackupVMMixin, qubes.tests.SystemTestCase, - globals=globals()))) + module=sys.modules[__name__]))) return tests diff --git a/qubes/tests/integ/basic.py b/qubes/tests/integ/basic.py index 5b0eb089..534b13aa 100644 --- a/qubes/tests/integ/basic.py +++ b/qubes/tests/integ/basic.py @@ -35,6 +35,8 @@ import collections import pkg_resources import shutil +import sys + import qubes import qubes.firewall import qubes.tests @@ -796,11 +798,11 @@ def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( qubes.tests.create_testcases_for_templates('TC_05_StandaloneVM', TC_05_StandaloneVMMixin, qubes.tests.SystemTestCase, - globals=globals()))) + module=sys.modules[__name__]))) tests.addTests(loader.loadTestsFromNames( qubes.tests.create_testcases_for_templates('TC_06_AppVM', TC_06_AppVMMixin, qubes.tests.SystemTestCase, - globals=globals()))) + module=sys.modules[__name__]))) return tests diff --git a/qubes/tests/integ/dispvm.py b/qubes/tests/integ/dispvm.py index 7b1abd71..b852d827 100644 --- a/qubes/tests/integ/dispvm.py +++ b/qubes/tests/integ/dispvm.py @@ -28,6 +28,8 @@ from distutils import spawn import asyncio +import sys + import qubes.tests class TC_04_DispVM(qubes.tests.SystemTestCase): @@ -285,5 +287,5 @@ def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( qubes.tests.create_testcases_for_templates('TC_20_DispVM', TC_20_DispVMMixin, qubes.tests.SystemTestCase, - globals=globals()))) + module=sys.modules[__name__]))) return tests diff --git a/qubes/tests/integ/network.py b/qubes/tests/integ/network.py index 68801820..8ebb07f0 100644 --- a/qubes/tests/integ/network.py +++ b/qubes/tests/integ/network.py @@ -1330,6 +1330,6 @@ def load_tests(loader, tests, pattern): module=sys.modules[__name__]))) tests.addTests(loader.loadTestsFromNames( qubes.tests.create_testcases_for_templates('VmUpdates', - VmUpdates, qubes.tests.SystemTestCase, + VmUpdatesMixin, qubes.tests.SystemTestCase, module=sys.modules[__name__]))) return tests From c688641363f431137f442e749fdd5d6aff19a313 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 15 Jul 2018 22:29:22 +0200 Subject: [PATCH 020/145] tests: fix DispVM related tests - fix regex for editor window search - 'disp*' matches 'disk_space.py' (a dom0 local widget...) - increase a timeout for automatic DispVM cleanup --- qubes/tests/integ/dispvm.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/qubes/tests/integ/dispvm.py b/qubes/tests/integ/dispvm.py index b852d827..b8a95c83 100644 --- a/qubes/tests/integ/dispvm.py +++ b/qubes/tests/integ/dispvm.py @@ -69,7 +69,7 @@ class TC_04_DispVM(qubes.tests.SystemTestCase): self.assertEqual(lines[0], "test") dispvm_name = lines[1] # wait for actual DispVM destruction - self.loop.run_until_complete(asyncio.sleep(1)) + self.loop.run_until_complete(asyncio.sleep(5)) self.assertNotIn(dispvm_name, self.app.domains) def test_003_cleanup_destroyed(self): @@ -88,7 +88,7 @@ class TC_04_DispVM(qubes.tests.SystemTestCase): p.stdin.write(b"sudo poweroff\n") # do not close p.stdin on purpose - wait to automatic disconnect when # domain is destroyed - timeout = 30 + timeout = 70 lines_task = asyncio.ensure_future(p.stdout.read()) self.loop.run_until_complete(asyncio.wait_for(p.wait(), timeout)) self.loop.run_until_complete(lines_task) @@ -171,7 +171,7 @@ class TC_20_DispVMMixin(object): del dispvm # give it a time for shutdown + cleanup - self.loop.run_until_complete(asyncio.sleep(2)) + self.loop.run_until_complete(asyncio.sleep(5)) self.assertNotIn(dispvm_name, self.app.domains, "DispVM not removed from qubes.xml") @@ -252,7 +252,8 @@ class TC_20_DispVMMixin(object): while True: search = self.loop.run_until_complete( asyncio.create_subprocess_exec( - 'xdotool', 'search', '--onlyvisible', '--class', 'disp*', + 'xdotool', 'search', '--onlyvisible', '--class', + 'disp[0-9]*', stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)) stdout, _ = self.loop.run_until_complete(search.communicate()) From be2465c1f95420bdcf89e03b9913f6572902d175 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 15 Jul 2018 23:08:23 +0200 Subject: [PATCH 021/145] Fix issues found by pylint 2.0 Resolve: - no-else-return - useless-object-inheritance - useless-return - consider-using-set-comprehension - consider-using-in - logging-not-lazy Ignore: - not-an-iterable - false possitives for asyncio coroutines Ignore all the above in qubespolicy/__init__.py, as the file will be moved to separate repository (core-qrexec) - it already has a copy there, don't desynchronize them. --- ci/pylintrc | 3 ++ qubes/__init__.py | 13 +++--- qubes/api/__init__.py | 2 +- qubes/api/admin.py | 2 +- qubes/app.py | 56 ++++++++++++------------- qubes/backup.py | 13 +++--- qubes/devices.py | 11 +++-- qubes/dochelpers.py | 4 +- qubes/events.py | 2 +- qubes/ext/__init__.py | 2 +- qubes/firewall.py | 8 ++-- qubes/rngdoc.py | 4 +- qubes/storage/__init__.py | 27 ++++++------ qubes/storage/file.py | 4 +- qubes/utils.py | 31 +++++++------- qubes/vm/__init__.py | 7 ++-- qubes/vm/adminvm.py | 13 +++--- qubes/vm/qubesvm.py | 52 ++++++++++------------- qubespolicy/__init__.py | 2 + qubespolicy/agent.py | 2 +- qubespolicy/cli.py | 10 ++--- qubespolicy/policycreateconfirmation.py | 2 +- 22 files changed, 132 insertions(+), 138 deletions(-) diff --git a/ci/pylintrc b/ci/pylintrc index be73b0cc..88465989 100644 --- a/ci/pylintrc +++ b/ci/pylintrc @@ -6,6 +6,8 @@ ignore=tests # abstract-class-little-used: see http://www.logilab.org/ticket/111138 # deprecated-method: # enable again after disabling py-3.4.3 asyncio.ensure_future compat hack +# not-an-iterable: +# a lot of false possitives for asyncio (yield from (some coroutine)) disable= abstract-class-little-used, bad-continuation, @@ -18,6 +20,7 @@ disable= locally-enabled, logging-format-interpolation, missing-docstring, + not-an-iterable, star-args, wrong-import-order diff --git a/qubes/__init__.py b/qubes/__init__.py index c6264246..2d6b08e8 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -42,7 +42,7 @@ __license__ = 'GPLv2 or later' __version__ = 'R3' -class Label(object): +class Label: '''Label definition for virtual machines Label specifies colour of the padlock displayed next to VM's name. @@ -134,7 +134,7 @@ class Label(object): self.icon_dispvm) + ".png" -class property(object): # pylint: disable=redefined-builtin,invalid-name +class property: # pylint: disable=redefined-builtin,invalid-name '''Qubes property. This class holds one property that can be saved to and loaded from @@ -350,11 +350,10 @@ class property(object): # pylint: disable=redefined-builtin,invalid-name raise qubes.exc.QubesValueError if self.type is bool: return self.bool(None, None, untrusted_newvalue) - else: - try: - return self.type(untrusted_newvalue) - except ValueError: - raise qubes.exc.QubesValueError + try: + return self.type(untrusted_newvalue) + except ValueError: + raise qubes.exc.QubesValueError else: # 'str' or not specified type try: diff --git a/qubes/api/__init__.py b/qubes/api/__init__.py index 370e05f9..fe078a24 100644 --- a/qubes/api/__init__.py +++ b/qubes/api/__init__.py @@ -97,7 +97,7 @@ def apply_filters(iterable, filters): return iterable -class AbstractQubesAPI(object): +class AbstractQubesAPI: '''Common code for Qubes Management Protocol handling Different interfaces can expose different API call sets, however they share diff --git a/qubes/api/admin.py b/qubes/api/admin.py index a4a803c5..96f74178 100644 --- a/qubes/api/admin.py +++ b/qubes/api/admin.py @@ -44,7 +44,7 @@ import qubes.vm.adminvm import qubes.vm.qubesvm -class QubesMgmtEventsDispatcher(object): +class QubesMgmtEventsDispatcher: def __init__(self, filters, send_event): self.filters = filters self.send_event = send_event diff --git a/qubes/app.py b/qubes/app.py index 0b50f6a0..a9e967e1 100644 --- a/qubes/app.py +++ b/qubes/app.py @@ -66,7 +66,7 @@ import qubes.vm.qubesvm import qubes.vm.templatevm # pylint: enable=wrong-import-position -class VirDomainWrapper(object): +class VirDomainWrapper: # pylint: disable=too-few-public-methods def __init__(self, connection, vm): @@ -97,7 +97,7 @@ class VirDomainWrapper(object): return wrapper -class VirConnectWrapper(object): +class VirConnectWrapper: # pylint: disable=too-few-public-methods def __init__(self, uri): @@ -134,7 +134,7 @@ class VirConnectWrapper(object): return wrapper -class VMMConnection(object): +class VMMConnection: '''Connection to Virtual Machine Manager (libvirt)''' def __init__(self, offline_mode=None): @@ -229,7 +229,7 @@ class VMMConnection(object): self._xc = None # and pray it will get garbage-collected -class QubesHost(object): +class QubesHost: '''Basic information about host machine :param qubes.Qubes app: Qubes application context (must have \ @@ -363,7 +363,7 @@ class QubesHost(object): return (current_time, current) -class VMCollection(object): +class VMCollection: '''A collection of Qubes VMs VMCollection supports ``in`` operator. You may test for ``qid``, ``name`` @@ -493,7 +493,7 @@ class VMCollection(object): self.app.fire_event('domain-delete', vm=vm) def __contains__(self, key): - return any((key == vm or key == vm.qid or key == vm.name) + return any((key in (vm, vm.qid, vm.name)) for vm in self) @@ -557,32 +557,32 @@ def _default_pool(app): ''' if 'default' in app.pools: return app.pools['default'] - else: - if 'DEFAULT_LVM_POOL' in os.environ: - thin_pool = os.environ['DEFAULT_LVM_POOL'] - for pool in app.pools.values(): - if pool.config.get('driver', None) != 'lvm_thin': - continue - if pool.config['thin_pool'] == thin_pool: - return pool - # no DEFAULT_LVM_POOL, or pool not defined - root_volume_group, root_thin_pool = \ - qubes.storage.DirectoryThinPool.thin_pool('/') - if root_thin_pool: - for pool in app.pools.values(): - if pool.config.get('driver', None) != 'lvm_thin': - continue - if (pool.config['volume_group'] == root_volume_group and - pool.config['thin_pool'] == root_thin_pool): - return pool - # not a thin volume? look for file pools + if 'DEFAULT_LVM_POOL' in os.environ: + thin_pool = os.environ['DEFAULT_LVM_POOL'] for pool in app.pools.values(): - if pool.config.get('driver', None) not in ('file', 'file-reflink'): + if pool.config.get('driver', None) != 'lvm_thin': continue - if pool.config['dir_path'] == qubes.config.qubes_base_dir: + if pool.config['thin_pool'] == thin_pool: return pool - raise AttributeError('Cannot determine default storage pool') + # no DEFAULT_LVM_POOL, or pool not defined + root_volume_group, root_thin_pool = \ + qubes.storage.DirectoryThinPool.thin_pool('/') + if root_thin_pool: + for pool in app.pools.values(): + if pool.config.get('driver', None) != 'lvm_thin': + continue + if (pool.config['volume_group'] == root_volume_group and + pool.config['thin_pool'] == root_thin_pool): + return pool + + # not a thin volume? look for file pools + for pool in app.pools.values(): + if pool.config.get('driver', None) not in ('file', 'file-reflink'): + continue + if pool.config['dir_path'] == qubes.config.qubes_base_dir: + return pool + raise AttributeError('Cannot determine default storage pool') def _setter_pool(app, prop, value): if isinstance(value, qubes.storage.Pool): diff --git a/qubes/backup.py b/qubes/backup.py index e3e6f1ec..cf974807 100644 --- a/qubes/backup.py +++ b/qubes/backup.py @@ -75,7 +75,7 @@ class BackupCanceledError(qubes.exc.QubesException): self.tmpdir = tmpdir -class BackupHeader(object): +class BackupHeader: '''Structure describing backup-header file included as the first file in backup archive ''' @@ -123,7 +123,7 @@ class BackupHeader(object): f_header.write("{!s}={!s}\n".format(key, getattr(self, attr))) -class SendWorker(object): +class SendWorker: # pylint: disable=too-few-public-methods def __init__(self, queue, base_dir, backup_stdout): super(SendWorker, self).__init__() @@ -147,6 +147,7 @@ class SendWorker(object): # verified before untaring. tar_final_cmd = ["tar", "-cO", "--posix", "-C", self.base_dir, filename] + # pylint: disable=not-an-iterable final_proc = yield from asyncio.create_subprocess_exec( *tar_final_cmd, stdout=self.backup_stdout) @@ -182,6 +183,7 @@ def launch_proc_with_pty(args, stdin=None, stdout=None, stderr=None, echo=True): termios_p[3] &= ~termios.ECHO termios.tcsetattr(ctty_fd, termios.TCSANOW, termios_p) (pty_master, pty_slave) = os.openpty() + # pylint: disable=not-an-iterable p = yield from asyncio.create_subprocess_exec(*args, stdin=stdin, stdout=stdout, @@ -227,7 +229,7 @@ def launch_scrypt(action, input_name, output_name, passphrase): return p -class Backup(object): +class Backup: '''Backup operation manager. Usage: >>> app = qubes.Qubes() @@ -250,7 +252,7 @@ class Backup(object): ''' # pylint: disable=too-many-instance-attributes - class FileToBackup(object): + class FileToBackup: # pylint: disable=too-few-public-methods def __init__(self, file_path, subdir=None, name=None, size=None): if size is None: @@ -280,7 +282,7 @@ class Backup(object): if name is not None: self.name = name - class VMToBackup(object): + class VMToBackup: # pylint: disable=too-few-public-methods def __init__(self, vm, files, subdir): self.vm = vm @@ -637,6 +639,7 @@ class Backup(object): # Pipe: tar-sparse | scrypt | tar | backup_target # TODO: log handle stderr + # pylint: disable=not-an-iterable tar_sparse = yield from asyncio.create_subprocess_exec( *tar_cmdline, stdout=subprocess.PIPE) diff --git a/qubes/devices.py b/qubes/devices.py index 565f325c..bccd9e39 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -67,7 +67,7 @@ class DeviceAlreadyAttached(qubes.exc.QubesException, KeyError): '''Trying to attach already attached device''' pass -class DeviceInfo(object): +class DeviceInfo: ''' Holds all information about a device ''' # pylint: disable=too-few-public-methods def __init__(self, backend_domain, ident, description=None, @@ -117,7 +117,7 @@ class DeviceInfo(object): return '{!s}:{!s}'.format(self.backend_domain, self.ident) -class DeviceAssignment(object): # pylint: disable=too-few-public-methods +class DeviceAssignment: # pylint: disable=too-few-public-methods ''' Maps a device to a frontend_domain. ''' def __init__(self, backend_domain, ident, options=None, persistent=False, @@ -158,7 +158,7 @@ class DeviceAssignment(object): # pylint: disable=too-few-public-methods return self.backend_domain.devices[self.bus][self.ident] -class DeviceCollection(object): +class DeviceCollection: '''Bag for devices. Used as default value for :py:meth:`DeviceManager.__missing__` factory. @@ -357,8 +357,7 @@ class DeviceCollection(object): if persistent is True: # don't break app.save() return self._set - else: - raise + raise result = set() for dev, options in devices: if dev in self._set and not persistent: @@ -433,7 +432,7 @@ class UnknownDevice(DeviceInfo): frontend_domain) -class PersistentCollection(object): +class PersistentCollection: ''' Helper object managing persistent `DeviceAssignment`s. ''' diff --git a/qubes/dochelpers.py b/qubes/dochelpers.py index 1fe20bd4..e55f9317 100644 --- a/qubes/dochelpers.py +++ b/qubes/dochelpers.py @@ -48,7 +48,7 @@ SUBCOMMANDS_TITLE = 'COMMANDS' OPTIONS_TITLE = 'OPTIONS' -class GithubTicket(object): +class GithubTicket: # pylint: disable=too-few-public-methods def __init__(self, data): self.number = data['number'] @@ -418,7 +418,7 @@ def parse_event(env, sig, signode): # -def break_to_pdb(app, *dummy): +def break_to_pdb(app, *_dummy): if not app.config.break_to_pdb: return import pdb diff --git a/qubes/events.py b/qubes/events.py index 34bcd9fd..2a0af524 100644 --- a/qubes/events.py +++ b/qubes/events.py @@ -94,7 +94,7 @@ class EmitterMeta(type): cls.__handlers__[event].add(attr) -class Emitter(object, metaclass=EmitterMeta): +class Emitter(metaclass=EmitterMeta): '''Subject that can emit events. By default all events are disabled not to interfere with loading from XML. diff --git a/qubes/ext/__init__.py b/qubes/ext/__init__.py index 2b4b668d..c10e29d5 100644 --- a/qubes/ext/__init__.py +++ b/qubes/ext/__init__.py @@ -29,7 +29,7 @@ import pkg_resources import qubes.events -class Extension(object): +class Extension: '''Base class for all extensions ''' # pylint: disable=too-few-public-methods diff --git a/qubes/firewall.py b/qubes/firewall.py index dbf2a9e3..5420654c 100644 --- a/qubes/firewall.py +++ b/qubes/firewall.py @@ -34,7 +34,7 @@ import qubes import qubes.vm.qubesvm -class RuleOption(object): +class RuleOption: def __init__(self, untrusted_value): # subset of string.punctuation safe_set = string.ascii_letters + string.digits + \ @@ -208,7 +208,7 @@ class Expire(RuleOption): @property def rule(self): - return None + pass @property def api_rule(self): @@ -232,7 +232,7 @@ class Comment(RuleOption): @property def rule(self): - return None + pass @property def api_rule(self): @@ -449,7 +449,7 @@ class Rule(qubes.PropertyHolder): return hash(self.api_rule) -class Firewall(object): +class Firewall: def __init__(self, vm, load=True): assert hasattr(vm, 'firewall_conf') self.vm = vm diff --git a/qubes/rngdoc.py b/qubes/rngdoc.py index 82f41897..eeb11864 100755 --- a/qubes/rngdoc.py +++ b/qubes/rngdoc.py @@ -26,7 +26,7 @@ import textwrap import lxml.etree -class Element(object): +class Element: def __init__(self, schema, xml): self.schema = schema self.xml = xml @@ -157,7 +157,7 @@ class Element(object): write_rst_table(stream, childtable, ('element', 'number')) -class Schema(object): +class Schema: # pylint: disable=too-few-public-methods nsmap = { 'rng': 'http://relaxng.org/ns/structure/1.0', diff --git a/qubes/storage/__init__.py b/qubes/storage/__init__.py index fdcdceb5..f506c12b 100644 --- a/qubes/storage/__init__.py +++ b/qubes/storage/__init__.py @@ -47,7 +47,7 @@ class StoragePoolException(qubes.exc.QubesException): pass -class BlockDevice(object): +class BlockDevice: ''' Represents a storage block device. ''' # pylint: disable=too-few-public-methods def __init__(self, path, name, script=None, rw=True, domain=None, @@ -62,7 +62,7 @@ class BlockDevice(object): self.devtype = devtype -class Volume(object): +class Volume: ''' Encapsulates all data about a volume for serialization to qubes.xml and libvirt config. @@ -334,14 +334,14 @@ class Volume(object): msg = msg.format(str(self.__class__.__name__), method_name) return NotImplementedError(msg) -class Storage(object): +class Storage: ''' Class for handling VM virtual disks. This is base class for all other implementations, mostly with Xen on Linux in mind. ''' - AVAILABLE_FRONTENDS = set(['xvd' + c for c in string.ascii_lowercase]) + AVAILABLE_FRONTENDS = {'xvd' + c for c in string.ascii_lowercase} def __init__(self, vm): #: Domain for which we manage storage @@ -655,9 +655,9 @@ class Storage(object): ''' Used device names ''' xml = self.vm.libvirt_domain.XMLDesc() parsed_xml = lxml.etree.fromstring(xml) - return set([target.get('dev', None) + return {target.get('dev', None) for target in parsed_xml.xpath( - "//domain/devices/disk/target")]) + "//domain/devices/disk/target")} def export(self, volume): ''' Helper function to export volume (pool.export(volume))''' @@ -688,7 +688,7 @@ class Storage(object): return self.vm.volumes[volume].import_data_end(success=success) -class VolumesCollection(object): +class VolumesCollection: '''Convenient collection wrapper for pool.get_volume and pool.list_volumes ''' @@ -706,8 +706,7 @@ class VolumesCollection(object): if isinstance(item, Volume): if item.pool == self._pool: return self[item.vid] - else: - raise KeyError(item) + raise KeyError(item) try: return self._pool.get_volume(item) except NotImplementedError: @@ -740,7 +739,7 @@ class VolumesCollection(object): return [vol for vol in self] -class Pool(object): +class Pool: ''' A Pool is used to manage different kind of volumes (File based/LVM/Btrfs/...). @@ -760,7 +759,7 @@ class Pool(object): def __eq__(self, other): if isinstance(other, Pool): return self.name == other.name - elif isinstance(other, str): + if isinstance(other, str): return self.name == other return NotImplemented @@ -829,12 +828,12 @@ class Pool(object): @property def size(self): ''' Storage pool size in bytes, or None if unknown ''' - return None + pass @property def usage(self): ''' Space used in the pool in bytes, or None if unknown ''' - return None + pass def _not_implemented(self, method_name): ''' Helper for emitting helpful `NotImplementedError` exceptions ''' @@ -898,7 +897,7 @@ def search_pool_containing_dir(pools, dir_path): return None -class VmCreationManager(object): +class VmCreationManager: ''' A `ContextManager` which cleans up if volume creation fails. ''' # pylint: disable=too-few-public-methods def __init__(self, vm): diff --git a/qubes/storage/file.py b/qubes/storage/file.py index ead07f4f..46bc3608 100644 --- a/qubes/storage/file.py +++ b/qubes/storage/file.py @@ -356,9 +356,9 @@ class FileVolume(qubes.storage.Volume): def script(self): if not self.snap_on_start and not self.save_on_stop: return None - elif not self.snap_on_start and self.save_on_stop: + if not self.snap_on_start and self.save_on_stop: return 'block-origin' - elif self.snap_on_start: + if self.snap_on_start: return 'block-snapshot' return None diff --git a/qubes/utils.py b/qubes/utils.py index 4fefe7df..63c49d90 100644 --- a/qubes/utils.py +++ b/qubes/utils.py @@ -41,7 +41,7 @@ def get_timezone(): if os.path.islink('/etc/localtime'): return '/'.join(os.readlink('/etc/localtime').split('/')[-2:]) # <=fc17 - elif os.path.exists('/etc/sysconfig/clock'): + if os.path.exists('/etc/sysconfig/clock'): clock_config = open('/etc/sysconfig/clock', "r") clock_config_lines = clock_config.readlines() clock_config.close() @@ -50,18 +50,17 @@ def get_timezone(): line_match = zone_re.match(line) if line_match: return line_match.group(1) - else: - # last resort way, some applications makes /etc/localtime - # hardlink instead of symlink... - tz_info = os.stat('/etc/localtime') - if not tz_info: - return None - if tz_info.st_nlink > 1: - p = subprocess.Popen(['find', '/usr/share/zoneinfo', - '-inum', str(tz_info.st_ino), '-print', '-quit'], - stdout=subprocess.PIPE) - tz_path = p.communicate()[0].strip() - return tz_path.replace('/usr/share/zoneinfo/', '') + # last resort way, some applications makes /etc/localtime + # hardlink instead of symlink... + tz_info = os.stat('/etc/localtime') + if not tz_info: + return None + if tz_info.st_nlink > 1: + p = subprocess.Popen(['find', '/usr/share/zoneinfo', + '-inum', str(tz_info.st_ino), '-print', '-quit'], + stdout=subprocess.PIPE) + tz_path = p.communicate()[0].strip() + return tz_path.replace(b'/usr/share/zoneinfo/', b'') return None @@ -132,9 +131,9 @@ def size_to_human(size): """Humane readable size, with 1/10 precision""" if size < 1024: return str(size) - elif size < 1024 * 1024: + if size < 1024 * 1024: return str(round(size / 1024.0, 1)) + ' KiB' - elif size < 1024 * 1024 * 1024: + if size < 1024 * 1024 * 1024: return str(round(size / (1024.0 * 1024), 1)) + ' MiB' return str(round(size / (1024.0 * 1024 * 1024), 1)) + ' GiB' @@ -181,6 +180,6 @@ def match_vm_name_with_special(vm, name): or @type:...''' if name.startswith('@tag:'): return name[len('@tag:'):] in vm.tags - elif name.startswith('@type:'): + if name.startswith('@type:'): return name[len('@type:'):] == vm.__class__.__name__ return name == vm.name diff --git a/qubes/vm/__init__.py b/qubes/vm/__init__.py index 33b49a96..c0f078cf 100644 --- a/qubes/vm/__init__.py +++ b/qubes/vm/__init__.py @@ -568,10 +568,9 @@ class VMProperty(qubes.property): if self.allow_none: super(VMProperty, self).__set__(instance, value) return - else: - raise ValueError( - 'Property {!r} does not allow setting to {!r}'.format( - self.__name__, value)) + raise ValueError( + 'Property {!r} does not allow setting to {!r}'.format( + self.__name__, value)) app = instance if isinstance(instance, qubes.Qubes) else instance.app diff --git a/qubes/vm/adminvm.py b/qubes/vm/adminvm.py index 2d044f67..e83f0ab9 100644 --- a/qubes/vm/adminvm.py +++ b/qubes/vm/adminvm.py @@ -147,12 +147,11 @@ class AdminVM(qubes.vm.BaseVM): if self.app.vmm.offline_mode: # default value passed on xen cmdline return 4096 - else: - try: - return self.app.vmm.libvirt_conn.getInfo()[1] - except libvirt.libvirtError as e: - self.log.warning('Failed to get memory limit for dom0: %s', e) - return 4096 + try: + return self.app.vmm.libvirt_conn.getInfo()[1] + except libvirt.libvirtError as e: + self.log.warning('Failed to get memory limit for dom0: %s', e) + return 4096 def verify_files(self): '''Always :py:obj:`True` @@ -181,7 +180,7 @@ class AdminVM(qubes.vm.BaseVM): @property def icon_path(self): - return None + pass @property def untrusted_qdb(self): diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 4730a2aa..1c966a9a 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -518,10 +518,9 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): except libvirt.libvirtError as e: if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN: return -1 - else: - self.log.exception('libvirt error code: {!r}'.format( - e.get_error_code())) - raise + self.log.exception('libvirt error code: {!r}'.format( + e.get_error_code())) + raise @qubes.stateless_property def stubdom_xid(self): @@ -1575,8 +1574,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): except libvirt.libvirtError as e: if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN: return 'Halted' - else: - raise + raise libvirt_domain = self.libvirt_domain if libvirt_domain is None: @@ -1587,19 +1585,17 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): # pylint: disable=line-too-long if libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_PAUSED: return "Paused" - elif libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_CRASHED: + if libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_CRASHED: return "Crashed" - elif libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_SHUTDOWN: + if libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_SHUTDOWN: return "Halting" - elif libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_SHUTOFF: + if libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_SHUTOFF: return "Dying" - elif libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_PMSUSPENDED: # nopep8 + if libvirt_domain.state()[0] == libvirt.VIR_DOMAIN_PMSUSPENDED: # nopep8 return "Suspended" - else: - if not self.is_fully_usable(): - return "Transient" - - return "Running" + if not self.is_fully_usable(): + return "Transient" + return "Running" return 'Halted' except libvirt.libvirtError as e: @@ -1639,8 +1635,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): except libvirt.libvirtError as e: if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN: return False - else: - raise + raise return bool(self.libvirt_domain.isActive()) @@ -1662,7 +1657,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): :py:obj:`False` otherwise. :rtype: bool ''' - if self.xid < 0: + if self.xid < 0: # pylint: disable=comparison-with-callable return False return os.path.exists('/var/run/qubes/qrexec.%s' % self.name) @@ -1706,10 +1701,9 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): libvirt.VIR_ERR_INTERNAL_ERROR): return 0 - else: - self.log.exception( - 'libvirt error code: {!r}'.format(e.get_error_code())) - raise + self.log.exception( + 'libvirt error code: {!r}'.format(e.get_error_code())) + raise def get_mem_static_max(self): '''Get maximum memory available to VM. @@ -1733,10 +1727,9 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): libvirt.VIR_ERR_INTERNAL_ERROR): return 0 - else: - self.log.exception( - 'libvirt error code: {!r}'.format(e.get_error_code())) - raise + self.log.exception( + 'libvirt error code: {!r}'.format(e.get_error_code())) + raise def get_cputime(self): '''Get total CPU time burned by this domain since start. @@ -1772,10 +1765,9 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): libvirt.VIR_ERR_INTERNAL_ERROR): return 0 - else: - self.log.exception( - 'libvirt error code: {!r}'.format(e.get_error_code())) - raise + self.log.exception( + 'libvirt error code: {!r}'.format(e.get_error_code())) + raise # miscellanous diff --git a/qubespolicy/__init__.py b/qubespolicy/__init__.py index c8c6c042..55c5913f 100755 --- a/qubespolicy/__init__.py +++ b/qubespolicy/__init__.py @@ -18,6 +18,8 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, see . +# pylint: disable=no-else-return,useless-object-inheritance,try-except-raise + ''' Qrexec policy parser and evaluator ''' import enum import itertools diff --git a/qubespolicy/agent.py b/qubespolicy/agent.py index 4ae465eb..661ec8a1 100644 --- a/qubespolicy/agent.py +++ b/qubespolicy/agent.py @@ -33,7 +33,7 @@ import qubespolicy.rpcconfirmation import qubespolicy.policycreateconfirmation # pylint: enable=wrong-import-position -class PolicyAgent(object): +class PolicyAgent: # pylint: disable=too-few-public-methods dbus = """ diff --git a/qubespolicy/cli.py b/qubespolicy/cli.py index da1d02a7..4b6887eb 100644 --- a/qubespolicy/cli.py +++ b/qubespolicy/cli.py @@ -72,12 +72,12 @@ def main(args=None): if not log.handlers: handler = logging.handlers.SysLogHandler(address='/dev/log') log.addHandler(handler) - log_prefix = 'qrexec: {}: {} -> {}: '.format( + log_prefix = 'qrexec: {}: {} -> {}:'.format( args.service_name, args.domain, args.target) try: system_info = qubespolicy.get_system_info() except qubespolicy.QubesMgmtException as e: - log.error(log_prefix + 'error getting system info: ' + str(e)) + log.error('%s error getting system info: %s', log_prefix, str(e)) return 1 try: try: @@ -130,13 +130,13 @@ def main(args=None): action.handle_user_response(True, response) else: action.handle_user_response(False) - log.info(log_prefix + 'allowed to {}'.format(action.target)) + log.info('%s allowed to %s', log_prefix, action.target) action.execute(caller_ident) except qubespolicy.PolicySyntaxError as e: - log.error(log_prefix + 'error loading policy: ' + str(e)) + log.error('%s error loading policy: %s', log_prefix, str(e)) return 1 except qubespolicy.AccessDenied as e: - log.info(log_prefix + 'denied: ' + str(e)) + log.info('%s denied: %s', log_prefix, str(e)) return 1 return 0 diff --git a/qubespolicy/policycreateconfirmation.py b/qubespolicy/policycreateconfirmation.py index 248bad15..3aeec822 100644 --- a/qubespolicy/policycreateconfirmation.py +++ b/qubespolicy/policycreateconfirmation.py @@ -28,7 +28,7 @@ gi.require_version('Gtk', '3.0') from gi.repository import Gtk # pylint: enable=import-error -class PolicyCreateConfirmationWindow(object): +class PolicyCreateConfirmationWindow: # pylint: disable=too-few-public-methods _source_file = pkg_resources.resource_filename('qubespolicy', os.path.join('glade', "PolicyCreateConfirmationWindow.glade")) From af7d54d3883505401ecf0a26567ae699877520e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 9 Jul 2018 19:42:18 +0200 Subject: [PATCH 022/145] Update windows-related feature requests Handle 'os' feature - if it's Windows, then set rpc-clipboard feature. Handle 'gui-emulated' feature - request for specifically stubdomain GUI. With 'gui' feature it is only possible to enable gui-agent based on, or disable GUI completely. Handle 'default-user' - verify it for weird characters and set 'default_user' property (if wasn't already set). QubesOS/qubes-issues#3585 --- qubes/api/misc.py | 3 +- qubes/ext/core_features.py | 4 +-- qubes/ext/windows.py | 64 ++++++++++++++++++++++++++++++++++++++ qubes/tests/api_misc.py | 10 ++++-- qubes/tests/ext.py | 51 ++++++++++++++++++++++++++++++ rpm_spec/core-dom0.spec.in | 1 + setup.py | 1 + 7 files changed, 129 insertions(+), 5 deletions(-) create mode 100644 qubes/ext/windows.py diff --git a/qubes/api/misc.py b/qubes/api/misc.py index 50652e3c..b95e75e3 100644 --- a/qubes/api/misc.py +++ b/qubes/api/misc.py @@ -76,7 +76,8 @@ class QubesMiscAPI(qubes.api.AbstractQubesAPI): untrusted_features = {} safe_set = string.ascii_letters + string.digits - expected_features = ('qrexec', 'gui', 'default-user') + expected_features = ('qrexec', 'gui', 'gui-emulated', 'default-user', + 'os') for feature in expected_features: untrusted_value = self.src.untrusted_qdb.read( '/qubes-tools/' + feature) diff --git a/qubes/ext/core_features.py b/qubes/ext/core_features.py index b2b77ea2..f07f5e93 100644 --- a/qubes/ext/core_features.py +++ b/qubes/ext/core_features.py @@ -32,7 +32,7 @@ class CoreFeatures(qubes.ext.Extension): return requested_features = {} - for feature in ('qrexec', 'gui', 'qubes-firewall'): + for feature in ('qrexec', 'gui', 'gui-emulated', 'qubes-firewall'): untrusted_value = untrusted_features.get(feature, None) if untrusted_value in ('1', '0'): requested_features[feature] = bool(int(untrusted_value)) @@ -44,7 +44,7 @@ class CoreFeatures(qubes.ext.Extension): # gui agent presence (0 or 1) qrexec_before = vm.features.get('qrexec', False) - for feature in ('qrexec', 'gui'): + for feature in ('qrexec', 'gui', 'gui-emulated'): # do not allow (Template)VM to override setting if already set # some other way if feature in requested_features and feature not in vm.features: diff --git a/qubes/ext/windows.py b/qubes/ext/windows.py new file mode 100644 index 00000000..57ddb1e8 --- /dev/null +++ b/qubes/ext/windows.py @@ -0,0 +1,64 @@ +# -*- encoding: utf-8 -*- +# +# The Qubes OS Project, http://www.qubes-os.org +# +# Copyright (C) 2017 Marek Marczykowski-Górecki +# +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see . + +import qubes.ext + +class WindowsFeatures(qubes.ext.Extension): + # pylint: disable=too-few-public-methods + @qubes.ext.handler('features-request') + def qubes_features_request(self, vm, event, untrusted_features): + '''Handle features provided requested by Qubes Windows Tools''' + # pylint: disable=no-self-use,unused-argument + if getattr(vm, 'template', None): + vm.log.warning( + 'Ignoring qubes.NotifyTools for template-based VM') + return + + guest_os = None + if 'os' in untrusted_features: + if untrusted_features['os'] in ['Windows']: + guest_os = untrusted_features['os'] + + qrexec = None + if 'qrexec' in untrusted_features: + if untrusted_features['qrexec'] == '1': + # qrexec feature is set by CoreFeatures extension + qrexec = True + + del untrusted_features + + if guest_os: + vm.features['os'] = guest_os + if guest_os == 'Windows' and qrexec: + vm.features['rpc-clipboard'] = True + + @qubes.ext.handler('domain-add', system=True) + def on_domain_add(self, app, _event, vm, **kwargs): + # pylint: disable=no-self-use,unused-argument + if getattr(vm, 'template', None) is None: + # handle only template-based vms + return + + template = vm.template + if template.features.check_with_template('os', None) != 'Windows': + # ignore non-windows templates + return + + # TODO: consider copying template's root volume here diff --git a/qubes/tests/api_misc.py b/qubes/tests/api_misc.py index b7ffc9a0..8eddd3e2 100644 --- a/qubes/tests/api_misc.py +++ b/qubes/tests/api_misc.py @@ -131,11 +131,14 @@ class TC_00_API_Misc(qubes.tests.QubesTestCase): self.assertEqual(self.src.mock_calls, [ mock.call.untrusted_qdb.read('/qubes-tools/qrexec'), mock.call.untrusted_qdb.read('/qubes-tools/gui'), + mock.call.untrusted_qdb.read('/qubes-tools/gui-emulated'), mock.call.untrusted_qdb.read('/qubes-tools/default-user'), + mock.call.untrusted_qdb.read('/qubes-tools/os'), mock.call.fire_event_async('features-request', untrusted_features={ 'gui': '1', 'default-user': 'user', - 'qrexec': '1'}), + 'qrexec': '1', + 'os': 'Linux'}), ('fire_event_async().__iter__', (), {}), ]) self.assertEqual(self.app.mock_calls, [mock.call.save()]) @@ -153,11 +156,14 @@ class TC_00_API_Misc(qubes.tests.QubesTestCase): self.assertEqual(self.src.mock_calls, [ mock.call.untrusted_qdb.read('/qubes-tools/qrexec'), mock.call.untrusted_qdb.read('/qubes-tools/gui'), + mock.call.untrusted_qdb.read('/qubes-tools/gui-emulated'), mock.call.untrusted_qdb.read('/qubes-tools/default-user'), + mock.call.untrusted_qdb.read('/qubes-tools/os'), mock.call.fire_event_async('features-request', untrusted_features={ 'gui': '1', 'default-user': 'user', - 'qrexec': '1'}), + 'qrexec': '1', + 'os': 'Linux'}), ('fire_event_async().__iter__', (), {}), ]) self.assertEqual(self.app.mock_calls, [mock.call.save()]) diff --git a/qubes/tests/ext.py b/qubes/tests/ext.py index 1197875c..bb937f68 100644 --- a/qubes/tests/ext.py +++ b/qubes/tests/ext.py @@ -21,6 +21,7 @@ from unittest import mock import qubes.ext.core_features +import qubes.ext.windows import qubes.tests @@ -163,3 +164,53 @@ class TC_00_CoreFeatures(qubes.tests.QubesTestCase): ('features.__contains__', ('qrexec',), {}), ('features.__contains__', ('gui',), {}), ]) + +class TC_10_WindowsFeatures(qubes.tests.QubesTestCase): + def setUp(self): + super().setUp() + self.ext = qubes.ext.windows.WindowsFeatures() + self.vm = mock.MagicMock() + self.features = {} + self.vm.configure_mock(**{ + 'features.get.side_effect': self.features.get, + 'features.__contains__.side_effect': self.features.__contains__, + 'features.__setitem__.side_effect': self.features.__setitem__, + }) + + def test_000_notify_tools_full(self): + del self.vm.template + self.ext.qubes_features_request(self.vm, 'features-request', + untrusted_features={ + 'gui': '1', + 'version': '1', + 'default-user': 'user', + 'qrexec': '1', + 'os': 'Windows'}) + self.assertEqual(self.vm.mock_calls, [ + ('features.__setitem__', ('os', 'Windows'), {}), + ('features.__setitem__', ('rpc-clipboard', True), {}), + ]) + + def test_001_notify_tools_no_qrexec(self): + del self.vm.template + self.ext.qubes_features_request(self.vm, 'features-request', + untrusted_features={ + 'gui': '1', + 'version': '1', + 'default-user': 'user', + 'qrexec': '0', + 'os': 'Windows'}) + self.assertEqual(self.vm.mock_calls, [ + ('features.__setitem__', ('os', 'Windows'), {}), + ]) + + def test_002_notify_tools_other_os(self): + del self.vm.template + self.ext.qubes_features_request(self.vm, 'features-request', + untrusted_features={ + 'gui': '1', + 'version': '1', + 'default-user': 'user', + 'qrexec': '1', + 'os': 'Linux'}) + self.assertEqual(self.vm.mock_calls, []) diff --git a/rpm_spec/core-dom0.spec.in b/rpm_spec/core-dom0.spec.in index 2e3a5f54..283434cb 100644 --- a/rpm_spec/core-dom0.spec.in +++ b/rpm_spec/core-dom0.spec.in @@ -284,6 +284,7 @@ fi %{python3_sitelib}/qubes/ext/qubesmanager.py %{python3_sitelib}/qubes/ext/r3compatibility.py %{python3_sitelib}/qubes/ext/services.py +%{python3_sitelib}/qubes/ext/windows.py %dir %{python3_sitelib}/qubes/tests %dir %{python3_sitelib}/qubes/tests/__pycache__ diff --git a/setup.py b/setup.py index 1f3ae9ce..d2110ac2 100644 --- a/setup.py +++ b/setup.py @@ -74,6 +74,7 @@ if __name__ == '__main__': 'qubes.ext.pci = qubes.ext.pci:PCIDeviceExtension', 'qubes.ext.block = qubes.ext.block:BlockDeviceExtension', 'qubes.ext.services = qubes.ext.services:ServicesExtension', + 'qubes.ext.windows = qubes.ext.windows:WindowsFeatures', ], 'qubes.devices': [ 'pci = qubes.ext.pci:PCIDevice', From af2435c0d464394e93994e2beffa8bd486c2d264 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 11 Jul 2018 04:35:36 +0200 Subject: [PATCH 023/145] Make some properties default to template's value (if any) Multiple properties are related to system installed inside the VM, so it makes sense to have them the same for all the VMs based on the same template. Modify default value getter to first try get the value from a template (if any) and only if it fails, fallback to original default value. This change is made to those properties: - default_user (it was already this way) - kernel - kernelopts - maxmem - memory - qrexec_timeout - vcpus - virt_mode This is especially useful for manually installed templates (like Windows). Related to QubesOS/qubes-issues#3585 --- qubes/tests/vm/qubesvm.py | 44 +++++++++++++++++++++++++++++ qubes/vm/qubesvm.py | 58 ++++++++++++++++++++++++++++----------- 2 files changed, 86 insertions(+), 16 deletions(-) diff --git a/qubes/tests/vm/qubesvm.py b/qubes/tests/vm/qubesvm.py index 168bd579..6d2e52ff 100644 --- a/qubes/tests/vm/qubesvm.py +++ b/qubes/tests/vm/qubesvm.py @@ -141,6 +141,50 @@ class TC_00_setters(qubes.tests.QubesTestCase): with self.assertRaises(ValueError): qubes.vm.qubesvm._setter_virt_mode(self.vm, self.prop, 'True') +class TC_10_default(qubes.tests.QubesTestCase): + def setUp(self): + super().setUp() + self.app = TestApp() + self.vm = TestVM(app=self.app) + self.prop = TestProp() + + def test_000_default_with_template_simple(self): + default_getter = qubes.vm.qubesvm._default_with_template('kernel', + 'dfl-kernel') + self.assertEqual(default_getter(self.vm), 'dfl-kernel') + self.vm.template = None + self.assertEqual(default_getter(self.vm), 'dfl-kernel') + self.vm.template = unittest.mock.Mock() + self.vm.template.kernel = 'template-kernel' + self.assertEqual(default_getter(self.vm), 'template-kernel') + + def test_001_default_with_template_callable(self): + default_getter = qubes.vm.qubesvm._default_with_template('kernel', + lambda x: x.app.default_kernel) + self.app.default_kernel = 'global-dfl-kernel' + self.assertEqual(default_getter(self.vm), 'global-dfl-kernel') + self.vm.template = None + self.assertEqual(default_getter(self.vm), 'global-dfl-kernel') + self.vm.template = unittest.mock.Mock() + self.vm.template.kernel = 'template-kernel' + self.assertEqual(default_getter(self.vm), 'template-kernel') + + def test_010_default_virt_mode(self): + default_getter = qubes.vm.qubesvm._default_with_template('kernel', + lambda x: x.app.default_kernel) + self.assertEqual(qubes.vm.qubesvm._default_virt_mode(self.vm), + 'pvh') + self.vm.template = unittest.mock.Mock() + self.vm.template.virt_mode = 'hvm' + self.assertEqual(qubes.vm.qubesvm._default_virt_mode(self.vm), + 'hvm') + self.vm.template = None + self.assertEqual(qubes.vm.qubesvm._default_virt_mode(self.vm), + 'pvh') + self.vm.devices['pci'].persistent().append('some-dev') + self.assertEqual(qubes.vm.qubesvm._default_virt_mode(self.vm), + 'hvm') + class QubesVMTestsMixin(object): property_no_default = object() diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 1c966a9a..8ee31086 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -102,7 +102,25 @@ def _setter_virt_mode(self, prop, value): def _default_virt_mode(self): if self.devices['pci'].persistent(): return 'hvm' - return 'pvh' + try: + return self.template.virt_mode + except AttributeError: + return 'pvh' + +def _default_with_template(prop, default): + '''Return a callable for 'default' argument of a property. Use a value + from a template (if any), otherwise *default* + ''' + + def _func(self): + try: + return getattr(self.template, prop) + except AttributeError: + if callable(default): + return default(self) + return default + + return _func class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): @@ -387,7 +405,8 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): type=str, setter=_setter_virt_mode, default=_default_virt_mode, doc='''Virtualisation mode: full virtualisation ("HVM"), - or paravirtualisation ("PV"), or hybrid ("PVH")''') + or paravirtualisation ("PV"), or hybrid ("PVH"). TemplateBasedVMs use its ' + 'template\'s value by default.''') installed_by_rpm = qubes.property('installed_by_rpm', type=bool, setter=qubes.property.bool, @@ -397,17 +416,19 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): memory = qubes.property('memory', type=int, setter=_setter_positive_int, - default=(lambda self: + default=_default_with_template('memory', lambda self: qubes.config.defaults[ 'hvm_memory' if self.virt_mode == 'hvm' else 'memory']), - doc='Memory currently available for this VM.') + doc='Memory currently available for this VM. TemplateBasedVMs use its ' + 'template\'s value by default.') maxmem = qubes.property('maxmem', type=int, setter=_setter_positive_int, - default=(lambda self: - int(min(self.app.host.memory_total / 1024 / 2, 4000))), + default=_default_with_template('maxmem', (lambda self: + int(min(self.app.host.memory_total / 1024 / 2, 4000)))), doc='''Maximum amount of memory available for this VM (for the purpose - of the memory balancer).''') + of the memory balancer). TemplateBasedVMs use its ' + 'template\'s value by default.''') stubdom_mem = qubes.property('stubdom_mem', type=int, setter=_setter_positive_int, @@ -417,14 +438,17 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): vcpus = qubes.property('vcpus', type=int, setter=_setter_positive_int, - default=2, - doc='Number of virtual CPUs for a qube') + default=_default_with_template('vcpus', 2), + doc='Number of virtual CPUs for a qube. TemplateBasedVMs use its ' + 'template\'s value by default.') # CORE2: swallowed uses_default_kernel kernel = qubes.property('kernel', type=str, setter=_setter_kernel, - default=(lambda self: self.app.default_kernel), - doc='Kernel used by this domain.') + default=_default_with_template('kernel', + lambda self: self.app.default_kernel), + doc='Kernel used by this domain. TemplateBasedVMs use its ' + 'template\'s value by default.') # CORE2: swallowed uses_default_kernelopts # pylint: disable=no-member @@ -434,7 +458,8 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): if list(self.devices['pci'].persistent()) else self.template.kernelopts if hasattr(self, 'template') else qubes.config.defaults['kernelopts']), - doc='Kernel command line passed to domain.') + doc='Kernel command line passed to domain. TemplateBasedVMs use its ' + 'template\'s value by default.') debug = qubes.property('debug', type=bool, default=False, setter=qubes.property.bool, @@ -445,10 +470,10 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): # only plain property? default_user = qubes.property('default_user', type=str, # pylint: disable=no-member - default=(lambda self: self.template.default_user - if hasattr(self, 'template') else 'user'), + default=_default_with_template('default_user', 'user'), setter=_setter_default_user, - doc='FIXME') + doc='Default user to start applications as. TemplateBasedVMs use its ' + 'template\'s value by default.') # pylint: enable=no-member @@ -459,7 +484,8 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): # else: # return self._default_user - qrexec_timeout = qubes.property('qrexec_timeout', type=int, default=60, + qrexec_timeout = qubes.property('qrexec_timeout', type=int, + default=_default_with_template('qrexec_timeout', 60), setter=_setter_positive_int, doc='''Time in seconds after which qrexec connection attempt is deemed failed. Operating system inside VM should be able to boot in this From e51efcf980f169433e1aa3164b652e7f490ac55a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 11 Jul 2018 04:50:37 +0200 Subject: [PATCH 024/145] vm: document domain-start-failed event --- qubes/vm/qubesvm.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 8ee31086..39713255 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -176,6 +176,17 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): *other arguments are as in :py:meth:`start`* + .. event:: domain-start-failed (subject, event, reason) + + Fired when :py:meth:`start` method fails. + *reason* argument is a textual error message. + + Handler for this event can be asynchronous (a coroutine). + + :param subject: Event emitter (the qube object) + :param event: Event name (``'domain-start'``) + + *other arguments are as in :py:meth:`start`* .. event:: domain-stopped (subject, event) Fired when domain has been stopped. From e6edbabf9421d9f74f7367aa992024d64f6acfac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 15 Jul 2018 22:08:06 +0200 Subject: [PATCH 025/145] tests: exclude windows templates from linux tests --- qubes/tests/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index 7965c7fe..0d3144c2 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -1130,7 +1130,8 @@ def list_templates(): try: app = qubes.Qubes() _templates = tuple(vm.name for vm in app.domains - if isinstance(vm, qubes.vm.templatevm.TemplateVM)) + if isinstance(vm, qubes.vm.templatevm.TemplateVM) and + vm.features.get('os', None) != 'Windows') app.close() del app except OSError: From 0e089ca38d3fe00ff9de54d6f46506e8dab7000b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 16 Jul 2018 22:02:05 +0200 Subject: [PATCH 026/145] ext/windows: copy private.img on windows TemplateBasedVM creation This is a workaround for missing private.img initialization in Qubes Windows Tools. QubesOS/qubes-issues#3585 --- qubes/ext/windows.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/qubes/ext/windows.py b/qubes/ext/windows.py index 57ddb1e8..75420975 100644 --- a/qubes/ext/windows.py +++ b/qubes/ext/windows.py @@ -17,6 +17,7 @@ # # You should have received a copy of the GNU General Public License along # with this program; if not, see . +import asyncio import qubes.ext @@ -49,8 +50,9 @@ class WindowsFeatures(qubes.ext.Extension): if guest_os == 'Windows' and qrexec: vm.features['rpc-clipboard'] = True - @qubes.ext.handler('domain-add', system=True) - def on_domain_add(self, app, _event, vm, **kwargs): + @qubes.ext.handler('domain-create-on-disk') + @asyncio.coroutine + def on_domain_create_on_disk(self, vm, _event, **kwargs): # pylint: disable=no-self-use,unused-argument if getattr(vm, 'template', None) is None: # handle only template-based vms @@ -61,4 +63,11 @@ class WindowsFeatures(qubes.ext.Extension): # ignore non-windows templates return - # TODO: consider copying template's root volume here + if vm.volumes['private'].save_on_stop: + # until windows tools get ability to prepare private.img on its own, + # copy one from the template + vm.log.info('Windows template - cloning private volume') + import_op = vm.volumes['private'].import_volume( + template.volumes['private']) + if asyncio.iscoroutine(import_op): + yield from import_op From e7f84dedd185799365835019bc0b5f3d2b2341df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 19 Jul 2018 15:21:21 +0200 Subject: [PATCH 027/145] version 4.0.28 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index f5cd8d2e..07ff330b 100644 --- a/version +++ b/version @@ -1 +1 @@ -4.0.27 +4.0.28 From b6a015b329a1dc0c0b89209a74d335537c91a4cd Mon Sep 17 00:00:00 2001 From: Patrick Schleizer Date: Fri, 20 Jul 2018 23:38:59 +0800 Subject: [PATCH 028/145] remove Whonix specific exceptions Whonix will use `qvm-open-in-dvm` so no more exceptions required. --- qubes-rpc-policy/qubes.OpenURL.policy | 3 --- 1 file changed, 3 deletions(-) diff --git a/qubes-rpc-policy/qubes.OpenURL.policy b/qubes-rpc-policy/qubes.OpenURL.policy index 27303cc9..41217337 100644 --- a/qubes-rpc-policy/qubes.OpenURL.policy +++ b/qubes-rpc-policy/qubes.OpenURL.policy @@ -3,8 +3,5 @@ ## Please use a single # to start your custom comments -sys-whonix anon-whonix allow -whonix-gw anon-whonix allow -whonix-ws anon-whonix allow $anyvm $dispvm allow $anyvm $anyvm ask From 6f311ee1ffb735d4315532e682f114eabda09b5e Mon Sep 17 00:00:00 2001 From: Patrick Schleizer Date: Fri, 20 Jul 2018 23:39:00 +0800 Subject: [PATCH 029/145] remove Whonix specific exceptions Whonix will use `qvm-open-in-dvm` so no more exceptions required. --- qubes-rpc-policy/qubes.OpenInVM.policy | 3 --- 1 file changed, 3 deletions(-) diff --git a/qubes-rpc-policy/qubes.OpenInVM.policy b/qubes-rpc-policy/qubes.OpenInVM.policy index 27303cc9..41217337 100644 --- a/qubes-rpc-policy/qubes.OpenInVM.policy +++ b/qubes-rpc-policy/qubes.OpenInVM.policy @@ -3,8 +3,5 @@ ## Please use a single # to start your custom comments -sys-whonix anon-whonix allow -whonix-gw anon-whonix allow -whonix-ws anon-whonix allow $anyvm $dispvm allow $anyvm $anyvm ask From e95ef5f61d4e41c17f0e251dd3434f19316c6963 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Ouellet Date: Sun, 29 Jul 2018 22:37:31 -0400 Subject: [PATCH 030/145] Add domain-paused/-unpaused events Needed for event-driven domains-tray UI updating and anti-GUI-DoS usability improvements. Catches errors from event handlers to protect libvirt, and logs to main qubesd logger singleton (by default meaning systemd journal). --- qubes/app.py | 10 ++++++++++ qubes/vm/qubesvm.py | 15 +++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/qubes/app.py b/qubes/app.py index a9e967e1..da668c51 100644 --- a/qubes/app.py +++ b/qubes/app.py @@ -1251,6 +1251,16 @@ class Qubes(qubes.PropertyHolder): if event == libvirt.VIR_DOMAIN_EVENT_STOPPED: vm.on_libvirt_domain_stopped() + elif event == libvirt.VIR_DOMAIN_EVENT_SUSPENDED: + try: + vm.fire_event('domain-paused') + except Exception: # pylint: disable=broad-except + self.log.exception('Uncaught exception from domain-paused handler for domain %s', vm.name) + elif event == libvirt.VIR_DOMAIN_EVENT_RESUMED: + try: + vm.fire_event('domain-unpaused') + except Exception: # pylint: disable=broad-except + self.log.exception('Uncaught exception from domain-unpaused handler for domain %s', vm.name) @qubes.events.handler('domain-pre-delete') def on_domain_pre_deleted(self, event, vm): diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 39713255..5c8f5843 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -187,6 +187,21 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): :param event: Event name (``'domain-start'``) *other arguments are as in :py:meth:`start`* + + .. event:: domain-paused (subject, event) + + Fired when the domain has been paused. + + :param subject: Event emitter (the qube object) + :param event: Event name (``'domain-paused'``) + + .. event:: domain-unpaused (subject, event) + + Fired when the domain has been unpaused. + + :param subject: Event emitter (the qube object) + :param event: Event name (``'domain-unpaused'``) + .. event:: domain-stopped (subject, event) Fired when domain has been stopped. From 65341a3468cb5d1d100b63131655c7ee26f84b27 Mon Sep 17 00:00:00 2001 From: Patrick Schleizer Date: Wed, 8 Aug 2018 00:29:07 +0800 Subject: [PATCH 031/145] $tag:anon-vm $anyvm deny --- qubes-rpc-policy/qubes.GetDate.policy | 1 + 1 file changed, 1 insertion(+) diff --git a/qubes-rpc-policy/qubes.GetDate.policy b/qubes-rpc-policy/qubes.GetDate.policy index 24c72be4..466d5396 100644 --- a/qubes-rpc-policy/qubes.GetDate.policy +++ b/qubes-rpc-policy/qubes.GetDate.policy @@ -3,4 +3,5 @@ ## Please use a single # to start your custom comments +$tag:anon-vm $anyvm deny $anyvm $anyvm allow,target=dom0 From 57424ef50b7c5157aa6e590d23694817c771d77b Mon Sep 17 00:00:00 2001 From: Patrick Schleizer Date: Wed, 8 Aug 2018 01:31:32 +0800 Subject: [PATCH 032/145] add Whonix defaults --- qubes-rpc-policy/qubes.UpdatesProxy.policy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qubes-rpc-policy/qubes.UpdatesProxy.policy b/qubes-rpc-policy/qubes.UpdatesProxy.policy index 21c68c56..4c82de84 100644 --- a/qubes-rpc-policy/qubes.UpdatesProxy.policy +++ b/qubes-rpc-policy/qubes.UpdatesProxy.policy @@ -3,6 +3,9 @@ ## Please use a single # to start your custom comments +$tag:whonix-updatevm $default allow,target=sys-whonix +$tag:whonix-updatevm $anyvm deny + # Default rule for all TemplateVMs - direct the connection to sys-net $type:TemplateVM $default allow,target=sys-net From fbdf460db853d11c5e93c4606a2a2fe79f1dfec5 Mon Sep 17 00:00:00 2001 From: Patrick Schleizer Date: Wed, 8 Aug 2018 09:38:45 +0000 Subject: [PATCH 033/145] comments --- qubes-rpc-policy/qubes.UpdatesProxy.policy | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/qubes-rpc-policy/qubes.UpdatesProxy.policy b/qubes-rpc-policy/qubes.UpdatesProxy.policy index 4c82de84..ff4f8434 100644 --- a/qubes-rpc-policy/qubes.UpdatesProxy.policy +++ b/qubes-rpc-policy/qubes.UpdatesProxy.policy @@ -3,7 +3,13 @@ ## Please use a single # to start your custom comments +# Upgrade all TemplateVMs through sys-whonix. +#$type:TemplateVM $default allow,target=sys-whonix + +# Upgrade Whonix TemplateVMs through sys-whonix. $tag:whonix-updatevm $default allow,target=sys-whonix + +# Deny Whonix TemplateVMs using UpdatesProxy of any other VM. $tag:whonix-updatevm $anyvm deny # Default rule for all TemplateVMs - direct the connection to sys-net From 6f04c8d65b5523184f4efdb15471256b761e8262 Mon Sep 17 00:00:00 2001 From: Galland Date: Tue, 21 Aug 2018 03:08:17 +0200 Subject: [PATCH 034/145] Fix error on non ASCII PCI IDs upon qvm-device list solves https://github.com/QubesOS/qubes-issues/issues/4229 --- qubes/ext/pci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes/ext/pci.py b/qubes/ext/pci.py index bd5f34cc..3c4a2f93 100644 --- a/qubes/ext/pci.py +++ b/qubes/ext/pci.py @@ -42,7 +42,7 @@ def load_pci_classes(): # subclass subclass_name <-- single tab # prog-if prog-if_name <-- two tabs result = {} - with open('/usr/share/hwdata/pci.ids') as pciids: + with open('/usr/share/hwdata/pci.ids', encoding='utf-8', errors='ignore') as pciids: class_id = None subclass_id = None for line in pciids.readlines(): From 8b2b26134e5ced7ec727be8f60eb75886cfb7846 Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Mon, 27 Aug 2018 23:31:16 +0000 Subject: [PATCH 035/145] tools/qvm-sync-clock: don't start clockvm Since bda9264, a qubes.GetDate call from a VM will not cause clockvm startup. Also avoid causing it with /etc/cron.d/qubes-sync-clock.cron. Fixes QubesOS/qubes-issues#3588 --- qvm-tools/qvm-sync-clock | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/qvm-tools/qvm-sync-clock b/qvm-tools/qvm-sync-clock index 0e607f19..18e0bcf8 100755 --- a/qvm-tools/qvm-sync-clock +++ b/qvm-tools/qvm-sync-clock @@ -32,6 +32,11 @@ def main(): app = Qubes() clockvm = app.clockvm + if not clockvm.is_running(): + sys.stderr.write('ClockVM {} is not running, aborting.\n'.format( + clockvm.name)) + sys.exit(0) + p = clockvm.run_service('qubes.GetDate') untrusted_date_out = p.stdout.read(25).decode('ascii', errors='strict') untrusted_date_out = untrusted_date_out.strip() From 890df9ba033c6d374d2973ead050a3710e9355d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 1 Sep 2018 19:05:31 +0200 Subject: [PATCH 036/145] qubespolicy: ease testing by calling str(target) only once Don't call it multiple times depending on number of registered loggers. --- qubespolicy/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubespolicy/cli.py b/qubespolicy/cli.py index 4b6887eb..0be0a4e7 100644 --- a/qubespolicy/cli.py +++ b/qubespolicy/cli.py @@ -130,7 +130,7 @@ def main(args=None): action.handle_user_response(True, response) else: action.handle_user_response(False) - log.info('%s allowed to %s', log_prefix, action.target) + log.info('%s allowed to %s', log_prefix, str(action.target)) action.execute(caller_ident) except qubespolicy.PolicySyntaxError as e: log.error('%s error loading policy: %s', log_prefix, str(e)) From 57c9b2edf7fc3f927ecc2a6f5463a843e22cd076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 2 Sep 2018 03:27:14 +0200 Subject: [PATCH 037/145] code style fixes --- qubes/app.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/qubes/app.py b/qubes/app.py index da668c51..5deff36f 100644 --- a/qubes/app.py +++ b/qubes/app.py @@ -1254,13 +1254,17 @@ class Qubes(qubes.PropertyHolder): elif event == libvirt.VIR_DOMAIN_EVENT_SUSPENDED: try: vm.fire_event('domain-paused') - except Exception: # pylint: disable=broad-except - self.log.exception('Uncaught exception from domain-paused handler for domain %s', vm.name) + except Exception: # pylint: disable=broad-except + self.log.exception( + 'Uncaught exception from domain-paused handler ' + 'for domain %s', vm.name) elif event == libvirt.VIR_DOMAIN_EVENT_RESUMED: try: vm.fire_event('domain-unpaused') - except Exception: # pylint: disable=broad-except - self.log.exception('Uncaught exception from domain-unpaused handler for domain %s', vm.name) + except Exception: # pylint: disable=broad-except + self.log.exception( + 'Uncaught exception from domain-unpaused handler ' + 'for domain %s', vm.name) @qubes.events.handler('domain-pre-delete') def on_domain_pre_deleted(self, event, vm): From ee25f7c7bb2f20a6113a3d2c7ada0499514d7bf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 3 Sep 2018 00:23:05 +0200 Subject: [PATCH 038/145] vm: fix error reporting on PVH without kernel set Fixes QubesOS/qubes-issues#4254 --- qubes/vm/qubesvm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 5c8f5843..4bfc0317 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -918,7 +918,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): qmemman_client = None try: - if self.virt_mode == 'pvh' and self.kernel is None: + if self.virt_mode == 'pvh' and not self.kernel: raise qubes.exc.QubesException( 'virt_mode PVH require kernel to be set') yield from self.storage.verify() From e1378f70fbf8d686f744a679f44bd1f08a0100d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 6 Sep 2018 01:16:50 +0200 Subject: [PATCH 039/145] doc: fix libvirt config path --- doc/libvirt.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/libvirt.rst b/doc/libvirt.rst index bd35fe49..7add6a66 100644 --- a/doc/libvirt.rst +++ b/doc/libvirt.rst @@ -22,14 +22,14 @@ The distributor may put a file at :file:`/usr/share/qubes/template/xen-dist.xml`) to override this file. User may put a file at either :file:`/etc/qubes/templates/libvirt/xen-user.xml` or -:file:`/etc/qubes/templates/libvirt/by-name/.xml`, where ```` is +:file:`/etc/qubes/templates/libvirt/xen/by-name/.xml`, where ```` is full name of the domain. Wildcards are not supported but symlinks are. Jinja has a concept of template names, which basically is the path below some load point, which in Qubes' case is :file:`/etc/qubes/templates` and :file:`/usr/share/qubes/templates`. Thus names of those templates are respectively ``'libvirt/xen.xml'``, ``'libvirt/xen-dist.xml'``, -``'libvirt/xen-user.xml'`` and ``'libvirt/by-name/.xml'``. +``'libvirt/xen-user.xml'`` and ``'libvirt/xen/by-name/.xml'``. This will be important later. .. note:: @@ -95,6 +95,9 @@ basic Contains ````, ````, ````, ```` and ```` nodes. +cpu + ```` node. + os Contents of ```` node. From d3a5799245c900d380c6ee374b66e3e2742d678b Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Thu, 6 Sep 2018 16:23:24 +0000 Subject: [PATCH 040/145] Order qubesd before systemd-user-sessions qubes-vm@.service would already cause this ordering, but not every user has any autostart=True VMs. Also needed to maybe f*x QubesOS/qubes-issues#3149 at some point. --- linux/systemd/qubesd.service | 1 + 1 file changed, 1 insertion(+) diff --git a/linux/systemd/qubesd.service b/linux/systemd/qubesd.service index 08791abf..44711761 100644 --- a/linux/systemd/qubesd.service +++ b/linux/systemd/qubesd.service @@ -1,5 +1,6 @@ [Unit] Description=Qubes OS daemon +Before=systemd-user-sessions.service [Service] Type=notify From 8ce34334064534f348d1efff9984e2d00d4cd555 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 7 Sep 2018 15:04:00 +0200 Subject: [PATCH 041/145] tests: drop sudo in tests already running as root Don't spam already trashed log. --- qubes/tests/__init__.py | 2 +- qubes/tests/integ/dom0_update.py | 27 +++++++++++---------------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index 0d3144c2..13555cef 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -849,7 +849,7 @@ class SystemTestCase(QubesTestCase): ''' try: volumes = subprocess.check_output( - ['sudo', 'lvs', '--noheadings', '-o', 'vg_name,name', + ['lvs', '--noheadings', '-o', 'vg_name,name', '--separator', '/']).decode() if ('/vm-' + prefix) not in volumes: return diff --git a/qubes/tests/integ/dom0_update.py b/qubes/tests/integ/dom0_update.py index a289396f..516264a3 100644 --- a/qubes/tests/integ/dom0_update.py +++ b/qubes/tests/integ/dom0_update.py @@ -78,23 +78,18 @@ Expire-Date: 0 cls.keyid = cls.generate_key(cls.tmpdir) - p = subprocess.Popen(['sudo', 'dd', - 'status=none', 'of=/etc/yum.repos.d/test.repo'], - stdin=subprocess.PIPE) - p.stdin.write(b''' + with open('/etc/yum.repos.d/test.repo', 'w') as repo_file: + repo_file.write(''' [test] name = Test baseurl = http://localhost:8080/ enabled = 1 ''') - p.stdin.close() - p.wait() @classmethod def tearDownClass(cls): - subprocess.check_call(['sudo', 'rm', '-f', - '/etc/yum.repos.d/test.repo']) + os.unlink('/etc/yum.repos.d/test.repo') shutil.rmtree(cls.tmpdir) @@ -113,9 +108,9 @@ enabled = 1 self.loop.run_until_complete(self.updatevm.create_on_disk()) self.app.updatevm = self.updatevm self.app.save() - subprocess.call(['sudo', 'rpm', '-e', self.pkg_name], + subprocess.call(['rpm', '-e', self.pkg_name], stderr=subprocess.DEVNULL) - subprocess.check_call(['sudo', 'rpm', '--import', + subprocess.check_call(['rpm', '--import', os.path.join(self.tmpdir, 'pubkey.asc')]) self.loop.run_until_complete(self.updatevm.start()) self.repo_running = False @@ -128,9 +123,9 @@ enabled = 1 del self.repo_proc super(TC_00_Dom0UpgradeMixin, self).tearDown() - subprocess.call(['sudo', 'rpm', '-e', self.pkg_name], + subprocess.call(['rpm', '-e', self.pkg_name], stderr=subprocess.DEVNULL) - subprocess.call(['sudo', 'rpm', '-e', 'gpg-pubkey-{}'.format( + subprocess.call(['rpm', '-e', 'gpg-pubkey-{}'.format( self.keyid)], stderr=subprocess.DEVNULL) for pkg in os.listdir(self.tmpdir): @@ -165,7 +160,7 @@ Test package spec_path]) pkg_path = os.path.join(dir, 'x86_64', '{}-{}-1.x86_64.rpm'.format(name, version)) - subprocess.check_call(['sudo', 'chmod', 'go-rw', '/dev/tty']) + subprocess.check_call(['chmod', 'go-rw', '/dev/tty']) subprocess.check_call( ['rpm', '--quiet', '--define=_gpg_path {}'.format(dir), '--define=_gpg_name {}'.format("Qubes test"), @@ -173,7 +168,7 @@ Test package stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) - subprocess.check_call(['sudo', 'chmod', 'go+rw', '/dev/tty']) + subprocess.check_call(['chmod', 'go+rw', '/dev/tty']) return pkg_path def send_pkg(self, filename): @@ -212,7 +207,7 @@ Test package - "updates pending" flag is cleared """ filename = self.create_pkg(self.tmpdir, self.pkg_name, '1.0') - subprocess.check_call(['sudo', 'rpm', '-i', filename]) + subprocess.check_call(['rpm', '-i', filename]) filename = self.create_pkg(self.tmpdir, self.pkg_name, '2.0') self.send_pkg(filename) open(self.update_flag_path, 'a').close() @@ -331,7 +326,7 @@ Test package self.pkg_name)) def test_020_install_wrong_sign(self): - subprocess.call(['sudo', 'rpm', '-e', 'gpg-pubkey-{}'.format( + subprocess.call(['rpm', '-e', 'gpg-pubkey-{}'.format( self.keyid)]) filename = self.create_pkg(self.tmpdir, self.pkg_name, '1.0') self.send_pkg(filename) From b200dc8df2dd101774932a62349d5062c82cc41b Mon Sep 17 00:00:00 2001 From: Wojtek Porczyk Date: Fri, 7 Sep 2018 19:44:29 +0200 Subject: [PATCH 042/145] Fix from Marek's revies - add load_extras() - change callouts to arrows for draw.io limitations Cc: @marmarek --- doc/loading.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/loading.svg b/doc/loading.svg index b0552d4e..fa6038a2 100644 --- a/doc/loading.svg +++ b/doc/loading.svg @@ -1,3 +1,3 @@ -
qubes.Qubes
qubes.Qubes
load()
load()
Labels
Pools
Labels<br>Pools
VMs
VMs
.load() cont.
.load() cont.
load_properties(
load_stage=3)
load_properties(<br>load_stage=3)
.domains.add()
.domains.add()
domain-add
domain-add
qubes.vm.QubesVM
qubes.vm.QubesVM
__init__()
__init__()
load_properties(
load_stage=2)
load_properties(<br>load_stage=2)
load_properties(
load_stage=4)
load_properties(<br>load_stage=4)
.fire_event('domain-load')
.fire_event('domain-load')
domain-load
domain-load
default_dispvm
kernelopts
netvm
template
[Not supported by viewer]
anything by default,
if not in load_stage=4
anything by default,<br>if not in load_stage=4
admin.vm.Create
admin.vm.CreateInPool
admin.vm.CreateDisposable
[Not supported by viewer]
app.add_new_vm(cls)
app.add_new_vm(cls)
qubes.vm.QubesVM
qubes.vm.QubesVM
__init__()
__init__()
here the domain is actually removed from the collection
here the domain is actually removed from the collection
domain-init
domain-init
admin.vm.Remove
admin.vm.Remove
del app.domains[vm]
del app.domains[vm]
domain-pre-delete
domain-pre-delete
domain-delete
domain-delete
Current as of v4.0.27 (2b2cdf4)
Not included: storage
Current as of v4.0.27 (2b2cdf4)<br>Not included: storage
all properties are set
or left as default
all properties are set<br>or left as default
everything in app
everything in app
events
events
API
API
Key:
Key:
the domain is removed
from disk
the domain is removed<br>from disk \ No newline at end of file +
qubes.Qubes
qubes.Qubes
load()
load()
Labels
Pools
[Not supported by viewer]
VMs
VMs
.load() cont.
.load() cont.
load_properties(
load_stage=3)
load_properties(<br>load_stage=3)
.domains.add()
.domains.add()
domain-add
domain-add
qubes.vm.QubesVM
qubes.vm.QubesVM
__init__()
__init__()
load_properties(
load_stage=2)
load_properties(<br>load_stage=2)
load_properties(
load_stage=4)
load_properties(<br>load_stage=4)
.fire_event('domain-load')
.fire_event('domain-load')
domain-load
domain-load
admin.vm.Create
admin.vm.CreateInPool
admin.vm.CreateDisposable
[Not supported by viewer]
app.add_new_vm(cls)
app.add_new_vm(cls)
qubes.vm.QubesVM
qubes.vm.QubesVM
__init__()
__init__()
domain-init
domain-init
admin.vm.Remove
admin.vm.Remove
del app.domains[vm]
del app.domains[vm]
domain-pre-delete
domain-pre-delete
domain-delete
domain-delete
Current as of v4.0.27 (2b2cdf4)
Not included: storage
Current as of v4.0.27 (2b2cdf4)<br>Not included: storage
events
events
API
API
Key:
Key:
load_extras()
load_extras()
everything in app
everything in app
default_dispvm
kernelopts
netvm
template
[Not supported by viewer]
features
devices
tags
[Not supported by viewer]
anything by default,
if not in load_stage=4
anything by default,<br>if not in load_stage=4
all properties are set
or left as default
all properties are set<br>or left as default
the domain is
removed
from disk
[Not supported by viewer]
here the domain is actually
removed from the collection
here the domain is actually<br>removed from the collection \ No newline at end of file From c102fa3d68269e74ac4c36922b6fb5a219abd232 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 8 Sep 2018 04:13:24 +0200 Subject: [PATCH 043/145] tests: add basic audio play/rec tests QubesOS/qubes-issues#4204 --- qubes/tests/integ/vm_qrexec_gui.py | 146 +++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) diff --git a/qubes/tests/integ/vm_qrexec_gui.py b/qubes/tests/integ/vm_qrexec_gui.py index 0ed3ee7f..b83ba0a6 100644 --- a/qubes/tests/integ/vm_qrexec_gui.py +++ b/qubes/tests/integ/vm_qrexec_gui.py @@ -24,11 +24,15 @@ import multiprocessing import os import subprocess import sys +import tempfile import unittest from distutils import spawn +import grp + import qubes.config +import qubes.devices import qubes.tests import qubes.vm.appvm import qubes.vm.templatevm @@ -779,6 +783,148 @@ class TC_00_AppVMMixin(object): finally: self.app.clockvm = None + @unittest.skipUnless(spawn.find_executable('parecord'), + "pulseaudio-utils not installed in dom0") + def test_220_audio_playback(self): + self.loop.run_until_complete(self.testvm1.start()) + try: + self.loop.run_until_complete( + self.testvm1.run_for_stdio('which parecord')) + except subprocess.CalledProcessError: + self.skipTest('pulseaudio-utils not installed in VM') + + self.loop.run_until_complete( + self.wait_for_session(self.testvm1)) + # and some more... + self.loop.run_until_complete(asyncio.sleep(1)) + # generate some "audio" data + audio_in = b'\x20' * 44100 + self.loop.run_until_complete( + self.testvm1.run_for_stdio('cat > audio_in.raw', input=audio_in)) + local_user = grp.getgrnam('qubes').gr_mem[0] + with tempfile.NamedTemporaryFile() as recorded_audio: + os.chmod(recorded_audio.name, 0o666) + # FIXME: -d 0 assumes only one audio device + p = subprocess.Popen(['sudo', '-E', '-u', local_user, + 'parecord', '-d', '0', '--raw', recorded_audio.name], + stdout=subprocess.PIPE) + self.loop.run_until_complete( + self.testvm1.run_for_stdio('paplay --raw audio_in.raw')) + # wait for possible parecord buffering + self.loop.run_until_complete(asyncio.sleep(1)) + p.terminate() + # for some reason sudo do not relay SIGTERM sent above + subprocess.check_call(['pkill', 'parecord']) + p.wait() + # allow few bytes missing, don't use assertIn, to avoid printing + # the whole data in error message + if audio_in[:-8] not in recorded_audio.file.read(): + self.fail('played sound not found in dom0') + + def _configure_audio_recording(self, vm): + '''Connect VM's output-source to sink monitor instead of mic''' + local_user = grp.getgrnam('qubes').gr_mem[0] + sudo = ['sudo', '-E', '-u', local_user] + source_outputs = subprocess.check_output( + sudo + ['pacmd', 'list-source-outputs']).decode() + + last_index = None + found = False + for line in source_outputs.splitlines(): + if line.startswith(' index: '): + last_index = line.split(':')[1].strip() + elif line.startswith('\t\tapplication.name = '): + app_name = line.split('=')[1].strip('" ') + if vm.name == app_name: + found = True + break + if not found: + self.fail('source-output for VM {} not found'.format(vm.name)) + + subprocess.check_call(sudo + + ['pacmd', 'move-source-output', last_index, '0']) + + @unittest.skipUnless(spawn.find_executable('parecord'), + "pulseaudio-utils not installed in dom0") + def test_221_audio_record_muted(self): + self.loop.run_until_complete(self.testvm1.start()) + try: + self.loop.run_until_complete( + self.testvm1.run_for_stdio('which parecord')) + except subprocess.CalledProcessError: + self.skipTest('pulseaudio-utils not installed in VM') + + self.loop.run_until_complete( + self.wait_for_session(self.testvm1)) + # and some more... + self.loop.run_until_complete(asyncio.sleep(1)) + # connect VM's recording source output monitor (instead of mic) + self._configure_audio_recording(self.testvm1) + + # generate some "audio" data + audio_in = b'\x20' * 44100 + local_user = grp.getgrnam('qubes').gr_mem[0] + record = self.loop.run_until_complete( + self.testvm1.run('parecord --raw audio_rec.raw')) + # give it time to start recording + self.loop.run_until_complete(asyncio.sleep(0.5)) + p = subprocess.Popen(['sudo', '-E', '-u', local_user, + 'paplay', '--raw'], + stdin=subprocess.PIPE) + p.communicate(audio_in) + # wait for possible parecord buffering + self.loop.run_until_complete(asyncio.sleep(1)) + self.loop.run_until_complete( + self.testvm1.run_for_stdio('pkill parecord')) + record.wait() + recorded_audio, _ = self.loop.run_until_complete( + self.testvm1.run_for_stdio('cat audio_rec.raw')) + # should be empty or silence, so check just a little fragment + if audio_in[:32] in recorded_audio: + self.fail('VM recorded something, even though mic disabled') + + @unittest.skipUnless(spawn.find_executable('parecord'), + "pulseaudio-utils not installed in dom0") + def test_222_audio_record_unmuted(self): + self.loop.run_until_complete(self.testvm1.start()) + try: + self.loop.run_until_complete( + self.testvm1.run_for_stdio('which parecord')) + except subprocess.CalledProcessError: + self.skipTest('pulseaudio-utils not installed in VM') + + self.loop.run_until_complete( + self.wait_for_session(self.testvm1)) + # and some more... + self.loop.run_until_complete(asyncio.sleep(1)) + da = qubes.devices.DeviceAssignment(self.app.domains[0], 'mic') + self.loop.run_until_complete( + self.testvm1.devices['mic'].attach(da)) + # connect VM's recording source output monitor (instead of mic) + self._configure_audio_recording(self.testvm1) + + # generate some "audio" data + audio_in = b'\x20' * 44100 + local_user = grp.getgrnam('qubes').gr_mem[0] + record = self.loop.run_until_complete( + self.testvm1.run('parecord --raw audio_rec.raw')) + # give it time to start recording + self.loop.run_until_complete(asyncio.sleep(0.5)) + p = subprocess.Popen(['sudo', '-E', '-u', local_user, + 'paplay', '--raw'], + stdin=subprocess.PIPE) + p.communicate(audio_in) + # wait for possible parecord buffering + self.loop.run_until_complete(asyncio.sleep(1)) + self.loop.run_until_complete( + self.testvm1.run_for_stdio('pkill parecord')) + record.wait() + recorded_audio, _ = self.loop.run_until_complete( + self.testvm1.run_for_stdio('cat audio_rec.raw')) + # allow few bytes to be missing + if audio_in[:-8] not in recorded_audio: + self.fail('VM not recorded expected data') + def test_250_resize_private_img(self): """ Test private.img resize, both offline and online From b2cc605f4bf9f9a0c4a6600eeb897517b576ad21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 9 Sep 2018 02:33:17 +0200 Subject: [PATCH 044/145] tests: clean local variables from traceback objects System tests are fragile for any object leaks, especially those holding open files. Instead of wrapping all tests with try/finally removing those local variables (as done in qubes.tests.integ.backup for example), apply generic solution: clean all traceback objects from local variables. Those aren't used to generate text report by either test runner (qubes.tests.run and nose2). If one wants to break into debugger and inspect tracebacks interactively, needs to comment out call to cleanup_traceback. --- qubes/tests/__init__.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index 13555cef..140e7f43 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -379,8 +379,20 @@ class QubesTestCase(unittest.TestCase): self.loop = asyncio.get_event_loop() self.addCleanup(self.cleanup_loop) + self.addCleanup(self.cleanup_traceback) self.addCleanup(qubes.ext.pci._cache_get.cache_clear) + def cleanup_traceback(self): + '''Remove local variables reference from tracebacks to allow garbage + collector to clean all Qubes*() objects, otherwise file descriptors + held by them will leak''' + for test_case, exc_info in self._outcome.errors: + if test_case is not self: + continue + if exc_info is None: + continue + traceback.clear_frames(exc_info[2]) + def cleanup_gc(self): gc.collect() leaked = [obj for obj in gc.get_objects() + gc.garbage From a371375f1d67841a2040c98322e690d29032a475 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 9 Sep 2018 06:38:16 +0200 Subject: [PATCH 045/145] version 4.0.29 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 07ff330b..3ced8af0 100644 --- a/version +++ b/version @@ -1 +1 @@ -4.0.28 +4.0.29 From 850778b52af89fd20bafe1abfd7eeb6131d1f49e Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Sun, 9 Sep 2018 20:01:11 +0000 Subject: [PATCH 046/145] storage/reflink: remove redundant format specifiers --- qubes/storage/reflink.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/qubes/storage/reflink.py b/qubes/storage/reflink.py index 574febb3..e541555c 100644 --- a/qubes/storage/reflink.py +++ b/qubes/storage/reflink.py @@ -132,7 +132,7 @@ class ReflinkVolume(qubes.storage.Volume): if img is None or os.path.exists(img): return True raise qubes.storage.StoragePoolException( - 'Missing image file {!r} for volume {!s}'.format(img, self.vid)) + 'Missing image file {!r} for volume {}'.format(img, self.vid)) def remove(self): ''' Drop volume object from pool; remove volume images from @@ -214,12 +214,12 @@ class ReflinkVolume(qubes.storage.Volume): ''' if not self.rw: raise qubes.storage.StoragePoolException( - 'Cannot resize: {!s} is read-only'.format(self.vid)) + 'Cannot resize: {} is read-only'.format(self.vid)) if size < self.size: raise qubes.storage.StoragePoolException( - 'For your own safety, shrinking of {!s} is disabled' - ' ({:d} < {:d}). If you really know what you are doing,' + 'For your own safety, shrinking of {} is disabled' + ' ({} < {}). If you really know what you are doing,' ' use "truncate" manually.'.format(self.vid, size, self.size)) try: # assume volume is not (cleanly) stopped ... @@ -240,7 +240,7 @@ class ReflinkVolume(qubes.storage.Volume): def _require_save_on_stop(self, method_name): if not self.save_on_stop: raise NotImplementedError( - 'Cannot {!s}: {!s} is not save_on_stop'.format( + 'Cannot {}: {} is not save_on_stop'.format( method_name, self.vid)) def export(self): @@ -408,7 +408,7 @@ def _cmd(*args): stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout except subprocess.CalledProcessError as ex: - msg = '{!s} err={!r} out={!r}'.format(ex, ex.stderr, ex.stdout) + msg = '{} err={!r} out={!r}'.format(ex, ex.stderr, ex.stdout) raise qubes.storage.StoragePoolException(msg) from ex def is_reflink_supported(dst_dir, src_dir=None): From 677183d8a6932aa9e8dae8de29c8047b222d6412 Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Sun, 9 Sep 2018 20:01:12 +0000 Subject: [PATCH 047/145] storage/reflink: add revision even if empty It's sort of useful to be able to revert a volume that has only ever been started once to its empty state. And the lvm_thin driver allows it too, so why not. --- qubes/storage/reflink.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/qubes/storage/reflink.py b/qubes/storage/reflink.py index e541555c..2a0321b0 100644 --- a/qubes/storage/reflink.py +++ b/qubes/storage/reflink.py @@ -184,8 +184,6 @@ class ReflinkVolume(qubes.storage.Volume): def _add_revision(self): if self.revisions_to_keep == 0: return - if _get_file_disk_usage(self._path_clean) == 0: - return ctime = os.path.getctime(self._path_clean) timestamp = qubes.storage.isodate(int(ctime)) _copy_file(self._path_clean, From 18f9356c2c70fef72f8e2cd837c3dde83ecb2e05 Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Sun, 9 Sep 2018 20:01:13 +0000 Subject: [PATCH 048/145] storage/reflink: refuse to revert() dirty volume --- qubes/storage/reflink.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qubes/storage/reflink.py b/qubes/storage/reflink.py index 2a0321b0..bcf2e3dc 100644 --- a/qubes/storage/reflink.py +++ b/qubes/storage/reflink.py @@ -197,6 +197,9 @@ class ReflinkVolume(qubes.storage.Volume): _remove_file(self._path_revision(number, timestamp)) def revert(self, revision=None): + if self.is_dirty(): + raise qubes.storage.StoragePoolException( + 'Cannot revert: {} is not cleanly stopped'.format(self.vid)) if revision is None: number, timestamp = list(self.revisions.items())[-1] else: From ef2698adb4aed4fdfb47e6bc186ece1e76c13337 Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Sun, 9 Sep 2018 20:01:14 +0000 Subject: [PATCH 049/145] storage/reflink: make revisions() more readable, use iglob --- qubes/storage/reflink.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qubes/storage/reflink.py b/qubes/storage/reflink.py index bcf2e3dc..20fa2a97 100644 --- a/qubes/storage/reflink.py +++ b/qubes/storage/reflink.py @@ -297,10 +297,10 @@ class ReflinkVolume(qubes.storage.Volume): @property def revisions(self): prefix = self._path_clean + '.' - paths = glob.glob(glob.escape(prefix) + '*@*Z') - items = sorted((path[len(prefix):-1].split('@') for path in paths), - key=lambda item: int(item[0])) - return collections.OrderedDict(items) + paths = glob.iglob(glob.escape(prefix) + '*@*Z') + items = (path[len(prefix):-1].split('@') for path in paths) + return collections.OrderedDict(sorted(items, + key=lambda item: int(item[0]))) @property def usage(self): From 75a4a1340e5f444d8b57eddac04cb9da520afee9 Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Sun, 9 Sep 2018 20:01:15 +0000 Subject: [PATCH 050/145] storage/reflink: don't recompute static properties per call --- qubes/storage/reflink.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/qubes/storage/reflink.py b/qubes/storage/reflink.py index 20fa2a97..e3582c8f 100644 --- a/qubes/storage/reflink.py +++ b/qubes/storage/reflink.py @@ -116,6 +116,13 @@ class ReflinkPool(qubes.storage.Pool): self.dir_path) class ReflinkVolume(qubes.storage.Volume): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._path_vid = os.path.join(self.pool.dir_path, self.vid) + self._path_clean = self._path_vid + '.img' + self._path_dirty = self._path_vid + '-dirty.img' + self.path = self._path_dirty + def create(self): if self.save_on_stop and not self.snap_on_start: _create_sparse_file(self._path_clean, self.size) @@ -275,18 +282,6 @@ class ReflinkVolume(qubes.storage.Volume): timestamp = self.revisions[number] return self._path_clean + '.' + number + '@' + timestamp + 'Z' - @property - def _path_clean(self): - return os.path.join(self.pool.dir_path, self.vid + '.img') - - @property - def _path_dirty(self): - return os.path.join(self.pool.dir_path, self.vid + '-dirty.img') - - @property - def path(self): - return self._path_dirty - @property def _next_revision_number(self): numbers = self.revisions.keys() From d301aa2e501a33981cc5537b5cbc119081693d8d Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Sun, 9 Sep 2018 20:01:17 +0000 Subject: [PATCH 051/145] storage/reflink: delete stale tempfiles on start and remove When the AT_REPLACE flag for linkat() finally lands in the Linux kernel, _replace_file() can be modified to use unnamed (O_TMPFILE) tempfiles. Until then, make sure stale tempfiles from previous crashes can't hang around for too long. --- qubes/storage/reflink.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/qubes/storage/reflink.py b/qubes/storage/reflink.py index e3582c8f..04467cce 100644 --- a/qubes/storage/reflink.py +++ b/qubes/storage/reflink.py @@ -146,12 +146,17 @@ class ReflinkVolume(qubes.storage.Volume): oldest to newest; remove empty VM directory. ''' self.pool._volumes.pop(self, None) # pylint: disable=protected-access + self._cleanup() self._prune_revisions(keep=0) _remove_file(self._path_clean) _remove_file(self._path_dirty) _remove_empty_dir(os.path.dirname(self._path_dirty)) return self + def _cleanup(self): + for tmp in glob.iglob(glob.escape(self._path_vid) + '*.img*~*'): + _remove_file(tmp) + def is_outdated(self): if self.snap_on_start: with suppress(FileNotFoundError): @@ -164,6 +169,7 @@ class ReflinkVolume(qubes.storage.Volume): return self.save_on_stop and os.path.exists(self._path_dirty) def start(self): + self._cleanup() if self.is_dirty(): # implies self.save_on_stop return self if self.snap_on_start: From 60bf68a74866a94e50e988b497a9709dec427ee3 Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Sun, 9 Sep 2018 20:01:18 +0000 Subject: [PATCH 052/145] storage/reflink: add _path_import (don't reuse _path_dirty) Import volume data to a new _path_import (instead of _path_dirty) before committing to _path_clean. In case the computer crashes while an import operation is running, the partially written file should not be attached to Xen on the next volume startup. Use -import.img as the filename like 'file' does, to be compatible with qubes.tests.api_admin/TC_00_VMs/test_510_vm_volume_import. --- qubes/storage/reflink.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/qubes/storage/reflink.py b/qubes/storage/reflink.py index 04467cce..264a8b5d 100644 --- a/qubes/storage/reflink.py +++ b/qubes/storage/reflink.py @@ -121,6 +121,7 @@ class ReflinkVolume(qubes.storage.Volume): self._path_vid = os.path.join(self.pool.dir_path, self.vid) self._path_clean = self._path_vid + '.img' self._path_dirty = self._path_vid + '-dirty.img' + self._path_import = self._path_vid + '-import.img' self.path = self._path_dirty def create(self): @@ -156,6 +157,7 @@ class ReflinkVolume(qubes.storage.Volume): def _cleanup(self): for tmp in glob.iglob(glob.escape(self._path_vid) + '*.img*~*'): _remove_file(tmp) + _remove_file(self._path_import) def is_outdated(self): if self.snap_on_start: @@ -183,16 +185,16 @@ class ReflinkVolume(qubes.storage.Volume): def stop(self): if self.save_on_stop: - self._commit() + self._commit(self._path_dirty) else: _remove_file(self._path_dirty) _remove_file(self._path_clean) return self - def _commit(self): + def _commit(self, path_from): self._add_revision() self._prune_revisions() - _rename_file(self._path_dirty, self._path_clean) + _rename_file(path_from, self._path_clean) def _add_revision(self): if self.revisions_to_keep == 0: @@ -263,20 +265,20 @@ class ReflinkVolume(qubes.storage.Volume): def import_data(self): self._require_save_on_stop('import_data') - _create_sparse_file(self._path_dirty, self.size) - return self._path_dirty + _create_sparse_file(self._path_import, self.size) + return self._path_import def import_data_end(self, success): if success: - self._commit() + self._commit(self._path_import) else: - _remove_file(self._path_dirty) + _remove_file(self._path_import) return self def import_volume(self, src_volume): self._require_save_on_stop('import_volume') try: - _copy_file(src_volume.export(), self._path_dirty) + _copy_file(src_volume.export(), self._path_import) except: self.import_data_end(False) raise From 6e8d7d42016514698a7eed0644ea30a699771fc4 Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Sun, 9 Sep 2018 20:01:19 +0000 Subject: [PATCH 053/145] storage/reflink: no-op import_volume() if not save_on_stop Instead of raising a NotImplementedError, just return self like 'file' and lvm_thin. This is needed when Storage.clone() is modified in another commit* to no longer swallow exceptions. * "storage: factor out _wait_and_reraise(); fix clone/create" --- qubes/storage/reflink.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qubes/storage/reflink.py b/qubes/storage/reflink.py index 264a8b5d..8b169c92 100644 --- a/qubes/storage/reflink.py +++ b/qubes/storage/reflink.py @@ -276,7 +276,8 @@ class ReflinkVolume(qubes.storage.Volume): return self def import_volume(self, src_volume): - self._require_save_on_stop('import_volume') + if not self.save_on_stop: + return self try: _copy_file(src_volume.export(), self._path_import) except: From e7b7c253acacb9cc8b9190091cf5679e22e7d65f Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Sun, 9 Sep 2018 20:01:20 +0000 Subject: [PATCH 054/145] storage/reflink: inline _require_self_on_stop() --- qubes/storage/reflink.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/qubes/storage/reflink.py b/qubes/storage/reflink.py index 8b169c92..69353eb1 100644 --- a/qubes/storage/reflink.py +++ b/qubes/storage/reflink.py @@ -253,18 +253,16 @@ class ReflinkVolume(qubes.storage.Volume): return self - def _require_save_on_stop(self, method_name): + def export(self): if not self.save_on_stop: raise NotImplementedError( - 'Cannot {}: {} is not save_on_stop'.format( - method_name, self.vid)) - - def export(self): - self._require_save_on_stop('export') + 'Cannot export: {} is not save_on_stop'.format(self.vid)) return self._path_clean def import_data(self): - self._require_save_on_stop('import_data') + if not self.save_on_stop: + raise NotImplementedError( + 'Cannot import_data: {} is not save_on_stop'.format(self.vid)) _create_sparse_file(self._path_import, self.size) return self._path_import From 385ba917727683942635748c84c14db332db31ae Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Sun, 9 Sep 2018 20:01:22 +0000 Subject: [PATCH 055/145] storage/reflink: resize(): don't look for loopdevs if clean --- qubes/storage/reflink.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qubes/storage/reflink.py b/qubes/storage/reflink.py index 69353eb1..88ef8394 100644 --- a/qubes/storage/reflink.py +++ b/qubes/storage/reflink.py @@ -240,10 +240,11 @@ class ReflinkVolume(qubes.storage.Volume): try: # assume volume is not (cleanly) stopped ... _resize_file(self._path_dirty, size) + self.size = size except FileNotFoundError: # ... but it actually is. _resize_file(self._path_clean, size) - - self.size = size + self.size = size + return self # resize any corresponding loop devices out = _cmd('losetup', '--associated', self._path_dirty) From ce794f33d83acd98e3ab28a40a35e29f685298ce Mon Sep 17 00:00:00 2001 From: xaki23 Date: Mon, 10 Sep 2018 17:24:35 +0200 Subject: [PATCH 056/145] add missing /sbin/ to hwclock call (so it will work as cronjob) --- qvm-tools/qvm-sync-clock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qvm-tools/qvm-sync-clock b/qvm-tools/qvm-sync-clock index 18e0bcf8..f2c6ecb3 100755 --- a/qvm-tools/qvm-sync-clock +++ b/qvm-tools/qvm-sync-clock @@ -47,7 +47,7 @@ def main(): date_out = untrusted_date_out subprocess.check_call(['date', '-u', '-Iseconds', '-s', date_out], stdout=subprocess.DEVNULL) - subprocess.check_call(['hwclock', '--systohc'], + subprocess.check_call(['/sbin/hwclock', '--systohc'], stdout=subprocess.DEVNULL) if __name__ == '__main__': From fb06a8089abf82f3a8075b7420d64caae745b839 Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Tue, 11 Sep 2018 23:50:10 +0000 Subject: [PATCH 057/145] storage/reflink: _update_loopdev_sizes() without losetup Factor out a function, and use the LOOP_SET_CAPACITY ioctl instead of going through losetup. --- qubes/storage/reflink.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/qubes/storage/reflink.py b/qubes/storage/reflink.py index 88ef8394..8dcfab2a 100644 --- a/qubes/storage/reflink.py +++ b/qubes/storage/reflink.py @@ -28,7 +28,6 @@ import fcntl import glob import logging import os -import re import subprocess import tempfile from contextlib import contextmanager, suppress @@ -36,7 +35,8 @@ from contextlib import contextmanager, suppress import qubes.storage BLKSIZE = 512 -FICLONE = 1074041865 # see ioctl_ficlone manpage +FICLONE = 1074041865 # defined in +LOOP_SET_CAPACITY = 0x4C07 # defined in LOGGER = logging.getLogger('qubes.storage.reflink') @@ -246,12 +246,7 @@ class ReflinkVolume(qubes.storage.Volume): self.size = size return self - # resize any corresponding loop devices - out = _cmd('losetup', '--associated', self._path_dirty) - for match in re.finditer(br'^(/dev/loop[0-9]+): ', out, re.MULTILINE): - loop_dev = match.group(1).decode('ascii') - _cmd('losetup', '--set-capacity', loop_dev) - + _update_loopdev_sizes(self._path_dirty) return self def export(self): @@ -395,6 +390,19 @@ def _create_sparse_file(path, size): tmp.truncate(size) LOGGER.info('Created sparse file: %s', tmp.name) +def _update_loopdev_sizes(img): + ''' Resolve img; update the size of loop devices backed by it. ''' + needle = os.fsencode(os.path.realpath(img)) + b'\n' + for sys_path in glob.iglob('/sys/block/loop[0-9]*/loop/backing_file'): + try: + with open(sys_path, 'rb') as sys_io: + if sys_io.read() != needle: + continue + except FileNotFoundError: + continue + with open('/dev/' + sys_path.split('/')[3]) as dev_io: + fcntl.ioctl(dev_io.fileno(), LOOP_SET_CAPACITY) + def _copy_file(src, dst): ''' Copy src to dst as a reflink if possible, sparse if not. ''' if not os.path.exists(src): From 69af0a48ecb4df096ebce910c563c66e9d2b2ae8 Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Tue, 11 Sep 2018 23:50:12 +0000 Subject: [PATCH 058/145] storage/reflink: inline and simplify _cmd() --- qubes/storage/reflink.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/qubes/storage/reflink.py b/qubes/storage/reflink.py index 8dcfab2a..bb7c4e4d 100644 --- a/qubes/storage/reflink.py +++ b/qubes/storage/reflink.py @@ -409,19 +409,10 @@ def _copy_file(src, dst): raise FileNotFoundError(src) with _replace_file(dst) as tmp: LOGGER.info('Copying file: %s -> %s', src, tmp.name) - _cmd('cp', '--sparse=always', '--reflink=auto', src, tmp.name) - -def _cmd(*args): - ''' Run command until finished; return stdout (as bytes) if it - exited 0. Otherwise, raise a detailed StoragePoolException. - ''' - try: - return subprocess.run(args, check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE).stdout - except subprocess.CalledProcessError as ex: - msg = '{} err={!r} out={!r}'.format(ex, ex.stderr, ex.stdout) - raise qubes.storage.StoragePoolException(msg) from ex + cmd = 'cp', '--sparse=always', '--reflink=auto', src, tmp.name + p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if p.returncode != 0: + raise qubes.storage.StoragePoolException(str(p)) def is_reflink_supported(dst_dir, src_dir=None): ''' Return whether destination directory supports reflink copies From edda3a1734505b7cf652c7624524319b778d58a6 Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Tue, 11 Sep 2018 23:50:13 +0000 Subject: [PATCH 059/145] storage/reflink: factor out _ficlone() --- qubes/storage/reflink.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/qubes/storage/reflink.py b/qubes/storage/reflink.py index bb7c4e4d..e8841964 100644 --- a/qubes/storage/reflink.py +++ b/qubes/storage/reflink.py @@ -403,6 +403,13 @@ def _update_loopdev_sizes(img): with open('/dev/' + sys_path.split('/')[3]) as dev_io: fcntl.ioctl(dev_io.fileno(), LOOP_SET_CAPACITY) +def _attempt_ficlone(src, dst): + try: + fcntl.ioctl(dst.fileno(), FICLONE, src.fileno()) + return True + except OSError: + return False + def _copy_file(src, dst): ''' Copy src to dst as a reflink if possible, sparse if not. ''' if not os.path.exists(src): @@ -424,9 +431,4 @@ def is_reflink_supported(dst_dir, src_dir=None): dst = tempfile.TemporaryFile(dir=dst_dir) src = tempfile.TemporaryFile(dir=src_dir) src.write(b'foo') # don't let any filesystem get clever with empty files - - try: - fcntl.ioctl(dst.fileno(), FICLONE, src.fileno()) - return True - except OSError: - return False + return _attempt_ficlone(src, dst) From 3d986be02a52d699c2b1e0484757b821177f562b Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Tue, 11 Sep 2018 23:50:14 +0000 Subject: [PATCH 060/145] storage/reflink: native FICLONE in _copy_file() happy path Avoid a subprocess launch, and distinguish reflink vs. fallback copy in the log. --- qubes/storage/reflink.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/qubes/storage/reflink.py b/qubes/storage/reflink.py index e8841964..c9862632 100644 --- a/qubes/storage/reflink.py +++ b/qubes/storage/reflink.py @@ -412,14 +412,17 @@ def _attempt_ficlone(src, dst): def _copy_file(src, dst): ''' Copy src to dst as a reflink if possible, sparse if not. ''' - if not os.path.exists(src): - raise FileNotFoundError(src) - with _replace_file(dst) as tmp: - LOGGER.info('Copying file: %s -> %s', src, tmp.name) - cmd = 'cp', '--sparse=always', '--reflink=auto', src, tmp.name + with _replace_file(dst) as tmp_io: + with open(src, 'rb') as src_io: + if _attempt_ficlone(src_io, tmp_io): + LOGGER.info('Reflinked file: %s -> %s', src, tmp_io.name) + return True + LOGGER.info('Copying file: %s -> %s', src, tmp_io.name) + cmd = 'cp', '--sparse=always', src, tmp_io.name p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if p.returncode != 0: raise qubes.storage.StoragePoolException(str(p)) + return False def is_reflink_supported(dst_dir, src_dir=None): ''' Return whether destination directory supports reflink copies From 1889c9b75fef2da7c120b108b77cd5228c4a4db2 Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Tue, 11 Sep 2018 23:50:15 +0000 Subject: [PATCH 061/145] storage/reflink: run synchronous volume methods in executor Convert create(), verify(), remove(), start(), stop(), revert(), resize(), and import_volume() into coroutine methods, via a decorator that runs them in the event loop's thread-based default executor. This reduces UI hangs by unblocking the event loop, and can e.g. speed up VM starts by starting multiple volumes in parallel. --- qubes/storage/reflink.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/qubes/storage/reflink.py b/qubes/storage/reflink.py index c9862632..d7f93a72 100644 --- a/qubes/storage/reflink.py +++ b/qubes/storage/reflink.py @@ -22,9 +22,11 @@ but not required. ''' +import asyncio import collections import errno import fcntl +import functools import glob import logging import os @@ -115,20 +117,37 @@ class ReflinkPool(qubes.storage.Pool): [pool for pool in app.pools.values() if pool is not self], self.dir_path) + +def _unblock(method): + ''' Decorator transforming a synchronous volume method into a + coroutine that runs the original method in the event loop's + thread-based default executor, under a per-volume lock. + ''' + @asyncio.coroutine + @functools.wraps(method) + def wrapper(self, *args, **kwargs): + with (yield from self._lock): # pylint: disable=protected-access + return (yield from asyncio.get_event_loop().run_in_executor( + None, functools.partial(method, self, *args, **kwargs))) + return wrapper + class ReflinkVolume(qubes.storage.Volume): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self._lock = asyncio.Lock() self._path_vid = os.path.join(self.pool.dir_path, self.vid) self._path_clean = self._path_vid + '.img' self._path_dirty = self._path_vid + '-dirty.img' self._path_import = self._path_vid + '-import.img' self.path = self._path_dirty + @_unblock def create(self): if self.save_on_stop and not self.snap_on_start: _create_sparse_file(self._path_clean, self.size) return self + @_unblock def verify(self): if self.snap_on_start: img = self.source._path_clean # pylint: disable=protected-access @@ -142,6 +161,7 @@ class ReflinkVolume(qubes.storage.Volume): raise qubes.storage.StoragePoolException( 'Missing image file {!r} for volume {}'.format(img, self.vid)) + @_unblock def remove(self): ''' Drop volume object from pool; remove volume images from oldest to newest; remove empty VM directory. @@ -170,6 +190,7 @@ class ReflinkVolume(qubes.storage.Volume): def is_dirty(self): return self.save_on_stop and os.path.exists(self._path_dirty) + @_unblock def start(self): self._cleanup() if self.is_dirty(): # implies self.save_on_stop @@ -183,6 +204,7 @@ class ReflinkVolume(qubes.storage.Volume): _create_sparse_file(self._path_dirty, self.size) return self + @_unblock def stop(self): if self.save_on_stop: self._commit(self._path_dirty) @@ -211,6 +233,7 @@ class ReflinkVolume(qubes.storage.Volume): for number, timestamp in list(self.revisions.items())[:-keep or None]: _remove_file(self._path_revision(number, timestamp)) + @_unblock def revert(self, revision=None): if self.is_dirty(): raise qubes.storage.StoragePoolException( @@ -224,6 +247,7 @@ class ReflinkVolume(qubes.storage.Volume): _rename_file(path_revision, self._path_clean) return self + @_unblock def resize(self, size): ''' Expand a read-write volume image; notify any corresponding loop devices of the size change. @@ -269,6 +293,7 @@ class ReflinkVolume(qubes.storage.Volume): _remove_file(self._path_import) return self + @_unblock def import_volume(self, src_volume): if not self.save_on_stop: return self From c75fe098141d3ada2bea5d90e69b8502600e6965 Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Tue, 11 Sep 2018 23:50:16 +0000 Subject: [PATCH 062/145] storage/reflink: is_reflink_supported() -> is_supported() --- qubes/storage/reflink.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qubes/storage/reflink.py b/qubes/storage/reflink.py index d7f93a72..b70094cb 100644 --- a/qubes/storage/reflink.py +++ b/qubes/storage/reflink.py @@ -55,7 +55,7 @@ class ReflinkPool(qubes.storage.Pool): def setup(self): created = _make_dir(self.dir_path) - if self.setup_check and not is_reflink_supported(self.dir_path): + if self.setup_check and not is_supported(self.dir_path): if created: _remove_empty_dir(self.dir_path) raise qubes.storage.StoragePoolException( @@ -449,7 +449,7 @@ def _copy_file(src, dst): raise qubes.storage.StoragePoolException(str(p)) return False -def is_reflink_supported(dst_dir, src_dir=None): +def is_supported(dst_dir, src_dir=None): ''' Return whether destination directory supports reflink copies from source directory. (A temporary file is created in each directory, using O_TMPFILE if possible.) From 266d90c2f954d44d3a84473be8b3656a0b226d2f Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Tue, 11 Sep 2018 23:50:18 +0000 Subject: [PATCH 063/145] storage: fix docstrings --- qubes/storage/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/qubes/storage/__init__.py b/qubes/storage/__init__.py index f506c12b..fe5e98bb 100644 --- a/qubes/storage/__init__.py +++ b/qubes/storage/__init__.py @@ -252,6 +252,8 @@ class Volume: def revert(self, revision=None): ''' Revert volume to previous revision + This can be implemented as a coroutine. + :param revision: revision to revert volume to, see :py:attr:`revisions` ''' # pylint: disable=unused-argument @@ -616,7 +618,7 @@ class Storage: @asyncio.coroutine def start(self): - ''' Execute the start method on each pool ''' + ''' Execute the start method on each volume ''' futures = [] for volume in self.vm.volumes.values(): ret = volume.start() @@ -631,7 +633,7 @@ class Storage: @asyncio.coroutine def stop(self): - ''' Execute the start method on each pool ''' + ''' Execute the stop method on each volume ''' futures = [] for volume in self.vm.volumes.values(): ret = volume.stop() From 44ca78523f915d217e85343c657fd649a4635d84 Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Tue, 11 Sep 2018 23:50:19 +0000 Subject: [PATCH 064/145] storage: insert missing NotImplementedError in Volume.stop() --- qubes/storage/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qubes/storage/__init__.py b/qubes/storage/__init__.py index fe5e98bb..b8e5f5b2 100644 --- a/qubes/storage/__init__.py +++ b/qubes/storage/__init__.py @@ -274,6 +274,7 @@ class Volume: This include committing data if :py:attr:`save_on_stop` is set. This can be implemented as a coroutine.''' + raise self._not_implemented("stop") def verify(self): ''' Verifies the volume. From f0ee73e63f7f836b8108373884607273e6accb7e Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Tue, 11 Sep 2018 23:50:20 +0000 Subject: [PATCH 065/145] storage: remove broken default parameter from isodate() With that syntax, the default timestamp would have been from the time of the function's definition (not invocation). But all callers are passing an explicit timestamp anyway. --- qubes/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes/storage/__init__.py b/qubes/storage/__init__.py index b8e5f5b2..59d0d210 100644 --- a/qubes/storage/__init__.py +++ b/qubes/storage/__init__.py @@ -874,7 +874,7 @@ def driver_parameters(name): return [p for p in params if p not in ignored_params] -def isodate(seconds=time.time()): +def isodate(seconds): ''' Helper method which returns an iso date ''' return datetime.utcfromtimestamp(seconds).isoformat("T") From d33bd3f2b629fc030afcc0e9b5fa97f7a57f6b0a Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Tue, 11 Sep 2018 23:50:21 +0000 Subject: [PATCH 066/145] storage: fix search_pool_containing_dir() Canonicalize both directories, resolving symlink components. Compare with commonpath() instead of startswith(), because /foo doesn't contain /foobar. --- qubes/storage/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/qubes/storage/__init__.py b/qubes/storage/__init__.py index 59d0d210..4631616b 100644 --- a/qubes/storage/__init__.py +++ b/qubes/storage/__init__.py @@ -884,17 +884,21 @@ def search_pool_containing_dir(pools, dir_path): This is useful for implementing Pool.included_in method ''' + real_dir_path = os.path.realpath(dir_path) + # prefer filesystem pools for pool in pools: if hasattr(pool, 'dir_path'): - if dir_path.startswith(pool.dir_path): + pool_real_dir_path = os.path.realpath(pool.dir_path) + if os.path.commonpath([pool_real_dir_path, real_dir_path]) == \ + pool_real_dir_path: return pool # then look for lvm for pool in pools: if hasattr(pool, 'thin_pool') and hasattr(pool, 'volume_group'): if (pool.volume_group, pool.thin_pool) == \ - DirectoryThinPool.thin_pool(dir_path): + DirectoryThinPool.thin_pool(real_dir_path): return pool return None From d181bf1aa4912013addc233cc93ba1de585b25c5 Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Tue, 11 Sep 2018 23:50:22 +0000 Subject: [PATCH 067/145] storage: factor out _wait_and_reraise(); fix clone/create _wait_and_reraise() is similar to asyncio.gather(), but it preserves the current behavior of waiting for all futures and only _then_ reraising the first exception (if there is any) in line. Also switch Storage.create() and Storage.clone() to _wait_and_reraise(). Previously, they called asyncio.wait() and implicitly swallowed all exceptions. --- qubes/storage/__init__.py | 43 ++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/qubes/storage/__init__.py b/qubes/storage/__init__.py index 4631616b..c81832a9 100644 --- a/qubes/storage/__init__.py +++ b/qubes/storage/__init__.py @@ -509,8 +509,7 @@ class Storage: ret = volume.create() if asyncio.iscoroutine(ret): coros.append(ret) - if coros: - yield from asyncio.wait(coros) + yield from _wait_and_reraise(coros) os.umask(old_umask) @@ -552,7 +551,7 @@ class Storage: self.vm.volumes = {} with VmCreationManager(self.vm): - yield from asyncio.wait([self.clone_volume(src_vm, vol_name) + yield from _wait_and_reraise([self.clone_volume(src_vm, vol_name) for vol_name in self.vm.volume_config.keys()]) @property @@ -584,11 +583,7 @@ class Storage: ret = volume.verify() if asyncio.iscoroutine(ret): futures.append(ret) - if futures: - done, _ = yield from asyncio.wait(futures) - for task in done: - # re-raise any exception from async task - task.result() + yield from _wait_and_reraise(futures) self.vm.fire_event('domain-verify-files') return True @@ -608,14 +603,10 @@ class Storage: except (IOError, OSError) as e: self.vm.log.exception("Failed to remove volume %s", name, e) - if futures: - try: - done, _ = yield from asyncio.wait(futures) - for task in done: - # re-raise any exception from async task - task.result() - except (IOError, OSError) as e: - self.vm.log.exception("Failed to remove some volume", e) + try: + yield from _wait_and_reraise(futures) + except (IOError, OSError) as e: + self.vm.log.exception("Failed to remove some volume", e) @asyncio.coroutine def start(self): @@ -626,11 +617,7 @@ class Storage: if asyncio.iscoroutine(ret): futures.append(ret) - if futures: - done, _ = yield from asyncio.wait(futures) - for task in done: - # re-raise any exception from async task - task.result() + yield from _wait_and_reraise(futures) @asyncio.coroutine def stop(self): @@ -641,11 +628,7 @@ class Storage: if asyncio.iscoroutine(ret): futures.append(ret) - if futures: - done, _ = yield from asyncio.wait(futures) - for task in done: - # re-raise any exception from async task - task.result() + yield from _wait_and_reraise(futures) def unused_frontend(self): ''' Find an unused device name ''' @@ -845,6 +828,14 @@ class Pool: return NotImplementedError(msg) +@asyncio.coroutine +def _wait_and_reraise(futures): + if futures: + done, _ = yield from asyncio.wait(futures) + for task in done: # (re-)raise first exception in line + task.result() + + def _sanitize_config(config): ''' Helper function to convert types to appropriate strings ''' # FIXME: find another solution for serializing basic types From 8eb9c64f2082e690dae997481a020a252b5234d1 Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Tue, 11 Sep 2018 23:50:24 +0000 Subject: [PATCH 068/145] tools/qubes-create: fix docstring --- qubes/tools/qubes_create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes/tools/qubes_create.py b/qubes/tools/qubes_create.py index 28059a09..34babd6d 100644 --- a/qubes/tools/qubes_create.py +++ b/qubes/tools/qubes_create.py @@ -18,7 +18,7 @@ # License along with this library; if not, see . # -'''qvm-create - Create new Qubes OS store''' +'''qubes-create - Create new Qubes OS store''' import sys import qubes From 53ef5ed431f47cc5ead3934529a78254c77c6e01 Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Tue, 11 Sep 2018 23:50:25 +0000 Subject: [PATCH 069/145] app: uncouple pool setup from loading initial configuration And ensure that setup is called on every type of these pools, not just lvm_thin. --- qubes/app.py | 23 +++++++++++++++++------ qubes/tests/api_admin.py | 1 + qubes/tools/qubes_create.py | 2 +- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/qubes/app.py b/qubes/app.py index 5deff36f..3609b6b1 100644 --- a/qubes/app.py +++ b/qubes/app.py @@ -21,6 +21,7 @@ # import collections +import copy import errno import functools import grp @@ -1064,15 +1065,20 @@ class Qubes(qubes.PropertyHolder): } assert max(self.labels.keys()) == qubes.config.max_default_label + pool_configs = copy.deepcopy(qubes.config.defaults['pool_configs']) + root_volume_group, root_thin_pool = \ qubes.storage.DirectoryThinPool.thin_pool('/') - if root_thin_pool: - self.add_pool( - volume_group=root_volume_group, thin_pool=root_thin_pool, - name='lvm', driver='lvm_thin') - # pool based on /var/lib/qubes will be created here: - for name, config in qubes.config.defaults['pool_configs'].items(): + lvm_config = { + 'name': 'lvm', + 'driver': 'lvm_thin', + 'volume_group': root_volume_group, + 'thin_pool': root_thin_pool + } + pool_configs[lvm_config['name']] = lvm_config + + for name, config in pool_configs.items(): self.pools[name] = self._get_pool(**config) self.default_pool_kernel = 'linux-kernel' @@ -1170,6 +1176,11 @@ class Qubes(qubes.PropertyHolder): raise KeyError(label) + def setup_pools(self): + """ Run implementation specific setup for each storage pool. """ + for pool in self.pools.values(): + pool.setup() + def add_pool(self, name, **kwargs): """ Add a storage pool to config.""" diff --git a/qubes/tests/api_admin.py b/qubes/tests/api_admin.py index 4a425d4b..9efdd75e 100644 --- a/qubes/tests/api_admin.py +++ b/qubes/tests/api_admin.py @@ -60,6 +60,7 @@ class AdminAPITestCase(qubes.tests.QubesTestCase): app = qubes.Qubes('/tmp/qubes-test.xml', load=False) app.vmm = unittest.mock.Mock(spec=qubes.app.VMMConnection) app.load_initial_values() + app.setup_pools() app.default_kernel = '1.0' app.default_netvm = None self.template = app.add_new_vm('TemplateVM', label='black', diff --git a/qubes/tools/qubes_create.py b/qubes/tools/qubes_create.py index 34babd6d..56e66c7f 100644 --- a/qubes/tools/qubes_create.py +++ b/qubes/tools/qubes_create.py @@ -38,7 +38,7 @@ def main(args=None): args = parser.parse_args(args) qubes.Qubes.create_empty_store(args.app, - offline_mode=args.offline_mode) + offline_mode=args.offline_mode).setup_pools() return 0 From 8d1913a8cc47c11ff53abe39c6a782ef48ede1c7 Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Tue, 11 Sep 2018 23:50:26 +0000 Subject: [PATCH 070/145] app: create /var/lib/qubes as file-reflink if supported Use the file-reflink storage driver if /var/lib/qubes is on a filesystem that supports reflinks, e.g. when the btrfs layout was selected in Anaconda. If it doesn't support reflinks (or if detection fails, e.g. in an unprivileged test environment), use 'file' as before. --- qubes/app.py | 12 +++++++++++- qubes/config.py | 3 +-- qubes/tests/storage.py | 6 ++++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/qubes/app.py b/qubes/app.py index 3609b6b1..7a31e822 100644 --- a/qubes/app.py +++ b/qubes/app.py @@ -61,6 +61,7 @@ import qubes import qubes.ext import qubes.utils import qubes.storage +import qubes.storage.reflink import qubes.vm import qubes.vm.adminvm import qubes.vm.qubesvm @@ -553,7 +554,7 @@ def _default_pool(app): 1. If there is one named 'default', use it. 2. Check if root fs is on LVM thin - use that - 3. Look for file-based pool pointing /var/lib/qubes + 3. Look for file(-reflink)-based pool pointing to /var/lib/qubes 4. Fail ''' if 'default' in app.pools: @@ -1079,6 +1080,15 @@ class Qubes(qubes.PropertyHolder): pool_configs[lvm_config['name']] = lvm_config for name, config in pool_configs.items(): + if 'driver' not in config and 'dir_path' in config: + config['driver'] = 'file' + try: + os.makedirs(config['dir_path'], exist_ok=True) + if qubes.storage.reflink.is_supported(config['dir_path']): + config['driver'] = 'file-reflink' + config['setup_check'] = 'no' # don't check twice + except PermissionError: # looks like a testing environment + pass # stay with 'file' self.pools[name] = self._get_pool(**config) self.default_pool_kernel = 'linux-kernel' diff --git a/qubes/config.py b/qubes/config.py index dfedfe23..da02dbb6 100644 --- a/qubes/config.py +++ b/qubes/config.py @@ -76,9 +76,8 @@ defaults = { 'root_img_size': 10*1024*1024*1024, 'pool_configs': { - # create file pool even when the default one is LVM + # create file(-reflink) pool even when the default one is LVM 'varlibqubes': {'dir_path': qubes_base_dir, - 'driver': 'file', 'name': 'varlibqubes'}, 'linux-kernel': { 'dir_path': os.path.join(qubes_base_dir, diff --git a/qubes/tests/storage.py b/qubes/tests/storage.py index 1af72a08..97d5932f 100644 --- a/qubes/tests/storage.py +++ b/qubes/tests/storage.py @@ -22,6 +22,7 @@ import qubes.storage from qubes.exc import QubesException from qubes.storage import pool_drivers from qubes.storage.file import FilePool +from qubes.storage.reflink import ReflinkPool from qubes.tests import SystemTestCase # :pylint: disable=invalid-name @@ -107,10 +108,11 @@ class TC_00_Pool(SystemTestCase): pool_drivers()) def test_002_get_pool_klass(self): - """ Expect the default pool to be `FilePool` """ + """ Expect the default pool to be `FilePool` or `ReflinkPool` """ # :pylint: disable=protected-access result = self.app.get_pool('varlibqubes') - self.assertIsInstance(result, FilePool) + self.assertTrue(isinstance(result, FilePool) + or isinstance(result, ReflinkPool)) def test_003_pool_exists_default(self): """ Expect the default pool to exists """ From bc30c6f3e8b18788930994fb1ac2888b65dccdb2 Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Tue, 11 Sep 2018 23:50:27 +0000 Subject: [PATCH 071/145] tests: delete orphaned Makefile --- Makefile | 2 -- tests/Makefile | 49 ------------------------------------------------- 2 files changed, 51 deletions(-) delete mode 100644 tests/Makefile diff --git a/Makefile b/Makefile index eb951965..c8787c2e 100644 --- a/Makefile +++ b/Makefile @@ -141,7 +141,6 @@ rpms-dom0: all: $(PYTHON) setup.py build $(MAKE) -C qubes-rpc all -# make all -C tests # Currently supported only on xen install: @@ -158,7 +157,6 @@ endif ln -s qvm-device.1.gz $(DESTDIR)/usr/share/man/man1/qvm-block.1.gz ln -s qvm-device.1.gz $(DESTDIR)/usr/share/man/man1/qvm-pci.1.gz ln -s qvm-device.1.gz $(DESTDIR)/usr/share/man/man1/qvm-usb.1.gz -# $(MAKE) install -C tests $(MAKE) install -C relaxng mkdir -p $(DESTDIR)/etc/qubes ifeq ($(BACKEND_VMM),xen) diff --git a/tests/Makefile b/tests/Makefile deleted file mode 100644 index 8b68f0b2..00000000 --- a/tests/Makefile +++ /dev/null @@ -1,49 +0,0 @@ -PYTHON_TESTSPATH = $(PYTHON_SITEPATH)/qubes/tests - -all: - python -m compileall . - python -O -m compileall . - -install: -ifndef PYTHON_SITEPATH - $(error PYTHON_SITEPATH not defined) -endif - mkdir -p $(DESTDIR)$(PYTHON_TESTSPATH) - cp __init__.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp __init__.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp backup.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp backup.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp backupcompatibility.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp backupcompatibility.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp basic.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp basic.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp block.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp block.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp dispvm.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp dispvm.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp dom0_update.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp dom0_update.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp extra.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp extra.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp hardware.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp hardware.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp hvm.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp hvm.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp mime.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp mime.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp network.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp network.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp pvgrub.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp pvgrub.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp regressions.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp regressions.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp run.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp run.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp storage.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp storage.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp storage_file.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp storage_file.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp storage_xen.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp storage_xen.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) - cp vm_qrexec_gui.py $(DESTDIR)$(PYTHON_TESTSPATH) - cp vm_qrexec_gui.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) From 49e7ce025f1fff4a2054e9c26a42147adb4d0aa1 Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Tue, 11 Sep 2018 23:50:28 +0000 Subject: [PATCH 072/145] tests/integ/backupcompatibility: Storage.verify() is a coro --- qubes/tests/integ/backupcompatibility.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qubes/tests/integ/backupcompatibility.py b/qubes/tests/integ/backupcompatibility.py index c52e0e62..43a2a4e1 100644 --- a/qubes/tests/integ/backupcompatibility.py +++ b/qubes/tests/integ/backupcompatibility.py @@ -19,6 +19,7 @@ from multiprocessing import Queue +import asyncio import os import shutil import subprocess @@ -382,7 +383,7 @@ class TC_00_BackupCompatibility( def assertRestored(self, name, **kwargs): with self.assertNotRaises((KeyError, qubes.exc.QubesException)): vm = self.app.domains[name] - vm.storage.verify() + asyncio.get_event_loop().run_until_complete(vm.storage.verify()) for prop, value in kwargs.items(): if prop == 'klass': self.assertIsInstance(vm, value) From 8c117549ad6591d340e2408a07a3e996949a9523 Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Tue, 11 Sep 2018 23:50:30 +0000 Subject: [PATCH 073/145] tests/integ/basic: use export() in get_rootimg_checksum() volume.path and volume.export() refer to the same thing in lvm_thin and 'file', but not in file-reflink (where volume.path is the -dirty.img, which doesn't exist if the volume is not started). --- qubes/tests/integ/basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes/tests/integ/basic.py b/qubes/tests/integ/basic.py index 07cea6cd..2b592182 100644 --- a/qubes/tests/integ/basic.py +++ b/qubes/tests/integ/basic.py @@ -572,7 +572,7 @@ class TC_03_QvmRevertTemplateChanges(qubes.tests.SystemTestCase): def get_rootimg_checksum(self): return subprocess.check_output( - ['sha1sum', self.test_template.volumes['root'].path]).\ + ['sha1sum', self.test_template.volumes['root'].export()]).\ decode().split(' ')[0] def _do_test(self): From b82e739346bab6fbcfb59a9f28e32be809ed1dc1 Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Tue, 11 Sep 2018 23:50:31 +0000 Subject: [PATCH 074/145] tests/integ/storage: add file-reflink integration tests --- qubes/tests/integ/storage.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/qubes/tests/integ/storage.py b/qubes/tests/integ/storage.py index 4d8f1569..253d8e72 100644 --- a/qubes/tests/integ/storage.py +++ b/qubes/tests/integ/storage.py @@ -26,6 +26,7 @@ import subprocess import qubes.storage.lvm import qubes.tests import qubes.tests.storage_lvm +import qubes.tests.storage_reflink import qubes.vm.appvm @@ -318,6 +319,28 @@ class StorageFile(StorageTestMixin, qubes.tests.SystemTestCase): super(StorageFile, self).tearDown() +class StorageReflinkMixin(StorageTestMixin): + def tearDown(self): + self.app.remove_pool(self.pool.name) + super().tearDown() + + def init_pool(self, fs_type, **kwargs): + name = 'test-reflink-integration-on-' + fs_type + dir_path = os.path.join('/var/tmp', name) + qubes.tests.storage_reflink.mkdir_fs(dir_path, fs_type, + cleanup_via=self.addCleanup) + self.pool = self.app.add_pool(name=name, dir_path=dir_path, + driver='file-reflink', **kwargs) + +class StorageReflinkOnBtrfs(StorageReflinkMixin, qubes.tests.SystemTestCase): + def init_pool(self): + super().init_pool('btrfs') + +class StorageReflinkOnExt4(StorageReflinkMixin, qubes.tests.SystemTestCase): + def init_pool(self): + super().init_pool('ext4', setup_check='no') + + @qubes.tests.storage_lvm.skipUnlessLvmPoolExists class StorageLVM(StorageTestMixin, qubes.tests.SystemTestCase): def init_pool(self): From 797bbc43a0676e3ee57e7db3d313f68d10f4085a Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Tue, 11 Sep 2018 23:50:32 +0000 Subject: [PATCH 075/145] tests/storage_reflink: test some file-reflink helpers Tested: - _copy_file() - _create_sparse_file() - _resize_file() - _update_loopdev_sizes() Smoke tested by calls from the functions above: - _replace_file() - _rename_file() - _make_dir() - _fsync_dir() --- qubes/tests/__init__.py | 1 + qubes/tests/storage_reflink.py | 154 +++++++++++++++++++++++++++++++++ rpm_spec/core-dom0.spec.in | 1 + 3 files changed, 156 insertions(+) create mode 100644 qubes/tests/storage_reflink.py diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index 140e7f43..b13f2cc0 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -1223,6 +1223,7 @@ def load_tests(loader, tests, pattern): # pylint: disable=unused-argument 'qubes.tests.vm.init', 'qubes.tests.storage', 'qubes.tests.storage_file', + 'qubes.tests.storage_reflink', 'qubes.tests.storage_lvm', 'qubes.tests.storage_kernels', 'qubes.tests.ext', diff --git a/qubes/tests/storage_reflink.py b/qubes/tests/storage_reflink.py new file mode 100644 index 00000000..fff26144 --- /dev/null +++ b/qubes/tests/storage_reflink.py @@ -0,0 +1,154 @@ +# +# The Qubes OS Project, https://www.qubes-os.org +# +# Copyright (C) 2018 Rusty Bird +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, see . +# + +''' Tests for the file-reflink storage driver ''' + +# pylint: disable=protected-access +# pylint: disable=invalid-name + +import os +import shutil +import subprocess +import sys + +import qubes.tests +from qubes.storage import reflink + + +class ReflinkMixin: + def setUp(self, fs_type='btrfs'): # pylint: disable=arguments-differ + super().setUp() + self.test_dir = '/var/tmp/test-reflink-units-on-' + fs_type + mkdir_fs(self.test_dir, fs_type, cleanup_via=self.addCleanup) + + def test_000_copy_file(self): + source = os.path.join(self.test_dir, 'source-file') + dest = os.path.join(self.test_dir, 'new-directory', 'dest-file') + content = os.urandom(1024**2) + + with open(source, 'wb') as source_io: + source_io.write(content) + + ficlone_succeeded = reflink._copy_file(source, dest) + self.assertEqual(ficlone_succeeded, self.ficlone_supported) + + self.assertNotEqual(os.stat(source).st_ino, os.stat(dest).st_ino) + with open(source, 'rb') as source_io: + self.assertEqual(source_io.read(), content) + with open(dest, 'rb') as dest_io: + self.assertEqual(dest_io.read(), content) + + def test_001_create_and_resize_files_and_update_loopdevs(self): + img_real = os.path.join(self.test_dir, 'img-real') + img_sym = os.path.join(self.test_dir, 'img-sym') + size_initial = 111 * 1024**2 + size_resized = 222 * 1024**2 + + os.symlink(img_real, img_sym) + reflink._create_sparse_file(img_real, size_initial) + self.assertEqual(reflink._get_file_disk_usage(img_real), 0) + self.assertEqual(os.stat(img_real).st_size, size_initial) + + dev_from_real = setup_loopdev(img_real, cleanup_via=self.addCleanup) + dev_from_sym = setup_loopdev(img_sym, cleanup_via=self.addCleanup) + + reflink._resize_file(img_real, size_resized) + self.assertEqual(reflink._get_file_disk_usage(img_real), 0) + self.assertEqual(os.stat(img_real).st_size, size_resized) + + reflink_update_loopdev_sizes(os.path.join(self.test_dir, 'unrelated')) + + for dev in (dev_from_real, dev_from_sym): + self.assertEqual(get_blockdev_size(dev), size_initial) + + reflink_update_loopdev_sizes(img_sym) + + for dev in (dev_from_real, dev_from_sym): + self.assertEqual(get_blockdev_size(dev), size_resized) + +class TC_00_ReflinkOnBtrfs(ReflinkMixin, qubes.tests.QubesTestCase): + def setUp(self): # pylint: disable=arguments-differ + super().setUp('btrfs') + self.ficlone_supported = True + +class TC_01_ReflinkOnExt4(ReflinkMixin, qubes.tests.QubesTestCase): + def setUp(self): # pylint: disable=arguments-differ + super().setUp('ext4') + self.ficlone_supported = False + + +def setup_loopdev(img, cleanup_via=None): + dev = str.strip(cmd('sudo', 'losetup', '-f', '--show', img).decode()) + if cleanup_via is not None: + cleanup_via(detach_loopdev, dev) + return dev + +def detach_loopdev(dev): + cmd('sudo', 'losetup', '-d', dev) + +def get_fs_type(directory): + # 'stat -f -c %T' would identify ext4 as 'ext2/ext3' + return cmd('df', '--output=fstype', directory).decode().splitlines()[1] + +def mkdir_fs(directory, fs_type, + accessible=True, max_size=100*1024**3, cleanup_via=None): + os.mkdir(directory) + + if get_fs_type(directory) != fs_type: + img = os.path.join(directory, 'img') + with open(img, 'xb') as img_io: + img_io.truncate(max_size) + cmd('mkfs.' + fs_type, img) + dev = setup_loopdev(img) + os.remove(img) + cmd('sudo', 'mount', dev, directory) + detach_loopdev(dev) + + if accessible: + cmd('sudo', 'chmod', '777', directory) + else: + cmd('sudo', 'chmod', '000', directory) + cmd('sudo', 'chattr', '+i', directory) # cause EPERM on write as root + + if cleanup_via is not None: + cleanup_via(rmtree_fs, directory) + +def rmtree_fs(directory): + if os.path.ismount(directory): + cmd('sudo', 'umount', '-l', directory) + # loop device and backing file are garbage collected automatically + cmd('sudo', 'chattr', '-i', directory) + cmd('sudo', 'chmod', '777', directory) + shutil.rmtree(directory) + +def get_blockdev_size(dev): + return int(cmd('sudo', 'blockdev', '--getsize64', dev)) + +def reflink_update_loopdev_sizes(img): + env = [k + '=' + v for k, v in os.environ.items() # 'sudo -E' alone would + if k.startswith('PYTHON')] # drop some of these + code = ('from qubes.storage import reflink\n' + 'reflink._update_loopdev_sizes(%r)' % img) + cmd('sudo', '-E', 'env', *env, sys.executable, '-c', code) + +def cmd(*argv): + p = subprocess.run(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if p.returncode != 0: + raise Exception(str(p)) # this will show stdout and stderr + return p.stdout diff --git a/rpm_spec/core-dom0.spec.in b/rpm_spec/core-dom0.spec.in index 283434cb..5dc6ddfc 100644 --- a/rpm_spec/core-dom0.spec.in +++ b/rpm_spec/core-dom0.spec.in @@ -305,6 +305,7 @@ fi %{python3_sitelib}/qubes/tests/init.py %{python3_sitelib}/qubes/tests/storage.py %{python3_sitelib}/qubes/tests/storage_file.py +%{python3_sitelib}/qubes/tests/storage_reflink.py %{python3_sitelib}/qubes/tests/storage_kernels.py %{python3_sitelib}/qubes/tests/storage_lvm.py %{python3_sitelib}/qubes/tests/tarwriter.py From cf1ea5cee1dfbe718a3d5328f97cff3d951560c3 Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Tue, 11 Sep 2018 23:50:33 +0000 Subject: [PATCH 076/145] tests/app: test varlibqubes pool driver selection --- qubes/tests/app.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/qubes/tests/app.py b/qubes/tests/app.py index 04ad2171..63b630c9 100644 --- a/qubes/tests/app.py +++ b/qubes/tests/app.py @@ -30,6 +30,7 @@ import qubes.events import qubes.tests import qubes.tests.init +import qubes.tests.storage_reflink class TestApp(qubes.tests.TestEmitter): pass @@ -264,6 +265,44 @@ class TC_30_VMCollection(qubes.tests.QubesTestCase): # pass +class TC_80_QubesInitialPools(qubes.tests.QubesTestCase): + def setUp(self): + super().setUp() + self.app = qubes.Qubes('/tmp/qubestest.xml', load=False, + offline_mode=True) + self.test_dir = '/var/tmp/test-varlibqubes' + self.test_patch = mock.patch.dict( + qubes.config.defaults['pool_configs']['varlibqubes'], + {'dir_path': self.test_dir}) + self.test_patch.start() + + def tearDown(self): + self.test_patch.stop() + self.app.close() + del self.app + + def get_driver(self, fs_type, accessible): + qubes.tests.storage_reflink.mkdir_fs(self.test_dir, fs_type, + accessible=accessible, cleanup_via=self.addCleanup) + self.app.load_initial_values() + + varlibqubes = self.app.pools['varlibqubes'] + self.assertEqual(varlibqubes.dir_path, self.test_dir) + return varlibqubes.driver + + def test_100_varlibqubes_btrfs_accessible(self): + self.assertEqual(self.get_driver('btrfs', True), 'file-reflink') + + def test_101_varlibqubes_btrfs_inaccessible(self): + self.assertEqual(self.get_driver('btrfs', False), 'file') + + def test_102_varlibqubes_ext4_accessible(self): + self.assertEqual(self.get_driver('ext4', True), 'file') + + def test_103_varlibqubes_ext4_inaccessible(self): + self.assertEqual(self.get_driver('ext4', False), 'file') + + class TC_89_QubesEmpty(qubes.tests.QubesTestCase): def tearDown(self): try: From 5aa35a1208d972f3ec3a2158ed91850e3fa0866e Mon Sep 17 00:00:00 2001 From: AJ Jordan Date: Thu, 30 Aug 2018 13:12:06 -0400 Subject: [PATCH 077/145] Make log location more explicit in error message See https://github.com/QubesOS/qubes-issues/issues/4224#issuecomment-414513721. --- qubes/app.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qubes/app.py b/qubes/app.py index 7a31e822..fe770fa1 100644 --- a/qubes/app.py +++ b/qubes/app.py @@ -1298,9 +1298,9 @@ class Qubes(qubes.PropertyHolder): self.log.error( 'Cannot remove %s, used by %s.%s', vm, obj, prop.__name__) - raise qubes.exc.QubesVMInUseError(vm, - 'Domain is in use: {!r}; details in system log' - .format(vm.name)) + raise qubes.exc.QubesVMInUseError(vm, 'Domain is in ' + 'use: {!r}; see /var/log/qubes/qubes.log in dom0 for ' + 'details'.format(vm.name)) except AttributeError: pass From b3983f5ef82b4b09f1699c0f518390639279cb2a Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Thu, 13 Sep 2018 19:46:45 +0000 Subject: [PATCH 078/145] 'except FileNotFoundError' instead of ENOENT check --- qubes/app.py | 5 ++--- qubes/vm/qubesvm.py | 8 ++------ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/qubes/app.py b/qubes/app.py index fe770fa1..36d1f6be 100644 --- a/qubes/app.py +++ b/qubes/app.py @@ -22,7 +22,6 @@ import collections import copy -import errno import functools import grp import itertools @@ -1008,8 +1007,8 @@ class Qubes(qubes.PropertyHolder): try: fd = os.open(self._store, os.O_RDWR | (os.O_CREAT * int(for_save))) - except OSError as e: - if not for_save and e.errno == errno.ENOENT: + except FileNotFoundError: + if not for_save: raise qubes.exc.QubesException( 'Qubes XML store {!r} is missing; ' 'use qubes-create tool'.format(self._store)) diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 85272c8e..506cfe99 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -24,7 +24,6 @@ from __future__ import absolute_import import asyncio import base64 -import errno import grp import os import os.path @@ -1461,11 +1460,8 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): try: # TODO: make it async? shutil.rmtree(self.dir_path) - except OSError as e: - if e.errno == errno.ENOENT: - pass - else: - raise + except FileNotFoundError: + pass yield from self.storage.remove() @asyncio.coroutine From 867baf47d1720d85a5a8a6e355e8a64180cb1907 Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Thu, 13 Sep 2018 19:46:46 +0000 Subject: [PATCH 079/145] api/admin: fix typo --- qubes/api/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes/api/admin.py b/qubes/api/admin.py index aa425795..60e0fa03 100644 --- a/qubes/api/admin.py +++ b/qubes/api/admin.py @@ -1099,7 +1099,7 @@ class QubesAdminAPI(qubes.api.AbstractQubesAPI): try: yield from self.dest.remove_from_disk() except: # pylint: disable=bare-except - self.app.log.exception('Error wile removing VM \'%s\' files', + self.app.log.exception('Error while removing VM \'%s\' files', self.dest.name) self.app.save() From 5756e870bd91ddd8ed4a81e79881adeec7d8d78a Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Thu, 13 Sep 2018 19:46:48 +0000 Subject: [PATCH 080/145] storage/reflink: use context managers in is_supported() Don't rely on garbage collection to close and remove the tempfiles. --- qubes/storage/reflink.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qubes/storage/reflink.py b/qubes/storage/reflink.py index b70094cb..3f250011 100644 --- a/qubes/storage/reflink.py +++ b/qubes/storage/reflink.py @@ -456,7 +456,7 @@ def is_supported(dst_dir, src_dir=None): ''' if src_dir is None: src_dir = dst_dir - dst = tempfile.TemporaryFile(dir=dst_dir) - src = tempfile.TemporaryFile(dir=src_dir) - src.write(b'foo') # don't let any filesystem get clever with empty files - return _attempt_ficlone(src, dst) + with tempfile.TemporaryFile(dir=src_dir) as src, \ + tempfile.TemporaryFile(dir=dst_dir) as dst: + src.write(b'foo') # don't let any fs get clever with empty files + return _attempt_ficlone(src, dst) From 5a1bf11d0d49964e64f355705d833af7cbc9a6c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 13 Sep 2018 16:42:30 +0200 Subject: [PATCH 081/145] tests: drop qvm-prefs tests Those are moved to qubes-core-admin-client repository. --- qubes/tests/integ/basic.py | 121 ------------------------------------- 1 file changed, 121 deletions(-) diff --git a/qubes/tests/integ/basic.py b/qubes/tests/integ/basic.py index 2b592182..5a7a73bd 100644 --- a/qubes/tests/integ/basic.py +++ b/qubes/tests/integ/basic.py @@ -424,127 +424,6 @@ class TC_01_Properties(qubes.tests.SystemTestCase): self.loop.run_until_complete(self.vm2.create_on_disk()) -class TC_02_QvmPrefs(qubes.tests.SystemTestCase): - # pylint: disable=attribute-defined-outside-init - - def setUp(self): - super(TC_02_QvmPrefs, self).setUp() - self.init_default_template() - self.sharedopts = ['--qubesxml', qubes.tests.XMLPATH] - - def setup_appvm(self): - self.testvm = self.app.add_new_vm( - qubes.vm.appvm.AppVM, - name=self.make_vm_name("vm"), - label='red') - self.loop.run_until_complete(self.testvm.create_on_disk()) - self.app.save() - - def setup_hvm(self): - self.testvm = self.app.add_new_vm( - qubes.vm.appvm.AppVM, - name=self.make_vm_name("hvm"), - label='red') - self.testvm.virt_mode = 'hvm' - self.loop.run_until_complete(self.testvm.create_on_disk()) - self.app.save() - - def pref_set(self, name, value, valid=True): - self.loop.run_until_complete(self._pref_set(name, value, valid)) - - @asyncio.coroutine - def _pref_set(self, name, value, valid=True): - cmd = ['qvm-prefs'] - if value != '-D': - cmd.append('--') - cmd.extend((self.testvm.name, name, value)) - p = yield from asyncio.create_subprocess_exec(*cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - (stdout, stderr) = yield from p.communicate() - if valid: - self.assertEqual(p.returncode, 0, - "qvm-prefs .. '{}' '{}' failed: {}{}".format( - name, value, stdout, stderr - )) - else: - self.assertNotEquals(p.returncode, 0, - "qvm-prefs should reject value '{}' for " - "property '{}'".format(value, name)) - - def pref_get(self, name): - self.loop.run_until_complete(self._pref_get(name)) - - @asyncio.coroutine - def _pref_get(self, name): - p = yield from asyncio.create_subprocess_exec( - 'qvm-prefs', *self.sharedopts, '--', self.testvm.name, name, - stdout=subprocess.PIPE) - (stdout, _) = yield from p.communicate() - self.assertEqual(p.returncode, 0) - return stdout.strip() - - bool_test_values = [ - ('true', 'True', True), - ('False', 'False', True), - ('0', 'False', True), - ('1', 'True', True), - ('invalid', '', False) - ] - - def execute_tests(self, name, values): - """ - Helper function, which executes tests for given property. - :param values: list of tuples (value, expected, valid), - where 'value' is what should be set and 'expected' is what should - qvm-prefs returns as a property value and 'valid' marks valid and - invalid values - if it's False, qvm-prefs should reject the value - :return: None - """ - for (value, expected, valid) in values: - self.pref_set(name, value, valid) - if valid: - self.assertEqual(self.pref_get(name), expected) - - @unittest.skip('test not converted to core3 API') - def test_006_template(self): - templates = [tpl for tpl in self.app.domains.values() if - isinstance(tpl, qubes.vm.templatevm.TemplateVM)] - if not templates: - self.skipTest("No templates installed") - some_template = templates[0].name - self.setup_appvm() - self.execute_tests('template', [ - (some_template, some_template, True), - ('invalid', '', False), - ]) - - @unittest.skip('test not converted to core3 API') - def test_014_pcidevs(self): - self.setup_appvm() - self.execute_tests('pcidevs', [ - ('[]', '[]', True), - ('[ "00:00.0" ]', "['00:00.0']", True), - ('invalid', '', False), - ('[invalid]', '', False), - # TODO: - # ('["12:12.0"]', '', False) - ]) - - @unittest.skip('test not converted to core3 API') - def test_024_pv_reject_hvm_props(self): - self.setup_appvm() - self.execute_tests('guiagent_installed', [('False', '', False)]) - self.execute_tests('qrexec_installed', [('False', '', False)]) - self.execute_tests('drive', [('/tmp/drive.img', '', False)]) - self.execute_tests('timezone', [('localtime', '', False)]) - - @unittest.skip('test not converted to core3 API') - def test_025_hvm_reject_pv_props(self): - self.setup_hvm() - self.execute_tests('kernel', [('default', '', False)]) - self.execute_tests('kernelopts', [('default', '', False)]) - class TC_03_QvmRevertTemplateChanges(qubes.tests.SystemTestCase): # pylint: disable=attribute-defined-outside-init From 556a08cb783e91085f2d0696c17331f820b8d07f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 13 Sep 2018 16:43:24 +0200 Subject: [PATCH 082/145] tests: improve shutdown timeout handling Instead of waiting 1sec, wait up to 5sec but skip when vm is shut off. This fix tests on slow machines, including openQA nested virt. --- qubes/tests/integ/basic.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/qubes/tests/integ/basic.py b/qubes/tests/integ/basic.py index 5a7a73bd..9c67fc1d 100644 --- a/qubes/tests/integ/basic.py +++ b/qubes/tests/integ/basic.py @@ -109,7 +109,10 @@ class TC_00_Basic(qubes.tests.SystemTestCase): # Type 'poweroff' subprocess.check_call(['xdotool', 'search', '--name', self.vm.name, 'type', 'poweroff\r']) - self.loop.run_until_complete(asyncio.sleep(1)) + for _ in range(5): + if not self.vm.is_running(): + break + self.loop.run_until_complete(asyncio.sleep(1)) self.assertFalse(self.vm.is_running()) def _test_200_on_domain_start(self, vm, event, **_kwargs): @@ -205,6 +208,8 @@ class TC_00_Basic(qubes.tests.SystemTestCase): if self.test_failure_reason: self.fail(self.test_failure_reason) + while self.vm.get_power_state() != 'Halted': + self.loop.run_until_complete(asyncio.sleep(1)) # and give a chance for both domain-shutdown handlers to execute self.loop.run_until_complete(asyncio.sleep(1)) @@ -670,7 +675,10 @@ class TC_06_AppVMMixin(object): # Type 'poweroff' subprocess.check_call(['xdotool', 'search', '--name', self.vm.name, 'type', 'poweroff\r']) - self.loop.run_until_complete(asyncio.sleep(1)) + for _ in range(5): + if not self.vm.is_running(): + break + self.loop.run_until_complete(asyncio.sleep(1)) self.assertFalse(self.vm.is_running()) From 576bcb158e2f2f9c10d5b98abebd5aac2ffcbcc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 13 Sep 2018 16:45:13 +0200 Subject: [PATCH 083/145] tests: skip tests not relevant on Whonix --- qubes/tests/integ/vm_qrexec_gui.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/qubes/tests/integ/vm_qrexec_gui.py b/qubes/tests/integ/vm_qrexec_gui.py index b83ba0a6..d0a07dc1 100644 --- a/qubes/tests/integ/vm_qrexec_gui.py +++ b/qubes/tests/integ/vm_qrexec_gui.py @@ -114,6 +114,8 @@ class TC_00_AppVMMixin(object): def test_011_run_gnome_terminal(self): if "minimal" in self.template: self.skipTest("Minimal template doesn't have 'gnome-terminal'") + if 'whonix' in self.template: + self.skipTest("Whonix template doesn't have 'gnome-terminal'") self.loop.run_until_complete(self.testvm1.start()) self.assertEqual(self.testvm1.get_power_state(), "Running") self.loop.run_until_complete(self.wait_for_session(self.testvm1)) @@ -786,6 +788,8 @@ class TC_00_AppVMMixin(object): @unittest.skipUnless(spawn.find_executable('parecord'), "pulseaudio-utils not installed in dom0") def test_220_audio_playback(self): + if 'whonix-gw' in self.template: + self.skipTest('whonix-gw have no audio') self.loop.run_until_complete(self.testvm1.start()) try: self.loop.run_until_complete( @@ -847,6 +851,8 @@ class TC_00_AppVMMixin(object): @unittest.skipUnless(spawn.find_executable('parecord'), "pulseaudio-utils not installed in dom0") def test_221_audio_record_muted(self): + if 'whonix-gw' in self.template: + self.skipTest('whonix-gw have no audio') self.loop.run_until_complete(self.testvm1.start()) try: self.loop.run_until_complete( @@ -886,6 +892,8 @@ class TC_00_AppVMMixin(object): @unittest.skipUnless(spawn.find_executable('parecord'), "pulseaudio-utils not installed in dom0") def test_222_audio_record_unmuted(self): + if 'whonix-gw' in self.template: + self.skipTest('whonix-gw have no audio') self.loop.run_until_complete(self.testvm1.start()) try: self.loop.run_until_complete( @@ -1028,10 +1036,10 @@ int main(int argc, char **argv) { input=allocator_c.encode()) try: - stdout, stderr = yield from self.testvm1.run_for_stdio( + yield from self.testvm1.run_for_stdio( 'gcc allocator.c -o allocator') - except subprocess.CalledProcessError: - self.skipTest('allocator compile failed: {}'.format(stderr)) + except subprocess.CalledProcessError as e: + self.skipTest('allocator compile failed: {}'.format(e.stderr)) # drop caches to have even more memory pressure yield from self.testvm1.run_for_stdio( From ac8b8a3ad43537073f5784e124def621146fe6bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 13 Sep 2018 16:45:40 +0200 Subject: [PATCH 084/145] tests: reenable some qrexec tests, convert them to py3k/asyncio --- qubes/tests/integ/vm_qrexec_gui.py | 136 +++++++++++++++++------------ 1 file changed, 82 insertions(+), 54 deletions(-) diff --git a/qubes/tests/integ/vm_qrexec_gui.py b/qubes/tests/integ/vm_qrexec_gui.py index d0a07dc1..a7136ec0 100644 --- a/qubes/tests/integ/vm_qrexec_gui.py +++ b/qubes/tests/integ/vm_qrexec_gui.py @@ -223,7 +223,6 @@ class TC_00_AppVMMixin(object): self.assertFalse(stderr, 'Some data was printed to stderr') - @unittest.skip('#2851, because there is no GUI in vm') def test_051_qrexec_simple_eof_reverse(self): """Test for EOF transmission VM->dom0""" @@ -241,7 +240,7 @@ class TC_00_AppVMMixin(object): p.stdin.write(TEST_DATA) yield from p.stdin.drain() p.stdin.close() - self.assertEqual(stdout.strip(), 'test', + self.assertEqual(stdout.strip(), b'test', 'Received data differs from what was expected') # this may hang in some buggy cases self.assertFalse((yield from p.stderr.read()), @@ -254,15 +253,18 @@ class TC_00_AppVMMixin(object): "probably EOF wasn't transferred from the VM process") self.loop.run_until_complete(self.testvm1.start()) + self.loop.run_until_complete(self.wait_for_session(self.testvm1)) self.loop.run_until_complete(run(self)) - @unittest.skip('#2851, because there is no GUI in vm') def test_052_qrexec_vm_service_eof(self): """Test for EOF transmission VM(src)->VM(dst)""" self.loop.run_until_complete(asyncio.wait([ self.testvm1.start(), self.testvm2.start()])) + self.loop.run_until_complete(asyncio.wait([ + self.wait_for_session(self.testvm1), + self.wait_for_session(self.testvm2)])) self.loop.run_until_complete(self.testvm2.run_for_stdio( 'cat > /etc/qubes-rpc/test.EOF', user='root', @@ -279,7 +281,7 @@ class TC_00_AppVMMixin(object): except asyncio.TimeoutError: self.fail("Timeout, probably EOF wasn't transferred") - self.assertEqual(stdout, b'test', + self.assertEqual(stdout, b'test\n', 'Received data differs from what was expected') @unittest.expectedFailure @@ -400,23 +402,14 @@ class TC_00_AppVMMixin(object): except asyncio.TimeoutError: self.fail('Timeout, probably deadlock') - @unittest.skip('localcmd= argument went away') def test_071_qrexec_dom0_simultaneous_write(self): """Test for simultaneous write in dom0(src)->VM(dst) connection Similar to test_070_qrexec_vm_simultaneous_write, but with dom0 as a source. """ - def run(self): - result.value = self.testvm2.run_service( - "test.write", localcmd="/bin/sh -c '" - # first write a lot of data to fill all the buffers - "dd if=/dev/zero bs=993 count=10000 iflag=fullblock & " - # then after some time start reading - "sleep 1; " - "dd of=/dev/null bs=993 count=10000 iflag=fullblock; " - "wait" - "'") + + self.loop.run_until_complete(self.testvm2.start()) self.create_remote_file(self.testvm2, '/etc/qubes-rpc/test.write', '''\ # first write a lot of data @@ -424,58 +417,93 @@ class TC_00_AppVMMixin(object): # and only then read something dd of=/dev/null bs=993 count=10000 iflag=fullblock ''') - self.create_local_file('/etc/qubes-rpc/policy/test.write', - '{} {} allow'.format(self.testvm1.name, self.testvm2.name)) - t = multiprocessing.Process(target=run, args=(self,)) - t.start() - t.join(timeout=10) - if t.is_alive(): - t.terminate() + # can't use subprocess.PIPE, because asyncio will claim those FDs + pipe1_r, pipe1_w = os.pipe() + pipe2_r, pipe2_w = os.pipe() + try: + local_proc = self.loop.run_until_complete( + asyncio.create_subprocess_shell( + # first write a lot of data to fill all the buffers + "dd if=/dev/zero bs=993 count=10000 iflag=fullblock & " + # then after some time start reading + "sleep 1; " + "dd of=/dev/null bs=993 count=10000 iflag=fullblock; " + "wait", stdin=pipe1_r, stdout=pipe2_w)) + + service_proc = self.loop.run_until_complete(self.testvm2.run_service( + "test.write", stdin=pipe2_r, stdout=pipe1_w)) + finally: + os.close(pipe1_r) + os.close(pipe1_w) + os.close(pipe2_r) + os.close(pipe2_w) + + try: + self.loop.run_until_complete( + asyncio.wait_for(service_proc.wait(), timeout=10)) + except asyncio.TimeoutError: self.fail("Timeout, probably deadlock") - self.assertEqual(result.value, 0, "Service call failed") + else: + self.assertEqual(service_proc.returncode, 0, + "Service call failed") + finally: + try: + service_proc.terminate() + except ProcessLookupError: + pass - @unittest.skip('localcmd= argument went away') def test_072_qrexec_to_dom0_simultaneous_write(self): """Test for simultaneous write in dom0(src)<-VM(dst) connection Similar to test_071_qrexec_dom0_simultaneous_write, but with dom0 as a "hanging" side. """ - result = multiprocessing.Value('i', -1) - - def run(self): - result.value = self.testvm2.run_service( - "test.write", localcmd="/bin/sh -c '" - # first write a lot of data to fill all the buffers - "dd if=/dev/zero bs=993 count=10000 iflag=fullblock " - # then, only when all written, read something - "dd of=/dev/null bs=993 count=10000 iflag=fullblock; " - "'") self.loop.run_until_complete(self.testvm2.start()) - p = self.testvm2.run("cat > /etc/qubes-rpc/test.write", user="root", - passio_popen=True) - # first write a lot of data - p.stdin.write(b"dd if=/dev/zero bs=993 count=10000 iflag=fullblock &\n") - # and only then read something - p.stdin.write(b"dd of=/dev/null bs=993 count=10000 iflag=fullblock\n") - p.stdin.write(b"sleep 1; \n") - p.stdin.write(b"wait\n") - p.stdin.close() - p.wait() - policy = open("/etc/qubes-rpc/policy/test.write", "w") - policy.write("%s %s allow" % (self.testvm1.name, self.testvm2.name)) - policy.close() - self.addCleanup(os.unlink, "/etc/qubes-rpc/policy/test.write") - t = multiprocessing.Process(target=run, args=(self,)) - t.start() - t.join(timeout=10) - if t.is_alive(): - t.terminate() + self.create_remote_file(self.testvm2, '/etc/qubes-rpc/test.write', '''\ + # first write a lot of data + dd if=/dev/zero bs=993 count=10000 iflag=fullblock & + # and only then read something + dd of=/dev/null bs=993 count=10000 iflag=fullblock + sleep 1; + wait + ''') + + # can't use subprocess.PIPE, because asyncio will claim those FDs + pipe1_r, pipe1_w = os.pipe() + pipe2_r, pipe2_w = os.pipe() + try: + local_proc = self.loop.run_until_complete( + asyncio.create_subprocess_shell( + # first write a lot of data to fill all the buffers + "dd if=/dev/zero bs=993 count=10000 iflag=fullblock & " + # then, only when all written, read something + "dd of=/dev/null bs=993 count=10000 iflag=fullblock; ", + stdin=pipe1_r, stdout=pipe2_w)) + + service_proc = self.loop.run_until_complete(self.testvm2.run_service( + "test.write", stdin=pipe2_r, stdout=pipe1_w)) + finally: + os.close(pipe1_r) + os.close(pipe1_w) + os.close(pipe2_r) + os.close(pipe2_w) + + try: + self.loop.run_until_complete( + asyncio.wait_for(service_proc.wait(), timeout=10)) + except asyncio.TimeoutError: self.fail("Timeout, probably deadlock") - self.assertEqual(result.value, 0, "Service call failed") + else: + self.assertEqual(service_proc.returncode, 0, + "Service call failed") + finally: + try: + service_proc.terminate() + except ProcessLookupError: + pass def test_080_qrexec_service_argument_allow_default(self): """Qrexec service call with argument""" From 240b1dd75e2a0116cd97de4c4dd4609e2f937cf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 15 Sep 2018 03:34:50 +0200 Subject: [PATCH 085/145] tests: exclude whonixcheck and NetworkManager from editor window search Those may pop up before actual editor is found, which fails the test as it can't handle such "editor". --- qubes/tests/integ/dispvm.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qubes/tests/integ/dispvm.py b/qubes/tests/integ/dispvm.py index b8a95c83..d88ff926 100644 --- a/qubes/tests/integ/dispvm.py +++ b/qubes/tests/integ/dispvm.py @@ -267,7 +267,9 @@ class TC_20_DispVMMixin(object): # ignore LibreOffice splash screen and window with no title # set yet if window_title and not window_title.startswith("LibreOffice")\ - and not window_title == 'VMapp command': + and not window_title == 'VMapp command' \ + and 'whonixcheck' not in window_title \ + and not window_title == 'NetworkManager Applet': break wait_count += 1 if wait_count > 100: From c4a84b3298721782775707cf02360e7b5b722f24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 15 Sep 2018 03:53:30 +0200 Subject: [PATCH 086/145] tests: wait for DispVM's qubes.VMShell exit It isn't enough to wait for window to disappear, the service may still be running. And if it is, test cleanup logic will complain about FD leak. To avoid deadlock on some test failure, do it with a timeout. --- qubes/tests/integ/dispvm.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/qubes/tests/integ/dispvm.py b/qubes/tests/integ/dispvm.py index d88ff926..5c26c7e0 100644 --- a/qubes/tests/integ/dispvm.py +++ b/qubes/tests/integ/dispvm.py @@ -23,6 +23,7 @@ import subprocess import tempfile import time import unittest +from contextlib import suppress from distutils import spawn @@ -162,8 +163,15 @@ class TC_20_DispVMMixin(object): self.enter_keys_in_window(window_title, ['Return']) # Wait for window to close self.wait_for_window(window_title, show=False) - finally: p.stdin.close() + self.loop.run_until_complete( + asyncio.wait_for(p.wait(), 30)) + except: + with suppress(ProcessLookupError): + p.terminate() + self.loop.run_until_complete(p.wait()) + raise + finally: del p finally: self.loop.run_until_complete(dispvm.cleanup()) From e26655bc823f9779f9e24485c21ab3e5883d42f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 15 Sep 2018 05:11:36 +0200 Subject: [PATCH 087/145] tests: fix time sync test qvm-sync-clock no longer fetches time from the network, by design. So, lets not break clockvm's time and check only if everything else correctly synchronize with it. --- qubes/tests/integ/vm_qrexec_gui.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/qubes/tests/integ/vm_qrexec_gui.py b/qubes/tests/integ/vm_qrexec_gui.py index a7136ec0..e70545b4 100644 --- a/qubes/tests/integ/vm_qrexec_gui.py +++ b/qubes/tests/integ/vm_qrexec_gui.py @@ -776,7 +776,8 @@ class TC_00_AppVMMixin(object): if self.template.startswith('whonix-'): self.skipTest('qvm-sync-clock disabled for Whonix VMs') self.loop.run_until_complete(asyncio.wait([ - self.testvm1.start()])) + self.testvm1.start(), + self.testvm2.start(),])) start_time = subprocess.check_output(['date', '-u', '+%s']) try: @@ -786,11 +787,11 @@ class TC_00_AppVMMixin(object): subprocess.check_call(['sudo', 'date', '-s', '2001-01-01T12:34:56'], stdout=subprocess.DEVNULL) self.loop.run_until_complete( - self.testvm1.run_for_stdio('date -s 2001-01-01T12:34:56', + self.testvm2.run_for_stdio('date -s 2001-01-01T12:34:56', user='root')) self.loop.run_until_complete( - self.testvm1.run_for_stdio('qvm-sync-clock', + self.testvm2.run_for_stdio('qvm-sync-clock', user='root')) p = self.loop.run_until_complete( @@ -799,7 +800,7 @@ class TC_00_AppVMMixin(object): self.loop.run_until_complete(p.wait()) self.assertEqual(p.returncode, 0) vm_time, _ = self.loop.run_until_complete( - self.testvm1.run_for_stdio('date -u +%s')) + self.testvm2.run_for_stdio('date -u +%s')) self.assertAlmostEquals(int(vm_time), int(start_time), delta=30) dom0_time = subprocess.check_output(['date', '-u', '+%s']) From 7feed2f680865af89ee9636b5b654009ba293e78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 16 Sep 2018 05:22:30 +0200 Subject: [PATCH 088/145] Handle qubes.skip_autostart option on kernel command line This allows to prevent automatically starting VMs at boot, mostly for troubleshooting. Fixes QubesOS/qubes-issues#4312 --- linux/systemd/qubes-vm@.service | 1 + 1 file changed, 1 insertion(+) diff --git a/linux/systemd/qubes-vm@.service b/linux/systemd/qubes-vm@.service index 0ff2255a..cb5adb1e 100644 --- a/linux/systemd/qubes-vm@.service +++ b/linux/systemd/qubes-vm@.service @@ -2,6 +2,7 @@ Description=Start Qubes VM %i Before=systemd-user-sessions.service After=qubesd.service qubes-meminfo-writer-dom0.service +ConditionKernelCommandLine=!qubes.skip_autostart [Service] Type=oneshot From bee69a98b9bf1d0b94f7ef4d18dc7275391733ee Mon Sep 17 00:00:00 2001 From: Rusty Bird Date: Sun, 16 Sep 2018 18:42:48 +0000 Subject: [PATCH 089/145] Add default_qrexec_timeout to qubes-prefs When a VM (or its template) does not explicitly set a qrexec_timeout, fall back to a global default_qrexec_timeout (with default value 60), instead of hardcoding the fallback value to 60. This makes it easy to set a higher timeout for the whole system, which helps users who habitually launch applications from several (not yet started) VMs at the same time. 60 seconds can be too short for that. --- qubes/app.py | 7 +++++++ qubes/tests/vm/__init__.py | 1 + qubes/tests/vm/qubesvm.py | 9 +++++++++ qubes/vm/qubesvm.py | 3 ++- 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/qubes/app.py b/qubes/app.py index 36d1f6be..8d0bd15e 100644 --- a/qubes/app.py +++ b/qubes/app.py @@ -718,6 +718,13 @@ class Qubes(qubes.PropertyHolder): setter=_setter_pool, doc='Default storage pool for kernel volumes') + default_qrexec_timeout = qubes.property('default_qrexec_timeout', + load_stage=3, + default=60, + type=int, + doc='''Default time in seconds after which qrexec connection attempt is + deemed failed''') + stats_interval = qubes.property('stats_interval', default=3, type=int, diff --git a/qubes/tests/vm/__init__.py b/qubes/tests/vm/__init__.py index d42148ec..54564b75 100644 --- a/qubes/tests/vm/__init__.py +++ b/qubes/tests/vm/__init__.py @@ -88,6 +88,7 @@ class TestApp(qubes.tests.TestEmitter): self.default_pool_root = 'default' self.default_pool_private = 'default' self.default_pool_kernel = 'linux-kernel' + self.default_qrexec_timeout = 60 self.default_netvm = None self.domains = TestVMsCollection() #: jinja2 environment for libvirt XML templates diff --git a/qubes/tests/vm/qubesvm.py b/qubes/tests/vm/qubesvm.py index 6d2e52ff..308fd782 100644 --- a/qubes/tests/vm/qubesvm.py +++ b/qubes/tests/vm/qubesvm.py @@ -459,6 +459,15 @@ class TC_90_QubesVM(QubesVMTestsMixin, qubes.tests.QubesTestCase): self.assertPropertyInvalidValue(vm, 'qrexec_timeout', '-2') self.assertPropertyInvalidValue(vm, 'qrexec_timeout', '') + def test_272_qrexec_timeout_global_changed(self): + self.app.default_qrexec_timeout = 123 + vm = self.get_vm() + self.assertPropertyDefaultValue(vm, 'qrexec_timeout', 123) + self.assertPropertyValue(vm, 'qrexec_timeout', 3, 3, '3') + del vm.qrexec_timeout + self.assertPropertyDefaultValue(vm, 'qrexec_timeout', 123) + self.assertPropertyValue(vm, 'qrexec_timeout', '3', 3, '3') + def test_280_autostart(self): vm = self.get_vm() # FIXME any better idea to not involve systemctl call at this stage? diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 506cfe99..7955297e 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -510,7 +510,8 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): # return self._default_user qrexec_timeout = qubes.property('qrexec_timeout', type=int, - default=_default_with_template('qrexec_timeout', 60), + default=_default_with_template('qrexec_timeout', + lambda self: self.app.default_qrexec_timeout), setter=_setter_positive_int, doc='''Time in seconds after which qrexec connection attempt is deemed failed. Operating system inside VM should be able to boot in this From b2387389d0918ad509ed3896ffc0b4d11d20146b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 19 Sep 2018 05:44:02 +0200 Subject: [PATCH 090/145] Update documentation for device-attach event --- qubes/devices.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qubes/devices.py b/qubes/devices.py index bccd9e39..9f6ad12a 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -168,13 +168,14 @@ class DeviceCollection: This class emits following events on VM object: - .. event:: device-attach: (device) + .. event:: device-attach: (device, options) Fired when device is attached to a VM. Handler for this event can be asynchronous (a coroutine). :param device: :py:class:`DeviceInfo` object to be attached + :param options: :py:class:`dict` of attachment options .. event:: device-pre-attach: (device) From 1efac373ddb2366856c26c1961ebc8b58e44fc15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 19 Sep 2018 05:46:07 +0200 Subject: [PATCH 091/145] version 4.0.30 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 3ced8af0..46280483 100644 --- a/version +++ b/version @@ -1 +1 @@ -4.0.29 +4.0.30 From b88fa398942ce1164162f6cfd05a3cd33cfcbf62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 29 Sep 2018 02:40:28 +0200 Subject: [PATCH 092/145] Fix mock-based build --- rpm_spec/core-dom0.spec.in | 1 + 1 file changed, 1 insertion(+) diff --git a/rpm_spec/core-dom0.spec.in b/rpm_spec/core-dom0.spec.in index 5dc6ddfc..17abc376 100644 --- a/rpm_spec/core-dom0.spec.in +++ b/rpm_spec/core-dom0.spec.in @@ -122,6 +122,7 @@ make -C doc PYTHON=%{__python3} SPHINXBUILD=sphinx-build-%{python3_version} man make install \ DESTDIR=$RPM_BUILD_ROOT \ + BACKEND_VMM=%{backend_vmm} \ UNITDIR=%{_unitdir} \ PYTHON_SITEPATH=%{python3_sitelib} \ SYSCONFDIR=%{_sysconfdir} From 5bc0baeafa27f7ec75779a10ef61633d364fd0d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 29 Sep 2018 02:40:43 +0200 Subject: [PATCH 093/145] tests: do not leak objects in object leaks checking function If any object is leaked, QubesTestCase.cleanup_gc() raises an exception, which have leaked objects list referenced in its traceback. This happens after cleanup_traceback(), so isn't cleaned, causing cleanup_gc() fail for all the further tests in the same test run. Avoid this, by dropping list just before checking if any object is leaked. --- qubes/tests/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index b13f2cc0..131dcb64 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -409,6 +409,8 @@ class QubesTestCase(unittest.TestCase): except ImportError: pass + # do not keep leaked object references in locals() + leaked = bool(leaked) assert not leaked def cleanup_loop(self): From 02f966116931612745d6f95e5be3c8ce41a2cb71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 1 Oct 2018 06:02:22 +0200 Subject: [PATCH 094/145] tests: migrate mime handlers test to core3 --- qubes/tests/__init__.py | 1 + {tests => qubes/tests/integ}/mime.py | 288 +++++++++++++-------------- rpm_spec/core-dom0.spec.in | 1 + 3 files changed, 141 insertions(+), 149 deletions(-) rename {tests => qubes/tests/integ}/mime.py (54%) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index 131dcb64..01937069 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -1268,6 +1268,7 @@ def load_tests(loader, tests, pattern): # pylint: disable=unused-argument 'qubes.tests.integ.network', 'qubes.tests.integ.dispvm', 'qubes.tests.integ.vm_qrexec_gui', + 'qubes.tests.integ.mime', 'qubes.tests.integ.salt', 'qubes.tests.integ.backup', 'qubes.tests.integ.backupcompatibility', diff --git a/tests/mime.py b/qubes/tests/integ/mime.py similarity index 54% rename from tests/mime.py rename to qubes/tests/integ/mime.py index 5128585d..e2a5ea0f 100644 --- a/tests/mime.py +++ b/qubes/tests/integ/mime.py @@ -21,112 +21,91 @@ # # from distutils import spawn -import os import re import subprocess import time import unittest +import itertools + +import asyncio + +import sys + import qubes.tests -import qubes.qubes -from qubes.qubes import QubesVmCollection +import qubes @unittest.skipUnless( spawn.find_executable('xprop') and spawn.find_executable('xdotool') and spawn.find_executable('wmctrl'), "xprop or xdotool or wmctrl not installed") -class TC_50_MimeHandlers(qubes.tests.SystemTestsMixin): - @classmethod - def setUpClass(cls): - if cls.template == 'whonix-gw' or 'minimal' in cls.template: - raise unittest.SkipTest( - 'Template {} not supported by this test'.format(cls.template)) - - if cls.template == 'whonix-ws': - # TODO remove when Whonix-based DispVMs will work (Whonix 13?) - raise unittest.SkipTest( - 'Template {} not supported by this test'.format(cls.template)) - - qc = QubesVmCollection() - - cls._kill_test_vms(qc, prefix=qubes.tests.CLSVMPREFIX) - - qc.lock_db_for_writing() - qc.load() - - cls._remove_test_vms(qc, qubes.qubes.vmm.libvirt_conn, - prefix=qubes.tests.CLSVMPREFIX) - - cls.source_vmname = cls.make_vm_name('source', True) - source_vm = qc.add_new_vm("QubesAppVm", - template=qc.get_vm_by_name(cls.template), - name=cls.source_vmname) - source_vm.create_on_disk(verbose=False) - - cls.target_vmname = cls.make_vm_name('target', True) - target_vm = qc.add_new_vm("QubesAppVm", - template=qc.get_vm_by_name(cls.template), - name=cls.target_vmname) - target_vm.create_on_disk(verbose=False) - - qc.save() - qc.unlock_db() - source_vm.start() - target_vm.start() - - # make sure that DispVMs will be started of the same template - retcode = subprocess.call(['/usr/bin/qvm-create-default-dvm', - cls.template], - stderr=open(os.devnull, 'w')) - assert retcode == 0, "Error preparing DispVM" - +class TC_50_MimeHandlers: def setUp(self): super(TC_50_MimeHandlers, self).setUp() - self.source_vm = self.qc.get_vm_by_name(self.source_vmname) - self.target_vm = self.qc.get_vm_by_name(self.target_vmname) + if self.template.startswith('whonix-gw') or 'minimal' in self.template: + raise unittest.SkipTest( + 'Template {} not supported by this test'.format(self.template)) + + self.source_vmname = self.make_vm_name('source') + self.source_vm = self.app.add_new_vm("AppVM", + template=self.template, + name=self.source_vmname, + label='red') + self.loop.run_until_complete(self.source_vm.create_on_disk()) + + self.target_vmname = self.make_vm_name('target') + self.target_vm = self.app.add_new_vm("AppVM", + template=self.template, + name=self.target_vmname, + label='red') + self.loop.run_until_complete(self.target_vm.create_on_disk()) + + self.target_vm.template_for_dispvms = True + self.source_vm.default_dispvm = self.target_vm + + done, not_done = self.loop.run_until_complete(asyncio.wait([ + self.source_vm.start(), + self.target_vm.start()])) + for result in itertools.chain(done, not_done): + # catch any exceptions + result.result() + def get_window_class(self, winid, dispvm=False): (vm_winid, _) = subprocess.Popen( ['xprop', '-id', winid, '_QUBES_VMWINDOWID'], stdout=subprocess.PIPE ).communicate() - vm_winid = vm_winid.split("#")[1].strip('\n" ') + vm_winid = vm_winid.decode().split("#")[1].strip('\n" ') if dispvm: (vmname, _) = subprocess.Popen( ['xprop', '-id', winid, '_QUBES_VMNAME'], stdout=subprocess.PIPE ).communicate() - vmname = vmname.split("=")[1].strip('\n" ') - window_class = None - while window_class is None: - # XXX to use self.qc.get_vm_by_name would require reloading - # qubes.xml, so use qvm-run instead - xprop = subprocess.Popen( - ['qvm-run', '-p', vmname, 'xprop -id {} WM_CLASS'.format( - vm_winid)], stdout=subprocess.PIPE) - (window_class, _) = xprop.communicate() - if xprop.returncode != 0: - self.skipTest("xprop failed, not installed?") - if 'not found' in window_class: - # WM_CLASS not set yet, wait a little - time.sleep(0.1) - window_class = None + vmname = vmname.decode().split("=")[1].strip('\n" ') + vm = self.app.domains[vmname] else: - window_class = None - while window_class is None: - xprop = self.target_vm.run( - 'xprop -id {} WM_CLASS'.format(vm_winid), - passio_popen=True) - (window_class, _) = xprop.communicate() - if xprop.returncode != 0: - self.skipTest("xprop failed, not installed?") - if 'not found' in window_class: - # WM_CLASS not set yet, wait a little - time.sleep(0.1) - window_class = None + vm = self.target_vm + window_class = None + while window_class is None: + try: + window_class, _ = self.loop.run_until_complete( + vm.run_for_stdio('xprop -id {} WM_CLASS'.format(vm_winid))) + except subprocess.CalledProcessError as e: + if e.returncode == 127: + self.skipTest('xprop not installed') + self.fail( + "xprop -id {} WM_CLASS failed: {}".format( + vm_winid, e.stderr.decode())) + if b'not found' in window_class: + # WM_CLASS not set yet, wait a little + time.sleep(0.1) + window_class = None + # output: WM_CLASS(STRING) = "gnome-terminal-server", "Gnome-terminal" try: + window_class = window_class.decode() window_class = window_class.split("=")[1].split(",")[0].strip('\n" ') except IndexError: raise Exception( @@ -136,44 +115,45 @@ class TC_50_MimeHandlers(qubes.tests.SystemTestsMixin): def open_file_and_check_viewer(self, filename, expected_app_titles, expected_app_classes, dispvm=False): - self.qc.unlock_db() if dispvm: - p = self.source_vm.run("qvm-open-in-dvm {}".format(filename), - passio_popen=True) - vmpattern = "disp*" + p = self.loop.run_until_complete(self.source_vm.run( + "qvm-open-in-dvm {}".format(filename), stdout=subprocess.PIPE)) + vmpattern = "disp[0-9]*" else: - self.qrexec_policy('qubes.OpenInVM', self.source_vm.name, - self.target_vmname) - self.qrexec_policy('qubes.OpenURL', self.source_vm.name, - self.target_vmname) - p = self.source_vm.run("qvm-open-in-vm {} {}".format( - self.target_vmname, filename), passio_popen=True) + p = self.loop.run_until_complete(self.source_vm.run( + "qvm-open-in-vm {} {}".format(self.target_vmname, filename), + stdout=subprocess.PIPE)) vmpattern = self.target_vmname wait_count = 0 winid = None - window_title = None - while True: - search = subprocess.Popen(['xdotool', 'search', - '--onlyvisible', '--class', vmpattern], - stdout=subprocess.PIPE, - stderr=open(os.path.devnull, 'w')) - retcode = search.wait() - if retcode == 0: - winid = search.stdout.read().strip() - # get window title - (window_title, _) = subprocess.Popen( - ['xdotool', 'getwindowname', winid], stdout=subprocess.PIPE). \ - communicate() - window_title = window_title.strip() - # ignore LibreOffice splash screen and window with no title - # set yet - if window_title and not window_title.startswith("LibreOffice")\ - and not window_title == 'VMapp command': - break - wait_count += 1 - if wait_count > 100: - self.fail("Timeout while waiting for editor window") - time.sleep(0.3) + with self.qrexec_policy('qubes.OpenInVM', self.source_vm.name, + self.target_vmname): + with self.qrexec_policy('qubes.OpenURL', self.source_vm.name, + self.target_vmname): + while True: + search = subprocess.Popen(['xdotool', 'search', + '--onlyvisible', '--class', vmpattern], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL) + retcode = search.wait() + if retcode == 0: + winid = search.stdout.read().strip() + # get window title + (window_title, _) = subprocess.Popen( + ['xdotool', 'getwindowname', winid], stdout=subprocess.PIPE). \ + communicate() + window_title = window_title.decode('utf8').strip() + # ignore LibreOffice splash screen and window with no title + # set yet + if window_title and \ + not window_title.startswith("LibreOffice") and\ + not window_title.startswith("NetworkManager") and\ + not window_title == 'VMapp command': + break + wait_count += 1 + if wait_count > 100: + self.fail("Timeout while waiting for editor window") + self.loop.run_until_complete(asyncio.sleep(0.3)) # get window class window_class = self.get_window_class(winid, dispvm) @@ -194,45 +174,66 @@ class TC_50_MimeHandlers(qubes.tests.SystemTestsMixin): expected_app_titles, expected_app_classes)) def prepare_txt(self, filename): - p = self.source_vm.run("cat > {}".format(filename), passio_popen=True) - p.stdin.write("This is test\n") - p.stdin.close() - retcode = p.wait() - assert retcode == 0, "Failed to write {} file".format(filename) + self.loop.run_until_complete( + self.source_vm.run_for_stdio("cat > {}".format(filename), + input=b'This is test\n')) def prepare_pdf(self, filename): self.prepare_txt("/tmp/source.txt") - cmd = "convert /tmp/source.txt {}".format(filename) - retcode = self.source_vm.run(cmd, wait=True) - assert retcode == 0, "Failed to run '{}'".format(cmd) + cmd = "convert text:/tmp/source.txt {}".format(filename) + try: + self.loop.run_until_complete( + self.source_vm.run_for_stdio(cmd)) + except subprocess.CalledProcessError as e: + self.fail('{} failed: {}'.format(cmd, e.stderr.decode())) def prepare_doc(self, filename): self.prepare_txt("/tmp/source.txt") cmd = "unoconv -f doc -o {} /tmp/source.txt".format(filename) - retcode = self.source_vm.run(cmd, wait=True) - if retcode != 0: - self.skipTest("Failed to run '{}', not installed?".format(cmd)) + try: + self.loop.run_until_complete( + self.source_vm.run_for_stdio(cmd)) + except subprocess.CalledProcessError as e: + if e.returncode == 127: + self.skipTest("unoconv not installed".format(cmd)) + self.skipTest("Failed to run '{}': {}".format(cmd, + e.stderr.decode())) def prepare_pptx(self, filename): self.prepare_txt("/tmp/source.txt") cmd = "unoconv -f pptx -o {} /tmp/source.txt".format(filename) - retcode = self.source_vm.run(cmd, wait=True) - if retcode != 0: - self.skipTest("Failed to run '{}', not installed?".format(cmd)) + try: + self.loop.run_until_complete( + self.source_vm.run_for_stdio(cmd)) + except subprocess.CalledProcessError as e: + if e.returncode == 127: + self.skipTest("unoconv not installed".format(cmd)) + self.skipTest("Failed to run '{}': {}".format(cmd, + e.stderr.decode())) def prepare_png(self, filename): self.prepare_txt("/tmp/source.txt") - cmd = "convert /tmp/source.txt {}".format(filename) - retcode = self.source_vm.run(cmd, wait=True) - if retcode != 0: - self.skipTest("Failed to run '{}', not installed?".format(cmd)) + cmd = "convert text:/tmp/source.txt {}".format(filename) + try: + self.loop.run_until_complete( + self.source_vm.run_for_stdio(cmd)) + except subprocess.CalledProcessError as e: + if e.returncode == 127: + self.skipTest("convert not installed".format(cmd)) + self.skipTest("Failed to run '{}': {}".format(cmd, + e.stderr.decode())) def prepare_jpg(self, filename): self.prepare_txt("/tmp/source.txt") - cmd = "convert /tmp/source.txt {}".format(filename) - retcode = self.source_vm.run(cmd, wait=True) - if retcode != 0: - self.skipTest("Failed to run '{}', not installed?".format(cmd)) + cmd = "convert text:/tmp/source.txt {}".format(filename) + try: + self.loop.run_until_complete( + self.source_vm.run_for_stdio(cmd)) + except subprocess.CalledProcessError as e: + if e.returncode == 127: + self.skipTest("convert not installed".format(cmd)) + self.skipTest("Failed to run '{}': {}".format(cmd, + e.stderr.decode())) def test_000_txt(self): filename = "/home/user/test_file.txt" @@ -335,19 +336,8 @@ class TC_50_MimeHandlers(qubes.tests.SystemTestsMixin): dispvm=True) def load_tests(loader, tests, pattern): - try: - qc = qubes.qubes.QubesVmCollection() - qc.lock_db_for_reading() - qc.load() - qc.unlock_db() - templates = [vm.name for vm in qc.values() if - isinstance(vm, qubes.qubes.QubesTemplateVm)] - except OSError: - templates = [] - for template in templates: - tests.addTests(loader.loadTestsFromTestCase( - type( - 'TC_50_MimeHandlers_' + template, - (TC_50_MimeHandlers, qubes.tests.QubesTestCase), - {'template': template}))) - return tests \ No newline at end of file + tests.addTests(loader.loadTestsFromNames( + qubes.tests.create_testcases_for_templates('TC_50_MimeHandlers', + TC_50_MimeHandlers, qubes.tests.SystemTestCase, + module=sys.modules[__name__]))) + return tests diff --git a/rpm_spec/core-dom0.spec.in b/rpm_spec/core-dom0.spec.in index 17abc376..5c5034db 100644 --- a/rpm_spec/core-dom0.spec.in +++ b/rpm_spec/core-dom0.spec.in @@ -342,6 +342,7 @@ fi %{python3_sitelib}/qubes/tests/integ/devices_pci.py %{python3_sitelib}/qubes/tests/integ/dispvm.py %{python3_sitelib}/qubes/tests/integ/dom0_update.py +%{python3_sitelib}/qubes/tests/integ/mime.py %{python3_sitelib}/qubes/tests/integ/network.py %{python3_sitelib}/qubes/tests/integ/pvgrub.py %{python3_sitelib}/qubes/tests/integ/salt.py From 7c91e82365e44ec10c20fce1238a29be55eeb52f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 1 Oct 2018 06:02:46 +0200 Subject: [PATCH 095/145] tests: handle KWrite editor in DispVM tests --- qubes/tests/integ/dispvm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes/tests/integ/dispvm.py b/qubes/tests/integ/dispvm.py index 5c26c7e0..1fd34445 100644 --- a/qubes/tests/integ/dispvm.py +++ b/qubes/tests/integ/dispvm.py @@ -191,7 +191,7 @@ class TC_20_DispVMMixin(object): window_title = window_title.decode().strip().\ replace('(', '\(').replace(')', '\)') time.sleep(1) - if "gedit" in window_title: + if "gedit" in window_title or 'KWrite' in window_title: subprocess.check_call(['xdotool', 'windowactivate', '--sync', winid, 'type', 'Test test 2']) subprocess.check_call(['xdotool', 'key', '--window', winid, From f33eca1e3fd2f54d54f2ed838a1d7d187a403f0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 1 Oct 2018 06:03:16 +0200 Subject: [PATCH 096/145] tests: remove old 'hvm' test file Those cases are already covered with basic core3 tests or are irrelevant now. --- tests/hvm.py | 124 --------------------------------------------------- 1 file changed, 124 deletions(-) delete mode 100644 tests/hvm.py diff --git a/tests/hvm.py b/tests/hvm.py deleted file mode 100644 index f4c42b7b..00000000 --- a/tests/hvm.py +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# -# The Qubes OS Project, http://www.qubes-os.org -# -# Copyright (C) 2016 Marek Marczykowski-Górecki -# -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, see . -# -# - -import qubes.tests -from qubes.qubes import QubesException - -class TC_10_HVM(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): - # TODO: test with some OS inside - # TODO: windows tools tests - - def test_000_create_start(self): - testvm1 = self.qc.add_new_vm("QubesHVm", - name=self.make_vm_name('vm1')) - testvm1.create_on_disk(verbose=False) - self.qc.save() - self.qc.unlock_db() - testvm1.start() - self.assertEquals(testvm1.get_power_state(), "Running") - - def test_010_create_start_template(self): - templatevm = self.qc.add_new_vm("QubesTemplateHVm", - name=self.make_vm_name('template')) - templatevm.create_on_disk(verbose=False) - self.qc.save() - self.qc.unlock_db() - - templatevm.start() - self.assertEquals(templatevm.get_power_state(), "Running") - - def test_020_create_start_template_vm(self): - templatevm = self.qc.add_new_vm("QubesTemplateHVm", - name=self.make_vm_name('template')) - templatevm.create_on_disk(verbose=False) - testvm2 = self.qc.add_new_vm("QubesHVm", - name=self.make_vm_name('vm2'), - template=templatevm) - testvm2.create_on_disk(verbose=False) - self.qc.save() - self.qc.unlock_db() - - testvm2.start() - self.assertEquals(testvm2.get_power_state(), "Running") - - def test_030_prevent_simultaneus_start(self): - templatevm = self.qc.add_new_vm("QubesTemplateHVm", - name=self.make_vm_name('template')) - templatevm.create_on_disk(verbose=False) - testvm2 = self.qc.add_new_vm("QubesHVm", - name=self.make_vm_name('vm2'), - template=templatevm) - testvm2.create_on_disk(verbose=False) - self.qc.save() - self.qc.unlock_db() - - templatevm.start() - self.assertEquals(templatevm.get_power_state(), "Running") - self.assertRaises(QubesException, testvm2.start) - templatevm.force_shutdown() - testvm2.start() - self.assertEquals(testvm2.get_power_state(), "Running") - self.assertRaises(QubesException, templatevm.start) - - def test_100_resize_root_img(self): - testvm1 = self.qc.add_new_vm("QubesHVm", - name=self.make_vm_name('vm1')) - testvm1.create_on_disk(verbose=False) - self.qc.save() - self.qc.unlock_db() - testvm1.resize_root_img(30*1024**3) - self.assertEquals(testvm1.get_root_img_sz(), 30*1024**3) - testvm1.start() - self.assertEquals(testvm1.get_power_state(), "Running") - # TODO: launch some OS there and check the size - - def test_200_start_invalid_drive(self): - """Regression test for #1619""" - testvm1 = self.qc.add_new_vm("QubesHVm", - name=self.make_vm_name('vm1')) - testvm1.create_on_disk(verbose=False) - testvm1.drive = 'hd:dom0:/invalid' - self.qc.save() - self.qc.unlock_db() - try: - testvm1.start() - except Exception as e: - self.assertIsInstance(e, QubesException) - else: - self.fail('No exception raised') - - def test_201_start_invalid_drive_cdrom(self): - """Regression test for #1619""" - testvm1 = self.qc.add_new_vm("QubesHVm", - name=self.make_vm_name('vm1')) - testvm1.create_on_disk(verbose=False) - testvm1.drive = 'cdrom:dom0:/invalid' - self.qc.save() - self.qc.unlock_db() - try: - testvm1.start() - except Exception as e: - self.assertIsInstance(e, QubesException) - else: - self.fail('No exception raised') - From 35c66987ab3c86b16916eede5feb75905339947f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 3 Oct 2018 22:39:54 +0200 Subject: [PATCH 097/145] tests: improve clearing tracebacks from Qubes* objects Clear also tracebacks of chained exceptions. --- qubes/tests/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index 01937069..e492eec1 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -391,7 +391,10 @@ class QubesTestCase(unittest.TestCase): continue if exc_info is None: continue - traceback.clear_frames(exc_info[2]) + ex = exc_info[1] + while ex is not None: + traceback.clear_frames(ex.__traceback__) + ex = ex.__context__ def cleanup_gc(self): gc.collect() From 7a607e3731edbdacdac7ffb8de80c67664f80d35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 3 Oct 2018 22:40:38 +0200 Subject: [PATCH 098/145] tests: add QUBES_TEST_TEMPLATES env variable Allow easily list templates to be tested, without enumerating all the test classes. This is especially useful with nose2 runner which can't use load tests protocol _and_ select subset of tests. --- qubes/tests/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index e492eec1..b5637ddd 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -1143,6 +1143,9 @@ _templates = None def list_templates(): '''Returns tuple of template names available in the system.''' global _templates + if _templates is None: + if 'QUBES_TEST_TEMPLATES' in os.environ: + _templates = os.environ['QUBES_TEST_TEMPLATES'].split() if _templates is None: try: app = qubes.Qubes() From c8929cfee9a39c6fdf9378aa493daf3267cd3964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 7 Oct 2018 15:46:07 +0200 Subject: [PATCH 099/145] tests: improve handling backups in core3 --- qubes/tests/integ/backup.py | 11 +---------- qubes/tests/integ/backupcompatibility.py | 1 + 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/qubes/tests/integ/backup.py b/qubes/tests/integ/backup.py index 36b4b98b..450054d3 100644 --- a/qubes/tests/integ/backup.py +++ b/qubes/tests/integ/backup.py @@ -480,16 +480,7 @@ class TC_00_Backup(BackupTestsMixin, qubes.tests.SystemTestCase): os.mkdir(test_dir) with open(os.path.join(test_dir, 'some-file.txt'), 'w') as f: f.write('test file\n') - self.restore_backup(expect_errors=[ - 'Error restoring VM test-inst-test-net, skipping: Got empty ' - 'response from qubesd. See journalctl in dom0 for details.', - 'Error setting test-inst-test1.netvm to test-inst-test-net: ' - '\'"No such domain: \\\'test-inst-test-net\\\'"\'', - ]) - del vms_info['test-inst-test-net'] - vms_info['test-inst-test1']['properties']['netvm'] = \ - str(self.app.default_netvm) - vms_info['test-inst-test1']['default']['netvm'] = True + self.restore_backup() self.assertCorrectlyRestored(vms_info, orig_hashes) finally: del vms diff --git a/qubes/tests/integ/backupcompatibility.py b/qubes/tests/integ/backupcompatibility.py index 43a2a4e1..53165d2c 100644 --- a/qubes/tests/integ/backupcompatibility.py +++ b/qubes/tests/integ/backupcompatibility.py @@ -124,6 +124,7 @@ class TC_00_BackupCompatibility( def tearDown(self): self.remove_test_vms(prefix="test-") + self.remove_test_vms(prefix="disp-test-") super(TC_00_BackupCompatibility, self).tearDown() def create_whitelisted_appmenus(self, filename): From 8dab298b893a9724297f346c49be528cb9822944 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 7 Oct 2018 19:44:48 +0200 Subject: [PATCH 100/145] tests: create testcases on module import if environment variable is set If QUBES_TEST_TEMPLATES or QUBES_TEST_LOAD_ALL is set, create testcases on modules import, instead of waiting until `load_tests` is called. The `QUBES_TEST_TEMPLATES` doesn't require `qubes.xml` access, so it should be safe to do regardless of the environment. The `QUBES_TEST_LOAD_ALL` force loading tests (and reading `qubes.xml`) regardless. This is useful for test runners not supporting load_tests protocol. Or with limited support - for example both default `unittest` runner and `nose2` can either use load_tests protocol _or_ select individual tests. Setting any of those variable allow to run a single test with those runners. With this feature used together load_tests protocol, tests could be registered twice. Avoid this by not listing already defined test classes in create_testcases_for_templates (according to load_tests protocol, those should already be registered). --- qubes/tests/__init__.py | 17 +++++++++++++++++ qubes/tests/integ/backup.py | 11 ++++++++--- qubes/tests/integ/basic.py | 17 ++++++++++------- qubes/tests/integ/dispvm.py | 12 +++++++++--- qubes/tests/integ/dom0_update.py | 11 ++++++++--- qubes/tests/integ/mime.py | 12 +++++++++--- qubes/tests/integ/network.py | 25 ++++++++++++++----------- qubes/tests/integ/pvgrub.py | 10 +++++++--- qubes/tests/integ/salt.py | 12 +++++++++--- qubes/tests/integ/vm_qrexec_gui.py | 10 +++++++--- 10 files changed, 98 insertions(+), 39 deletions(-) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index b5637ddd..98922195 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -1193,12 +1193,29 @@ def create_testcases_for_templates(name, *bases, module, **kwds): for template in list_templates(): clsname = name + '_' + template + if hasattr(module, clsname): + continue cls = type(clsname, bases, {'template': template, **kwds}) cls.__module__ = module.__name__ # XXX I wonder what other __dunder__ attrs did I miss setattr(module, clsname, cls) yield '.'.join((module.__name__, clsname)) +def maybe_create_testcases_on_import(create_testcases_gen): + '''If certain conditions are met, call *create_testcases_gen* to create + testcases for templates tests. The purpose is to use it on integration + tests module(s) import, so the test runner could discover tests without + using load tests protocol. + + The conditions - any of: + - QUBES_TEST_TEMPLATES present in the environment (it's possible to + create test cases without opening qubes.xml) + - QUBES_TEST_LOAD_ALL present in the environment + ''' + if 'QUBES_TEST_TEMPLATES' in os.environ or \ + 'QUBES_TEST_LOAD_ALL' in os.environ: + list(create_testcases_gen()) + def extra_info(obj): '''Return short info identifying object. diff --git a/qubes/tests/integ/backup.py b/qubes/tests/integ/backup.py index 450054d3..73ad8db7 100644 --- a/qubes/tests/integ/backup.py +++ b/qubes/tests/integ/backup.py @@ -641,9 +641,14 @@ class TC_10_BackupVMMixin(BackupTestsMixin): del vms +def create_testcases_for_templates(): + return qubes.tests.create_testcases_for_templates('TC_10_BackupVM', + TC_10_BackupVMMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('TC_10_BackupVM', - TC_10_BackupVMMixin, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) + create_testcases_for_templates())) return tests + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) diff --git a/qubes/tests/integ/basic.py b/qubes/tests/integ/basic.py index 9c67fc1d..f7565460 100644 --- a/qubes/tests/integ/basic.py +++ b/qubes/tests/integ/basic.py @@ -681,17 +681,20 @@ class TC_06_AppVMMixin(object): self.loop.run_until_complete(asyncio.sleep(1)) self.assertFalse(self.vm.is_running()) +def create_testcases_for_templates(): + yield from qubes.tests.create_testcases_for_templates('TC_05_StandaloneVM', + TC_05_StandaloneVMMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + yield from qubes.tests.create_testcases_for_templates('TC_06_AppVM', + TC_06_AppVMMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('TC_05_StandaloneVM', - TC_05_StandaloneVMMixin, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) - tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('TC_06_AppVM', - TC_06_AppVMMixin, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) + create_testcases_for_templates())) return tests +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) + # vim: ts=4 sw=4 et diff --git a/qubes/tests/integ/dispvm.py b/qubes/tests/integ/dispvm.py index 1fd34445..af6f4f7b 100644 --- a/qubes/tests/integ/dispvm.py +++ b/qubes/tests/integ/dispvm.py @@ -294,9 +294,15 @@ class TC_20_DispVMMixin(object): test_txt_content = test_txt_content[3:] self.assertEqual(test_txt_content, b"Test test 2\ntest1\n") + +def create_testcases_for_templates(): + return qubes.tests.create_testcases_for_templates('TC_20_DispVM', + TC_20_DispVMMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('TC_20_DispVM', - TC_20_DispVMMixin, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) + create_testcases_for_templates())) return tests + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) diff --git a/qubes/tests/integ/dom0_update.py b/qubes/tests/integ/dom0_update.py index 516264a3..d103418d 100644 --- a/qubes/tests/integ/dom0_update.py +++ b/qubes/tests/integ/dom0_update.py @@ -380,9 +380,14 @@ Test package 'UNSIGNED package {}-1.0 installed'.format(self.pkg_name)) +def create_testcases_for_templates(): + return qubes.tests.create_testcases_for_templates('TC_00_Dom0Upgrade', + TC_00_Dom0UpgradeMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('TC_00_Dom0Upgrade', - TC_00_Dom0UpgradeMixin, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) + create_testcases_for_templates())) return tests + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) diff --git a/qubes/tests/integ/mime.py b/qubes/tests/integ/mime.py index e2a5ea0f..9a2105d7 100644 --- a/qubes/tests/integ/mime.py +++ b/qubes/tests/integ/mime.py @@ -20,6 +20,7 @@ # License along with this library; if not, see . # # +import os from distutils import spawn import re import subprocess @@ -335,9 +336,14 @@ class TC_50_MimeHandlers: ["Firefox", "Iceweasel", "Navigator"], dispvm=True) +def create_testcases_for_templates(): + return qubes.tests.create_testcases_for_templates('TC_50_MimeHandlers', + TC_50_MimeHandlers, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('TC_50_MimeHandlers', - TC_50_MimeHandlers, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) + create_testcases_for_templates())) return tests + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) diff --git a/qubes/tests/integ/network.py b/qubes/tests/integ/network.py index 8ebb07f0..30f4f32e 100644 --- a/qubes/tests/integ/network.py +++ b/qubes/tests/integ/network.py @@ -1319,17 +1319,20 @@ SHA256: self.assertIn(self.loop.run_until_complete(p.wait()), self.exit_code_ok, '{}: {}\n{}'.format(self.update_cmd, stdout, stderr)) +def create_testcases_for_templates(): + yield from qubes.tests.create_testcases_for_templates('VmNetworking', + VmNetworkingMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + yield from qubes.tests.create_testcases_for_templates('VmIPv6Networking', + VmIPv6NetworkingMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + yield from qubes.tests.create_testcases_for_templates('VmUpdates', + VmUpdatesMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('VmNetworking', - VmNetworkingMixin, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) - tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('VmIPv6Networking', - VmIPv6NetworkingMixin, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) - tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('VmUpdates', - VmUpdatesMixin, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) + create_testcases_for_templates())) return tests + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) diff --git a/qubes/tests/integ/pvgrub.py b/qubes/tests/integ/pvgrub.py index 53a699bd..5c1e7cb1 100644 --- a/qubes/tests/integ/pvgrub.py +++ b/qubes/tests/integ/pvgrub.py @@ -137,10 +137,14 @@ class TC_40_PVGrub(object): self.test_template.run_for_stdio('uname -r')) self.assertEquals(actual_kver.strip(), kver) +def create_testcases_for_templates(): + return qubes.tests.create_testcases_for_templates('TC_40_PVGrub', + TC_40_PVGrub, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('TC_40_PVGrub', - TC_40_PVGrub, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) + create_testcases_for_templates())) return tests + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) diff --git a/qubes/tests/integ/salt.py b/qubes/tests/integ/salt.py index f598c7e7..bfbbd2b2 100644 --- a/qubes/tests/integ/salt.py +++ b/qubes/tests/integ/salt.py @@ -391,9 +391,15 @@ class SaltVMTestMixin(SaltTestMixin): self.assertEqual(stderr, b'') +def create_testcases_for_templates(): + return qubes.tests.create_testcases_for_templates('TC_10_VMSalt', + SaltVMTestMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) + + def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('TC_10_VMSalt', - SaltVMTestMixin, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) + create_testcases_for_templates())) return tests + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) diff --git a/qubes/tests/integ/vm_qrexec_gui.py b/qubes/tests/integ/vm_qrexec_gui.py index e70545b4..c93dd49a 100644 --- a/qubes/tests/integ/vm_qrexec_gui.py +++ b/qubes/tests/integ/vm_qrexec_gui.py @@ -1194,10 +1194,14 @@ class TC_10_Generic(qubes.tests.SystemTestCase): 'Flag file created (service was run) even though should be denied,' ' qrexec-client-vm output: {} {}'.format(stdout, stderr)) +def create_testcases_for_templates(): + return qubes.tests.create_testcases_for_templates('TC_00_AppVM', + TC_00_AppVMMixin, qubes.tests.SystemTestCase, + module=sys.modules[__name__]) def load_tests(loader, tests, pattern): tests.addTests(loader.loadTestsFromNames( - qubes.tests.create_testcases_for_templates('TC_00_AppVM', - TC_00_AppVMMixin, qubes.tests.SystemTestCase, - module=sys.modules[__name__]))) + create_testcases_for_templates())) return tests + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) From 124560645314ea18027b38a004791f01ffe1bb4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 7 Oct 2018 21:47:25 +0200 Subject: [PATCH 101/145] tests: fix cleanup of dom0_update tests Reset updatevm to None before removing VMs, otherwise removing updatevm will fail. --- qubes/tests/integ/dom0_update.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qubes/tests/integ/dom0_update.py b/qubes/tests/integ/dom0_update.py index d103418d..14ec7f1a 100644 --- a/qubes/tests/integ/dom0_update.py +++ b/qubes/tests/integ/dom0_update.py @@ -121,6 +121,7 @@ enabled = 1 self.repo_proc.terminate() self.loop.run_until_complete(self.repo_proc.wait()) del self.repo_proc + self.app.updatevm = None super(TC_00_Dom0UpgradeMixin, self).tearDown() subprocess.call(['rpm', '-e', self.pkg_name], From 399d2138bddb661934cd1c29db5d5fae2de8c305 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 7 Oct 2018 21:48:06 +0200 Subject: [PATCH 102/145] tests: remove old hardware.py tests, migrated to devices_pci.py --- tests/hardware.py | 74 ----------------------------------------------- 1 file changed, 74 deletions(-) delete mode 100644 tests/hardware.py diff --git a/tests/hardware.py b/tests/hardware.py deleted file mode 100644 index 619267f5..00000000 --- a/tests/hardware.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/python2 -# -*- coding: utf-8 -*- -# -# The Qubes OS Project, http://www.qubes-os.org -# -# Copyright (C) 2016 Marek Marczykowski-Górecki -# -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, see . -# -# -import os - -import qubes.tests -import time -import subprocess -from unittest import expectedFailure - - -class TC_00_HVM(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): - def setUp(self): - super(TC_00_HVM, self).setUp() - self.vm = self.qc.add_new_vm("QubesHVm", - name=self.make_vm_name('vm1')) - self.vm.create_on_disk(verbose=False) - - @expectedFailure - def test_000_pci_passthrough_presence(self): - pcidev = os.environ.get('QUBES_TEST_PCIDEV', None) - if pcidev is None: - self.skipTest('Specify PCI device with QUBES_TEST_PCIDEV ' - 'environment variable') - self.vm.pcidevs = [pcidev] - self.vm.pci_strictreset = False - self.qc.save() - self.qc.unlock_db() - - init_script = ( - "#!/bin/sh\n" - "set -e\n" - "lspci -n > /dev/xvdb\n" - "poweroff\n" - ) - - self.prepare_hvm_system_linux(self.vm, init_script, - ['/usr/sbin/lspci']) - self.vm.start() - timeout = 60 - while timeout > 0: - if not self.vm.is_running(): - break - time.sleep(1) - timeout -= 1 - if self.vm.is_running(): - self.fail("Timeout while waiting for VM shutdown") - - with open(self.vm.storage.private_img, 'r') as f: - lspci_vm = f.read(512).strip('\0') - p = subprocess.Popen(['lspci', '-ns', pcidev], stdout=subprocess.PIPE) - (lspci_host, _) = p.communicate() - # strip BDF, as it is different in VM - pcidev_desc = ' '.join(lspci_host.strip().split(' ')[1:]) - self.assertIn(pcidev_desc, lspci_vm) From d23636fa02abe49ff342ce5b0554cd68e2935613 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 7 Oct 2018 22:37:02 +0200 Subject: [PATCH 103/145] tests: migrate qvm-block tests to core3 --- qubes/tests/__init__.py | 1 + .../tests/integ/devices_block.py | 167 ++++++++---------- rpm_spec/core-dom0.spec.in | 1 + 3 files changed, 78 insertions(+), 91 deletions(-) rename tests/block.py => qubes/tests/integ/devices_block.py (63%) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index 98922195..f7ea3465 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -1286,6 +1286,7 @@ def load_tests(loader, tests, pattern): # pylint: disable=unused-argument 'qubes.tests.integ.basic', 'qubes.tests.integ.storage', 'qubes.tests.integ.pvgrub', + 'qubes.tests.integ.devices_block', 'qubes.tests.integ.devices_pci', 'qubes.tests.integ.dom0_update', 'qubes.tests.integ.network', diff --git a/tests/block.py b/qubes/tests/integ/devices_block.py similarity index 63% rename from tests/block.py rename to qubes/tests/integ/devices_block.py index 6220a977..1ad55e42 100644 --- a/tests/block.py +++ b/qubes/tests/integ/devices_block.py @@ -2,7 +2,7 @@ # # The Qubes OS Project, https://www.qubes-os.org/ # -# Copyright (C) 2016 +# Copyright (C) 2018 # Marek Marczykowski-Górecki # # This library is free software; you can redistribute it and/or @@ -20,42 +20,44 @@ # import os +import sys + +import qubes import qubes.tests -import qubes.qubesutils import subprocess # the same class for both dom0 and VMs -class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): +class TC_00_List(qubes.tests.SystemTestCase): template = None def setUp(self): - super(TC_00_List, self).setUp() + super().setUp() self.img_path = '/tmp/test.img' self.mount_point = '/tmp/test-dir' if self.template is not None: - self.vm = self.qc.add_new_vm( - "QubesAppVm", - name=self.make_vm_name("vm"), - template=self.qc.get_vm_by_name(self.template)) - self.vm.create_on_disk(verbose=False) + self.vm = self.app.add_new_vm( + "AppVM", + label='red', + name=self.make_vm_name("vm")) + self.loop.run_until_complete( + self.vm.create_on_disk()) self.app.save() - self.qc.unlock_db() - self.vm.start() + self.loop.run_until_complete(self.vm.start()) else: - self.qc.unlock_db() - self.vm = self.qc[0] + self.vm = self.app.domains[0] def tearDown(self): - super(TC_00_List, self).tearDown() + super().tearDown() if self.template is None: if os.path.exists(self.mount_point): subprocess.call(['sudo', 'umount', self.mount_point]) subprocess.call(['sudo', 'rmdir', self.mount_point]) - subprocess.call(['sudo', 'dmsetup', 'remove', 'test-dm']) + if os.path.exists('/dev/mapper/test-dm'): + subprocess.call(['sudo', 'dmsetup', 'remove', 'test-dm']) if os.path.exists(self.img_path): loopdev = subprocess.check_output(['losetup', '-j', self.img_path]) - for dev in loopdev.splitlines(): + for dev in loopdev.decode().splitlines(): subprocess.call( ['sudo', 'losetup', '-d', dev.split(':')[0]]) subprocess.call(['sudo', 'rm', '-f', self.img_path]) @@ -67,9 +69,8 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): elif user == "root": subprocess.check_call(['sudo', 'sh', '-c', script]) else: - retcode = self.vm.run(script, user=user, wait=True) - if retcode != 0: - raise subprocess.CalledProcessError + self.loop.run_until_complete( + self.vm.run_for_stdio(script, user=user)) def test_000_list_loop(self): if self.template is None: @@ -80,19 +81,18 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): "losetup -f {path}; " "udevadm settle".format(path=self.img_path), user="root") - dev_list = qubes.qubesutils.block_list_vm(self.vm) + dev_list = list(self.vm.devices['block']) found = False - for dev in dev_list.keys(): - if dev_list[dev]['desc'] == self.img_path: - self.assertTrue(dev.startswith(self.vm.name + ':loop')) - self.assertEquals(dev_list[dev]['mode'], 'w') - self.assertEquals(dev_list[dev]['size'], 1024 * 1024 * 128) - self.assertEquals( - dev_list[dev]['device'], '/dev/' + dev.split(':')[1]) + for dev in dev_list: + if dev.description == self.img_path: + self.assertTrue(dev.ident.startswith('loop')) + self.assertEquals(dev.mode, 'w') + self.assertEquals(dev.size, 1024 * 1024 * 128) found = True if not found: - self.fail("Device {} not found in {!r}".format(self.img_path, dev_list)) + self.fail("Device {} not found in {!r}".format( + self.img_path, dev_list)) def test_001_list_loop_mounted(self): if self.template is None: @@ -108,9 +108,9 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): mntdir=self.mount_point), user="root") - dev_list = qubes.qubesutils.block_list_vm(self.vm) - for dev in dev_list.keys(): - if dev_list[dev]['desc'] == self.img_path: + dev_list = list(self.vm.devices['block']) + for dev in dev_list: + if dev.description == self.img_path: self.fail( 'Device {} ({}) should not be listed because is mounted' .format(dev, self.img_path)) @@ -125,19 +125,17 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): "/sys/block/$(basename $loopdev)/dev) 0\";" "udevadm settle".format(path=self.img_path), user="root") - dev_list = qubes.qubesutils.block_list_vm(self.vm) + dev_list = list(self.vm.devices['block']) found = False - for dev in dev_list.keys(): - if dev.startswith(self.vm.name + ':loop'): - self.assertNotEquals(dev_list[dev]['desc'], self.img_path, + for dev in dev_list: + if dev.ident.startswith('loop'): + self.assertNotEquals(dev.description, self.img_path, "Device {} ({}) should not be listed as it is used in " "device-mapper".format(dev, self.img_path) ) - elif dev_list[dev]['desc'] == 'test-dm': - self.assertEquals(dev_list[dev]['mode'], 'w') - self.assertEquals(dev_list[dev]['size'], 1024 * 1024 * 128) - self.assertEquals( - dev_list[dev]['device'], '/dev/' + dev.split(':')[1]) + elif dev.description == 'test-dm': + self.assertEquals(dev.mode, 'w') + self.assertEquals(dev.size, 1024 * 1024 * 128) found = True if not found: @@ -159,15 +157,15 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): mntdir=self.mount_point), user="root") - dev_list = qubes.qubesutils.block_list_vm(self.vm) - for dev in dev_list.keys(): - if dev.startswith(self.vm.name + ':loop'): - self.assertNotEquals(dev_list[dev]['desc'], self.img_path, + dev_list = list(self.vm.devices['block']) + for dev in dev_list: + if dev.ident.startswith('loop'): + self.assertNotEquals(dev.description, self.img_path, "Device {} ({}) should not be listed as it is used in " "device-mapper".format(dev, self.img_path) ) else: - self.assertNotEquals(dev_list[dev]['desc'], 'test-dm', + self.assertNotEquals(dev.description, 'test-dm', "Device {} ({}) should not be listed as it is " "mounted".format(dev, 'test-dm') ) @@ -183,19 +181,17 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): "/sys/block/$(basename $loopdev)/dev) 0\";" "udevadm settle".format(path=self.img_path), user="root") - dev_list = qubes.qubesutils.block_list_vm(self.vm) + dev_list = list(self.vm.devices['block']) found = False - for dev in dev_list.keys(): - if dev.startswith(self.vm.name + ':loop'): - self.assertNotEquals(dev_list[dev]['desc'], self.img_path, + for dev in dev_list: + if dev.ident.startswith('loop'): + self.assertNotEquals(dev.description, self.img_path, "Device {} ({}) should not be listed as it is used in " "device-mapper".format(dev, self.img_path) ) - elif dev_list[dev]['desc'] == 'test-dm': - self.assertEquals(dev_list[dev]['mode'], 'w') - self.assertEquals(dev_list[dev]['size'], 1024 * 1024 * 128) - self.assertEquals( - dev_list[dev]['device'], '/dev/' + dev.split(':')[1]) + elif dev.description == 'test-dm': + self.assertEquals(dev.mode, 'w') + self.assertEquals(dev.size, 1024 * 1024 * 128) found = True if not found: @@ -216,15 +212,13 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): "dmsetup remove test-dm;" "udevadm settle".format(path=self.img_path), user="root") - dev_list = qubes.qubesutils.block_list_vm(self.vm) + dev_list = list(self.vm.devices['block']) found = False - for dev in dev_list.keys(): - if dev_list[dev]['desc'] == self.img_path: - self.assertTrue(dev.startswith(self.vm.name + ':loop')) - self.assertEquals(dev_list[dev]['mode'], 'w') - self.assertEquals(dev_list[dev]['size'], 1024 * 1024 * 128) - self.assertEquals( - dev_list[dev]['device'], '/dev/' + dev.split(':')[1]) + for dev in dev_list: + if dev.description == self.img_path: + self.assertTrue(dev.ident.startswith('loop')) + self.assertEquals(dev.mode, 'w') + self.assertEquals(dev.size, 1024 * 1024 * 128) found = True if not found: @@ -242,16 +236,14 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): "blockdev --rereadpt $loopdev; " "udevadm settle".format(path=self.img_path), user="root") - dev_list = qubes.qubesutils.block_list_vm(self.vm) + dev_list = list(self.vm.devices['block']) found = False - for dev in dev_list.keys(): - if dev_list[dev]['desc'] == self.img_path: - self.assertTrue(dev.startswith(self.vm.name + ':loop')) - self.assertEquals(dev_list[dev]['mode'], 'w') - self.assertEquals(dev_list[dev]['size'], 1024 * 1024 * 128) - self.assertEquals( - dev_list[dev]['device'], '/dev/' + dev.split(':')[1]) - self.assertIn(dev + 'p1', dev_list) + for dev in dev_list: + if dev.description == self.img_path: + self.assertTrue(dev.ident.startswith('loop')) + self.assertEquals(dev.mode, 'w') + self.assertEquals(dev.size, 1024 * 1024 * 128) + self.assertIn(dev.ident + 'p1', [d.ident for d in dev_list]) found = True if not found: @@ -274,14 +266,14 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): path=self.img_path, mntdir=self.mount_point), user="root") - dev_list = qubes.qubesutils.block_list_vm(self.vm) - for dev in dev_list.keys(): - if dev_list[dev]['desc'] == self.img_path: + dev_list = list(self.vm.devices['block']) + for dev in dev_list: + if dev.description == self.img_path: self.fail( 'Device {} ({}) should not be listed because its ' 'partition is mounted' .format(dev, self.img_path)) - elif dev.startswith(self.vm.name + ':loop') and dev.endswith('p1'): + elif dev.ident.startswith('loop') and dev.ident.endswith('p1'): # FIXME: risky assumption that only tests create partitioned # loop devices self.fail( @@ -289,21 +281,14 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): .format(dev, self.img_path)) -def load_tests(loader, tests, pattern): - try: - qc = qubes.qubes.QubesVmCollection() - qc.lock_db_for_reading() - qc.load() - qc.unlock_db() - templates = [vm.name for vm in qc.values() if - isinstance(vm, qubes.qubes.QubesTemplateVm)] - except OSError: - templates = [] - for template in templates: - tests.addTests(loader.loadTestsFromTestCase( - type( - 'TC_00_List_' + template, - (TC_00_List, qubes.tests.QubesTestCase), - {'template': template}))) +def create_testcases_for_templates(): + return qubes.tests.create_testcases_for_templates('TC_00_List', + TC_00_List, + module=sys.modules[__name__]) +def load_tests(loader, tests, pattern): + tests.addTests(loader.loadTestsFromNames( + create_testcases_for_templates())) return tests + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) diff --git a/rpm_spec/core-dom0.spec.in b/rpm_spec/core-dom0.spec.in index 5c5034db..34705ca8 100644 --- a/rpm_spec/core-dom0.spec.in +++ b/rpm_spec/core-dom0.spec.in @@ -339,6 +339,7 @@ fi %{python3_sitelib}/qubes/tests/integ/backup.py %{python3_sitelib}/qubes/tests/integ/backupcompatibility.py %{python3_sitelib}/qubes/tests/integ/basic.py +%{python3_sitelib}/qubes/tests/integ/devices_block.py %{python3_sitelib}/qubes/tests/integ/devices_pci.py %{python3_sitelib}/qubes/tests/integ/dispvm.py %{python3_sitelib}/qubes/tests/integ/dom0_update.py From c45ce78ee46558c9c085136c93dcc6325c1e69a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 10 Oct 2018 02:02:27 +0200 Subject: [PATCH 104/145] version 4.0.31 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 46280483..6634c5d5 100644 --- a/version +++ b/version @@ -1 +1 @@ -4.0.30 +4.0.31 From 00c0b4c69f09470d3f85bc584b2135533eb131e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 12 Oct 2018 14:41:35 +0200 Subject: [PATCH 105/145] tests: cleanup tracebacks also for expectedFailure exception Continuation of 5bc0baea "tests: do not leak objects in object leaks checking function". --- qubes/tests/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index f7ea3465..32bc888b 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -386,9 +386,11 @@ class QubesTestCase(unittest.TestCase): '''Remove local variables reference from tracebacks to allow garbage collector to clean all Qubes*() objects, otherwise file descriptors held by them will leak''' - for test_case, exc_info in self._outcome.errors: - if test_case is not self: - continue + exc_infos = [e for test_case, e in self._outcome.errors + if test_case is self] + if self._outcome.expectedFailure: + exc_infos.append(self._outcome.expectedFailure) + for exc_info in exc_infos: if exc_info is None: continue ex = exc_info[1] From 3e28ccefde8041f7b8d2a9ebbe46686abddd67c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 14 Oct 2018 03:29:30 +0200 Subject: [PATCH 106/145] tests: fix cleanup after backup compatibility tests Allow removing VMs based on multiple prefixes at once. Removing them separately doesn't handle all the dependencies (default_netvm, netvm) correctly. This is needed for backup compatibility tests, where VMs are created with `test-` prefix and `disp-tests-`. Additionally backup code will create `disp-no-netvm`, which also may need to be removed. --- qubes/tests/__init__.py | 18 +++++++++++++----- qubes/tests/integ/backupcompatibility.py | 6 ++++-- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index 32bc888b..6bdf5977 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -902,9 +902,16 @@ class SystemTestCase(QubesTestCase): self._remove_vm_qubes(vm) def remove_test_vms(self, xmlpath=XMLPATH, prefix=VMPREFIX): - '''Aggresively remove any domain that has name in testing namespace. + '''Aggressively remove any domain that has name in testing namespace. + + :param prefix: name prefix of VMs to remove, can be a list of prefixes ''' + if isinstance(prefix, str): + prefixes = [prefix] + else: + prefixes = prefix + del prefix # first, remove them Qubes-way if os.path.exists(xmlpath): try: @@ -917,7 +924,7 @@ class SystemTestCase(QubesTestCase): except AttributeError: host_app = qubes.Qubes() self.remove_vms([vm for vm in app.domains - if vm.name.startswith(prefix) or + if any(vm.name.startswith(prefix) for prefix in prefixes) or (isinstance(vm, qubes.vm.dispvm.DispVM) and vm.name not in host_app.domains)]) if not hasattr(self, 'host_app'): @@ -933,7 +940,7 @@ class SystemTestCase(QubesTestCase): # now remove what was only in libvirt conn = libvirt.open(qubes.config.defaults['libvirt_uri']) for dom in conn.listAllDomains(): - if dom.name().startswith(prefix): + if any(dom.name().startswith(prefix) for prefix in prefixes): self._remove_vm_libvirt(dom) conn.close() @@ -948,11 +955,12 @@ class SystemTestCase(QubesTestCase): if not os.path.exists(dirpath): continue for name in os.listdir(dirpath): - if name.startswith(prefix): + if any(name.startswith(prefix) for prefix in prefixes): vmnames.add(name) for vmname in vmnames: self._remove_vm_disk(vmname) - self._remove_vm_disk_lvm(prefix) + for prefix in prefixes: + self._remove_vm_disk_lvm(prefix) def qrexec_policy(self, service, source, destination, allow=True, action=None): diff --git a/qubes/tests/integ/backupcompatibility.py b/qubes/tests/integ/backupcompatibility.py index 53165d2c..c7a7f6c4 100644 --- a/qubes/tests/integ/backupcompatibility.py +++ b/qubes/tests/integ/backupcompatibility.py @@ -123,8 +123,10 @@ class TC_00_BackupCompatibility( qubes.tests.integ.backup.BackupTestsMixin, qubes.tests.SystemTestCase): def tearDown(self): - self.remove_test_vms(prefix="test-") - self.remove_test_vms(prefix="disp-test-") + prefixes = ["test-", "disp-test-"] + if 'disp-no-netvm' not in self.host_app.domains: + prefixes.append('disp-no-netvm') + self.remove_test_vms(prefix=prefixes) super(TC_00_BackupCompatibility, self).tearDown() def create_whitelisted_appmenus(self, filename): From 29a26e7d69d0025c9fd040e9ad4ce63fe74d8bbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 14 Oct 2018 03:32:17 +0200 Subject: [PATCH 107/145] tests: make timeout it shutdown test even longer Reduce false positives when testing on busy machine. --- qubes/tests/integ/basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes/tests/integ/basic.py b/qubes/tests/integ/basic.py index f7565460..354a39c4 100644 --- a/qubes/tests/integ/basic.py +++ b/qubes/tests/integ/basic.py @@ -211,7 +211,7 @@ class TC_00_Basic(qubes.tests.SystemTestCase): while self.vm.get_power_state() != 'Halted': self.loop.run_until_complete(asyncio.sleep(1)) # and give a chance for both domain-shutdown handlers to execute - self.loop.run_until_complete(asyncio.sleep(1)) + self.loop.run_until_complete(asyncio.sleep(3)) if self.test_failure_reason: self.fail(self.test_failure_reason) From 9887b925b428ca57fd04920d1d1ba5ee704fb00d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 14 Oct 2018 05:27:36 +0200 Subject: [PATCH 108/145] tests: increase timeout for vm shutdown start_standalone_with_cdrom_vm test essence is somewhere else, let not fail it for just slow shutdown (LVM cleanup etc). --- qubes/tests/integ/basic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qubes/tests/integ/basic.py b/qubes/tests/integ/basic.py index 354a39c4..fee89ab6 100644 --- a/qubes/tests/integ/basic.py +++ b/qubes/tests/integ/basic.py @@ -109,7 +109,7 @@ class TC_00_Basic(qubes.tests.SystemTestCase): # Type 'poweroff' subprocess.check_call(['xdotool', 'search', '--name', self.vm.name, 'type', 'poweroff\r']) - for _ in range(5): + for _ in range(10): if not self.vm.is_running(): break self.loop.run_until_complete(asyncio.sleep(1)) @@ -675,7 +675,7 @@ class TC_06_AppVMMixin(object): # Type 'poweroff' subprocess.check_call(['xdotool', 'search', '--name', self.vm.name, 'type', 'poweroff\r']) - for _ in range(5): + for _ in range(10): if not self.vm.is_running(): break self.loop.run_until_complete(asyncio.sleep(1)) From 3b6703f2bd6c449a9f1ebce3e7a39508e4bf963b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 14 Oct 2018 05:48:25 +0200 Subject: [PATCH 109/145] tests: fix race condition in gui_memory_pinning test Don't rely on top update timing, pause it updates for taking screenshots. --- qubes/tests/integ/vm_qrexec_gui.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qubes/tests/integ/vm_qrexec_gui.py b/qubes/tests/integ/vm_qrexec_gui.py index c93dd49a..a847a71d 100644 --- a/qubes/tests/integ/vm_qrexec_gui.py +++ b/qubes/tests/integ/vm_qrexec_gui.py @@ -1142,6 +1142,9 @@ int main(int argc, char **argv) { # wait for damage notify - top updates every 3 sec by default yield from asyncio.sleep(6) + # stop changing the window content + subprocess.check_call(['xdotool', 'key', '--window', winid, 'd']) + # now take screenshot of the window, from dom0 and VM # choose pnm format, as it doesn't have any useless metadata - easy # to compare From fc3b28608e9ac87c1359ec13bba2e8f035e75cb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 14 Oct 2018 06:05:04 +0200 Subject: [PATCH 110/145] version 4.0.32 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 6634c5d5..4d1ccce0 100644 --- a/version +++ b/version @@ -1 +1 @@ -4.0.31 +4.0.32 From 375688837c3849cdce81130e961ff4a01f59423d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 15 Oct 2018 00:30:17 +0200 Subject: [PATCH 111/145] tests/integ/network: add type annotations Make PyCharm understand what mixin those objects are for. --- qubes/tests/integ/network.py | 142 +++++++++++++++++++++++++++++++---- 1 file changed, 126 insertions(+), 16 deletions(-) diff --git a/qubes/tests/integ/network.py b/qubes/tests/integ/network.py index 30f4f32e..733c4b06 100644 --- a/qubes/tests/integ/network.py +++ b/qubes/tests/integ/network.py @@ -50,6 +50,9 @@ class VmNetworkingMixin(object): template = None def run_cmd(self, vm, cmd, user="root"): + ''' + :type self: qubes.tests.SystemTestCase | VmNetworkingMixin + ''' try: self.loop.run_until_complete(vm.run_for_stdio(cmd, user=user)) except subprocess.CalledProcessError as e: @@ -57,6 +60,9 @@ class VmNetworkingMixin(object): return 0 def check_nc_version(self, vm): + ''' + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' if self.run_cmd(vm, 'nc -h >/dev/null 2>&1') != 0: self.skipTest('nc not installed') if self.run_cmd(vm, 'nc -h 2>&1|grep -q nmap.org') == 0: @@ -65,6 +71,9 @@ class VmNetworkingMixin(object): return NcVersion.Trad def setUp(self): + ''' + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' super(VmNetworkingMixin, self).setUp() if self.template.startswith('whonix-'): self.skipTest("Test not supported here - Whonix uses its own " @@ -86,6 +95,9 @@ class VmNetworkingMixin(object): def configure_netvm(self): + ''' + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' def run_netvm_cmd(cmd): if self.run_cmd(self.testnetvm, cmd) != 0: self.fail("Command '%s' failed" % cmd) @@ -113,12 +125,18 @@ class VmNetworkingMixin(object): def test_000_simple_networking(self): + ''' + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.loop.run_until_complete(self.testvm1.start()) self.assertEqual(self.run_cmd(self.testvm1, self.ping_ip), 0) self.assertEqual(self.run_cmd(self.testvm1, self.ping_name), 0) def test_010_simple_proxyvm(self): + ''' + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=self.make_vm_name('proxy'), label='red') @@ -144,6 +162,9 @@ class VmNetworkingMixin(object): @unittest.skipUnless(spawn.find_executable('xdotool'), "xdotool not installed") def test_020_simple_proxyvm_nm(self): + ''' + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=self.make_vm_name('proxy'), label='red') @@ -189,6 +210,9 @@ class VmNetworkingMixin(object): def test_030_firewallvm_firewall(self): + ''' + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=self.make_vm_name('proxy'), label='red') @@ -290,6 +314,9 @@ class VmNetworkingMixin(object): def test_040_inter_vm(self): + ''' + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=self.make_vm_name('proxy'), label='red') @@ -327,7 +354,10 @@ class VmNetworkingMixin(object): self.ping_cmd.format(target=self.testvm1.ip)), 0) def test_050_spoof_ip(self): - """Test if VM IP spoofing is blocked""" + '''Test if VM IP spoofing is blocked + + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.loop.run_until_complete(self.testvm1.start()) self.assertEqual(self.run_cmd(self.testvm1, self.ping_ip), 0) @@ -353,7 +383,10 @@ class VmNetworkingMixin(object): self.assertEquals(packets, '0', 'Some packet hit the INPUT rule') def test_100_late_xldevd_startup(self): - """Regression test for #1990""" + '''Regression test for #1990 + + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' # Simulater late xl devd startup cmd = "systemctl stop xendriverdomain" if self.run_cmd(self.testnetvm, cmd) != 0: @@ -367,7 +400,10 @@ class VmNetworkingMixin(object): self.assertEqual(self.run_cmd(self.testvm1, self.ping_ip), 0) def test_200_fake_ip_simple(self): - '''Test hiding VM real IP''' + '''Test hiding VM real IP + + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.testvm1.features['net.fake-ip'] = '192.168.1.128' self.testvm1.features['net.fake-gateway'] = '192.168.1.1' self.testvm1.features['net.fake-netmask'] = '255.255.255.0' @@ -398,7 +434,10 @@ class VmNetworkingMixin(object): self.assertNotIn(str(self.testvm1.netvm.ip), output) def test_201_fake_ip_without_gw(self): - '''Test hiding VM real IP''' + '''Test hiding VM real IP + + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.testvm1.features['net.fake-ip'] = '192.168.1.128' self.app.save() self.loop.run_until_complete(self.testvm1.start()) @@ -417,7 +456,10 @@ class VmNetworkingMixin(object): self.assertNotIn(str(self.testvm1.ip), output) def test_202_fake_ip_firewall(self): - '''Test hiding VM real IP, firewall''' + '''Test hiding VM real IP, firewall + + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.testvm1.features['net.fake-ip'] = '192.168.1.128' self.testvm1.features['net.fake-gateway'] = '192.168.1.1' self.testvm1.features['net.fake-netmask'] = '255.255.255.0' @@ -468,7 +510,10 @@ class VmNetworkingMixin(object): self.loop.run_until_complete(nc.wait()) def test_203_fake_ip_inter_vm_allow(self): - '''Access VM with "fake IP" from other VM (when firewall allows)''' + '''Access VM with "fake IP" from other VM (when firewall allows) + + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=self.make_vm_name('proxy'), label='red') @@ -521,7 +566,10 @@ class VmNetworkingMixin(object): 'Packets didn\'t managed to the VM') def test_204_fake_ip_proxy(self): - '''Test hiding VM real IP''' + '''Test hiding VM real IP + + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=self.make_vm_name('proxy'), label='red') @@ -582,7 +630,10 @@ class VmNetworkingMixin(object): self.assertNotIn(str(self.proxy.ip), output) def test_210_custom_ip_simple(self): - '''Custom AppVM IP''' + '''Custom AppVM IP + + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.testvm1.ip = '192.168.1.1' self.app.save() self.loop.run_until_complete(self.testvm1.start()) @@ -590,7 +641,10 @@ class VmNetworkingMixin(object): self.assertEqual(self.run_cmd(self.testvm1, self.ping_name), 0) def test_211_custom_ip_proxy(self): - '''Custom ProxyVM IP''' + '''Custom ProxyVM IP + + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=self.make_vm_name('proxy'), label='red') @@ -607,7 +661,10 @@ class VmNetworkingMixin(object): self.assertEqual(self.run_cmd(self.testvm1, self.ping_name), 0) def test_212_custom_ip_firewall(self): - '''Custom VM IP and firewall''' + '''Custom VM IP and firewall + + :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + ''' self.testvm1.ip = '192.168.1.1' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, @@ -666,6 +723,9 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): self.ping6_name = self.ping6_cmd.format(target=self.test_name) def configure_netvm(self): + ''' + :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin + ''' self.testnetvm.features['ipv6'] = True super(VmIPv6NetworkingMixin, self).configure_netvm() @@ -683,12 +743,18 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): format(ip=self.test_ip, ip6=self.test_ip6, name=self.test_name)) def test_500_ipv6_simple_networking(self): + ''' + :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin + ''' self.loop.run_until_complete(self.testvm1.start()) self.assertEqual(self.run_cmd(self.testvm1, self.ping6_ip), 0) self.assertEqual(self.run_cmd(self.testvm1, self.ping6_name), 0) def test_510_ipv6_simple_proxyvm(self): + ''' + :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin + ''' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=self.make_vm_name('proxy'), label='red') @@ -714,6 +780,9 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): @unittest.skipUnless(spawn.find_executable('xdotool'), "xdotool not installed") def test_520_ipv6_simple_proxyvm_nm(self): + ''' + :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin + ''' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=self.make_vm_name('proxy'), label='red') @@ -764,6 +833,9 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): def test_530_ipv6_firewallvm_firewall(self): + ''' + :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin + ''' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=self.make_vm_name('proxy'), label='red') @@ -881,6 +953,9 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): def test_540_ipv6_inter_vm(self): + ''' + :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin + ''' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=self.make_vm_name('proxy'), label='red') @@ -920,7 +995,10 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): def test_550_ipv6_spoof_ip(self): - """Test if VM IP spoofing is blocked""" + '''Test if VM IP spoofing is blocked + + :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin + ''' self.loop.run_until_complete(self.testvm1.start()) self.assertEqual(self.run_cmd(self.testvm1, self.ping6_ip), 0) @@ -949,7 +1027,10 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): self.assertEquals(packets, '0', 'Some packet hit the INPUT rule') def test_710_ipv6_custom_ip_simple(self): - '''Custom AppVM IP''' + '''Custom AppVM IP + + :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin + ''' self.testvm1.ip6 = '2000:aaaa:bbbb::1' self.app.save() self.loop.run_until_complete(self.testvm1.start()) @@ -957,7 +1038,10 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): self.assertEqual(self.run_cmd(self.testvm1, self.ping6_name), 0) def test_711_ipv6_custom_ip_proxy(self): - '''Custom ProxyVM IP''' + '''Custom ProxyVM IP + + :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin + ''' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=self.make_vm_name('proxy'), label='red') @@ -974,7 +1058,10 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): self.assertEqual(self.run_cmd(self.testvm1, self.ping6_name), 0) def test_712_ipv6_custom_ip_firewall(self): - '''Custom VM IP and firewall''' + '''Custom VM IP and firewall + + :type self: qubes.tests.SystemTestCase | VmIPv6NetworkingMixin + ''' self.testvm1.ip6 = '2000:aaaa:bbbb::1' self.proxy = self.app.add_new_vm(qubes.vm.appvm.AppVM, @@ -1099,6 +1186,10 @@ class VmUpdatesMixin(object): ) def run_cmd(self, vm, cmd, user="root"): + ''' + + :type self: qubes.tests.SystemTestCase | VmUpdatesMixin + ''' try: self.loop.run_until_complete(vm.run_for_stdio(cmd)) except subprocess.CalledProcessError as e: @@ -1106,6 +1197,9 @@ class VmUpdatesMixin(object): return 0 def setUp(self): + ''' + :type self: qubes.tests.SystemTestCase | VmUpdatesMixin + ''' if not self.template.count('debian') and \ not self.template.count('fedora'): self.skipTest("Template {} not supported by this test".format( @@ -1142,6 +1236,9 @@ class VmUpdatesMixin(object): self.loop.run_until_complete(self.testvm1.create_on_disk()) def test_000_simple_update(self): + ''' + :type self: qubes.tests.SystemTestCase | VmUpdatesMixin + ''' self.app.save() # reload the VM to have all the properties properly set (especially # default netvm) @@ -1155,6 +1252,9 @@ class VmUpdatesMixin(object): '{}: {}\n{}'.format(self.update_cmd, stdout, stderr)) def create_repo_apt(self): + ''' + :type self: qubes.tests.SystemTestCase | VmUpdatesMixin + ''' pkg_file_name = "test-pkg_1.0-1_amd64.deb" self.loop.run_until_complete(self.netvm_repo.run_for_stdio(''' mkdir /tmp/apt-repo \ @@ -1209,6 +1309,9 @@ SHA256: ''')) def create_repo_yum(self): + ''' + :type self: qubes.tests.SystemTestCase | VmUpdatesMixin + ''' pkg_file_name = "test-pkg-1.0-1.fc21.x86_64.rpm" self.loop.run_until_complete(self.netvm_repo.run_for_stdio(''' mkdir /tmp/yum-repo \ @@ -1221,6 +1324,9 @@ SHA256: 'createrepo /tmp/yum-repo')) def create_repo_and_serve(self): + ''' + :type self: qubes.tests.SystemTestCase | VmUpdatesMixin + ''' if self.template.count("debian") or self.template.count("whonix"): self.create_repo_apt() self.loop.run_until_complete(self.netvm_repo.run( @@ -1242,6 +1348,8 @@ SHA256: The critical part is to use "localhost" - this will work only when accessed through update proxy and this is exactly what we want to test here. + + :type self: qubes.tests.SystemTestCase | VmUpdatesMixin """ if self.template.count("debian") or self.template.count("whonix"): @@ -1266,9 +1374,11 @@ SHA256: self.template)) def test_010_update_via_proxy(self): - """ + ''' Test both whether updates proxy works and whether is actually used by the VM - """ + + :type self: qubes.tests.SystemTestCase | VmUpdatesMixin + ''' if self.template.count("minimal"): self.skipTest("Template {} not supported by this test".format( self.template)) From 15140255d5244e9f3b5e0e9f31c573d23ec98546 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 15 Oct 2018 00:38:38 +0200 Subject: [PATCH 112/145] tests/integ/network: few more code style improvement Remove unused imports and unused variables, add some more docstrings. --- qubes/tests/integ/network.py | 40 +++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/qubes/tests/integ/network.py b/qubes/tests/integ/network.py index 733c4b06..980deec0 100644 --- a/qubes/tests/integ/network.py +++ b/qubes/tests/integ/network.py @@ -22,8 +22,6 @@ from distutils import spawn import asyncio -import multiprocessing -import os import subprocess import sys import time @@ -31,13 +29,15 @@ import unittest import qubes.tests import qubes.firewall +import qubes.vm.qubesvm import qubes.vm.appvm class NcVersion: Trad = 1 Nmap = 2 -# noinspection PyAttributeOutsideInit + +# noinspection PyAttributeOutsideInit,PyPep8Naming class VmNetworkingMixin(object): test_ip = '192.168.123.45' test_name = 'test.example.com' @@ -50,8 +50,12 @@ class VmNetworkingMixin(object): template = None def run_cmd(self, vm, cmd, user="root"): - ''' + '''Run a command *cmd* in a *vm* as *user*. Return its exit code. :type self: qubes.tests.SystemTestCase | VmNetworkingMixin + :param qubes.vm.qubesvm.QubesVM vm: VM object to run command in + :param str cmd: command to execute + :param std user: user to execute command as + :return int: command exit code ''' try: self.loop.run_until_complete(vm.run_for_stdio(cmd, user=user)) @@ -62,6 +66,7 @@ class VmNetworkingMixin(object): def check_nc_version(self, vm): ''' :type self: qubes.tests.SystemTestCase | VMNetworkingMixin + :param vm: VM where check ncat version in ''' if self.run_cmd(vm, 'nc -h >/dev/null 2>&1') != 0: self.skipTest('nc not installed') @@ -535,9 +540,9 @@ class VmNetworkingMixin(object): self.loop.run_until_complete(self.testvm1.start()) self.loop.run_until_complete(self.testvm2.start()) + cmd = 'iptables -I FORWARD -s {} -d {} -j ACCEPT'.format( + self.testvm2.ip, self.testvm1.ip) try: - cmd = 'iptables -I FORWARD -s {} -d {} -j ACCEPT'.format( - self.testvm2.ip, self.testvm1.ip) self.loop.run_until_complete(self.proxy.run_for_stdio( cmd, user='root')) except subprocess.CalledProcessError as e: @@ -593,7 +598,7 @@ class VmNetworkingMixin(object): (output, _) = self.loop.run_until_complete( self.proxy.run_for_stdio( 'ip addr show dev eth0', user='root')) - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError: self.fail('ip addr show dev eth0 failed') output = output.decode() self.assertIn('192.168.1.128', output) @@ -603,7 +608,7 @@ class VmNetworkingMixin(object): (output, _) = self.loop.run_until_complete( self.proxy.run_for_stdio( 'ip route show', user='root')) - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError: self.fail('ip route show failed') output = output.decode() self.assertIn('192.168.1.1', output) @@ -613,7 +618,7 @@ class VmNetworkingMixin(object): (output, _) = self.loop.run_until_complete( self.testvm1.run_for_stdio( 'ip addr show dev eth0', user='root')) - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError: self.fail('ip addr show dev eth0 failed') output = output.decode() self.assertNotIn('192.168.1.128', output) @@ -623,7 +628,7 @@ class VmNetworkingMixin(object): (output, _) = self.loop.run_until_complete( self.testvm1.run_for_stdio( 'ip route show', user='root')) - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError: self.fail('ip route show failed') output = output.decode() self.assertIn('192.168.1.128', output) @@ -712,6 +717,7 @@ class VmNetworkingMixin(object): nc.terminate() self.loop.run_until_complete(nc.wait()) +# noinspection PyAttributeOutsideInit,PyPep8Naming class VmIPv6NetworkingMixin(VmNetworkingMixin): test_ip6 = '2000:abcd::1' @@ -903,7 +909,8 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): # block all except target self.testvm1.firewall.rules = [ - qubes.firewall.Rule(None, action='accept', dsthost=self.test_ip6, + qubes.firewall.Rule(None, action='accept', + dsthost=self.test_ip6, proto='tcp', dstports=1234), ] self.testvm1.firewall.save() @@ -1109,7 +1116,7 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): nc.terminate() self.loop.run_until_complete(nc.wait()) -# noinspection PyAttributeOutsideInit +# noinspection PyAttributeOutsideInit,PyPep8Naming class VmUpdatesMixin(object): """ Tests for VM updates @@ -1186,9 +1193,13 @@ class VmUpdatesMixin(object): ) def run_cmd(self, vm, cmd, user="root"): - ''' + '''Run a command *cmd* in a *vm* as *user*. Return its exit code. :type self: qubes.tests.SystemTestCase | VmUpdatesMixin + :param qubes.vm.qubesvm.QubesVM vm: VM object to run command in + :param str cmd: command to execute + :param std user: user to execute command as + :return int: command exit code ''' try: self.loop.run_until_complete(vm.run_for_stdio(cmd)) @@ -1375,7 +1386,8 @@ SHA256: def test_010_update_via_proxy(self): ''' - Test both whether updates proxy works and whether is actually used by the VM + Test both whether updates proxy works and whether is actually used + by the VM :type self: qubes.tests.SystemTestCase | VmUpdatesMixin ''' From 3f5618dbb0aa7004252ba4239e8073280ec16c5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 15 Oct 2018 00:39:32 +0200 Subject: [PATCH 113/145] tests/integ/network: make the tests independend of default netvm Network tests create own temporary netvm. Make it disconnected from the real netvm, to not interefere in tests. --- qubes/tests/integ/network.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qubes/tests/integ/network.py b/qubes/tests/integ/network.py index 980deec0..524dd5b8 100644 --- a/qubes/tests/integ/network.py +++ b/qubes/tests/integ/network.py @@ -89,6 +89,7 @@ class VmNetworkingMixin(object): label='red') self.loop.run_until_complete(self.testnetvm.create_on_disk()) self.testnetvm.provides_network = True + self.testnetvm.netvm = None self.testvm1 = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=self.make_vm_name('vm1'), label='red') From f1621c01e907d339e8a3fbbd978d5f162fbf0b96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 15 Oct 2018 05:08:25 +0200 Subject: [PATCH 114/145] tests: add search based on window class to wait_for_window Searching based on class is used in many tests, searching by class, not only by name in wait_for_window will allow to reduce code duplication. While at it, improve it additionally: - avoid active waiting for window and use `xdotool search --sync` instead - return found window id - add wait_for_window_coro() for use where coroutine is needed - when waiting for window to disappear, check window id once and wait for that particular window to disappear (avoid xdotool race conditions on window enumeration) Besides reducing code duplication, this also move various xdotool imperfections handling into one place. --- qubes/tests/__init__.py | 82 +++++++++++++++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 12 deletions(-) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index 6bdf5977..c3f3a2fd 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -977,7 +977,25 @@ class SystemTestCase(QubesTestCase): return _QrexecPolicyContext(service, source, destination, allow=allow, action=action) - def wait_for_window(self, title, timeout=30, show=True): + @asyncio.coroutine + def wait_for_window_hide_coro(self, title, winid, timeout=30): + """ + Wait for window do disappear + :param winid: window id + :return: + """ + wait_count = 0 + while subprocess.call(['xdotool', 'getwindowname', str(winid)], + stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) == 0: + wait_count += 1 + if wait_count > timeout * 10: + self.fail("Timeout while waiting for {}({}) window to " + "disappear".format(title, winid)) + yield from asyncio.sleep(0.1) + + @asyncio.coroutine + def wait_for_window_coro(self, title, search_class=False, timeout=30, + show=True): """ Wait for a window with a given title. Depending on show parameter, it will wait for either window to show or to disappear. @@ -986,19 +1004,59 @@ class SystemTestCase(QubesTestCase): :param timeout: timeout of the operation, in seconds :param show: if True - wait for the window to be visible, otherwise - to not be visible - :return: None + :param search_class: search based on window class instead of title + :return: window id of found window, if show=True """ - wait_count = 0 - while subprocess.call(['xdotool', 'search', '--name', title], - stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) \ - != int(not show): - wait_count += 1 - if wait_count > timeout*10: - self.fail("Timeout while waiting for {} window to {}".format( - title, "show" if show else "hide") - ) - self.loop.run_until_complete(asyncio.sleep(0.1)) + xdotool_search = ['xdotool', 'search', '--onlyvisible'] + if search_class: + xdotool_search.append('--class') + else: + xdotool_search.append('--name') + if show: + xdotool_search.append('--sync') + if not show: + try: + winid = subprocess.check_output(xdotool_search + [title], + stderr=subprocess.DEVNULL).decode() + except subprocess.CalledProcessError: + # already gone + return + yield from self.wait_for_window_hide_coro(winid, title, + timeout=timeout) + return + + winid = None + while not winid: + p = yield from asyncio.create_subprocess_exec( + *xdotool_search, title, + stderr=subprocess.DEVNULL, stdout=subprocess.PIPE) + try: + (winid, _) = yield from asyncio.wait_for( + p.communicate(), timeout) + # don't check exit code, getting winid on stdout is enough + # indicator of success; specifically ignore xdotool failing + # with BadWindow or such - when some window appears only for a + # moment by xdotool didn't manage to get its properties + except asyncio.TimeoutError: + self.fail( + "Timeout while waiting for {} window to show".format(title)) + return winid.decode().strip() + + def wait_for_window(self, *args, **kwargs): + """ + Wait for a window with a given title. Depending on show parameter, + it will wait for either window to show or to disappear. + + :param title: title of the window to wait for + :param timeout: timeout of the operation, in seconds + :param show: if True - wait for the window to be visible, + otherwise - to not be visible + :param search_class: search based on window class instead of title + :return: window id of found window, if show=True + """ + return self.loop.run_until_complete( + self.wait_for_window_coro(*args, **kwargs)) def enter_keys_in_window(self, title, keys): """ From 9e81087b252bdd194ff6fb6b1010744f39194706 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 15 Oct 2018 05:16:29 +0200 Subject: [PATCH 115/145] tests: use improved wait_for_window in various tests Replace manual `xdotool search calls` with wait_for_window(), where compatible. --- qubes/tests/integ/dispvm.py | 46 ++++++++++-------------- qubes/tests/integ/vm_qrexec_gui.py | 58 +++++------------------------- 2 files changed, 27 insertions(+), 77 deletions(-) diff --git a/qubes/tests/integ/dispvm.py b/qubes/tests/integ/dispvm.py index af6f4f7b..77c47cd3 100644 --- a/qubes/tests/integ/dispvm.py +++ b/qubes/tests/integ/dispvm.py @@ -255,34 +255,26 @@ class TC_20_DispVMMixin(object): p = self.loop.run_until_complete( self.testvm1.run("qvm-open-in-dvm /home/user/test.txt")) - wait_count = 0 + # if first 5 windows isn't expected editor, there is no hope winid = None - while True: - search = self.loop.run_until_complete( - asyncio.create_subprocess_exec( - 'xdotool', 'search', '--onlyvisible', '--class', - 'disp[0-9]*', - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL)) - stdout, _ = self.loop.run_until_complete(search.communicate()) - if search.returncode == 0: - winid = stdout.strip() - # get window title - (window_title, _) = subprocess.Popen( - ['xdotool', 'getwindowname', winid], stdout=subprocess.PIPE). \ - communicate() - window_title = window_title.decode().strip() - # ignore LibreOffice splash screen and window with no title - # set yet - if window_title and not window_title.startswith("LibreOffice")\ - and not window_title == 'VMapp command' \ - and 'whonixcheck' not in window_title \ - and not window_title == 'NetworkManager Applet': - break - wait_count += 1 - if wait_count > 100: - self.fail("Timeout while waiting for editor window") - self.loop.run_until_complete(asyncio.sleep(0.3)) + for _ in range(5): + winid = self.wait_for_window('disp[0-9]*', search_class=True) + # get window title + (window_title, _) = subprocess.Popen( + ['xdotool', 'getwindowname', winid], stdout=subprocess.PIPE). \ + communicate() + window_title = window_title.decode().strip() + # ignore LibreOffice splash screen and window with no title + # set yet + if window_title and not window_title.startswith("LibreOffice")\ + and not window_title == 'VMapp command' \ + and 'whonixcheck' not in window_title \ + and not window_title == 'NetworkManager Applet': + break + self.loop.run_until_complete(asyncio.sleep(1)) + winid = None + if winid is None: + self.fail('Timeout waiting for editor window') time.sleep(0.5) self._handle_editor(winid) diff --git a/qubes/tests/integ/vm_qrexec_gui.py b/qubes/tests/integ/vm_qrexec_gui.py index a847a71d..7b87e786 100644 --- a/qubes/tests/integ/vm_qrexec_gui.py +++ b/qubes/tests/integ/vm_qrexec_gui.py @@ -76,32 +76,17 @@ class TC_00_AppVMMixin(object): self.loop.run_until_complete(self.wait_for_session(self.testvm1)) p = self.loop.run_until_complete(self.testvm1.run('xterm')) try: - wait_count = 0 title = 'user@{}'.format(self.testvm1.name) if self.template.count("whonix"): title = 'user@host' - while subprocess.call( - ['xdotool', 'search', '--name', title], - stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) > 0: - wait_count += 1 - if wait_count > 100: - self.fail("Timeout while waiting for xterm window") - self.loop.run_until_complete(asyncio.sleep(0.1)) + self.wait_for_window(title) self.loop.run_until_complete(asyncio.sleep(0.5)) subprocess.check_call( ['xdotool', 'search', '--name', title, 'windowactivate', 'type', 'exit\n']) - wait_count = 0 - while subprocess.call(['xdotool', 'search', '--name', title], - stdout=open(os.path.devnull, 'w'), - stderr=subprocess.STDOUT) == 0: - wait_count += 1 - if wait_count > 100: - self.fail("Timeout while waiting for xterm " - "termination") - self.loop.run_until_complete(asyncio.sleep(0.1)) + self.wait_for_window(title, show=False) finally: try: p.terminate() @@ -124,15 +109,7 @@ class TC_00_AppVMMixin(object): title = 'user@{}'.format(self.testvm1.name) if self.template.count("whonix"): title = 'user@host' - wait_count = 0 - while subprocess.call( - ['xdotool', 'search', '--name', title], - stdout=open(os.path.devnull, 'w'), - stderr=subprocess.STDOUT) > 0: - wait_count += 1 - if wait_count > 100: - self.fail("Timeout while waiting for gnome-terminal window") - self.loop.run_until_complete(asyncio.sleep(0.1)) + self.wait_for_window(title) self.loop.run_until_complete(asyncio.sleep(0.5)) subprocess.check_call( @@ -178,30 +155,14 @@ class TC_00_AppVMMixin(object): title = 'user@{}'.format(self.testvm1.name) if self.template.count("whonix"): title = 'user@host' - wait_count = 0 - while subprocess.call( - ['xdotool', 'search', '--name', title], - stdout=open(os.path.devnull, 'w'), - stderr=subprocess.STDOUT) > 0: - wait_count += 1 - if wait_count > 100: - self.fail("Timeout while waiting for xterm window") - self.loop.run_until_complete(asyncio.sleep(0.1)) + self.wait_for_window(title) self.loop.run_until_complete(asyncio.sleep(0.5)) subprocess.check_call( ['xdotool', 'search', '--name', title, 'windowactivate', '--sync', 'type', 'exit\n']) - wait_count = 0 - while subprocess.call(['xdotool', 'search', '--name', title], - stdout=open(os.path.devnull, 'w'), - stderr=subprocess.STDOUT) == 0: - wait_count += 1 - if wait_count > 100: - self.fail("Timeout while waiting for xterm " - "termination") - self.loop.run_until_complete(asyncio.sleep(0.1)) + self.wait_for_window(title, show=False) def test_050_qrexec_simple_eof(self): """Test for data and EOF transmission dom0->VM""" @@ -1111,15 +1072,12 @@ int main(int argc, char **argv) { proc = yield from self.testvm1.run( 'xterm -maximized -e top') - # help xdotool a little... - yield from asyncio.sleep(2) if proc.returncode is not None: self.fail('xterm failed to start') # get window ID - winid = (yield from asyncio.get_event_loop().run_in_executor(None, - subprocess.check_output, - ['xdotool', 'search', '--sync', '--onlyvisible', '--class', - self.testvm1.name + ':xterm'])).decode() + winid = yield from self.wait_for_window_coro( + self.testvm1.name + ':xterm', + search_class=True) xprop = yield from asyncio.get_event_loop().run_in_executor(None, subprocess.check_output, ['xprop', '-notype', '-id', winid, '_QUBES_VMWINDOWID']) From fd9f2e2a6c7360bab0391dc2b2ba58e3006cb6b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 15 Oct 2018 05:17:19 +0200 Subject: [PATCH 116/145] tests: type commands into specific found window Make sure events are sent to specific window found with xdotool search, not the one having the focus. In case of Whonix, it can be first connection wizard or whonixcheck report. --- qubes/tests/integ/basic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qubes/tests/integ/basic.py b/qubes/tests/integ/basic.py index fee89ab6..6d535e7e 100644 --- a/qubes/tests/integ/basic.py +++ b/qubes/tests/integ/basic.py @@ -108,7 +108,7 @@ class TC_00_Basic(qubes.tests.SystemTestCase): self.assertTrue(self.vm.is_running()) # Type 'poweroff' subprocess.check_call(['xdotool', 'search', '--name', self.vm.name, - 'type', 'poweroff\r']) + 'type', '--window', '%1', 'poweroff\r']) for _ in range(10): if not self.vm.is_running(): break @@ -674,7 +674,7 @@ class TC_06_AppVMMixin(object): self.assertTrue(self.vm.is_running()) # Type 'poweroff' subprocess.check_call(['xdotool', 'search', '--name', self.vm.name, - 'type', 'poweroff\r']) + 'type', '--window', '%1', 'poweroff\r']) for _ in range(10): if not self.vm.is_running(): break From e8dc6cb916f7828c34e71a135e7ada7586f52ffa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 15 Oct 2018 05:24:24 +0200 Subject: [PATCH 117/145] tests: use smaller root.img in backupcompatibility tests 1GB image easily exceed available space on openQA instances. Use 100MB instead. --- qubes/tests/integ/backupcompatibility.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes/tests/integ/backupcompatibility.py b/qubes/tests/integ/backupcompatibility.py index c7a7f6c4..88c3708a 100644 --- a/qubes/tests/integ/backupcompatibility.py +++ b/qubes/tests/integ/backupcompatibility.py @@ -237,7 +237,7 @@ class TC_00_BackupCompatibility( self.create_sparse(self.fullpath( "vm-templates/test-template-clone/root.img"), 10*2**30) self.fill_image(self.fullpath( - "vm-templates/test-template-clone/root.img"), 1*2**30, True) + "vm-templates/test-template-clone/root.img"), 100*2**20, True) self.create_volatile_img(self.fullpath( "vm-templates/test-template-clone/volatile.img")) subprocess.check_call([ From 133219f6d390b9bb2fdf3831951d38a4ea4911c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 15 Oct 2018 06:05:05 +0200 Subject: [PATCH 118/145] Do not generate R3 compat firewall rules if R4 format is supported R3 format had limitation of ~40 rules per VM. Do not generate compat rules (possibly hitting that limitation) if new format, free of that limitation is supported. Fixes QubesOS/qubes-issues#1570 Fixes QubesOS/qubes-issues#4228 --- qubes/ext/r3compatibility.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qubes/ext/r3compatibility.py b/qubes/ext/r3compatibility.py index acc1dd08..2fa8ec3a 100644 --- a/qubes/ext/r3compatibility.py +++ b/qubes/ext/r3compatibility.py @@ -80,6 +80,9 @@ class R3Compatibility(qubes.ext.Extension): def write_iptables_qubesdb_entry(self, firewallvm): # pylint: disable=no-self-use + # skip compatibility rules if new format support is advertised + if firewallvm.features.check_with_template('qubes-firewall', False): + return firewallvm.untrusted_qdb.rm("/qubes-iptables-domainrules/") iptables = "# Generated by Qubes Core on {0}\n".format( datetime.datetime.now().ctime()) From c01ae06feee4c933aefd7d282afd5fcfccaafad3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 17 Oct 2018 17:37:02 +0200 Subject: [PATCH 119/145] tests: add basic ServicesExtension tests --- qubes/tests/ext.py | 55 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/qubes/tests/ext.py b/qubes/tests/ext.py index bb937f68..ef4e7252 100644 --- a/qubes/tests/ext.py +++ b/qubes/tests/ext.py @@ -21,6 +21,7 @@ from unittest import mock import qubes.ext.core_features +import qubes.ext.services import qubes.ext.windows import qubes.tests @@ -214,3 +215,57 @@ class TC_10_WindowsFeatures(qubes.tests.QubesTestCase): 'qrexec': '1', 'os': 'Linux'}) self.assertEqual(self.vm.mock_calls, []) + +class TC_20_Services(qubes.tests.QubesTestCase): + def setUp(self): + super().setUp() + self.ext = qubes.ext.services.ServicesExtension() + self.vm = mock.MagicMock() + self.features = {} + self.vm.configure_mock(**{ + 'is_running.return_value': True, + 'features.get.side_effect': self.features.get, + 'features.items.side_effect': self.features.items, + 'features.__iter__.side_effect': self.features.__iter__, + 'features.__contains__.side_effect': self.features.__contains__, + 'features.__setitem__.side_effect': self.features.__setitem__, + 'features.__delitem__.side_effect': self.features.__delitem__, + }) + + def test_000_write_to_qdb(self): + self.features['service.test1'] = '1' + self.features['service.test2'] = '' + + self.ext.on_domain_qdb_create(self.vm, 'domain-qdb-create') + self.assertEqual(sorted(self.vm.untrusted_qdb.mock_calls), [ + ('write', ('/qubes-service/test1', '1'), {}), + ('write', ('/qubes-service/test2', '0'), {}), + ]) + + def test_001_feature_set(self): + self.ext.on_domain_feature_set(self.vm, + 'feature-set:service.test_no_oldvalue', + 'service.test_no_oldvalue', '1') + self.ext.on_domain_feature_set(self.vm, + 'feature-set:service.test_oldvalue', + 'service.test_oldvalue', '1', '') + self.ext.on_domain_feature_set(self.vm, + 'feature-set:service.test_disable', + 'service.test_disable', '', '1') + self.ext.on_domain_feature_set(self.vm, + 'feature-set:service.test_disable_no_oldvalue', + 'service.test_disable_no_oldvalue', '') + + self.assertEqual(sorted(self.vm.untrusted_qdb.mock_calls), sorted([ + ('write', ('/qubes-service/test_no_oldvalue', '1'), {}), + ('write', ('/qubes-service/test_oldvalue', '1'), {}), + ('write', ('/qubes-service/test_disable', '0'), {}), + ('write', ('/qubes-service/test_disable_no_oldvalue', '0'), {}), + ])) + + def test_002_feature_delete(self): + self.ext.on_domain_feature_delete(self.vm, + 'feature-delete:service.test3', 'service.test3') + self.assertEqual(sorted(self.vm.untrusted_qdb.mock_calls), [ + ('rm', ('/qubes-service/test3',), {}), + ]) From ba210c41ee644a31fa605266bc06880b49301274 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 18 Oct 2018 00:01:45 +0200 Subject: [PATCH 120/145] qubesvm: don't crash VM creation if icon symlink already exists It can be leftover from previous failed attempt. Don't crash on it, and replace it instead. QubesOS/qubes-issues#3438 --- qubes/vm/qubesvm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 7955297e..165df53e 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -1439,6 +1439,8 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): 'creation'.format(self.dir_path)) raise + if os.path.exists(self.icon_path): + os.unlink(self.icon_path) self.log.info('Creating icon symlink: {} -> {}'.format( self.icon_path, self.label.icon_path)) if hasattr(os, "symlink"): From 58bcec2a64e5c49bf42633776f4502a5615dfee8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 18 Oct 2018 00:03:05 +0200 Subject: [PATCH 121/145] qubesvm: improve error message about same-pool requirement Make it clear that volume creation fails because it needs to be in the same pool as its parent. This message is shown in context of `qvm-create -p root=MyPool` for example and the previous message didn't make sense at all. Fixes QubesOS/qubes-issues#3438 --- qubes/vm/qubesvm.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 165df53e..7b023776 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -1990,7 +1990,9 @@ def _patch_pool_config(config, pool=None, pools=None): if not is_snapshot: config['pool'] = str(pools[name]) else: - msg = "Can't clone a snapshot volume {!s} to pool {!s} " \ + msg = "Snapshot volume {0!s} must be in the same pool as its " \ + "origin ({0!s} volume of template)," \ + "cannot move to pool {1!s} " \ .format(name, pools[name]) raise qubes.exc.QubesException(msg) return config From d1f5cb5d15d7cd39bcffec9d0cb743d75b2df470 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 18 Oct 2018 05:44:08 +0200 Subject: [PATCH 122/145] ext/services: mechanism for advertising supported services Support 'supported-service.*' features requests coming from VMs. Set such features directly (allow only value '1') and remove any not reported in given call. This way uninstalling package providing given service will automatically remove related 'supported-service...' feature. Fixes QubesOS/qubes-issues#4402 --- qubes/ext/services.py | 37 +++++++++++++++++++++++++++++++++++++ qubes/tests/ext.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/qubes/ext/services.py b/qubes/ext/services.py index 77e94cdb..b09e2fda 100644 --- a/qubes/ext/services.py +++ b/qubes/ext/services.py @@ -62,3 +62,40 @@ class ServicesExtension(qubes.ext.Extension): return service = feature[len('service.'):] vm.untrusted_qdb.rm('/qubes-service/{}'.format(service)) + + @qubes.ext.handler('features-request') + def supported_services(self, vm, event, untrusted_features): + '''Handle advertisement of supported services''' + # pylint: disable=no-self-use,unused-argument + + if getattr(vm, 'template', None): + vm.log.warning( + 'Ignoring qubes.FeaturesRequest from template-based VM') + return + + new_supported_services = set() + for requested_service in untrusted_features: + if not requested_service.startswith('supported-service.'): + continue + if untrusted_features[requested_service] == '1': + # only allow to advertise service as supported, lack of entry + # means service is not supported + new_supported_services.add(requested_service) + del untrusted_features + + # if no service is supported, ignore the whole thing - do not clear + # all services in case of empty request (manual or such) + if not new_supported_services: + return + + old_supported_services = set( + feat for feat in vm.features + if feat.startswith('supported-service.') and vm.features[feat]) + + for feature in new_supported_services.difference( + old_supported_services): + vm.features[feature] = True + + for feature in old_supported_services.difference( + new_supported_services): + del vm.features[feature] diff --git a/qubes/tests/ext.py b/qubes/tests/ext.py index ef4e7252..5acd183f 100644 --- a/qubes/tests/ext.py +++ b/qubes/tests/ext.py @@ -223,6 +223,7 @@ class TC_20_Services(qubes.tests.QubesTestCase): self.vm = mock.MagicMock() self.features = {} self.vm.configure_mock(**{ + 'template': None, 'is_running.return_value': True, 'features.get.side_effect': self.features.get, 'features.items.side_effect': self.features.items, @@ -269,3 +270,38 @@ class TC_20_Services(qubes.tests.QubesTestCase): self.assertEqual(sorted(self.vm.untrusted_qdb.mock_calls), [ ('rm', ('/qubes-service/test3',), {}), ]) + + def test_010_supported_services(self): + self.ext.supported_services(self.vm, 'features-request', + untrusted_features={ + 'supported-service.test1': '1', # ok + 'supported-service.test2': '0', # ignored + 'supported-service.test3': 'some text', # ignored + 'no-service': '1', # ignored + }) + self.assertEqual(self.features, { + 'supported-service.test1': True, + }) + + def test_011_supported_services_add(self): + self.features['supported-service.test1'] = '1' + self.ext.supported_services(self.vm, 'features-request', + untrusted_features={ + 'supported-service.test1': '1', # ok + 'supported-service.test2': '1', # ok + }) + # also check if existing one is untouched + self.assertEqual(self.features, { + 'supported-service.test1': '1', + 'supported-service.test2': True, + }) + + def test_012_supported_services_remove(self): + self.features['supported-service.test1'] = '1' + self.ext.supported_services(self.vm, 'features-request', + untrusted_features={ + 'supported-service.test2': '1', # ok + }) + self.assertEqual(self.features, { + 'supported-service.test2': True, + }) From 295705a708b0f46fa514ccc4053dba927dd368cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 18 Oct 2018 06:30:38 +0200 Subject: [PATCH 123/145] doc: document features, qvm-features-request and services Fixes QubesOS/qubes-issues#2829 --- doc/index.rst | 1 + doc/qubes-features.rst | 193 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 194 insertions(+) create mode 100644 doc/qubes-features.rst diff --git a/doc/index.rst b/doc/index.rst index 5d0e71d4..b375f67b 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -16,6 +16,7 @@ manpages and API documentation. For primary user documentation, see qubes qubes-vm/index qubes-events + qubes-features qubes-storage qubes-exc qubes-ext diff --git a/doc/qubes-features.rst b/doc/qubes-features.rst new file mode 100644 index 00000000..a334cff9 --- /dev/null +++ b/doc/qubes-features.rst @@ -0,0 +1,193 @@ +:py:class:`qubes.vm.Features` - Qubes VM features, services +============================================================ + +Features are generic mechanism for storing key-value pairs attached to a +VM. The primary use case for them is data storage for extensions (you can think +of them as more flexible properties, defined by extensions), but some are also +used in the qubes core itself. There is no definite list of supported features, +each extension can set their own and there is no requirement of registration, +but :program:`qvm-features` man page contains well known ones. +In addition, there is a mechanism for VM request setting a feature. This is +useful for extensions to discover if its VM part is present. + +Features can have three distinct values: no value (not present in mapping, +which is closest thing to :py:obj:`None`), empty string (which is +interpreted as :py:obj:`False`) and non-empty string, which is +:py:obj:`True`. Anything assigned to the mapping is coerced to strings, +however if you assign instances of :py:class:`bool`, they are converted as +described above. Be aware that assigning the number `0` (which is considered +false in Python) will result in string `'0'`, which is considered true. + +:py:class:`qubes.vm.Features` inherits from :py:class:`dict`, so provide all the +standard functions to get, list and set values. Additionally provide helper +functions to check if given feature is set on the VM and default to the value +on the VM's template or netvm. This is useful for features which nature is +inherited from other VMs, like "is package X is installed" or "is VM behind a +VPN". + +Example usage of features in extension: + +.. code-block:: python + + import qubes.exc + import qubes.ext + + class ExampleExtension(qubes.ext.Extension): + @qubes.ext.handler('domain-pre-start') + def on_domain_start(self, vm, event, **kwargs): + if vm.features.get('do-not-start', False): + raise qubes.exc.QubesVMError(vm, + 'Start prohibited because of do-not-start feature') + + if vm.features.check_with_template('something-installed', False): + # do something + +The above extension does two things: + + - prevent starting a qube with ``do-not-start`` feature set + - do something when ``something-installed`` feature is set on the qube, or its template + + +qvm-features-request, qubes.PostInstall service +------------------------------------------------ + +When some package in the VM want to request feature to be set (aka advertise +support for it), it should place a shell script in ``/etc/qubes/post-install.d``. +This script should call :program:`qvm-features-request` with ``FEATURE=VALUE`` pair(s) as +arguments to request those features. It is recommended to use very simple +values here (for example ``1``). The script should be named in form +``XX-package-name.sh`` where ``XX`` is two-digits number below 90 and +``package-name`` is unique name specific to this package (preferably actual +package name). The script needs executable bit set. + +``qubes.PostInstall`` service will call all those scripts after any package +installation and also after initial template installation. +This way package have a chance to report to dom0 if any feature is +added/removed. + +The features flow to dom0 according to the diagram below. Important part is +that qubes core :py:class:`qubes.ext.Extension` is responsible for handling such request in +``features-request`` event handler. If no extension handles given feature request, +it will be ignored. The extension should carefuly validate requested +features (ignoring those not recognized - may be for another extension) and +only then set appropriate value on VM object +(:py:attr:`qubes.vm.BaseVM.features`). It is recommended to make the +verification code as bulletproof as possible (for example allow only specific +simple values, instead of complex structures), because feature requests +come from untrusted sources. The features actually set on the VM in some cases +may not be necessary those requested. Similar for values. + +.. graphviz:: + + digraph { + + "qubes.PostInstall"; + "/etc/qubes/post-install.d/ scripts"; + "qvm-features-request"; + "qubes.FeaturesRequest"; + "qubes core extensions"; + "VM features"; + + "qubes.PostInstall" -> "/etc/qubes/post-install.d/ scripts"; + "/etc/qubes/post-install.d/ scripts" -> "qvm-features-request" + [xlabel="each script calls"]; + "qvm-features-request" -> "qubes.FeaturesRequest" + [xlabel="last script call the service to dom0"]; + "qubes.FeaturesRequest" -> "qubes core extensions" + [xlabel="features-request event"]; + "qubes core extensions" -> "VM features" + [xlabel="verification"]; + + } + +Example ``/etc/qubes/post-install.d/20-example.sh`` file: + +.. code-block:: shell + + #!/bin/sh + + qvm-features-request example-feature=1 + +Example extension handling the above: + +.. code-block:: python + + import qubes.ext + + class ExampleExtension(qubes.ext.Extension): + # the last argument must be named untrusted_features + @qubes.ext.handler('features-request') + def on_features_request(self, vm, event, untrusted_features): + # don't allow TemplateBasedVMs to request the feature - should be + # requested by the template instead + if hasattr(vm, 'template'): + return + + untrusted_value = untrusted_features.get('example-feature', None) + # check if feature is advertised and verify its value + if untrusted_value != '1': + return + value = untrusted_value + + # and finally set the value + vm.features['example-feature'] = value + +Services +--------- + +`Qubes services `_ are implemented +as features with ``service.`` prefix. The +:py:class:`qubes.ext.services.ServicesExtension` enumerate all the features +in form of ``service.`` prefix and write them to QubesDB as +``/qubes-service/`` and value either ``0`` or ``1``. +VM startup scripts list those entries for for each with value of ``1``, create +``/var/run/qubes-service/`` file. Then, it can be conveniently +used by other scripts to check whether dom0 wishes service to be enabled or +disabled. + +VM package can advertise what services are supported. For that, it needs to +request ``supported-service.`` feature with value ``1`` according +to description above. The :py:class:`qubes.ext.services.ServicesExtension` will +handle such request and set this feature on VM object. ``supported-service.`` +features that stop being advertised with ``qvm-features-request`` call are +removed. This way, it's enough to remove the file from +``/etc/qubes/post-install.d`` (for example by uninstalling package providing +the service) to tell dom0 the service is no longer supported. Services +advertised by TemplateBasedVMs are currently ignored (related +``supported-service.`` features are not set), but retrieving them may be added +in the future. Applications checking for specific service support should use +``vm.features.check_with_template('supported-service.', False)`` +call on desired VM object. When enumerating all supported services, application +should consider both the vm and its template (if any). + +Various tools will use this information to discover if given service is +supported. The API does not enforce service being first advertised before being +enabled (means: there can be service which is enabled, but without matching +``supported-service.`` feature). The list of well known services is in +:program:`qvm-service` man page. + +Example ``/etc/qubes/post-install.d/20-my-service.sh``: + +.. code-block:: shell + + #!/bin/sh + + qvm-features-request supported-service.my-service=1 + +Services and features can be then inspected from dom0 using +:program:`qvm-features` tool, for example: + +.. code-block:: shell + + $ qvm-features my-qube + supported-service.my-service 1 + +Module contents +--------------- + +.. autoclass:: qubes.vm.Features + :members: + :show-inheritance: + +.. vim: ts=3 sw=3 et + From 6170edb291fa016ba978fd97a4317999c2736b7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 19 Oct 2018 01:29:03 +0200 Subject: [PATCH 124/145] storage: allow import_data and import_data_end be coroutines On some storage pools this operation can also be time consuming - for example require creating temporary volume, and volume.create() already can be a coroutine. This is also requirement for making common code used by start()/create() etc be a coroutine, otherwise neither of them can be and will block other operations. Related to QubesOS/qubes-issues#4283 --- qubes/api/admin.py | 2 +- qubes/api/internal.py | 3 ++- qubes/storage/__init__.py | 22 ++++++++++++++++++---- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/qubes/api/admin.py b/qubes/api/admin.py index 60e0fa03..5a2b765e 100644 --- a/qubes/api/admin.py +++ b/qubes/api/admin.py @@ -479,7 +479,7 @@ class QubesAdminAPI(qubes.api.AbstractQubesAPI): if not self.dest.is_halted(): raise qubes.exc.QubesVMNotHaltedError(self.dest) - path = self.dest.storage.import_data(self.arg) + path = yield from self.dest.storage.import_data(self.arg) assert ' ' not in path size = self.dest.volumes[self.arg].size diff --git a/qubes/api/internal.py b/qubes/api/internal.py index 0773a98f..3af5848f 100644 --- a/qubes/api/internal.py +++ b/qubes/api/internal.py @@ -68,7 +68,8 @@ class QubesInternalAPI(qubes.api.AbstractQubesAPI): success = untrusted_payload == b'ok' try: - self.dest.storage.import_data_end(self.arg, success=success) + yield from self.dest.storage.import_data_end(self.arg, + success=success) except: self.dest.fire_event('domain-volume-import-end', volume=self.arg, success=False) diff --git a/qubes/storage/__init__.py b/qubes/storage/__init__.py index c81832a9..2a6afaa3 100644 --- a/qubes/storage/__init__.py +++ b/qubes/storage/__init__.py @@ -198,6 +198,8 @@ class Volume: volume data require something more than just writing to a file ( for example connecting to some other domain, or converting data on the fly), the returned path may be a pipe. + + This can be implemented as a coroutine. ''' raise self._not_implemented("import") @@ -207,6 +209,8 @@ class Volume: This method is called regardless the operation was successful or not. + This can be implemented as a coroutine. + :param success: True if data import was successful, otherwise False ''' # by default do nothing @@ -654,24 +658,34 @@ class Storage: return self.vm.volumes[volume].export() + @asyncio.coroutine def import_data(self, volume): ''' Helper function to import volume data (pool.import_data(volume))''' assert isinstance(volume, (Volume, str)), \ "You need to pass a Volume or pool name as str" if isinstance(volume, Volume): - return volume.import_data() + ret = volume.import_data() + else: + ret = self.vm.volumes[volume].import_data() - return self.vm.volumes[volume].import_data() + if asyncio.iscoroutine(ret): + ret = yield from ret + return ret + @asyncio.coroutine def import_data_end(self, volume, success): ''' Helper function to finish/cleanup data import (pool.import_data_end( volume))''' assert isinstance(volume, (Volume, str)), \ "You need to pass a Volume or pool name as str" if isinstance(volume, Volume): - return volume.import_data_end(success=success) + ret = volume.import_data_end(success=success) + else: + ret = self.vm.volumes[volume].import_data_end(success=success) - return self.vm.volumes[volume].import_data_end(success=success) + if asyncio.iscoroutine(ret): + ret = yield from ret + return ret class VolumesCollection: From 299c5146474fd654cf152bb7a913f7c97642f47e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 19 Oct 2018 02:28:09 +0200 Subject: [PATCH 125/145] tests: fix asyncio usage in storage_lvm.TC_01_ThinPool Both vm.create_on_disk() and vm.start() are coroutines. Tests in this class didn't run them, so basically didn't test anything. Wrap couroutine calls with self.loop.run_until_complete(). Additionally, don't fail if LVM pool is named differently. In that case, the test is rather sily, as it probably use the same pool for source and destination (operation already tested elsewhere). But it isn't a reason for failing the test. --- qubes/tests/storage_lvm.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/qubes/tests/storage_lvm.py b/qubes/tests/storage_lvm.py index f6b67d5b..3bead18e 100644 --- a/qubes/tests/storage_lvm.py +++ b/qubes/tests/storage_lvm.py @@ -946,26 +946,27 @@ class TC_01_ThinPool(ThinPoolBase, qubes.tests.SystemTestCase): vm = self.app.add_new_vm(qubes.vm.templatevm.TemplateVM, name=name, label='red') vm.clone_properties(template_vm) - vm.clone_disk_files(template_vm, pool='test-lvm') + self.loop.run_until_complete( + vm.clone_disk_files(template_vm, pool=self.pool.name)) for v_name, volume in vm.volumes.items(): if volume.save_on_stop: expected = "/dev/{!s}/vm-{!s}-{!s}".format( DEFAULT_LVM_POOL.split('/')[0], vm.name, v_name) self.assertEqual(volume.path, expected) with self.assertNotRaises(qubes.exc.QubesException): - vm.start() + self.loop.run_until_complete(vm.start()) def test_005_create_appvm(self): vm = self.app.add_new_vm(cls=qubes.vm.appvm.AppVM, name=self.make_vm_name('appvm'), label='red') - vm.create_on_disk(pool='test-lvm') + self.loop.run_until_complete(vm.create_on_disk(pool=self.pool.name)) for v_name, volume in vm.volumes.items(): if volume.save_on_stop: expected = "/dev/{!s}/vm-{!s}-{!s}".format( DEFAULT_LVM_POOL.split('/')[0], vm.name, v_name) self.assertEqual(volume.path, expected) with self.assertNotRaises(qubes.exc.QubesException): - vm.start() + self.loop.run_until_complete(vm.start()) @skipUnlessLvmPoolExists class TC_02_StorageHelpers(ThinPoolBase): From b65fdf97005c2dbea3070d6e19aec5cf12b1160c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 19 Oct 2018 02:38:47 +0200 Subject: [PATCH 126/145] storage: convert lvm driver to async version LVM operations can take significant amount of time. This is especially visible when stopping a VM (`vm.storage.stop()`) - in that time the whole qubesd freeze for about 2 seconds. Fix this by making all the ThinVolume methods a coroutines (where supported). Each public coroutine is also wrapped with locking on volume._lock to avoid concurrency-related problems. This all also require changing internal helper functions to coroutines. There are two functions that still needs to be called from non-coroutine call sites: - init_cache/reset_cache (initial cache fill, ThinPool.setup()) - qubes_lvm (ThinVolume.export() So, those two functions need to live in two variants. Extract its common code to separate functions to reduce code duplications. Fixes QubesOS/qubes-issues#4283 --- qubes/storage/lvm.py | 240 ++++++++++++++++++++++++++----------- qubes/tests/storage_lvm.py | 156 ++++++++++++------------ 2 files changed, 245 insertions(+), 151 deletions(-) diff --git a/qubes/storage/lvm.py b/qubes/storage/lvm.py index 178019dd..8943bb06 100644 --- a/qubes/storage/lvm.py +++ b/qubes/storage/lvm.py @@ -18,7 +18,7 @@ # ''' Driver for storing vm images in a LVM thin pool ''' - +import functools import logging import os import subprocess @@ -195,26 +195,14 @@ class ThinPool(qubes.storage.Pool): return 0 -def init_cache(log=logging.getLogger('qubes.storage.lvm')): - cmd = ['lvs', '--noheadings', '-o', - 'vg_name,pool_lv,name,lv_size,data_percent,lv_attr,origin', - '--units', 'b', '--separator', ';'] - if os.getuid() != 0: - cmd.insert(0, 'sudo') - environ = os.environ.copy() - environ['LC_ALL'] = 'C.utf8' - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - close_fds=True, env=environ) - out, err = p.communicate() - return_code = p.returncode - if return_code == 0 and err: - log.warning(err) - elif return_code != 0: - raise qubes.storage.StoragePoolException(err) +_init_cache_cmd = ['lvs', '--noheadings', '-o', + 'vg_name,pool_lv,name,lv_size,data_percent,lv_attr,origin', + '--units', 'b', '--separator', ';'] +def _parse_lvm_cache(lvm_output): result = {} - for line in out.splitlines(): + for line in lvm_output.splitlines(): line = line.decode().strip() pool_name, pool_lv, name, size, usage_percent, attr, \ origin = line.split(';', 6) @@ -228,6 +216,42 @@ def init_cache(log=logging.getLogger('qubes.storage.lvm')): return result +def init_cache(log=logging.getLogger('qubes.storage.lvm')): + cmd = _init_cache_cmd + if os.getuid() != 0: + cmd.insert(0, 'sudo') + environ = os.environ.copy() + environ['LC_ALL'] = 'C.utf8' + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + close_fds=True, env=environ) + out, err = p.communicate() + return_code = p.returncode + if return_code == 0 and err: + log.warning(err) + elif return_code != 0: + raise qubes.storage.StoragePoolException(err) + + return _parse_lvm_cache(out) + +@asyncio.coroutine +def init_cache_coro(log=logging.getLogger('qubes.storage.lvm')): + cmd = _init_cache_cmd + if os.getuid() != 0: + cmd = ['sudo'] + cmd + environ = os.environ.copy() + environ['LC_ALL'] = 'C.utf8' + p = yield from asyncio.create_subprocess_exec(*cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=True, env=environ) + out, err = yield from p.communicate() + return_code = p.returncode + if return_code == 0 and err: + log.warning(err) + elif return_code != 0: + raise qubes.storage.StoragePoolException(err) + + return _parse_lvm_cache(out) size_cache = init_cache() @@ -243,6 +267,21 @@ def _revision_sort_key(revision): revision = revision.split('-')[0] return int(revision) +def locked(method): + '''Decorator running given Volume's coroutine under a lock. + Needs to be added after wrapping with @asyncio.coroutine, for example: + + >>>@locked + >>>@asyncio.coroutine + >>>def start(self): + >>> pass + ''' + @asyncio.coroutine + @functools.wraps(method) + def wrapper(self, *args, **kwargs): + with (yield from self._lock): # pylint: disable=protected-access + return (yield from method(self, *args, **kwargs)) + return wrapper class ThinVolume(qubes.storage.Volume): ''' Default LVM thin volume implementation @@ -260,6 +299,7 @@ class ThinVolume(qubes.storage.Volume): self._vid_import = self.vid + '-import' self._size = size + self._lock = asyncio.Lock() @property def path(self): @@ -307,6 +347,7 @@ class ThinVolume(qubes.storage.Volume): raise qubes.storage.StoragePoolException( "You shouldn't use lvm size setter") + @asyncio.coroutine def _reset(self): ''' Resets a volatile volume ''' assert not self.snap_on_start and not self.save_on_stop, \ @@ -314,14 +355,15 @@ class ThinVolume(qubes.storage.Volume): self.log.debug('Resetting volatile %s', self.vid) try: cmd = ['remove', self.vid] - qubes_lvm(cmd, self.log) + yield from qubes_lvm_coro(cmd, self.log) except qubes.storage.StoragePoolException: pass # pylint: disable=protected-access cmd = ['create', self.pool._pool_id, self.vid.split('/')[1], str(self.size)] - qubes_lvm(cmd, self.log) + yield from qubes_lvm_coro(cmd, self.log) + @asyncio.coroutine def _remove_revisions(self, revisions=None): '''Remove old volume revisions. @@ -342,10 +384,11 @@ class ThinVolume(qubes.storage.Volume): assert rev_id != self._vid_current try: cmd = ['remove', self.vid + '-' + rev_id] - qubes_lvm(cmd, self.log) + yield from qubes_lvm_coro(cmd, self.log) except qubes.storage.StoragePoolException: pass + @asyncio.coroutine def _commit(self, vid_to_commit=None, keep=False): ''' Commit temporary volume into current one. By default @@ -368,8 +411,7 @@ class ThinVolume(qubes.storage.Volume): assert hasattr(self, '_vid_snap') vid_to_commit = self._vid_snap - # TODO: when converting this function to coroutine, this _must_ be - # under a lock + assert self._lock.locked() if not os.path.exists('/dev/' + vid_to_commit): # nothing to commit return @@ -377,21 +419,23 @@ class ThinVolume(qubes.storage.Volume): if self._vid_current == self.vid: cmd = ['rename', self.vid, '{}-{}-back'.format(self.vid, int(time.time()))] - qubes_lvm(cmd, self.log) - reset_cache() + yield from qubes_lvm_coro(cmd, self.log) + yield from reset_cache_coro() cmd = ['clone' if keep else 'rename', vid_to_commit, self.vid] - qubes_lvm(cmd, self.log) - reset_cache() + yield from qubes_lvm_coro(cmd, self.log) + yield from reset_cache_coro() # make sure the one we've committed right now is properly # detected as the current one - before removing anything assert self._vid_current == self.vid # and remove old snapshots, if needed - self._remove_revisions() + yield from self._remove_revisions() + @locked + @asyncio.coroutine def create(self): assert self.vid assert self.size @@ -405,32 +449,34 @@ class ThinVolume(qubes.storage.Volume): self.vid.split('/', 1)[1], str(self.size) ] - qubes_lvm(cmd, self.log) - reset_cache() + yield from qubes_lvm_coro(cmd, self.log) + yield from reset_cache_coro() return self + @locked + @asyncio.coroutine def remove(self): assert self.vid try: if os.path.exists('/dev/' + self._vid_snap): cmd = ['remove', self._vid_snap] - qubes_lvm(cmd, self.log) + yield from qubes_lvm_coro(cmd, self.log) except AttributeError: pass try: if os.path.exists('/dev/' + self._vid_import): cmd = ['remove', self._vid_import] - qubes_lvm(cmd, self.log) + yield from qubes_lvm_coro(cmd, self.log) except AttributeError: pass - self._remove_revisions(self.revisions.keys()) + yield from self._remove_revisions(self.revisions.keys()) if not os.path.exists(self.path): return cmd = ['remove', self.path] - qubes_lvm(cmd, self.log) - reset_cache() + yield from qubes_lvm_coro(cmd, self.log) + yield from reset_cache_coro() # pylint: disable=protected-access self.pool._volume_objects_cache.pop(self.vid, None) @@ -441,6 +487,7 @@ class ThinVolume(qubes.storage.Volume): devpath = self.path return devpath + @locked @asyncio.coroutine def import_volume(self, src_volume): if not src_volume.save_on_stop: @@ -456,13 +503,13 @@ class ThinVolume(qubes.storage.Volume): # pylint: disable=line-too-long if isinstance(src_volume.pool, ThinPool) and \ src_volume.pool.thin_pool == self.pool.thin_pool: # NOQA - self._commit(src_volume.path[len('/dev/'):], keep=True) + yield from self._commit(src_volume.path[len('/dev/'):], keep=True) else: cmd = ['create', self.pool._pool_id, # pylint: disable=protected-access self._vid_import.split('/')[1], str(src_volume.size)] - qubes_lvm(cmd, self.log) + yield from qubes_lvm_coro(cmd, self.log) src_path = src_volume.export() cmd = ['dd', 'if=' + src_path, 'of=/dev/' + self._vid_import, 'conv=sparse', 'status=none'] @@ -474,14 +521,16 @@ class ThinVolume(qubes.storage.Volume): yield from p.wait() if p.returncode != 0: cmd = ['remove', self._vid_import] - qubes_lvm(cmd, self.log) + yield from qubes_lvm_coro(cmd, self.log) raise qubes.storage.StoragePoolException( 'Failed to import volume {!r}, dd exit code: {}'.format( src_volume, p.returncode)) - self._commit(self._vid_import) + yield from self._commit(self._vid_import) return self + @locked + @asyncio.coroutine def import_data(self): ''' Returns an object that can be `open()`. ''' if self.is_dirty(): @@ -492,21 +541,23 @@ class ThinVolume(qubes.storage.Volume): # pylint: disable=protected-access cmd = ['create', self.pool._pool_id, self._vid_import.split('/')[1], str(self.size)] - qubes_lvm(cmd, self.log) - reset_cache() + yield from qubes_lvm_coro(cmd, self.log) + yield from reset_cache_coro() devpath = '/dev/' + self._vid_import return devpath + @locked + @asyncio.coroutine def import_data_end(self, success): '''Either commit imported data, or discard temporary volume''' if not os.path.exists('/dev/' + self._vid_import): raise qubes.storage.StoragePoolException( 'No import operation in progress on {}'.format(self.vid)) if success: - self._commit(self._vid_import) + yield from self._commit(self._vid_import) else: cmd = ['remove', self._vid_import] - qubes_lvm(cmd, self.log) + yield from qubes_lvm_coro(cmd, self.log) def abort_if_import_in_progress(self): try: @@ -531,6 +582,8 @@ class ThinVolume(qubes.storage.Volume): return (size_cache[self._vid_snap]['origin'] != self.source.path.split('/')[-1]) + @locked + @asyncio.coroutine def revert(self, revision=None): if self.is_dirty(): raise qubes.storage.StoragePoolException( @@ -547,12 +600,14 @@ class ThinVolume(qubes.storage.Volume): if self.vid in size_cache: cmd = ['remove', self.vid] - qubes_lvm(cmd, self.log) + yield from qubes_lvm_coro(cmd, self.log) cmd = ['clone', self.vid + '-' + revision, self.vid] - qubes_lvm(cmd, self.log) - reset_cache() + yield from qubes_lvm_coro(cmd, self.log) + yield from reset_cache_coro() return self + @locked + @asyncio.coroutine def resize(self, size): ''' Expands volume, throws :py:class:`qubst.storage.qubes.storage.StoragePoolException` if @@ -574,20 +629,21 @@ class ThinVolume(qubes.storage.Volume): if self.is_dirty(): cmd = ['extend', self._vid_snap, str(size)] - qubes_lvm(cmd, self.log) + yield from qubes_lvm_coro(cmd, self.log) elif hasattr(self, '_vid_import') and \ os.path.exists('/dev/' + self._vid_import): cmd = ['extend', self._vid_import, str(size)] - qubes_lvm(cmd, self.log) + yield from qubes_lvm_coro(cmd, self.log) elif self.save_on_stop or not self.snap_on_start: cmd = ['extend', self._vid_current, str(size)] - qubes_lvm(cmd, self.log) - reset_cache() + yield from qubes_lvm_coro(cmd, self.log) + yield from reset_cache_coro() + @asyncio.coroutine def _snapshot(self): try: cmd = ['remove', self._vid_snap] - qubes_lvm(cmd, self.log) + yield from qubes_lvm_coro(cmd, self.log) except: # pylint: disable=bare-except pass @@ -595,32 +651,36 @@ class ThinVolume(qubes.storage.Volume): cmd = ['clone', self._vid_current, self._vid_snap] else: cmd = ['clone', self.source.path, self._vid_snap] - qubes_lvm(cmd, self.log) + yield from qubes_lvm_coro(cmd, self.log) + @locked + @asyncio.coroutine def start(self): self.abort_if_import_in_progress() try: if self.snap_on_start or self.save_on_stop: if not self.save_on_stop or not self.is_dirty(): - self._snapshot() + yield from self._snapshot() else: - self._reset() + yield from self._reset() finally: - reset_cache() + yield from reset_cache_coro() return self + @locked + @asyncio.coroutine def stop(self): try: if self.save_on_stop: - self._commit() + yield from self._commit() if self.snap_on_start and not self.save_on_stop: cmd = ['remove', self._vid_snap] - qubes_lvm(cmd, self.log) + yield from qubes_lvm_coro(cmd, self.log) elif not self.snap_on_start and not self.save_on_stop: cmd = ['remove', self.vid] - qubes_lvm(cmd, self.log) + yield from qubes_lvm_coro(cmd, self.log) finally: - reset_cache() + yield from reset_cache_coro() return self def verify(self): @@ -671,9 +731,14 @@ def pool_exists(pool_id): except KeyError: return False +def _get_lvm_cmdline(cmd): + ''' Build command line for :program:`lvm` call. + The purpose of this function is to keep all the detailed lvm options in + one place. -def qubes_lvm(cmd, log=logging.getLogger('qubes.storage.lvm')): - ''' Call :program:`lvm` to execute an LVM operation ''' + :param cmd: array of str, where cmd[0] is action and the rest are arguments + :return array of str appropriate for subprocess.Popen + ''' action = cmd[0] if action == 'remove': lvm_cmd = ['lvremove', '-f', cmd[1]] @@ -698,28 +763,57 @@ def qubes_lvm(cmd, log=logging.getLogger('qubes.storage.lvm')): cmd = ['sudo', 'lvm'] + lvm_cmd else: cmd = ['lvm'] + lvm_cmd - environ = os.environ.copy() - environ['LC_ALL'] = 'C.utf8' - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - close_fds=True, env=environ) - out, err = p.communicate() - err = err.decode() + + return cmd + +def _process_lvm_output(returncode, stdout, stderr, log): + '''Process output of LVM, determine if the call was successful and + possibly log warnings.''' # Filter out warning about intended over-provisioning. # Upstream discussion about missing option to silence it: # https://bugzilla.redhat.com/1347008 - err = '\n'.join(line for line in err.splitlines() + err = '\n'.join(line for line in stderr.decode().splitlines() if 'exceeds the size of thin pool' not in line) - return_code = p.returncode - if out: - log.debug(out) - if return_code == 0 and err: + if stdout: + log.debug(stdout) + if returncode == 0 and err: log.warning(err) - elif return_code != 0: + elif returncode != 0: assert err, "Command exited unsuccessful, but printed nothing to stderr" err = err.replace('%', '%%') raise qubes.storage.StoragePoolException(err) return True +def qubes_lvm(cmd, log=logging.getLogger('qubes.storage.lvm')): + ''' Call :program:`lvm` to execute an LVM operation ''' + # the only caller for this non-coroutine version is ThinVolume.export() + cmd = _get_lvm_cmdline(cmd) + environ = os.environ.copy() + environ['LC_ALL'] = 'C.utf8' + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + close_fds=True, env=environ) + out, err = p.communicate() + return _process_lvm_output(p.returncode, out, err, log) + +@asyncio.coroutine +def qubes_lvm_coro(cmd, log=logging.getLogger('qubes.storage.lvm')): + ''' Call :program:`lvm` to execute an LVM operation + + Coroutine version of :py:func:`qubes_lvm`''' + cmd = _get_lvm_cmdline(cmd) + environ = os.environ.copy() + environ['LC_ALL'] = 'C.utf8' + p = yield from asyncio.create_subprocess_exec(*cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=True, env=environ) + out, err = yield from p.communicate() + return _process_lvm_output(p.returncode, out, err, log) + def reset_cache(): qubes.storage.lvm.size_cache = init_cache() + +@asyncio.coroutine +def reset_cache_coro(): + qubes.storage.lvm.size_cache = yield from init_cache_coro() diff --git a/qubes/tests/storage_lvm.py b/qubes/tests/storage_lvm.py index 3bead18e..3f320790 100644 --- a/qubes/tests/storage_lvm.py +++ b/qubes/tests/storage_lvm.py @@ -136,10 +136,10 @@ class TC_00_ThinPool(ThinPoolBase): self.assertEqual(volume.name, 'root') self.assertEqual(volume.pool, self.pool.name) self.assertEqual(volume.size, qubes.config.defaults['root_img_size']) - volume.create() + self.loop.run_until_complete(volume.create()) path = "/dev/%s" % volume.vid self.assertTrue(os.path.exists(path), path) - volume.remove() + self.loop.run_until_complete(volume.remove()) def test_003_read_write_volume(self): ''' Test read-write volume creation ''' @@ -156,10 +156,10 @@ class TC_00_ThinPool(ThinPoolBase): self.assertEqual(volume.name, 'root') self.assertEqual(volume.pool, self.pool.name) self.assertEqual(volume.size, qubes.config.defaults['root_img_size']) - volume.create() + self.loop.run_until_complete(volume.create()) path = "/dev/%s" % volume.vid self.assertTrue(os.path.exists(path), path) - volume.remove() + self.loop.run_until_complete(volume.remove()) def test_004_size(self): with self.assertNotRaises(NotImplementedError): @@ -207,11 +207,11 @@ class TC_00_ThinPool(ThinPoolBase): } vm = qubes.tests.storage.TestVM(self) volume = self.app.get_pool(self.pool.name).init_volume(vm, config) - volume.create() - self.addCleanup(volume.remove) + self.loop.run_until_complete(volume.create()) + self.addCleanup(self.loop.run_until_complete, volume.remove()) path = "/dev/%s" % volume.vid new_size = 64 * 1024 ** 2 - volume.resize(new_size) + self.loop.run_until_complete(volume.resize(new_size)) self.assertEqual(self._get_size(path), new_size) self.assertEqual(volume.size, new_size) @@ -226,17 +226,17 @@ class TC_00_ThinPool(ThinPoolBase): } vm = qubes.tests.storage.TestVM(self) volume = self.app.get_pool(self.pool.name).init_volume(vm, config) - volume.create() - self.addCleanup(volume.remove) - volume.start() + self.loop.run_until_complete(volume.create()) + self.addCleanup(self.loop.run_until_complete, volume.remove()) + self.loop.run_until_complete(volume.start()) path = "/dev/%s" % volume.vid path2 = "/dev/%s" % volume._vid_snap new_size = 64 * 1024 ** 2 - volume.resize(new_size) + self.loop.run_until_complete(volume.resize(new_size)) self.assertEqual(self._get_size(path), old_size) self.assertEqual(self._get_size(path2), new_size) self.assertEqual(volume.size, new_size) - volume.stop() + self.loop.run_until_complete(volume.stop()) self.assertEqual(self._get_size(path), new_size) self.assertEqual(volume.size, new_size) @@ -271,18 +271,18 @@ class TC_00_ThinPool(ThinPoolBase): } vm = qubes.tests.storage.TestVM(self) volume = self.app.get_pool(self.pool.name).init_volume(vm, config) - volume.create() + self.loop.run_until_complete(volume.create()) path_snap = '/dev/' + volume._vid_snap self.assertFalse(os.path.exists(path_snap), path_snap) origin_uuid = self._get_lv_uuid(volume.path) - volume.start() + self.loop.run_until_complete(volume.start()) snap_uuid = self._get_lv_uuid(path_snap) self.assertNotEqual(origin_uuid, snap_uuid) path = volume.path self.assertTrue(path.startswith('/dev/' + volume.vid), '{} does not start with /dev/{}'.format(path, volume.vid)) self.assertTrue(os.path.exists(path), path) - volume.remove() + self.loop.run_until_complete(volume.remove()) def test_009_interrupted_commit(self): ''' Test volume changes commit''' @@ -317,7 +317,7 @@ class TC_00_ThinPool(ThinPoolBase): revisions[1].lstrip('-'): '2018-03-14T22:18:25', } self.assertEqual(volume.revisions, expected_revisions) - volume.start() + self.loop.run_until_complete(volume.start()) self.assertEqual(volume.revisions, expected_revisions) snap_uuid = self._get_lv_uuid(path_snap) self.assertEqual(orig_uuids['-snap'], snap_uuid) @@ -326,7 +326,7 @@ class TC_00_ThinPool(ThinPoolBase): '/dev/' + volume.vid + revisions[1]) with unittest.mock.patch('time.time') as mock_time: mock_time.side_effect = [521065906] - volume.stop() + self.loop.run_until_complete(volume.stop()) expected_revisions = { revisions[0].lstrip('-'): '2018-03-14T22:18:24', revisions[1].lstrip('-'): '2018-03-14T22:18:25', @@ -337,7 +337,7 @@ class TC_00_ThinPool(ThinPoolBase): self.assertEqual(snap_uuid, self._get_lv_uuid(volume.path)) self.assertFalse(os.path.exists(path_snap), path_snap) - volume.remove() + self.loop.run_until_complete(volume.remove()) def test_010_migration1(self): '''Start with old revisions, then start interacting using new code''' @@ -371,7 +371,7 @@ class TC_00_ThinPool(ThinPoolBase): self.assertEqual(volume.revisions, expected_revisions) self.assertEqual(volume.path, '/dev/' + volume.vid) - volume.start() + self.loop.run_until_complete(volume.start()) snap_uuid = self._get_lv_uuid(path_snap) self.assertNotEqual(orig_uuids[''], snap_uuid) snap_origin_uuid = self._get_lv_origin_uuid(path_snap) @@ -382,7 +382,7 @@ class TC_00_ThinPool(ThinPoolBase): with unittest.mock.patch('time.time') as mock_time: mock_time.side_effect = ('1521065906', '1521065907') - volume.stop() + self.loop.run_until_complete(volume.stop()) revisions.extend(['-1521065906-back']) expected_revisions = { revisions[2].lstrip('-'): '2018-03-14T22:18:25', @@ -397,7 +397,7 @@ class TC_00_ThinPool(ThinPoolBase): prev_path = '/dev/' + volume.vid + revisions[3] self.assertEqual(self._get_lv_uuid(prev_path), orig_uuids['']) - volume.remove() + self.loop.run_until_complete(volume.remove()) for rev in revisions: path = '/dev/' + volume.vid + rev self.assertFalse(os.path.exists(path), path) @@ -438,7 +438,7 @@ class TC_00_ThinPool(ThinPoolBase): with unittest.mock.patch('time.time') as mock_time: mock_time.side_effect = ('1521065906', '1521065907') - volume.stop() + self.loop.run_until_complete(volume.stop()) revisions.extend(['-1521065906-back']) expected_revisions = { revisions[2].lstrip('-'): '2018-03-14T22:18:26', @@ -452,7 +452,7 @@ class TC_00_ThinPool(ThinPoolBase): prev_path = '/dev/' + volume.vid + revisions[2] self.assertEqual(self._get_lv_uuid(prev_path), orig_uuids['']) - volume.remove() + self.loop.run_until_complete(volume.remove()) for rev in revisions: path = '/dev/' + volume.vid + rev self.assertFalse(os.path.exists(path), path) @@ -487,14 +487,14 @@ class TC_00_ThinPool(ThinPoolBase): self.assertTrue(volume.path, '/dev/' + volume.vid) self.assertTrue(volume.is_dirty()) - volume.start() + self.loop.run_until_complete(volume.start()) self.assertEqual(volume.revisions, expected_revisions) self.assertEqual(volume.path, '/dev/' + volume.vid) # -snap LV should be unchanged self.assertEqual(self._get_lv_uuid(volume._vid_snap), orig_uuids['-snap']) - volume.remove() + self.loop.run_until_complete(volume.remove()) for rev in revisions: path = '/dev/' + volume.vid + rev self.assertFalse(os.path.exists(path), path) @@ -531,12 +531,12 @@ class TC_00_ThinPool(ThinPoolBase): with unittest.mock.patch('time.time') as mock_time: mock_time.side_effect = ('1521065906', '1521065907') - volume.stop() + self.loop.run_until_complete(volume.stop()) expected_revisions = {} self.assertEqual(volume.revisions, expected_revisions) self.assertEqual(volume.path, '/dev/' + volume.vid) - volume.remove() + self.loop.run_until_complete(volume.remove()) for rev in revisions: path = '/dev/' + volume.vid + rev self.assertFalse(os.path.exists(path), path) @@ -555,13 +555,13 @@ class TC_00_ThinPool(ThinPoolBase): volume = self.app.get_pool(self.pool.name).init_volume(vm, config) # mock logging, to not interfere with time.time() mock volume.log = unittest.mock.Mock() - volume.create() + self.loop.run_until_complete(volume.create()) self.assertFalse(volume.is_dirty()) path = volume.path expected_revisions = {} self.assertEqual(volume.revisions, expected_revisions) - volume.start() + self.loop.run_until_complete(volume.start()) self.assertEqual(volume.revisions, expected_revisions) path_snap = '/dev/' + volume._vid_snap snap_uuid = self._get_lv_uuid(path_snap) @@ -570,14 +570,14 @@ class TC_00_ThinPool(ThinPoolBase): with unittest.mock.patch('time.time') as mock_time: mock_time.side_effect = [521065906] - volume.stop() + self.loop.run_until_complete(volume.stop()) self.assertFalse(volume.is_dirty()) self.assertEqual(volume.revisions, {}) self.assertEqual(volume.path, '/dev/' + volume.vid) self.assertEqual(snap_uuid, self._get_lv_uuid(volume.path)) self.assertFalse(os.path.exists(path_snap), path_snap) - volume.remove() + self.loop.run_until_complete(volume.remove()) def test_020_revert_last(self): ''' Test volume revert''' @@ -591,11 +591,11 @@ class TC_00_ThinPool(ThinPoolBase): } vm = qubes.tests.storage.TestVM(self) volume = self.app.get_pool(self.pool.name).init_volume(vm, config) - volume.create() - volume.start() - volume.stop() - volume.start() - volume.stop() + self.loop.run_until_complete(volume.create()) + self.loop.run_until_complete(volume.start()) + self.loop.run_until_complete(volume.stop()) + self.loop.run_until_complete(volume.start()) + self.loop.run_until_complete(volume.stop()) self.assertEqual(len(volume.revisions), 2) revisions = volume.revisions revision_id = max(revisions.keys()) @@ -604,7 +604,7 @@ class TC_00_ThinPool(ThinPoolBase): rev_uuid = self._get_lv_uuid(volume.vid + '-' + revision_id) self.assertFalse(volume.is_dirty()) self.assertNotEqual(current_uuid, rev_uuid) - volume.revert() + self.loop.run_until_complete(volume.revert()) path_snap = '/dev/' + volume._vid_snap self.assertFalse(os.path.exists(path_snap), path_snap) self.assertEqual(current_path, volume.path) @@ -612,7 +612,7 @@ class TC_00_ThinPool(ThinPoolBase): self.assertEqual(new_uuid, rev_uuid) self.assertEqual(volume.revisions, revisions) - volume.remove() + self.loop.run_until_complete(volume.remove()) def test_021_revert_earlier(self): ''' Test volume revert''' @@ -626,11 +626,11 @@ class TC_00_ThinPool(ThinPoolBase): } vm = qubes.tests.storage.TestVM(self) volume = self.app.get_pool(self.pool.name).init_volume(vm, config) - volume.create() - volume.start() - volume.stop() - volume.start() - volume.stop() + self.loop.run_until_complete(volume.create()) + self.loop.run_until_complete(volume.start()) + self.loop.run_until_complete(volume.stop()) + self.loop.run_until_complete(volume.start()) + self.loop.run_until_complete(volume.stop()) self.assertEqual(len(volume.revisions), 2) revisions = volume.revisions revision_id = min(revisions.keys()) @@ -639,7 +639,7 @@ class TC_00_ThinPool(ThinPoolBase): rev_uuid = self._get_lv_uuid(volume.vid + '-' + revision_id) self.assertFalse(volume.is_dirty()) self.assertNotEqual(current_uuid, rev_uuid) - volume.revert(revision_id) + self.loop.run_until_complete(volume.revert(revision_id)) path_snap = '/dev/' + volume._vid_snap self.assertFalse(os.path.exists(path_snap), path_snap) self.assertEqual(current_path, volume.path) @@ -647,7 +647,7 @@ class TC_00_ThinPool(ThinPoolBase): self.assertEqual(new_uuid, rev_uuid) self.assertEqual(volume.revisions, revisions) - volume.remove() + self.loop.run_until_complete(volume.remove()) def test_030_import_data(self): ''' Test volume import''' @@ -661,14 +661,14 @@ class TC_00_ThinPool(ThinPoolBase): } vm = qubes.tests.storage.TestVM(self) volume = self.app.get_pool(self.pool.name).init_volume(vm, config) - volume.create() + self.loop.run_until_complete(volume.create()) current_uuid = self._get_lv_uuid(volume.path) self.assertFalse(volume.is_dirty()) - import_path = volume.import_data() + import_path = self.loop.run_until_complete(volume.import_data()) import_uuid = self._get_lv_uuid(import_path) self.assertNotEqual(current_uuid, import_uuid) # success - commit data - volume.import_data_end(True) + self.loop.run_until_complete(volume.import_data_end(True)) new_current_uuid = self._get_lv_uuid(volume.path) self.assertEqual(new_current_uuid, import_uuid) revisions = volume.revisions @@ -678,7 +678,7 @@ class TC_00_ThinPool(ThinPoolBase): self._get_lv_uuid(volume.vid + '-' + revision)) self.assertFalse(os.path.exists(import_path), import_path) - volume.remove() + self.loop.run_until_complete(volume.remove()) def test_031_import_data_fail(self): ''' Test volume import''' @@ -692,21 +692,21 @@ class TC_00_ThinPool(ThinPoolBase): } vm = qubes.tests.storage.TestVM(self) volume = self.app.get_pool(self.pool.name).init_volume(vm, config) - volume.create() + self.loop.run_until_complete(volume.create()) current_uuid = self._get_lv_uuid(volume.path) self.assertFalse(volume.is_dirty()) - import_path = volume.import_data() + import_path = self.loop.run_until_complete(volume.import_data()) import_uuid = self._get_lv_uuid(import_path) self.assertNotEqual(current_uuid, import_uuid) # fail - discard data - volume.import_data_end(False) + self.loop.run_until_complete(volume.import_data_end(False)) new_current_uuid = self._get_lv_uuid(volume.path) self.assertEqual(new_current_uuid, current_uuid) revisions = volume.revisions self.assertEqual(len(revisions), 0) self.assertFalse(os.path.exists(import_path), import_path) - volume.remove() + self.loop.run_until_complete(volume.remove()) def test_032_import_volume_same_pool(self): '''Import volume from the same pool''' @@ -721,7 +721,7 @@ class TC_00_ThinPool(ThinPoolBase): } vm = qubes.tests.storage.TestVM(self) source_volume = self.app.get_pool(self.pool.name).init_volume(vm, config) - source_volume.create() + self.loop.run_until_complete(source_volume.create()) source_uuid = self._get_lv_uuid(source_volume.path) @@ -738,7 +738,7 @@ class TC_00_ThinPool(ThinPoolBase): volume.log = unittest.mock.Mock() with unittest.mock.patch('time.time') as mock_time: mock_time.side_effect = [1521065905] - volume.create() + self.loop.run_until_complete(volume.create()) self.assertEqual(volume.revisions, {}) uuid_before = self._get_lv_uuid(volume.path) @@ -760,8 +760,8 @@ class TC_00_ThinPool(ThinPoolBase): } self.assertEqual(volume.revisions, expected_revisions) - volume.remove() - source_volume.remove() + self.loop.run_until_complete(volume.remove()) + self.loop.run_until_complete(source_volume.remove()) def test_033_import_volume_different_pool(self): '''Import volume from a different pool''' @@ -780,7 +780,7 @@ class TC_00_ThinPool(ThinPoolBase): volume.log = unittest.mock.Mock() with unittest.mock.patch('time.time') as mock_time: mock_time.side_effect = [1521065905] - volume.create() + self.loop.run_until_complete(volume.create()) self.assertEqual(volume.revisions, {}) uuid_before = self._get_lv_uuid(volume.path) @@ -807,7 +807,7 @@ class TC_00_ThinPool(ThinPoolBase): } self.assertEqual(volume.revisions, expected_revisions) - volume.remove() + self.loop.run_until_complete(volume.remove()) def test_040_volatile(self): '''Volatile volume test''' @@ -821,21 +821,21 @@ class TC_00_ThinPool(ThinPoolBase): volume = self.app.get_pool(self.pool.name).init_volume(vm, config) # volatile volume don't need any file, verify should succeed self.assertTrue(volume.verify()) - volume.create() + self.loop.run_until_complete(volume.create()) self.assertTrue(volume.verify()) self.assertFalse(volume.save_on_stop) self.assertFalse(volume.snap_on_start) path = volume.path self.assertEqual(path, '/dev/' + volume.vid) self.assertFalse(os.path.exists(path)) - volume.start() + self.loop.run_until_complete(volume.start()) self.assertTrue(os.path.exists(path)) vol_uuid = self._get_lv_uuid(path) - volume.start() + self.loop.run_until_complete(volume.start()) self.assertTrue(os.path.exists(path)) vol_uuid2 = self._get_lv_uuid(path) self.assertNotEqual(vol_uuid, vol_uuid2) - volume.stop() + self.loop.run_until_complete(volume.stop()) self.assertFalse(os.path.exists(path)) def test_050_snapshot_volume(self): @@ -850,7 +850,7 @@ class TC_00_ThinPool(ThinPoolBase): vm = qubes.tests.storage.TestVM(self) volume_origin = self.app.get_pool(self.pool.name).init_volume( vm, config_origin) - volume_origin.create() + self.loop.run_until_complete(volume_origin.create()) config_snapshot = { 'name': 'root2', 'pool': self.pool.name, @@ -868,11 +868,11 @@ class TC_00_ThinPool(ThinPoolBase): # only origin volume really needs to exist, verify should succeed # even before create self.assertTrue(volume.verify()) - volume.create() + self.loop.run_until_complete(volume.create()) path = volume.path self.assertEqual(path, '/dev/' + volume.vid) self.assertFalse(os.path.exists(path), path) - volume.start() + self.loop.run_until_complete(volume.start()) # snapshot volume isn't considered dirty at any time self.assertFalse(volume.is_dirty()) # not outdated yet @@ -882,13 +882,13 @@ class TC_00_ThinPool(ThinPoolBase): self.assertEqual(origin_uuid, snap_origin_uuid) # now make it outdated - volume_origin.start() - volume_origin.stop() + self.loop.run_until_complete(volume_origin.start()) + self.loop.run_until_complete(volume_origin.stop()) self.assertTrue(volume.is_outdated()) origin_uuid = self._get_lv_uuid(volume_origin.path) self.assertNotEqual(origin_uuid, snap_origin_uuid) - volume.stop() + self.loop.run_until_complete(volume.stop()) # stopped volume is never outdated self.assertFalse(volume.is_outdated()) path = volume.path @@ -896,8 +896,8 @@ class TC_00_ThinPool(ThinPoolBase): path = '/dev/' + volume._vid_snap self.assertFalse(os.path.exists(path), path) - volume.remove() - volume_origin.remove() + self.loop.run_until_complete(volume.remove()) + self.loop.run_until_complete(volume_origin.remove()) def test_100_pool_list_volumes(self): config = { @@ -911,24 +911,24 @@ class TC_00_ThinPool(ThinPoolBase): config2 = config.copy() vm = qubes.tests.storage.TestVM(self) volume1 = self.app.get_pool(self.pool.name).init_volume(vm, config) - volume1.create() + self.loop.run_until_complete(volume1.create()) config2['name'] = 'private' volume2 = self.app.get_pool(self.pool.name).init_volume(vm, config2) - volume2.create() + self.loop.run_until_complete(volume2.create()) # create some revisions - volume1.start() - volume1.stop() + self.loop.run_until_complete(volume1.start()) + self.loop.run_until_complete(volume1.stop()) # and have one in dirty state - volume2.start() + self.loop.run_until_complete(volume2.start()) self.assertIn(volume1, list(self.pool.volumes)) self.assertIn(volume2, list(self.pool.volumes)) - volume1.remove() + self.loop.run_until_complete(volume1.remove()) self.assertNotIn(volume1, list(self.pool.volumes)) self.assertIn(volume2, list(self.pool.volumes)) - volume2.remove() + self.loop.run_until_complete(volume2.remove()) self.assertNotIn(volume1, list(self.pool.volumes)) self.assertNotIn(volume1, list(self.pool.volumes)) From e1f65bdf7b823562db25a8714ec3994e6ef547e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 21 Oct 2018 04:36:13 +0200 Subject: [PATCH 127/145] vm: add shutdown_timeout property, make vm.shutdown(wait=True) use it vm.shutdown(wait=True) waited indefinitely for the shutdown, which makes useless without some boilerplate handling the timeout. Since the timeout may depend on the operating system inside, add a per-VM property for it, with value inheritance from template and then from global default_shutdown_timeout property. When timeout is reached, the method raises exception - whether to kill it or not is left to the caller. Fixes QubesOS/qubes-issues#1696 --- qubes/app.py | 6 ++++++ qubes/exc.py | 8 ++++++++ qubes/vm/qubesvm.py | 24 +++++++++++++++++++++--- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/qubes/app.py b/qubes/app.py index 8d0bd15e..5e72159f 100644 --- a/qubes/app.py +++ b/qubes/app.py @@ -725,6 +725,12 @@ class Qubes(qubes.PropertyHolder): doc='''Default time in seconds after which qrexec connection attempt is deemed failed''') + default_shutdown_timeout = qubes.property('default_shutdown_timeout', + load_stage=3, + default=60, + type=int, + doc='''Default time in seconds for VM shutdown to complete''') + stats_interval = qubes.property('stats_interval', default=3, type=int, diff --git a/qubes/exc.py b/qubes/exc.py index 427ae4fa..3c7797d1 100644 --- a/qubes/exc.py +++ b/qubes/exc.py @@ -101,6 +101,14 @@ class QubesVMNotHaltedError(QubesVMError): super(QubesVMNotHaltedError, self).__init__(vm, msg or 'Domain is not powered off: {!r}'.format(vm.name)) +class QubesVMShutdownTimeoutError(QubesVMError): + '''Domain shutdown timed out. + + ''' + def __init__(self, vm, msg=None): + super(QubesVMShutdownTimeoutError, self).__init__(vm, + msg or 'Domain shutdown timed out: {!r}'.format(vm.name)) + class QubesNoTemplateError(QubesVMError): '''Cannot start domain, because there is no template''' diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 7b023776..b2f97587 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -517,6 +517,14 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): failed. Operating system inside VM should be able to boot in this time.''') + shutdown_timeout = qubes.property('shutdown_timeout', type=int, + default=_default_with_template('shutdown_timeout', + lambda self: self.app.default_shutdown_timeout), + setter=_setter_positive_int, + doc='''Time in seconds for shutdown of the VM, after which VM may be + forcefully powered off. Operating system inside VM should be + able to fully shutdown in this time.''') + autostart = qubes.property('autostart', default=False, type=bool, setter=qubes.property.bool, doc='''Setting this to `True` means that VM should be autostarted on @@ -1055,9 +1063,13 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): self.name) @asyncio.coroutine - def shutdown(self, force=False, wait=False): + def shutdown(self, force=False, wait=False, timeout=None): '''Shutdown domain. + :param force: ignored + :param wait: wait for shutdown to complete + :param timeout: shutdown wait timeout (for *wait*=True), defaults to + :py:attr:`shutdown_timeout` :raises qubes.exc.QubesVMNotStartedError: \ when domain is already shut down. ''' @@ -1070,8 +1082,14 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): self.libvirt_domain.shutdown() - while wait and not self.is_halted(): - yield from asyncio.sleep(0.25) + if wait: + if timeout is None: + timeout = self.shutdown_timeout + while timeout > 0 and not self.is_halted(): + yield from asyncio.sleep(0.25) + timeout -= 0.25 + if not self.is_halted(): + raise qubes.exc.QubesVMShutdownTimeoutError(self) return self From 5be003d53904fe3585dbc63e9cc4585e4dd32961 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 21 Oct 2018 04:44:47 +0200 Subject: [PATCH 128/145] vm/dispvm: fix DispVM cleanup First unregister the domain from collection, and only then call remove_from_disk(). Removing it from collection prevent further calls being made to it. Or if anything else keep a reference to it (for example as a netvm), then abort the operation. Additionally this makes it unnecessary to take startup lock when cleaning it up in tests. --- qubes/tests/__init__.py | 7 ------- qubes/vm/dispvm.py | 4 ++-- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index c3f3a2fd..78777599 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -789,13 +789,6 @@ class SystemTestCase(QubesTestCase): vmname = vm.name app = vm.app - # avoid race with DispVM.auto_cleanup=True - try: - self.loop.run_until_complete( - asyncio.wait_for(vm.startup_lock.acquire(), 10)) - except asyncio.TimeoutError: - pass - try: # XXX .is_running() may throw libvirtError if undefined if vm.is_running(): diff --git a/qubes/vm/dispvm.py b/qubes/vm/dispvm.py index 2179dd9f..0c6389f3 100644 --- a/qubes/vm/dispvm.py +++ b/qubes/vm/dispvm.py @@ -142,8 +142,8 @@ class DispVM(qubes.vm.qubesvm.QubesVM): def _auto_cleanup(self): '''Do auto cleanup if enabled''' if self.auto_cleanup and self in self.app.domains: - yield from self.remove_from_disk() del self.app.domains[self] + yield from self.remove_from_disk() self.app.save() @classmethod @@ -193,8 +193,8 @@ class DispVM(qubes.vm.qubesvm.QubesVM): pass # if auto_cleanup is set, this will be done automatically if not self.auto_cleanup: - yield from self.remove_from_disk() del self.app.domains[self] + yield from self.remove_from_disk() self.app.save() @asyncio.coroutine From 2c1629da042d6495c6dae5ae1fdc411cc105f825 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 21 Oct 2018 04:52:27 +0200 Subject: [PATCH 129/145] vm: call after-shutdown cleanup also from vm.kill and vm.shutdown Cleaning up after domain shutdown (domain-stopped and domain-shutdown events) relies on libvirt events which may be unreliable in some cases (events may be processed with some delay, of if libvirt was restarted in the meantime, may not happen at all). So, instead of ensuring only proper ordering between shutdown cleanup and next startup, also trigger the cleanup when we know for sure domain isn't running: - at vm.kill() - after libvirt confirms domain was destroyed - at vm.shutdown(wait=True) - after successful shutdown - at vm.remove_from_disk() - after ensuring it isn't running but just before actually removing it This fixes various race conditions: - qvm-kill && qvm-remove: remove could happen before shutdown cleanup was done and storage driver would be confused about that - qvm-shutdown --wait && qvm-clone: clone could happen before new content was commited to the original volume, making the copy of previous VM state (and probably more) Previously it wasn't such a big issue on default configuration, because LVM driver was fully synchronous, effectively blocking the whole qubesd for the time the cleanup happened. To avoid code duplication, factor out _ensure_shutdown_handled function calling actual cleanup (and possibly canceling one called with libvirt event). Note that now, "Duplicated stopped event from libvirt received!" warning may happen in normal circumstances, not only because of some bug. It is very important that post-shutdown cleanup happen when domain is not running. To ensure that, take startup_lock and under it 1) ensure its halted and only then 2) execute the cleanup. This isn't necessary when removing it from disk, because its already removed from the collection at that time, which also avoids other calls to it (see also "vm/dispvm: fix DispVM cleanup" commit). Actually, taking the startup_lock in remove_from_disk function would cause a deadlock in DispVM auto cleanup code: - vm.kill (or other trigger for the cleanup) - vm.startup_lock acquire <==== - vm._ensure_shutdown_handled - domain-shutdown event - vm._auto_cleanup (in DispVM class) - vm.remove_from_disk - cannot take vm.startup_lock again --- qubes/vm/qubesvm.py | 90 ++++++++++++++++++++++++++++----------------- 1 file changed, 56 insertions(+), 34 deletions(-) diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index b2f97587..faa7467b 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -878,6 +878,36 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): # methods for changing domain state # + @asyncio.coroutine + def _ensure_shutdown_handled(self): + '''Make sure previous shutdown is fully handled. + MUST NOT be called when domain is running. + ''' + with (yield from self._domain_stopped_lock): + # Don't accept any new stopped event's till a new VM has been + # created. If we didn't received any stopped event or it wasn't + # handled yet we will handle this in the next lines. + self._domain_stopped_event_received = True + + if self._domain_stopped_future is not None: + # Libvirt stopped event was already received, so cancel the + # future. If it didn't generate the Qubes events yet we + # will do it below. + self._domain_stopped_future.cancel() + self._domain_stopped_future = None + + if not self._domain_stopped_event_handled: + # No Qubes domain-stopped events have been generated yet. + # So do this now. + + # Set this immediately such that we don't generate events + # twice if an exception gets thrown. + self._domain_stopped_event_handled = True + + yield from self.fire_event_async('domain-stopped') + yield from self.fire_event_async('domain-shutdown') + + @asyncio.coroutine def start(self, start_guid=True, notify_function=None, mem_required=None): @@ -894,29 +924,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): if self.get_power_state() != 'Halted': return self - with (yield from self._domain_stopped_lock): - # Don't accept any new stopped event's till a new VM has been - # created. If we didn't received any stopped event or it wasn't - # handled yet we will handle this in the next lines. - self._domain_stopped_event_received = True - - if self._domain_stopped_future is not None: - # Libvirt stopped event was already received, so cancel the - # future. If it didn't generate the Qubes events yet we - # will do it below. - self._domain_stopped_future.cancel() - self._domain_stopped_future = None - - if not self._domain_stopped_event_handled: - # No Qubes domain-stopped events have been generated yet. - # So do this now. - - # Set this immediately such that we don't generate events - # twice if an exception gets thrown. - self._domain_stopped_event_handled = True - - yield from self.fire_event_async('domain-stopped') - yield from self.fire_event_async('domain-shutdown') + yield from self._ensure_shutdown_handled() self.log.info('Starting {}'.format(self.name)) @@ -1030,8 +1038,8 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): return if self._domain_stopped_event_received: - self.log.warning('Duplicated stopped event from libvirt received!') - # ignore this unexpected event + # ignore this event - already triggered by shutdown(), kill(), + # or subsequent start() return self._domain_stopped_event_received = True @@ -1088,8 +1096,12 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): while timeout > 0 and not self.is_halted(): yield from asyncio.sleep(0.25) timeout -= 0.25 - if not self.is_halted(): - raise qubes.exc.QubesVMShutdownTimeoutError(self) + with (yield from self.startup_lock): + if self.is_halted(): + # make sure all shutdown tasks are completed + yield from self._ensure_shutdown_handled() + else: + raise qubes.exc.QubesVMShutdownTimeoutError(self) return self @@ -1104,13 +1116,17 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): if not self.is_running() and not self.is_paused(): raise qubes.exc.QubesVMNotStartedError(self) - try: - self.libvirt_domain.destroy() - except libvirt.libvirtError as e: - if e.get_error_code() == libvirt.VIR_ERR_OPERATION_INVALID: - raise qubes.exc.QubesVMNotStartedError(self) - else: - raise + with (yield from self.startup_lock): + try: + self.libvirt_domain.destroy() + except libvirt.libvirtError as e: + if e.get_error_code() == libvirt.VIR_ERR_OPERATION_INVALID: + raise qubes.exc.QubesVMNotStartedError(self) + else: + raise + + # make sure all shutdown tasks are completed + yield from self._ensure_shutdown_handled() return self @@ -1477,6 +1493,12 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): "Can't remove VM {!s}, beacuse it's in state {!r}.".format( self, self.get_power_state())) + # make sure shutdown is handled before removing anything, but only if + # handling is pending; if not, we may be called from within + # domain-shutdown event (DispVM._auto_cleanup), which would deadlock + if not self._domain_stopped_event_handled: + yield from self._ensure_shutdown_handled() + yield from self.fire_event_async('domain-remove-from-disk') try: # TODO: make it async? From cf8b6219a9f6c13246894ce45dc6c74e56a4250b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 21 Oct 2018 05:11:24 +0200 Subject: [PATCH 130/145] tests: make use of vm.shutdown(wait=True) --- qubes/tests/__init__.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index 78777599..2940c7c7 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -1070,15 +1070,12 @@ class SystemTestCase(QubesTestCase): subprocess.check_call(command) def shutdown_and_wait(self, vm, timeout=60): - self.loop.run_until_complete(vm.shutdown()) - while timeout > 0: - if not vm.is_running(): - return - self.loop.run_until_complete(asyncio.sleep(1)) - timeout -= 1 - name = vm.name - del vm - self.fail("Timeout while waiting for VM {} shutdown".format(name)) + try: + self.loop.run_until_complete(vm.shutdown(wait=True, timeout=timeout)) + except qubes.exc.QubesException: + name = vm.name + del vm + self.fail("Timeout while waiting for VM {} shutdown".format(name)) def prepare_hvm_system_linux(self, vm, init_script, extra_files=None): if not os.path.exists('/usr/lib/grub/i386-pc'): From 4e762788a9b574d93c5a77f6d34ba5af3b6f14b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 21 Oct 2018 05:12:27 +0200 Subject: [PATCH 131/145] tests: check if qubes-vm@ service is disabled on domain removal Test for QubesOS/qubes-issues#4014 --- qubes/tests/integ/basic.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/qubes/tests/integ/basic.py b/qubes/tests/integ/basic.py index 6d535e7e..dfb57e39 100644 --- a/qubes/tests/integ/basic.py +++ b/qubes/tests/integ/basic.py @@ -115,6 +115,26 @@ class TC_00_Basic(qubes.tests.SystemTestCase): self.loop.run_until_complete(asyncio.sleep(1)) self.assertFalse(self.vm.is_running()) + def test_130_autostart_disable_on_remove(self): + vm = self.app.add_new_vm(qubes.vm.appvm.AppVM, + name=self.make_vm_name('vm'), + template=self.app.default_template, + label='red') + + self.assertIsNotNone(vm) + self.loop.run_until_complete(vm.create_on_disk()) + vm.autostart = True + self.assertTrue(os.path.exists( + '/etc/systemd/system/multi-user.target.wants/' + 'qubes-vm@{}.service'.format(vm.name)), + "systemd service not enabled by autostart=True") + del self.app.domains[vm] + self.loop.run_until_complete(vm.remove_from_disk()) + self.assertFalse(os.path.exists( + '/etc/systemd/system/multi-user.target.wants/' + 'qubes-vm@{}.service'.format(vm.name)), + "systemd service not disabled on domain remove") + def _test_200_on_domain_start(self, vm, event, **_kwargs): '''Simulate domain crash just after startup''' vm.libvirt_domain.destroy() From 08ddeee9fbb55768fe930d22947991fce60472fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 21 Oct 2018 05:19:07 +0200 Subject: [PATCH 132/145] tests: improve VMs cleanup wrt custom templates Cleanup VMs in template reverse topological order, not network one. Network can be set to None to break dependency, but template can't. For netvm to be changed, kill VMs first (kill doesn't check network dependency), so netvm change will not trigger side effects (runtime change, which could fail). This fixes cleanup for tests creating custom templates - previously order was undefined and if template was tried removed before its child VMs, it fails. All the relevant files were removed later anyway, but it lead to python objects leaks. --- qubes/tests/__init__.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index 2940c7c7..b66a4ee7 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -789,13 +789,6 @@ class SystemTestCase(QubesTestCase): vmname = vm.name app = vm.app - try: - # XXX .is_running() may throw libvirtError if undefined - if vm.is_running(): - self.loop.run_until_complete(vm.kill()) - except: # pylint: disable=bare-except - pass - try: self.loop.run_until_complete(vm.remove_from_disk()) except: # pylint: disable=bare-except @@ -876,18 +869,36 @@ class SystemTestCase(QubesTestCase): vms = list(vms) if not vms: return + # first kill all the domains, to avoid side effects of changing netvm + for vm in vms: + try: + # XXX .is_running() may throw libvirtError if undefined + if vm.is_running(): + self.loop.run_until_complete(vm.kill()) + except: # pylint: disable=bare-except + pass # break dependencies for vm in vms: vm.default_dispvm = None - # then remove in reverse topological order (wrt netvm), using naive + vm.netvm = None + # take app instance from any VM to be removed + app = vms[0].app + if app.default_dispvm in vms: + app.default_dispvm = None + if app.default_netvm in vms: + app.default_netvm = None + del app + # then remove in reverse topological order (wrt template), using naive # algorithm - # this heavily depends on lack of netvm loops + # this heavily depends on lack of template loops, but those are + # impossible while vms: vm = vms.pop(0) # make sure that all connected VMs are going to be removed, # otherwise this will loop forever - assert all(x in vms for x in vm.connected_vms) - if list(vm.connected_vms): + child_vms = list(getattr(vm, 'appvms', [])) + assert all(x in vms for x in child_vms) + if child_vms: # if still something use this VM, put it at the end of queue # and try next one vms.append(vm) From a972c61914e0a5c12a2e91d9b7f6e5157fc49cfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 21 Oct 2018 16:26:39 +0200 Subject: [PATCH 133/145] tests: use socat instead of nc socat have only one variant, so one command line syntax to handle. It's also installed by default in Qubes VMs. --- qubes/tests/integ/network.py | 125 +++++++++++------------------------ 1 file changed, 39 insertions(+), 86 deletions(-) diff --git a/qubes/tests/integ/network.py b/qubes/tests/integ/network.py index 524dd5b8..1b4c9e04 100644 --- a/qubes/tests/integ/network.py +++ b/qubes/tests/integ/network.py @@ -32,10 +32,6 @@ import qubes.firewall import qubes.vm.qubesvm import qubes.vm.appvm -class NcVersion: - Trad = 1 - Nmap = 2 - # noinspection PyAttributeOutsideInit,PyPep8Naming class VmNetworkingMixin(object): @@ -63,18 +59,6 @@ class VmNetworkingMixin(object): return e.returncode return 0 - def check_nc_version(self, vm): - ''' - :type self: qubes.tests.SystemTestCase | VMNetworkingMixin - :param vm: VM where check ncat version in - ''' - if self.run_cmd(vm, 'nc -h >/dev/null 2>&1') != 0: - self.skipTest('nc not installed') - if self.run_cmd(vm, 'nc -h 2>&1|grep -q nmap.org') == 0: - return NcVersion.Nmap - else: - return NcVersion.Trad - def setUp(self): ''' :type self: qubes.tests.SystemTestCase | VMNetworkingMixin @@ -228,8 +212,6 @@ class VmNetworkingMixin(object): self.testvm1.netvm = self.proxy self.app.save() - nc_version = self.check_nc_version(self.testnetvm) - # block all for first self.testvm1.firewall.rules = [qubes.firewall.Rule(action='drop')] @@ -237,10 +219,8 @@ class VmNetworkingMixin(object): self.loop.run_until_complete(self.testvm1.start()) self.assertTrue(self.proxy.is_running()) - nc = self.loop.run_until_complete(self.testnetvm.run( - 'nc -l --send-only -e /bin/hostname -k 1234' - if nc_version == NcVersion.Nmap - else 'while nc -l -e /bin/hostname -p 1234; do true; done')) + server = self.loop.run_until_complete(self.testnetvm.run( + 'socat TCP-LISTEN:1234,fork EXEC:/bin/hostname')) try: self.assertEqual(self.run_cmd(self.proxy, self.ping_ip), 0, @@ -250,11 +230,8 @@ class VmNetworkingMixin(object): self.assertNotEqual(self.run_cmd(self.testvm1, self.ping_ip), 0, "Ping by IP should be blocked") - if nc_version == NcVersion.Nmap: - nc_cmd = "nc -w 1 --recv-only {} 1234".format(self.test_ip) - else: - nc_cmd = "nc -w 1 {} 1234".format(self.test_ip) - self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0, + client_cmd = "socat TCP:{}:1234 -".format(self.test_ip) + self.assertNotEqual(self.run_cmd(self.testvm1, client_cmd), 0, "TCP connection should be blocked") # block all except ICMP @@ -283,7 +260,7 @@ class VmNetworkingMixin(object): time.sleep(3) self.assertEqual(self.run_cmd(self.testvm1, self.ping_name), 0, "Ping by name failed (should be allowed now)") - self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0, + self.assertNotEqual(self.run_cmd(self.testvm1, client_cmd), 0, "TCP connection should be blocked") # block all except target @@ -297,7 +274,7 @@ class VmNetworkingMixin(object): # Ugly hack b/c there is no feedback when the rules are actually # applied time.sleep(3) - self.assertEqual(self.run_cmd(self.testvm1, nc_cmd), 0, + self.assertEqual(self.run_cmd(self.testvm1, client_cmd), 0, "TCP connection failed (should be allowed now)") # allow all except target @@ -312,11 +289,11 @@ class VmNetworkingMixin(object): # Ugly hack b/c there is no feedback when the rules are actually # applied time.sleep(3) - self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0, + self.assertNotEqual(self.run_cmd(self.testvm1, client_cmd), 0, "TCP connection should be blocked") finally: - nc.terminate() - self.loop.run_until_complete(nc.wait()) + server.terminate() + self.loop.run_until_complete(server.wait()) def test_040_inter_vm(self): @@ -479,8 +456,6 @@ class VmNetworkingMixin(object): self.testvm1.netvm = self.proxy self.app.save() - nc_version = self.check_nc_version(self.testnetvm) - # block all but ICMP and DNS self.testvm1.firewall.rules = [ @@ -491,10 +466,8 @@ class VmNetworkingMixin(object): self.loop.run_until_complete(self.testvm1.start()) self.assertTrue(self.proxy.is_running()) - nc = self.loop.run_until_complete(self.testnetvm.run( - 'nc -l --send-only -e /bin/hostname -k 1234' - if nc_version == NcVersion.Nmap - else 'while nc -l -e /bin/hostname -p 1234; do true; done')) + server = self.loop.run_until_complete(self.testnetvm.run( + 'socat TCP-LISTEN:1234,fork EXEC:/bin/hostname')) try: self.assertEqual(self.run_cmd(self.proxy, self.ping_ip), 0, @@ -505,15 +478,12 @@ class VmNetworkingMixin(object): "Ping by IP should be allowed") self.assertEqual(self.run_cmd(self.testvm1, self.ping_name), 0, "Ping by name should be allowed") - if nc_version == NcVersion.Nmap: - nc_cmd = "nc -w 1 --recv-only {} 1234".format(self.test_ip) - else: - nc_cmd = "nc -w 1 {} 1234".format(self.test_ip) - self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0, + client_cmd = "socat TCP:{}:1234 -".format(self.test_ip) + self.assertNotEqual(self.run_cmd(self.testvm1, client_cmd), 0, "TCP connection should be blocked") finally: - nc.terminate() - self.loop.run_until_complete(nc.wait()) + server.terminate() + self.loop.run_until_complete(server.wait()) def test_203_fake_ip_inter_vm_allow(self): '''Access VM with "fake IP" from other VM (when firewall allows) @@ -682,8 +652,6 @@ class VmNetworkingMixin(object): self.testvm1.netvm = self.proxy self.app.save() - nc_version = self.check_nc_version(self.testnetvm) - # block all but ICMP and DNS self.testvm1.firewall.rules = [ @@ -694,10 +662,8 @@ class VmNetworkingMixin(object): self.loop.run_until_complete(self.testvm1.start()) self.assertTrue(self.proxy.is_running()) - nc = self.loop.run_until_complete(self.testnetvm.run( - 'nc -l --send-only -e /bin/hostname -k 1234' - if nc_version == NcVersion.Nmap - else 'while nc -l -e /bin/hostname -p 1234; do true; done')) + server = self.loop.run_until_complete(self.testnetvm.run( + 'socat TCP-LISTEN:1234,fork EXEC:/bin/hostname')) try: self.assertEqual(self.run_cmd(self.proxy, self.ping_ip), 0, @@ -708,15 +674,12 @@ class VmNetworkingMixin(object): "Ping by IP should be allowed") self.assertEqual(self.run_cmd(self.testvm1, self.ping_name), 0, "Ping by name should be allowed") - if nc_version == NcVersion.Nmap: - nc_cmd = "nc -w 1 --recv-only {} 1234".format(self.test_ip) - else: - nc_cmd = "nc -w 1 {} 1234".format(self.test_ip) - self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0, + client_cmd = "socat TCP:{}:1234 -".format(self.test_ip) + self.assertNotEqual(self.run_cmd(self.testvm1, client_cmd), 0, "TCP connection should be blocked") finally: - nc.terminate() - self.loop.run_until_complete(nc.wait()) + server.terminate() + self.loop.run_until_complete(server.wait()) # noinspection PyAttributeOutsideInit,PyPep8Naming class VmIPv6NetworkingMixin(VmNetworkingMixin): @@ -852,9 +815,6 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): self.testvm1.netvm = self.proxy self.app.save() - if self.run_cmd(self.testnetvm, 'ncat -h') != 0: - self.skipTest('nmap ncat not installed') - # block all for first self.testvm1.firewall.rules = [qubes.firewall.Rule(action='drop')] @@ -862,8 +822,8 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): self.loop.run_until_complete(self.testvm1.start()) self.assertTrue(self.proxy.is_running()) - nc = self.loop.run_until_complete(self.testnetvm.run( - 'ncat -l --send-only -e /bin/hostname -k 1234')) + server = self.loop.run_until_complete(self.testnetvm.run( + 'socat TCP6-LISTEN:1234,fork EXEC:/bin/hostname')) try: self.assertEqual(self.run_cmd(self.proxy, self.ping6_ip), 0, @@ -873,8 +833,9 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): self.assertNotEqual(self.run_cmd(self.testvm1, self.ping6_ip), 0, "Ping by IP should be blocked") - nc_cmd = "ncat -w 1 --recv-only {} 1234".format(self.test_ip6) - self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0, + client6_cmd = "socat TCP:[{}]:1234 -".format(self.test_ip6) + client4_cmd = "socat TCP:{}:1234 -".format(self.test_ip) + self.assertNotEqual(self.run_cmd(self.testvm1, client6_cmd), 0, "TCP connection should be blocked") # block all except ICMP @@ -904,7 +865,7 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): time.sleep(3) self.assertEqual(self.run_cmd(self.testvm1, self.ping6_name), 0, "Ping by name failed (should be allowed now)") - self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0, + self.assertNotEqual(self.run_cmd(self.testvm1, client6_cmd), 0, "TCP connection should be blocked") # block all except target @@ -919,7 +880,7 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): # Ugly hack b/c there is no feedback when the rules are actually # applied time.sleep(3) - self.assertEqual(self.run_cmd(self.testvm1, nc_cmd), 0, + self.assertEqual(self.run_cmd(self.testvm1, client6_cmd), 0, "TCP connection failed (should be allowed now)") # block all except target - by name @@ -934,10 +895,9 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): # Ugly hack b/c there is no feedback when the rules are actually # applied time.sleep(3) - self.assertEqual(self.run_cmd(self.testvm1, nc_cmd), 0, + self.assertEqual(self.run_cmd(self.testvm1, client6_cmd), 0, "TCP (IPv6) connection failed (should be allowed now)") - self.assertEqual(self.run_cmd(self.testvm1, - nc_cmd.replace(self.test_ip6, self.test_ip)), + self.assertEqual(self.run_cmd(self.testvm1, client4_cmd), 0, "TCP (IPv4) connection failed (should be allowed now)") @@ -953,11 +913,11 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): # Ugly hack b/c there is no feedback when the rules are actually # applied time.sleep(3) - self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0, + self.assertNotEqual(self.run_cmd(self.testvm1, client6_cmd), 0, "TCP connection should be blocked") finally: - nc.terminate() - self.loop.run_until_complete(nc.wait()) + server.terminate() + self.loop.run_until_complete(server.wait()) def test_540_ipv6_inter_vm(self): @@ -1081,8 +1041,6 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): self.testvm1.netvm = self.proxy self.app.save() - nc_version = self.check_nc_version(self.testnetvm) - # block all but ICMP and DNS self.testvm1.firewall.rules = [ @@ -1093,10 +1051,8 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): self.loop.run_until_complete(self.testvm1.start()) self.assertTrue(self.proxy.is_running()) - nc = self.loop.run_until_complete(self.testnetvm.run( - 'nc -l --send-only -e /bin/hostname -k 1234' - if nc_version == NcVersion.Nmap - else 'while nc -l -e /bin/hostname -p 1234; do true; done')) + server = self.loop.run_until_complete(self.testnetvm.run( + 'socat TCP6-LISTEN:1234,fork EXEC:/bin/hostname')) try: self.assertEqual(self.run_cmd(self.proxy, self.ping6_ip), 0, @@ -1107,15 +1063,12 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): "Ping by IP should be allowed") self.assertEqual(self.run_cmd(self.testvm1, self.ping6_name), 0, "Ping by name should be allowed") - if nc_version == NcVersion.Nmap: - nc_cmd = "nc -w 1 --recv-only {} 1234".format(self.test_ip6) - else: - nc_cmd = "nc -w 1 {} 1234".format(self.test_ip6) - self.assertNotEqual(self.run_cmd(self.testvm1, nc_cmd), 0, + client_cmd = "socat TCP:[{}]:1234 -".format(self.test_ip6) + self.assertNotEqual(self.run_cmd(self.testvm1, client_cmd), 0, "TCP connection should be blocked") finally: - nc.terminate() - self.loop.run_until_complete(nc.wait()) + server.terminate() + self.loop.run_until_complete(server.wait()) # noinspection PyAttributeOutsideInit,PyPep8Naming class VmUpdatesMixin(object): From 0b7aa546c6a7cf3b744860e7763f05d1b01a8360 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 21 Oct 2018 16:49:20 +0200 Subject: [PATCH 134/145] tests: remove VM reference from QubesVMError Yet another place wheren object references are leaked. --- qubes/tests/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index b66a4ee7..991692bb 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -395,6 +395,8 @@ class QubesTestCase(unittest.TestCase): continue ex = exc_info[1] while ex is not None: + if isinstance(ex, qubes.exc.QubesVMError): + ex.vm = None traceback.clear_frames(ex.__traceback__) ex = ex.__context__ From f13029219b7b1312bb33cebc5299657c5056b5e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 21 Oct 2018 18:19:36 +0200 Subject: [PATCH 135/145] vm: disable/enable qubes-vm@ service when domain is removed/created If domain is set to autostart, qubes-vm@ systemd service is used to start it at boot. Cleanup the service when domain is removed, and similarly enable the service when domain is created and already have autostart=True. Fixes QubesOS/qubes-issues#4014 --- qubes/vm/qubesvm.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index faa7467b..bb4c1e03 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -874,6 +874,22 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): raise qubes.exc.QubesException( 'Failed to reset autostart for VM in systemd') + @qubes.events.handler('domain-remove-from-disk') + def on_remove_from_disk(self, event, **kwargs): + # pylint: disable=unused-argument + if self.autostart: + subprocess.call( + ['sudo', 'systemctl', 'disable', + 'qubes-vm@{}.service'.format(self.name)]) + + @qubes.events.handler('domain-create-on-disk') + def on_create_on_disk(self, event, **kwargs): + # pylint: disable=unused-argument + if self.autostart: + subprocess.call( + ['sudo', 'systemctl', 'enable', + 'qubes-vm@{}.service'.format(self.name)]) + # # methods for changing domain state # From e244c192ae1f87e4d252747eae66f8efb8c2a2f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 21 Oct 2018 19:38:21 +0200 Subject: [PATCH 136/145] tests: use /bin/uname instead of /bin/hostname as dummy output generator Use something included in coreutils installed everywhere. --- qubes/tests/integ/network.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/qubes/tests/integ/network.py b/qubes/tests/integ/network.py index 1b4c9e04..12a196b4 100644 --- a/qubes/tests/integ/network.py +++ b/qubes/tests/integ/network.py @@ -220,7 +220,7 @@ class VmNetworkingMixin(object): self.assertTrue(self.proxy.is_running()) server = self.loop.run_until_complete(self.testnetvm.run( - 'socat TCP-LISTEN:1234,fork EXEC:/bin/hostname')) + 'socat TCP-LISTEN:1234,fork EXEC:/bin/uname')) try: self.assertEqual(self.run_cmd(self.proxy, self.ping_ip), 0, @@ -467,7 +467,7 @@ class VmNetworkingMixin(object): self.assertTrue(self.proxy.is_running()) server = self.loop.run_until_complete(self.testnetvm.run( - 'socat TCP-LISTEN:1234,fork EXEC:/bin/hostname')) + 'socat TCP-LISTEN:1234,fork EXEC:/bin/uname')) try: self.assertEqual(self.run_cmd(self.proxy, self.ping_ip), 0, @@ -663,7 +663,7 @@ class VmNetworkingMixin(object): self.assertTrue(self.proxy.is_running()) server = self.loop.run_until_complete(self.testnetvm.run( - 'socat TCP-LISTEN:1234,fork EXEC:/bin/hostname')) + 'socat TCP-LISTEN:1234,fork EXEC:/bin/uname')) try: self.assertEqual(self.run_cmd(self.proxy, self.ping_ip), 0, @@ -823,7 +823,7 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): self.assertTrue(self.proxy.is_running()) server = self.loop.run_until_complete(self.testnetvm.run( - 'socat TCP6-LISTEN:1234,fork EXEC:/bin/hostname')) + 'socat TCP6-LISTEN:1234,fork EXEC:/bin/uname')) try: self.assertEqual(self.run_cmd(self.proxy, self.ping6_ip), 0, @@ -1052,7 +1052,7 @@ class VmIPv6NetworkingMixin(VmNetworkingMixin): self.assertTrue(self.proxy.is_running()) server = self.loop.run_until_complete(self.testnetvm.run( - 'socat TCP6-LISTEN:1234,fork EXEC:/bin/hostname')) + 'socat TCP6-LISTEN:1234,fork EXEC:/bin/uname')) try: self.assertEqual(self.run_cmd(self.proxy, self.ping6_ip), 0, From 8be70c9e4d22e901fb41d21b55a3828306de7f90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Tue, 23 Oct 2018 23:29:23 +0200 Subject: [PATCH 137/145] ext/services: allow for os=Linux feature request from VM It's weird to set it for Windows, but not Linux. --- qubes/ext/windows.py | 2 +- qubes/tests/ext.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qubes/ext/windows.py b/qubes/ext/windows.py index 75420975..0b417886 100644 --- a/qubes/ext/windows.py +++ b/qubes/ext/windows.py @@ -34,7 +34,7 @@ class WindowsFeatures(qubes.ext.Extension): guest_os = None if 'os' in untrusted_features: - if untrusted_features['os'] in ['Windows']: + if untrusted_features['os'] in ['Windows', 'Linux']: guest_os = untrusted_features['os'] qrexec = None diff --git a/qubes/tests/ext.py b/qubes/tests/ext.py index 5acd183f..6f59132f 100644 --- a/qubes/tests/ext.py +++ b/qubes/tests/ext.py @@ -213,7 +213,7 @@ class TC_10_WindowsFeatures(qubes.tests.QubesTestCase): 'version': '1', 'default-user': 'user', 'qrexec': '1', - 'os': 'Linux'}) + 'os': 'other'}) self.assertEqual(self.vm.mock_calls, []) class TC_20_Services(qubes.tests.QubesTestCase): From 2f3a9847425e1dbbf13ac5172518d6f2d05228c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 24 Oct 2018 14:13:07 +0200 Subject: [PATCH 138/145] exc: Make QubesMemoryError inherit from QubesVMError Same as other vm-related errors. This helps QubesTestCase.cleanup_traceback() cleanup VM reference. --- qubes/exc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes/exc.py b/qubes/exc.py index 3c7797d1..df31f60e 100644 --- a/qubes/exc.py +++ b/qubes/exc.py @@ -159,7 +159,7 @@ class BackupCancelledError(QubesException): msg or 'Backup cancelled') -class QubesMemoryError(QubesException, MemoryError): +class QubesMemoryError(QubesVMError, MemoryError): '''Cannot start domain, because not enough memory is available''' def __init__(self, vm, msg=None): super(QubesMemoryError, self).__init__( From 4742a630f21ae0506c59ed60da98f62a711469f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 24 Oct 2018 14:15:18 +0200 Subject: [PATCH 139/145] tests: use iptables --wait QubesOS/qubes-issues#3665 affects also tests... --- qubes/tests/integ/network.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qubes/tests/integ/network.py b/qubes/tests/integ/network.py index 12a196b4..8869c28b 100644 --- a/qubes/tests/integ/network.py +++ b/qubes/tests/integ/network.py @@ -104,7 +104,8 @@ class VmNetworkingMixin(object): run_netvm_cmd("ip link add test0 type dummy") run_netvm_cmd("ip link set test0 up") run_netvm_cmd("ip addr add {}/24 dev test0".format(self.test_ip)) - run_netvm_cmd("iptables -I INPUT -d {} -j ACCEPT".format(self.test_ip)) + run_netvm_cmd("iptables -I INPUT -d {} -j ACCEPT --wait".format( + self.test_ip)) # ignore failure self.run_cmd(self.testnetvm, "killall --wait dnsmasq") run_netvm_cmd("dnsmasq -a {ip} -A /{name}/{ip} -i test0 -z".format( From fb14f589cb0c449e32141893ebea8f508387ce7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 27 Oct 2018 01:41:30 +0200 Subject: [PATCH 140/145] tests: wait for full user session before doing rest of the test Clean VM shutdown may timeout if its initiated before full startup, so make sure the full startup is completed first. --- qubes/tests/integ/basic.py | 2 ++ qubes/tests/integ/storage.py | 6 ++++++ qubes/tests/integ/vm_qrexec_gui.py | 1 + 3 files changed, 9 insertions(+) diff --git a/qubes/tests/integ/basic.py b/qubes/tests/integ/basic.py index dfb57e39..e11c63ee 100644 --- a/qubes/tests/integ/basic.py +++ b/qubes/tests/integ/basic.py @@ -254,6 +254,7 @@ class TC_00_Basic(qubes.tests.SystemTestCase): try: # first boot, mkfs private volume self.loop.run_until_complete(vm.start()) + self.loop.run_until_complete(self.wait_for_session(vm)) # get private volume UUID private_uuid, _ = self.loop.run_until_complete( vm.run_for_stdio('blkid -o value /dev/xvdb', user='root')) @@ -482,6 +483,7 @@ class TC_03_QvmRevertTemplateChanges(qubes.tests.SystemTestCase): def _do_test(self): checksum_before = self.get_rootimg_checksum() self.loop.run_until_complete(self.test_template.start()) + self.loop.run_until_complete(self.wait_for_session(self.test_template)) self.shutdown_and_wait(self.test_template) checksum_changed = self.get_rootimg_checksum() if checksum_before == checksum_changed: diff --git a/qubes/tests/integ/storage.py b/qubes/tests/integ/storage.py index 253d8e72..a4766e18 100644 --- a/qubes/tests/integ/storage.py +++ b/qubes/tests/integ/storage.py @@ -77,6 +77,7 @@ class StorageTestMixin(object): del coro_maybe self.app.save() yield from (self.vm1.start()) + yield from self.wait_for_session(self.vm1) # volatile image not clean yield from (self.vm1.run_for_stdio( @@ -112,6 +113,7 @@ class StorageTestMixin(object): del coro_maybe self.app.save() yield from self.vm1.start() + yield from self.wait_for_session(self.vm1) # non-volatile image not clean yield from self.vm1.run_for_stdio( 'head -c {} /dev/zero 2>&1 | diff -q /dev/xvde - 2>&1'.format(size), @@ -197,6 +199,9 @@ class StorageTestMixin(object): self.app.save() yield from self.vm1.start() yield from self.vm2.start() + yield from asyncio.wait( + [self.wait_for_session(self.vm1), self.wait_for_session(self.vm2)]) + try: yield from self.vm1.run_for_stdio( @@ -285,6 +290,7 @@ class StorageTestMixin(object): del coro_maybe self.app.save() yield from self.vm2.start() + yield from self.wait_for_session(self.vm2) # snapshot image not clean yield from self.vm2.run_for_stdio( diff --git a/qubes/tests/integ/vm_qrexec_gui.py b/qubes/tests/integ/vm_qrexec_gui.py index 7b87e786..a6e532a1 100644 --- a/qubes/tests/integ/vm_qrexec_gui.py +++ b/qubes/tests/integ/vm_qrexec_gui.py @@ -64,6 +64,7 @@ class TC_00_AppVMMixin(object): # TODO: wait_for, timeout self.loop.run_until_complete(self.testvm1.start()) self.assertEqual(self.testvm1.get_power_state(), "Running") + self.loop.run_until_complete(self.wait_for_session(self.testvm1)) self.loop.run_until_complete(self.testvm1.shutdown(wait=True)) self.assertEqual(self.testvm1.get_power_state(), "Halted") From 84d3547f0943d910e4aaffad0c7fa7b8eb6376a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 27 Oct 2018 16:21:27 +0200 Subject: [PATCH 141/145] tests: adjust extra tests loader to work with nose2 Nose loader do not provide loader.loadTestsFromTestCase(), use loader.loadTestsFromNames() instead. --- qubes/tests/extra.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qubes/tests/extra.py b/qubes/tests/extra.py index 9eb55c92..d4ee2f34 100644 --- a/qubes/tests/extra.py +++ b/qubes/tests/extra.py @@ -195,7 +195,8 @@ def load_tests(loader, tests, pattern): for entry in pkg_resources.iter_entry_points('qubes.tests.extra'): try: for test_case in entry.load()(): - tests.addTests(loader.loadTestsFromTestCase(test_case)) + tests.addTests(loader.loadTestsFromNames([ + '{}.{}'.format(test_case.__module__, test_case.__name__)])) except Exception as err: # pylint: disable=broad-except def runTest(self): raise err From 84c321b923f215f064867e1dfcc3eabfde74847d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 27 Oct 2018 16:22:10 +0200 Subject: [PATCH 142/145] tests: increase session startup timeout for whonix-ws based VMs First boot of whonix-ws based VM take extended period of time, because a lot of files needs to be copied to private volume. This takes even more time, when verbose logging through console is enabled. Extend the timeout for that. --- qubes/tests/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index 991692bb..cd44bef5 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -1204,10 +1204,15 @@ class SystemTestCase(QubesTestCase): @asyncio.coroutine def wait_for_session(self, vm): + timeout = 30 + if getattr(vm, 'template', None) and 'whonix-ws' in vm.template.name: + # first boot of whonix-ws takes more time because of /home + # initialization, including Tor Browser copying + timeout = 120 yield from asyncio.wait_for( vm.run_service_for_stdio( 'qubes.WaitForSession', input=vm.default_user.encode()), - timeout=30) + timeout=timeout) _templates = None From 42061cb1947669c93a583ecfd06991368a14d817 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 29 Oct 2018 01:20:57 +0100 Subject: [PATCH 143/145] tests: try to collect qvm-open-in-dvm output if no editor window is shown Try to collect more details about why the test failed. This will help only if qvm-open-in-dvm exist early. On the other hand, if it hang, or remote side fails to find the right editor (which results in GUI error message), this change will not provide any more details. --- qubes/tests/integ/dispvm.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/qubes/tests/integ/dispvm.py b/qubes/tests/integ/dispvm.py index 77c47cd3..c9e2f697 100644 --- a/qubes/tests/integ/dispvm.py +++ b/qubes/tests/integ/dispvm.py @@ -253,12 +253,24 @@ class TC_20_DispVMMixin(object): self.testvm1.run_for_stdio("echo test1 > /home/user/test.txt")) p = self.loop.run_until_complete( - self.testvm1.run("qvm-open-in-dvm /home/user/test.txt")) + self.testvm1.run("qvm-open-in-dvm /home/user/test.txt", + stdout=subprocess.PIPE, stderr=subprocess.STDOUT)) # if first 5 windows isn't expected editor, there is no hope winid = None for _ in range(5): - winid = self.wait_for_window('disp[0-9]*', search_class=True) + try: + winid = self.wait_for_window('disp[0-9]*', search_class=True) + except Exception as e: + try: + self.loop.run_until_complete(asyncio.wait_for(p.wait(), 1)) + except asyncio.TimeoutError: + raise e + else: + stdout = self.loop.run_until_complete(p.stdout.read()) + self.fail( + 'qvm-open-in-dvm exited prematurely with {}: {}'.format( + p.returncode, stdout)) # get window title (window_title, _) = subprocess.Popen( ['xdotool', 'getwindowname', winid], stdout=subprocess.PIPE). \ @@ -278,7 +290,7 @@ class TC_20_DispVMMixin(object): time.sleep(0.5) self._handle_editor(winid) - self.loop.run_until_complete(p.wait()) + self.loop.run_until_complete(p.communicate()) (test_txt_content, _) = self.loop.run_until_complete( self.testvm1.run_for_stdio("cat /home/user/test.txt")) # Drop BOM if added by editor From 26a553737f62a5714fa66a5992bd0ede0ee4fe29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 29 Oct 2018 05:16:23 +0100 Subject: [PATCH 144/145] storage/lvm: minor fix for lvs command building Do not prepend 'sudo' each time - do a copy of array if that's necessary. --- qubes/storage/lvm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes/storage/lvm.py b/qubes/storage/lvm.py index 8943bb06..47324621 100644 --- a/qubes/storage/lvm.py +++ b/qubes/storage/lvm.py @@ -219,7 +219,7 @@ def _parse_lvm_cache(lvm_output): def init_cache(log=logging.getLogger('qubes.storage.lvm')): cmd = _init_cache_cmd if os.getuid() != 0: - cmd.insert(0, 'sudo') + cmd = ['sudo'] + cmd environ = os.environ.copy() environ['LC_ALL'] = 'C.utf8' p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, From 114a9db09abe0d8e3d2c9501e7bad97fa2f30d35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 29 Oct 2018 05:45:56 +0100 Subject: [PATCH 145/145] version 4.0.33 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 4d1ccce0..a6e85764 100644 --- a/version +++ b/version @@ -1 +1 @@ -4.0.32 +4.0.33