2017-01-18 22:16:46 +01:00
|
|
|
#
|
2015-11-07 19:01:25 +01:00
|
|
|
# The Qubes OS Project, https://www.qubes-os.org/
|
|
|
|
#
|
|
|
|
# Copyright (C) 2015 Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
|
|
|
|
#
|
2017-10-12 00:11:50 +02:00
|
|
|
# This library is free software; you can redistribute it and/or
|
|
|
|
# modify it under the terms of the GNU Lesser General Public
|
|
|
|
# License as published by the Free Software Foundation; either
|
|
|
|
# version 2.1 of the License, or (at your option) any later version.
|
2015-11-07 19:01:25 +01:00
|
|
|
#
|
2017-10-12 00:11:50 +02:00
|
|
|
# This library is distributed in the hope that it will be useful,
|
2015-11-07 19:01:25 +01:00
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
2017-10-12 00:11:50 +02:00
|
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
# Lesser General Public License for more details.
|
2015-11-07 19:01:25 +01:00
|
|
|
#
|
2017-10-12 00:11:50 +02:00
|
|
|
# You should have received a copy of the GNU Lesser General Public
|
|
|
|
# License along with this library; if not, see <https://www.gnu.org/licenses/>.
|
2017-01-18 22:16:46 +01:00
|
|
|
#
|
2019-02-19 00:54:17 +01:00
|
|
|
import shutil
|
2017-04-26 01:01:52 +02:00
|
|
|
import unittest.mock
|
2016-02-11 05:41:42 +01:00
|
|
|
import qubes.log
|
2017-07-25 05:44:10 +02:00
|
|
|
import qubes.storage
|
2016-04-15 20:10:53 +02:00
|
|
|
from qubes.exc import QubesException
|
|
|
|
from qubes.storage import pool_drivers
|
2016-04-30 20:42:46 +02:00
|
|
|
from qubes.storage.file import FilePool
|
2018-09-12 01:50:26 +02:00
|
|
|
from qubes.storage.reflink import ReflinkPool
|
2019-02-19 00:54:17 +01:00
|
|
|
from qubes.tests import SystemTestCase, QubesTestCase
|
2016-04-15 20:10:53 +02:00
|
|
|
|
|
|
|
# :pylint: disable=invalid-name
|
2016-03-20 20:29:46 +01:00
|
|
|
|
2015-11-07 19:01:25 +01:00
|
|
|
|
2017-04-26 01:01:52 +02:00
|
|
|
class TestPool(unittest.mock.Mock):
|
2020-06-22 16:03:21 +02:00
|
|
|
def __init__(self, **kwargs):
|
|
|
|
super(TestPool, self).__init__(spec=qubes.storage.Pool, **kwargs)
|
2017-07-25 05:44:10 +02:00
|
|
|
try:
|
|
|
|
self.name = kwargs['name']
|
|
|
|
except KeyError:
|
|
|
|
pass
|
2017-04-26 01:01:52 +02:00
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
return 'test'
|
|
|
|
|
2017-07-25 05:44:10 +02:00
|
|
|
def init_volume(self, vm, volume_config):
|
|
|
|
vol = unittest.mock.Mock(spec=qubes.storage.Volume)
|
|
|
|
vol.configure_mock(**volume_config)
|
|
|
|
vol.pool = self
|
|
|
|
vol.import_data.return_value = '/tmp/test-' + vm.name
|
|
|
|
return vol
|
|
|
|
|
2017-04-26 01:01:52 +02:00
|
|
|
|
2016-02-11 05:41:42 +01:00
|
|
|
class TestVM(object):
|
2016-04-15 20:10:53 +02:00
|
|
|
def __init__(self, test, template=None):
|
|
|
|
self.app = test.app
|
|
|
|
self.name = test.make_vm_name('appvm')
|
2019-07-28 22:06:30 +02:00
|
|
|
self.dir_path_prefix = 'appvms'
|
2016-07-12 18:05:10 +02:00
|
|
|
self.dir_path = '/var/lib/qubes/appvms/' + self.name
|
2016-02-11 05:41:42 +01:00
|
|
|
self.log = qubes.log.get_vm_logger(self.name)
|
|
|
|
|
2016-04-15 20:10:53 +02:00
|
|
|
if template:
|
|
|
|
self.template = template
|
|
|
|
|
2016-02-11 05:41:42 +01:00
|
|
|
def is_template(self):
|
2016-04-15 20:10:53 +02:00
|
|
|
# :pylint: disable=no-self-use
|
2016-02-11 05:41:42 +01:00
|
|
|
return False
|
|
|
|
|
|
|
|
def is_disposablevm(self):
|
2016-04-15 20:10:53 +02:00
|
|
|
# :pylint: disable=no-self-use
|
2016-02-11 05:41:42 +01:00
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
|
|
class TestTemplateVM(TestVM):
|
2016-04-15 20:10:53 +02:00
|
|
|
dir_path_prefix = qubes.config.system_path['qubes_templates_dir']
|
|
|
|
|
2016-07-12 18:05:10 +02:00
|
|
|
def __init__(self, test, template=None):
|
|
|
|
super(TestTemplateVM, self).__init__(test, template)
|
|
|
|
self.dir_path = '/var/lib/qubes/vm-templates/' + self.name
|
|
|
|
|
2016-02-11 05:41:42 +01:00
|
|
|
def is_template(self):
|
|
|
|
return True
|
|
|
|
|
2016-03-20 20:29:46 +01:00
|
|
|
|
2016-02-11 05:41:42 +01:00
|
|
|
class TestDisposableVM(TestVM):
|
|
|
|
def is_disposablevm(self):
|
|
|
|
return True
|
|
|
|
|
2016-06-16 21:09:48 +02:00
|
|
|
class TestApp(qubes.Qubes):
|
2016-07-12 18:05:10 +02:00
|
|
|
def __init__(self, *args, **kwargs): # pylint: disable=unused-argument
|
2016-06-16 21:09:48 +02:00
|
|
|
super(TestApp, self).__init__('/tmp/qubes-test.xml',
|
|
|
|
load=False, offline_mode=True, **kwargs)
|
|
|
|
self.load_initial_values()
|
2019-02-19 00:54:17 +01:00
|
|
|
self.default_pool = self.pools['varlibqubes']
|
2015-11-07 19:01:25 +01:00
|
|
|
|
2019-02-19 00:54:17 +01:00
|
|
|
class TC_00_Pool(QubesTestCase):
|
2015-11-08 21:23:07 +01:00
|
|
|
""" This class tests the utility methods from :mod:``qubes.storage`` """
|
|
|
|
|
2016-02-11 05:41:42 +01:00
|
|
|
def setUp(self):
|
2016-03-20 20:29:46 +01:00
|
|
|
super(TC_00_Pool, self).setUp()
|
2019-02-19 00:54:17 +01:00
|
|
|
self.basedir_patch = unittest.mock.patch('qubes.config.qubes_base_dir',
|
|
|
|
'/tmp/qubes-test-basedir')
|
|
|
|
self.basedir_patch.start()
|
2016-06-16 21:09:48 +02:00
|
|
|
self.app = TestApp()
|
2016-03-20 20:29:46 +01:00
|
|
|
|
2019-02-19 00:54:17 +01:00
|
|
|
def tearDown(self):
|
|
|
|
self.basedir_patch.stop()
|
|
|
|
self.app.close()
|
|
|
|
del self.app
|
|
|
|
shutil.rmtree('/tmp/qubes-test-basedir', ignore_errors=True)
|
|
|
|
super().tearDown()
|
|
|
|
|
2016-03-20 20:29:46 +01:00
|
|
|
def test_000_unknown_pool_driver(self):
|
|
|
|
# :pylint: disable=protected-access
|
|
|
|
""" Expect an exception when unknown pool is requested"""
|
2016-04-15 20:10:53 +02:00
|
|
|
with self.assertRaises(QubesException):
|
|
|
|
self.app.get_pool('foo-bar')
|
2016-03-20 20:29:46 +01:00
|
|
|
|
|
|
|
def test_001_all_pool_drivers(self):
|
file-reflink, a storage driver optimized for CoW filesystems
This adds the file-reflink storage driver. It is never selected
automatically for pool creation, especially not the creation of
'varlibqubes' (though it can be used if set up manually).
The code is quite small:
reflink.py lvm.py file.py + block-snapshot
sloccount 334 lines 447 (134%) 570 (171%)
Background: btrfs and XFS (but not yet ZFS) support instant copies of
individual files through the 'FICLONE' ioctl behind 'cp --reflink'.
Which file-reflink uses to snapshot VM image files without an extra
device-mapper layer. All the snapshots are essentially freestanding;
there's no functional origin vs. snapshot distinction.
In contrast to 'file'-on-btrfs, file-reflink inherently avoids
CoW-on-CoW. Which is a bigger issue now on R4.0, where even AppVMs'
private volumes are CoW. (And turning off the lower, filesystem-level
CoW for 'file'-on-btrfs images would turn off data checksums too, i.e.
protection against bit rot.)
Also in contrast to 'file', all storage features are supported,
including
- any number of revisions_to_keep
- volume.revert()
- volume.is_outdated
- online fstrim/discard
Example tree of a file-reflink pool - *-dirty.img are connected to Xen:
- /var/lib/testpool/appvms/foo/volatile-dirty.img
- /var/lib/testpool/appvms/foo/root-dirty.img
- /var/lib/testpool/appvms/foo/root.img
- /var/lib/testpool/appvms/foo/private-dirty.img
- /var/lib/testpool/appvms/foo/private.img
- /var/lib/testpool/appvms/foo/private.img@2018-01-02T03:04:05Z
- /var/lib/testpool/appvms/foo/private.img@2018-01-02T04:05:06Z
- /var/lib/testpool/appvms/foo/private.img@2018-01-02T05:06:07Z
- /var/lib/testpool/appvms/bar/...
- /var/lib/testpool/appvms/...
- /var/lib/testpool/template-vms/fedora-26/...
- /var/lib/testpool/template-vms/...
It looks similar to a 'file' pool tree, and in fact file-reflink is
drop-in compatible:
$ qvm-shutdown --all --wait
$ systemctl stop qubesd
$ sed 's/ driver="file"/ driver="file-reflink"/g' -i.bak /var/lib/qubes/qubes.xml
$ systemctl start qubesd
$ sudo rm -f /path/to/pool/*/*/*-cow.img*
If the user tries to create a fresh file-reflink pool on a filesystem
that doesn't support reflinks, qvm-pool will abort and mention the
'setup_check=no' option. Which can be passed to force a fallback on
regular sparse copies, with of course lots of time/space overhead. The
same fallback code is also used when initially cloning a VM from a
foreign pool, or from another file-reflink pool on a different
mountpoint.
'journalctl -fu qubesd' will show all file-reflink copy/rename/remove
operations on VM creation/startup/shutdown/etc.
2018-02-12 22:20:05 +01:00
|
|
|
""" Expect all our pool drivers (and only them) """
|
|
|
|
self.assertCountEqual(
|
2020-06-30 12:45:33 +02:00
|
|
|
['linux-kernel', 'lvm_thin', 'file', 'file-reflink', 'callback'],
|
file-reflink, a storage driver optimized for CoW filesystems
This adds the file-reflink storage driver. It is never selected
automatically for pool creation, especially not the creation of
'varlibqubes' (though it can be used if set up manually).
The code is quite small:
reflink.py lvm.py file.py + block-snapshot
sloccount 334 lines 447 (134%) 570 (171%)
Background: btrfs and XFS (but not yet ZFS) support instant copies of
individual files through the 'FICLONE' ioctl behind 'cp --reflink'.
Which file-reflink uses to snapshot VM image files without an extra
device-mapper layer. All the snapshots are essentially freestanding;
there's no functional origin vs. snapshot distinction.
In contrast to 'file'-on-btrfs, file-reflink inherently avoids
CoW-on-CoW. Which is a bigger issue now on R4.0, where even AppVMs'
private volumes are CoW. (And turning off the lower, filesystem-level
CoW for 'file'-on-btrfs images would turn off data checksums too, i.e.
protection against bit rot.)
Also in contrast to 'file', all storage features are supported,
including
- any number of revisions_to_keep
- volume.revert()
- volume.is_outdated
- online fstrim/discard
Example tree of a file-reflink pool - *-dirty.img are connected to Xen:
- /var/lib/testpool/appvms/foo/volatile-dirty.img
- /var/lib/testpool/appvms/foo/root-dirty.img
- /var/lib/testpool/appvms/foo/root.img
- /var/lib/testpool/appvms/foo/private-dirty.img
- /var/lib/testpool/appvms/foo/private.img
- /var/lib/testpool/appvms/foo/private.img@2018-01-02T03:04:05Z
- /var/lib/testpool/appvms/foo/private.img@2018-01-02T04:05:06Z
- /var/lib/testpool/appvms/foo/private.img@2018-01-02T05:06:07Z
- /var/lib/testpool/appvms/bar/...
- /var/lib/testpool/appvms/...
- /var/lib/testpool/template-vms/fedora-26/...
- /var/lib/testpool/template-vms/...
It looks similar to a 'file' pool tree, and in fact file-reflink is
drop-in compatible:
$ qvm-shutdown --all --wait
$ systemctl stop qubesd
$ sed 's/ driver="file"/ driver="file-reflink"/g' -i.bak /var/lib/qubes/qubes.xml
$ systemctl start qubesd
$ sudo rm -f /path/to/pool/*/*/*-cow.img*
If the user tries to create a fresh file-reflink pool on a filesystem
that doesn't support reflinks, qvm-pool will abort and mention the
'setup_check=no' option. Which can be passed to force a fallback on
regular sparse copies, with of course lots of time/space overhead. The
same fallback code is also used when initially cloning a VM from a
foreign pool, or from another file-reflink pool on a different
mountpoint.
'journalctl -fu qubesd' will show all file-reflink copy/rename/remove
operations on VM creation/startup/shutdown/etc.
2018-02-12 22:20:05 +01:00
|
|
|
pool_drivers())
|
2016-03-20 20:29:46 +01:00
|
|
|
|
|
|
|
def test_002_get_pool_klass(self):
|
2018-09-12 01:50:26 +02:00
|
|
|
""" Expect the default pool to be `FilePool` or `ReflinkPool` """
|
2016-03-20 20:29:46 +01:00
|
|
|
# :pylint: disable=protected-access
|
2017-07-25 05:23:17 +02:00
|
|
|
result = self.app.get_pool('varlibqubes')
|
2018-09-12 01:50:26 +02:00
|
|
|
self.assertTrue(isinstance(result, FilePool)
|
|
|
|
or isinstance(result, ReflinkPool))
|
2015-11-07 19:01:25 +01:00
|
|
|
|
2016-03-20 20:29:46 +01:00
|
|
|
def test_003_pool_exists_default(self):
|
2015-11-07 19:32:03 +01:00
|
|
|
""" Expect the default pool to exists """
|
2017-07-25 05:23:17 +02:00
|
|
|
self.assertPoolExists('varlibqubes')
|
2015-11-07 19:32:03 +01:00
|
|
|
|
2016-04-15 20:10:53 +02:00
|
|
|
def test_004_add_remove_pool(self):
|
2015-11-07 20:53:50 +01:00
|
|
|
""" Tries to adding and removing a pool. """
|
|
|
|
pool_name = 'asdjhrp89132'
|
|
|
|
|
|
|
|
# make sure it's really does not exist
|
2019-02-18 21:13:24 +01:00
|
|
|
self.loop.run_until_complete(self.app.remove_pool(pool_name))
|
2016-04-15 20:10:53 +02:00
|
|
|
self.assertFalse(self.assertPoolExists(pool_name))
|
|
|
|
|
2019-02-18 21:13:24 +01:00
|
|
|
self.loop.run_until_complete(
|
|
|
|
self.app.add_pool(name=pool_name,
|
2016-04-30 20:42:46 +02:00
|
|
|
driver='file',
|
2019-02-18 21:13:24 +01:00
|
|
|
dir_path='/tmp/asdjhrp89132'))
|
2016-04-15 20:10:53 +02:00
|
|
|
self.assertTrue(self.assertPoolExists(pool_name))
|
2015-11-07 20:53:50 +01:00
|
|
|
|
2019-02-18 21:13:24 +01:00
|
|
|
self.loop.run_until_complete(self.app.remove_pool(pool_name))
|
2016-04-15 20:10:53 +02:00
|
|
|
self.assertFalse(self.assertPoolExists(pool_name))
|
2015-11-07 20:53:50 +01:00
|
|
|
|
2016-04-15 20:10:53 +02:00
|
|
|
def assertPoolExists(self, pool):
|
|
|
|
""" Check if specified pool exists """
|
|
|
|
return pool in self.app.pools.keys()
|
2019-02-19 00:54:44 +01:00
|
|
|
|
|
|
|
def test_005_remove_used(self):
|
|
|
|
pool_name = 'test-pool-asdf'
|
|
|
|
|
|
|
|
dir_path = '/tmp/{}'.format(pool_name)
|
|
|
|
pool = self.loop.run_until_complete(
|
|
|
|
self.app.add_pool(name=pool_name,
|
|
|
|
driver='file',
|
|
|
|
dir_path=dir_path))
|
|
|
|
self.addCleanup(shutil.rmtree, dir_path)
|
|
|
|
vm = self.app.add_new_vm('StandaloneVM', label='red',
|
|
|
|
name=self.make_vm_name('vm'))
|
|
|
|
self.loop.run_until_complete(vm.create_on_disk(pool=pool))
|
|
|
|
with self.assertRaises(qubes.exc.QubesPoolInUseError):
|
|
|
|
self.loop.run_until_complete(self.app.remove_pool(pool_name))
|
2019-11-15 17:45:09 +01:00
|
|
|
|
|
|
|
def test_006_pool_usage_qubespool(self):
|
|
|
|
qubes_pool = self.app.get_pool('varlibqubes')
|
|
|
|
|
|
|
|
usage_details = qubes_pool.usage_details
|
|
|
|
|
|
|
|
self.assertIsNotNone(usage_details['data_size'],
|
|
|
|
"Data size incorrectly reported as None")
|
|
|
|
self.assertIsNotNone(usage_details['data_usage'],
|
|
|
|
"Data usage incorrectly reported as None")
|
|
|
|
|
|
|
|
def test_007_pool_kernels_usage(self):
|
|
|
|
pool_name = 'kernels_test_pools'
|
|
|
|
dir_path = '/tmp/{}'.format(pool_name)
|
|
|
|
pool = self.loop.run_until_complete(
|
|
|
|
self.app.add_pool(name=pool_name,
|
|
|
|
driver='linux-kernel',
|
|
|
|
dir_path=dir_path))
|
|
|
|
self.assertTrue(self.assertPoolExists(pool_name))
|
|
|
|
|
|
|
|
qubes_pool = self.app.get_pool(pool_name)
|
|
|
|
|
|
|
|
usage_details = qubes_pool.usage_details
|
|
|
|
|
|
|
|
self.assertEqual(usage_details,
|
|
|
|
{},
|
|
|
|
"Kernels pool should not report usage details.")
|
|
|
|
|
|
|
|
self.loop.run_until_complete(self.app.remove_pool(pool_name))
|