Qubes/core-admin
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'storage-properties'

Browse Source 
* storage-properties:
  storage: use None for size/usage properties if unknown
  tests: call search_pool_containing_dir with various dirs and pools
  storage: make DirectoryThinPool helper less verbose, add sudo
  api/admin: add 'included_in' to admin.pool.Info call
  storage: add Pool.included_in() method for checking nested pools
  storage: move and generalize RootThinPool helper class
  storage/kernels: refuse changes to 'rw' and 'revisions_to_keep'
  api/admin: implement admin.vm.volume.Set.rw method
  api/admin: include 'revisions_to_keep' and 'is_outdated' in volume info
This commit is contained in:
Marek Marczykowski-Górecki 2018-03-21 01:43:53 +01:00
commit e5413a3036
No known key found for this signature in database
GPG Key ID: 063938BA42CFA724
9 changed files with 312 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -30,6 +30,7 @@ ADMIN_API_METHODS_SIMPLE = \
admin.pool.volume.Resize \
admin.pool.volume.Revert \
admin.pool.volume.Set.revisions_to_keep \
admin.pool.volume.Set.rw \
admin.pool.volume.Snapshot \
admin.property.Get \
admin.property.GetDefault \
@ -100,6 +101,7 @@ ADMIN_API_METHODS_SIMPLE = \
admin.vm.volume.Resize \
admin.vm.volume.Revert \
admin.vm.volume.Set.revisions_to_keep \
admin.vm.volume.Set.rw \
admin.vm.Stats \
$(null)

53
qubes/api/admin.py
View File

@ -336,9 +336,16 @@ class QubesAdminAPI(qubes.api.AbstractQubesAPI):
# properties defined in API
volume_properties = [
'pool', 'vid', 'size', 'usage', 'rw', 'source',
'save_on_stop', 'snap_on_start']
return ''.join('{}={}\n'.format(key, getattr(volume, key)) for key in
volume_properties)
'save_on_stop', 'snap_on_start', 'revisions_to_keep', 'is_outdated']
def _serialize(value):
if callable(value):
value = value()
if value is None:
value = ''
return str(value)
return ''.join('{}={}\n'.format(key, _serialize(getattr(volume, key)))
for key in volume_properties)
@qubes.api.method('admin.vm.volume.ListSnapshots', no_payload=True,
scope='local', read=True)
@ -496,6 +503,26 @@ class QubesAdminAPI(qubes.api.AbstractQubesAPI):
self.dest.volumes[self.arg].revisions_to_keep = newvalue
self.app.save()
@qubes.api.method('admin.vm.volume.Set.rw',
scope='local', write=True)
@asyncio.coroutine
def vm_volume_set_rw(self, untrusted_payload):
assert self.arg in self.dest.volumes.keys()
try:
newvalue = qubes.property.bool(None, None,
untrusted_payload.decode('ascii'))
except (UnicodeDecodeError, ValueError):
raise qubes.api.ProtocolError('Invalid value')
del untrusted_payload
self.fire_event_for_permission(newvalue=newvalue)
if not self.dest.is_halted():
raise qubes.exc.QubesVMNotHaltedError(self.dest)
self.dest.volumes[self.arg].rw = newvalue
self.app.save()
@qubes.api.method('admin.vm.tag.List', no_payload=True,
scope='local', read=True)
@asyncio.coroutine
@ -579,19 +606,25 @@ class QubesAdminAPI(qubes.api.AbstractQubesAPI):
self.fire_event_for_permission(pool=pool)
size_info = ''
other_info = ''
pool_size = pool.size
if pool_size is not None:
other_info += 'size={}\n'.format(pool_size)
pool_usage = pool.usage
if pool_usage is not None:
other_info += 'usage={}\n'.format(pool_usage)
try:
size_info += 'size={}\n'.format(pool.size)
except NotImplementedError:
pass
try:
size_info += 'usage={}\n'.format(pool.usage)
included_in = pool.included_in(self.app)
if included_in:
other_info += 'included_in={}\n'.format(str(included_in))
except NotImplementedError:
pass
return ''.join('{}={}\n'.format(prop, val)
for prop, val in sorted(pool.config.items())) + \
size_info
other_info
@qubes.api.method('admin.pool.Add',
scope='global', write=True)

62
qubes/app.py
View File

@ -24,21 +24,19 @@ import collections
import errno
import functools
import grp
import itertools
import logging
import os
import random
import subprocess
import sys
import tempfile
import time
import traceback
import uuid
import itertools
import lxml.etree
import jinja2
import libvirt
import lxml.etree
try:
import xen.lowlevel.xs # pylint: disable=wrong-import-order
@ -548,54 +546,6 @@ class VMCollection(object):
'https://xkcd.com/221/',
'http://dilbert.com/strip/2001-10-25')[random.randint(0, 1)])
# pylint: disable=too-few-public-methods
class RootThinPool:
'''The thin pool containing the rootfs device'''
_inited = False
_volume_group = None
_thin_pool = None
@classmethod
def _init(cls):
'''Find out the thin pool containing the root device'''
if not cls._inited:
cls._inited = True
try:
rootfs = os.stat('/')
root_major = (rootfs.st_dev & 0xff00) >> 8
root_minor = rootfs.st_dev & 0xff
root_table = subprocess.check_output(["dmsetup",
"-j", str(root_major), "-m", str(root_minor),
"table"])
_start, _sectors, target_type, target_args = \
root_table.decode().split(" ", 3)
if target_type == "thin":
thin_pool_devnum, _thin_pool_id = target_args.split(" ")
with open("/sys/dev/block/{}/dm/name"
.format(thin_pool_devnum), "r") as thin_pool_tpool_f:
thin_pool_tpool = thin_pool_tpool_f.read().rstrip('\n')
if thin_pool_tpool.endswith("-tpool"):
volume_group, thin_pool, _tpool = \
thin_pool_tpool.rsplit("-", 2)
cls._volume_group = volume_group
cls._thin_pool = thin_pool
except: # pylint: disable=bare-except
pass
@classmethod
def volume_group(cls):
'''Volume group of the thin pool containing the rootfs device'''
cls._init()
return cls._volume_group
@classmethod
def thin_pool(cls):
'''Thin pool name containing the rootfs device'''
cls._init()
return cls._thin_pool
def _default_pool(app):
''' Default storage pool.
@ -616,8 +566,8 @@ def _default_pool(app):
if pool.config['thin_pool'] == thin_pool:
return pool
# no DEFAULT_LVM_POOL, or pool not defined
root_volume_group = RootThinPool.volume_group()
root_thin_pool = RootThinPool.thin_pool()
root_volume_group, root_thin_pool = \
qubes.storage.DirectoryThinPool.thin_pool('/')
if root_thin_pool:
for pool in app.pools.values():
if pool.config.get('driver', None) != 'lvm_thin':
@ -1114,8 +1064,8 @@ class Qubes(qubes.PropertyHolder):
}
assert max(self.labels.keys()) == qubes.config.max_default_label
root_volume_group = RootThinPool.volume_group()
root_thin_pool = RootThinPool.thin_pool()
root_volume_group, root_thin_pool = \
qubes.storage.DirectoryThinPool.thin_pool('/')
if root_thin_pool:
self.add_pool(

83
qubes/storage/__init__.py
View File

@ -815,15 +815,26 @@ class Pool(object):
'''
raise self._not_implemented("get_volume")
def included_in(self, app):
''' Check if this pool is physically included in another one
This works on best-effort basis, because one pool driver may not know
all the other drivers.
:param app: Qubes() object to lookup other pools in
:returns pool or None
'''
pass
@property
def size(self):
''' Storage pool size in bytes '''
raise self._not_implemented("size")
''' Storage pool size in bytes, or None if unknown '''
return None
@property
def usage(self):
''' Space used in the pool, in bytes '''
raise self._not_implemented("usage")
''' Space used in the pool in bytes, or None if unknown '''
return None
def _not_implemented(self, method_name):
''' Helper for emitting helpful `NotImplementedError` exceptions '''
@ -865,6 +876,27 @@ def isodate(seconds=time.time()):
''' Helper method which returns an iso date '''
return datetime.utcfromtimestamp(seconds).isoformat("T")
def search_pool_containing_dir(pools, dir_path):
''' Helper function looking for a pool containing given directory.
This is useful for implementing Pool.included_in method
'''
# prefer filesystem pools
for pool in pools:
if hasattr(pool, 'dir_path'):
if dir_path.startswith(pool.dir_path):
return pool
# then look for lvm
for pool in pools:
if hasattr(pool, 'thin_pool') and hasattr(pool, 'volume_group'):
if (pool.volume_group, pool.thin_pool) == \
DirectoryThinPool.thin_pool(dir_path):
return pool
return None
class VmCreationManager(object):
''' A `ContextManager` which cleans up if volume creation fails.
@ -883,3 +915,46 @@ class VmCreationManager(object):
except Exception: # pylint: disable=broad-except
pass
os.rmdir(self.vm.dir_path)
# pylint: disable=too-few-public-methods
class DirectoryThinPool:
'''The thin pool containing the device of given filesystem'''
_thin_pool = {}
@classmethod
def _init(cls, dir_path):
'''Find out the thin pool containing given filesystem'''
if dir_path not in cls._thin_pool:
cls._thin_pool[dir_path] = None, None
try:
fs_stat = os.stat(dir_path)
fs_major = (fs_stat.st_dev & 0xff00) >> 8
fs_minor = fs_stat.st_dev & 0xff
sudo = []
if os.getuid():
sudo = ['sudo']
root_table = subprocess.check_output(sudo + ["dmsetup",
"-j", str(fs_major), "-m", str(fs_minor),
"table"], stderr=subprocess.DEVNULL)
_start, _sectors, target_type, target_args = \
root_table.decode().split(" ", 3)
if target_type == "thin":
thin_pool_devnum, _thin_pool_id = target_args.split(" ")
with open("/sys/dev/block/{}/dm/name"
.format(thin_pool_devnum), "r") as thin_pool_tpool_f:
thin_pool_tpool = thin_pool_tpool_f.read().rstrip('\n')
if thin_pool_tpool.endswith("-tpool"):
volume_group, thin_pool, _tpool = \
thin_pool_tpool.rsplit("-", 2)
cls._thin_pool[dir_path] = volume_group, thin_pool
except: # pylint: disable=bare-except
pass
@classmethod
def thin_pool(cls, dir_path):
'''Thin tuple (volume group, pool name) containing given filesystem'''
cls._init(dir_path)
return cls._thin_pool[dir_path]

6
qubes/storage/file.py
View File

@ -163,6 +163,12 @@ class FilePool(qubes.storage.Pool):
statvfs = os.statvfs(self.dir_path)
return statvfs.f_frsize * (statvfs.f_blocks - statvfs.f_bfree)
def included_in(self, app):
''' Check if there is pool containing this one - either as a
filesystem or its LVM volume'''
return qubes.storage.search_pool_containing_dir(
[pool for pool in app.pools.values() if pool is not self],
self.dir_path)
class FileVolume(qubes.storage.Volume):
''' Parent class for the xen volumes implementation which expects a

46
qubes/storage/kernels.py
View File

@ -23,18 +23,21 @@
import os
import qubes.exc
import qubes.storage
from qubes.storage import Pool, StoragePoolException, Volume
class LinuxModules(Volume):
''' A volume representing a ro linux kernel '''
rw = False
def __init__(self, target_dir, kernel_version, **kwargs):
kwargs['vid'] = ''
super(LinuxModules, self).__init__(**kwargs)
self._kernel_version = kernel_version
self.target_dir = target_dir
assert self.revisions_to_keep == 0
assert self.rw is False
@property
def vid(self):
@ -104,6 +107,28 @@ class LinuxModules(Volume):
def is_outdated(self):
return False
@property
def revisions_to_keep(self):
return 0
@revisions_to_keep.setter
def revisions_to_keep(self, value):
# pylint: disable=no-self-use
if value:
raise qubes.exc.QubesValueError(
'LinuxModules supports only revisions_to_keep=0')
@property
def rw(self):
return False
@rw.setter
def rw(self, value):
# pylint: disable=no-self-use
if value:
raise qubes.exc.QubesValueError(
'LinuxModules supports only read-only volumes')
def start(self):
return self
@ -131,7 +156,7 @@ class LinuxKernel(Pool):
def __init__(self, name=None, dir_path=None):
assert dir_path, 'Missing dir_path'
super(LinuxKernel, self).__init__(name=name)
super(LinuxKernel, self).__init__(name=name, revisions_to_keep=0)
self.dir_path = dir_path
def init_volume(self, vm, volume_config):
@ -167,6 +192,23 @@ class LinuxKernel(Pool):
def setup(self):
pass
@property
def revisions_to_keep(self):
return 0
@revisions_to_keep.setter
def revisions_to_keep(self, value):
# pylint: disable=no-self-use
if value:
raise qubes.exc.QubesValueError(
'LinuxModules supports only revisions_to_keep=0')
def included_in(self, app):
''' Check if there is pool containing /var/lib/qubes/vm-kernels '''
return qubes.storage.search_pool_containing_dir(
[pool for pool in app.pools.values() if pool is not self],
self.dir_path)
@property
def volumes(self):
''' Return all known kernel volumes '''

6
qubes/storage/reflink.py
View File

@ -108,6 +108,12 @@ class ReflinkPool(qubes.storage.Pool):
statvfs = os.statvfs(self.dir_path)
return statvfs.f_frsize * (statvfs.f_blocks - statvfs.f_bfree)
def included_in(self, app):
''' Check if there is pool containing this one - either as a
filesystem or its LVM volume'''
return qubes.storage.search_pool_containing_dir(
[pool for pool in app.pools.values() if pool is not self],
self.dir_path)
class ReflinkVolume(qubes.storage.Volume):
def create(self):

57
qubes/tests/api_admin.py
View File

@ -40,7 +40,7 @@ import qubes.storage
# properties defined in API
volume_properties = [
'pool', 'vid', 'size', 'usage', 'rw', 'source',
'save_on_stop', 'snap_on_start']
'save_on_stop', 'snap_on_start', 'revisions_to_keep', 'is_outdated']
class AdminAPITestCase(qubes.tests.QubesTestCase):
@ -557,6 +557,7 @@ class TC_00_VMs(AdminAPITestCase):
usage=102400,
size=204800)
}
self.app.pools['pool1'].included_in.return_value = None
value = self.call_mgmt_func(b'admin.pool.Info', b'dom0', b'pool1')
self.assertEqual(value,
@ -566,18 +567,34 @@ class TC_00_VMs(AdminAPITestCase):
def test_151_pool_info_unsupported_size(self):
self.app.pools = {
'pool1': unittest.mock.Mock(config={
'param1': 'value1', 'param2': 'value2'})
'param1': 'value1', 'param2': 'value2'},
size=None, usage=None),
}
type(self.app.pools['pool1']).size = unittest.mock.PropertyMock(
side_effect=NotImplementedError)
type(self.app.pools['pool1']).usage = unittest.mock.PropertyMock(
side_effect=NotImplementedError)
self.app.pools['pool1'].included_in.return_value = None
value = self.call_mgmt_func(b'admin.pool.Info', b'dom0', b'pool1')
self.assertEqual(value,
'param1=value1\nparam2=value2\n')
self.assertFalse(self.app.save.called)
def test_152_pool_info_included_in(self):
self.app.pools = {
'pool1': unittest.mock.MagicMock(config={
'param1': 'value1',
'param2': 'value2'},
usage=102400,
size=204800)
}
self.app.pools['pool1'].included_in.return_value = \
self.app.pools['pool1']
self.app.pools['pool1'].__str__.return_value = 'pool1'
value = self.call_mgmt_func(b'admin.pool.Info', b'dom0', b'pool1')
self.assertEqual(value,
'param1=value1\nparam2=value2\nsize=204800\nusage=102400'
'\nincluded_in=pool1\n')
self.assertFalse(self.app.save.called)
@unittest.mock.patch('qubes.storage.pool_drivers')
@unittest.mock.patch('qubes.storage.driver_parameters')
def test_160_pool_add(self, mock_parameters, mock_drivers):
@ -2373,6 +2390,34 @@ class TC_00_VMs(AdminAPITestCase):
self.call_mgmt_func(b'admin.vm.volume.Set.revisions_to_keep',
b'test-vm1', b'private', b'abc')
def test_680_vm_volume_set_rw(self):
self.vm.volumes = unittest.mock.MagicMock()
volumes_conf = {
'keys.return_value': ['root', 'private', 'volatile', 'kernel'],
}
self.vm.volumes.configure_mock(**volumes_conf)
self.vm.storage = unittest.mock.Mock()
value = self.call_mgmt_func(b'admin.vm.volume.Set.rw',
b'test-vm1', b'private', b'True')
self.assertIsNone(value)
self.assertEqual(self.vm.volumes.mock_calls,
[unittest.mock.call.keys(),
('__getitem__', ('private',), {})])
self.assertEqual(self.vm.volumes['private'].rw, True)
self.app.save.assert_called_once_with()
def test_681_vm_volume_set_rw_invalid(self):
self.vm.volumes = unittest.mock.MagicMock()
volumes_conf = {
'keys.return_value': ['root', 'private', 'volatile', 'kernel'],
}
self.vm.volumes.configure_mock(**volumes_conf)
self.vm.storage = unittest.mock.Mock()
with self.assertRaises(AssertionError):
self.call_mgmt_func(b'admin.vm.volume.Set.revisions_to_keep',
b'test-vm1', b'private', b'abc')
self.assertFalse(self.app.save.called)
def test_990_vm_unexpected_payload(self):
methods_with_no_payload = [
b'admin.vm.List',

75
qubes/tests/storage_lvm.py
View File

@ -27,9 +27,11 @@
import os
import subprocess
import tempfile
import unittest
import qubes.tests
import qubes.storage
from qubes.storage.lvm import ThinPool, ThinVolume
if 'DEFAULT_LVM_POOL' in os.environ.keys():
@ -267,3 +269,76 @@ class TC_01_ThinPool(ThinPoolBase, qubes.tests.SystemTestCase):
self.assertEqual(volume.path, expected)
with self.assertNotRaises(qubes.exc.QubesException):
vm.start()
@skipUnlessLvmPoolExists
class TC_02_StorageHelpers(ThinPoolBase):
def setUp(self):
xml_path = '/tmp/qubes-test.xml'
self.app = qubes.Qubes.create_empty_store(store=xml_path,
clockvm=None,
updatevm=None,
offline_mode=True,
)
os.environ['QUBES_XML_PATH'] = xml_path
super(TC_02_StorageHelpers, self).setUp()
# reset cache
qubes.storage.DirectoryThinPool._thin_pool = {}
self.thin_dir = tempfile.TemporaryDirectory()
subprocess.check_call(
['sudo', 'lvcreate', '-q', '-V', '32M',
'-T', DEFAULT_LVM_POOL, '-n',
'test-file-pool'], stdout=subprocess.DEVNULL)
self.thin_dev = '/dev/{}/test-file-pool'.format(
DEFAULT_LVM_POOL.split('/')[0])
subprocess.check_call(
['sudo', 'mkfs.ext4', '-q', self.thin_dev])
subprocess.check_call(['sudo', 'mount', self.thin_dev,
self.thin_dir.name])
subprocess.check_call(['sudo', 'chmod', '777',
self.thin_dir.name])
def tearDown(self):
subprocess.check_call(['sudo', 'umount', self.thin_dir.name])
subprocess.check_call(
['sudo', 'lvremove', '-q', '-f', self.thin_dev],
stdout = subprocess.DEVNULL)
self.thin_dir.cleanup()
super(TC_02_StorageHelpers, self).tearDown()
os.unlink(self.app.store)
del self.app
for attr in dir(self):
if isinstance(getattr(self, attr), qubes.vm.BaseVM):
delattr(self, attr)
def test_000_search_thin_pool(self):
pool = qubes.storage.search_pool_containing_dir(
self.app.pools.values(), self.thin_dir.name)
self.assertEqual(pool, self.pool)
def test_001_search_none(self):
pool = qubes.storage.search_pool_containing_dir(
self.app.pools.values(), '/tmp')
self.assertIsNone(pool)
def test_002_search_subdir(self):
subdir = os.path.join(self.thin_dir.name, 'some-dir')
os.mkdir(subdir)
pool = qubes.storage.search_pool_containing_dir(
self.app.pools.values(), subdir)
self.assertEqual(pool, self.pool)
def test_003_search_file_pool(self):
subdir = os.path.join(self.thin_dir.name, 'some-dir')
file_pool_config = {
'name': 'test-file-pool',
'driver': 'file',
'dir_path': subdir
}
pool2 = self.app.add_pool(**file_pool_config)
pool = qubes.storage.search_pool_containing_dir(
self.app.pools.values(), subdir)
self.assertEqual(pool, pool2)
pool = qubes.storage.search_pool_containing_dir(
self.app.pools.values(), self.thin_dir.name)
self.assertEqual(pool, self.pool)