Refactor Storage, Pool and XenPool

- Remove all *_dev_config methods
 - Checks if a storage image exists moved to XenPool
 - Storage.remove wraps Pool.remove()
 - Stop volumes on domain sutdown/kill
 - Warn when using deprecated methods
This commit is contained in:
Bahtiar `kalkin-` Gadimov 2016-04-15 20:40:53 +02:00
parent 3c66d4b54c
commit bdfb85ac19
4 changed files with 406 additions and 557 deletions

View File

@ -39,7 +39,6 @@ import qubes.exc
import qubes.utils
from qubes.devices import BlockDevice
BLKSIZE = 512
STORAGE_ENTRY_POINT = 'qubes.storage'
@ -93,91 +92,20 @@ class Storage(object):
in mind.
'''
modules_dev = 'xvdd'
def __init__(self, vm):
#: Domain for which we manage storage
self.vm = vm
self.log = self.vm.log
#: Additional drive (currently used only by HVM)
self.drive = None
self.pools = {}
if hasattr(vm, 'volume_config'):
for name, conf in self.vm.volume_config.items():
assert 'pool' in conf
pool = get_pool(conf['pool'], self.vm)
self.vm.volumes[name] = pool.init_volume(conf)
@property
def root_img(self):
pool = self.get_pool()
return pool.root_img
@property
def private_img(self):
pool = self.get_pool()
return pool.private_img
@property
def volatile_img(self):
pool = self.get_pool()
return pool.volatile_img
@property
def rootcow_img(self):
pool = self.get_pool()
return pool.rootcow_img
def get_config_params(self):
args = {}
args['rootdev'] = self.root_dev_config()
args['privatedev'] = self.private_dev_config()
args['volatiledev'] = self.volatile_dev_config()
args['otherdevs'] = self.other_dev_config()
args['kerneldir'] = self.kernels_dir
return args
def root_dev_config(self):
pool = self.get_pool()
return pool.root_dev_config()
def private_dev_config(self):
pool = self.get_pool()
return pool.private_dev_config()
def volatile_dev_config(self):
pool = self.get_pool()
return pool.volatile_dev_config()
def modules_dev_config(self):
return self.format_disk_dev(self.modules_img,
'kernel',
rw=self.modules_img_rw)
def other_dev_config(self):
if self.modules_img is not None:
return self.modules_dev_config()
elif self.drive is not None:
(drive_type, drive_domain, drive_path) = self.drive.split(":")
if drive_type == 'hd':
drive_type = 'disk'
rw = (drive_type == 'disk')
if drive_domain.lower() == "dom0":
drive_domain = None
return self.format_disk_dev(drive_path,
'other',
rw=rw,
devtype=drive_type,
domain=drive_domain)
else:
return ''
def format_disk_dev(self, path, name, script=None, rw=True, devtype='disk',
domain=None):
return BlockDevice(path, name, script, rw, domain, devtype)
assert 'pool' in conf, "Pool missing in volume_config" % str(
conf)
pool = self.vm.app.get_pool(conf['pool'])
self.vm.volumes[name] = pool.init_volume(self.vm, conf)
self.pools[name] = pool
@property
def kernels_dir(self):
@ -186,51 +114,33 @@ class Storage(object):
If :py:attr:`self.vm.kernel` is :py:obj:`None`, the this points inside
:py:attr:`self.vm.dir_path`
'''
return os.path.join(qubes.config.system_path['qubes_base_dir'],
qubes.config.system_path['qubes_kernels_base_dir'], self.vm.kernel)\
if self.vm.kernel is not None \
else os.path.join(self.vm.dir_path,
qubes.config.vm_files['kernels_subdir'])
@property
def modules_img(self):
'''Path to image with modules.
Depending on domain, this may be global or inside domain's dir.
'''
modules_path = os.path.join(self.kernels_dir, 'modules.img')
if os.path.exists(modules_path):
return modules_path
else:
return None
@property
def modules_img_rw(self):
''':py:obj:`True` if module image should be mounted RW, :py:obj:`False`
otherwise.'''
return self.vm.kernel is None
assert 'kernel' in self.vm.volumes, "VM has no kernel pool"
return self.vm.volumes['kernel'].kernels_dir
def get_disk_utilization(self):
return get_disk_usage(self.vm.dir_path)
''' Returns summed up disk utilization for all domain volumes '''
result = 0
for volume in self.vm.volumes.values():
result += volume.usage
return result
# TODO Remove this wrapper
def get_disk_utilization_private_img(self):
# pylint: disable=invalid-name
return get_disk_usage(self.private_img)
# pylint: disable=invalid-name,missing-docstring
return self.vm.volume['private'].usage
# TODO Remove this wrapper
def get_private_img_sz(self):
if not os.path.exists(self.private_img):
return 0
# :pylint: disable=missing-docstring
return self.vm.volume['private'].size
return os.path.getsize(self.private_img)
def resize_private_img(self, size):
raise NotImplementedError()
def resize(self, volume, size):
''' Resize volume '''
self.get_pool(volume).resize(volume, size)
# TODO rename it to create()
def create_on_disk(self, source_template=None):
# :pylint: disable=missing-docstring
if source_template is None and hasattr(self.vm, 'template'):
source_template = self.vm.template
@ -238,14 +148,17 @@ class Storage(object):
self.vm.log.info('Creating directory: {0}'.format(self.vm.dir_path))
os.makedirs(self.vm.dir_path)
pool = self.get_pool()
pool.create_on_disk_private_img(source_template)
pool.create_on_disk_root_img(source_template)
pool.reset_volatile_storage()
for name, volume in self.vm.volumes.items():
source_volume = None
if source_template and hasattr(source_template, 'volumes'):
source_volume = source_template.volumes[name]
self.get_pool(volume).create(volume, source_volume=source_volume)
os.umask(old_umask)
# TODO migrate this
def clone_disk_files(self, src_vm):
# :pylint: disable=missing-docstring
self.vm.log.info('Creating directory: {0}'.format(self.vm.dir_path))
os.mkdir(self.vm.dir_path)
@ -255,14 +168,15 @@ class Storage(object):
self._copy_file(src_vm.private_img, self.vm.private_img)
if src_vm.updateable and hasattr(src_vm, 'root_img'):
self.vm.log.info('Copying the root image: {} -> {}'.format(
src_vm.root_img, self.root_img))
self._copy_file(src_vm.root_img, self.root_img)
# TODO: modules?
# XXX which modules? -woju
self.vm.log.info(
'Copying the root image: {} -> {}'.format(
src_vm.volume['root'].path_origin,
self.vm.volume['root'].path_origin)
)
self._copy_file(src_vm.volume['root'].path_origin,
self.vm.volume['root'].path_origin)
# TODO migrate this
@staticmethod
def rename(newpath, oldpath):
'''Move storage directory, most likely during domain's rename.
@ -280,133 +194,87 @@ class Storage(object):
os.rename(oldpath, newpath)
def verify_files(self):
'''Verify that the storage is sane.
On success, returns normally. On failure, raises exception.
'''
if not os.path.exists(self.vm.dir_path):
raise qubes.exc.QubesVMError(self.vm,
raise qubes.exc.QubesVMError(
self.vm,
'VM directory does not exist: {}'.format(self.vm.dir_path))
if hasattr(self.vm, 'root_img') and not os.path.exists(self.root_img):
raise qubes.exc.QubesVMError(self.vm,
'VM root image file does not exist: {}'.format(self.root_img))
if hasattr(self.vm, 'private_img') \
and not os.path.exists(self.private_img):
raise qubes.exc.QubesVMError(self.vm,
'VM private image file does not exist: {}'.format(
self.private_img))
if self.modules_img is not None \
and not os.path.exists(self.modules_img):
raise qubes.exc.QubesVMError(self.vm,
'VM kernel modules image does not exists: {}'.format(
self.modules_img))
def remove_from_disk(self):
def remove(self):
for name, volume in self.vm.volumes.items():
self.log.info('Removing volume %s: %s' % (name, volume.vid))
self.get_pool(volume).remove(volume)
shutil.rmtree(self.vm.dir_path)
def start(self):
''' Execute the start method on each pool '''
for volume in self.vm.volumes.values():
self.get_pool(volume).start(volume)
def stop(self):
''' Execute the start method on each pool '''
for volume in self.vm.volumes.values():
self.get_pool(volume).stop(volume)
def prepare_for_vm_startup(self):
pool = get_pool(self.vm.pool_name, self.vm)
pool.reset_volatile_storage()
if hasattr(self.vm, 'private_img') \
and not os.path.exists(self.private_img):
self.vm.log.info('Creating empty VM private image file: {0}'.format(
pool.private_img))
pool.create_on_disk_private_img()
def get_pool(self):
return get_pool(self.vm.pool_name, self.vm)
def get_pool(self, volume):
''' Helper function '''
assert isinstance(volume, Volume), "You need to pass a Volume"
return self.pools[volume.name]
def commit_template_changes(self):
pool = self.get_pool()
pool.commit_template_changes()
for volume in self.vm.volumes.values():
if volume.volume_type == 'origin':
self.get_pool(volume).commit_template_changes(volume)
def get_disk_usage_one(st):
'''Extract disk usage of one inode from its stat_result struct.
If known, get real disk usage, as written to device by filesystem, not
logical file size. Those values may be different for sparse files.
:param os.stat_result st: stat result
:returns: disk usage
'''
try:
return st.st_blocks * BLKSIZE
except AttributeError:
return st.st_size
def get_disk_usage(path):
'''Get real disk usage of given path (file or directory).
When *path* points to directory, then it is evaluated recursively.
This function tries estiate real disk usage. See documentation of
:py:func:`get_disk_usage_one`.
:param str path: path to evaluate
:returns: disk usage
'''
try:
st = os.lstat(path)
except OSError:
return 0
ret = get_disk_usage_one(st)
# if path is not a directory, this is skipped
for dirpath, dirnames, filenames in os.walk(path):
for name in dirnames + filenames:
ret += get_disk_usage_one(os.lstat(os.path.join(dirpath, name)))
return ret
class Pool(object):
''' A Pool is used to manage different kind of volumes (File
based/LVM/Btrfs/...).
3rd Parties providing own storage implementations will need to extend
this class.
'''
private_img_size = qubes.config.defaults['private_img_size']
root_img_size = qubes.config.defaults['root_img_size']
def __init__(self, vm=None, name=None, **kwargs):
assert vm
def __init__(self, name=None, **kwargs):
# :pylint: disable=unused-argument
assert name, "Pool name is missing"
self.vm = vm
self.name = name
def root_dev_config(self):
raise NotImplementedError()
def create(self, volume, source_volume):
''' Create the given volume on disk or copy from provided
`source_volume`.
'''
raise NotImplementedError("Pool %s has create() not implemented" %
self.name)
def private_dev_config(self):
raise NotImplementedError()
def commit_template_changes(self, volume):
''' Update origin device '''
raise NotImplementedError(
"Pool %s has commit_template_changes() not implemented" %
self.name)
def volatile_dev_config(self):
raise NotImplementedError()
@property
def config(self):
''' Returns the pool config to be written to qubes.xml '''
raise NotImplementedError("Pool %s has config() not implemented" %
self.name)
def create_on_disk_private_img(self, source_template=None):
raise NotImplementedError()
def create_on_disk_root_img(self, source_template=None):
raise NotImplementedError()
def create_dir_if_not_exists(self, path):
""" Check if a directory exists in if not create it.
This method does not create any parent directories.
"""
if not os.path.exists(path):
os.mkdir(path)
def commit_template_changes(self):
raise NotImplementedError()
@staticmethod
def _copy_file(source, destination):
'''Effective file copy, preserving sparse files etc.
'''
# TODO: Windows support
# We prefer to use Linux's cp, because it nicely handles sparse files
assert os.path.exists(source), \
"Missing the source %s to copy from" % source
assert not os.path.exists(destination), \
"Destination %s already exists" % destination
try:
subprocess.check_call(['cp', '--reflink=auto', source, destination
])
@ -414,32 +282,32 @@ class Pool(object):
raise IOError('Error while copying {!r} to {!r}'.format(
source, destination))
def format_disk_dev(self, path, vdev, script=None, rw=True, devtype='disk',
domain=None):
def remove(self, volume):
''' Remove volume'''
raise NotImplementedError("Pool %s has remove() not implemented" %
self.name)
return BlockDevice(path, vdev, script, rw, domain, devtype)
def reset_volatile_storage(self):
# Re-create only for template based VMs
try:
if self.vm.template is not None and self.volatile_img:
if os.path.exists(self.volatile_img):
os.remove(self.volatile_img)
except AttributeError: # self.vm.template
pass
def clone(self, source, target):
''' Clone volume '''
raise NotImplementedError("Pool %s has clone() not implemented" %
self.name)
# For StandaloneVM create it only if not already exists
# (eg after backup-restore)
if hasattr(self, 'volatile_img') \
and not os.path.exists(self.volatile_img):
self.vm.log.info(
'Creating volatile image: {0}'.format(self.volatile_img))
subprocess.check_call(
[qubes.config.system_path["prepare_volatile_img_cmd"],
self.volatile_img,
str(self.root_img_size / 1024 / 1024)])
def start(self, volume):
''' Do what ever is needed on start '''
raise NotImplementedError("Pool %s has start() not implemented" %
self.name)
def stop(self, volume):
''' Do what ever is needed on stop'''
raise NotImplementedError("Pool %s has stop() not implemented" %
self.name)
def init_volume(self, volume_config):
raise NotImplementedError()
''' Initialize a :py:class:`qubes.storage.Volume` from `volume_config`.
'''
raise NotImplementedError("Pool %s has init_volume() not implemented" %
self.name)
def pool_drivers():
""" Return a list of EntryPoints names """

View File

@ -31,136 +31,69 @@ import os.path
import re
import subprocess
import qubes
import qubes.config
import qubes.vm.templatevm
from qubes.storage import Pool, StoragePoolException, Volume
BLKSIZE = 512
class XenPool(Pool):
''' File based 'original' disk implementation '''
root_dev = 'xvda'
private_dev = 'xvdb'
volatile_dev = 'xvdc'
def __init__(self, vm=None, name=None, dir_path=None):
super(XenPool, self).__init__(vm=vm, name=name)
def __init__(self, name=None, dir_path=None):
super(XenPool, self).__init__(name=name)
assert dir_path, "No pool dir_path specified"
self.dir_path = os.path.normpath(dir_path)
self.create_dir_if_not_exists(self.dir_path)
create_dir_if_not_exists(self.dir_path)
appvms_path = os.path.join(self.dir_path, 'appvms')
self.create_dir_if_not_exists(appvms_path)
create_dir_if_not_exists(appvms_path)
vm_templates_path = os.path.join(self.dir_path, 'vm-templates')
self.create_dir_if_not_exists(vm_templates_path)
create_dir_if_not_exists(vm_templates_path)
@property
def private_img(self):
'''Path to the private image'''
return self.abspath(qubes.config.vm_files['private_img'])
def create(self, volume, source_volume=None):
_type = volume.volume_type
size = volume.size
if _type == 'origin':
create_sparse_file(volume.path_origin, size)
create_sparse_file(volume.path_cow, size)
elif _type in ['read-write'] and source_volume:
copy_file(source_volume.path, volume.path)
elif _type in ['read-write', 'volatile']:
create_sparse_file(volume.path, size)
@property
def root_img(self):
'''Path to the root image'''
return self.vm.template.storage.root_img \
if hasattr(self.vm, 'template') and self.vm.template \
else self.abspath(qubes.config.vm_files['root_img'])
return volume
@property
def rootcow_img(self):
'''Path to the root COW image'''
def resize(self, volume, size):
''' Expands volume, throws
:py:class:`qubst.storage.StoragePoolException` if given size is
less than current_size
'''
_type = volume.volume_type
if _type not in ['origin', 'read-write', 'volatile']:
raise StoragePoolException('Can not resize a %s volume %s' %
(_type, volume.vid))
if isinstance(self.vm, qubes.vm.templatevm.TemplateVM):
return self.abspath(qubes.config.vm_files['rootcow_img'])
if size <= volume.size:
raise StoragePoolException(
'For your own safety, shrinking of %s is'
' disabled. If you really know what you'
' are doing, use `truncate` on %s manually.' %
(volume.name, volume.vid))
return None
if _type == 'origin':
path = volume.path_origin
elif _type in ['read-write', 'volatile']:
path = volume.path
@property
def volatile_img(self):
'''Path to the volatile image'''
return self.abspath(qubes.config.vm_files['volatile_img'])
if size <= volume.size:
raise StoragePoolException('Can not shring volume %s' %
volume.name)
def root_dev_config(self):
dev_name = 'root'
if isinstance(self.vm, qubes.vm.templatevm.TemplateVM):
return self.format_disk_dev(
'{root}:{rootcow}'.format(
root=self.root_img,
rootcow=self.rootcow_img),
dev_name,
script='block-origin')
elif self.vm.hvm and hasattr(self.vm, 'template'):
# HVM template-based VM - only one device-mapper layer, in dom0
# (root+volatile)
# HVM detection based on 'kernel' property is massive hack,
# but taken from assumption that VM needs Qubes-specific kernel
# (actually initramfs) to assemble the second layer of device-mapper
return self.format_disk_dev(
'{root}:{volatile}'.format(
root=self.vm.template.storage.root_img,
volatile=self.volatile_img),
dev_name,
script='block-snapshot')
elif hasattr(self.vm, 'template'):
# any other template-based VM - two device-mapper layers: one
# in dom0 (here) from root+root-cow, and another one from
# this+volatile.img
path = '{root}:{template_rootcow}'.format(
root=self.root_img,
template_rootcow=self.vm.template.storage.rootcow_img)
return self.format_disk_dev(path=path,
vdev=self.root_dev,
script='block-snapshot',
rw=False)
else:
# standalone qube
return self.format_disk_dev(self.root_img, dev_name)
def private_dev_config(self):
return self.format_disk_dev(self.private_img, 'private')
def volatile_dev_config(self):
return self.format_disk_dev(self.volatile_img, 'volatile')
def create_on_disk_private_img(self, source_template=None):
if not os.path.exists(self.target_dir):
os.makedirs(self.target_dir)
if source_template is None:
f_private = open(self.private_img, 'a+b')
f_private.truncate(self.private_img_size)
f_private.close()
else:
self.vm.log.info("Copying the template's private image: {}".format(
source_template.storage.private_img))
self._copy_file(source_template.storage.private_img, self.private_img)
def create_on_disk_root_img(self, source_template=None):
if not os.path.exists(self.target_dir):
os.makedirs(self.target_dir)
if source_template is None:
fd = open(self.root_img, 'a+b')
fd.truncate(self.root_img_size)
fd.close()
elif self.vm.updateable:
# if not updateable, just use template's disk
self.vm.log.info(
"--> Copying the template's root image: {}".format(
source_template.storage.root_img))
self._copy_file(source_template.storage.root_img, self.root_img)
def resize_private_img(self, size):
fd = open(self.private_img, 'a+b')
with open(path, 'a+b') as fd:
fd.truncate(size)
fd.close()
# find loop device if any
p = subprocess.Popen(
['sudo', 'losetup', '--associated', self.private_img],
p = subprocess.Popen(['sudo', 'losetup', '--associated', path],
stdout=subprocess.PIPE)
result = p.communicate()
@ -172,106 +105,46 @@ class XenPool(Pool):
subprocess.check_call(['sudo', 'losetup', '--set-capacity',
loop_dev])
def commit_template_changes(self):
assert isinstance(self.vm, qubes.vm.templatevm.TemplateVM)
def commit_template_changes(self, volume):
if volume.volume_type != 'origin':
return volume
# TODO: move rootcow_img to this class; the same for vm.is_outdated()
if os.path.exists(self.vm.rootcow_img):
os.rename(self.vm.rootcow_img, self.vm.rootcow_img + '.old')
if os.path.exists(volume.path_cow):
os.rename(volume.path_cow, volume.path_cow + '.old')
old_umask = os.umask(002)
f_cow = open(self.vm.rootcow_img, 'w')
f_root = open(self.root_img, 'r')
f_root.seek(0, os.SEEK_END)
# make empty sparse file of the same size as root.img
f_cow.truncate(f_root.tell())
f_cow.close()
f_root.close()
with open(volume.path_cow, 'w') as f_cow:
f_cow.truncate(volume.size)
os.umask(old_umask)
return volume
def start(self, volume):
if volume.volume_type == 'volatile':
self._reset_volume(volume)
if volume.volume_type in ['origin', 'snapshot']:
_check_path(volume.path_origin)
_check_path(volume.path_cow)
else:
_check_path(volume.path)
return volume
def stop(self, volume):
pass
def _reset_volume(self, volume):
''' Remove and recreate a volatile volume '''
assert volume.volume_type == 'volatile', "Not a volatile volume"
size = self.vm.volume_config[volume.name]['size']
assert size
assert volume.size
if os.path.exists(volume.path):
os.remove(volume.path)
_remove_if_exists(volume)
with open(volume.path, "w") as f_volatile:
f_volatile.truncate(volume.size)
return volume
def reset_volatile_storage(self):
try:
# no template set, in any way (Standalone VM, Template VM)
if self.vm.template is None:
raise AttributeError
# template-based HVM with only one device-mapper layer -
# volatile.img used as upper layer on root.img, no root-cow.img
# intermediate layer
if self.vm.hvm:
if os.path.exists(self.volatile_img):
if self.vm.debug:
if os.path.getmtime(self.vm.template.storage.root_img) \
> os.path.getmtime(self.volatile_img):
self.vm.log.warning(
'Template have changed, resetting root.img')
else:
self.vm.log.warning(
'Debug mode: not resetting root.img; if you'
' want to force root.img reset, either'
' update template VM, or remove volatile.img'
' file.')
return
os.remove(self.volatile_img)
# FIXME stat on f_root; with open() ...
f_volatile = open(self.volatile_img, "w")
f_root = open(self.vm.template.storage.root_img, "r")
# make empty sparse file of the same size as root.img
f_root.seek(0, os.SEEK_END)
f_volatile.truncate(f_root.tell())
f_volatile.close()
f_root.close()
return # XXX why is that? super() does not run
except AttributeError: # self.vm.template
pass
super(XenPool, self).reset_volatile_storage()
def prepare_for_vm_startup(self):
super(XenPool, self).prepare_for_vm_startup()
if self.drive is not None:
# pylint: disable=unused-variable
(drive_type, drive_domain, drive_path) = self.drive.split(":")
if drive_domain.lower() != "dom0":
# XXX "VM '{}' holding '{}' does not exists".format(
drive_vm = self.vm.app.domains[drive_domain]
if not drive_vm.is_running():
raise qubes.exc.QubesVMNotRunningError(
drive_vm, 'VM {!r} holding {!r} isn\'t running'.format(
drive_domain, drive_path))
if self.rootcow_img and not os.path.exists(self.rootcow_img):
self.commit_template_changes()
# XXX there is also a class attribute on the domain classes which does
# exactly that -- which one should prevail?
@property
def target_dir(self):
def target_dir(self, vm):
""" Returns the path to vmdir depending on the type of the VM.
The default QubesOS file storage saves the vm images in three
@ -288,7 +161,6 @@ class XenPool(Pool):
string (str) absolute path to the directory where the vm files
are stored
"""
vm = self.vm
if vm.is_template():
subdir = 'vm-templates'
elif vm.is_disposablevm():
@ -300,16 +172,10 @@ class XenPool(Pool):
return os.path.join(self.dir_path, subdir, vm.name)
def abspath(self, file_name):
return os.path.join(self.target_dir, file_name)
def init_volume(self, volume_config):
def init_volume(self, vm, volume_config):
assert 'volume_type' in volume_config, "Volume type missing " \
+ str(volume_config)
target_dir = self.target_dir
assert target_dir, "Pool target_dir not set"
volume_type = volume_config['volume_type']
volume_config['target_dir'] = target_dir
known_types = {
'read-write': ReadWriteFile,
'read-only': ReadOnlyFile,
@ -320,125 +186,124 @@ class XenPool(Pool):
if volume_type not in known_types:
raise StoragePoolException("Unknown volume type " + volume_type)
if volume_type == 'snapshot':
path = qubes.storage.get_pool(volume_config['pool'], self.vm.template).target_dir
volume_config['vid'] = os.path.join(path, volume_config['name'] + '.img')
if volume_type in ['snapshot', 'read-only']:
origin_pool = vm.app.get_pool(volume_config['pool'])
assert isinstance(origin_pool,
XenPool), 'Origin volume not a xen volume'
volume_config['target_dir'] = origin_pool.target_dir(vm.template)
name = volume_config['name']
volume_config['size'] = vm.template.volume_config[name]['size']
else:
volume_config['target_dir'] = self.target_dir(vm)
return known_types[volume_type](**volume_config)
class SizeMixIn(Volume):
class XenVolume(Volume):
''' Parent class for the xen volumes implementation '''
def __init__(self, target_dir, **kwargs):
self.target_dir = target_dir
assert self.target_dir, "target_dir not specified"
super(XenVolume, self).__init__(**kwargs)
class SizeMixIn(XenVolume):
''' A mix in which expects a `size` param to be > 0 on initialization and
provides a usage property wrapper.
'''
def __init__(self, name=None, pool=None, vid=None, target_dir=None, size=0,
**kwargs):
assert size > 0, 'Size for volume ' + name + ' is <=0'
super(SizeMixIn, self).__init__(name=name,
pool=pool,
vid=vid,
size=size,
**kwargs)
self._size = size
self.target_dir = target_dir
@property
def size(self):
if self.vid and os.path.exists(self.vid):
return qubes.storage.get_disk_usage(self.vid)
else:
return self._size
def usage(self):
''' Returns the actualy used space '''
return get_disk_usage(self.vid)
class ReadWriteFile(SizeMixIn):
# :pylint: disable=too-few-public-methods
# :pylint: disable=missing-docstring
def __init__(self, **kwargs):
super(ReadWriteFile, self).__init__(**kwargs)
self.path = os.path.join(self.target_dir, self.name + '.img')
self.vid = self.path
@property
def size(self):
if self.vid and os.path.exists(self.vid):
return qubes.storage.get_disk_usage(self.vid)
else:
return self._size
def create(self):
create_file(self.path, self.size)
@property
def created(self):
return os.path.exists(self.path)
class ReadOnlyFile(Volume):
# :pylint: disable=missing-docstring
usage = 0
def __init__(self, name=None, pool=None, vid=None, target_dir=None,
**kwargs):
size=0, **kwargs):
# :pylint: disable=unused-argument
assert os.path.exists(vid), "read-only volume missing vid"
super(ReadOnlyFile, self).__init__(name=name,
pool=pool,
vid=vid,
size=size,
**kwargs)
self.path = self.vid
@property
def size(self):
return qubes.storage.get_disk_usage(self.vid)
class OriginFile(SizeMixIn):
# :pylint: disable=missing-docstring
script = 'block-origin'
def __init__(self, **kwargs):
super(OriginFile, self).__init__(**kwargs)
self.path = os.path.join(self.target_dir, self.name + '.img')
self.path_origin = os.path.join(self.target_dir, self.name + '.img')
self.path_cow = os.path.join(self.target_dir, self.name + '-cow.img')
self.vid = self.path
def create(self):
create_file(self.path, self.size)
create_file(self.path_cow, self.size)
self.path = '%s:%s' % (self.path_origin, self.path_cow)
self.vid = self.path_origin
def commit(self):
raise NotImplementedError
@property
def size(self):
if self.vid and os.path.exists(self.vid):
return qubes.storage.get_disk_usage(self.vid)
else:
return self._size
@property
def created(self):
return os.path.exists(self.path) and os.path.exists(self.path_cow)
def usage(self):
result = 0
if os.path.exists(self.path_origin):
result += get_disk_usage(self.path_origin)
if os.path.exists(self.path_cow):
result += get_disk_usage(self.path_cow)
return result
class SnapshotFile(Volume):
# :pylint: disable=too-few-public-methods
# :pylint: disable=missing-docstring
script = 'block-snapshot'
rw = False
usage = 0
def __init__(self, name=None, pool=None, vid=None, target_dir=None,
**kwargs):
assert vid, "SnapshotVolume missing a vid to OriginVolume"
assert os.path.exists(vid), "OriginVolume does not exist"
size=None, **kwargs):
assert size
super(SnapshotFile, self).__init__(name=name,
pool=pool,
vid=vid,
size=size,
**kwargs)
self.path = os.path.join(target_dir, name + '.img')
self.path_origin = os.path.join(target_dir, name + '.img')
self.path_cow = os.path.join(target_dir, name + '-cow.img')
self.path = '%s:%s' % (self.path_origin, self.path_cow)
self.vid = self.path_origin
@property
def created(self):
return os.path.exists(self.path) and os.path.exists(self.path_cow)
@property
def size(self):
return qubes.storage.get_disk_usage(self.vid)
return os.path.exists(self.path_origin) and os.path.exists(
self.path_cow)
class VolatileFile(SizeMixIn):
# :pylint: disable=missing-docstring
def __init__(self, **kwargs):
super(VolatileFile, self).__init__(**kwargs)
@ -446,8 +311,94 @@ class VolatileFile(SizeMixIn):
self.vid = self.path
def create_file(path, size):
def create_sparse_file(path, size):
''' Create an empty sparse file '''
if os.path.exists(path):
raise IOError("Volume %s already exists", path)
parent_dir = os.path.dirname(path)
if not os.path.exists(parent_dir):
os.makedirs(parent_dir)
with open(path, 'a+b') as fh:
fh.truncate(size)
def get_disk_usage_one(st):
'''Extract disk usage of one inode from its stat_result struct.
If known, get real disk usage, as written to device by filesystem, not
logical file size. Those values may be different for sparse files.
:param os.stat_result st: stat result
:returns: disk usage
'''
try:
return st.st_blocks * BLKSIZE
except AttributeError:
return st.st_size
def get_disk_usage(path):
'''Get real disk usage of given path (file or directory).
When *path* points to directory, then it is evaluated recursively.
This function tries estiate real disk usage. See documentation of
:py:func:`get_disk_usage_one`.
:param str path: path to evaluate
:returns: disk usage
'''
try:
st = os.lstat(path)
except OSError:
return 0
ret = get_disk_usage_one(st)
# if path is not a directory, this is skipped
for dirpath, dirnames, filenames in os.walk(path):
for name in dirnames + filenames:
ret += get_disk_usage_one(os.lstat(os.path.join(dirpath, name)))
return ret
def create_dir_if_not_exists(path):
""" Check if a directory exists in if not create it.
This method does not create any parent directories.
"""
if not os.path.exists(path):
os.mkdir(path)
def copy_file(source, destination):
'''Effective file copy, preserving sparse files etc.
'''
# TODO: Windows support
# We prefer to use Linux's cp, because it nicely handles sparse files
assert os.path.exists(source), \
"Missing the source %s to copy from" % source
assert not os.path.exists(destination), \
"Destination %s already exists" % destination
parent_dir = os.path.dirname(destination)
if not os.path.exists(parent_dir):
os.makedirs(parent_dir)
try:
subprocess.check_call(['cp', '--reflink=auto', source, destination])
except subprocess.CalledProcessError:
raise IOError('Error while copying {!r} to {!r}'.format(source,
destination))
def _remove_if_exists(volume):
if os.path.exists(volume.path):
os.remove(volume.path)
def _check_path(path):
''' Raise an StoragePoolException if ``path`` does not exist'''
if not os.path.exists(path):
raise StoragePoolException('Missing image file: %s' % path)

View File

@ -333,10 +333,10 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
@property
def block_devices(self):
return [self.storage.root_dev_config(),
self.storage.private_dev_config(),
self.storage.volatile_dev_config(),
self.storage.other_dev_config()]
''' Return all :py:class:`qubes.devices.BlockDevice`s for current domain
for serialization in the libvirt XML template as <disk>.
'''
return [v.block_device() for v in self.volumes.values()]
@property
def qdb(self):
@ -353,21 +353,27 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
def private_img(self):
'''Location of private image of the VM (that contains :file:`/rw` \
and :file:`/home`).'''
return self.storage.private_img
warnings.warn("volatile_img is deprecated, use volumes['private'].vid",
DeprecationWarning)
return self.volumes['private'].vid
# XXX this should go to to AppVM? or TemplateVM?
@property
def root_img(self):
'''Location of root image.'''
return self.storage.root_img
warnings.warn("root_img is deprecated, use volumes['root'].vid",
DeprecationWarning)
return self.volumes['root'].vid
# XXX and this should go to exactly where? DispVM has it.
@property
def volatile_img(self):
'''Volatile image that overlays :py:attr:`root_img`.'''
return self.storage.volatile_img
warnings.warn("volatile_img is deprecated, use volumes['volatile'].vid",
DeprecationWarning)
return self.volumes['volatile'].vid
# XXX shouldn't this go elsewhere?
@ -425,8 +431,10 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
# constructor
#
def __init__(self, app, xml, **kwargs):
def __init__(self, app, xml, volume_config={}, **kwargs):
super(QubesVM, self).__init__(app, xml, **kwargs)
if hasattr(self, 'volume_config'):
dict_merge(self.volume_config, volume_config)
import qubes.vm.adminvm # pylint: disable=redefined-outer-name
@ -637,7 +645,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
self.netvm.start(start_guid=start_guid,
notify_function=notify_function)
self.storage.prepare_for_vm_startup()
self.storage.start()
self._update_libvirt_domain()
qmemman_client = self.request_memory(mem_required)
@ -729,6 +737,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
exc_info=1)
self.libvirt_domain.shutdown()
self.storage.stop()
def kill(self):
@ -742,6 +751,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
raise qubes.exc.QubesVMNotStartedError(self)
self.libvirt_domain.destroy()
self.storage.stop()
def force_shutdown(self, *args, **kwargs):
@ -1025,7 +1035,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
p.communicate(input=self.default_user)
# TODO move to storage
# TODO rename to create
def create_on_disk(self, source_template=None):
'''Create files needed for VM.
@ -1115,11 +1125,11 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
while self.is_running(): #1696
time.sleep(1)
def remove_from_disk(self):
'''Remove domain remnants from disk.'''
self.fire_event('domain-remove-from-disk')
self.storage.remove_from_disk()
self.storage.remove()
shutil.rmtree(self.vm.dir_path)
def clone_disk_files(self, src):
@ -1454,68 +1464,69 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
# XXX shouldn't this go only to vms that have root image?
def get_disk_utilization_root_img(self):
'''Get space that is actually ocuppied by :py:attr:`root_img`.
Root image is a sparse file, so it is probably much less than logical
available space.
'''Get space that is actually ocuppied by :py:attr:`volumes['root']`.
This is a temporary wrapper for backwards compatibility. You should
call directly :py:attr:`volumes[name].utilization`
:returns: domain's real disk image size [FIXME unit]
:rtype: FIXME
.. seealso:: :py:meth:`get_root_img_sz`
'''
return qubes.storage.get_disk_usage(self.root_img)
warnings.warn(
"get_disk_utilization_root_img is deprecated, use volumes['root'].utilization",
DeprecationWarning)
return qubes.storage.get_disk_usage(self.volumes['root'].utilization)
# XXX shouldn't this go only to vms that have root image?
def get_root_img_sz(self):
'''Get image size of :py:attr:`root_img`.
Root image is a sparse file, so it is probably much more than ocuppied
physical space.
'''Get the size of the :py:attr:`volumes['root']`.
This is a temporary wrapper for backwards compatibility. You should
call directly :py:attr:`volumes[name].size`
:returns: domain's virtual disk size [FIXME unit]
:rtype: FIXME
.. seealso:: :py:meth:`get_disk_utilization_root_img`
'''
if not os.path.exists(self.root_img):
return 0
return os.path.getsize(self.root_img)
warnings.warn(
"get_disk_root_img_sz is deprecated, use volumes['root'].size",
DeprecationWarning)
return qubes.storage.get_disk_usage(self.volumes['root'].size)
def get_disk_utilization_private_img(self):
'''Get space that is actually ocuppied by :py:attr:`private_img`.
Private image is a sparse file, so it is probably much less than
logical available space.
'''Get space that is actually ocuppied by :py:attr:`volumes['private']`.
This is a temporary wrapper for backwards compatibility. You should
call directly :py:attr:`volumes[name].utilization`
:returns: domain's real disk image size [FIXME unit]
:rtype: FIXME
'''
.. seealso:: :py:meth:`get_private_img_sz`
''' # pylint: disable=invalid-name
return qubes.storage.get_disk_usage(self.private_img)
warnings.warn(
"get_disk_utilization_private_img is deprecated, use volumes['private'].utilization",
DeprecationWarning)
return qubes.storage.get_disk_usage(self.volumes[
'private'].utilization)
def get_private_img_sz(self):
'''Get image size of :py:attr:`private_img`.
Private image is a sparse file, so it is probably much more than
ocuppied physical space.
'''Get the size of the :py:attr:`volumes['private']`.
This is a temporary wrapper for backwards compatibility. You should
call directly :py:attr:`volumes[name].size`
:returns: domain's virtual disk size [FIXME unit]
:rtype: FIXME
.. seealso:: :py:meth:`get_disk_utilization_private_img`
'''
return self.storage.get_private_img_sz()
warnings.warn(
"get_disk_private_img_sz is deprecated, use volumes['private'].size",
DeprecationWarning)
return qubes.storage.get_disk_usage(self.volumes['private'].size)
def get_disk_utilization(self):
'''Return total space actually occuppied by all files belonging to \
@ -1527,7 +1538,6 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
return qubes.storage.get_disk_usage(self.dir_path)
# TODO move to storage
def verify_files(self):
'''Verify that files accessed by this machine are sane.
@ -1751,3 +1761,20 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
# if self.is_qrexec_running():
# #TODO: kill qrexec daemon
# pass
def dict_merge(dct, merge_dct):
""" Recursive dict merge. Inspired by :meth:``dict.update()``, instead of
updating only top-level keys, dict_merge recurses down into dicts nested
to an arbitrary depth, updating keys. The ``merge_dct`` is merged into
``dct``. (Source https://gist.github.com/angstwad/bf22d1822c38a92ec0a9)
:param dct: dict onto which the merge is executed
:param merge_dct: dct merged into dct
:return: None
"""
for k, v in merge_dct.iteritems():
if (k in dct and isinstance(dct[k], dict)
and isinstance(merge_dct[k], dict)):
dict_merge(dct[k], merge_dct[k])
else:
dct[k] = merge_dct[k]

View File

@ -1,13 +1,16 @@
#!/usr/bin/python2 -O
# vim: fileencoding=utf-8
import warnings
import qubes
import qubes.config
import qubes.vm.qubesvm
from qubes.config import defaults
from qubes.vm.qubesvm import QubesVM
class TemplateVM(qubes.vm.qubesvm.QubesVM):
class TemplateVM(QubesVM):
'''Template for AppVM'''
dir_path_prefix = qubes.config.system_path['qubes_templates_dir']
@ -15,7 +18,9 @@ class TemplateVM(qubes.vm.qubesvm.QubesVM):
@property
def rootcow_img(self):
'''COW image'''
return self.storage.rootcow_img
warnings.warn("rootcow_img is deprecated, use "
"volumes['root'].path_origin", DeprecationWarning)
return self.volumes['root'].path_cow
@property
def appvms(self):
@ -67,6 +72,4 @@ class TemplateVM(qubes.vm.qubesvm.QubesVM):
assert not self.is_running(), \
'Attempt to commit changes on running Template VM!'
self.log.info('Commiting template update; COW: {}'.format(
self.rootcow_img))
self.storage.commit_template_changes()