Qubes/core-admin
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge remote-tracking branch 'woju/pull/5/head' into core3-devel

Browse Source 
Makefile                    |   1 -
doc/manpages/qvm-create.rst |   5 +
etc/storage.conf            |  13 -
qubes/__init__.py           |  82 +++++-
qubes/config.py             |  14 +-
qubes/devices.py            |  15 ++
qubes/storage/__init__.py   | 591 +++++++++++++---------------------------
qubes/storage/kernels.py    | 109 ++++++++
qubes/storage/xen.py        | 643 +++++++++++++++++++++++++++++---------------
qubes/tests/__init__.py     |  13 +-
qubes/tests/int/basic.py    |   5 +-
qubes/tests/storage.py      |  70 ++---
qubes/tests/storage_xen.py  | 366 +++++++++++++++----------
qubes/tools/qvm_create.py   |  18 +-
qubes/vm/appvm.py           |  38 ++-
qubes/vm/qubesvm.py         | 192 ++++++-------
qubes/vm/templatevm.py      |  45 +++-
rpm_spec/core-dom0.spec     |   2 +-
setup.py                    |   1 +
templates/libvirt/xen.xml   |  32 ++-
20 files changed, 1313 insertions(+), 942 deletions(-)
This commit is contained in:
Wojtek Porczyk 2016-04-26 11:09:18 +02:00
commit 487411be4c
20 changed files with 1299 additions and 928 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile
View File

@ -60,7 +60,6 @@ endif
# $(MAKE) install -C tests
$(MAKE) install -C relaxng
mkdir -p $(DESTDIR)/etc/qubes
cp etc/storage.conf $(DESTDIR)/etc/qubes/
ifeq ($(BACKEND_VMM),xen)
# Currently supported only on xen
cp etc/qmemman.conf $(DESTDIR)/etc/qubes/

5
doc/manpages/qvm-create.rst
View File

@ -53,6 +53,10 @@ Options
Use provided :file:`root.img` instead of default/empty one (file will be
*moved*). This option is mutually exclusive with :option:`--root-copy-from`.
.. option:: --pool=POOL_NAME:VOLUME_NAME, -P POOL_NAME:VOLUME_NAME
Specify the pool to use for a volume
Options for internal use
------------------------
@ -71,5 +75,6 @@ Authors
| Rafal Wojtczuk <rafal at invisiblethingslab dot com>
| Marek Marczykowski <marmarek at invisiblethingslab dot com>
| Wojtek Porczyk <woju at invisiblethingslab dot com>
| Bahtiar `kalkin-` Gadimov <bahtiar at gadimov dot de>
.. vim: ts=3 sw=3 et tw=80

13
etc/storage.conf
View File

@ -1,13 +0,0 @@
[default] ; poolname
driver=xen ; the default xen storage
; class = qubes.storage.xen.XenStorage ; class always overwrites the driver
;
; To use our own pool driver it needs to provide `qubes.storage` entry_point
; see also: https://pythonhosted.org/setuptools/setuptools.html#dynamic-discovery-of-services-and-plugins
; class name
; [pool-b]
; driver = foo
;
; [test-dummy]
; driver=dummy

82
qubes/__init__.py
View File

@ -36,8 +36,6 @@ __author__ = 'Invisible Things Lab'
__license__ = 'GPLv2 or later'
__version__ = 'R3'
import ast
import atexit
import collections
import errno
import grp
@ -47,12 +45,9 @@ import os.path
import sys
import tempfile
import time
import warnings
import __builtin__
import docutils.core
import docutils.io
import jinja2
import lxml.etree
import pkg_resources
@ -1181,7 +1176,6 @@ class Qubes(PropertyHolder):
default=True,
doc='check for updates inside qubes')
def __init__(self, store=None, load=True, **kwargs):
#: logger instance for logging global messages
self.log = logging.getLogger('app')
@ -1194,6 +1188,9 @@ class Qubes(PropertyHolder):
#: collection of all available labels for VMs
self.labels = {}
#: collection of all pools
self.pools = {}
#: Connection to VMM
self.vmm = VMMConnection()
@ -1235,7 +1232,7 @@ class Qubes(PropertyHolder):
'''
try:
fd = os.open(self._store, os.O_RDWR) # no O_CREAT
fd = os.open(self._store, os.O_RDWR) # no O_CREAT
except OSError as e:
if e.errno != errno.ENOENT:
raise
@ -1256,11 +1253,19 @@ class Qubes(PropertyHolder):
self.xml = lxml.etree.parse(fh)
# stage 1: load labels
# stage 1: load labels and pools
for node in self.xml.xpath('./labels/label'):
label = Label.fromxml(node)
self.labels[label.index] = label
for node in self.xml.xpath('./pools/pool'):
name = node.get('name')
assert name, "Pool name '%s' is invalid " % name
try:
self.pools[name] = self._get_pool(**node.attrib)
except qubes.exc.QubesException as e:
self.log.error(e.message)
# stage 2: load VMs
for node in self.xml.xpath('./domains/domain'):
# pylint: disable=no-member
@ -1270,7 +1275,7 @@ class Qubes(PropertyHolder):
vm.init_log()
self.domains.add(vm)
if not 0 in self.domains:
if 0 not in self.domains:
self.domains.add(qubes.vm.adminvm.AdminVM(
self, None, qid=0, name='dom0'))
@ -1310,11 +1315,17 @@ class Qubes(PropertyHolder):
fh.close()
del fh
def __xml__(self):
element = lxml.etree.Element('qubes')
element.append(self.xml_labels())
pools_xml = lxml.etree.Element('pools')
for pool in self.pools.values():
pools_xml.append(pool.__xml__())
element.append(pools_xml)
element.append(self.xml_properties())
domains = lxml.etree.Element('domains')
@ -1395,6 +1406,10 @@ class Qubes(PropertyHolder):
7: Label(7, '0x75507b', 'purple'),
8: Label(8, '0x000000', 'black'),
}
for name, config in qubes.config.defaults['pool_configs'].items():
self.pools[name] = self._get_pool(**config)
self.domains.add(
qubes.vm.adminvm.AdminVM(self, None, qid=0, name='dom0'))
self.save()
@ -1413,7 +1428,6 @@ class Qubes(PropertyHolder):
labels.append(label.__xml__())
return labels
def get_vm_class(self, clsname):
'''Find the class for a domain.
@ -1473,6 +1487,52 @@ class Qubes(PropertyHolder):
raise KeyError(label)
def add_pool(self, **kwargs):
""" Add a storage pool to config."""
name = kwargs['name']
assert name not in self.pools.keys(), \
"Pool named %s already exists" % name
pool = self._get_pool(**kwargs)
pool.setup()
self.pools[name] = pool
def remove_pool(self, name):
""" Remove a storage pool from config file. """
try:
pool = self.pools[name]
del self.pools[name]
pool.destroy()
except KeyError:
return
def get_pool(self, name):
''' Returns a :py:class:`qubes.storage.Pool` instance '''
try:
return self.pools[name]
except KeyError:
raise qubes.exc.QubesException('Unknown storage pool ' + name)
def _get_pool(self, **kwargs):
try:
name = kwargs['name']
assert name, 'Name needs to be an non empty string'
except KeyError:
raise qubes.exc.QubesException('No pool name for pool')
try:
driver = kwargs['driver']
except KeyError:
raise qubes.exc.QubesException('No driver specified for pool ' +
name)
try:
klass = qubes.get_entry_point_one(
qubes.storage.STORAGE_ENTRY_POINT, driver)
del kwargs['driver']
return klass(**kwargs)
except KeyError:
raise qubes.exc.QubesException('Driver %s for pool %s' %
(driver, name))
@qubes.events.handler('domain-pre-delete')
def on_domain_pre_deleted(self, event, vm):

14
qubes/config.py
View File

@ -28,6 +28,8 @@
# make a real /etc/qubes/master.conf or whatever
#
import os.path
'''Constants which can be configured in one place'''
qubes_base_dir = "/var/lib/qubes"
@ -83,7 +85,17 @@ defaults = {
'private_img_size': 2*1024*1024*1024,
'root_img_size': 10*1024*1024*1024,
'pool_config': {'dir_path': '/var/lib/qubes'},
'pool_configs': {
'default': {'dir_path': qubes_base_dir,
'driver': 'xen',
'name': 'default'},
'linux-kernel': {
'dir_path': os.path.join(qubes_base_dir,
system_path['qubes_kernels_base_dir']),
'driver': 'linux-kernel',
'name': 'linux-kernel'
}
},
# how long (in sec) to wait for VMs to shutdown,
# before killing them (when used qvm-run with --wait option),

15
qubes/devices.py
View File

@ -6,6 +6,7 @@
#
# Copyright (C) 2010-2016 Joanna Rutkowska <joanna@invisiblethingslab.com>
# Copyright (C) 2015-2016 Wojtek Porczyk <woju@invisiblethingslab.com>
# Copyright (C) 2016 Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -26,6 +27,7 @@ import re
import qubes
class DeviceCollection(object):
'''Bag for devices.
@ -121,3 +123,16 @@ class RegexDevice(str):
class PCIDevice(RegexDevice):
regex = re.compile(
r'^(?P<bus>[0-9a-f]+):(?P<device>[0-9a-f]+)\.(?P<function>[0-9a-f]+)$')
class BlockDevice(object):
def __init__(self, path, name, script=None, rw=True, domain=None,
devtype='disk'):
assert name, 'Missing device name'
assert path, 'Missing device path'
self.path = path
self.name = name
self.rw = rw
self.script = script
self.domain = domain
self.devtype = devtype

575
qubes/storage/__init__.py
View File

@ -23,24 +23,22 @@
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
""" Qubes storage system"""
from __future__ import absolute_import
import ConfigParser
import os
import os.path
import shutil
import subprocess
import pkg_resources
import qubes
import qubes.exc
import qubes.utils
from qubes.devices import BlockDevice
import lxml.etree
BLKSIZE = 512
CONFIG_FILE = '/etc/qubes/storage.conf'
STORAGE_ENTRY_POINT = 'qubes.storage'
@ -48,84 +46,70 @@ class StoragePoolException(qubes.exc.QubesException):
pass
class Volume(object):
''' Encapsulates all data about a volume for serialization to qubes.xml and
libvirt config.
'''
devtype = 'disk'
domain = None
path = None
rw = True
script = None
usage = 0
def __init__(self, name, pool, volume_type, vid=None, size=0, **kwargs):
super(Volume, self).__init__(**kwargs)
self.name = str(name)
self.pool = str(pool)
self.vid = vid
self.size = size
self.volume_type = volume_type
def __xml__(self):
return lxml.etree.Element('volume', **self.config)
@property
def config(self):
''' return config data for serialization to qubes.xml '''
return {'name': self.name,
'pool': self.pool,
'volume_type': self.volume_type}
def __repr__(self):
return '{}(name={!s}, pool={!r}, vid={!r}, volume_type={!r})'.format(
self.__class__.__name__, self.name, self.pool, self.vid,
self.volume_type)
def block_device(self):
''' Return :py:class:`qubes.devices.BlockDevice` for serialization in
the libvirt XML template as <disk>.
'''
return BlockDevice(self.path, self.name, self.script, self.rw,
self.domain, self.devtype)
class Storage(object):
'''Class for handling VM virtual disks.
''' Class for handling VM virtual disks.
This is base class for all other implementations, mostly with Xen on Linux
in mind.
'''
root_img = None
private_img = None
volatile_img = None
modules_dev = None
def __init__(self, vm, private_img_size=None, root_img_size=None):
def __init__(self, vm):
#: Domain for which we manage storage
self.vm = vm
#: Size of the private image
self.private_img_size = private_img_size \
if private_img_size is not None \
else qubes.config.defaults['private_img_size']
#: Size of the root image
self.root_img_size = root_img_size \
if root_img_size is not None \
else qubes.config.defaults['root_img_size']
self.log = self.vm.log
#: Additional drive (currently used only by HVM)
self.drive = None
def get_config_params(self):
args = {}
args['rootdev'] = self.root_dev_config()
args['privatedev'] = self.private_dev_config()
args['volatiledev'] = self.volatile_dev_config()
args['otherdevs'] = self.other_dev_config()
args['kerneldir'] = self.kernels_dir
return args
def root_dev_config(self):
raise NotImplementedError()
def private_dev_config(self):
raise NotImplementedError()
def volatile_dev_config(self):
raise NotImplementedError()
def other_dev_config(self):
if self.modules_img is not None:
return self.format_disk_dev(self.modules_img, self.modules_dev,
rw=self.modules_img_rw)
elif self.drive is not None:
(drive_type, drive_domain, drive_path) = self.drive.split(":")
if drive_type == 'hd':
drive_type = 'disk'
rw = (drive_type == 'disk')
if drive_domain.lower() == "dom0":
drive_domain = None
return self.format_disk_dev(drive_path,
self.modules_dev,
rw=rw,
devtype=drive_type,
domain=drive_domain)
else:
return ''
def format_disk_dev(self, path, vdev, script=None, rw=True, devtype='disk',
domain=None):
raise NotImplementedError()
self.pools = {}
if hasattr(vm, 'volume_config'):
for name, conf in self.vm.volume_config.items():
assert 'pool' in conf, "Pool missing in volume_config" % str(
conf)
pool = self.vm.app.get_pool(conf['pool'])
self.vm.volumes[name] = pool.init_volume(self.vm, conf)
self.pools[name] = pool
@property
def kernels_dir(self):
@ -134,370 +118,183 @@ class Storage(object):
If :py:attr:`self.vm.kernel` is :py:obj:`None`, the this points inside
:py:attr:`self.vm.dir_path`
'''
return os.path.join(qubes.config.system_path['qubes_base_dir'],
qubes.config.system_path['qubes_kernels_base_dir'], self.vm.kernel)\
if self.vm.kernel is not None \
else os.path.join(self.vm.dir_path,
qubes.config.vm_files['kernels_subdir'])
@property
def modules_img(self):
'''Path to image with modules.
Depending on domain, this may be global or inside domain's dir.
'''
modules_path = os.path.join(self.kernels_dir, 'modules.img')
if os.path.exists(modules_path):
return modules_path
else:
return None
@property
def modules_img_rw(self):
''':py:obj:`True` if module image should be mounted RW, :py:obj:`False`
otherwise.'''
return self.vm.kernel is None
def abspath(self, path, rel=None):
'''Make absolute path.
If given path is relative, it is interpreted as relative to
:py:attr:`self.vm.dir_path` or given *rel*.
'''
return path if os.path.isabs(path) \
else os.path.join(rel or self.vm.dir_path, path)
@staticmethod
def _copy_file(source, destination):
'''Effective file copy, preserving sparse files etc.
'''
# TODO: Windows support
# We prefer to use Linux's cp, because it nicely handles sparse files
try:
subprocess.check_call(['cp', '--reflink=auto', source, destination])
except subprocess.CalledProcessError:
raise IOError('Error while copying {!r} to {!r}'.format(
source, destination))
assert 'kernel' in self.vm.volumes, "VM has no kernel pool"
return self.vm.volumes['kernel'].kernels_dir
def get_disk_utilization(self):
return get_disk_usage(self.vm.dir_path)
''' Returns summed up disk utilization for all domain volumes '''
result = 0
for volume in self.vm.volumes.values():
result += volume.usage
return result
# TODO Remove this wrapper
def get_disk_utilization_private_img(self):
# pylint: disable=invalid-name
return get_disk_usage(self.private_img)
# pylint: disable=invalid-name,missing-docstring
return self.vm.volume['private'].usage
# TODO Remove this wrapper
def get_private_img_sz(self):
if not os.path.exists(self.private_img):
return 0
# :pylint: disable=missing-docstring
return self.vm.volume['private'].size
return os.path.getsize(self.private_img)
def resize_private_img(self, size):
raise NotImplementedError()
def create_on_disk_private_img(self, source_template=None):
raise NotImplementedError()
def create_on_disk_root_img(self, source_template=None):
raise NotImplementedError()
def resize(self, volume, size):
''' Resize volume '''
self.get_pool(volume).resize(volume, size)
# TODO rename it to create()
def create_on_disk(self, source_template=None):
# :pylint: disable=missing-docstring
if source_template is None and hasattr(self.vm, 'template'):
source_template = self.vm.template
old_umask = os.umask(002)
self.vm.log.info('Creating directory: {0}'.format(self.vm.dir_path))
os.mkdir(self.vm.dir_path)
self.create_on_disk_private_img(source_template)
self.create_on_disk_root_img(source_template)
self.reset_volatile_storage()
self.log.info('Creating directory: {0}'.format(self.vm.dir_path))
os.makedirs(self.vm.dir_path)
for name, volume in self.vm.volumes.items():
source_volume = None
if source_template and hasattr(source_template, 'volumes'):
source_volume = source_template.volumes[name]
self.get_pool(volume).create(volume, source_volume=source_volume)
os.umask(old_umask)
def clone_disk_files(self, src_vm):
def clone(self, src_vm):
self.vm.log.info('Creating directory: {0}'.format(self.vm.dir_path))
os.mkdir(self.vm.dir_path)
if hasattr(src_vm, 'private_img'):
self.vm.log.info('Copying the private image: {} -> {}'.format(
src_vm.private_img, self.vm.private_img))
self._copy_file(src_vm.private_img, self.vm.private_img)
if src_vm.updateable and hasattr(src_vm, 'root_img'):
self.vm.log.info('Copying the root image: {} -> {}'.format(
src_vm.root_img, self.root_img))
self._copy_file(src_vm.root_img, self.root_img)
# TODO: modules?
# XXX which modules? -woju
@staticmethod
def rename(newpath, oldpath):
'''Move storage directory, most likely during domain's rename.
.. note::
The arguments are in different order than in :program:`cp` utility.
.. versionchange:: 4.0
This is now dummy method that just passes everything to
:py:func:`os.rename`.
:param str newpath: New path
:param str oldpath: Old path
'''
os.rename(oldpath, newpath)
if not os.path.exists(self.vm.dir_path):
self.log.info('Creating directory: {0}'.format(self.vm.dir_path))
os.makedirs(self.vm.dir_path)
for name, target in self.vm.volumes.items():
pool = self.get_pool(target)
source = src_vm.volumes[name]
volume = pool.clone(source, target)
assert volume, "%s.clone() returned '%s'" % (pool.__class__,
volume)
self.vm.volumes[name] = volume
def rename(self, old_name, new_name):
''' Notify the pools that the domain was renamed '''
volumes = self.vm.volumes
for name, volume in volumes.items():
pool = self.get_pool(volume)
volumes[name] = pool.rename(volume, old_name, new_name)
def verify_files(self):
'''Verify that the storage is sane.
On success, returns normally. On failure, raises exception.
'''
if not os.path.exists(self.vm.dir_path):
raise qubes.exc.QubesVMError(self.vm,
raise qubes.exc.QubesVMError(
self.vm,
'VM directory does not exist: {}'.format(self.vm.dir_path))
if hasattr(self.vm, 'root_img') and not os.path.exists(self.root_img):
raise qubes.exc.QubesVMError(self.vm,
'VM root image file does not exist: {}'.format(self.root_img))
if hasattr(self.vm, 'private_img') \
and not os.path.exists(self.private_img):
raise qubes.exc.QubesVMError(self.vm,
'VM private image file does not exist: {}'.format(
self.private_img))
if self.modules_img is not None \
and not os.path.exists(self.modules_img):
raise qubes.exc.QubesVMError(self.vm,
'VM kernel modules image does not exists: {}'.format(
self.modules_img))
def remove_from_disk(self):
def remove(self):
for name, volume in self.vm.volumes.items():
self.log.info('Removing volume %s: %s' % (name, volume.vid))
self.get_pool(volume).remove(volume)
shutil.rmtree(self.vm.dir_path)
def start(self):
''' Execute the start method on each pool '''
for volume in self.vm.volumes.values():
self.get_pool(volume).start(volume)
def reset_volatile_storage(self):
# Re-create only for template based VMs
try:
if self.vm.template is not None and self.volatile_img:
if os.path.exists(self.volatile_img):
os.remove(self.volatile_img)
except AttributeError: # self.vm.template
pass
def stop(self):
''' Execute the start method on each pool '''
for volume in self.vm.volumes.values():
self.get_pool(volume).stop(volume)
# For StandaloneVM create it only if not already exists
# (eg after backup-restore)
if hasattr(self, 'volatile_img') \
and not os.path.exists(self.volatile_img):
self.vm.log.info(
'Creating volatile image: {0}'.format(self.volatile_img))
subprocess.check_call(
[qubes.config.system_path["prepare_volatile_img_cmd"],
self.volatile_img,
str(self.root_img_size / 1024 / 1024)])
def get_pool(self, volume):
''' Helper function '''
assert isinstance(volume, Volume), "You need to pass a Volume"
return self.pools[volume.name]
def commit_template_changes(self):
for volume in self.vm.volumes.values():
if volume.volume_type == 'origin':
self.get_pool(volume).commit_template_changes(volume)
def prepare_for_vm_startup(self):
self.reset_volatile_storage()
if hasattr(self.vm, 'private_img') \
and not os.path.exists(self.private_img):
self.vm.log.info('Creating empty VM private image file: {0}'.format(
self.private_img))
self.create_on_disk_private_img()
def get_disk_usage_one(st):
'''Extract disk usage of one inode from its stat_result struct.
If known, get real disk usage, as written to device by filesystem, not
logical file size. Those values may be different for sparse files.
:param os.stat_result st: stat result
:returns: disk usage
'''
try:
return st.st_blocks * BLKSIZE
except AttributeError:
return st.st_size
def get_disk_usage(path):
'''Get real disk usage of given path (file or directory).
When *path* points to directory, then it is evaluated recursively.
This function tries estiate real disk usage. See documentation of
:py:func:`get_disk_usage_one`.
:param str path: path to evaluate
:returns: disk usage
'''
try:
st = os.lstat(path)
except OSError:
return 0
ret = get_disk_usage_one(st)
# if path is not a directory, this is skipped
for dirpath, dirnames, filenames in os.walk(path):
for name in dirnames + filenames:
ret += get_disk_usage_one(os.lstat(os.path.join(dirpath, name)))
return ret
def get_pool(name, vm):
""" Instantiates the storage for the specified vm """
config = _get_storage_config_parser()
klass = _get_pool_klass(name, config)
keys = [k for k in config.options(name) if k != 'driver' and k != 'class']
values = [config.get(name, o) for o in keys]
config_kwargs = dict(zip(keys, values))
if name == 'default':
kwargs = qubes.config.defaults['pool_config'].copy()
kwargs.update(keys)
else:
kwargs = config_kwargs
return klass(vm, **kwargs)
def pool_exists(name):
""" Check if the specified pool exists """
try:
_get_pool_klass(name)
return True
except StoragePoolException:
return False
def add_pool(name, **kwargs):
""" Add a storage pool to config."""
config = _get_storage_config_parser()
config.add_section(name)
for key, value in kwargs.iteritems():
config.set(name, key, value)
_write_config(config)
def remove_pool(name):
""" Remove a storage pool from config file. """
config = _get_storage_config_parser()
config.remove_section(name)
_write_config(config)
def _write_config(config):
with open(CONFIG_FILE, 'w') as configfile:
config.write(configfile)
def _get_storage_config_parser():
""" Instantiates a `ConfigParaser` for specified storage config file.
Returns:
RawConfigParser
"""
config = ConfigParser.RawConfigParser()
config.read(CONFIG_FILE)
return config
def _get_pool_klass(name, config=None):
""" Returns the storage klass for the specified pool.
Args:
name: The pool name.
config: If ``config`` is not specified
`_get_storage_config_parser()` is called.
Returns:
type: A class inheriting from `QubesVmStorage`
"""
if config is None:
config = _get_storage_config_parser()
if not config.has_section(name):
raise StoragePoolException('Uknown storage pool ' + name)
elif not config.has_option(name, 'driver'):
raise StoragePoolException('No driver specified for pool ' + name)
driver = config.get(name, 'driver')
try:
return qubes.get_entry_point_one(STORAGE_ENTRY_POINT, driver)
except KeyError:
raise StoragePoolException('Driver %s for pool %s' % (driver, name))
class Pool(object):
def __init__(self, vm, dir_path):
assert vm is not None
assert dir_path is not None
''' A Pool is used to manage different kind of volumes (File
based/LVM/Btrfs/...).
self.vm = vm
self.dir_path = dir_path
3rd Parties providing own storage implementations will need to extend
this class.
'''
private_img_size = qubes.config.defaults['private_img_size']
root_img_size = qubes.config.defaults['root_img_size']
self.create_dir_if_not_exists(self.dir_path)
def __init__(self, name, **kwargs):
super(Pool, self).__init__(**kwargs)
self.name = name
kwargs['name'] = self.name
self.vmdir = self.vmdir_path(vm, self.dir_path)
def __xml__(self):
return lxml.etree.Element('pool', **self.config)
appvms_path = os.path.join(self.dir_path, 'appvms')
self.create_dir_if_not_exists(appvms_path)
def create(self, volume, source_volume):
''' Create the given volume on disk or copy from provided
`source_volume`.
'''
raise NotImplementedError("Pool %s has create() not implemented" %
self.name)
servicevms_path = os.path.join(self.dir_path, 'servicevms')
self.create_dir_if_not_exists(servicevms_path)
def commit_template_changes(self, volume):
''' Update origin device '''
raise NotImplementedError(
"Pool %s has commit_template_changes() not implemented" %
self.name)
vm_templates_path = os.path.join(self.dir_path, 'vm-templates')
self.create_dir_if_not_exists(vm_templates_path)
@property
def config(self):
''' Returns the pool config to be written to qubes.xml '''
raise NotImplementedError("Pool %s has config() not implemented" %
self.name)
# XXX there is also a class attribute on the domain classes which does
# exactly that -- which one should prevail?
def vmdir_path(self, vm, pool_dir):
""" Returns the path to vmdir depending on the type of the VM.
def clone(self, source, target):
''' Clone volume '''
raise NotImplementedError("Pool %s has clone() not implemented" %
self.name)
The default QubesOS file storage saves the vm images in three
different directories depending on the ``QubesVM`` type:
def destroy(self):
raise NotImplementedError("Pool %s has destroy() not implemented" %
self.name)
* ``appvms`` for ``QubesAppVm`` or ``QubesHvm``
* ``vm-templates`` for ``QubesTemplateVm`` or ``QubesTemplateHvm``
def remove(self, volume):
''' Remove volume'''
raise NotImplementedError("Pool %s has remove() not implemented" %
self.name)
Args:
vm: a QubesVM
pool_dir: the root directory of the pool
def rename(self, volume, old_name, new_name):
''' Called when the domain changes its name '''
raise NotImplementedError("Pool %s has rename() not implemented" %
self.name)
Returns:
string (str) absolute path to the directory where the vm files
are stored
"""
if vm.is_template():
subdir = 'vm-templates'
elif vm.is_disposablevm():
subdir = 'appvms'
return os.path.join(pool_dir, subdir, vm.template.name + '-dvm')
else:
subdir = 'appvms'
def start(self, volume):
''' Do what ever is needed on start '''
raise NotImplementedError("Pool %s has start() not implemented" %
self.name)
return os.path.join(pool_dir, subdir, vm.name)
def setup(self):
raise NotImplementedError("Pool %s has setup() not implemented" %
self.name)
def create_dir_if_not_exists(self, path):
""" Check if a directory exists in if not create it.
def stop(self, volume):
''' Do what ever is needed on stop'''
raise NotImplementedError("Pool %s has stop() not implemented" %
self.name)
This method does not create any parent directories.
"""
if not os.path.exists(path):
os.mkdir(path)
def init_volume(self, volume_config):
''' Initialize a :py:class:`qubes.storage.Volume` from `volume_config`.
'''
raise NotImplementedError("Pool %s has init_volume() not implemented" %
self.name)
def pool_drivers():
""" Return a list of EntryPoints names """
return [ep.name
for ep in pkg_resources.iter_entry_points(STORAGE_ENTRY_POINT)]
for ep in pkg_resources.iter_entry_points(STORAGE_ENTRY_POINT)]

109
qubes/storage/kernels.py Normal file
View File

@ -0,0 +1,109 @@
#!/usr/bin/python2
# -*- encoding: utf8 -*-
#
# The Qubes OS Project, http://www.qubes-os.org
#
# Copyright (C) 2010-2015 Joanna Rutkowska <joanna@invisiblethingslab.com>
# Copyright (C) 2015 Wojtek Porczyk <woju@invisiblethingslab.com>
# Copyright (C) 2016 Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
import os
from qubes.storage import Pool, StoragePoolException, Volume
class LinuxModules(Volume):
rw = False
def __init__(self, target_dir, kernel_version, **kwargs):
super(LinuxModules, self).__init__(**kwargs)
self.kernels_dir = os.path.join(target_dir, kernel_version)
self.path = os.path.join(self.kernels_dir, 'modules.img')
self.vid = self.path
self.vmlinuz = os.path.join(self.kernels_dir, 'vmlinuz')
self.initramfs = os.path.join(self.kernels_dir, 'initramfs')
class LinuxKernel(Pool):
driver = 'linux-kernel'
def __init__(self, name=None, dir_path=None):
assert dir_path, 'Missing dir_path'
super(LinuxKernel, self).__init__(name=name)
self.dir_path = dir_path
def init_volume(self, vm, volume_config):
assert 'volume_type' in volume_config, "Volume type missing " \
+ str(volume_config)
volume_type = volume_config['volume_type']
if volume_type != 'read-only':
raise StoragePoolException("Unknown volume type " + volume_type)
volume = LinuxModules(self.dir_path, vm.kernel, **volume_config)
_check_path(volume.path)
_check_path(volume.vmlinuz)
_check_path(volume.initramfs)
return volume
def clone(self, source, target):
return target
def create(self, volume, source_volume):
return volume
def commit_template_changes(self, volume):
return volume
@property
def config(self):
return {
'name': self.name,
'dir_path': self.dir_path,
'driver': LinuxKernel.driver,
}
def destroy(self):
pass
def remove(self, volume):
pass
def rename(self, volume, old_name, new_name):
return volume
def setup(self):
pass
def start(self, volume):
path = volume.path
if not os.path.exists(path):
raise StoragePoolException('Missing kernel modules: %s' % path)
return volume
def stop(self, volume):
pass
def _check_path(path):
''' Raise an :py:class:`qubes.storage.StoragePoolException` if ``path`` does
not exist.
'''
if not os.path.exists(path):
raise StoragePoolException('Missing file: %s' % path)

635
qubes/storage/xen.py
View File

@ -31,173 +31,110 @@ import os.path
import re
import subprocess
import lxml.etree
from qubes.storage import Pool, StoragePoolException, Volume
import qubes
import qubes.config
import qubes.storage
import qubes.vm.templatevm
BLKSIZE = 512
class XenStorage(qubes.storage.Storage):
'''Class for VM storage of Xen VMs.
'''
class XenPool(Pool):
''' File based 'original' disk implementation '''
driver = 'xen'
root_dev = 'xvda'
private_dev = 'xvdb'
volatile_dev = 'xvdc'
modules_dev = 'xvdd'
def __init__(self, name=None, dir_path=None):
super(XenPool, self).__init__(name=name)
assert dir_path, "No pool dir_path specified"
self.dir_path = os.path.normpath(dir_path)
def __init__(self, vm, vmdir, **kwargs):
""" Instantiate the storage.
def clone(self, source, target):
''' Clones the volume if the `source.pool` if the source is a
:py:class:`XenVolume`.
'''
if issubclass(XenVolume, source.__class__):
raise StoragePoolException('Volumes %s and %s use different pools'
% (source.__class__, target.__class__))
Args:
vm: a QubesVM
vmdir: the root directory of the pool
"""
assert vm is not None
assert vmdir is not None
if source.volume_type not in ['origin', 'read-write']:
return target
super(XenStorage, self).__init__(vm, **kwargs)
copy_file(source.vid, target.vid)
return target
self.vmdir = vmdir
def create(self, volume, source_volume=None):
_type = volume.volume_type
size = volume.size
if _type == 'origin':
create_sparse_file(volume.path_origin, size)
create_sparse_file(volume.path_cow, size)
elif _type in ['read-write'] and source_volume:
copy_file(source_volume.path, volume.path)
elif _type in ['read-write', 'volatile']:
create_sparse_file(volume.path, size)
return volume
@property
def private_img(self):
'''Path to the private image'''
return self.abspath(qubes.config.vm_files['private_img'])
def config(self):
return {
'name': self.name,
'dir_path': self.dir_path,
'driver': XenPool.driver,
}
def resize(self, volume, size):
''' Expands volume, throws
:py:class:`qubst.storage.StoragePoolException` if given size is
less than current_size
'''
_type = volume.volume_type
if _type not in ['origin', 'read-write', 'volatile']:
raise StoragePoolException('Can not resize a %s volume %s' %
(_type, volume.vid))
@property
def root_img(self):
'''Path to the root image'''
return self.vm.template.storage.root_img \
if hasattr(self.vm, 'template') and self.vm.template \
else self.abspath(qubes.config.vm_files['root_img'])
if size <= volume.size:
raise StoragePoolException(
'For your own safety, shrinking of %s is'
' disabled. If you really know what you'
' are doing, use `truncate` on %s manually.' %
(volume.name, volume.vid))
if _type == 'origin':
path = volume.path_origin
elif _type in ['read-write', 'volatile']:
path = volume.path
@property
def rootcow_img(self):
'''Path to the root COW image'''
with open(path, 'a+b') as fd:
fd.truncate(size)
if isinstance(self.vm, qubes.vm.templatevm.TemplateVM):
return self.abspath(qubes.config.vm_files['rootcow_img'])
self._resize_loop_device(path)
return None
def remove(self, volume):
if volume.volume_type in ['read-write', 'volatile']:
_remove_if_exists(volume.vid)
elif volume.volume_type == 'origin':
_remove_if_exists(volume.vid)
_remove_if_exists(volume.path_cow)
def rename(self, volume, old_name, new_name):
assert issubclass(volume.__class__, XenVolume)
old_dir = os.path.dirname(volume.path)
new_dir = os.path.join(os.path.dirname(old_dir), new_name)
@property
def volatile_img(self):
'''Path to the volatile image'''
return self.abspath(qubes.config.vm_files['volatile_img'])
if not os.path.exists(new_dir):
os.makedirs(new_dir)
if volume.volume_type == 'read-write':
volume.rename_target_dir(new_name, new_dir)
elif volume.volume_type == 'read-only':
volume.rename_target_dir(old_name, new_dir)
elif volume.volume_type in ['origin', 'volatile']:
volume.rename_target_dir(new_dir)
def format_disk_dev(self, path, vdev, script=None, rw=True, devtype='disk',
domain=None):
if path is None:
return ''
element = lxml.etree.Element('disk')
element.set('type', 'block')
element.set('device', devtype)
element.append(lxml.etree.Element('driver', name='phy'))
element.append(lxml.etree.Element('source', dev=path))
element.append(lxml.etree.Element('target', dev=vdev))
if not rw:
element.append(lxml.etree.Element('readonly'))
if domain is not None:
# XXX vm.name?
element.append(lxml.etree.Element('domain', name=domain))
if script:
element.append(lxml.etree.Element('script', path=script))
# TODO return element
return lxml.etree.tostring(element)
def root_dev_config(self):
if isinstance(self.vm, qubes.vm.templatevm.TemplateVM):
return self.format_disk_dev(
'{root}:{rootcow}'.format(
root=self.root_img,
rootcow=self.rootcow_img),
self.root_dev,
script='block-origin')
elif self.vm.hvm and hasattr(self.vm, 'template'):
# HVM template-based VM - only one device-mapper layer, in dom0
# (root+volatile)
# HVM detection based on 'kernel' property is massive hack,
# but taken from assumption that VM needs Qubes-specific kernel
# (actually initramfs) to assemble the second layer of device-mapper
return self.format_disk_dev(
'{root}:{volatile}'.format(
root=self.vm.template.storage.root_img,
volatile=self.volatile_img),
self.root_dev,
script='block-snapshot')
elif hasattr(self.vm, 'template'):
# any other template-based VM - two device-mapper layers: one
# in dom0 (here) from root+root-cow, and another one from
# this+volatile.img
return self.format_disk_dev(
'{root}:{template_rootcow}'.format(
root=self.root_img,
template_rootcow=self.vm.template.storage.rootcow_img),
self.root_dev,
script='block-snapshot',
rw=False)
else:
# standalone qube
return self.format_disk_dev(self.root_img, self.root_dev)
def private_dev_config(self):
return self.format_disk_dev(self.private_img, self.private_dev)
def volatile_dev_config(self):
return self.format_disk_dev(self.volatile_img, self.volatile_dev)
def create_on_disk_private_img(self, source_template=None):
if source_template is None:
f_private = open(self.private_img, 'a+b')
f_private.truncate(self.private_img_size)
f_private.close()
else:
self.vm.log.info("Copying the template's private image: {}".format(
source_template.storage.private_img))
self._copy_file(source_template.storage.private_img,
self.private_img)
def create_on_disk_root_img(self, source_template=None):
if source_template is None:
fd = open(self.root_img, 'a+b')
fd.truncate(self.root_img_size)
fd.close()
elif self.vm.updateable:
# if not updateable, just use template's disk
self.vm.log.info("--> Copying the template's root image: {}".format(
source_template.storage.root_img))
self._copy_file(source_template.storage.root_img, self.root_img)
def resize_private_img(self, size):
fd = open(self.private_img, 'a+b')
fd.truncate(size)
fd.close()
return volume
def _resize_loop_device(self, path):
# find loop device if any
p = subprocess.Popen(
['sudo', 'losetup', '--associated', self.private_img],
['sudo', 'losetup', '--associated', path],
stdout=subprocess.PIPE)
result = p.communicate()
@ -206,89 +143,359 @@ class XenStorage(qubes.storage.Storage):
loop_dev = m.group(1)
# resize loop device
subprocess.check_call(
['sudo', 'losetup', '--set-capacity', loop_dev])
subprocess.check_call(['sudo', 'losetup', '--set-capacity',
loop_dev])
def commit_template_changes(self, volume):
if volume.volume_type != 'origin':
return volume
def commit_template_changes(self):
assert isinstance(self.vm, qubes.vm.templatevm.TemplateVM)
# TODO: move rootcow_img to this class; the same for vm.is_outdated()
if os.path.exists(self.vm.rootcow_img):
os.rename(self.vm.rootcow_img, self.vm.rootcow_img + '.old')
if os.path.exists(volume.path_cow):
os.rename(volume.path_cow, volume.path_cow + '.old')
old_umask = os.umask(002)
f_cow = open(self.vm.rootcow_img, 'w')
f_root = open(self.root_img, 'r')
f_root.seek(0, os.SEEK_END)
# make empty sparse file of the same size as root.img
f_cow.truncate(f_root.tell())
f_cow.close()
f_root.close()
with open(volume.path_cow, 'w') as f_cow:
f_cow.truncate(volume.size)
os.umask(old_umask)
return volume
def destroy(self):
pass
def setup(self):
create_dir_if_not_exists(self.dir_path)
appvms_path = os.path.join(self.dir_path, 'appvms')
create_dir_if_not_exists(appvms_path)
vm_templates_path = os.path.join(self.dir_path, 'vm-templates')
create_dir_if_not_exists(vm_templates_path)
def start(self, volume):
if volume.volume_type == 'volatile':
self._reset_volume(volume)
if volume.volume_type in ['origin', 'snapshot']:
_check_path(volume.path_origin)
_check_path(volume.path_cow)
else:
_check_path(volume.path)
return volume
def stop(self, volume):
pass
def _reset_volume(self, volume):
''' Remove and recreate a volatile volume '''
assert volume.volume_type == 'volatile', "Not a volatile volume"
assert volume.size
_remove_if_exists(volume)
with open(volume.path, "w") as f_volatile:
f_volatile.truncate(volume.size)
return volume
def target_dir(self, vm):
""" Returns the path to vmdir depending on the type of the VM.
The default QubesOS file storage saves the vm images in three
different directories depending on the ``QubesVM`` type:
* ``appvms`` for ``QubesAppVm`` or ``QubesHvm``
* ``vm-templates`` for ``QubesTemplateVm`` or ``QubesTemplateHvm``
Args:
vm: a QubesVM
pool_dir: the root directory of the pool
Returns:
string (str) absolute path to the directory where the vm files
are stored
"""
if vm.is_template():
subdir = 'vm-templates'
elif vm.is_disposablevm():
subdir = 'appvms'
return os.path.join(self.dir_path, subdir,
vm.template.name + '-dvm')
else:
subdir = 'appvms'
return os.path.join(self.dir_path, subdir, vm.name)
def init_volume(self, vm, volume_config):
assert 'volume_type' in volume_config, "Volume type missing " \
+ str(volume_config)
volume_type = volume_config['volume_type']
known_types = {
'read-write': ReadWriteFile,
'read-only': ReadOnlyFile,
'origin': OriginFile,
'snapshot': SnapshotFile,
'volatile': VolatileFile,
}
if volume_type not in known_types:
raise StoragePoolException("Unknown volume type " + volume_type)
if volume_type in ['snapshot', 'read-only']:
origin_pool = vm.app.get_pool(volume_config['pool'])
assert isinstance(origin_pool,
XenPool), 'Origin volume not a xen volume'
volume_config['target_dir'] = origin_pool.target_dir(vm.template)
name = volume_config['name']
volume_config['size'] = vm.template.volume_config[name]['size']
else:
volume_config['target_dir'] = self.target_dir(vm)
return known_types[volume_type](**volume_config)
def reset_volatile_storage(self):
try:
# no template set, in any way (Standalone VM, Template VM)
if self.vm.template is None:
raise AttributeError
class XenVolume(Volume):
''' Parent class for the xen volumes implementation which expects a
`target_dir` param on initialization.
'''
# template-based HVM with only one device-mapper layer -
# volatile.img used as upper layer on root.img, no root-cow.img
# intermediate layer
if self.vm.hvm:
if os.path.exists(self.volatile_img):
if self.vm.debug:
if os.path.getmtime(self.vm.template.storage.root_img) \
> os.path.getmtime(self.volatile_img):
self.vm.log.warning(
'Template have changed, resetting root.img')
else:
self.vm.log.warning(
'Debug mode: not resetting root.img; if you'
' want to force root.img reset, either'
' update template VM, or remove volatile.img'
' file.')
return
os.remove(self.volatile_img)
# FIXME stat on f_root; with open() ...
f_volatile = open(self.volatile_img, "w")
f_root = open(self.vm.template.storage.root_img, "r")
# make empty sparse file of the same size as root.img
f_root.seek(0, os.SEEK_END)
f_volatile.truncate(f_root.tell())
f_volatile.close()
f_root.close()
return # XXX why is that? super() does not run
except AttributeError: # self.vm.template
pass
super(XenStorage, self).reset_volatile_storage()
def __init__(self, target_dir, **kwargs):
self.target_dir = target_dir
assert self.target_dir, "target_dir not specified"
super(XenVolume, self).__init__(**kwargs)
def prepare_for_vm_startup(self):
super(XenStorage, self).prepare_for_vm_startup()
class SizeMixIn(XenVolume):
''' A mix in which expects a `size` param to be > 0 on initialization and
provides a usage property wrapper.
'''
if self.drive is not None:
# pylint: disable=unused-variable
(drive_type, drive_domain, drive_path) = self.drive.split(":")
def __init__(self, size=0, **kwargs):
assert size, 'Empty size provided'
assert size > 0, 'Size for volume ' + kwargs['name'] + ' is <=0'
super(SizeMixIn, self).__init__(size=int(size), **kwargs)
if drive_domain.lower() != "dom0":
# XXX "VM '{}' holding '{}' does not exists".format(
drive_vm = self.vm.app.domains[drive_domain]
@property
def usage(self):
''' Returns the actualy used space '''
return get_disk_usage(self.vid)
if not drive_vm.is_running():
raise qubes.exc.QubesVMNotRunningError(drive_vm,
'VM {!r} holding {!r} isn\'t running'.format(
drive_domain, drive_path))
@property
def config(self):
''' return config data for serialization to qubes.xml '''
return {'name': self.name,
'pool': self.pool,
'size': str(self.size),
'volume_type': self.volume_type}
if self.rootcow_img and not os.path.exists(self.rootcow_img):
self.commit_template_changes()
class XenPool(qubes.storage.Pool):
def get_storage(self):
""" Returns an instantiated ``XenStorage``. """
return XenStorage(self.vm, vmdir=self.vmdir)
class ReadWriteFile(SizeMixIn):
''' Represents a readable & writable file image based volume '''
def __init__(self, **kwargs):
super(ReadWriteFile, self).__init__(**kwargs)
self.path = os.path.join(self.target_dir, self.name + '.img')
self.vid = self.path
def rename_target_dir(self, new_name, new_dir):
''' Called by :py:class:`XenPool` when a domain changes it's name '''
old_path = self.path
file_name = os.path.basename(self.path)
new_path = os.path.join(new_dir, file_name)
os.rename(old_path, new_path)
self.target_dir = new_dir
self.path = new_path
self.vid = self.path
class ReadOnlyFile(XenVolume):
''' Represents a readonly file image based volume '''
usage = 0
def __init__(self, size=0, **kwargs):
super(ReadOnlyFile, self).__init__(size=int(size), **kwargs)
self.path = self.vid
def rename_target_dir(self, old_name, new_dir):
""" Called by :py:class:`XenPool` when a domain changes it's name.
Only copies the volume if it belongs to the domain being renamed.
Currently if a volume is in a directory named the same as the domain,
it's ”owned” by the domain.
"""
if os.path.basename(self.target_dir) == old_name:
file_name = os.path.basename(self.path)
new_path = os.path.join(new_dir, file_name)
old_path = self.path
os.rename(old_path, new_path)
self.target_dir = new_dir
self.path = new_path
self.vid = self.path
class OriginFile(SizeMixIn):
''' Represents a readable, writeable & snapshotable file image based volume.
This is used for TemplateVM's
'''
script = 'block-origin'
def __init__(self, **kwargs):
super(OriginFile, self).__init__(**kwargs)
self.path_origin = os.path.join(self.target_dir, self.name + '.img')
self.path_cow = os.path.join(self.target_dir, self.name + '-cow.img')
self.path = '%s:%s' % (self.path_origin, self.path_cow)
self.vid = self.path_origin
def commit(self):
''' Commit Template changes '''
raise NotImplementedError
def rename_target_dir(self, new_dir):
''' Called by :py:class:`XenPool` when a domain changes it's name '''
old_path_origin = self.path_origin
old_path_cow = self.path_cow
new_path_origin = os.path.join(new_dir, self.name + '.img')
new_path_cow = os.path.join(new_dir, self.name + '-cow.img')
os.rename(old_path_origin, new_path_origin)
os.rename(old_path_cow, new_path_cow)
self.target_dir = new_dir
self.path_origin = new_path_origin
self.path_cow = new_path_cow
self.path = '%s:%s' % (self.path_origin, self.path_cow)
self.vid = self.path_origin
@property
def usage(self):
result = 0
if os.path.exists(self.path_origin):
result += get_disk_usage(self.path_origin)
if os.path.exists(self.path_cow):
result += get_disk_usage(self.path_cow)
return result
class SnapshotFile(XenVolume):
''' Represents a readonly snapshot of an :py:class:`OriginFile` volume '''
script = 'block-snapshot'
rw = False
usage = 0
def __init__(self, name=None, size=None, **kwargs):
assert size
super(SnapshotFile, self).__init__(name=name, size=int(size), **kwargs)
self.path_origin = os.path.join(self.target_dir, name + '.img')
self.path_cow = os.path.join(self.target_dir, name + '-cow.img')
self.path = '%s:%s' % (self.path_origin, self.path_cow)
self.vid = self.path_origin
class VolatileFile(SizeMixIn):
''' Represents a readable & writeable file based volume, which will be
discarded and recreated at each startup.
'''
def __init__(self, **kwargs):
super(VolatileFile, self).__init__(**kwargs)
self.path = os.path.join(self.target_dir, self.name + '.img')
self.vid = self.path
def rename_target_dir(self, new_dir):
''' Called by :py:class:`XenPool` when a domain changes it's name '''
_remove_if_exists(self)
file_name = os.path.basename(self.path)
self.target_dir = new_dir
new_path = os.path.join(new_dir, file_name)
self.path = new_path
self.vid = self.path
def create_sparse_file(path, size):
''' Create an empty sparse file '''
if os.path.exists(path):
raise IOError("Volume %s already exists", path)
parent_dir = os.path.dirname(path)
if not os.path.exists(parent_dir):
os.makedirs(parent_dir)
with open(path, 'a+b') as fh:
fh.truncate(size)
def get_disk_usage_one(st):
'''Extract disk usage of one inode from its stat_result struct.
If known, get real disk usage, as written to device by filesystem, not
logical file size. Those values may be different for sparse files.
:param os.stat_result st: stat result
:returns: disk usage
'''
try:
return st.st_blocks * BLKSIZE
except AttributeError:
return st.st_size
def get_disk_usage(path):
'''Get real disk usage of given path (file or directory).
When *path* points to directory, then it is evaluated recursively.
This function tries estiate real disk usage. See documentation of
:py:func:`get_disk_usage_one`.
:param str path: path to evaluate
:returns: disk usage
'''
try:
st = os.lstat(path)
except OSError:
return 0
ret = get_disk_usage_one(st)
# if path is not a directory, this is skipped
for dirpath, dirnames, filenames in os.walk(path):
for name in dirnames + filenames:
ret += get_disk_usage_one(os.lstat(os.path.join(dirpath, name)))
return ret
def create_dir_if_not_exists(path):
""" Check if a directory exists in if not create it.
This method does not create any parent directories.
"""
if not os.path.exists(path):
os.mkdir(path)
def copy_file(source, destination):
'''Effective file copy, preserving sparse files etc.
'''
# TODO: Windows support
# We prefer to use Linux's cp, because it nicely handles sparse files
assert os.path.exists(source), \
"Missing the source %s to copy from" % source
assert not os.path.exists(destination), \
"Destination %s already exists" % destination
parent_dir = os.path.dirname(destination)
if not os.path.exists(parent_dir):
os.makedirs(parent_dir)
try:
subprocess.check_call(['cp', '--reflink=auto', source, destination])
except subprocess.CalledProcessError:
raise IOError('Error while copying {!r} to {!r}'.format(source,
destination))
def _remove_if_exists(volume):
if os.path.exists(volume.path):
os.remove(volume.path)
def _check_path(path):
''' Raise an StoragePoolException if ``path`` does not exist'''
if not os.path.exists(path):
raise StoragePoolException('Missing image file: %s' % path)

13
qubes/tests/__init__.py
View File

@ -820,7 +820,7 @@ class BackupTestsMixin(SystemTestsMixin):
name=vmname, template=template, provides_network=True, label='red')
testnet.create_on_disk()
vms.append(testnet)
self.fill_image(testnet.private_img, 20*1024*1024)
self.fill_image(testnet.volumes['private'].vid, 20*1024*1024)
vmname = self.make_vm_name('test1')
if self.verbose:
@ -831,16 +831,17 @@ class BackupTestsMixin(SystemTestsMixin):
testvm1.netvm = testnet
testvm1.create_on_disk()
vms.append(testvm1)
self.fill_image(testvm1.private_img, 100*1024*1024)
self.fill_image(testvm1.volumes['private'].vid, 100*1024*1024)
vmname = self.make_vm_name('testhvm1')
if self.verbose:
print >>sys.stderr, "-> Creating %s" % vmname
testvm2 = self.app.add_new_vm(qubes.vm.standalonevm.StandaloneVM,
name=vmname,
hvm=True, label='red')
testvm2.create_on_disk()
self.fill_image(testvm2.root_img, 1024*1024*1024, True)
name=vmname,
hvm=True,
label='red')
testvm2.create_on_disk(verbose=self.verbose)
self.fill_image(testvm2.volumes['root'].vid, 1024 * 1024 * 1024, True)
vms.append(testvm2)
vmname = self.make_vm_name('template')

5
qubes/tests/int/basic.py
View File

@ -392,8 +392,9 @@ class TC_03_QvmRevertTemplateChanges(qubes.tests.SystemTestsMixin,
self.save_and_reload_db()
def get_rootimg_checksum(self):
p = subprocess.Popen(['sha1sum', self.test_template.root_img],
stdout=subprocess.PIPE)
p = subprocess.Popen(
['sha1sum', self.test_template.volumes['root'].vid],
stdout=subprocess.PIPE)
return p.communicate()[0]
def _do_test(self):

70
qubes/tests/storage.py
View File

@ -17,9 +17,12 @@
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import qubes.log
from qubes.storage import StoragePoolException, pool_drivers
from qubes.exc import QubesException
from qubes.storage import pool_drivers
from qubes.storage.xen import XenPool
from qubes.tests import QubesTestCase
from qubes.tests import QubesTestCase, SystemTestsMixin
# :pylint: disable=invalid-name
class TestApp(qubes.tests.TestEmitter):
@ -27,30 +30,26 @@ class TestApp(qubes.tests.TestEmitter):
class TestVM(object):
def __init__(self, app, qid, name, pool_name, template=None):
super(TestVM, self).__init__()
self.app = app
self.qid = qid
self.name = name
self.pool_name = pool_name
self.template = template
self.hvm = False
self.storage = qubes.storage.get_pool(self.pool_name,
self).get_storage()
def __init__(self, test, template=None):
self.app = test.app
self.name = test.make_vm_name('appvm')
self.log = qubes.log.get_vm_logger(self.name)
if template:
self.template = template
def is_template(self):
# :pylint: disable=no-self-use
return False
def is_disposablevm(self):
# :pylint: disable=no-self-use
return False
@property
def dir_path(self):
return self.storage.vmdir
class TestTemplateVM(TestVM):
dir_path_prefix = qubes.config.system_path['qubes_templates_dir']
def is_template(self):
return True
@ -60,46 +59,47 @@ class TestDisposableVM(TestVM):
return True
class TC_00_Pool(QubesTestCase):
class TC_00_Pool(SystemTestsMixin, QubesTestCase):
""" This class tests the utility methods from :mod:``qubes.storage`` """
def setUp(self):
super(TC_00_Pool, self).setUp()
self.init_default_template()
def test_000_unknown_pool_driver(self):
# :pylint: disable=protected-access
""" Expect an exception when unknown pool is requested"""
with self.assertRaises(StoragePoolException):
qubes.storage._get_pool_klass('foo-bar')
with self.assertRaises(QubesException):
self.app.get_pool('foo-bar')
def test_001_all_pool_drivers(self):
""" The only predefined pool driver is file """
self.assertEquals(["xen"], pool_drivers())
""" The only predefined pool driver is xen """
self.assertEquals(['linux-kernel', 'xen'], pool_drivers())
def test_002_get_pool_klass(self):
""" Expect the default pool to be `XenPool` """
# :pylint: disable=protected-access
result = qubes.storage._get_pool_klass('default')
self.assertTrue(result is XenPool)
result = self.app.get_pool('default')
self.assertIsInstance(result, XenPool)
def test_003_pool_exists_default(self):
""" Expect the default pool to exists """
self.assertTrue(qubes.storage.pool_exists('default'))
self.assertPoolExists('default')
def test_004_pool_exists_random(self):
""" Expect this pool to not a exist """
self.assertFalse(qubes.storage.pool_exists(
'asdh312096r832598213iudhas'))
def test_005_add_remove_pool(self):
def test_004_add_remove_pool(self):
""" Tries to adding and removing a pool. """
pool_name = 'asdjhrp89132'
# make sure it's really does not exist
qubes.storage.remove_pool(pool_name)
self.app.remove_pool(pool_name)
self.assertFalse(self.assertPoolExists(pool_name))
qubes.storage.add_pool(pool_name, driver='xen')
self.assertTrue(qubes.storage.pool_exists(pool_name))
self.app.add_pool(name=pool_name, driver='xen', dir_path='/tmp/asdjhrp89132')
self.assertTrue(self.assertPoolExists(pool_name))
qubes.storage.remove_pool(pool_name)
self.assertFalse(qubes.storage.pool_exists(pool_name))
self.app.remove_pool(pool_name)
self.assertFalse(self.assertPoolExists(pool_name))
def assertPoolExists(self, pool):
""" Check if specified pool exists """
return pool in self.app.pools.keys()

362
qubes/tests/storage_xen.py
View File

@ -18,13 +18,21 @@
import os
import shutil
import unittest
import qubes.storage
import qubes.tests.storage
from qubes.config import defaults
from qubes.storage import Storage
from qubes.storage.xen import (OriginFile, ReadOnlyFile, ReadWriteFile,
SnapshotFile, VolatileFile)
from qubes.tests import QubesTestCase, SystemTestsMixin
from qubes.storage.xen import XenStorage
from qubes.tests.storage import TestVM
class TC_00_XenPool(QubesTestCase):
# :pylint: disable=invalid-name
class TC_00_XenPool(SystemTestsMixin, QubesTestCase):
""" This class tests some properties of the 'default' pool. """
@ -34,32 +42,166 @@ class TC_00_XenPool(QubesTestCase):
.. sealso::
Data :data:``qubes.qubes.defaults['pool_config']``.
"""
vm = self._init_app_vm()
result = qubes.storage.get_pool("default", vm).dir_path
result = self.app.get_pool("default").dir_path
expected = '/var/lib/qubes'
self.assertEquals(result, expected)
def test001_default_storage_class(self):
""" Check when using default pool the Storage is ``XenStorage``. """
""" Check when using default pool the Storage is ``Storage``. """
result = self._init_app_vm().storage
self.assertIsInstance(result, XenStorage)
def test_002_default_pool_name(self):
""" Default pool_name is 'default'. """
vm = self._init_app_vm()
self.assertEquals(vm.pool_name, "default")
self.assertIsInstance(result, Storage)
def _init_app_vm(self):
""" Return initalised, but not created, AppVm. """
app = qubes.tests.storage.TestApp()
vmname = self.make_vm_name('appvm')
template = qubes.tests.storage.TestTemplateVM(app, 1,
self.make_vm_name('template'), 'default')
return qubes.tests.storage.TestVM(app, qid=2, name=vmname,
template=template, pool_name='default')
self.init_default_template()
return self.app.add_new_vm(qubes.vm.appvm.AppVM,
name=vmname,
template=self.app.default_template,
label='red')
class TC_01_XenVolumes(SystemTestsMixin, QubesTestCase):
POOL_DIR = '/var/lib/qubes/test-pool'
POOL_NAME = 'test-pool'
POOL_CONF = {'driver': 'xen', 'dir_path': POOL_DIR, 'name': POOL_NAME}
def setUp(self):
""" Add a test file based storage pool """
super(TC_01_XenVolumes, self).setUp()
self.init_default_template()
self.app.add_pool(**self.POOL_CONF)
def tearDown(self):
""" Remove the file based storage pool after testing """
self.app.remove_pool("test-pool")
super(TC_01_XenVolumes, self).tearDown()
shutil.rmtree(self.POOL_DIR, ignore_errors=True)
def test_000_origin_volume(self):
config = {
'name': 'root',
'pool': self.POOL_NAME,
'volume_type': 'origin',
'size': defaults['root_img_size'],
}
vm = TestVM(self)
result = self.app.get_pool(self.POOL_NAME).init_volume(vm, config)
self.assertIsInstance(result, OriginFile)
self.assertEqual(result.name, 'root')
self.assertEqual(result.pool, self.POOL_NAME)
self.assertEqual(result.size, defaults['root_img_size'])
def test_001_snapshot_volume(self):
original_path = '/var/lib/qubes/vm-templates/fedora-23/root.img'
original_size = qubes.config.defaults['root_img_size']
config = {
'name': 'root',
'pool': 'default',
'volume_type': 'snapshot',
'vid': original_path,
}
vm = TestVM(self, template=self.app.default_template)
result = self.app.get_pool(self.POOL_NAME).init_volume(vm, config)
self.assertIsInstance(result, SnapshotFile)
self.assertEqual(result.name, 'root')
self.assertEqual(result.pool, 'default')
self.assertEqual(result.size, original_size)
def test_002_read_write_volume(self):
config = {
'name': 'root',
'pool': self.POOL_NAME,
'volume_type': 'read-write',
'size': defaults['root_img_size'],
}
vm = TestVM(self)
result = self.app.get_pool(self.POOL_NAME).init_volume(vm, config)
self.assertIsInstance(result, ReadWriteFile)
self.assertEqual(result.name, 'root')
self.assertEqual(result.pool, self.POOL_NAME)
self.assertEqual(result.size, defaults['root_img_size'])
def test_003_read_volume(self):
template = self.app.default_template
original_path = template.volumes['root'].vid
original_size = qubes.config.defaults['root_img_size']
config = {
'name': 'root',
'pool': 'default',
'volume_type': 'read-only',
'vid': original_path
}
vm = TestVM(self, template=template)
result = self.app.get_pool(self.POOL_NAME).init_volume(vm, config)
self.assertIsInstance(result, ReadOnlyFile)
self.assertEqual(result.name, 'root')
self.assertEqual(result.pool, 'default')
self.assertEqual(result.size, original_size)
def test_004_volatile_volume(self):
config = {
'name': 'root',
'pool': self.POOL_NAME,
'volume_type': 'volatile',
'size': defaults['root_img_size'],
}
vm = TestVM(self)
result = self.app.get_pool(self.POOL_NAME).init_volume(vm, config)
self.assertIsInstance(result, VolatileFile)
self.assertEqual(result.name, 'root')
self.assertEqual(result.pool, self.POOL_NAME)
self.assertEqual(result.size, defaults['root_img_size'])
def test_005_appvm_volumes(self):
''' Check if AppVM volumes are propertly initialized '''
vmname = self.make_vm_name('appvm')
vm = self.app.add_new_vm(qubes.vm.appvm.AppVM,
name=vmname,
template=self.app.default_template,
label='red')
volumes = vm.volumes
self.assertIsInstance(volumes['root'], SnapshotFile)
self.assertIsInstance(volumes['private'], ReadWriteFile)
self.assertIsInstance(volumes['volatile'], VolatileFile)
expected = vm.template.dir_path + '/root.img:' + vm.template.dir_path \
+ '/root-cow.img'
self.assertVolumePath(vm, 'root', expected, rw=False)
expected = vm.dir_path + '/private.img'
self.assertVolumePath(vm, 'private', expected, rw=True)
expected = vm.dir_path + '/volatile.img'
self.assertVolumePath(vm, 'volatile', expected, rw=True)
def test_006_template_volumes(self):
''' Check if TemplateVM volumes are propertly initialized '''
vmname = self.make_vm_name('appvm')
vm = self.app.add_new_vm(qubes.vm.templatevm.TemplateVM,
name=vmname,
label='red')
volumes = vm.volumes
self.assertIsInstance(volumes['root'], OriginFile)
self.assertIsInstance(volumes['private'], ReadWriteFile)
self.assertIsInstance(volumes['volatile'], VolatileFile)
expected = vm.dir_path + '/root.img:' + vm.dir_path + '/root-cow.img'
self.assertVolumePath(vm, 'root', expected, rw=True)
expected = vm.dir_path + '/private.img'
self.assertVolumePath(vm, 'private', expected, rw=True)
expected = vm.dir_path + '/volatile.img'
self.assertVolumePath(vm, 'volatile', expected, rw=True)
def assertVolumePath(self, vm, dev_name, expected, rw=True):
# :pylint: disable=invalid-name
volumes = vm.volumes
b_dev = volumes[dev_name].block_device()
self.assertEqual(b_dev.rw, rw)
self.assertEquals(b_dev.path, expected)
@qubes.tests.skipUnlessDom0
class TC_01_XenPool(QubesTestCase):
class TC_03_XenPool(SystemTestsMixin, QubesTestCase):
""" Test the paths for the default Xen file based storage (``XenStorage``).
"""
@ -68,172 +210,124 @@ class TC_01_XenPool(QubesTestCase):
APPVMS_DIR = '/var/lib/qubes/test-pool/appvms'
TEMPLATES_DIR = '/var/lib/qubes/test-pool/vm-templates'
SERVICE_DIR = '/var/lib/qubes/test-pool/servicevms'
POOL_NAME = 'test-pool'
POOL_CONFIG = {'driver': 'xen', 'dir_path': POOL_DIR, 'name': POOL_NAME}
def setUp(self):
""" Add a test file based storage pool """
super(TC_01_XenPool, self).setUp()
qubes.storage.add_pool('test-pool', driver='xen',
dir_path=self.POOL_DIR)
self.app = qubes.tests.storage.TestApp()
self.template = qubes.tests.storage.TestTemplateVM(self.app, 1,
self.make_vm_name('template'), 'default')
super(TC_03_XenPool, self).setUp()
self.init_default_template()
self.app.add_pool(**self.POOL_CONFIG)
def tearDown(self):
""" Remove the file based storage pool after testing """
super(TC_01_XenPool, self).tearDown()
qubes.storage.remove_pool("test-pool")
self.app.remove_pool("test-pool")
super(TC_03_XenPool, self).tearDown()
shutil.rmtree(self.POOL_DIR, ignore_errors=True)
def test_001_pool_exists(self):
""" Check if the storage pool was added to the storage pool config """
self.assertTrue(qubes.storage.pool_exists('test-pool'))
self.assertIn('test-pool', self.app.pools.keys())
def test_002_pool_dir_create(self):
""" Check if the storage pool dir and subdirs were created """
# The dir should not exists before
self.assertFalse(os.path.exists(self.POOL_DIR))
pool_name = 'foo'
pool_dir = '/tmp/foo'
appvms_dir = '/tmp/foo/appvms'
templates_dir = '/tmp/foo/vm-templates'
vmname = self.make_vm_name('appvm')
qubes.tests.storage.TestVM(self.app, qid=2, name=vmname,
template=self.template, pool_name='test-pool')
self.assertFalse(os.path.exists(pool_dir))
self.assertTrue(os.path.exists(self.POOL_DIR))
self.assertTrue(os.path.exists(self.APPVMS_DIR))
self.assertTrue(os.path.exists(self.SERVICE_DIR))
self.assertTrue(os.path.exists(self.TEMPLATES_DIR))
self.app.add_pool(name=pool_name, dir_path=pool_dir, driver='xen')
def test_003_pool_dir(self):
""" Check if the vm storage pool_dir is the same as specified """
vmname = self.make_vm_name('appvm')
vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname,
template=self.template, pool_name='test-pool')
result = qubes.storage.get_pool('test-pool', vm).dir_path
self.assertEquals(self.POOL_DIR, result)
self.assertTrue(os.path.exists(pool_dir))
self.assertTrue(os.path.exists(appvms_dir))
self.assertTrue(os.path.exists(templates_dir))
def test_004_app_vmdir(self):
""" Check the vm storage dir for an AppVm"""
vmname = self.make_vm_name('appvm')
vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname,
template=self.template, pool_name='test-pool')
expected = os.path.join(self.APPVMS_DIR, vm.name)
result = vm.storage.vmdir
self.assertEquals(expected, result)
def test_005_hvm_vmdir(self):
""" Check the vm storage dir for a HVM"""
vmname = self.make_vm_name('hvm')
vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname,
template=self.template, pool_name='test-pool')
vm.hvm = True
expected = os.path.join(self.APPVMS_DIR, vm.name)
result = vm.storage.vmdir
self.assertEquals(expected, result)
@unittest.skip('TODO - servicevms dir?')
def test_006_net_vmdir(self):
""" Check the vm storage dir for a Netvm"""
vmname = self.make_vm_name('hvm')
vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname,
template=self.template, pool_name='test-pool')
expected = os.path.join(self.SERVICE_DIR, vm.name)
result = vm.storage.vmdir
self.assertEquals(expected, result)
@unittest.skip('TODO - servicevms dir?')
def test_007_proxy_vmdir(self):
""" Check the vm storage dir for a ProxyVm"""
vmname = self.make_vm_name('proxyvm')
vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname,
template=self.template, pool_name='test-pool')
expected = os.path.join(self.SERVICE_DIR, vm.name)
result = vm.storage.vmdir
self.assertEquals(expected, result)
def test_008_admin_vmdir(self):
""" Check the vm storage dir for a AdminVm"""
# TODO How to test AdminVm?
pass
def test_009_template_vmdir(self):
""" Check the vm storage dir for a TemplateVm"""
vmname = self.make_vm_name('templatevm')
vm = qubes.tests.storage.TestTemplateVM(self.app, qid=2, name=vmname,
pool_name='test-pool')
expected = os.path.join(self.TEMPLATES_DIR, vm.name)
result = vm.storage.vmdir
self.assertEquals(expected, result)
def test_010_template_hvm_vmdir(self):
""" Check the vm storage dir for a TemplateHVm"""
vmname = self.make_vm_name('templatehvm')
vm = qubes.tests.storage.TestTemplateVM(self.app, qid=2, name=vmname,
pool_name='test-pool')
expected = os.path.join(self.TEMPLATES_DIR, vm.name)
result = vm.storage.vmdir
self.assertEquals(expected, result)
shutil.rmtree(pool_dir, ignore_errors=True)
def test_011_appvm_file_images(self):
""" Check if all the needed image files are created for an AppVm"""
vmname = self.make_vm_name('appvm')
vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname,
pool_name='test-pool')
vm = self.app.add_new_vm(qubes.vm.appvm.AppVM,
name=vmname,
template=self.app.default_template,
volume_config={
'private': {
'pool': 'test-pool'
},
'volatile': {
'pool': 'test-pool'
}
},
label='red')
vm.storage.create_on_disk()
expected_vmdir = os.path.join(self.APPVMS_DIR, vm.name)
self.assertEqualsAndExists(vm.storage.vmdir, expected_vmdir)
expected_private_path = os.path.join(expected_vmdir, 'private.img')
self.assertEqualsAndExists(vm.storage.private_img,
self.assertEqualsAndExists(vm.volumes['private'].path,
expected_private_path)
expected_volatile_path = os.path.join(expected_vmdir, 'volatile.img')
self.assertEqualsAndExists(vm.storage.volatile_img,
self.assertEqualsAndExists(vm.volumes['volatile'].path,
expected_volatile_path)
def test_012_hvm_file_images(self):
""" Check if all the needed image files are created for a HVm"""
def test_013_template_file_images(self):
""" Check if root.img, private.img, volatile.img and root-cow.img are
created propertly by the storage system
"""
vmname = self.make_vm_name('tmvm')
vm = self.app.add_new_vm(qubes.vm.templatevm.TemplateVM,
name=vmname,
volume_config={
'root': {
'pool': 'test-pool'
},
'private': {
'pool': 'test-pool'
},
'volatile': {
'pool': 'test-pool'
}
},
label='red')
vm.create_on_disk()
vmname = self.make_vm_name('hvm')
vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname,
pool_name='test-pool')
vm.hvm = True
vm.storage.create_on_disk()
expected_vmdir = os.path.join(self.TEMPLATES_DIR, vm.name)
expected_vmdir = os.path.join(self.APPVMS_DIR, vm.name)
self.assertEqualsAndExists(vm.storage.vmdir, expected_vmdir)
expected_root_origin_path = os.path.join(expected_vmdir, 'root.img')
expected_root_cow_path = os.path.join(expected_vmdir, 'root-cow.img')
expected_root_path = '%s:%s' % (expected_root_origin_path,
expected_root_cow_path)
self.assertEquals(vm.volumes['root'].path, expected_root_path)
self.assertExist(vm.volumes['root'].path_origin)
expected_private_path = os.path.join(expected_vmdir, 'private.img')
self.assertEqualsAndExists(vm.storage.private_img,
self.assertEqualsAndExists(vm.volumes['private'].path,
expected_private_path)
expected_root_path = os.path.join(expected_vmdir, 'root.img')
self.assertEqualsAndExists(vm.storage.root_img, expected_root_path)
expected_volatile_path = os.path.join(expected_vmdir, 'volatile.img')
self.assertEqualsAndExists(vm.storage.volatile_img,
self.assertEqualsAndExists(vm.volumes['volatile'].path,
expected_volatile_path)
@unittest.skip('test not implemented') # TODO
def test_013_template_based_file_images(self):
pass
vm.storage.commit_template_changes()
expected_rootcow_path = os.path.join(expected_vmdir, 'root-cow.img')
self.assertEqualsAndExists(vm.volumes['root'].path_cow,
expected_rootcow_path)
def assertEqualsAndExists(self, result_path, expected_path):
""" Check if the ``result_path``, matches ``expected_path`` and exists.
See also: :meth:``assertExist``
"""
# :pylint: disable=invalid-name
self.assertEquals(result_path, expected_path)
self.assertExist(result_path)
def assertExist(self, path):
""" Assert that the given path exists. """
self.assertTrue(os.path.exists(path))
# :pylint: disable=invalid-name
self.assertTrue(os.path.exists(path), "Path %s does not exist" % path)

18
qubes/tools/qvm_create.py
View File

@ -46,9 +46,10 @@ parser.add_argument('--property', '--prop', '-p',
action=qubes.tools.PropertyAction,
help='set domain\'s property, like "internal", "memory" or "vcpus"')
parser.add_argument('--pool-name', '--pool', '-P',
action=qubes.tools.SinglePropertyAction,
help='specify the storage pool to use')
parser.add_argument('--pool', '-P',
action='append',
metavar='POOL_NAME:VOLUME_NAME',
help='specify the pool to use for a volume')
parser.add_argument('--template', '-t',
action=qubes.tools.SinglePropertyAction,
@ -79,6 +80,17 @@ parser.add_argument('name', metavar='VMNAME',
def main(args=None):
args = parser.parse_args(args)
if args.pool:
args.properties['volume_config'] = {}
for pool_vol in args.pool:
try:
pool_name, volume_name = pool_vol.split(':')
config = {'pool': pool_name, 'name': volume_name}
args.properties['volume_config'][volume_name] = config
except ValueError:
parser.error(
'Pool argument must be of form: -P pool_name:volume_name')
if 'label' not in args.properties:
parser.error('--label option is mandatory')

38
qubes/vm/appvm.py
View File

@ -3,16 +3,44 @@
import qubes.events
import qubes.vm.qubesvm
from qubes.config import defaults
class AppVM(qubes.vm.qubesvm.QubesVM):
'''Application VM'''
template = qubes.VMProperty('template', load_stage=4,
vmclass=qubes.vm.templatevm.TemplateVM,
ls_width=31,
doc='Template, on which this AppVM is based.')
template = qubes.VMProperty('template',
load_stage=4,
vmclass=qubes.vm.templatevm.TemplateVM,
ls_width=31,
doc='Template, on which this AppVM is based.')
def __init__(self, *args, **kwargs):
self.volumes = {}
self.volume_config = {
'root': {
'name': 'root',
'pool': 'default',
'volume_type': 'snapshot',
},
'private': {
'name': 'private',
'pool': 'default',
'volume_type': 'read-write',
'size': defaults['private_img_size'],
},
'volatile': {
'name': 'volatile',
'pool': 'default',
'volume_type': 'volatile',
'size': defaults['root_img_size'],
},
'kernel': {
'name': 'kernel',
'pool': 'linux-kernel',
'volume_type': 'read-only',
}
}
super(AppVM, self).__init__(*args, **kwargs)
@qubes.events.handler('domain-load')
@ -20,4 +48,4 @@ class AppVM(qubes.vm.qubesvm.QubesVM):
# pylint: disable=unused-argument
# Some additional checks for template based VM
assert self.template
#self.template.appvms.add(self) # XXX
# self.template.appvms.add(self) # XXX

192
qubes/vm/qubesvm.py
View File

@ -29,6 +29,7 @@ from __future__ import absolute_import
import base64
import datetime
import itertools
import lxml
import os
import os.path
import re
@ -209,7 +210,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
default='default',
doc='storage pool for this qube devices')
dir_path = property((lambda self: self.storage.vmdir),
dir_path = property( (lambda self: os.path.join(qubes.config.system_path['qubes_base_dir'], self.dir_path_prefix, self.name)),
doc='Root directory for files related to this domain')
# XXX swallowed uses_default_kernel
@ -331,6 +332,12 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
raise
return self._libvirt_domain
@property
def block_devices(self):
''' Return all :py:class:`qubes.devices.BlockDevice`s for current domain
for serialization in the libvirt XML template as <disk>.
'''
return [v.block_device() for v in self.volumes.values()]
@property
def qdb(self):
@ -347,21 +354,27 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
def private_img(self):
'''Location of private image of the VM (that contains :file:`/rw` \
and :file:`/home`).'''
return self.storage.private_img
warnings.warn("volatile_img is deprecated, use volumes['private'].vid",
DeprecationWarning)
return self.volumes['private'].vid
# XXX this should go to to AppVM? or TemplateVM?
@property
def root_img(self):
'''Location of root image.'''
return self.storage.root_img
warnings.warn("root_img is deprecated, use volumes['root'].vid",
DeprecationWarning)
return self.volumes['root'].vid
# XXX and this should go to exactly where? DispVM has it.
@property
def volatile_img(self):
'''Volatile image that overlays :py:attr:`root_img`.'''
return self.storage.volatile_img
warnings.warn("volatile_img is deprecated, use volumes['volatile'].vid",
DeprecationWarning)
return self.volumes['volatile'].vid
# XXX shouldn't this go elsewhere?
@ -419,12 +432,27 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
# constructor
#
def __init__(self, app, xml, **kwargs):
def __init__(self, app, xml, volume_config={}, **kwargs):
super(QubesVM, self).__init__(app, xml, **kwargs)
if hasattr(self, 'volume_config'):
if xml is not None:
for node in xml.xpath('volume-config/volume'):
name = node.get('name')
assert name
for k, v in node.items():
self.volume_config[name][k] = v
import qubes.vm.adminvm # pylint: disable=redefined-outer-name
for name, conf in volume_config.items():
for k, v in conf.items():
self.volume_config[name][k] = v
elif volume_config:
raise TypeError(
'volume_config specified, but {} did not expect that.' %
self.__class__.__name__)
#Init private attrs
import qubes.vm.adminvm # pylint: disable=redefined-outer-name
# Init private attrs
self._libvirt_domain = None
self._qdb_connection = None
@ -458,13 +486,23 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
self.features['check-updates'] = None
# will be initialized after loading all the properties
self.storage = None
# fire hooks
if xml is None:
self.events_enabled = True
self.fire_event('domain-init')
def __xml__(self):
element = super(QubesVM, self).__xml__()
if hasattr(self, 'volumes'):
volume_config_node = lxml.etree.Element('volume-config')
for volume in self.volumes.values():
volume_config_node.append(volume.__xml__())
element.append(volume_config_node)
return element
#
# event handlers
@ -477,8 +515,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
self.uuid = uuid.uuid4()
# Initialize VM image storage class
self.storage = qubes.storage.get_pool(
self.pool_name, self).get_storage()
self.storage = qubes.storage.Storage(self)
@qubes.events.handler('property-set:label')
@ -532,18 +569,18 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
self._qdb_connection.close()
self._qdb_connection = None
self.storage.rename(
os.path.join(qubes.config.system_path['qubes_base_dir'],
self.dir_path_prefix, new_name),
os.path.join(qubes.config.system_path['qubes_base_dir'],
self.dir_path_prefix, old_name))
self.storage.rename(old_name, new_name)
prefix = os.path.join(qubes.config.system_path['qubes_base_dir'], self.dir_path_prefix)
old_config = os.path.join(prefix, old_name, old_name + '.conf')
new_config = os.path.join(prefix, new_name, new_name + '.conf')
os.rename(old_config, new_config)
self._update_libvirt_domain()
if self.autostart:
self.autostart = self.autostart
@qubes.events.handler('property-pre-set:autostart')
def on_property_pre_set_autostart(self, event, prop, name, value,
oldvalue=None):
@ -633,7 +670,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
self.netvm.start(start_guid=start_guid,
notify_function=notify_function)
self.storage.prepare_for_vm_startup()
self.storage.start()
self._update_libvirt_domain()
qmemman_client = self.request_memory(mem_required)
@ -725,6 +762,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
exc_info=1)
self.libvirt_domain.shutdown()
self.storage.stop()
def kill(self):
@ -738,6 +776,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
raise qubes.exc.QubesVMNotStartedError(self)
self.libvirt_domain.destroy()
self.storage.stop()
def force_shutdown(self, *args, **kwargs):
@ -1021,7 +1060,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
p.communicate(input=self.default_user)
# TODO move to storage
# TODO rename to create
def create_on_disk(self, source_template=None):
'''Create files needed for VM.
@ -1050,11 +1089,11 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
def resize_private_img(self, size):
'''Resize private image.'''
if size >= self.get_private_img_sz():
raise qubes.exc.QubesValueError('Cannot shrink private.img')
warnings.warn(
"resize_private_img is deprecated, use volumes[name].resize()",
DeprecationWarning)
# resize the image
self.storage.resize_private_img(size)
self.volumes['private'].resize(size)
# and then the filesystem
# FIXME move this to qubes.storage.xen.XenVMStorage
@ -1070,29 +1109,12 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
if retcode != 0:
raise qubes.exc.QubesException('resize2fs failed')
# TODO move to storage
def resize_root_img(self, size, allow_start=False):
if hasattr(self, 'template'):
raise qubes.exc.QubesVMError(self,
'Cannot resize root.img of template based qube. Resize the'
' root.img of the template instead.')
warnings.warn(
"resize_root_img is deprecated, use volumes[name].resize()",
DeprecationWarning)
# TODO self.is_halted
if self.is_running():
raise qubes.exc.QubesVMNotHaltedError(self,
'Cannot resize root.img of a running qube')
if size < self.get_root_img_sz():
raise qubes.exc.QubesValueError(
'For your own safety, shrinking of root.img is disabled. If you'
' really know what you are doing, use `truncate` manually.')
with open(self.root_img, 'a+b') as fd:
fd.truncate(size)
if False: #self.hvm:
return
self.volumes['root'].resize(size)
if not allow_start:
raise qubes.exc.QubesException(
@ -1111,12 +1133,11 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
while self.is_running(): #1696
time.sleep(1)
def remove_from_disk(self):
'''Remove domain remnants from disk.'''
self.fire_event('domain-remove-from-disk')
self.storage.remove_from_disk()
self.storage.remove()
shutil.rmtree(self.vm.dir_path)
def clone_disk_files(self, src):
'''Clone files from other vm.
@ -1124,11 +1145,14 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
:param qubes.vm.qubesvm.QubesVM src: source VM
'''
if src.is_running(): # XXX what about paused?
if src.is_running(): # XXX what about paused?
raise qubes.exc.QubesVMNotHaltedError(
self, 'Cannot clone a running domain {!r}'.format(self.name))
self.storage.clone_disk_files(src)
if hasattr(src, 'volume_config'):
self.volume_config = src.volume_config
self.storage = qubes.storage.Storage(self)
self.storage.clone(src)
if src.icon_path is not None \
and os.path.exists(src.dir_path) \
@ -1450,68 +1474,69 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
# XXX shouldn't this go only to vms that have root image?
def get_disk_utilization_root_img(self):
'''Get space that is actually ocuppied by :py:attr:`root_img`.
Root image is a sparse file, so it is probably much less than logical
available space.
'''Get space that is actually ocuppied by :py:attr:`volumes['root']`.
This is a temporary wrapper for backwards compatibility. You should
call directly :py:attr:`volumes[name].utilization`
:returns: domain's real disk image size [FIXME unit]
:rtype: FIXME
.. seealso:: :py:meth:`get_root_img_sz`
'''
return qubes.storage.get_disk_usage(self.root_img)
warnings.warn(
"get_disk_utilization_root_img is deprecated, use volumes['root'].utilization",
DeprecationWarning)
return qubes.storage.get_disk_usage(self.volumes['root'].utilization)
# XXX shouldn't this go only to vms that have root image?
def get_root_img_sz(self):
'''Get image size of :py:attr:`root_img`.
Root image is a sparse file, so it is probably much more than ocuppied
physical space.
'''Get the size of the :py:attr:`volumes['root']`.
This is a temporary wrapper for backwards compatibility. You should
call directly :py:attr:`volumes[name].size`
:returns: domain's virtual disk size [FIXME unit]
:rtype: FIXME
.. seealso:: :py:meth:`get_disk_utilization_root_img`
'''
if not os.path.exists(self.root_img):
return 0
return os.path.getsize(self.root_img)
warnings.warn(
"get_disk_root_img_sz is deprecated, use volumes['root'].size",
DeprecationWarning)
return qubes.storage.get_disk_usage(self.volumes['root'].size)
def get_disk_utilization_private_img(self):
'''Get space that is actually ocuppied by :py:attr:`private_img`.
Private image is a sparse file, so it is probably much less than
logical available space.
'''Get space that is actually ocuppied by :py:attr:`volumes['private']`.
This is a temporary wrapper for backwards compatibility. You should
call directly :py:attr:`volumes[name].utilization`
:returns: domain's real disk image size [FIXME unit]
:rtype: FIXME
'''
.. seealso:: :py:meth:`get_private_img_sz`
''' # pylint: disable=invalid-name
return qubes.storage.get_disk_usage(self.private_img)
warnings.warn(
"get_disk_utilization_private_img is deprecated, use volumes['private'].utilization",
DeprecationWarning)
return qubes.storage.get_disk_usage(self.volumes[
'private'].utilization)
def get_private_img_sz(self):
'''Get image size of :py:attr:`private_img`.
Private image is a sparse file, so it is probably much more than
ocuppied physical space.
'''Get the size of the :py:attr:`volumes['private']`.
This is a temporary wrapper for backwards compatibility. You should
call directly :py:attr:`volumes[name].size`
:returns: domain's virtual disk size [FIXME unit]
:rtype: FIXME
.. seealso:: :py:meth:`get_disk_utilization_private_img`
'''
return self.storage.get_private_img_sz()
warnings.warn(
"get_disk_private_img_sz is deprecated, use volumes['private'].size",
DeprecationWarning)
return qubes.storage.get_disk_usage(self.volumes['private'].size)
def get_disk_utilization(self):
'''Return total space actually occuppied by all files belonging to \
@ -1523,7 +1548,6 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
return qubes.storage.get_disk_usage(self.dir_path)
# TODO move to storage
def verify_files(self):
'''Verify that files accessed by this machine are sane.
@ -1533,18 +1557,6 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
self.storage.verify_files()
if not os.path.exists(
os.path.join(self.storage.kernels_dir, 'vmlinuz')):
raise qubes.exc.QubesException(
'VM kernel does not exist: {0}'.format(
os.path.join(self.storage.kernels_dir, 'vmlinuz')))
if not os.path.exists(
os.path.join(self.storage.kernels_dir, 'initramfs')):
raise qubes.exc.QubesException(
'VM initramfs does not exist: {0}'.format(
os.path.join(self.storage.kernels_dir, 'initramfs')))
self.fire_event('domain-verify-files')
return True

45
qubes/vm/templatevm.py
View File

@ -1,11 +1,16 @@
#!/usr/bin/python2 -O
# vim: fileencoding=utf-8
import warnings
import qubes
import qubes.config
import qubes.vm.qubesvm
from qubes.config import defaults
from qubes.vm.qubesvm import QubesVM
class TemplateVM(qubes.vm.qubesvm.QubesVM):
class TemplateVM(QubesVM):
'''Template for AppVM'''
dir_path_prefix = qubes.config.system_path['qubes_templates_dir']
@ -13,7 +18,9 @@ class TemplateVM(qubes.vm.qubesvm.QubesVM):
@property
def rootcow_img(self):
'''COW image'''
return self.storage.rootcow_img
warnings.warn("rootcow_img is deprecated, use "
"volumes['root'].path_origin", DeprecationWarning)
return self.volumes['root'].path_cow
@property
def appvms(self):
@ -23,20 +30,40 @@ class TemplateVM(qubes.vm.qubesvm.QubesVM):
def __init__(self, *args, **kwargs):
assert 'template' not in kwargs, "A TemplateVM can not have a template"
self.volumes = {}
self.volume_config = {
'root': {
'name': 'root',
'pool': 'default',
'volume_type': 'origin',
'size': defaults['root_img_size'],
},
'private': {
'name': 'private',
'pool': 'default',
'volume_type': 'read-write',
'size': defaults['private_img_size'],
},
'volatile': {
'name': 'volatile',
'pool': 'default',
'size': defaults['root_img_size'],
'volume_type': 'volatile',
},
'kernel': {
'name': 'kernel',
'pool': 'linux-kernel',
'volume_type': 'read-only',
}
}
super(TemplateVM, self).__init__(*args, **kwargs)
# Some additional checks for template based VM
# TODO find better way
# assert self.root_img is not None, "Missing root_img for standalone VM!"
def clone_disk_files(self, src):
super(TemplateVM, self).clone_disk_files(src)
# Create root-cow.img
self.commit_changes()
def commit_changes(self):
'''Commit changes to template'''
self.log.debug('commit_changes()')
@ -45,6 +72,4 @@ class TemplateVM(qubes.vm.qubesvm.QubesVM):
assert not self.is_running(), \
'Attempt to commit changes on running Template VM!'
self.log.info(
'Commiting template update; COW: {}'.format(self.rootcow_img))
self.storage.commit_template_changes()

2
rpm_spec/core-dom0.spec
View File

@ -197,7 +197,6 @@ fi
%files
%defattr(-,root,root,-)
%config(noreplace) %attr(0664,root,qubes) %{_sysconfdir}/qubes/qmemman.conf
%config(noreplace) %attr(0664,root,qubes) %{_sysconfdir}/qubes/storage.conf
/usr/bin/qvm-*
/usr/bin/qubes-*
/usr/bin/qmemmand
@ -234,6 +233,7 @@ fi
%dir %{python_sitelib}/qubes/storage
%{python_sitelib}/qubes/storage/__init__.py*
%{python_sitelib}/qubes/storage/xen.py*
%{python_sitelib}/qubes/storage/kernels.py*
%dir %{python_sitelib}/qubes/tools
%{python_sitelib}/qubes/tools/__init__.py*

1
setup.py
View File

@ -45,5 +45,6 @@ if __name__ == '__main__':
],
'qubes.storage': [
'xen = qubes.storage.xen:XenPool',
'linux-kernel = qubes.storage.kernels:LinuxKernel',
]
})

32
templates/libvirt/xen.xml
View File

@ -45,12 +45,27 @@
<on_reboot>destroy</on_reboot>
<on_crash>destroy</on_crash>
<devices>
{#
{% for device in vm.storage %}
<disk type="block" device="{{ device.type }}">
{% set i = 0 %}
{# TODO Allow more volumes out of the box #}
{% set dd = ['e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p',
'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y']
%}
{% for device in vm.block_devices %}
<disk type="block" device="{{ device.devtype }}">
<driver name="phy" />
<source dev="{{ device.path }}" />
<target dev="{{ device.vdev }}" />
{% if device.name == 'root' %}
<target dev="xvda" />
{% elif device.name == 'private' %}
<target dev="xvdb" />
{% elif device.name == 'volatile' %}
<target dev="xvdc" />
{% elif device.name == 'kernel' %}
<target dev="xvdd" />
{% else %}
<target dev="xvd{{dd[i]}}" />
{% set i = i + 1 %}
{% endif %}
{% if not device.rw %}
<readonly />
@ -65,15 +80,6 @@
{% endif %}
</disk>
{% endfor %}
#}
{{ vm.storage.root_dev_config() }}
{% if not prepare_dvm %}{{ vm.storage.private_dev_config() }}{% endif %}
{{ vm.storage.other_dev_config() }}
{% if not vm.hvm %}
{{ vm.storage.volatile_dev_config() }}
{% endif %}
{% if vm.netvm %}
<interface type="ethernet">