From 2f99efa4b8e647c02e73f42058c1fb0f0c9c15ab Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Thu, 24 Mar 2016 22:15:24 +0100 Subject: [PATCH 01/44] Add BlockDevice --- qubes/devices.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/qubes/devices.py b/qubes/devices.py index 0911344f..b9799b50 100644 --- a/qubes/devices.py +++ b/qubes/devices.py @@ -6,6 +6,7 @@ # # Copyright (C) 2010-2016 Joanna Rutkowska # Copyright (C) 2015-2016 Wojtek Porczyk +# Copyright (C) 2016 Bahtiar `kalkin-` Gadimov # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -26,6 +27,7 @@ import re import qubes + class DeviceCollection(object): '''Bag for devices. @@ -121,3 +123,16 @@ class RegexDevice(str): class PCIDevice(RegexDevice): regex = re.compile( r'^(?P[0-9a-f]+):(?P[0-9a-f]+)\.(?P[0-9a-f]+)$') + + +class BlockDevice(object): + def __init__(self, path, name, script=None, rw=True, domain=None, + devtype='disk'): + assert name, 'Missing device name' + assert path, 'Missing device path' + self.path = path + self.name = name + self.rw = rw + self.script = script + self.domain = domain + self.devtype = devtype From cc7dd625d947f9ef30a7641c6ba792cfa59aebbf Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sun, 27 Mar 2016 22:55:09 +0200 Subject: [PATCH 02/44] Loop over QubesVM.block_devices in libvirt xml --- qubes/vm/qubesvm.py | 6 ++++++ templates/libvirt/xen.xml | 32 +++++++++++++++++++------------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 09519010..9db842c1 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -331,6 +331,12 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): raise return self._libvirt_domain + @property + def block_devices(self): + return [self.storage.root_dev_config(), + self.storage.private_dev_config(), + self.storage.volatile_dev_config(), + self.storage.other_dev_config()] @property def qdb(self): diff --git a/templates/libvirt/xen.xml b/templates/libvirt/xen.xml index 13b15e51..a16ff581 100644 --- a/templates/libvirt/xen.xml +++ b/templates/libvirt/xen.xml @@ -45,12 +45,27 @@ destroy destroy - {# - {% for device in vm.storage %} - + {% set i = 0 %} + {# TODO Allow more volumes out of the box #} + {% set dd = ['e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', + 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y'] + %} + {% for device in vm.block_devices %} + - + {% if device.name == 'root' %} + + {% elif device.name == 'private' %} + + {% elif device.name == 'volatile' %} + + {% elif device.name == 'kernel' %} + + {% else %} + + {% set i = i + 1 %} + {% endif %} {% if not device.rw %} @@ -65,15 +80,6 @@ {% endif %} {% endfor %} - #} - - {{ vm.storage.root_dev_config() }} - {% if not prepare_dvm %}{{ vm.storage.private_dev_config() }}{% endif %} - {{ vm.storage.other_dev_config() }} - - {% if not vm.hvm %} - {{ vm.storage.volatile_dev_config() }} - {% endif %} {% if vm.netvm %} From 47e89d84b6f78ae30ff1d10302662d9434f0c443 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sun, 27 Mar 2016 22:30:33 +0200 Subject: [PATCH 03/44] XenStorage.format_disk_dev returns now BlockDevice --- qubes/storage/__init__.py | 16 +++++++-------- qubes/storage/xen.py | 41 ++++++++++----------------------------- 2 files changed, 17 insertions(+), 40 deletions(-) diff --git a/qubes/storage/__init__.py b/qubes/storage/__init__.py index 4ddbe34b..bcbdb0d9 100644 --- a/qubes/storage/__init__.py +++ b/qubes/storage/__init__.py @@ -101,8 +101,7 @@ class Storage(object): def other_dev_config(self): if self.modules_img is not None: - return self.format_disk_dev(self.modules_img, self.modules_dev, - rw=self.modules_img_rw) + return BlockDevice(self.modules_img, 'kernel', rw=False) elif self.drive is not None: (drive_type, drive_domain, drive_path) = self.drive.split(":") if drive_type == 'hd': @@ -114,19 +113,18 @@ class Storage(object): drive_domain = None return self.format_disk_dev(drive_path, - self.modules_dev, - rw=rw, - devtype=drive_type, - domain=drive_domain) + 'other', + rw=rw, + devtype=drive_type, + domain=drive_domain) else: return '' - def format_disk_dev(self, path, vdev, script=None, rw=True, devtype='disk', - domain=None): + def format_disk_dev(self, path, name, script=None, rw=True, devtype='disk', + domain=None): raise NotImplementedError() - @property def kernels_dir(self): '''Directory where kernel resides. diff --git a/qubes/storage/xen.py b/qubes/storage/xen.py index 57c0ff66..097bc705 100644 --- a/qubes/storage/xen.py +++ b/qubes/storage/xen.py @@ -31,12 +31,11 @@ import os.path import re import subprocess -import lxml.etree - import qubes import qubes.config import qubes.storage import qubes.vm.templatevm +from qubes.devices import BlockDevice class XenStorage(qubes.storage.Storage): @@ -92,39 +91,19 @@ class XenStorage(qubes.storage.Storage): '''Path to the volatile image''' return self.abspath(qubes.config.vm_files['volatile_img']) + def format_disk_dev(self, path, name, script=None, rw=True, devtype='disk', + domain=None): - def format_disk_dev(self, path, vdev, script=None, rw=True, devtype='disk', - domain=None): - if path is None: - return '' - - element = lxml.etree.Element('disk') - element.set('type', 'block') - element.set('device', devtype) - - element.append(lxml.etree.Element('driver', name='phy')) - element.append(lxml.etree.Element('source', dev=path)) - element.append(lxml.etree.Element('target', dev=vdev)) - - if not rw: - element.append(lxml.etree.Element('readonly')) - if domain is not None: - # XXX vm.name? - element.append(lxml.etree.Element('domain', name=domain)) - if script: - element.append(lxml.etree.Element('script', path=script)) - - # TODO return element - return lxml.etree.tostring(element) - + return BlockDevice(path, name, script, rw, domain, devtype) def root_dev_config(self): + dev_name = 'root' if isinstance(self.vm, qubes.vm.templatevm.TemplateVM): return self.format_disk_dev( '{root}:{rootcow}'.format( root=self.root_img, rootcow=self.rootcow_img), - self.root_dev, + dev_name, script='block-origin') elif self.vm.hvm and hasattr(self.vm, 'template'): @@ -137,7 +116,7 @@ class XenStorage(qubes.storage.Storage): '{root}:{volatile}'.format( root=self.vm.template.storage.root_img, volatile=self.volatile_img), - self.root_dev, + dev_name, script='block-snapshot') elif hasattr(self.vm, 'template'): @@ -154,14 +133,14 @@ class XenStorage(qubes.storage.Storage): else: # standalone qube - return self.format_disk_dev(self.root_img, self.root_dev) + return self.format_disk_dev(self.root_img, dev_name) def private_dev_config(self): - return self.format_disk_dev(self.private_img, self.private_dev) + return self.format_disk_dev(self.private_img, 'private') def volatile_dev_config(self): - return self.format_disk_dev(self.volatile_img, self.volatile_dev) + return self.format_disk_dev(self.volatile_img, 'volatile') def create_on_disk_private_img(self, source_template=None): From c791cb1935666b4d997ead0f4433ed50d7f00c7a Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Mon, 21 Mar 2016 12:00:53 +0100 Subject: [PATCH 04/44] Serialize pool configuration to XML --- qubes/__init__.py | 30 ++++++++++++++++++++++-------- qubes/config.py | 2 +- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/qubes/__init__.py b/qubes/__init__.py index a5e5c7eb..bd4ab466 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -36,8 +36,6 @@ __author__ = 'Invisible Things Lab' __license__ = 'GPLv2 or later' __version__ = 'R3' -import ast -import atexit import collections import errno import grp @@ -47,7 +45,6 @@ import os.path import sys import tempfile import time -import warnings import __builtin__ @@ -1181,7 +1178,6 @@ class Qubes(PropertyHolder): default=True, doc='check for updates inside qubes') - def __init__(self, store=None, load=True, **kwargs): #: logger instance for logging global messages self.log = logging.getLogger('app') @@ -1194,6 +1190,9 @@ class Qubes(PropertyHolder): #: collection of all available labels for VMs self.labels = {} + #: collection of all available pool configurations + self.pool_configs = {} + #: Connection to VMM self.vmm = VMMConnection() @@ -1235,7 +1234,7 @@ class Qubes(PropertyHolder): ''' try: - fd = os.open(self._store, os.O_RDWR) # no O_CREAT + fd = os.open(self._store, os.O_RDWR) # no O_CREAT except OSError as e: if e.errno != errno.ENOENT: raise @@ -1256,11 +1255,17 @@ class Qubes(PropertyHolder): self.xml = lxml.etree.parse(fh) - # stage 1: load labels + # stage 1: load labels and pools for node in self.xml.xpath('./labels/label'): label = Label.fromxml(node) self.labels[label.index] = label + for node in self.xml.xpath('./pools/pool'): + name = node.get('name') + config_data = node.attrib + del(config_data['name']) + self.pool_configs[name] = config_data + # stage 2: load VMs for node in self.xml.xpath('./domains/domain'): # pylint: disable=no-member @@ -1270,7 +1275,7 @@ class Qubes(PropertyHolder): vm.init_log() self.domains.add(vm) - if not 0 in self.domains: + if 0 not in self.domains: self.domains.add(qubes.vm.adminvm.AdminVM( self, None, qid=0, name='dom0')) @@ -1310,11 +1315,11 @@ class Qubes(PropertyHolder): fh.close() del fh - def __xml__(self): element = lxml.etree.Element('qubes') element.append(self.xml_labels()) + element.append(self.xml_pool_configs()) element.append(self.xml_properties()) domains = lxml.etree.Element('domains') @@ -1395,6 +1400,7 @@ class Qubes(PropertyHolder): 7: Label(7, '0x75507b', 'purple'), 8: Label(8, '0x000000', 'black'), } + self.pool_configs['default'] = qubes.config.defaults['pool_config'] self.domains.add( qubes.vm.adminvm.AdminVM(self, None, qid=0, name='dom0')) self.save() @@ -1413,6 +1419,14 @@ class Qubes(PropertyHolder): labels.append(label.__xml__()) return labels + def xml_pool_configs(self): + """ Helper for converting pools config to xml """ + pools = lxml.etree.Element('pools') + for config_data in self.pool_configs.values(): + p = lxml.etree.Element('pool', **config_data) + pools.append(p) + + return pools def get_vm_class(self, clsname): '''Find the class for a domain. diff --git a/qubes/config.py b/qubes/config.py index a09e2f35..9984238d 100644 --- a/qubes/config.py +++ b/qubes/config.py @@ -83,7 +83,7 @@ defaults = { 'private_img_size': 2*1024*1024*1024, 'root_img_size': 10*1024*1024*1024, - 'pool_config': {'dir_path': '/var/lib/qubes'}, + 'pool_config': {'dir_path': '/var/lib/qubes/', 'driver': 'xen'}, # how long (in sec) to wait for VMs to shutdown, # before killing them (when used qvm-run with --wait option), From 36470310a2ae835ca2b05f37030e1510cf227f95 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Mon, 21 Mar 2016 12:20:19 +0100 Subject: [PATCH 05/44] Replace pool config parsing logic - Move add_pool/remove_pool to Qubes class - Add Qubes.get_pool - Remove storage.conf --- Makefile | 1 - etc/storage.conf | 13 ------ qubes/__init__.py | 70 ++++++++++++++++++++++++++------ qubes/storage/__init__.py | 85 --------------------------------------- rpm_spec/core-dom0.spec | 1 - 5 files changed, 57 insertions(+), 113 deletions(-) delete mode 100644 etc/storage.conf diff --git a/Makefile b/Makefile index 01edd38a..99d1491d 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,6 @@ endif # $(MAKE) install -C tests $(MAKE) install -C relaxng mkdir -p $(DESTDIR)/etc/qubes - cp etc/storage.conf $(DESTDIR)/etc/qubes/ ifeq ($(BACKEND_VMM),xen) # Currently supported only on xen cp etc/qmemman.conf $(DESTDIR)/etc/qubes/ diff --git a/etc/storage.conf b/etc/storage.conf deleted file mode 100644 index 78fd2a3a..00000000 --- a/etc/storage.conf +++ /dev/null @@ -1,13 +0,0 @@ -[default] ; poolname -driver=xen ; the default xen storage -; class = qubes.storage.xen.XenStorage ; class always overwrites the driver -; -; To use our own pool driver it needs to provide `qubes.storage` entry_point -; see also: https://pythonhosted.org/setuptools/setuptools.html#dynamic-discovery-of-services-and-plugins -; class name -; [pool-b] -; driver = foo -; -; [test-dummy] -; driver=dummy - diff --git a/qubes/__init__.py b/qubes/__init__.py index bd4ab466..e25b1e1d 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -48,8 +48,6 @@ import time import __builtin__ -import docutils.core -import docutils.io import jinja2 import lxml.etree import pkg_resources @@ -1190,8 +1188,8 @@ class Qubes(PropertyHolder): #: collection of all available labels for VMs self.labels = {} - #: collection of all available pool configurations - self.pool_configs = {} + #: collection of all pools + self.pools = {} #: Connection to VMM self.vmm = VMMConnection() @@ -1262,9 +1260,11 @@ class Qubes(PropertyHolder): for node in self.xml.xpath('./pools/pool'): name = node.get('name') - config_data = node.attrib - del(config_data['name']) - self.pool_configs[name] = config_data + assert name, "Pool name '%s' is invalid " % name + try: + self.pools[name] = self._get_pool(**node.attrib) + except qubes.exc.QubesException as e: + self.log.error(e.message) # stage 2: load VMs for node in self.xml.xpath('./domains/domain'): @@ -1400,7 +1400,9 @@ class Qubes(PropertyHolder): 7: Label(7, '0x75507b', 'purple'), 8: Label(8, '0x000000', 'black'), } - self.pool_configs['default'] = qubes.config.defaults['pool_config'] + for name, config in qubes.config.defaults['pool_configs'].items(): + self.pools[name] = self._get_pool(**config) + self.domains.add( qubes.vm.adminvm.AdminVM(self, None, qid=0, name='dom0')) self.save() @@ -1421,12 +1423,12 @@ class Qubes(PropertyHolder): def xml_pool_configs(self): """ Helper for converting pools config to xml """ - pools = lxml.etree.Element('pools') - for config_data in self.pool_configs.values(): - p = lxml.etree.Element('pool', **config_data) - pools.append(p) + pools_xml = lxml.etree.Element('pools') + for pool in self.pools.values(): + p = lxml.etree.Element('pool', **pool.config) + pools_xml.append(p) - return pools + return pools_xml def get_vm_class(self, clsname): '''Find the class for a domain. @@ -1487,6 +1489,48 @@ class Qubes(PropertyHolder): raise KeyError(label) + def add_pool(self, **kwargs): + """ Add a storage pool to config.""" + name = kwargs['name'] + self.pools[name] = self._get_pool(**kwargs) + self.save() + + def remove_pool(self, name): + """ Remove a storage pool from config file. """ + try: + del self.pools[name] + except KeyError: + return + + self.save() + + def get_pool(self, name): + ''' Returns a :py:class:`qubes.storage.Pool` instance ''' + try: + return self.pools[name] + except KeyError: + raise qubes.exc.QubesException('Unknown storage pool ' + name) + + def _get_pool(self, **kwargs): + try: + name = kwargs['name'] + assert name, 'Name needs to be an non empty string' + except KeyError: + raise qubes.exc.QubesException('No pool name for pool') + + try: + driver = kwargs['driver'] + except KeyError: + raise qubes.exc.QubesException('No driver specified for pool ' + + name) + try: + klass = qubes.get_entry_point_one( + qubes.storage.STORAGE_ENTRY_POINT, driver) + del kwargs['driver'] + return klass(**kwargs) + except KeyError: + raise qubes.exc.QubesException('Driver %s for pool %s' % + (driver, name)) @qubes.events.handler('domain-pre-delete') def on_domain_pre_deleted(self, event, vm): diff --git a/qubes/storage/__init__.py b/qubes/storage/__init__.py index bcbdb0d9..c53e68c6 100644 --- a/qubes/storage/__init__.py +++ b/qubes/storage/__init__.py @@ -28,7 +28,6 @@ from __future__ import absolute_import -import ConfigParser import os import os.path import shutil @@ -40,7 +39,6 @@ import qubes.exc import qubes.utils BLKSIZE = 512 -CONFIG_FILE = '/etc/qubes/storage.conf' STORAGE_ENTRY_POINT = 'qubes.storage' @@ -353,89 +351,6 @@ def get_disk_usage(path): return ret - -def get_pool(name, vm): - """ Instantiates the storage for the specified vm """ - config = _get_storage_config_parser() - - klass = _get_pool_klass(name, config) - - keys = [k for k in config.options(name) if k != 'driver' and k != 'class'] - values = [config.get(name, o) for o in keys] - config_kwargs = dict(zip(keys, values)) - - if name == 'default': - kwargs = qubes.config.defaults['pool_config'].copy() - kwargs.update(keys) - else: - kwargs = config_kwargs - - return klass(vm, **kwargs) - - -def pool_exists(name): - """ Check if the specified pool exists """ - try: - _get_pool_klass(name) - return True - except StoragePoolException: - return False - -def add_pool(name, **kwargs): - """ Add a storage pool to config.""" - config = _get_storage_config_parser() - config.add_section(name) - for key, value in kwargs.iteritems(): - config.set(name, key, value) - _write_config(config) - -def remove_pool(name): - """ Remove a storage pool from config file. """ - config = _get_storage_config_parser() - config.remove_section(name) - _write_config(config) - -def _write_config(config): - with open(CONFIG_FILE, 'w') as configfile: - config.write(configfile) - -def _get_storage_config_parser(): - """ Instantiates a `ConfigParaser` for specified storage config file. - - Returns: - RawConfigParser - """ - config = ConfigParser.RawConfigParser() - config.read(CONFIG_FILE) - return config - - -def _get_pool_klass(name, config=None): - """ Returns the storage klass for the specified pool. - - Args: - name: The pool name. - config: If ``config`` is not specified - `_get_storage_config_parser()` is called. - - Returns: - type: A class inheriting from `QubesVmStorage` - """ - if config is None: - config = _get_storage_config_parser() - - if not config.has_section(name): - raise StoragePoolException('Uknown storage pool ' + name) - elif not config.has_option(name, 'driver'): - raise StoragePoolException('No driver specified for pool ' + name) - - - driver = config.get(name, 'driver') - try: - return qubes.get_entry_point_one(STORAGE_ENTRY_POINT, driver) - except KeyError: - raise StoragePoolException('Driver %s for pool %s' % (driver, name)) - class Pool(object): def __init__(self, vm, dir_path): assert vm is not None diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec index 942e3f66..d1832259 100644 --- a/rpm_spec/core-dom0.spec +++ b/rpm_spec/core-dom0.spec @@ -197,7 +197,6 @@ fi %files %defattr(-,root,root,-) %config(noreplace) %attr(0664,root,qubes) %{_sysconfdir}/qubes/qmemman.conf -%config(noreplace) %attr(0664,root,qubes) %{_sysconfdir}/qubes/storage.conf /usr/bin/qvm-* /usr/bin/qubes-* /usr/bin/qmemmand From 22d73e8fa973641117bbead6d2dd412ea9ec9bab Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Mon, 21 Mar 2016 12:22:01 +0100 Subject: [PATCH 06/44] Fix Pool dir_path normalization --- qubes/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes/storage/__init__.py b/qubes/storage/__init__.py index c53e68c6..cea2360a 100644 --- a/qubes/storage/__init__.py +++ b/qubes/storage/__init__.py @@ -357,7 +357,7 @@ class Pool(object): assert dir_path is not None self.vm = vm - self.dir_path = dir_path + self.dir_path = os.path.normpath(dir_path) self.create_dir_if_not_exists(self.dir_path) From bd4674b6589f545f57f6a160e1b44c2873e85de3 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Thu, 14 Apr 2016 19:02:41 +0200 Subject: [PATCH 07/44] Remove obsolete tests --- qubes/tests/storage_xen.py | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/qubes/tests/storage_xen.py b/qubes/tests/storage_xen.py index 5954fba6..9cbf67f7 100644 --- a/qubes/tests/storage_xen.py +++ b/qubes/tests/storage_xen.py @@ -122,39 +122,6 @@ class TC_01_XenPool(QubesTestCase): result = vm.storage.vmdir self.assertEquals(expected, result) - def test_005_hvm_vmdir(self): - """ Check the vm storage dir for a HVM""" - vmname = self.make_vm_name('hvm') - vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname, - template=self.template, pool_name='test-pool') - vm.hvm = True - - expected = os.path.join(self.APPVMS_DIR, vm.name) - result = vm.storage.vmdir - self.assertEquals(expected, result) - - @unittest.skip('TODO - servicevms dir?') - def test_006_net_vmdir(self): - """ Check the vm storage dir for a Netvm""" - vmname = self.make_vm_name('hvm') - vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname, - template=self.template, pool_name='test-pool') - - expected = os.path.join(self.SERVICE_DIR, vm.name) - result = vm.storage.vmdir - self.assertEquals(expected, result) - - @unittest.skip('TODO - servicevms dir?') - def test_007_proxy_vmdir(self): - """ Check the vm storage dir for a ProxyVm""" - vmname = self.make_vm_name('proxyvm') - vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname, - template=self.template, pool_name='test-pool') - - expected = os.path.join(self.SERVICE_DIR, vm.name) - result = vm.storage.vmdir - self.assertEquals(expected, result) - def test_008_admin_vmdir(self): """ Check the vm storage dir for a AdminVm""" # TODO How to test AdminVm? From 3c798bc825e1a1a95e84226c541f58ab8469da59 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Wed, 23 Mar 2016 12:11:58 +0100 Subject: [PATCH 08/44] Pool configuration include the pool name --- qubes/config.py | 4 +++- qubes/storage/__init__.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/qubes/config.py b/qubes/config.py index 9984238d..636b0b07 100644 --- a/qubes/config.py +++ b/qubes/config.py @@ -83,7 +83,9 @@ defaults = { 'private_img_size': 2*1024*1024*1024, 'root_img_size': 10*1024*1024*1024, - 'pool_config': {'dir_path': '/var/lib/qubes/', 'driver': 'xen'}, + 'pool_config': {'dir_path': '/var/lib/qubes/', + 'driver': 'xen', + 'name': 'default'}, # how long (in sec) to wait for VMs to shutdown, # before killing them (when used qvm-run with --wait option), diff --git a/qubes/storage/__init__.py b/qubes/storage/__init__.py index cea2360a..ccd30ad8 100644 --- a/qubes/storage/__init__.py +++ b/qubes/storage/__init__.py @@ -352,11 +352,13 @@ def get_disk_usage(path): return ret class Pool(object): - def __init__(self, vm, dir_path): + def __init__(self, vm, dir_path, name): assert vm is not None assert dir_path is not None + assert name self.vm = vm + self.name = name self.dir_path = os.path.normpath(dir_path) self.create_dir_if_not_exists(self.dir_path) From c3d8c899ccc7e9ad1fe00e3112023f85b89d5319 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Mon, 21 Mar 2016 17:25:27 +0100 Subject: [PATCH 09/44] Add TemplateVM test for storage_xen --- qubes/tests/storage_xen.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/qubes/tests/storage_xen.py b/qubes/tests/storage_xen.py index 9cbf67f7..a8e0b1f7 100644 --- a/qubes/tests/storage_xen.py +++ b/qubes/tests/storage_xen.py @@ -189,9 +189,34 @@ class TC_01_XenPool(QubesTestCase): self.assertEqualsAndExists(vm.storage.volatile_img, expected_volatile_path) - @unittest.skip('test not implemented') # TODO - def test_013_template_based_file_images(self): - pass + def test_013_template_file_images(self): + """ Check if root.img, private.img, volatile.img and root-cow.img are + created propertly by the storage system + """ + vmname = self.make_vm_name('tmvm') + vm = self.app.add_new_vm(qubes.vm.templatevm.TemplateVM, name=vmname, + pool_name='test-pool', label='red') + vm.storage.create_on_disk() + + expected_vmdir = os.path.join(self.TEMPLATES_DIR, vm.name) + self.assertEqualsAndExists(vm.storage.vmdir, expected_vmdir) + + expected_root_path = os.path.join(expected_vmdir, 'root.img') + self.assertEqualsAndExists(vm.storage.root_img, expected_root_path) + + expected_private_path = os.path.join(expected_vmdir, 'private.img') + self.assertEqualsAndExists(vm.storage.private_img, + expected_private_path) + + expected_volatile_path = os.path.join(expected_vmdir, 'volatile.img') + self.assertEqualsAndExists(vm.storage.volatile_img, + expected_volatile_path) + + vm.storage.commit_template_changes() + expected_rootcow_path = os.path.join(expected_vmdir, 'root-cow.img') + self.assertEqualsAndExists(vm.storage.rootcow_img, + expected_rootcow_path) + def assertEqualsAndExists(self, result_path, expected_path): """ Check if the ``result_path``, matches ``expected_path`` and exists. From 428dd5bc1bcde6ece10dc3c574c8d264acb6d036 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Mon, 28 Mar 2016 00:01:00 +0200 Subject: [PATCH 10/44] QubesVM.dir_path is set independent of storage --- qubes/vm/qubesvm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 9db842c1..27ec98d7 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -209,7 +209,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): default='default', doc='storage pool for this qube devices') - dir_path = property((lambda self: self.storage.vmdir), + dir_path = property( (lambda self: os.path.join(qubes.config.system_path['qubes_base_dir'], self.dir_path_prefix, self.name)), doc='Root directory for files related to this domain') # XXX swallowed uses_default_kernel From 24193c4308ae7b99357f75c347812854a3d55b26 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Mon, 28 Mar 2016 02:55:38 +0200 Subject: [PATCH 11/44] Add Volume class --- qubes/storage/__init__.py | 41 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/qubes/storage/__init__.py b/qubes/storage/__init__.py index ccd30ad8..db9a03af 100644 --- a/qubes/storage/__init__.py +++ b/qubes/storage/__init__.py @@ -46,8 +46,47 @@ class StoragePoolException(qubes.exc.QubesException): pass +class Volume(object): + ''' Encapsulates all data about a volume for serialization to qubes.xml and + libvirt config. + ''' + + devtype = 'disk' + domain = None + path = None + rw = True + script = None + usage = 0 + + def __init__(self, name=None, pool=None, volume_type=None, vid=None, + size=0): + assert name and pool and volume_type + self.name = str(name) + self.pool = str(pool) + self.vid = vid + self.size = size + self.volume_type = volume_type + + @property + def config(self): + ''' return config data for serialization to qubes.xml ''' + return {'name': self.name, + 'pool': self.pool, + 'volume_type': self.volume_type} + + def __str__(self): + return str({'name': self.name, 'pool': self.pool, 'vid': self.vid}) + + def block_device(self): + ''' Return :py:class:`qubes.devices.BlockDevice` for serialization in + the libvirt XML template as . + ''' + return BlockDevice(self.path, self.name, self.script, self.rw, + self.domain, self.devtype) + + class Storage(object): - '''Class for handling VM virtual disks. + ''' Class for handling VM virtual disks. This is base class for all other implementations, mostly with Xen on Linux in mind. From 9d646aabd326bc30626b5ee64ef02fd229b57ee1 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 15 Apr 2016 13:09:50 +0200 Subject: [PATCH 12/44] Add volume_config to AppVM and TemplateVM --- qubes/vm/appvm.py | 38 +++++++++++++++++++++++++++++++++----- qubes/vm/qubesvm.py | 1 - qubes/vm/templatevm.py | 38 ++++++++++++++++++++++++++++++-------- 3 files changed, 63 insertions(+), 14 deletions(-) diff --git a/qubes/vm/appvm.py b/qubes/vm/appvm.py index aa2e3966..d310ca99 100644 --- a/qubes/vm/appvm.py +++ b/qubes/vm/appvm.py @@ -3,16 +3,44 @@ import qubes.events import qubes.vm.qubesvm +from qubes.config import defaults + class AppVM(qubes.vm.qubesvm.QubesVM): '''Application VM''' - template = qubes.VMProperty('template', load_stage=4, - vmclass=qubes.vm.templatevm.TemplateVM, - ls_width=31, - doc='Template, on which this AppVM is based.') + template = qubes.VMProperty('template', + load_stage=4, + vmclass=qubes.vm.templatevm.TemplateVM, + ls_width=31, + doc='Template, on which this AppVM is based.') def __init__(self, *args, **kwargs): + self.volumes = {} + self.volume_config = { + 'root': { + 'name': 'root', + 'pool': 'default', + 'volume_type': 'snapshot', + }, + 'private': { + 'name': 'private', + 'pool': 'default', + 'volume_type': 'read-write', + 'size': defaults['private_img_size'], + }, + 'volatile': { + 'name': 'volatile', + 'pool': 'default', + 'volume_type': 'volatile', + 'size': defaults['root_img_size'], + }, + 'kernel': { + 'name': 'kernel', + 'pool': 'linux-kernel', + 'volume_type': 'read-only', + } + } super(AppVM, self).__init__(*args, **kwargs) @qubes.events.handler('domain-load') @@ -20,4 +48,4 @@ class AppVM(qubes.vm.qubesvm.QubesVM): # pylint: disable=unused-argument # Some additional checks for template based VM assert self.template - #self.template.appvms.add(self) # XXX + # self.template.appvms.add(self) # XXX diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 27ec98d7..9aef6f71 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -464,7 +464,6 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): self.features['check-updates'] = None # will be initialized after loading all the properties - self.storage = None # fire hooks if xml is None: diff --git a/qubes/vm/templatevm.py b/qubes/vm/templatevm.py index f0c7fe2f..7eadc134 100644 --- a/qubes/vm/templatevm.py +++ b/qubes/vm/templatevm.py @@ -4,6 +4,8 @@ import qubes import qubes.config import qubes.vm.qubesvm +from qubes.config import defaults + class TemplateVM(qubes.vm.qubesvm.QubesVM): '''Template for AppVM''' @@ -23,20 +25,40 @@ class TemplateVM(qubes.vm.qubesvm.QubesVM): def __init__(self, *args, **kwargs): assert 'template' not in kwargs, "A TemplateVM can not have a template" + self.volumes = {} + self.volume_config = { + 'root': { + 'name': 'root', + 'pool': 'default', + 'volume_type': 'origin', + 'size': defaults['root_img_size'], + }, + 'private': { + 'name': 'private', + 'pool': 'default', + 'volume_type': 'read-write', + 'size': defaults['private_img_size'], + }, + 'volatile': { + 'name': 'volatile', + 'pool': 'default', + 'size': defaults['root_img_size'], + 'volume_type': 'volatile', + }, + 'kernel': { + 'name': 'kernel', + 'pool': 'linux-kernel', + 'volume_type': 'read-only', + } + } super(TemplateVM, self).__init__(*args, **kwargs) - # Some additional checks for template based VM - # TODO find better way -# assert self.root_img is not None, "Missing root_img for standalone VM!" - - def clone_disk_files(self, src): super(TemplateVM, self).clone_disk_files(src) # Create root-cow.img self.commit_changes() - def commit_changes(self): '''Commit changes to template''' self.log.debug('commit_changes()') @@ -45,6 +67,6 @@ class TemplateVM(qubes.vm.qubesvm.QubesVM): assert not self.is_running(), \ 'Attempt to commit changes on running Template VM!' - self.log.info( - 'Commiting template update; COW: {}'.format(self.rootcow_img)) + self.log.info('Commiting template update; COW: {}'.format( + self.rootcow_img)) self.storage.commit_template_changes() From 32255a79164cd087c481e051e87528a91e847073 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 15 Apr 2016 20:13:06 +0200 Subject: [PATCH 13/44] =?UTF-8?q?Reverted=20Storage=20=E2=86=90=E2=86=92?= =?UTF-8?q?=20Pool=20dependency?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Storage() operates on a pool and in future on multiple pools --- qubes/storage/__init__.py | 234 ++++++++++++++++++-------------------- qubes/vm/qubesvm.py | 3 +- 2 files changed, 113 insertions(+), 124 deletions(-) diff --git a/qubes/storage/__init__.py b/qubes/storage/__init__.py index db9a03af..3bfb4762 100644 --- a/qubes/storage/__init__.py +++ b/qubes/storage/__init__.py @@ -37,6 +37,7 @@ import pkg_resources import qubes import qubes.exc import qubes.utils +from qubes.devices import BlockDevice BLKSIZE = 512 STORAGE_ENTRY_POINT = 'qubes.storage' @@ -92,29 +93,37 @@ class Storage(object): in mind. ''' - root_img = None - private_img = None - volatile_img = None - - modules_dev = None - - def __init__(self, vm, private_img_size=None, root_img_size=None): + modules_dev = 'xvdd' + def __init__(self, vm): #: Domain for which we manage storage self.vm = vm - - #: Size of the private image - self.private_img_size = private_img_size \ - if private_img_size is not None \ - else qubes.config.defaults['private_img_size'] - - #: Size of the root image - self.root_img_size = root_img_size \ - if root_img_size is not None \ - else qubes.config.defaults['root_img_size'] - #: Additional drive (currently used only by HVM) self.drive = None + for name, conf in self.vm.volume_config.items(): + assert 'pool' in conf + pool = get_pool(conf['pool'], self.vm) + self.vm.volumes[name] = pool.init_volume(conf) + + @property + def root_img(self): + pool = self.get_pool() + return pool.root_img + + @property + def private_img(self): + pool = self.get_pool() + return pool.private_img + + @property + def volatile_img(self): + pool = self.get_pool() + return pool.volatile_img + + @property + def rootcow_img(self): + pool = self.get_pool() + return pool.rootcow_img def get_config_params(self): args = {} @@ -128,17 +137,25 @@ class Storage(object): return args def root_dev_config(self): - raise NotImplementedError() + pool = self.get_pool() + return pool.root_dev_config() def private_dev_config(self): - raise NotImplementedError() + pool = self.get_pool() + return pool.private_dev_config() def volatile_dev_config(self): - raise NotImplementedError() + pool = self.get_pool() + return pool.volatile_dev_config() + + def modules_dev_config(self): + return self.format_disk_dev(self.modules_img, + 'kernel', + rw=self.modules_img_rw) def other_dev_config(self): if self.modules_img is not None: - return BlockDevice(self.modules_img, 'kernel', rw=False) + return self.modules_dev_config() elif self.drive is not None: (drive_type, drive_domain, drive_path) = self.drive.split(":") if drive_type == 'hd': @@ -160,7 +177,7 @@ class Storage(object): def format_disk_dev(self, path, name, script=None, rw=True, devtype='disk', domain=None): - raise NotImplementedError() + return BlockDevice(path, name, script, rw, domain, devtype) @property def kernels_dir(self): @@ -175,7 +192,6 @@ class Storage(object): else os.path.join(self.vm.dir_path, qubes.config.vm_files['kernels_subdir']) - @property def modules_img(self): '''Path to image with modules. @@ -198,29 +214,6 @@ class Storage(object): return self.vm.kernel is None - def abspath(self, path, rel=None): - '''Make absolute path. - - If given path is relative, it is interpreted as relative to - :py:attr:`self.vm.dir_path` or given *rel*. - ''' - return path if os.path.isabs(path) \ - else os.path.join(rel or self.vm.dir_path, path) - - - @staticmethod - def _copy_file(source, destination): - '''Effective file copy, preserving sparse files etc. - ''' - # TODO: Windows support - - # We prefer to use Linux's cp, because it nicely handles sparse files - try: - subprocess.check_call(['cp', '--reflink=auto', source, destination]) - except subprocess.CalledProcessError: - raise IOError('Error while copying {!r} to {!r}'.format( - source, destination)) - def get_disk_utilization(self): return get_disk_usage(self.vm.dir_path) @@ -237,12 +230,6 @@ class Storage(object): def resize_private_img(self, size): raise NotImplementedError() - def create_on_disk_private_img(self, source_template=None): - raise NotImplementedError() - - def create_on_disk_root_img(self, source_template=None): - raise NotImplementedError() - def create_on_disk(self, source_template=None): if source_template is None and hasattr(self.vm, 'template'): source_template = self.vm.template @@ -250,10 +237,11 @@ class Storage(object): old_umask = os.umask(002) self.vm.log.info('Creating directory: {0}'.format(self.vm.dir_path)) - os.mkdir(self.vm.dir_path) - self.create_on_disk_private_img(source_template) - self.create_on_disk_root_img(source_template) - self.reset_volatile_storage() + os.makedirs(self.vm.dir_path) + pool = self.get_pool() + pool.create_on_disk_private_img(source_template) + pool.create_on_disk_root_img(source_template) + pool.reset_volatile_storage() os.umask(old_umask) @@ -319,36 +307,23 @@ class Storage(object): shutil.rmtree(self.vm.dir_path) - def reset_volatile_storage(self): - # Re-create only for template based VMs - try: - if self.vm.template is not None and self.volatile_img: - if os.path.exists(self.volatile_img): - os.remove(self.volatile_img) - except AttributeError: # self.vm.template - pass - - # For StandaloneVM create it only if not already exists - # (eg after backup-restore) - if hasattr(self, 'volatile_img') \ - and not os.path.exists(self.volatile_img): - self.vm.log.info( - 'Creating volatile image: {0}'.format(self.volatile_img)) - subprocess.check_call( - [qubes.config.system_path["prepare_volatile_img_cmd"], - self.volatile_img, - str(self.root_img_size / 1024 / 1024)]) - def prepare_for_vm_startup(self): - self.reset_volatile_storage() + pool = get_pool(self.vm.pool_name, self.vm) + pool.reset_volatile_storage() if hasattr(self.vm, 'private_img') \ and not os.path.exists(self.private_img): self.vm.log.info('Creating empty VM private image file: {0}'.format( - self.private_img)) - self.create_on_disk_private_img() + pool.private_img)) + pool.create_on_disk_private_img() + def get_pool(self): + return get_pool(self.vm.pool_name, self.vm) + + def commit_template_changes(self): + pool = self.get_pool() + pool.commit_template_changes() def get_disk_usage_one(st): '''Extract disk usage of one inode from its stat_result struct. @@ -391,56 +366,29 @@ def get_disk_usage(path): return ret class Pool(object): - def __init__(self, vm, dir_path, name): - assert vm is not None - assert dir_path is not None - assert name + private_img_size = qubes.config.defaults['private_img_size'] + root_img_size = qubes.config.defaults['root_img_size'] + def __init__(self, vm=None, name=None, **kwargs): + assert vm + assert name, "Pool name is missing" self.vm = vm self.name = name - self.dir_path = os.path.normpath(dir_path) - self.create_dir_if_not_exists(self.dir_path) + def root_dev_config(self): + raise NotImplementedError() - self.vmdir = self.vmdir_path(vm, self.dir_path) + def private_dev_config(self): + raise NotImplementedError() - appvms_path = os.path.join(self.dir_path, 'appvms') - self.create_dir_if_not_exists(appvms_path) + def volatile_dev_config(self): + raise NotImplementedError() - servicevms_path = os.path.join(self.dir_path, 'servicevms') - self.create_dir_if_not_exists(servicevms_path) + def create_on_disk_private_img(self, source_template=None): + raise NotImplementedError() - vm_templates_path = os.path.join(self.dir_path, 'vm-templates') - self.create_dir_if_not_exists(vm_templates_path) - - # XXX there is also a class attribute on the domain classes which does - # exactly that -- which one should prevail? - def vmdir_path(self, vm, pool_dir): - """ Returns the path to vmdir depending on the type of the VM. - - The default QubesOS file storage saves the vm images in three - different directories depending on the ``QubesVM`` type: - - * ``appvms`` for ``QubesAppVm`` or ``QubesHvm`` - * ``vm-templates`` for ``QubesTemplateVm`` or ``QubesTemplateHvm`` - - Args: - vm: a QubesVM - pool_dir: the root directory of the pool - - Returns: - string (str) absolute path to the directory where the vm files - are stored - """ - if vm.is_template(): - subdir = 'vm-templates' - elif vm.is_disposablevm(): - subdir = 'appvms' - return os.path.join(pool_dir, subdir, vm.template.name + '-dvm') - else: - subdir = 'appvms' - - return os.path.join(pool_dir, subdir, vm.name) + def create_on_disk_root_img(self, source_template=None): + raise NotImplementedError() def create_dir_if_not_exists(self, path): """ Check if a directory exists in if not create it. @@ -450,6 +398,48 @@ class Pool(object): if not os.path.exists(path): os.mkdir(path) + def commit_template_changes(self): + raise NotImplementedError() + @staticmethod + def _copy_file(source, destination): + '''Effective file copy, preserving sparse files etc. + ''' + # TODO: Windows support + + # We prefer to use Linux's cp, because it nicely handles sparse files + try: + subprocess.check_call(['cp', '--reflink=auto', source, destination + ]) + except subprocess.CalledProcessError: + raise IOError('Error while copying {!r} to {!r}'.format( + source, destination)) + + def format_disk_dev(self, path, vdev, script=None, rw=True, devtype='disk', + domain=None): + + return BlockDevice(path, vdev, script, rw, domain, devtype) + def reset_volatile_storage(self): + # Re-create only for template based VMs + try: + if self.vm.template is not None and self.volatile_img: + if os.path.exists(self.volatile_img): + os.remove(self.volatile_img) + except AttributeError: # self.vm.template + pass + + # For StandaloneVM create it only if not already exists + # (eg after backup-restore) + if hasattr(self, 'volatile_img') \ + and not os.path.exists(self.volatile_img): + self.vm.log.info( + 'Creating volatile image: {0}'.format(self.volatile_img)) + subprocess.check_call( + [qubes.config.system_path["prepare_volatile_img_cmd"], + self.volatile_img, + str(self.root_img_size / 1024 / 1024)]) + + def init_volume(self, volume_config): + raise NotImplementedError() def pool_drivers(): """ Return a list of EntryPoints names """ diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 9aef6f71..c6203121 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -482,8 +482,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): self.uuid = uuid.uuid4() # Initialize VM image storage class - self.storage = qubes.storage.get_pool( - self.pool_name, self).get_storage() + self.storage = qubes.storage.Storage(self) @qubes.events.handler('property-set:label') From 4d4b846ce87a8185ed6564875a3208bb591a9e43 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 15 Apr 2016 20:15:18 +0200 Subject: [PATCH 14/44] Replace XenStorage with XenPool --- qubes/storage/xen.py | 125 +++++++++++++++++++++++-------------------- 1 file changed, 68 insertions(+), 57 deletions(-) diff --git a/qubes/storage/xen.py b/qubes/storage/xen.py index 097bc705..49a03be9 100644 --- a/qubes/storage/xen.py +++ b/qubes/storage/xen.py @@ -33,41 +33,32 @@ import subprocess import qubes import qubes.config -import qubes.storage import qubes.vm.templatevm -from qubes.devices import BlockDevice +from qubes.storage import Pool, StoragePoolException, Volume -class XenStorage(qubes.storage.Storage): - '''Class for VM storage of Xen VMs. - ''' +class XenPool(Pool): root_dev = 'xvda' private_dev = 'xvdb' volatile_dev = 'xvdc' - modules_dev = 'xvdd' - def __init__(self, vm, vmdir, **kwargs): - """ Instantiate the storage. - - Args: - vm: a QubesVM - vmdir: the root directory of the pool - """ - assert vm is not None - assert vmdir is not None - - super(XenStorage, self).__init__(vm, **kwargs) - - self.vmdir = vmdir + def __init__(self, vm=None, name=None, dir_path=None): + super(XenPool, self).__init__(vm=vm, name=name) + assert dir_path, "No pool dir_path specified" + self.dir_path = os.path.normpath(dir_path) + self.create_dir_if_not_exists(self.dir_path) + appvms_path = os.path.join(self.dir_path, 'appvms') + self.create_dir_if_not_exists(appvms_path) + vm_templates_path = os.path.join(self.dir_path, 'vm-templates') + self.create_dir_if_not_exists(vm_templates_path) @property def private_img(self): '''Path to the private image''' return self.abspath(qubes.config.vm_files['private_img']) - @property def root_img(self): '''Path to the root image''' @@ -75,7 +66,6 @@ class XenStorage(qubes.storage.Storage): if hasattr(self.vm, 'template') and self.vm.template \ else self.abspath(qubes.config.vm_files['root_img']) - @property def rootcow_img(self): '''Path to the root COW image''' @@ -85,17 +75,11 @@ class XenStorage(qubes.storage.Storage): return None - @property def volatile_img(self): '''Path to the volatile image''' return self.abspath(qubes.config.vm_files['volatile_img']) - def format_disk_dev(self, path, name, script=None, rw=True, devtype='disk', - domain=None): - - return BlockDevice(path, name, script, rw, domain, devtype) - def root_dev_config(self): dev_name = 'root' if isinstance(self.vm, qubes.vm.templatevm.TemplateVM): @@ -123,27 +107,27 @@ class XenStorage(qubes.storage.Storage): # any other template-based VM - two device-mapper layers: one # in dom0 (here) from root+root-cow, and another one from # this+volatile.img - return self.format_disk_dev( - '{root}:{template_rootcow}'.format( - root=self.root_img, - template_rootcow=self.vm.template.storage.rootcow_img), - self.root_dev, - script='block-snapshot', - rw=False) + path = '{root}:{template_rootcow}'.format( + root=self.root_img, + template_rootcow=self.vm.template.storage.rootcow_img) + return self.format_disk_dev(path=path, + vdev=self.root_dev, + script='block-snapshot', + rw=False) else: # standalone qube return self.format_disk_dev(self.root_img, dev_name) - def private_dev_config(self): return self.format_disk_dev(self.private_img, 'private') def volatile_dev_config(self): return self.format_disk_dev(self.volatile_img, 'volatile') - def create_on_disk_private_img(self, source_template=None): + if not os.path.exists(self.target_dir): + os.makedirs(self.target_dir) if source_template is None: f_private = open(self.private_img, 'a+b') f_private.truncate(self.private_img_size) @@ -152,11 +136,11 @@ class XenStorage(qubes.storage.Storage): else: self.vm.log.info("Copying the template's private image: {}".format( source_template.storage.private_img)) - self._copy_file(source_template.storage.private_img, - self.private_img) - + self._copy_file(source_template.storage.private_img, self.private_img) def create_on_disk_root_img(self, source_template=None): + if not os.path.exists(self.target_dir): + os.makedirs(self.target_dir) if source_template is None: fd = open(self.root_img, 'a+b') fd.truncate(self.root_img_size) @@ -164,11 +148,11 @@ class XenStorage(qubes.storage.Storage): elif self.vm.updateable: # if not updateable, just use template's disk - self.vm.log.info("--> Copying the template's root image: {}".format( - source_template.storage.root_img)) + self.vm.log.info( + "--> Copying the template's root image: {}".format( + source_template.storage.root_img)) self._copy_file(source_template.storage.root_img, self.root_img) - def resize_private_img(self, size): fd = open(self.private_img, 'a+b') fd.truncate(size) @@ -185,9 +169,8 @@ class XenStorage(qubes.storage.Storage): loop_dev = m.group(1) # resize loop device - subprocess.check_call( - ['sudo', 'losetup', '--set-capacity', loop_dev]) - + subprocess.check_call(['sudo', 'losetup', '--set-capacity', + loop_dev]) def commit_template_changes(self): assert isinstance(self.vm, qubes.vm.templatevm.TemplateVM) @@ -206,7 +189,6 @@ class XenStorage(qubes.storage.Storage): f_root.close() os.umask(old_umask) - def reset_volatile_storage(self): try: # no template set, in any way (Standalone VM, Template VM) @@ -241,15 +223,14 @@ class XenStorage(qubes.storage.Storage): f_volatile.truncate(f_root.tell()) f_volatile.close() f_root.close() - return # XXX why is that? super() does not run - except AttributeError: # self.vm.template + return # XXX why is that? super() does not run + except AttributeError: # self.vm.template pass - super(XenStorage, self).reset_volatile_storage() - + super(XenPool, self).reset_volatile_storage() def prepare_for_vm_startup(self): - super(XenStorage, self).prepare_for_vm_startup() + super(XenPool, self).prepare_for_vm_startup() if self.drive is not None: # pylint: disable=unused-variable @@ -260,14 +241,44 @@ class XenStorage(qubes.storage.Storage): drive_vm = self.vm.app.domains[drive_domain] if not drive_vm.is_running(): - raise qubes.exc.QubesVMNotRunningError(drive_vm, - 'VM {!r} holding {!r} isn\'t running'.format( + raise qubes.exc.QubesVMNotRunningError( + drive_vm, 'VM {!r} holding {!r} isn\'t running'.format( drive_domain, drive_path)) if self.rootcow_img and not os.path.exists(self.rootcow_img): self.commit_template_changes() -class XenPool(qubes.storage.Pool): - def get_storage(self): - """ Returns an instantiated ``XenStorage``. """ - return XenStorage(self.vm, vmdir=self.vmdir) + # XXX there is also a class attribute on the domain classes which does + # exactly that -- which one should prevail? + @property + def target_dir(self): + """ Returns the path to vmdir depending on the type of the VM. + + The default QubesOS file storage saves the vm images in three + different directories depending on the ``QubesVM`` type: + + * ``appvms`` for ``QubesAppVm`` or ``QubesHvm`` + * ``vm-templates`` for ``QubesTemplateVm`` or ``QubesTemplateHvm`` + + Args: + vm: a QubesVM + pool_dir: the root directory of the pool + + Returns: + string (str) absolute path to the directory where the vm files + are stored + """ + vm = self.vm + if vm.is_template(): + subdir = 'vm-templates' + elif vm.is_disposablevm(): + subdir = 'appvms' + return os.path.join(self.dir_path, subdir, + vm.template.name + '-dvm') + else: + subdir = 'appvms' + + return os.path.join(self.dir_path, subdir, vm.name) + + def abspath(self, file_name): + return os.path.join(self.target_dir, file_name) From f02f9e3a412a2270e3a9928baadf3e73dc85fc4b Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Thu, 14 Apr 2016 19:00:52 +0200 Subject: [PATCH 15/44] Add XenPool init_volume --- qubes/storage/xen.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/qubes/storage/xen.py b/qubes/storage/xen.py index 49a03be9..303cfa33 100644 --- a/qubes/storage/xen.py +++ b/qubes/storage/xen.py @@ -282,3 +282,22 @@ class XenPool(Pool): def abspath(self, file_name): return os.path.join(self.target_dir, file_name) + + def init_volume(self, volume_config): + assert 'volume_type' in volume_config, "Volume type missing " \ + + str(volume_config) + target_dir = self.target_dir + assert target_dir, "Pool target_dir not set" + volume_type = volume_config['volume_type'] + volume_config['target_dir'] = target_dir + known_types = { + 'read-write': ReadWriteFile, + 'read-only': ReadOnlyFile, + 'origin': OriginFile, + 'snapshot': SnapshotFile, + 'volatile': VolatileFile, + } + if volume_type not in known_types: + raise StoragePoolException("Unknown volume type " + volume_type) + return known_types[volume_type](**volume_config) + From 792d94959f10c81d2b8654d757df6a98d2c8d549 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 15 Apr 2016 19:59:40 +0200 Subject: [PATCH 16/44] Add implementations of xen volumes --- qubes/storage/xen.py | 125 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/qubes/storage/xen.py b/qubes/storage/xen.py index 303cfa33..bf3e4ad6 100644 --- a/qubes/storage/xen.py +++ b/qubes/storage/xen.py @@ -301,3 +301,128 @@ class XenPool(Pool): raise StoragePoolException("Unknown volume type " + volume_type) return known_types[volume_type](**volume_config) + +class SizeMixIn(Volume): + + def __init__(self, name=None, pool=None, vid=None, target_dir=None, size=0, + **kwargs): + assert size > 0, 'Size for volume ' + name + ' is <=0' + super(SizeMixIn, self).__init__(name=name, + pool=pool, + vid=vid, + **kwargs) + self._size = size + self.target_dir = target_dir + + @property + def size(self): + if self.vid and os.path.exists(self.vid): + return qubes.storage.get_disk_usage(self.vid) + else: + return self._size + + +class ReadWriteFile(SizeMixIn): + # :pylint: disable=too-few-public-methods + def __init__(self, **kwargs): + super(ReadWriteFile, self).__init__(**kwargs) + self.path = os.path.join(self.target_dir, self.name + '.img') + self.vid = self.path + + @property + def size(self): + if self.vid and os.path.exists(self.vid): + return qubes.storage.get_disk_usage(self.vid) + else: + return self._size + + def create(self): + create_file(self.path, self.size) + + @property + def created(self): + return os.path.exists(self.path) + + +class ReadOnlyFile(Volume): + + def __init__(self, name=None, pool=None, vid=None, target_dir=None, + **kwargs): + assert os.path.exists(vid), "read-only volume missing vid" + super(ReadOnlyFile, self).__init__(name=name, + pool=pool, + vid=vid, + **kwargs) + self.path = self.vid + + @property + def size(self): + return qubes.storage.get_disk_usage(self.vid) + + +class OriginFile(SizeMixIn): + script = 'block-origin' + + def __init__(self, **kwargs): + super(OriginFile, self).__init__(**kwargs) + self.path = os.path.join(self.target_dir, self.name + '.img') + self.path_cow = os.path.join(self.target_dir, self.name + '-cow.img') + self.vid = self.path + + def create(self): + create_file(self.path, self.size) + create_file(self.path_cow, self.size) + + def commit(self): + raise NotImplementedError + + @property + def size(self): + if self.vid and os.path.exists(self.vid): + return qubes.storage.get_disk_usage(self.vid) + else: + return self._size + + @property + def created(self): + return os.path.exists(self.path) and os.path.exists(self.path_cow) + + +class SnapshotFile(Volume): + # :pylint: disable=too-few-public-methods + script = 'block-snapshot' + rw = False + + def __init__(self, name=None, pool=None, vid=None, target_dir=None, + **kwargs): + assert vid, "SnapshotVolume missing a vid to OriginVolume" + assert os.path.exists(vid), "OriginVolume does not exist" + super(SnapshotFile, self).__init__(name=name, + pool=pool, + vid=vid, + **kwargs) + self.path = os.path.join(target_dir, name + '.img') + self.path_cow = os.path.join(target_dir, name + '-cow.img') + + @property + def created(self): + return os.path.exists(self.path) and os.path.exists(self.path_cow) + + @property + def size(self): + return qubes.storage.get_disk_usage(self.vid) + + +class VolatileFile(SizeMixIn): + + def __init__(self, **kwargs): + super(VolatileFile, self).__init__(**kwargs) + self.path = os.path.join(self.target_dir, 'volatile.img') + self.vid = self.path + + +def create_file(path, size): + if os.path.exists(path): + raise IOError("Volume %s already exists", path) + with open(path, 'a+b') as fh: + fh.truncate(size) From 3dab5193c695633daac3d34fba7cde1906c2d902 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 15 Apr 2016 20:40:48 +0200 Subject: [PATCH 17/44] XenPool add snapshot handling --- qubes/storage/xen.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/qubes/storage/xen.py b/qubes/storage/xen.py index bf3e4ad6..144082a3 100644 --- a/qubes/storage/xen.py +++ b/qubes/storage/xen.py @@ -299,6 +299,11 @@ class XenPool(Pool): } if volume_type not in known_types: raise StoragePoolException("Unknown volume type " + volume_type) + + if volume_type == 'snapshot': + path = qubes.storage.get_pool(volume_config['pool'], self.vm.template).target_dir + volume_config['vid'] = os.path.join(path, volume_config['name'] + '.img') + return known_types[volume_type](**volume_config) From 88238c80f3875dbac1acfae338e025e2f52259e7 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 1 Apr 2016 13:44:29 +0200 Subject: [PATCH 18/44] Add XenPool._reset_volume --- qubes/storage/xen.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/qubes/storage/xen.py b/qubes/storage/xen.py index 144082a3..dc1bced6 100644 --- a/qubes/storage/xen.py +++ b/qubes/storage/xen.py @@ -188,6 +188,26 @@ class XenPool(Pool): f_cow.close() f_root.close() os.umask(old_umask) + return volume + + def start(self, volume): + if volume.volume_type == 'volatile': + self._reset_volume(volume) + return volume + + def _reset_volume(self, volume): + ''' Remove and recreate a volatile volume ''' + assert volume.volume_type == 'volatile', "Not a volatile volume" + + size = self.vm.volume_config[volume.name]['size'] + assert size + + if os.path.exists(volume.path): + os.remove(volume.path) + + with open(volume.path, "w") as f_volatile: + f_volatile.truncate(volume.size) + return volume def reset_volatile_storage(self): try: From 79ac3d3770acee5b6c60341833c1b3e5a2a36326 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 15 Apr 2016 20:10:53 +0200 Subject: [PATCH 19/44] Fix storage test and simplify TestVM --- qubes/tests/storage.py | 68 +++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/qubes/tests/storage.py b/qubes/tests/storage.py index 8cb9f9fc..7e777bdf 100644 --- a/qubes/tests/storage.py +++ b/qubes/tests/storage.py @@ -17,9 +17,12 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import qubes.log -from qubes.storage import StoragePoolException, pool_drivers +from qubes.exc import QubesException +from qubes.storage import pool_drivers from qubes.storage.xen import XenPool -from qubes.tests import QubesTestCase +from qubes.tests import QubesTestCase, SystemTestsMixin + +# :pylint: disable=invalid-name class TestApp(qubes.tests.TestEmitter): @@ -27,30 +30,26 @@ class TestApp(qubes.tests.TestEmitter): class TestVM(object): - def __init__(self, app, qid, name, pool_name, template=None): - super(TestVM, self).__init__() - self.app = app - self.qid = qid - self.name = name - self.pool_name = pool_name - self.template = template - self.hvm = False - self.storage = qubes.storage.get_pool(self.pool_name, - self).get_storage() + def __init__(self, test, template=None): + self.app = test.app + self.name = test.make_vm_name('appvm') self.log = qubes.log.get_vm_logger(self.name) + if template: + self.template = template + def is_template(self): + # :pylint: disable=no-self-use return False def is_disposablevm(self): + # :pylint: disable=no-self-use return False - @property - def dir_path(self): - return self.storage.vmdir - class TestTemplateVM(TestVM): + dir_path_prefix = qubes.config.system_path['qubes_templates_dir'] + def is_template(self): return True @@ -60,46 +59,47 @@ class TestDisposableVM(TestVM): return True -class TC_00_Pool(QubesTestCase): +class TC_00_Pool(SystemTestsMixin, QubesTestCase): """ This class tests the utility methods from :mod:``qubes.storage`` """ def setUp(self): super(TC_00_Pool, self).setUp() + self.init_default_template() def test_000_unknown_pool_driver(self): # :pylint: disable=protected-access """ Expect an exception when unknown pool is requested""" - with self.assertRaises(StoragePoolException): - qubes.storage._get_pool_klass('foo-bar') + with self.assertRaises(QubesException): + self.app.get_pool('foo-bar') def test_001_all_pool_drivers(self): - """ The only predefined pool driver is file """ + """ The only predefined pool driver is xen """ self.assertEquals(["xen"], pool_drivers()) def test_002_get_pool_klass(self): """ Expect the default pool to be `XenPool` """ # :pylint: disable=protected-access - result = qubes.storage._get_pool_klass('default') - self.assertTrue(result is XenPool) + result = self.app.get_pool('default') + self.assertIsInstance(result, XenPool) def test_003_pool_exists_default(self): """ Expect the default pool to exists """ - self.assertTrue(qubes.storage.pool_exists('default')) + self.assertPoolExists('default') - def test_004_pool_exists_random(self): - """ Expect this pool to not a exist """ - self.assertFalse(qubes.storage.pool_exists( - 'asdh312096r832598213iudhas')) - - def test_005_add_remove_pool(self): + def test_004_add_remove_pool(self): """ Tries to adding and removing a pool. """ pool_name = 'asdjhrp89132' # make sure it's really does not exist - qubes.storage.remove_pool(pool_name) + self.app.remove_pool(pool_name) + self.assertFalse(self.assertPoolExists(pool_name)) - qubes.storage.add_pool(pool_name, driver='xen') - self.assertTrue(qubes.storage.pool_exists(pool_name)) + self.app.add_pool(name=pool_name, driver='xen', dir_path='/tmp/asdjhrp89132') + self.assertTrue(self.assertPoolExists(pool_name)) - qubes.storage.remove_pool(pool_name) - self.assertFalse(qubes.storage.pool_exists(pool_name)) + self.app.remove_pool(pool_name) + self.assertFalse(self.assertPoolExists(pool_name)) + + def assertPoolExists(self, pool): + """ Check if specified pool exists """ + return pool in self.app.pools.keys() From 3c66d4b54ccf265b50e966484a351cf016992ac7 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 15 Apr 2016 20:11:02 +0200 Subject: [PATCH 20/44] Fix storage_xen test --- qubes/tests/storage_xen.py | 336 ++++++++++++++++++++++++------------- 1 file changed, 219 insertions(+), 117 deletions(-) diff --git a/qubes/tests/storage_xen.py b/qubes/tests/storage_xen.py index a8e0b1f7..0dd9557c 100644 --- a/qubes/tests/storage_xen.py +++ b/qubes/tests/storage_xen.py @@ -18,13 +18,21 @@ import os import shutil -import unittest + import qubes.storage import qubes.tests.storage +from qubes.config import defaults +from qubes.storage import Storage +from qubes.storage.xen import (OriginFile, ReadOnlyFile, ReadWriteFile, + SnapshotFile, VolatileFile) from qubes.tests import QubesTestCase, SystemTestsMixin -from qubes.storage.xen import XenStorage +from qubes.tests.storage import TestVM -class TC_00_XenPool(QubesTestCase): + +# :pylint: disable=invalid-name + + +class TC_00_XenPool(SystemTestsMixin, QubesTestCase): """ This class tests some properties of the 'default' pool. """ @@ -34,32 +42,166 @@ class TC_00_XenPool(QubesTestCase): .. sealso:: Data :data:``qubes.qubes.defaults['pool_config']``. """ - vm = self._init_app_vm() - result = qubes.storage.get_pool("default", vm).dir_path + result = self.app.get_pool("default").dir_path expected = '/var/lib/qubes' self.assertEquals(result, expected) def test001_default_storage_class(self): - """ Check when using default pool the Storage is ``XenStorage``. """ + """ Check when using default pool the Storage is ``Storage``. """ result = self._init_app_vm().storage - self.assertIsInstance(result, XenStorage) - - def test_002_default_pool_name(self): - """ Default pool_name is 'default'. """ - vm = self._init_app_vm() - self.assertEquals(vm.pool_name, "default") + self.assertIsInstance(result, Storage) def _init_app_vm(self): """ Return initalised, but not created, AppVm. """ - app = qubes.tests.storage.TestApp() vmname = self.make_vm_name('appvm') - template = qubes.tests.storage.TestTemplateVM(app, 1, - self.make_vm_name('template'), 'default') - return qubes.tests.storage.TestVM(app, qid=2, name=vmname, - template=template, pool_name='default') + self.init_default_template() + return self.app.add_new_vm(qubes.vm.appvm.AppVM, + name=vmname, + template=self.app.default_template, + label='red') + + +class TC_01_XenVolumes(SystemTestsMixin, QubesTestCase): + POOL_DIR = '/var/lib/qubes/test-pool' + POOL_NAME = 'test-pool' + POOL_CONF = {'driver': 'xen', 'dir_path': POOL_DIR, 'name': POOL_NAME} + + def setUp(self): + """ Add a test file based storage pool """ + super(TC_01_XenVolumes, self).setUp() + self.init_default_template() + self.app.add_pool(**self.POOL_CONF) + + def tearDown(self): + """ Remove the file based storage pool after testing """ + self.app.remove_pool("test-pool") + super(TC_01_XenVolumes, self).tearDown() + shutil.rmtree(self.POOL_DIR, ignore_errors=True) + + def test_000_origin_volume(self): + config = { + 'name': 'root', + 'pool': self.POOL_NAME, + 'volume_type': 'origin', + 'size': defaults['root_img_size'], + } + vm = TestVM(self) + result = self.app.get_pool(self.POOL_NAME).init_volume(vm, config) + self.assertIsInstance(result, OriginFile) + self.assertEqual(result.name, 'root') + self.assertEqual(result.pool, self.POOL_NAME) + self.assertEqual(result.size, defaults['root_img_size']) + + def test_001_snapshot_volume(self): + original_path = '/var/lib/qubes/vm-templates/fedora-23/root.img' + original_size = qubes.config.defaults['root_img_size'] + config = { + 'name': 'root', + 'pool': 'default', + 'volume_type': 'snapshot', + 'vid': original_path, + } + vm = TestVM(self, template=self.app.default_template) + result = self.app.get_pool(self.POOL_NAME).init_volume(vm, config) + self.assertIsInstance(result, SnapshotFile) + self.assertEqual(result.name, 'root') + self.assertEqual(result.pool, 'default') + self.assertEqual(result.size, original_size) + + def test_002_read_write_volume(self): + config = { + 'name': 'root', + 'pool': self.POOL_NAME, + 'volume_type': 'read-write', + 'size': defaults['root_img_size'], + } + vm = TestVM(self) + result = self.app.get_pool(self.POOL_NAME).init_volume(vm, config) + self.assertIsInstance(result, ReadWriteFile) + self.assertEqual(result.name, 'root') + self.assertEqual(result.pool, self.POOL_NAME) + self.assertEqual(result.size, defaults['root_img_size']) + + def test_003_read_volume(self): + template = self.app.default_template + original_path = template.volumes['root'].vid + original_size = qubes.config.defaults['root_img_size'] + config = { + 'name': 'root', + 'pool': 'default', + 'volume_type': 'read-only', + 'vid': original_path + } + vm = TestVM(self, template=template) + + result = self.app.get_pool(self.POOL_NAME).init_volume(vm, config) + self.assertIsInstance(result, ReadOnlyFile) + self.assertEqual(result.name, 'root') + self.assertEqual(result.pool, 'default') + self.assertEqual(result.size, original_size) + + def test_004_volatile_volume(self): + config = { + 'name': 'root', + 'pool': self.POOL_NAME, + 'volume_type': 'volatile', + 'size': defaults['root_img_size'], + } + vm = TestVM(self) + result = self.app.get_pool(self.POOL_NAME).init_volume(vm, config) + self.assertIsInstance(result, VolatileFile) + self.assertEqual(result.name, 'root') + self.assertEqual(result.pool, self.POOL_NAME) + self.assertEqual(result.size, defaults['root_img_size']) + + def test_005_appvm_volumes(self): + ''' Check if AppVM volumes are propertly initialized ''' + vmname = self.make_vm_name('appvm') + vm = self.app.add_new_vm(qubes.vm.appvm.AppVM, + name=vmname, + template=self.app.default_template, + label='red') + + volumes = vm.volumes + self.assertIsInstance(volumes['root'], SnapshotFile) + self.assertIsInstance(volumes['private'], ReadWriteFile) + self.assertIsInstance(volumes['volatile'], VolatileFile) + expected = vm.template.dir_path + '/root.img:' + vm.template.dir_path \ + + '/root-cow.img' + self.assertVolumePath(vm, 'root', expected, rw=False) + expected = vm.dir_path + '/private.img' + self.assertVolumePath(vm, 'private', expected, rw=True) + expected = vm.dir_path + '/volatile.img' + self.assertVolumePath(vm, 'volatile', expected, rw=True) + + def test_006_template_volumes(self): + ''' Check if TemplateVM volumes are propertly initialized ''' + vmname = self.make_vm_name('appvm') + vm = self.app.add_new_vm(qubes.vm.templatevm.TemplateVM, + name=vmname, + label='red') + + volumes = vm.volumes + self.assertIsInstance(volumes['root'], OriginFile) + self.assertIsInstance(volumes['private'], ReadWriteFile) + self.assertIsInstance(volumes['volatile'], VolatileFile) + expected = vm.dir_path + '/root.img:' + vm.dir_path + '/root-cow.img' + self.assertVolumePath(vm, 'root', expected, rw=True) + expected = vm.dir_path + '/private.img' + self.assertVolumePath(vm, 'private', expected, rw=True) + expected = vm.dir_path + '/volatile.img' + self.assertVolumePath(vm, 'volatile', expected, rw=True) + + def assertVolumePath(self, vm, dev_name, expected, rw=True): + # :pylint: disable=invalid-name + volumes = vm.volumes + b_dev = volumes[dev_name].block_device() + self.assertEqual(b_dev.rw, rw) + self.assertEquals(b_dev.path, expected) + @qubes.tests.skipUnlessDom0 -class TC_01_XenPool(QubesTestCase): +class TC_03_XenPool(SystemTestsMixin, QubesTestCase): """ Test the paths for the default Xen file based storage (``XenStorage``). """ @@ -68,125 +210,69 @@ class TC_01_XenPool(QubesTestCase): APPVMS_DIR = '/var/lib/qubes/test-pool/appvms' TEMPLATES_DIR = '/var/lib/qubes/test-pool/vm-templates' SERVICE_DIR = '/var/lib/qubes/test-pool/servicevms' + POOL_NAME = 'test-pool' + POOL_CONFIG = {'driver': 'xen', 'dir_path': POOL_DIR, 'name': POOL_NAME} def setUp(self): """ Add a test file based storage pool """ - super(TC_01_XenPool, self).setUp() - qubes.storage.add_pool('test-pool', driver='xen', - dir_path=self.POOL_DIR) - self.app = qubes.tests.storage.TestApp() - self.template = qubes.tests.storage.TestTemplateVM(self.app, 1, - self.make_vm_name('template'), 'default') - + super(TC_03_XenPool, self).setUp() + self.init_default_template() + self.app.add_pool(**self.POOL_CONFIG) def tearDown(self): """ Remove the file based storage pool after testing """ - super(TC_01_XenPool, self).tearDown() - qubes.storage.remove_pool("test-pool") + self.app.remove_pool("test-pool") + super(TC_03_XenPool, self).tearDown() shutil.rmtree(self.POOL_DIR, ignore_errors=True) def test_001_pool_exists(self): """ Check if the storage pool was added to the storage pool config """ - self.assertTrue(qubes.storage.pool_exists('test-pool')) + self.assertIn('test-pool', self.app.pools.keys()) def test_002_pool_dir_create(self): """ Check if the storage pool dir and subdirs were created """ - # The dir should not exists before - self.assertFalse(os.path.exists(self.POOL_DIR)) + pool_name = 'foo' + pool_dir = '/tmp/foo' + appvms_dir = '/tmp/foo/appvms' + templates_dir = '/tmp/foo/vm-templates' - vmname = self.make_vm_name('appvm') - qubes.tests.storage.TestVM(self.app, qid=2, name=vmname, - template=self.template, pool_name='test-pool') + self.assertFalse(os.path.exists(pool_dir)) - self.assertTrue(os.path.exists(self.POOL_DIR)) - self.assertTrue(os.path.exists(self.APPVMS_DIR)) - self.assertTrue(os.path.exists(self.SERVICE_DIR)) - self.assertTrue(os.path.exists(self.TEMPLATES_DIR)) + self.app.add_pool(name=pool_name, dir_path=pool_dir, driver='xen') - def test_003_pool_dir(self): - """ Check if the vm storage pool_dir is the same as specified """ - vmname = self.make_vm_name('appvm') - vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname, - template=self.template, pool_name='test-pool') - result = qubes.storage.get_pool('test-pool', vm).dir_path - self.assertEquals(self.POOL_DIR, result) + self.assertTrue(os.path.exists(pool_dir)) + self.assertTrue(os.path.exists(appvms_dir)) + self.assertTrue(os.path.exists(templates_dir)) - def test_004_app_vmdir(self): - """ Check the vm storage dir for an AppVm""" - vmname = self.make_vm_name('appvm') - vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname, - template=self.template, pool_name='test-pool') - - expected = os.path.join(self.APPVMS_DIR, vm.name) - result = vm.storage.vmdir - self.assertEquals(expected, result) - - def test_008_admin_vmdir(self): - """ Check the vm storage dir for a AdminVm""" - # TODO How to test AdminVm? - pass - - def test_009_template_vmdir(self): - """ Check the vm storage dir for a TemplateVm""" - vmname = self.make_vm_name('templatevm') - vm = qubes.tests.storage.TestTemplateVM(self.app, qid=2, name=vmname, - pool_name='test-pool') - - expected = os.path.join(self.TEMPLATES_DIR, vm.name) - result = vm.storage.vmdir - self.assertEquals(expected, result) - - def test_010_template_hvm_vmdir(self): - """ Check the vm storage dir for a TemplateHVm""" - vmname = self.make_vm_name('templatehvm') - vm = qubes.tests.storage.TestTemplateVM(self.app, qid=2, name=vmname, - pool_name='test-pool') - - expected = os.path.join(self.TEMPLATES_DIR, vm.name) - result = vm.storage.vmdir - self.assertEquals(expected, result) + shutil.rmtree(pool_dir, ignore_errors=True) def test_011_appvm_file_images(self): """ Check if all the needed image files are created for an AppVm""" vmname = self.make_vm_name('appvm') - vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname, - pool_name='test-pool') + vm = self.app.add_new_vm(qubes.vm.appvm.AppVM, + name=vmname, + template=self.app.default_template, + volume_config={ + 'private': { + 'pool': 'test-pool' + }, + 'volatile': { + 'pool': 'test-pool' + } + }, + label='red') vm.storage.create_on_disk() expected_vmdir = os.path.join(self.APPVMS_DIR, vm.name) - self.assertEqualsAndExists(vm.storage.vmdir, expected_vmdir) expected_private_path = os.path.join(expected_vmdir, 'private.img') - self.assertEqualsAndExists(vm.storage.private_img, + self.assertEqualsAndExists(vm.volumes['private'].path, expected_private_path) expected_volatile_path = os.path.join(expected_vmdir, 'volatile.img') - self.assertEqualsAndExists(vm.storage.volatile_img, - expected_volatile_path) - - def test_012_hvm_file_images(self): - """ Check if all the needed image files are created for a HVm""" - - vmname = self.make_vm_name('hvm') - vm = qubes.tests.storage.TestVM(self.app, qid=2, name=vmname, - pool_name='test-pool') - vm.hvm = True - vm.storage.create_on_disk() - - expected_vmdir = os.path.join(self.APPVMS_DIR, vm.name) - self.assertEqualsAndExists(vm.storage.vmdir, expected_vmdir) - - expected_private_path = os.path.join(expected_vmdir, 'private.img') - self.assertEqualsAndExists(vm.storage.private_img, - expected_private_path) - - expected_root_path = os.path.join(expected_vmdir, 'root.img') - self.assertEqualsAndExists(vm.storage.root_img, expected_root_path) - - expected_volatile_path = os.path.join(expected_vmdir, 'volatile.img') - self.assertEqualsAndExists(vm.storage.volatile_img, + self.assertEqualsAndExists(vm.volumes['volatile'].path, expected_volatile_path) def test_013_template_file_images(self): @@ -194,38 +280,54 @@ class TC_01_XenPool(QubesTestCase): created propertly by the storage system """ vmname = self.make_vm_name('tmvm') - vm = self.app.add_new_vm(qubes.vm.templatevm.TemplateVM, name=vmname, - pool_name='test-pool', label='red') - vm.storage.create_on_disk() + vm = self.app.add_new_vm(qubes.vm.templatevm.TemplateVM, + name=vmname, + volume_config={ + 'root': { + 'pool': 'test-pool' + }, + 'private': { + 'pool': 'test-pool' + }, + 'volatile': { + 'pool': 'test-pool' + } + }, + label='red') + vm.create_on_disk() expected_vmdir = os.path.join(self.TEMPLATES_DIR, vm.name) - self.assertEqualsAndExists(vm.storage.vmdir, expected_vmdir) - expected_root_path = os.path.join(expected_vmdir, 'root.img') - self.assertEqualsAndExists(vm.storage.root_img, expected_root_path) + expected_root_origin_path = os.path.join(expected_vmdir, 'root.img') + expected_root_cow_path = os.path.join(expected_vmdir, 'root-cow.img') + expected_root_path = '%s:%s' % (expected_root_origin_path, + expected_root_cow_path) + self.assertEquals(vm.volumes['root'].path, expected_root_path) + self.assertExist(vm.volumes['root'].path_origin) expected_private_path = os.path.join(expected_vmdir, 'private.img') - self.assertEqualsAndExists(vm.storage.private_img, + self.assertEqualsAndExists(vm.volumes['private'].path, expected_private_path) expected_volatile_path = os.path.join(expected_vmdir, 'volatile.img') - self.assertEqualsAndExists(vm.storage.volatile_img, + self.assertEqualsAndExists(vm.volumes['volatile'].path, expected_volatile_path) vm.storage.commit_template_changes() expected_rootcow_path = os.path.join(expected_vmdir, 'root-cow.img') - self.assertEqualsAndExists(vm.storage.rootcow_img, + self.assertEqualsAndExists(vm.volumes['root'].path_cow, expected_rootcow_path) - def assertEqualsAndExists(self, result_path, expected_path): """ Check if the ``result_path``, matches ``expected_path`` and exists. See also: :meth:``assertExist`` """ + # :pylint: disable=invalid-name self.assertEquals(result_path, expected_path) self.assertExist(result_path) def assertExist(self, path): """ Assert that the given path exists. """ - self.assertTrue(os.path.exists(path)) + # :pylint: disable=invalid-name + self.assertTrue(os.path.exists(path), "Path %s does not exist" % path) From bdfb85ac1926ac987c471af817dcfe50157bce2c Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 15 Apr 2016 20:40:53 +0200 Subject: [PATCH 21/44] Refactor Storage, Pool and XenPool - Remove all *_dev_config methods - Checks if a storage image exists moved to XenPool - Storage.remove wraps Pool.remove() - Stop volumes on domain sutdown/kill - Warn when using deprecated methods --- qubes/storage/__init__.py | 368 ++++++++++------------------- qubes/storage/xen.py | 473 +++++++++++++++++--------------------- qubes/vm/qubesvm.py | 111 +++++---- qubes/vm/templatevm.py | 11 +- 4 files changed, 406 insertions(+), 557 deletions(-) diff --git a/qubes/storage/__init__.py b/qubes/storage/__init__.py index 3bfb4762..da257d69 100644 --- a/qubes/storage/__init__.py +++ b/qubes/storage/__init__.py @@ -39,7 +39,6 @@ import qubes.exc import qubes.utils from qubes.devices import BlockDevice -BLKSIZE = 512 STORAGE_ENTRY_POINT = 'qubes.storage' @@ -93,91 +92,20 @@ class Storage(object): in mind. ''' - modules_dev = 'xvdd' - def __init__(self, vm): #: Domain for which we manage storage self.vm = vm + self.log = self.vm.log #: Additional drive (currently used only by HVM) self.drive = None - for name, conf in self.vm.volume_config.items(): - assert 'pool' in conf - pool = get_pool(conf['pool'], self.vm) - self.vm.volumes[name] = pool.init_volume(conf) - - @property - def root_img(self): - pool = self.get_pool() - return pool.root_img - - @property - def private_img(self): - pool = self.get_pool() - return pool.private_img - - @property - def volatile_img(self): - pool = self.get_pool() - return pool.volatile_img - - @property - def rootcow_img(self): - pool = self.get_pool() - return pool.rootcow_img - - def get_config_params(self): - args = {} - args['rootdev'] = self.root_dev_config() - args['privatedev'] = self.private_dev_config() - args['volatiledev'] = self.volatile_dev_config() - args['otherdevs'] = self.other_dev_config() - - args['kerneldir'] = self.kernels_dir - - return args - - def root_dev_config(self): - pool = self.get_pool() - return pool.root_dev_config() - - def private_dev_config(self): - pool = self.get_pool() - return pool.private_dev_config() - - def volatile_dev_config(self): - pool = self.get_pool() - return pool.volatile_dev_config() - - def modules_dev_config(self): - return self.format_disk_dev(self.modules_img, - 'kernel', - rw=self.modules_img_rw) - - def other_dev_config(self): - if self.modules_img is not None: - return self.modules_dev_config() - elif self.drive is not None: - (drive_type, drive_domain, drive_path) = self.drive.split(":") - if drive_type == 'hd': - drive_type = 'disk' - - rw = (drive_type == 'disk') - - if drive_domain.lower() == "dom0": - drive_domain = None - - return self.format_disk_dev(drive_path, - 'other', - rw=rw, - devtype=drive_type, - domain=drive_domain) - - else: - return '' - - def format_disk_dev(self, path, name, script=None, rw=True, devtype='disk', - domain=None): - return BlockDevice(path, name, script, rw, domain, devtype) + self.pools = {} + if hasattr(vm, 'volume_config'): + for name, conf in self.vm.volume_config.items(): + assert 'pool' in conf, "Pool missing in volume_config" % str( + conf) + pool = self.vm.app.get_pool(conf['pool']) + self.vm.volumes[name] = pool.init_volume(self.vm, conf) + self.pools[name] = pool @property def kernels_dir(self): @@ -186,51 +114,33 @@ class Storage(object): If :py:attr:`self.vm.kernel` is :py:obj:`None`, the this points inside :py:attr:`self.vm.dir_path` ''' - return os.path.join(qubes.config.system_path['qubes_base_dir'], - qubes.config.system_path['qubes_kernels_base_dir'], self.vm.kernel)\ - if self.vm.kernel is not None \ - else os.path.join(self.vm.dir_path, - qubes.config.vm_files['kernels_subdir']) - - @property - def modules_img(self): - '''Path to image with modules. - - Depending on domain, this may be global or inside domain's dir. - ''' - - modules_path = os.path.join(self.kernels_dir, 'modules.img') - - if os.path.exists(modules_path): - return modules_path - else: - return None - - - @property - def modules_img_rw(self): - ''':py:obj:`True` if module image should be mounted RW, :py:obj:`False` - otherwise.''' - return self.vm.kernel is None - + assert 'kernel' in self.vm.volumes, "VM has no kernel pool" + return self.vm.volumes['kernel'].kernels_dir def get_disk_utilization(self): - return get_disk_usage(self.vm.dir_path) + ''' Returns summed up disk utilization for all domain volumes ''' + result = 0 + for volume in self.vm.volumes.values(): + result += volume.usage + return result + # TODO Remove this wrapper def get_disk_utilization_private_img(self): - # pylint: disable=invalid-name - return get_disk_usage(self.private_img) + # pylint: disable=invalid-name,missing-docstring + return self.vm.volume['private'].usage + # TODO Remove this wrapper def get_private_img_sz(self): - if not os.path.exists(self.private_img): - return 0 + # :pylint: disable=missing-docstring + return self.vm.volume['private'].size - return os.path.getsize(self.private_img) - - def resize_private_img(self, size): - raise NotImplementedError() + def resize(self, volume, size): + ''' Resize volume ''' + self.get_pool(volume).resize(volume, size) + # TODO rename it to create() def create_on_disk(self, source_template=None): + # :pylint: disable=missing-docstring if source_template is None and hasattr(self.vm, 'template'): source_template = self.vm.template @@ -238,14 +148,17 @@ class Storage(object): self.vm.log.info('Creating directory: {0}'.format(self.vm.dir_path)) os.makedirs(self.vm.dir_path) - pool = self.get_pool() - pool.create_on_disk_private_img(source_template) - pool.create_on_disk_root_img(source_template) - pool.reset_volatile_storage() + for name, volume in self.vm.volumes.items(): + source_volume = None + if source_template and hasattr(source_template, 'volumes'): + source_volume = source_template.volumes[name] + self.get_pool(volume).create(volume, source_volume=source_volume) os.umask(old_umask) + # TODO migrate this def clone_disk_files(self, src_vm): + # :pylint: disable=missing-docstring self.vm.log.info('Creating directory: {0}'.format(self.vm.dir_path)) os.mkdir(self.vm.dir_path) @@ -255,14 +168,15 @@ class Storage(object): self._copy_file(src_vm.private_img, self.vm.private_img) if src_vm.updateable and hasattr(src_vm, 'root_img'): - self.vm.log.info('Copying the root image: {} -> {}'.format( - src_vm.root_img, self.root_img)) - self._copy_file(src_vm.root_img, self.root_img) - - # TODO: modules? - # XXX which modules? -woju - + self.vm.log.info( + 'Copying the root image: {} -> {}'.format( + src_vm.volume['root'].path_origin, + self.vm.volume['root'].path_origin) + ) + self._copy_file(src_vm.volume['root'].path_origin, + self.vm.volume['root'].path_origin) + # TODO migrate this @staticmethod def rename(newpath, oldpath): '''Move storage directory, most likely during domain's rename. @@ -280,133 +194,87 @@ class Storage(object): os.rename(oldpath, newpath) - def verify_files(self): + '''Verify that the storage is sane. + + On success, returns normally. On failure, raises exception. + ''' if not os.path.exists(self.vm.dir_path): - raise qubes.exc.QubesVMError(self.vm, + raise qubes.exc.QubesVMError( + self.vm, 'VM directory does not exist: {}'.format(self.vm.dir_path)) - if hasattr(self.vm, 'root_img') and not os.path.exists(self.root_img): - raise qubes.exc.QubesVMError(self.vm, - 'VM root image file does not exist: {}'.format(self.root_img)) - - if hasattr(self.vm, 'private_img') \ - and not os.path.exists(self.private_img): - raise qubes.exc.QubesVMError(self.vm, - 'VM private image file does not exist: {}'.format( - self.private_img)) - - if self.modules_img is not None \ - and not os.path.exists(self.modules_img): - raise qubes.exc.QubesVMError(self.vm, - 'VM kernel modules image does not exists: {}'.format( - self.modules_img)) - - - def remove_from_disk(self): + def remove(self): + for name, volume in self.vm.volumes.items(): + self.log.info('Removing volume %s: %s' % (name, volume.vid)) + self.get_pool(volume).remove(volume) shutil.rmtree(self.vm.dir_path) + def start(self): + ''' Execute the start method on each pool ''' + for volume in self.vm.volumes.values(): + self.get_pool(volume).start(volume) + def stop(self): + ''' Execute the start method on each pool ''' + for volume in self.vm.volumes.values(): + self.get_pool(volume).stop(volume) - def prepare_for_vm_startup(self): - pool = get_pool(self.vm.pool_name, self.vm) - pool.reset_volatile_storage() - - if hasattr(self.vm, 'private_img') \ - and not os.path.exists(self.private_img): - self.vm.log.info('Creating empty VM private image file: {0}'.format( - pool.private_img)) - pool.create_on_disk_private_img() - - def get_pool(self): - return get_pool(self.vm.pool_name, self.vm) + def get_pool(self, volume): + ''' Helper function ''' + assert isinstance(volume, Volume), "You need to pass a Volume" + return self.pools[volume.name] def commit_template_changes(self): - pool = self.get_pool() - pool.commit_template_changes() + for volume in self.vm.volumes.values(): + if volume.volume_type == 'origin': + self.get_pool(volume).commit_template_changes(volume) -def get_disk_usage_one(st): - '''Extract disk usage of one inode from its stat_result struct. - - If known, get real disk usage, as written to device by filesystem, not - logical file size. Those values may be different for sparse files. - - :param os.stat_result st: stat result - :returns: disk usage - ''' - try: - return st.st_blocks * BLKSIZE - except AttributeError: - return st.st_size - - -def get_disk_usage(path): - '''Get real disk usage of given path (file or directory). - - When *path* points to directory, then it is evaluated recursively. - - This function tries estiate real disk usage. See documentation of - :py:func:`get_disk_usage_one`. - - :param str path: path to evaluate - :returns: disk usage - ''' - try: - st = os.lstat(path) - except OSError: - return 0 - - ret = get_disk_usage_one(st) - - # if path is not a directory, this is skipped - for dirpath, dirnames, filenames in os.walk(path): - for name in dirnames + filenames: - ret += get_disk_usage_one(os.lstat(os.path.join(dirpath, name))) - - return ret class Pool(object): + ''' A Pool is used to manage different kind of volumes (File + based/LVM/Btrfs/...). + + 3rd Parties providing own storage implementations will need to extend + this class. + ''' private_img_size = qubes.config.defaults['private_img_size'] root_img_size = qubes.config.defaults['root_img_size'] - def __init__(self, vm=None, name=None, **kwargs): - assert vm + def __init__(self, name=None, **kwargs): + # :pylint: disable=unused-argument assert name, "Pool name is missing" - self.vm = vm self.name = name - def root_dev_config(self): - raise NotImplementedError() + def create(self, volume, source_volume): + ''' Create the given volume on disk or copy from provided + `source_volume`. + ''' + raise NotImplementedError("Pool %s has create() not implemented" % + self.name) - def private_dev_config(self): - raise NotImplementedError() + def commit_template_changes(self, volume): + ''' Update origin device ''' + raise NotImplementedError( + "Pool %s has commit_template_changes() not implemented" % + self.name) - def volatile_dev_config(self): - raise NotImplementedError() + @property + def config(self): + ''' Returns the pool config to be written to qubes.xml ''' + raise NotImplementedError("Pool %s has config() not implemented" % + self.name) - def create_on_disk_private_img(self, source_template=None): - raise NotImplementedError() - - def create_on_disk_root_img(self, source_template=None): - raise NotImplementedError() - - def create_dir_if_not_exists(self, path): - """ Check if a directory exists in if not create it. - - This method does not create any parent directories. - """ - if not os.path.exists(path): - os.mkdir(path) - - def commit_template_changes(self): - raise NotImplementedError() @staticmethod def _copy_file(source, destination): '''Effective file copy, preserving sparse files etc. ''' # TODO: Windows support - # We prefer to use Linux's cp, because it nicely handles sparse files + assert os.path.exists(source), \ + "Missing the source %s to copy from" % source + assert not os.path.exists(destination), \ + "Destination %s already exists" % destination try: subprocess.check_call(['cp', '--reflink=auto', source, destination ]) @@ -414,34 +282,34 @@ class Pool(object): raise IOError('Error while copying {!r} to {!r}'.format( source, destination)) - def format_disk_dev(self, path, vdev, script=None, rw=True, devtype='disk', - domain=None): + def remove(self, volume): + ''' Remove volume''' + raise NotImplementedError("Pool %s has remove() not implemented" % + self.name) - return BlockDevice(path, vdev, script, rw, domain, devtype) - def reset_volatile_storage(self): - # Re-create only for template based VMs - try: - if self.vm.template is not None and self.volatile_img: - if os.path.exists(self.volatile_img): - os.remove(self.volatile_img) - except AttributeError: # self.vm.template - pass + def clone(self, source, target): + ''' Clone volume ''' + raise NotImplementedError("Pool %s has clone() not implemented" % + self.name) - # For StandaloneVM create it only if not already exists - # (eg after backup-restore) - if hasattr(self, 'volatile_img') \ - and not os.path.exists(self.volatile_img): - self.vm.log.info( - 'Creating volatile image: {0}'.format(self.volatile_img)) - subprocess.check_call( - [qubes.config.system_path["prepare_volatile_img_cmd"], - self.volatile_img, - str(self.root_img_size / 1024 / 1024)]) + def start(self, volume): + ''' Do what ever is needed on start ''' + raise NotImplementedError("Pool %s has start() not implemented" % + self.name) + + def stop(self, volume): + ''' Do what ever is needed on stop''' + raise NotImplementedError("Pool %s has stop() not implemented" % + self.name) def init_volume(self, volume_config): - raise NotImplementedError() + ''' Initialize a :py:class:`qubes.storage.Volume` from `volume_config`. + ''' + raise NotImplementedError("Pool %s has init_volume() not implemented" % + self.name) + def pool_drivers(): """ Return a list of EntryPoints names """ return [ep.name - for ep in pkg_resources.iter_entry_points(STORAGE_ENTRY_POINT)] + for ep in pkg_resources.iter_entry_points(STORAGE_ENTRY_POINT)] diff --git a/qubes/storage/xen.py b/qubes/storage/xen.py index dc1bced6..a50cd23f 100644 --- a/qubes/storage/xen.py +++ b/qubes/storage/xen.py @@ -31,137 +31,70 @@ import os.path import re import subprocess -import qubes -import qubes.config -import qubes.vm.templatevm from qubes.storage import Pool, StoragePoolException, Volume +BLKSIZE = 512 + class XenPool(Pool): + ''' File based 'original' disk implementation ''' - root_dev = 'xvda' - private_dev = 'xvdb' - volatile_dev = 'xvdc' - - def __init__(self, vm=None, name=None, dir_path=None): - super(XenPool, self).__init__(vm=vm, name=name) + def __init__(self, name=None, dir_path=None): + super(XenPool, self).__init__(name=name) assert dir_path, "No pool dir_path specified" self.dir_path = os.path.normpath(dir_path) - self.create_dir_if_not_exists(self.dir_path) + create_dir_if_not_exists(self.dir_path) appvms_path = os.path.join(self.dir_path, 'appvms') - self.create_dir_if_not_exists(appvms_path) + create_dir_if_not_exists(appvms_path) vm_templates_path = os.path.join(self.dir_path, 'vm-templates') - self.create_dir_if_not_exists(vm_templates_path) + create_dir_if_not_exists(vm_templates_path) - @property - def private_img(self): - '''Path to the private image''' - return self.abspath(qubes.config.vm_files['private_img']) + def create(self, volume, source_volume=None): + _type = volume.volume_type + size = volume.size + if _type == 'origin': + create_sparse_file(volume.path_origin, size) + create_sparse_file(volume.path_cow, size) + elif _type in ['read-write'] and source_volume: + copy_file(source_volume.path, volume.path) + elif _type in ['read-write', 'volatile']: + create_sparse_file(volume.path, size) - @property - def root_img(self): - '''Path to the root image''' - return self.vm.template.storage.root_img \ - if hasattr(self.vm, 'template') and self.vm.template \ - else self.abspath(qubes.config.vm_files['root_img']) + return volume - @property - def rootcow_img(self): - '''Path to the root COW image''' + def resize(self, volume, size): + ''' Expands volume, throws + :py:class:`qubst.storage.StoragePoolException` if given size is + less than current_size + ''' + _type = volume.volume_type + if _type not in ['origin', 'read-write', 'volatile']: + raise StoragePoolException('Can not resize a %s volume %s' % + (_type, volume.vid)) - if isinstance(self.vm, qubes.vm.templatevm.TemplateVM): - return self.abspath(qubes.config.vm_files['rootcow_img']) + if size <= volume.size: + raise StoragePoolException( + 'For your own safety, shrinking of %s is' + ' disabled. If you really know what you' + ' are doing, use `truncate` on %s manually.' % + (volume.name, volume.vid)) - return None + if _type == 'origin': + path = volume.path_origin + elif _type in ['read-write', 'volatile']: + path = volume.path - @property - def volatile_img(self): - '''Path to the volatile image''' - return self.abspath(qubes.config.vm_files['volatile_img']) + if size <= volume.size: + raise StoragePoolException('Can not shring volume %s' % + volume.name) - def root_dev_config(self): - dev_name = 'root' - if isinstance(self.vm, qubes.vm.templatevm.TemplateVM): - return self.format_disk_dev( - '{root}:{rootcow}'.format( - root=self.root_img, - rootcow=self.rootcow_img), - dev_name, - script='block-origin') - - elif self.vm.hvm and hasattr(self.vm, 'template'): - # HVM template-based VM - only one device-mapper layer, in dom0 - # (root+volatile) - # HVM detection based on 'kernel' property is massive hack, - # but taken from assumption that VM needs Qubes-specific kernel - # (actually initramfs) to assemble the second layer of device-mapper - return self.format_disk_dev( - '{root}:{volatile}'.format( - root=self.vm.template.storage.root_img, - volatile=self.volatile_img), - dev_name, - script='block-snapshot') - - elif hasattr(self.vm, 'template'): - # any other template-based VM - two device-mapper layers: one - # in dom0 (here) from root+root-cow, and another one from - # this+volatile.img - path = '{root}:{template_rootcow}'.format( - root=self.root_img, - template_rootcow=self.vm.template.storage.rootcow_img) - return self.format_disk_dev(path=path, - vdev=self.root_dev, - script='block-snapshot', - rw=False) - - else: - # standalone qube - return self.format_disk_dev(self.root_img, dev_name) - - def private_dev_config(self): - return self.format_disk_dev(self.private_img, 'private') - - def volatile_dev_config(self): - return self.format_disk_dev(self.volatile_img, 'volatile') - - def create_on_disk_private_img(self, source_template=None): - if not os.path.exists(self.target_dir): - os.makedirs(self.target_dir) - if source_template is None: - f_private = open(self.private_img, 'a+b') - f_private.truncate(self.private_img_size) - f_private.close() - - else: - self.vm.log.info("Copying the template's private image: {}".format( - source_template.storage.private_img)) - self._copy_file(source_template.storage.private_img, self.private_img) - - def create_on_disk_root_img(self, source_template=None): - if not os.path.exists(self.target_dir): - os.makedirs(self.target_dir) - if source_template is None: - fd = open(self.root_img, 'a+b') - fd.truncate(self.root_img_size) - fd.close() - - elif self.vm.updateable: - # if not updateable, just use template's disk - self.vm.log.info( - "--> Copying the template's root image: {}".format( - source_template.storage.root_img)) - self._copy_file(source_template.storage.root_img, self.root_img) - - def resize_private_img(self, size): - fd = open(self.private_img, 'a+b') - fd.truncate(size) - fd.close() + with open(path, 'a+b') as fd: + fd.truncate(size) # find loop device if any - p = subprocess.Popen( - ['sudo', 'losetup', '--associated', self.private_img], - stdout=subprocess.PIPE) + p = subprocess.Popen(['sudo', 'losetup', '--associated', path], + stdout=subprocess.PIPE) result = p.communicate() m = re.match(r'^(/dev/loop\d+):\s', result[0]) @@ -172,106 +105,46 @@ class XenPool(Pool): subprocess.check_call(['sudo', 'losetup', '--set-capacity', loop_dev]) - def commit_template_changes(self): - assert isinstance(self.vm, qubes.vm.templatevm.TemplateVM) + def commit_template_changes(self, volume): + if volume.volume_type != 'origin': + return volume - # TODO: move rootcow_img to this class; the same for vm.is_outdated() - if os.path.exists(self.vm.rootcow_img): - os.rename(self.vm.rootcow_img, self.vm.rootcow_img + '.old') + if os.path.exists(volume.path_cow): + os.rename(volume.path_cow, volume.path_cow + '.old') old_umask = os.umask(002) - f_cow = open(self.vm.rootcow_img, 'w') - f_root = open(self.root_img, 'r') - f_root.seek(0, os.SEEK_END) - # make empty sparse file of the same size as root.img - f_cow.truncate(f_root.tell()) - f_cow.close() - f_root.close() + with open(volume.path_cow, 'w') as f_cow: + f_cow.truncate(volume.size) os.umask(old_umask) return volume def start(self, volume): if volume.volume_type == 'volatile': self._reset_volume(volume) + if volume.volume_type in ['origin', 'snapshot']: + _check_path(volume.path_origin) + _check_path(volume.path_cow) + else: + _check_path(volume.path) + return volume + def stop(self, volume): + pass + def _reset_volume(self, volume): ''' Remove and recreate a volatile volume ''' assert volume.volume_type == 'volatile', "Not a volatile volume" - size = self.vm.volume_config[volume.name]['size'] - assert size + assert volume.size - if os.path.exists(volume.path): - os.remove(volume.path) + _remove_if_exists(volume) with open(volume.path, "w") as f_volatile: f_volatile.truncate(volume.size) return volume - def reset_volatile_storage(self): - try: - # no template set, in any way (Standalone VM, Template VM) - if self.vm.template is None: - raise AttributeError - - # template-based HVM with only one device-mapper layer - - # volatile.img used as upper layer on root.img, no root-cow.img - # intermediate layer - if self.vm.hvm: - if os.path.exists(self.volatile_img): - if self.vm.debug: - if os.path.getmtime(self.vm.template.storage.root_img) \ - > os.path.getmtime(self.volatile_img): - self.vm.log.warning( - 'Template have changed, resetting root.img') - else: - self.vm.log.warning( - 'Debug mode: not resetting root.img; if you' - ' want to force root.img reset, either' - ' update template VM, or remove volatile.img' - ' file.') - return - - os.remove(self.volatile_img) - - # FIXME stat on f_root; with open() ... - f_volatile = open(self.volatile_img, "w") - f_root = open(self.vm.template.storage.root_img, "r") - # make empty sparse file of the same size as root.img - f_root.seek(0, os.SEEK_END) - f_volatile.truncate(f_root.tell()) - f_volatile.close() - f_root.close() - return # XXX why is that? super() does not run - except AttributeError: # self.vm.template - pass - - super(XenPool, self).reset_volatile_storage() - - def prepare_for_vm_startup(self): - super(XenPool, self).prepare_for_vm_startup() - - if self.drive is not None: - # pylint: disable=unused-variable - (drive_type, drive_domain, drive_path) = self.drive.split(":") - - if drive_domain.lower() != "dom0": - # XXX "VM '{}' holding '{}' does not exists".format( - drive_vm = self.vm.app.domains[drive_domain] - - if not drive_vm.is_running(): - raise qubes.exc.QubesVMNotRunningError( - drive_vm, 'VM {!r} holding {!r} isn\'t running'.format( - drive_domain, drive_path)) - - if self.rootcow_img and not os.path.exists(self.rootcow_img): - self.commit_template_changes() - - # XXX there is also a class attribute on the domain classes which does - # exactly that -- which one should prevail? - @property - def target_dir(self): + def target_dir(self, vm): """ Returns the path to vmdir depending on the type of the VM. The default QubesOS file storage saves the vm images in three @@ -288,7 +161,6 @@ class XenPool(Pool): string (str) absolute path to the directory where the vm files are stored """ - vm = self.vm if vm.is_template(): subdir = 'vm-templates' elif vm.is_disposablevm(): @@ -300,16 +172,10 @@ class XenPool(Pool): return os.path.join(self.dir_path, subdir, vm.name) - def abspath(self, file_name): - return os.path.join(self.target_dir, file_name) - - def init_volume(self, volume_config): + def init_volume(self, vm, volume_config): assert 'volume_type' in volume_config, "Volume type missing " \ + str(volume_config) - target_dir = self.target_dir - assert target_dir, "Pool target_dir not set" volume_type = volume_config['volume_type'] - volume_config['target_dir'] = target_dir known_types = { 'read-write': ReadWriteFile, 'read-only': ReadOnlyFile, @@ -320,125 +186,124 @@ class XenPool(Pool): if volume_type not in known_types: raise StoragePoolException("Unknown volume type " + volume_type) - if volume_type == 'snapshot': - path = qubes.storage.get_pool(volume_config['pool'], self.vm.template).target_dir - volume_config['vid'] = os.path.join(path, volume_config['name'] + '.img') + if volume_type in ['snapshot', 'read-only']: + origin_pool = vm.app.get_pool(volume_config['pool']) + assert isinstance(origin_pool, + XenPool), 'Origin volume not a xen volume' + volume_config['target_dir'] = origin_pool.target_dir(vm.template) + name = volume_config['name'] + volume_config['size'] = vm.template.volume_config[name]['size'] + else: + volume_config['target_dir'] = self.target_dir(vm) return known_types[volume_type](**volume_config) -class SizeMixIn(Volume): +class XenVolume(Volume): + ''' Parent class for the xen volumes implementation ''' + def __init__(self, target_dir, **kwargs): + self.target_dir = target_dir + assert self.target_dir, "target_dir not specified" + super(XenVolume, self).__init__(**kwargs) + + +class SizeMixIn(XenVolume): + ''' A mix in which expects a `size` param to be > 0 on initialization and + provides a usage property wrapper. + ''' def __init__(self, name=None, pool=None, vid=None, target_dir=None, size=0, **kwargs): assert size > 0, 'Size for volume ' + name + ' is <=0' super(SizeMixIn, self).__init__(name=name, pool=pool, vid=vid, + size=size, **kwargs) - self._size = size self.target_dir = target_dir @property - def size(self): - if self.vid and os.path.exists(self.vid): - return qubes.storage.get_disk_usage(self.vid) - else: - return self._size + def usage(self): + ''' Returns the actualy used space ''' + return get_disk_usage(self.vid) + class ReadWriteFile(SizeMixIn): - # :pylint: disable=too-few-public-methods + # :pylint: disable=missing-docstring def __init__(self, **kwargs): super(ReadWriteFile, self).__init__(**kwargs) self.path = os.path.join(self.target_dir, self.name + '.img') self.vid = self.path - @property - def size(self): - if self.vid and os.path.exists(self.vid): - return qubes.storage.get_disk_usage(self.vid) - else: - return self._size - - def create(self): - create_file(self.path, self.size) - - @property - def created(self): - return os.path.exists(self.path) - class ReadOnlyFile(Volume): + # :pylint: disable=missing-docstring + usage = 0 def __init__(self, name=None, pool=None, vid=None, target_dir=None, - **kwargs): + size=0, **kwargs): + # :pylint: disable=unused-argument assert os.path.exists(vid), "read-only volume missing vid" super(ReadOnlyFile, self).__init__(name=name, - pool=pool, - vid=vid, - **kwargs) + pool=pool, + vid=vid, + size=size, + **kwargs) self.path = self.vid - @property - def size(self): - return qubes.storage.get_disk_usage(self.vid) - class OriginFile(SizeMixIn): + # :pylint: disable=missing-docstring script = 'block-origin' def __init__(self, **kwargs): super(OriginFile, self).__init__(**kwargs) - self.path = os.path.join(self.target_dir, self.name + '.img') + self.path_origin = os.path.join(self.target_dir, self.name + '.img') self.path_cow = os.path.join(self.target_dir, self.name + '-cow.img') - self.vid = self.path - - def create(self): - create_file(self.path, self.size) - create_file(self.path_cow, self.size) + self.path = '%s:%s' % (self.path_origin, self.path_cow) + self.vid = self.path_origin def commit(self): raise NotImplementedError @property - def size(self): - if self.vid and os.path.exists(self.vid): - return qubes.storage.get_disk_usage(self.vid) - else: - return self._size - - @property - def created(self): - return os.path.exists(self.path) and os.path.exists(self.path_cow) + def usage(self): + result = 0 + if os.path.exists(self.path_origin): + result += get_disk_usage(self.path_origin) + if os.path.exists(self.path_cow): + result += get_disk_usage(self.path_cow) + return result class SnapshotFile(Volume): - # :pylint: disable=too-few-public-methods + # :pylint: disable=missing-docstring script = 'block-snapshot' rw = False + usage = 0 def __init__(self, name=None, pool=None, vid=None, target_dir=None, - **kwargs): - assert vid, "SnapshotVolume missing a vid to OriginVolume" - assert os.path.exists(vid), "OriginVolume does not exist" + size=None, **kwargs): + assert size super(SnapshotFile, self).__init__(name=name, - pool=pool, - vid=vid, - **kwargs) - self.path = os.path.join(target_dir, name + '.img') + pool=pool, + vid=vid, + size=size, + **kwargs) + self.path_origin = os.path.join(target_dir, name + '.img') self.path_cow = os.path.join(target_dir, name + '-cow.img') + self.path = '%s:%s' % (self.path_origin, self.path_cow) + self.vid = self.path_origin @property def created(self): - return os.path.exists(self.path) and os.path.exists(self.path_cow) - - @property - def size(self): - return qubes.storage.get_disk_usage(self.vid) + return os.path.exists(self.path_origin) and os.path.exists( + self.path_cow) class VolatileFile(SizeMixIn): + # :pylint: disable=missing-docstring def __init__(self, **kwargs): super(VolatileFile, self).__init__(**kwargs) @@ -446,8 +311,94 @@ class VolatileFile(SizeMixIn): self.vid = self.path -def create_file(path, size): +def create_sparse_file(path, size): + ''' Create an empty sparse file ''' if os.path.exists(path): raise IOError("Volume %s already exists", path) + parent_dir = os.path.dirname(path) + if not os.path.exists(parent_dir): + os.makedirs(parent_dir) with open(path, 'a+b') as fh: fh.truncate(size) + + +def get_disk_usage_one(st): + '''Extract disk usage of one inode from its stat_result struct. + + If known, get real disk usage, as written to device by filesystem, not + logical file size. Those values may be different for sparse files. + + :param os.stat_result st: stat result + :returns: disk usage + ''' + try: + return st.st_blocks * BLKSIZE + except AttributeError: + return st.st_size + + +def get_disk_usage(path): + '''Get real disk usage of given path (file or directory). + + When *path* points to directory, then it is evaluated recursively. + + This function tries estiate real disk usage. See documentation of + :py:func:`get_disk_usage_one`. + + :param str path: path to evaluate + :returns: disk usage + ''' + try: + st = os.lstat(path) + except OSError: + return 0 + + ret = get_disk_usage_one(st) + + # if path is not a directory, this is skipped + for dirpath, dirnames, filenames in os.walk(path): + for name in dirnames + filenames: + ret += get_disk_usage_one(os.lstat(os.path.join(dirpath, name))) + + return ret + + +def create_dir_if_not_exists(path): + """ Check if a directory exists in if not create it. + + This method does not create any parent directories. + """ + if not os.path.exists(path): + os.mkdir(path) + + +def copy_file(source, destination): + '''Effective file copy, preserving sparse files etc. + ''' + # TODO: Windows support + # We prefer to use Linux's cp, because it nicely handles sparse files + assert os.path.exists(source), \ + "Missing the source %s to copy from" % source + assert not os.path.exists(destination), \ + "Destination %s already exists" % destination + + parent_dir = os.path.dirname(destination) + if not os.path.exists(parent_dir): + os.makedirs(parent_dir) + + try: + subprocess.check_call(['cp', '--reflink=auto', source, destination]) + except subprocess.CalledProcessError: + raise IOError('Error while copying {!r} to {!r}'.format(source, + destination)) + + +def _remove_if_exists(volume): + if os.path.exists(volume.path): + os.remove(volume.path) + + +def _check_path(path): + ''' Raise an StoragePoolException if ``path`` does not exist''' + if not os.path.exists(path): + raise StoragePoolException('Missing image file: %s' % path) diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index c6203121..91be2f7d 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -333,10 +333,10 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): @property def block_devices(self): - return [self.storage.root_dev_config(), - self.storage.private_dev_config(), - self.storage.volatile_dev_config(), - self.storage.other_dev_config()] + ''' Return all :py:class:`qubes.devices.BlockDevice`s for current domain + for serialization in the libvirt XML template as . + ''' + return [v.block_device() for v in self.volumes.values()] @property def qdb(self): @@ -353,21 +353,27 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): def private_img(self): '''Location of private image of the VM (that contains :file:`/rw` \ and :file:`/home`).''' - return self.storage.private_img + warnings.warn("volatile_img is deprecated, use volumes['private'].vid", + DeprecationWarning) + return self.volumes['private'].vid # XXX this should go to to AppVM? or TemplateVM? @property def root_img(self): '''Location of root image.''' - return self.storage.root_img + warnings.warn("root_img is deprecated, use volumes['root'].vid", + DeprecationWarning) + return self.volumes['root'].vid # XXX and this should go to exactly where? DispVM has it. @property def volatile_img(self): '''Volatile image that overlays :py:attr:`root_img`.''' - return self.storage.volatile_img + warnings.warn("volatile_img is deprecated, use volumes['volatile'].vid", + DeprecationWarning) + return self.volumes['volatile'].vid # XXX shouldn't this go elsewhere? @@ -425,8 +431,10 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): # constructor # - def __init__(self, app, xml, **kwargs): + def __init__(self, app, xml, volume_config={}, **kwargs): super(QubesVM, self).__init__(app, xml, **kwargs) + if hasattr(self, 'volume_config'): + dict_merge(self.volume_config, volume_config) import qubes.vm.adminvm # pylint: disable=redefined-outer-name @@ -637,7 +645,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): self.netvm.start(start_guid=start_guid, notify_function=notify_function) - self.storage.prepare_for_vm_startup() + self.storage.start() self._update_libvirt_domain() qmemman_client = self.request_memory(mem_required) @@ -729,6 +737,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): exc_info=1) self.libvirt_domain.shutdown() + self.storage.stop() def kill(self): @@ -742,6 +751,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): raise qubes.exc.QubesVMNotStartedError(self) self.libvirt_domain.destroy() + self.storage.stop() def force_shutdown(self, *args, **kwargs): @@ -1025,7 +1035,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): p.communicate(input=self.default_user) - # TODO move to storage + # TODO rename to create def create_on_disk(self, source_template=None): '''Create files needed for VM. @@ -1115,11 +1125,11 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): while self.is_running(): #1696 time.sleep(1) - def remove_from_disk(self): '''Remove domain remnants from disk.''' self.fire_event('domain-remove-from-disk') - self.storage.remove_from_disk() + self.storage.remove() + shutil.rmtree(self.vm.dir_path) def clone_disk_files(self, src): @@ -1454,68 +1464,69 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): # XXX shouldn't this go only to vms that have root image? def get_disk_utilization_root_img(self): - '''Get space that is actually ocuppied by :py:attr:`root_img`. - - Root image is a sparse file, so it is probably much less than logical - available space. + '''Get space that is actually ocuppied by :py:attr:`volumes['root']`. + This is a temporary wrapper for backwards compatibility. You should + call directly :py:attr:`volumes[name].utilization` :returns: domain's real disk image size [FIXME unit] :rtype: FIXME .. seealso:: :py:meth:`get_root_img_sz` ''' - return qubes.storage.get_disk_usage(self.root_img) + warnings.warn( + "get_disk_utilization_root_img is deprecated, use volumes['root'].utilization", + DeprecationWarning) + return qubes.storage.get_disk_usage(self.volumes['root'].utilization) # XXX shouldn't this go only to vms that have root image? def get_root_img_sz(self): - '''Get image size of :py:attr:`root_img`. - - Root image is a sparse file, so it is probably much more than ocuppied - physical space. + '''Get the size of the :py:attr:`volumes['root']`. + This is a temporary wrapper for backwards compatibility. You should + call directly :py:attr:`volumes[name].size` :returns: domain's virtual disk size [FIXME unit] :rtype: FIXME .. seealso:: :py:meth:`get_disk_utilization_root_img` ''' - if not os.path.exists(self.root_img): - return 0 - - return os.path.getsize(self.root_img) - + warnings.warn( + "get_disk_root_img_sz is deprecated, use volumes['root'].size", + DeprecationWarning) + return qubes.storage.get_disk_usage(self.volumes['root'].size) def get_disk_utilization_private_img(self): - '''Get space that is actually ocuppied by :py:attr:`private_img`. - - Private image is a sparse file, so it is probably much less than - logical available space. + '''Get space that is actually ocuppied by :py:attr:`volumes['private']`. + This is a temporary wrapper for backwards compatibility. You should + call directly :py:attr:`volumes[name].utilization` :returns: domain's real disk image size [FIXME unit] :rtype: FIXME + ''' - .. seealso:: :py:meth:`get_private_img_sz` - ''' # pylint: disable=invalid-name - - return qubes.storage.get_disk_usage(self.private_img) - + warnings.warn( + "get_disk_utilization_private_img is deprecated, use volumes['private'].utilization", + DeprecationWarning) + return qubes.storage.get_disk_usage(self.volumes[ + 'private'].utilization) def get_private_img_sz(self): - '''Get image size of :py:attr:`private_img`. - - Private image is a sparse file, so it is probably much more than - ocuppied physical space. + '''Get the size of the :py:attr:`volumes['private']`. + This is a temporary wrapper for backwards compatibility. You should + call directly :py:attr:`volumes[name].size` :returns: domain's virtual disk size [FIXME unit] :rtype: FIXME .. seealso:: :py:meth:`get_disk_utilization_private_img` ''' - return self.storage.get_private_img_sz() - + warnings.warn( + "get_disk_private_img_sz is deprecated, use volumes['private'].size", + DeprecationWarning) + return qubes.storage.get_disk_usage(self.volumes['private'].size) def get_disk_utilization(self): '''Return total space actually occuppied by all files belonging to \ @@ -1527,7 +1538,6 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): return qubes.storage.get_disk_usage(self.dir_path) - # TODO move to storage def verify_files(self): '''Verify that files accessed by this machine are sane. @@ -1751,3 +1761,20 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): # if self.is_qrexec_running(): # #TODO: kill qrexec daemon # pass + + +def dict_merge(dct, merge_dct): + """ Recursive dict merge. Inspired by :meth:``dict.update()``, instead of + updating only top-level keys, dict_merge recurses down into dicts nested + to an arbitrary depth, updating keys. The ``merge_dct`` is merged into + ``dct``. (Source https://gist.github.com/angstwad/bf22d1822c38a92ec0a9) + :param dct: dict onto which the merge is executed + :param merge_dct: dct merged into dct + :return: None + """ + for k, v in merge_dct.iteritems(): + if (k in dct and isinstance(dct[k], dict) + and isinstance(merge_dct[k], dict)): + dict_merge(dct[k], merge_dct[k]) + else: + dct[k] = merge_dct[k] diff --git a/qubes/vm/templatevm.py b/qubes/vm/templatevm.py index 7eadc134..39432ac8 100644 --- a/qubes/vm/templatevm.py +++ b/qubes/vm/templatevm.py @@ -1,13 +1,16 @@ #!/usr/bin/python2 -O # vim: fileencoding=utf-8 +import warnings + import qubes import qubes.config import qubes.vm.qubesvm from qubes.config import defaults +from qubes.vm.qubesvm import QubesVM -class TemplateVM(qubes.vm.qubesvm.QubesVM): +class TemplateVM(QubesVM): '''Template for AppVM''' dir_path_prefix = qubes.config.system_path['qubes_templates_dir'] @@ -15,7 +18,9 @@ class TemplateVM(qubes.vm.qubesvm.QubesVM): @property def rootcow_img(self): '''COW image''' - return self.storage.rootcow_img + warnings.warn("rootcow_img is deprecated, use " + "volumes['root'].path_origin", DeprecationWarning) + return self.volumes['root'].path_cow @property def appvms(self): @@ -67,6 +72,4 @@ class TemplateVM(qubes.vm.qubesvm.QubesVM): assert not self.is_running(), \ 'Attempt to commit changes on running Template VM!' - self.log.info('Commiting template update; COW: {}'.format( - self.rootcow_img)) self.storage.commit_template_changes() From 973c83cedd402a6260c74d191449680e3a8a75da Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Tue, 5 Apr 2016 20:04:20 +0200 Subject: [PATCH 22/44] Move most resize logic to XenPool --- qubes/storage/xen.py | 12 ++++++------ qubes/vm/qubesvm.py | 33 ++++++++------------------------- 2 files changed, 14 insertions(+), 31 deletions(-) diff --git a/qubes/storage/xen.py b/qubes/storage/xen.py index a50cd23f..ada83f50 100644 --- a/qubes/storage/xen.py +++ b/qubes/storage/xen.py @@ -85,16 +85,16 @@ class XenPool(Pool): elif _type in ['read-write', 'volatile']: path = volume.path - if size <= volume.size: - raise StoragePoolException('Can not shring volume %s' % - volume.name) - with open(path, 'a+b') as fd: fd.truncate(size) + self._resize_loop_device(path) + + def _resize_loop_device(self, path): # find loop device if any - p = subprocess.Popen(['sudo', 'losetup', '--associated', path], - stdout=subprocess.PIPE) + p = subprocess.Popen( + ['sudo', 'losetup', '--associated', path], + stdout=subprocess.PIPE) result = p.communicate() m = re.match(r'^(/dev/loop\d+):\s', result[0]) diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 91be2f7d..974e7ed8 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -1064,11 +1064,11 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): def resize_private_img(self, size): '''Resize private image.''' - if size >= self.get_private_img_sz(): - raise qubes.exc.QubesValueError('Cannot shrink private.img') + warnings.warn( + "resize_private_img is deprecated, use volumes[name].resize()", + DeprecationWarning) - # resize the image - self.storage.resize_private_img(size) + self.volumes['private'].resize(size) # and then the filesystem # FIXME move this to qubes.storage.xen.XenVMStorage @@ -1084,29 +1084,12 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): if retcode != 0: raise qubes.exc.QubesException('resize2fs failed') - - # TODO move to storage def resize_root_img(self, size, allow_start=False): - if hasattr(self, 'template'): - raise qubes.exc.QubesVMError(self, - 'Cannot resize root.img of template based qube. Resize the' - ' root.img of the template instead.') + warnings.warn( + "resize_root_img is deprecated, use volumes[name].resize()", + DeprecationWarning) - # TODO self.is_halted - if self.is_running(): - raise qubes.exc.QubesVMNotHaltedError(self, - 'Cannot resize root.img of a running qube') - - if size < self.get_root_img_sz(): - raise qubes.exc.QubesValueError( - 'For your own safety, shrinking of root.img is disabled. If you' - ' really know what you are doing, use `truncate` manually.') - - with open(self.root_img, 'a+b') as fd: - fd.truncate(size) - - if False: #self.hvm: - return + self.volumes['root'].resize(size) if not allow_start: raise qubes.exc.QubesException( From 5f7cb41a2147a2a3f076224b4b9a6a5b4a2451ae Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 15 Apr 2016 15:16:25 +0200 Subject: [PATCH 23/44] Move Storage.clone_disk_files logic to XenPool - Add XenVolume to identify volumes which can be cloned even if they are not in the same pool --- qubes/storage/__init__.py | 56 ++++++++++--------------------------- qubes/storage/xen.py | 59 ++++++++++++++++++++------------------- qubes/vm/qubesvm.py | 8 ++++-- 3 files changed, 50 insertions(+), 73 deletions(-) diff --git a/qubes/storage/__init__.py b/qubes/storage/__init__.py index da257d69..df01efae 100644 --- a/qubes/storage/__init__.py +++ b/qubes/storage/__init__.py @@ -31,7 +31,6 @@ from __future__ import absolute_import import os import os.path import shutil -import subprocess import pkg_resources import qubes @@ -146,7 +145,7 @@ class Storage(object): old_umask = os.umask(002) - self.vm.log.info('Creating directory: {0}'.format(self.vm.dir_path)) + self.log.info('Creating directory: {0}'.format(self.vm.dir_path)) os.makedirs(self.vm.dir_path) for name, volume in self.vm.volumes.items(): source_volume = None @@ -156,25 +155,17 @@ class Storage(object): os.umask(old_umask) - # TODO migrate this - def clone_disk_files(self, src_vm): - # :pylint: disable=missing-docstring + def clone(self, src_vm): self.vm.log.info('Creating directory: {0}'.format(self.vm.dir_path)) - os.mkdir(self.vm.dir_path) - - if hasattr(src_vm, 'private_img'): - self.vm.log.info('Copying the private image: {} -> {}'.format( - src_vm.private_img, self.vm.private_img)) - self._copy_file(src_vm.private_img, self.vm.private_img) - - if src_vm.updateable and hasattr(src_vm, 'root_img'): - self.vm.log.info( - 'Copying the root image: {} -> {}'.format( - src_vm.volume['root'].path_origin, - self.vm.volume['root'].path_origin) - ) - self._copy_file(src_vm.volume['root'].path_origin, - self.vm.volume['root'].path_origin) + if not os.path.exists(self.vm.dir_path): + self.log.info('Creating directory: {0}'.format(self.vm.dir_path)) + os.makedirs(self.vm.dir_path) + for name, target in self.vm.volumes.items(): + pool = self.get_pool(target) + source = src_vm.volumes[name] + volume = pool.clone(source, target) + assert volume, "%s.clone() returned '%s'" % (pool.__class__, volume) + self.vm.volumes[name] = volume # TODO migrate this @staticmethod @@ -265,33 +256,16 @@ class Pool(object): raise NotImplementedError("Pool %s has config() not implemented" % self.name) - @staticmethod - def _copy_file(source, destination): - '''Effective file copy, preserving sparse files etc. - ''' - # TODO: Windows support - # We prefer to use Linux's cp, because it nicely handles sparse files - assert os.path.exists(source), \ - "Missing the source %s to copy from" % source - assert not os.path.exists(destination), \ - "Destination %s already exists" % destination - try: - subprocess.check_call(['cp', '--reflink=auto', source, destination - ]) - except subprocess.CalledProcessError: - raise IOError('Error while copying {!r} to {!r}'.format( - source, destination)) + def clone(self, source, target): + ''' Clone volume ''' + raise NotImplementedError("Pool %s has clone() not implemented" % + self.name) def remove(self, volume): ''' Remove volume''' raise NotImplementedError("Pool %s has remove() not implemented" % self.name) - def clone(self, source, target): - ''' Clone volume ''' - raise NotImplementedError("Pool %s has clone() not implemented" % - self.name) - def start(self, volume): ''' Do what ever is needed on start ''' raise NotImplementedError("Pool %s has start() not implemented" % diff --git a/qubes/storage/xen.py b/qubes/storage/xen.py index ada83f50..020b3270 100644 --- a/qubes/storage/xen.py +++ b/qubes/storage/xen.py @@ -50,6 +50,20 @@ class XenPool(Pool): vm_templates_path = os.path.join(self.dir_path, 'vm-templates') create_dir_if_not_exists(vm_templates_path) + def clone(self, source, target): + ''' Clones the volume if the `source.pool` if the source is a + :py:class:`XenVolume`. + ''' + if issubclass(XenVolume, source.__class__): + raise StoragePoolException('Volumes %s and %s use different pools' + % (source.__class__, target.__class__)) + + if source.volume_type not in ['origin', 'read-write']: + return target + + copy_file(source.vid, target.vid) + return target + def create(self, volume, source_volume=None): _type = volume.volume_type size = volume.size @@ -200,7 +214,9 @@ class XenPool(Pool): class XenVolume(Volume): - ''' Parent class for the xen volumes implementation ''' + ''' Parent class for the xen volumes implementation which expects a + `target_dir` param on initialization. + ''' def __init__(self, target_dir, **kwargs): self.target_dir = target_dir @@ -212,15 +228,11 @@ class SizeMixIn(XenVolume): ''' A mix in which expects a `size` param to be > 0 on initialization and provides a usage property wrapper. ''' - def __init__(self, name=None, pool=None, vid=None, target_dir=None, size=0, - **kwargs): - assert size > 0, 'Size for volume ' + name + ' is <=0' - super(SizeMixIn, self).__init__(name=name, - pool=pool, - vid=vid, - size=size, - **kwargs) - self.target_dir = target_dir + + def __init__(self, size=0, **kwargs): + super(SizeMixIn, self).__init__(size=int(size), **kwargs) + assert size, 'Empty size provided' + assert size > 0, 'Size for volume ' + kwargs['name'] + ' is <=0' @property def usage(self): @@ -237,19 +249,13 @@ class ReadWriteFile(SizeMixIn): self.vid = self.path -class ReadOnlyFile(Volume): +class ReadOnlyFile(XenVolume): # :pylint: disable=missing-docstring usage = 0 - def __init__(self, name=None, pool=None, vid=None, target_dir=None, - size=0, **kwargs): + def __init__(self, size=0, **kwargs): # :pylint: disable=unused-argument - assert os.path.exists(vid), "read-only volume missing vid" - super(ReadOnlyFile, self).__init__(name=name, - pool=pool, - vid=vid, - size=size, - **kwargs) + super(ReadOnlyFile, self).__init__(size=int(size), **kwargs) self.path = self.vid @@ -277,22 +283,17 @@ class OriginFile(SizeMixIn): return result -class SnapshotFile(Volume): +class SnapshotFile(XenVolume): # :pylint: disable=missing-docstring script = 'block-snapshot' rw = False usage = 0 - def __init__(self, name=None, pool=None, vid=None, target_dir=None, - size=None, **kwargs): + def __init__(self, name=None, size=None, **kwargs): assert size - super(SnapshotFile, self).__init__(name=name, - pool=pool, - vid=vid, - size=size, - **kwargs) - self.path_origin = os.path.join(target_dir, name + '.img') - self.path_cow = os.path.join(target_dir, name + '-cow.img') + super(SnapshotFile, self).__init__(name=name, size=int(size), **kwargs) + self.path_origin = os.path.join(self.target_dir, name + '.img') + self.path_cow = os.path.join(self.target_dir, name + '-cow.img') self.path = '%s:%s' % (self.path_origin, self.path_cow) self.vid = self.path_origin diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 974e7ed8..086fbba7 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -1114,18 +1114,20 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): self.storage.remove() shutil.rmtree(self.vm.dir_path) - def clone_disk_files(self, src): '''Clone files from other vm. :param qubes.vm.qubesvm.QubesVM src: source VM ''' - if src.is_running(): # XXX what about paused? + if src.is_running(): # XXX what about paused? raise qubes.exc.QubesVMNotHaltedError( self, 'Cannot clone a running domain {!r}'.format(self.name)) - self.storage.clone_disk_files(src) + if hasattr(src, 'volume_config'): + self.volume_config = src.volume_config + self.storage = qubes.storage.Storage(self) + self.storage.clone(src) if src.icon_path is not None \ and os.path.exists(src.dir_path) \ From 971c4ae91df9f82a08bf2db113b15b5e0024760c Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 15 Apr 2016 16:01:27 +0200 Subject: [PATCH 24/44] Add XenPool.driver field --- qubes/storage/xen.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/qubes/storage/xen.py b/qubes/storage/xen.py index 020b3270..01463b39 100644 --- a/qubes/storage/xen.py +++ b/qubes/storage/xen.py @@ -38,6 +38,7 @@ BLKSIZE = 512 class XenPool(Pool): ''' File based 'original' disk implementation ''' + driver = 'xen' def __init__(self, name=None, dir_path=None): super(XenPool, self).__init__(name=name) @@ -77,6 +78,14 @@ class XenPool(Pool): return volume + @property + def config(self): + return { + 'name': self.name, + 'dir_path': self.dir_path, + 'driver': XenPool.driver, + } + def resize(self, volume, size): ''' Expands volume, throws :py:class:`qubst.storage.StoragePoolException` if given size is From a37fc2464a31b7499b14f67e87aca9278faad8f7 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 15 Apr 2016 19:57:23 +0200 Subject: [PATCH 25/44] Add XenPool.config() --- qubes/storage/xen.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/qubes/storage/xen.py b/qubes/storage/xen.py index 01463b39..c1b0f803 100644 --- a/qubes/storage/xen.py +++ b/qubes/storage/xen.py @@ -248,6 +248,13 @@ class SizeMixIn(XenVolume): ''' Returns the actualy used space ''' return get_disk_usage(self.vid) + @property + def config(self): + ''' return config data for serialization to qubes.xml ''' + return {'name': self.name, + 'pool': self.pool, + 'size': str(self.size), + 'volume_type': self.volume_type} class ReadWriteFile(SizeMixIn): From d1a0542c851535c5c26f8be3fe0c6ecad64b67ed Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 15 Apr 2016 17:10:55 +0200 Subject: [PATCH 26/44] Add XenPool.remove() --- qubes/storage/xen.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/qubes/storage/xen.py b/qubes/storage/xen.py index c1b0f803..aad54d9b 100644 --- a/qubes/storage/xen.py +++ b/qubes/storage/xen.py @@ -113,6 +113,13 @@ class XenPool(Pool): self._resize_loop_device(path) + def remove(self, volume): + if volume.volume_type in ['read-write', 'volatile']: + _remove_if_exists(volume.vid) + elif volume.volume_type == 'origin': + _remove_if_exists(volume.vid) + _remove_if_exists(volume.path_cow) + def _resize_loop_device(self, path): # find loop device if any p = subprocess.Popen( From 9674d0308835ee6d6746fcb813619b60ed1e55dc Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 1 Apr 2016 15:10:44 +0200 Subject: [PATCH 27/44] Add pool LinuxKernel --- qubes/storage/kernels.py | 88 ++++++++++++++++++++++++++++++++++++++++ rpm_spec/core-dom0.spec | 1 + setup.py | 1 + 3 files changed, 90 insertions(+) create mode 100644 qubes/storage/kernels.py diff --git a/qubes/storage/kernels.py b/qubes/storage/kernels.py new file mode 100644 index 00000000..d7441b29 --- /dev/null +++ b/qubes/storage/kernels.py @@ -0,0 +1,88 @@ +#!/usr/bin/python2 +# -*- encoding: utf8 -*- +# +# The Qubes OS Project, http://www.qubes-os.org +# +# Copyright (C) 2010-2015 Joanna Rutkowska +# Copyright (C) 2015 Wojtek Porczyk +# Copyright (C) 2016 Bahtiar `kalkin-` Gadimov +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +import os + +from qubes.exc import QubesVMError +from qubes.storage import Pool, StoragePoolException, Volume + + +class LinuxModules(Volume): + rw = False + + def __init__(self, target_dir, kernel_version, **kwargs): + super(LinuxModules, self).__init__(**kwargs) + self.kernels_dir = os.path.join(target_dir, kernel_version) + self.path = os.path.join(self.kernels_dir, 'modules.img') + self.vid = self.path + + +class LinuxKernel(Pool): + driver = 'linux-kernel' + + def __init__(self, name=None, dir_path=None): + assert dir_path, 'Missing dir_path' + super(LinuxKernel, self).__init__(name=name) + self.dir_path = dir_path + + def init_volume(self, volume_config): + assert 'volume_type' in volume_config, "Volume type missing " \ + + str(volume_config) + volume_type = volume_config['volume_type'] + if volume_type != 'read-only': + raise StoragePoolException("Unknown volume type " + volume_type) + + return LinuxModules(self.dir_path, self.vm.kernel, **volume_config) + + def clone(self, source, target): + return target + + def create(self, volume, source_volume): + return volume + + def commit_template_changes(self, volume): + return volume + + @property + def config(self): + return { + 'name': self.name, + 'dir_path': self.dir_path, + 'driver': LinuxKernel.driver, + } + + def remove(self, volume): + pass + + def rename(self, volume, old_name, new_name): + return volume + + def start(self, volume): + path = volume.path + if not os.path.exists(path): + msg = 'VM %s is missing modules: %s' % (self.vm.name, path) + raise QubesVMError(self.vm, msg) + return volume + + def stop(self, volume): + pass diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec index d1832259..7f0b7b48 100644 --- a/rpm_spec/core-dom0.spec +++ b/rpm_spec/core-dom0.spec @@ -233,6 +233,7 @@ fi %dir %{python_sitelib}/qubes/storage %{python_sitelib}/qubes/storage/__init__.py* %{python_sitelib}/qubes/storage/xen.py* +%{python_sitelib}/qubes/storage/kernels.py* %dir %{python_sitelib}/qubes/tools %{python_sitelib}/qubes/tools/__init__.py* diff --git a/setup.py b/setup.py index 533981f1..ad3a0f90 100644 --- a/setup.py +++ b/setup.py @@ -45,5 +45,6 @@ if __name__ == '__main__': ], 'qubes.storage': [ 'xen = qubes.storage.xen:XenPool', + 'linux-kernel = qubes.storage.kernels:LinuxKernel', ] }) From 97d04791b756628bad0c777e96054533ec8cdff4 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 15 Apr 2016 20:31:56 +0200 Subject: [PATCH 28/44] After add/remove_pool execute Pool.setup/destroy --- qubes/__init__.py | 8 +++++++- qubes/storage/__init__.py | 13 +++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/qubes/__init__.py b/qubes/__init__.py index e25b1e1d..aa79ad3c 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -1492,13 +1492,19 @@ class Qubes(PropertyHolder): def add_pool(self, **kwargs): """ Add a storage pool to config.""" name = kwargs['name'] - self.pools[name] = self._get_pool(**kwargs) + assert name not in self.pools.keys(), \ + "Pool named %s already exists" % name + pool = self._get_pool(**kwargs) + pool.setup() + self.pools[name] = pool self.save() def remove_pool(self, name): """ Remove a storage pool from config file. """ try: + pool = self.pools[name] del self.pools[name] + pool.destroy() except KeyError: return diff --git a/qubes/storage/__init__.py b/qubes/storage/__init__.py index df01efae..308aaae8 100644 --- a/qubes/storage/__init__.py +++ b/qubes/storage/__init__.py @@ -23,7 +23,6 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # - """ Qubes storage system""" from __future__ import absolute_import @@ -164,7 +163,8 @@ class Storage(object): pool = self.get_pool(target) source = src_vm.volumes[name] volume = pool.clone(source, target) - assert volume, "%s.clone() returned '%s'" % (pool.__class__, volume) + assert volume, "%s.clone() returned '%s'" % (pool.__class__, + volume) self.vm.volumes[name] = volume # TODO migrate this @@ -236,6 +236,7 @@ class Pool(object): # :pylint: disable=unused-argument assert name, "Pool name is missing" self.name = name + kwargs['name'] = self.name def create(self, volume, source_volume): ''' Create the given volume on disk or copy from provided @@ -261,6 +262,10 @@ class Pool(object): raise NotImplementedError("Pool %s has clone() not implemented" % self.name) + def destroy(self): + raise NotImplementedError("Pool %s has destroy() not implemented" % + self.name) + def remove(self, volume): ''' Remove volume''' raise NotImplementedError("Pool %s has remove() not implemented" % @@ -271,6 +276,10 @@ class Pool(object): raise NotImplementedError("Pool %s has start() not implemented" % self.name) + def setup(self): + raise NotImplementedError("Pool %s has setup() not implemented" % + self.name) + def stop(self, volume): ''' Do what ever is needed on stop''' raise NotImplementedError("Pool %s has stop() not implemented" % From 62c81044c5cd2beea1e6e749b8c35dc7468fee5d Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 15 Apr 2016 20:33:04 +0200 Subject: [PATCH 29/44] Add XenPool.setup/destroy --- qubes/storage/kernels.py | 6 ++++++ qubes/storage/xen.py | 16 ++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/qubes/storage/kernels.py b/qubes/storage/kernels.py index d7441b29..8242aafa 100644 --- a/qubes/storage/kernels.py +++ b/qubes/storage/kernels.py @@ -71,12 +71,18 @@ class LinuxKernel(Pool): 'driver': LinuxKernel.driver, } + def destroy(self): + pass + def remove(self, volume): pass def rename(self, volume, old_name, new_name): return volume + def setup(self): + pass + def start(self, volume): path = volume.path if not os.path.exists(path): diff --git a/qubes/storage/xen.py b/qubes/storage/xen.py index aad54d9b..7bb0dfb3 100644 --- a/qubes/storage/xen.py +++ b/qubes/storage/xen.py @@ -45,12 +45,6 @@ class XenPool(Pool): assert dir_path, "No pool dir_path specified" self.dir_path = os.path.normpath(dir_path) - create_dir_if_not_exists(self.dir_path) - appvms_path = os.path.join(self.dir_path, 'appvms') - create_dir_if_not_exists(appvms_path) - vm_templates_path = os.path.join(self.dir_path, 'vm-templates') - create_dir_if_not_exists(vm_templates_path) - def clone(self, source, target): ''' Clones the volume if the `source.pool` if the source is a :py:class:`XenVolume`. @@ -148,6 +142,16 @@ class XenPool(Pool): os.umask(old_umask) return volume + def destroy(self): + pass + + def setup(self): + create_dir_if_not_exists(self.dir_path) + appvms_path = os.path.join(self.dir_path, 'appvms') + create_dir_if_not_exists(appvms_path) + vm_templates_path = os.path.join(self.dir_path, 'vm-templates') + create_dir_if_not_exists(vm_templates_path) + def start(self, volume): if volume.volume_type == 'volatile': self._reset_volume(volume) From ef485ca32a0e6f448a0c08b39705de49d78d8f57 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 1 Apr 2016 16:43:59 +0200 Subject: [PATCH 30/44] Add linux-kernel to defaults['pool_config'] --- qubes/__init__.py | 1 + qubes/config.py | 12 +++++++++++- qubes/tests/storage.py | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/qubes/__init__.py b/qubes/__init__.py index aa79ad3c..22fe7c98 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -1400,6 +1400,7 @@ class Qubes(PropertyHolder): 7: Label(7, '0x75507b', 'purple'), 8: Label(8, '0x000000', 'black'), } + for name, config in qubes.config.defaults['pool_configs'].items(): self.pools[name] = self._get_pool(**config) diff --git a/qubes/config.py b/qubes/config.py index 636b0b07..80a2981b 100644 --- a/qubes/config.py +++ b/qubes/config.py @@ -28,6 +28,8 @@ # make a real /etc/qubes/master.conf or whatever # +import os.path + '''Constants which can be configured in one place''' qubes_base_dir = "/var/lib/qubes" @@ -83,9 +85,17 @@ defaults = { 'private_img_size': 2*1024*1024*1024, 'root_img_size': 10*1024*1024*1024, - 'pool_config': {'dir_path': '/var/lib/qubes/', + 'pool_configs': { + 'default': {'dir_path': qubes_base_dir, 'driver': 'xen', 'name': 'default'}, + 'linux-kernel': { + 'dir_path': os.path.join(qubes_base_dir, + system_path['qubes_kernels_base_dir']), + 'driver': 'linux-kernel', + 'name': 'linux-kernel' + } + }, # how long (in sec) to wait for VMs to shutdown, # before killing them (when used qvm-run with --wait option), diff --git a/qubes/tests/storage.py b/qubes/tests/storage.py index 7e777bdf..5fa82222 100644 --- a/qubes/tests/storage.py +++ b/qubes/tests/storage.py @@ -74,7 +74,7 @@ class TC_00_Pool(SystemTestsMixin, QubesTestCase): def test_001_all_pool_drivers(self): """ The only predefined pool driver is xen """ - self.assertEquals(["xen"], pool_drivers()) + self.assertEquals(['linux-kernel', 'xen'], pool_drivers()) def test_002_get_pool_klass(self): """ Expect the default pool to be `XenPool` """ From fe6a35155e55874ed6d978665c41bbf1218cbcf1 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 1 Apr 2016 17:30:11 +0200 Subject: [PATCH 31/44] Move kernel file checks to LinuxKernel pool --- qubes/storage/kernels.py | 25 ++++++++++++++++++++----- qubes/vm/qubesvm.py | 12 ------------ 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/qubes/storage/kernels.py b/qubes/storage/kernels.py index 8242aafa..febbc13c 100644 --- a/qubes/storage/kernels.py +++ b/qubes/storage/kernels.py @@ -23,7 +23,6 @@ # import os -from qubes.exc import QubesVMError from qubes.storage import Pool, StoragePoolException, Volume @@ -35,6 +34,8 @@ class LinuxModules(Volume): self.kernels_dir = os.path.join(target_dir, kernel_version) self.path = os.path.join(self.kernels_dir, 'modules.img') self.vid = self.path + self.vmlinuz = os.path.join(self.kernels_dir, 'vmlinuz') + self.initramfs = os.path.join(self.kernels_dir, 'initramfs') class LinuxKernel(Pool): @@ -45,14 +46,20 @@ class LinuxKernel(Pool): super(LinuxKernel, self).__init__(name=name) self.dir_path = dir_path - def init_volume(self, volume_config): + def init_volume(self, vm, volume_config): assert 'volume_type' in volume_config, "Volume type missing " \ + str(volume_config) volume_type = volume_config['volume_type'] if volume_type != 'read-only': raise StoragePoolException("Unknown volume type " + volume_type) - return LinuxModules(self.dir_path, self.vm.kernel, **volume_config) + volume = LinuxModules(self.dir_path, vm.kernel, **volume_config) + + _check_path(volume.path) + _check_path(volume.vmlinuz) + _check_path(volume.initramfs) + + return volume def clone(self, source, target): return target @@ -86,9 +93,17 @@ class LinuxKernel(Pool): def start(self, volume): path = volume.path if not os.path.exists(path): - msg = 'VM %s is missing modules: %s' % (self.vm.name, path) - raise QubesVMError(self.vm, msg) + raise StoragePoolException('Missing kernel modules: %s' % path) + return volume def stop(self, volume): pass + + +def _check_path(path): + ''' Raise an :py:class:`qubes.storage.StoragePoolException` if ``path`` does + not exist. + ''' + if not os.path.exists(path): + raise StoragePoolException('Missing file: %s' % path) diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 086fbba7..d971f14f 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -1532,18 +1532,6 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): self.storage.verify_files() - if not os.path.exists( - os.path.join(self.storage.kernels_dir, 'vmlinuz')): - raise qubes.exc.QubesException( - 'VM kernel does not exist: {0}'.format( - os.path.join(self.storage.kernels_dir, 'vmlinuz'))) - - if not os.path.exists( - os.path.join(self.storage.kernels_dir, 'initramfs')): - raise qubes.exc.QubesException( - 'VM initramfs does not exist: {0}'.format( - os.path.join(self.storage.kernels_dir, 'initramfs'))) - self.fire_event('domain-verify-files') return True From 2c2a778a1d43870193f1365d3e49d2b87a8275a1 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 1 Apr 2016 19:12:44 +0200 Subject: [PATCH 32/44] Serialize volume_config from qubes.xml --- qubes/vm/qubesvm.py | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index d971f14f..7aad4ac8 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -29,6 +29,7 @@ from __future__ import absolute_import import base64 import datetime import itertools +import lxml import os import os.path import re @@ -434,7 +435,16 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): def __init__(self, app, xml, volume_config={}, **kwargs): super(QubesVM, self).__init__(app, xml, **kwargs) if hasattr(self, 'volume_config'): - dict_merge(self.volume_config, volume_config) + if xml is not None: + for node in xml.xpath('volume-config/volume'): + name = node.get('name') + assert name + for k, v in node.items(): + self.volume_config[name][k] = v + + for name, conf in volume_config.items(): + for k, v in conf.items(): + self.volume_config[name][k] = v import qubes.vm.adminvm # pylint: disable=redefined-outer-name @@ -478,6 +488,18 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): self.events_enabled = True self.fire_event('domain-init') + def __xml__(self): + element = super(QubesVM, self).__xml__() + if hasattr(self, 'volumes'): + volume_config_node = lxml.etree.Element('volume-config') + for volume in self.volumes.values(): + volume_node = lxml.etree.Element('volume', **volume.config) + volume_config_node.append(volume_node) + + element.append(volume_config_node) + + + return element # # event handlers @@ -1734,20 +1756,3 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): # if self.is_qrexec_running(): # #TODO: kill qrexec daemon # pass - - -def dict_merge(dct, merge_dct): - """ Recursive dict merge. Inspired by :meth:``dict.update()``, instead of - updating only top-level keys, dict_merge recurses down into dicts nested - to an arbitrary depth, updating keys. The ``merge_dct`` is merged into - ``dct``. (Source https://gist.github.com/angstwad/bf22d1822c38a92ec0a9) - :param dct: dict onto which the merge is executed - :param merge_dct: dct merged into dct - :return: None - """ - for k, v in merge_dct.iteritems(): - if (k in dct and isinstance(dct[k], dict) - and isinstance(merge_dct[k], dict)): - dict_merge(dct[k], merge_dct[k]) - else: - dct[k] = merge_dct[k] From 8cc31e86a77bced0650c1db7b657736f5243aa1b Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 1 Apr 2016 20:02:39 +0200 Subject: [PATCH 33/44] qvm-create handle --pool argument --- doc/manpages/qvm-create.rst | 5 +++++ qubes/tools/qvm_create.py | 18 +++++++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/doc/manpages/qvm-create.rst b/doc/manpages/qvm-create.rst index c5ec3cee..c287b4b6 100644 --- a/doc/manpages/qvm-create.rst +++ b/doc/manpages/qvm-create.rst @@ -53,6 +53,10 @@ Options Use provided :file:`root.img` instead of default/empty one (file will be *moved*). This option is mutually exclusive with :option:`--root-copy-from`. +.. option:: --pool=POOL_NAME:VOLUME_NAME, -P POOL_NAME:VOLUME_NAME + + Specify the pool to use for a volume + Options for internal use ------------------------ @@ -71,5 +75,6 @@ Authors | Rafal Wojtczuk | Marek Marczykowski | Wojtek Porczyk +| Bahtiar `kalkin-` Gadimov .. vim: ts=3 sw=3 et tw=80 diff --git a/qubes/tools/qvm_create.py b/qubes/tools/qvm_create.py index 3ee077d6..83f1fb35 100644 --- a/qubes/tools/qvm_create.py +++ b/qubes/tools/qvm_create.py @@ -46,9 +46,10 @@ parser.add_argument('--property', '--prop', '-p', action=qubes.tools.PropertyAction, help='set domain\'s property, like "internal", "memory" or "vcpus"') -parser.add_argument('--pool-name', '--pool', '-P', - action=qubes.tools.SinglePropertyAction, - help='specify the storage pool to use') +parser.add_argument('--pool', '-P', + action='append', + metavar='POOL_NAME:VOLUME_NAME', + help='specify the pool to use for a volume') parser.add_argument('--template', '-t', action=qubes.tools.SinglePropertyAction, @@ -79,6 +80,17 @@ parser.add_argument('name', metavar='VMNAME', def main(args=None): args = parser.parse_args(args) + if args.pool: + args.properties['volume_config'] = {} + for pool_vol in args.pool: + try: + pool_name, volume_name = pool_vol.split(':') + config = {'pool': pool_name, 'name': volume_name} + args.properties['volume_config'][volume_name] = config + except ValueError: + parser.error( + 'Pool argument must be of form: -P pool_name:volume_name') + if 'label' not in args.properties: parser.error('--label option is mandatory') From d7fd66070aed88690c10690f3b02c90f852aae7b Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 15 Apr 2016 20:37:35 +0200 Subject: [PATCH 34/44] Fix revert template changes test --- qubes/tests/int/basic.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qubes/tests/int/basic.py b/qubes/tests/int/basic.py index 5a107fdc..e740eeb2 100644 --- a/qubes/tests/int/basic.py +++ b/qubes/tests/int/basic.py @@ -392,8 +392,9 @@ class TC_03_QvmRevertTemplateChanges(qubes.tests.SystemTestsMixin, self.save_and_reload_db() def get_rootimg_checksum(self): - p = subprocess.Popen(['sha1sum', self.test_template.root_img], - stdout=subprocess.PIPE) + p = subprocess.Popen( + ['sha1sum', self.test_template.volumes['root'].vid], + stdout=subprocess.PIPE) return p.communicate()[0] def _do_test(self): From e3ae6cdc1bbfedad13bb443ca29a3ad9a4d1e1e9 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Tue, 5 Apr 2016 18:58:21 +0200 Subject: [PATCH 35/44] BackupTestsMixin.create_backup_vms uses volumes Instead of using root_img to access the path it uses now the proper volumes --- qubes/tests/__init__.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index 1505a07d..538a02bb 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -820,7 +820,7 @@ class BackupTestsMixin(SystemTestsMixin): name=vmname, template=template, provides_network=True, label='red') testnet.create_on_disk() vms.append(testnet) - self.fill_image(testnet.private_img, 20*1024*1024) + self.fill_image(testnet.volumes['private'].vid, 20*1024*1024) vmname = self.make_vm_name('test1') if self.verbose: @@ -831,16 +831,17 @@ class BackupTestsMixin(SystemTestsMixin): testvm1.netvm = testnet testvm1.create_on_disk() vms.append(testvm1) - self.fill_image(testvm1.private_img, 100*1024*1024) + self.fill_image(testvm1.volumes['private'].vid, 100*1024*1024) vmname = self.make_vm_name('testhvm1') if self.verbose: print >>sys.stderr, "-> Creating %s" % vmname testvm2 = self.app.add_new_vm(qubes.vm.standalonevm.StandaloneVM, - name=vmname, - hvm=True, label='red') - testvm2.create_on_disk() - self.fill_image(testvm2.root_img, 1024*1024*1024, True) + name=vmname, + hvm=True, + label='red') + testvm2.create_on_disk(verbose=self.verbose) + self.fill_image(testvm2.volumes['root'].vid, 1024 * 1024 * 1024, True) vms.append(testvm2) vmname = self.make_vm_name('template') From 49b495138994991792c5fa23c856d7dc5c804447 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 15 Apr 2016 17:11:42 +0200 Subject: [PATCH 36/44] Storage move rename() logic to XenPool - Fix config renaming --- qubes/storage/__init__.py | 34 ++++++++++---------- qubes/storage/xen.py | 65 ++++++++++++++++++++++++++++++++++++++- qubes/vm/qubesvm.py | 12 ++++---- 3 files changed, 86 insertions(+), 25 deletions(-) diff --git a/qubes/storage/__init__.py b/qubes/storage/__init__.py index 308aaae8..a1dbb0f9 100644 --- a/qubes/storage/__init__.py +++ b/qubes/storage/__init__.py @@ -56,7 +56,11 @@ class Volume(object): script = None usage = 0 - def __init__(self, name=None, pool=None, volume_type=None, vid=None, + def __init__(self, + name=None, + pool=None, + volume_type=None, + vid=None, size=0): assert name and pool and volume_type self.name = str(name) @@ -167,23 +171,12 @@ class Storage(object): volume) self.vm.volumes[name] = volume - # TODO migrate this - @staticmethod - def rename(newpath, oldpath): - '''Move storage directory, most likely during domain's rename. - - .. note:: - The arguments are in different order than in :program:`cp` utility. - - .. versionchange:: 4.0 - This is now dummy method that just passes everything to - :py:func:`os.rename`. - - :param str newpath: New path - :param str oldpath: Old path - ''' - - os.rename(oldpath, newpath) + def rename(self, old_name, new_name): + ''' Notify the pools that the domain was renamed ''' + volumes = self.vm.volumes + for name, volume in volumes.items(): + pool = self.get_pool(volume) + volumes[name] = pool.rename(volume, old_name, new_name) def verify_files(self): '''Verify that the storage is sane. @@ -271,6 +264,11 @@ class Pool(object): raise NotImplementedError("Pool %s has remove() not implemented" % self.name) + def rename(self, volume, old_name, new_name): + ''' Called when the domain changes its name ''' + raise NotImplementedError("Pool %s has rename() not implemented" % + self.name) + def start(self, volume): ''' Do what ever is needed on start ''' raise NotImplementedError("Pool %s has start() not implemented" % diff --git a/qubes/storage/xen.py b/qubes/storage/xen.py index 7bb0dfb3..7af5ff63 100644 --- a/qubes/storage/xen.py +++ b/qubes/storage/xen.py @@ -114,6 +114,23 @@ class XenPool(Pool): _remove_if_exists(volume.vid) _remove_if_exists(volume.path_cow) + def rename(self, volume, old_name, new_name): + assert issubclass(volume.__class__, XenVolume) + old_dir = os.path.dirname(volume.path) + new_dir = os.path.join(os.path.dirname(old_dir), new_name) + + if not os.path.exists(new_dir): + os.makedirs(new_dir) + + if volume.volume_type == 'read-write': + volume.rename_target_dir(new_name, new_dir) + elif volume.volume_type == 'read-only': + volume.rename_target_dir(old_name, new_dir) + elif volume.volume_type in ['origin', 'volatile']: + volume.rename_target_dir(new_dir) + + return volume + def _resize_loop_device(self, path): # find loop device if any p = subprocess.Popen( @@ -275,6 +292,17 @@ class ReadWriteFile(SizeMixIn): self.path = os.path.join(self.target_dir, self.name + '.img') self.vid = self.path + def rename_target_dir(self, new_name, new_dir): + # :pylint: disable=unused-argument + old_path = self.path + file_name = os.path.basename(self.path) + new_path = os.path.join(new_dir, file_name) + + os.rename(old_path, new_path) + self.target_dir = new_dir + self.path = new_path + self.vid = self.path + class ReadOnlyFile(XenVolume): # :pylint: disable=missing-docstring @@ -285,6 +313,20 @@ class ReadOnlyFile(XenVolume): super(ReadOnlyFile, self).__init__(size=int(size), **kwargs) self.path = self.vid + def rename_target_dir(self, old_name, new_dir): + # only copy the read-only volume if it's "owned" by the current vm + # "owned" means that it's in a directory named the same as the vm + if os.path.basename(self.target_dir) == old_name: + file_name = os.path.basename(self.path) + new_path = os.path.join(new_dir, file_name) + old_path = self.path + + os.rename(old_path, new_path) + + self.target_dir = new_dir + self.path = new_path + self.vid = self.path + class OriginFile(SizeMixIn): # :pylint: disable=missing-docstring @@ -300,6 +342,19 @@ class OriginFile(SizeMixIn): def commit(self): raise NotImplementedError + def rename_target_dir(self, new_dir): + old_path_origin = self.path_origin + old_path_cow = self.path_cow + new_path_origin = os.path.join(new_dir, self.name + '.img') + new_path_cow = os.path.join(new_dir, self.name + '-cow.img') + os.rename(old_path_origin, new_path_origin) + os.rename(old_path_cow, new_path_cow) + self.target_dir = new_dir + self.path_origin = new_path_origin + self.path_cow = new_path_cow + self.path = '%s:%s' % (self.path_origin, self.path_cow) + self.vid = self.path_origin + @property def usage(self): result = 0 @@ -335,7 +390,15 @@ class VolatileFile(SizeMixIn): def __init__(self, **kwargs): super(VolatileFile, self).__init__(**kwargs) - self.path = os.path.join(self.target_dir, 'volatile.img') + self.path = os.path.join(self.target_dir, self.name + '.img') + self.vid = self.path + + def rename_target_dir(self, new_dir): + _remove_if_exists(self) + file_name = os.path.basename(self.path) + self.target_dir = new_dir + new_path = os.path.join(new_dir, file_name) + self.path = new_path self.vid = self.path diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 7aad4ac8..4a77b181 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -566,18 +566,18 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): self._qdb_connection.close() self._qdb_connection = None - self.storage.rename( - os.path.join(qubes.config.system_path['qubes_base_dir'], - self.dir_path_prefix, new_name), - os.path.join(qubes.config.system_path['qubes_base_dir'], - self.dir_path_prefix, old_name)) + self.storage.rename(old_name, new_name) + + prefix = os.path.join(qubes.config.system_path['qubes_base_dir'], self.dir_path_prefix) + old_config = os.path.join(prefix, old_name, old_name + '.conf') + new_config = os.path.join(prefix, new_name, new_name + '.conf') + os.rename(old_config, new_config) self._update_libvirt_domain() if self.autostart: self.autostart = self.autostart - @qubes.events.handler('property-pre-set:autostart') def on_property_pre_set_autostart(self, event, prop, name, value, oldvalue=None): From 2e28849c90150557d3e235659f509ea82b76817f Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 22 Apr 2016 14:16:41 +0200 Subject: [PATCH 37/44] Move pool xml config from Qubes to Pool --- qubes/__init__.py | 17 +++++++---------- qubes/storage/__init__.py | 5 +++++ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/qubes/__init__.py b/qubes/__init__.py index 22fe7c98..5d35e970 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -1319,7 +1319,13 @@ class Qubes(PropertyHolder): element = lxml.etree.Element('qubes') element.append(self.xml_labels()) - element.append(self.xml_pool_configs()) + + pools_xml = lxml.etree.Element('pools') + for pool in self.pools.values(): + pools_xml.append(pool.__xml__()) + + element.append(pools_xml) + element.append(self.xml_properties()) domains = lxml.etree.Element('domains') @@ -1422,15 +1428,6 @@ class Qubes(PropertyHolder): labels.append(label.__xml__()) return labels - def xml_pool_configs(self): - """ Helper for converting pools config to xml """ - pools_xml = lxml.etree.Element('pools') - for pool in self.pools.values(): - p = lxml.etree.Element('pool', **pool.config) - pools_xml.append(p) - - return pools_xml - def get_vm_class(self, clsname): '''Find the class for a domain. diff --git a/qubes/storage/__init__.py b/qubes/storage/__init__.py index a1dbb0f9..e940ebd8 100644 --- a/qubes/storage/__init__.py +++ b/qubes/storage/__init__.py @@ -37,6 +37,8 @@ import qubes.exc import qubes.utils from qubes.devices import BlockDevice +import lxml.etree + STORAGE_ENTRY_POINT = 'qubes.storage' @@ -231,6 +233,9 @@ class Pool(object): self.name = name kwargs['name'] = self.name + def __xml__(self): + return lxml.etree.Element('pool', **self.config) + def create(self, volume, source_volume): ''' Create the given volume on disk or copy from provided `source_volume`. From d7ff4b90577c6da516dd3a73fc28a63c36fec09f Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 22 Apr 2016 14:29:30 +0200 Subject: [PATCH 38/44] Move volume xml config from QubesVM to Volume --- qubes/storage/__init__.py | 3 +++ qubes/vm/qubesvm.py | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/qubes/storage/__init__.py b/qubes/storage/__init__.py index e940ebd8..56c9fbcd 100644 --- a/qubes/storage/__init__.py +++ b/qubes/storage/__init__.py @@ -71,6 +71,9 @@ class Volume(object): self.size = size self.volume_type = volume_type + def __xml__(self): + return lxml.etree.Element('volume', **self.config) + @property def config(self): ''' return config data for serialization to qubes.xml ''' diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 4a77b181..e0932292 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -493,8 +493,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): if hasattr(self, 'volumes'): volume_config_node = lxml.etree.Element('volume-config') for volume in self.volumes.values(): - volume_node = lxml.etree.Element('volume', **volume.config) - volume_config_node.append(volume_node) + volume_config_node.append(volume.__xml__()) element.append(volume_config_node) From 04536c59505b4559bfd9e23984bc15707052e01a Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 22 Apr 2016 14:39:04 +0200 Subject: [PATCH 39/44] Don't exec app.save() after add_pool & remove_pool --- qubes/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/qubes/__init__.py b/qubes/__init__.py index 5d35e970..d14dd520 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -1495,7 +1495,6 @@ class Qubes(PropertyHolder): pool = self._get_pool(**kwargs) pool.setup() self.pools[name] = pool - self.save() def remove_pool(self, name): """ Remove a storage pool from config file. """ @@ -1506,7 +1505,6 @@ class Qubes(PropertyHolder): except KeyError: return - self.save() def get_pool(self, name): ''' Returns a :py:class:`qubes.storage.Pool` instance ''' From 591134833b67fbf7e50d4c809f7ff7b5b41ae93a Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 22 Apr 2016 14:47:00 +0200 Subject: [PATCH 40/44] Replace Volume.__str__ with enhanced __repr__ --- qubes/storage/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/qubes/storage/__init__.py b/qubes/storage/__init__.py index 56c9fbcd..98aeddcf 100644 --- a/qubes/storage/__init__.py +++ b/qubes/storage/__init__.py @@ -81,8 +81,10 @@ class Volume(object): 'pool': self.pool, 'volume_type': self.volume_type} - def __str__(self): - return str({'name': self.name, 'pool': self.pool, 'vid': self.vid}) + def __repr__(self): + return '{}(name={!s}, pool={!r}, vid={!r}, volume_type={!r})'.format( + self.__class__.__name__, self.name, self.pool, self.vid, + self.volume_type) def block_device(self): ''' Return :py:class:`qubes.devices.BlockDevice` for serialization in From 8f060a87460219a271543e7c2a90e364d90db626 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 22 Apr 2016 14:58:21 +0200 Subject: [PATCH 41/44] Fix Pool and Volume __init__ --- qubes/storage/__init__.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/qubes/storage/__init__.py b/qubes/storage/__init__.py index 98aeddcf..5250b20c 100644 --- a/qubes/storage/__init__.py +++ b/qubes/storage/__init__.py @@ -58,13 +58,8 @@ class Volume(object): script = None usage = 0 - def __init__(self, - name=None, - pool=None, - volume_type=None, - vid=None, - size=0): - assert name and pool and volume_type + def __init__(self, name, pool, volume_type, vid=None, size=0, **kwargs): + super(Volume, self).__init__(**kwargs) self.name = str(name) self.pool = str(pool) self.vid = vid @@ -232,9 +227,8 @@ class Pool(object): private_img_size = qubes.config.defaults['private_img_size'] root_img_size = qubes.config.defaults['root_img_size'] - def __init__(self, name=None, **kwargs): - # :pylint: disable=unused-argument - assert name, "Pool name is missing" + def __init__(self, name, **kwargs): + super(Pool, self).__init__(**kwargs) self.name = name kwargs['name'] = self.name From 37ca33b0d14718193f3444bddeed18319f59bc66 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 22 Apr 2016 15:21:33 +0200 Subject: [PATCH 42/44] Add docstring to xen volumes implementations --- qubes/storage/xen.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/qubes/storage/xen.py b/qubes/storage/xen.py index 7af5ff63..bbbcb0a5 100644 --- a/qubes/storage/xen.py +++ b/qubes/storage/xen.py @@ -286,14 +286,14 @@ class SizeMixIn(XenVolume): class ReadWriteFile(SizeMixIn): - # :pylint: disable=missing-docstring + ''' Represents a readable & writable file image based volume ''' def __init__(self, **kwargs): super(ReadWriteFile, self).__init__(**kwargs) self.path = os.path.join(self.target_dir, self.name + '.img') self.vid = self.path def rename_target_dir(self, new_name, new_dir): - # :pylint: disable=unused-argument + ''' Called by :py:class:`XenPool` when a domain changes it's name ''' old_path = self.path file_name = os.path.basename(self.path) new_path = os.path.join(new_dir, file_name) @@ -305,17 +305,20 @@ class ReadWriteFile(SizeMixIn): class ReadOnlyFile(XenVolume): - # :pylint: disable=missing-docstring + ''' Represents a readonly file image based volume ''' usage = 0 def __init__(self, size=0, **kwargs): - # :pylint: disable=unused-argument super(ReadOnlyFile, self).__init__(size=int(size), **kwargs) self.path = self.vid def rename_target_dir(self, old_name, new_dir): - # only copy the read-only volume if it's "owned" by the current vm - # "owned" means that it's in a directory named the same as the vm + """ Called by :py:class:`XenPool` when a domain changes it's name. + + Only copies the volume if it belongs to the domain being renamed. + Currently if a volume is in a directory named the same as the domain, + it's ”owned” by the domain. + """ if os.path.basename(self.target_dir) == old_name: file_name = os.path.basename(self.path) new_path = os.path.join(new_dir, file_name) @@ -329,7 +332,11 @@ class ReadOnlyFile(XenVolume): class OriginFile(SizeMixIn): - # :pylint: disable=missing-docstring + ''' Represents a readable, writeable & snapshotable file image based volume. + + This is used for TemplateVM's + ''' + script = 'block-origin' def __init__(self, **kwargs): @@ -340,9 +347,11 @@ class OriginFile(SizeMixIn): self.vid = self.path_origin def commit(self): + ''' Commit Template changes ''' raise NotImplementedError def rename_target_dir(self, new_dir): + ''' Called by :py:class:`XenPool` when a domain changes it's name ''' old_path_origin = self.path_origin old_path_cow = self.path_cow new_path_origin = os.path.join(new_dir, self.name + '.img') @@ -366,7 +375,7 @@ class OriginFile(SizeMixIn): class SnapshotFile(XenVolume): - # :pylint: disable=missing-docstring + ''' Represents a readonly snapshot of an :py:class:`OriginFile` volume ''' script = 'block-snapshot' rw = False usage = 0 @@ -379,21 +388,18 @@ class SnapshotFile(XenVolume): self.path = '%s:%s' % (self.path_origin, self.path_cow) self.vid = self.path_origin - @property - def created(self): - return os.path.exists(self.path_origin) and os.path.exists( - self.path_cow) - class VolatileFile(SizeMixIn): - # :pylint: disable=missing-docstring - + ''' Represents a readable & writeable file based volume, which will be + discarded and recreated at each startup. + ''' def __init__(self, **kwargs): super(VolatileFile, self).__init__(**kwargs) self.path = os.path.join(self.target_dir, self.name + '.img') self.vid = self.path def rename_target_dir(self, new_dir): + ''' Called by :py:class:`XenPool` when a domain changes it's name ''' _remove_if_exists(self) file_name = os.path.basename(self.path) self.target_dir = new_dir From 29f4be0f10f8ff771b1611bf8fa0f9403a7d4ab2 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 22 Apr 2016 15:40:36 +0200 Subject: [PATCH 43/44] If vm doesnt support volume_config raise TypeError --- qubes/vm/qubesvm.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index e0932292..6b5135ec 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -445,10 +445,14 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): for name, conf in volume_config.items(): for k, v in conf.items(): self.volume_config[name][k] = v + elif volume_config: + raise TypeError( + 'volume_config specified, but {} did not expect that.' % + self.__class__.__name__) - import qubes.vm.adminvm # pylint: disable=redefined-outer-name + import qubes.vm.adminvm # pylint: disable=redefined-outer-name - #Init private attrs + # Init private attrs self._libvirt_domain = None self._qdb_connection = None From 04a3e803113fbc916b0b8fc3e7f59ee5789425dc Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Fri, 22 Apr 2016 16:25:55 +0200 Subject: [PATCH 44/44] SizeMixIn first assert than call super() --- qubes/storage/xen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes/storage/xen.py b/qubes/storage/xen.py index bbbcb0a5..2071e45e 100644 --- a/qubes/storage/xen.py +++ b/qubes/storage/xen.py @@ -267,9 +267,9 @@ class SizeMixIn(XenVolume): ''' def __init__(self, size=0, **kwargs): - super(SizeMixIn, self).__init__(size=int(size), **kwargs) assert size, 'Empty size provided' assert size > 0, 'Size for volume ' + kwargs['name'] + ' is <=0' + super(SizeMixIn, self).__init__(size=int(size), **kwargs) @property def usage(self):