From 7550fccf940822440cde1cdb7950ee051a17f118 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sat, 7 Nov 2015 16:54:03 +0100 Subject: [PATCH 01/19] Removed WNI --- core/settings-wni-Windows_NT.py | 45 ----------- core/storage/README.md | 3 + core/storage/wni.py | 138 -------------------------------- 3 files changed, 3 insertions(+), 183 deletions(-) delete mode 100644 core/settings-wni-Windows_NT.py create mode 100644 core/storage/README.md delete mode 100644 core/storage/wni.py diff --git a/core/settings-wni-Windows_NT.py b/core/settings-wni-Windows_NT.py deleted file mode 100644 index 6e646c9d..00000000 --- a/core/settings-wni-Windows_NT.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/python2 - -from __future__ import absolute_import -import _winreg -import os -import sys - -from qubes.storage.wni import QubesWniVmStorage - -DEFAULT_INSTALLDIR = 'c:\\program files\\Invisible Things Lab\\Qubes WNI' -DEFAULT_STOREDIR = 'c:\\qubes' - -def apply(system_path, vm_files, defaults): - system_path['qubes_base_dir'] = DEFAULT_STOREDIR - installdir = DEFAULT_INSTALLDIR - try: - reg_key = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, - "Software\\Invisible Things Lab\\Qubes WNI") - installdir = _winreg.QueryValueEx(reg_key, "InstallDir")[0] - system_path['qubes_base_dir'] = \ - _winreg.QueryValueEx(reg_key, "StoreDir")[0] - except WindowsError as e: - print >>sys.stderr, \ - "WARNING: invalid installation: missing registry entries (%s)" \ - % str(e) - - system_path['config_template_pv'] = \ - os.path.join(installdir, 'vm-template.xml') - system_path['config_template_hvm'] = \ - os.path.join(installdir, 'vm-template-hvm.xml') - system_path['qubes_icon_dir'] = os.path.join(installdir, 'icons') - system_path['qubesdb_daemon_path'] = \ - os.path.join(installdir, 'bin\\qubesdb-daemon.exe') - system_path['qrexec_daemon_path'] = \ - os.path.join(installdir, 'bin\\qrexec-daemon.exe') - system_path['qrexec_client_path'] = \ - os.path.join(installdir, 'bin\\qrexec-client.exe') - system_path['qrexec_policy_dir'] = \ - os.path.join(installdir, 'qubes-rpc\\policy') - # Specific to WNI - normally VM have this file - system_path['qrexec_agent_path'] = \ - os.path.join(installdir, 'bin\\qrexec-agent.exe') - - defaults['libvirt_uri'] = 'wni:///' - defaults['storage_class'] = QubesWniVmStorage diff --git a/core/storage/README.md b/core/storage/README.md new file mode 100644 index 00000000..7258512f --- /dev/null +++ b/core/storage/README.md @@ -0,0 +1,3 @@ +# WNI File storage +Before v3.1 there existed a draft wni storage. You can find it in the git +history diff --git a/core/storage/wni.py b/core/storage/wni.py deleted file mode 100644 index a3571765..00000000 --- a/core/storage/wni.py +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/python2 -# -# The Qubes OS Project, http://www.qubes-os.org -# -# Copyright (C) 2013 Marek Marczykowski -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# - -from __future__ import absolute_import - -import sys -import os -import os.path -import win32api -import win32net -import win32netcon -import win32security -import win32profile -import pywintypes -import random - -from qubes.storage import QubesVmStorage -from qubes.qubes import QubesException,system_path - -class QubesWniVmStorage(QubesVmStorage): - """ - Class for VM storage of WNI VMs. - """ - - def __init__(self, *args, **kwargs): - super(QubesWniVmStorage, self).__init__(*args, **kwargs) - # Use the user profile as "private.img" - self.home_root = win32profile.GetProfilesDirectory() - # FIXME: the assignment below may not always be correct, - # but GetUserProfileDirectory needs a user token... - self.private_img = os.path.join(self.home_root, self._get_username()) - - # Pass paths for WNI libvirt driver - os.putenv("WNI_DRIVER_QUBESDB_PATH", system_path['qubesdb_daemon_path']) - os.putenv("WNI_DRIVER_QREXEC_AGENT_PATH", system_path['qrexec_agent_path']) - - def _get_username(self, vmname = None): - if vmname is None: - vmname = self.vm.name - return "qubes-vm-%s" % vmname - - def _get_random_password(self, vmname = None): - if vmname is None: - vmname = self.vm.name - return '%x' % random.SystemRandom().getrandombits(256) - - def get_config_params(self): - return {} - - def create_on_disk_private_img(self, verbose, source_template = None): - # FIXME: this may not always be correct - home_dir = os.path.join(self.home_root, self._get_username()) - # Create user data in information level 1 (PyUSER_INFO_1) format. - user_data = {} - user_data['name'] = self._get_username() - user_data['full_name'] = self._get_username() - # libvirt driver doesn't need to know the password anymore - user_data['password'] = self._get_random_password() - user_data['flags'] = ( - win32netcon.UF_NORMAL_ACCOUNT | - win32netcon.UF_SCRIPT | - win32netcon.UF_DONT_EXPIRE_PASSWD | - win32netcon.UF_PASSWD_CANT_CHANGE - ) - user_data['priv'] = win32netcon.USER_PRIV_USER - user_data['home_dir'] = home_dir - user_data['max_storage'] = win32netcon.USER_MAXSTORAGE_UNLIMITED - # TODO: catch possible exception - win32net.NetUserAdd(None, 1, user_data) - - def create_on_disk_root_img(self, verbose, source_template = None): - pass - - def remove_from_disk(self): - try: - sid = win32security.LookupAccountName(None, self._get_username())[0] - string_sid = win32security.ConvertSidToStringSid(sid) - win32profile.DeleteProfile(string_sid) - win32net.NetUserDel(None, self._get_username()) - except pywintypes.error, details: - if details[0] == 2221: - # "The user name cannot be found." - raise IOError("User %s doesn't exist" % self._get_username()) - else: - raise - - super(QubesWniVmStorage, self).remove_from_disk() - - def rename(self, old_name, new_name): - super(QubesWniVmStorage, self).rename(old_name, new_name) - user_data = {} - user_data['name'] = self._get_username(new_name) - win32net.NetUserSetInfo(None, - self._get_username(old_name), 0, user_data) - #TODO: rename user profile - - def verify_files(self): - if not os.path.exists (self.vmdir): - raise QubesException ( - "VM directory doesn't exist: {0}".\ - format(self.vmdir)) - - try: - # TemplateVm in WNI is quite virtual, so do not require the user - if not self.vm.is_template(): - win32net.NetUserGetInfo(None, self._get_username(), 0) - except pywintypes.error, details: - if details[0] == 2221: - # "The user name cannot be found." - raise QubesException("User %s doesn't exist" % self._get_username()) - else: - raise - - def reset_volatile_storage(self, verbose = False, source_template = None): - pass - - def prepare_for_vm_startup(self, verbose = False): - if self.vm.is_template(): - raise QubesException("Starting TemplateVM is not supported") From 15d5e6edbb1222d79b2deb3fa68f299ad2c76c2a Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sat, 7 Nov 2015 18:58:49 +0100 Subject: [PATCH 02/19] Add Pool & XenPool --- core/settings-xen-Linux.py | 4 +++- core/storage/__init__.py | 13 ++++++++----- core/storage/xen.py | 12 ++++++++---- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/core/settings-xen-Linux.py b/core/settings-xen-Linux.py index de5084f5..8b23b447 100644 --- a/core/settings-xen-Linux.py +++ b/core/settings-xen-Linux.py @@ -2,7 +2,9 @@ from __future__ import absolute_import -from qubes.storage.xen import QubesXenVmStorage +from qubes.storage.xen import QubesXenVmStorage, XenPool + def apply(system_path, vm_files, defaults): defaults['storage_class'] = QubesXenVmStorage + defaults['pool_types'] = {'xen': XenPool} diff --git a/core/storage/__init__.py b/core/storage/__init__.py index 29cf8654..795aa37a 100644 --- a/core/storage/__init__.py +++ b/core/storage/__init__.py @@ -16,22 +16,21 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. # from __future__ import absolute_import import os import os.path -import re import shutil import subprocess import sys -from qubes.qubes import vm_files,system_path,defaults -from qubes.qubes import QubesException import qubes.qubesutils +from qubes.qubes import QubesException, defaults, system_path, vm_files + class QubesVmStorage(object): """ @@ -199,3 +198,7 @@ class QubesVmStorage(object): print >>sys.stderr, "WARNING: Creating empty VM private image file: {0}".\ format(self.private_img) self.create_on_disk_private_img(verbose=False) + + +class Pool(object): + pass diff --git a/core/storage/xen.py b/core/storage/xen.py index 00c3b3da..f66126a6 100644 --- a/core/storage/xen.py +++ b/core/storage/xen.py @@ -16,20 +16,20 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. # from __future__ import absolute_import import os import os.path +import re import subprocess import sys -import re -from qubes.storage import QubesVmStorage from qubes.qubes import QubesException, vm_files +from qubes.storage import Pool, QubesVmStorage class QubesXenVmStorage(QubesVmStorage): @@ -249,3 +249,7 @@ class QubesXenVmStorage(QubesVmStorage): raise QubesException( "VM '{}' holding '{}' does not exists".format( drive_domain, drive_path)) + + +class XenPool(Pool): + pass From bfaf37dae534d0266c4e3a56808cd11ba0c8c0ef Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sat, 7 Nov 2015 19:01:25 +0100 Subject: [PATCH 03/19] Add pool config parsing --- core/storage/Makefile | 3 ++ core/storage/__init__.py | 95 +++++++++++++++++++++++++++++++++++++++ core/storage/storage.conf | 11 +++++ rpm_spec/core-dom0.spec | 1 + tests/Makefile | 2 + tests/__init__.py | 1 + tests/storage.py | 63 ++++++++++++++++++++++++++ 7 files changed, 176 insertions(+) create mode 100644 core/storage/storage.conf create mode 100644 tests/storage.py diff --git a/core/storage/Makefile b/core/storage/Makefile index ec59cc64..7c7af60e 100644 --- a/core/storage/Makefile +++ b/core/storage/Makefile @@ -1,5 +1,6 @@ OS ?= Linux +SYSCONFDIR ?= /etc PYTHON_QUBESPATH = $(PYTHON_SITEPATH)/qubes all: @@ -13,6 +14,8 @@ endif mkdir -p $(DESTDIR)$(PYTHON_QUBESPATH)/storage cp __init__.py $(DESTDIR)$(PYTHON_QUBESPATH)/storage cp __init__.py[co] $(DESTDIR)$(PYTHON_QUBESPATH)/storage + mkdir -p $(DESTDIR)$(SYSCONFDIR)/qubes + cp storage.conf $(DESTDIR)$(SYSCONFDIR)/qubes/ ifneq ($(BACKEND_VMM),) if [ -r $(BACKEND_VMM).py ]; then \ cp $(BACKEND_VMM).py $(DESTDIR)$(PYTHON_QUBESPATH)/storage && \ diff --git a/core/storage/__init__.py b/core/storage/__init__.py index 795aa37a..b9810881 100644 --- a/core/storage/__init__.py +++ b/core/storage/__init__.py @@ -22,6 +22,7 @@ from __future__ import absolute_import +import ConfigParser import os import os.path import shutil @@ -31,6 +32,8 @@ import sys import qubes.qubesutils from qubes.qubes import QubesException, defaults, system_path, vm_files +CONFIG_FILE = '/etc/qubes/storage.conf' + class QubesVmStorage(object): """ @@ -200,5 +203,97 @@ class QubesVmStorage(object): self.create_on_disk_private_img(verbose=False) +def dump(o): + """ Returns a string represention of the given object + + Args: + o (object): anything that response to `__module__` and `__class__` + + Given the class :class:`qubes.storage.QubesVmStorage` it returns + 'qubes.storage.QubesVmStorage' as string + """ + return o.__module__ + '.' + o.__class__.__name__ + + +def load(string): + """ Given a dotted full module string representation of a class it loads it + + Args: + string (str) i.e. 'qubes.storage.xen.QubesXenVmStorage' + + Returns: + type + + See also: + :func:`qubes.storage.dump` + """ + if not type(string) is str: + # This is a hack which allows giving a real class to a vm instead of a + # string as string_class parameter. + return string + + components = string.split(".") + module_path = ".".join(components[:-1]) + klass = components[-1:][0] + module = __import__(module_path, fromlist=[klass]) + return getattr(module, klass) + + +def get_pool(vm): + """ Instantiates the storage for the specified vm """ + config = _get_storage_config_parser() + + name = vm.storage_pool + klass = _get_pool_klass(name, config) + + keys = [k for k in config.options(name) if k != 'type' and k != 'class'] + values = [config.get(name, o) for o in keys] + kwargs = dict(zip(keys, values)) + return klass(vm, **kwargs) + + +def _get_storage_config_parser(): + """ Instantiates a `ConfigParaser` for specified storage config file. + + Returns: + RawConfigParser + """ + config = ConfigParser.RawConfigParser() + config.read(CONFIG_FILE) + return config + + +def _get_pool_klass(name, config=None): + """ Returns the storage klass for the specified pool. + + Args: + name: The pool name. + config: If ``config`` is not specified + `_get_storage_config_parser()` is called. + + Returns: + type: A class inheriting from `QubesVmStorage` + """ + if config is None: + config = _get_storage_config_parser() + + if not config.has_section(name): + raise StoragePoolException('Uknown storage pool ' + name) + + if config.has_option(name, 'class'): + klass = load(config.get(name, 'class')) + elif config.has_option(name, 'type'): + pool_type = config.get(name, 'type') + klass = defaults['pool_types'][pool_type] + + if klass is None: + raise StoragePoolException('Uknown storage pool type ' + name) + return klass + + +class StoragePoolException(QubesException): + pass + class Pool(object): pass + diff --git a/core/storage/storage.conf b/core/storage/storage.conf new file mode 100644 index 00000000..8fa44c7e --- /dev/null +++ b/core/storage/storage.conf @@ -0,0 +1,11 @@ +[default] ; poolname +type=xen ; the default xen storage +; class = qubes.storage.xen.XenStorage ; class always overwrites type +; +; To use our own storage adapter, you need just to specify the module path and +; class name +; [pool-b] +; class = foo.bar.MyStorage +; + + diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec index 86612c48..84e106ba 100644 --- a/rpm_spec/core-dom0.spec +++ b/rpm_spec/core-dom0.spec @@ -176,6 +176,7 @@ fi %files %defattr(-,root,root,-) %config(noreplace) %attr(0664,root,qubes) %{_sysconfdir}/qubes/qmemman.conf +%config(noreplace) %attr(0664,root,qubes) %{_sysconfdir}/qubes/storage.conf /usr/bin/qvm-* /usr/bin/qubes-* %dir %{python_sitearch}/qubes diff --git a/tests/Makefile b/tests/Makefile index 8523adde..f2dff357 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -27,3 +27,5 @@ endif cp regressions.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) cp run.py $(DESTDIR)$(PYTHON_TESTSPATH) cp run.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) + cp storage.py $(DESTDIR)$(PYTHON_TESTSPATH) + cp storage.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) diff --git a/tests/__init__.py b/tests/__init__.py index 8abea4b8..8af1afbf 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -547,6 +547,7 @@ def load_tests(loader, tests, pattern): 'qubes.tests.backup', 'qubes.tests.backupcompatibility', 'qubes.tests.regressions', + 'qubes.tests.storage', ): tests.addTests(loader.loadTestsFromName(modname)) diff --git a/tests/storage.py b/tests/storage.py new file mode 100644 index 00000000..30b9cd2d --- /dev/null +++ b/tests/storage.py @@ -0,0 +1,63 @@ +# The Qubes OS Project, https://www.qubes-os.org/ +# +# Copyright (C) 2015 Bahtiar `kalkin-` Gadimov +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +from qubes.tests import QubesTestCase, SystemTestsMixin +from qubes.qubes import defaults + +import qubes.storage +from qubes.storage.xen import QubesXenVmStorage, XenPool + + +class TC_00_Storage(SystemTestsMixin, QubesTestCase): + + def test_000_dump(self): + """ Dumps storage instance to a storage string """ + vmname = self.make_vm_name('appvm') + template = self.qc.get_default_template() + storage = self.qc.add_new_vm('QubesAppVm', name=vmname, pool='default', + template=template).storage + result = qubes.storage.dump(storage) + expected = 'qubes.storage.xen.QubesXenVmStorage' + self.assertEquals(result, expected) + + def test_001_load(self): + """ Loads storage type from a storage string """ + result = qubes.storage.load('qubes.storage.xen.QubesXenVmStorage') + self.assertTrue(result is QubesXenVmStorage) + + def test_002_default_pool_types(self): + """ The only predifined pool type is xen """ + result = defaults['pool_types'].keys() + expected = ["xen"] + self.assertEquals(result, expected) + + def test_003_get_pool_klass(self): + """ Expect the default pool to be `XenPool` """ + result = qubes.storage._get_pool_klass('default') + self.assertTrue(result is XenPool) + + +class TC_01_Storage(SystemTestsMixin, QubesTestCase): + + def test_000_vm_use_default_pool(self): + vmname = self.make_vm_name('appvm') + template = self.qc.get_default_template() + vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template, + pool='default') + self.assertIsInstance(vm.storage, QubesXenVmStorage) From 8e0207a199f9e7a3503646acef2fac4f21e1a945 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sat, 7 Nov 2015 19:04:51 +0100 Subject: [PATCH 04/19] Rename QubesXenVmStorage to XenStorage --- core/settings-xen-Linux.py | 4 ++-- core/storage/xen.py | 10 +++++----- tests/storage.py | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/core/settings-xen-Linux.py b/core/settings-xen-Linux.py index 8b23b447..30669407 100644 --- a/core/settings-xen-Linux.py +++ b/core/settings-xen-Linux.py @@ -2,9 +2,9 @@ from __future__ import absolute_import -from qubes.storage.xen import QubesXenVmStorage, XenPool +from qubes.storage.xen import XenStorage, XenPool def apply(system_path, vm_files, defaults): - defaults['storage_class'] = QubesXenVmStorage + defaults['storage_class'] = XenStorage defaults['pool_types'] = {'xen': XenPool} diff --git a/core/storage/xen.py b/core/storage/xen.py index f66126a6..9c1a75cc 100644 --- a/core/storage/xen.py +++ b/core/storage/xen.py @@ -32,13 +32,13 @@ from qubes.qubes import QubesException, vm_files from qubes.storage import Pool, QubesVmStorage -class QubesXenVmStorage(QubesVmStorage): +class XenStorage(QubesVmStorage): """ Class for VM storage of Xen VMs. """ def __init__(self, vm, **kwargs): - super(QubesXenVmStorage, self).__init__(vm, **kwargs) + super(XenStorage, self).__init__(vm, **kwargs) self.root_dev = "xvda" self.private_dev = "xvdb" @@ -158,7 +158,7 @@ class QubesXenVmStorage(QubesVmStorage): self.commit_template_changes() def rename(self, old_name, new_name): - super(QubesXenVmStorage, self).rename(old_name, new_name) + super(XenStorage, self).rename(old_name, new_name) old_dirpath = os.path.join(os.path.dirname(self.vmdir), old_name) if self.rootcow_img: @@ -226,11 +226,11 @@ class QubesXenVmStorage(QubesVmStorage): f_volatile.close() f_root.close() return - super(QubesXenVmStorage, self).reset_volatile_storage( + super(XenStorage, self).reset_volatile_storage( verbose=verbose, source_template=source_template) def prepare_for_vm_startup(self, verbose): - super(QubesXenVmStorage, self).prepare_for_vm_startup(verbose=verbose) + super(XenStorage, self).prepare_for_vm_startup(verbose=verbose) if self.drive is not None: (drive_type, drive_domain, drive_path) = self.drive.split(":") diff --git a/tests/storage.py b/tests/storage.py index 30b9cd2d..a52b62b4 100644 --- a/tests/storage.py +++ b/tests/storage.py @@ -21,7 +21,7 @@ from qubes.tests import QubesTestCase, SystemTestsMixin from qubes.qubes import defaults import qubes.storage -from qubes.storage.xen import QubesXenVmStorage, XenPool +from qubes.storage.xen import XenStorage, XenPool class TC_00_Storage(SystemTestsMixin, QubesTestCase): @@ -33,13 +33,13 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase): storage = self.qc.add_new_vm('QubesAppVm', name=vmname, pool='default', template=template).storage result = qubes.storage.dump(storage) - expected = 'qubes.storage.xen.QubesXenVmStorage' + expected = 'qubes.storage.xen.XenStorage' self.assertEquals(result, expected) def test_001_load(self): """ Loads storage type from a storage string """ - result = qubes.storage.load('qubes.storage.xen.QubesXenVmStorage') - self.assertTrue(result is QubesXenVmStorage) + result = qubes.storage.load('qubes.storage.xen.XenStorage') + self.assertTrue(result is XenStorage) def test_002_default_pool_types(self): """ The only predifined pool type is xen """ @@ -60,4 +60,4 @@ class TC_01_Storage(SystemTestsMixin, QubesTestCase): template = self.qc.get_default_template() vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template, pool='default') - self.assertIsInstance(vm.storage, QubesXenVmStorage) + self.assertIsInstance(vm.storage, XenStorage) From d1685a13df6a9114c883f35c3908db9f1e11e511 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sat, 7 Nov 2015 19:32:03 +0100 Subject: [PATCH 05/19] Add storage.pool_exists --- core/storage/__init__.py | 9 +++++++++ tests/storage.py | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/core/storage/__init__.py b/core/storage/__init__.py index b9810881..aabb2f65 100644 --- a/core/storage/__init__.py +++ b/core/storage/__init__.py @@ -291,6 +291,15 @@ def _get_pool_klass(name, config=None): return klass +def pool_exists(name): + """ Check if the specified pool exists """ + try: + _get_pool_klass(name) + return True + except StoragePoolException: + return False + + class StoragePoolException(QubesException): pass diff --git a/tests/storage.py b/tests/storage.py index a52b62b4..e3fb4b29 100644 --- a/tests/storage.py +++ b/tests/storage.py @@ -52,6 +52,15 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase): result = qubes.storage._get_pool_klass('default') self.assertTrue(result is XenPool) + def test_004_pool_exists_default(self): + """ Expect the default pool to exists """ + self.assertTrue(qubes.storage.pool_exists('default')) + + def test_005_pool_exists_random(self): + """ Expect this pool to not a exist """ + self.assertFalse( + qubes.storage.pool_exists('asdh312096r832598213iudhas')) + class TC_01_Storage(SystemTestsMixin, QubesTestCase): From 78891dd70f344bcdca2faa5f7145d642e620cc88 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sat, 7 Nov 2015 19:32:38 +0100 Subject: [PATCH 06/19] QubesVm save the name of the storage pool used --- core-modules/000QubesVm.py | 7 ++++--- core/storage/__init__.py | 3 +-- core/storage/xen.py | 8 ++++++-- tests/storage.py | 4 ++-- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py index f8fdd76c..615652db 100644 --- a/core-modules/000QubesVm.py +++ b/core-modules/000QubesVm.py @@ -40,12 +40,12 @@ import signal from qubes import qmemman from qubes import qmemman_algo import libvirt -import warnings from qubes.qubes import dry_run,vmm from qubes.qubes import register_qubes_vm_class from qubes.qubes import QubesVmCollection,QubesException,QubesHost,QubesVmLabels from qubes.qubes import defaults,system_path,vm_files,qubes_max_qid +from qubes.storage import get_pool qmemman_present = False try: @@ -109,6 +109,7 @@ class QubesVm(object): "name": { "order": 1 }, "uuid": { "order": 0, "eval": 'uuid.UUID(value) if value else None' }, "dir_path": { "default": None, "order": 2 }, + "pool_name": { "default":"default" }, "conf_file": { "func": lambda value: self.absolute_path(value, self.name + ".conf"), @@ -198,7 +199,7 @@ class QubesVm(object): 'kernelopts', 'services', 'installed_by_rpm',\ 'uses_default_netvm', 'include_in_backups', 'debug',\ 'qrexec_timeout', 'autostart', 'uses_default_dispvm_netvm', - 'backup_content', 'backup_size', 'backup_path' ]: + 'backup_content', 'backup_size', 'backup_path', 'pool_name' ]: attrs[prop]['save'] = lambda prop=prop: str(getattr(self, prop)) # Simple paths for prop in ['conf_file', 'firewall_conf']: @@ -345,7 +346,7 @@ class QubesVm(object): self.services['qubes-update-check'] = False # Initialize VM image storage class - self.storage = defaults["storage_class"](self) + self.storage = get_pool(self.pool_name, self).getStorage() if hasattr(self, 'kernels_dir'): modules_path = os.path.join(self.kernels_dir, "modules.img") diff --git a/core/storage/__init__.py b/core/storage/__init__.py index aabb2f65..4636dc47 100644 --- a/core/storage/__init__.py +++ b/core/storage/__init__.py @@ -239,11 +239,10 @@ def load(string): return getattr(module, klass) -def get_pool(vm): +def get_pool(name, vm): """ Instantiates the storage for the specified vm """ config = _get_storage_config_parser() - name = vm.storage_pool klass = _get_pool_klass(name, config) keys = [k for k in config.options(name) if k != 'type' and k != 'class'] diff --git a/core/storage/xen.py b/core/storage/xen.py index 9c1a75cc..01ed0277 100644 --- a/core/storage/xen.py +++ b/core/storage/xen.py @@ -28,7 +28,7 @@ import re import subprocess import sys -from qubes.qubes import QubesException, vm_files +from qubes.qubes import QubesException, defaults, vm_files from qubes.storage import Pool, QubesVmStorage @@ -252,4 +252,8 @@ class XenStorage(QubesVmStorage): class XenPool(Pool): - pass + def __init__(self, vm): + self.vm = vm + + def getStorage(self): + return defaults['storage_class'](self.vm) diff --git a/tests/storage.py b/tests/storage.py index e3fb4b29..3115535d 100644 --- a/tests/storage.py +++ b/tests/storage.py @@ -30,7 +30,7 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase): """ Dumps storage instance to a storage string """ vmname = self.make_vm_name('appvm') template = self.qc.get_default_template() - storage = self.qc.add_new_vm('QubesAppVm', name=vmname, pool='default', + storage = self.qc.add_new_vm('QubesAppVm', name=vmname, pool_name='default', template=template).storage result = qubes.storage.dump(storage) expected = 'qubes.storage.xen.XenStorage' @@ -68,5 +68,5 @@ class TC_01_Storage(SystemTestsMixin, QubesTestCase): vmname = self.make_vm_name('appvm') template = self.qc.get_default_template() vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template, - pool='default') + pool_name='default') self.assertIsInstance(vm.storage, XenStorage) From 26711e7e9aa64b18d9b8e00e567c6dc4e2f86992 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sat, 7 Nov 2015 20:19:17 +0100 Subject: [PATCH 07/19] Use default pool_config if pool name is 'default' --- core/settings-xen-Linux.py | 1 + core/storage/__init__.py | 9 ++++++++- core/storage/xen.py | 6 +++++- tests/storage.py | 34 +++++++++++++++++++++++++++------- 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/core/settings-xen-Linux.py b/core/settings-xen-Linux.py index 30669407..74503524 100644 --- a/core/settings-xen-Linux.py +++ b/core/settings-xen-Linux.py @@ -8,3 +8,4 @@ from qubes.storage.xen import XenStorage, XenPool def apply(system_path, vm_files, defaults): defaults['storage_class'] = XenStorage defaults['pool_types'] = {'xen': XenPool} + defaults['pool_config'] = {'dir': '/var/lib/qubes/'} diff --git a/core/storage/__init__.py b/core/storage/__init__.py index 4636dc47..39f623d5 100644 --- a/core/storage/__init__.py +++ b/core/storage/__init__.py @@ -247,7 +247,14 @@ def get_pool(name, vm): keys = [k for k in config.options(name) if k != 'type' and k != 'class'] values = [config.get(name, o) for o in keys] - kwargs = dict(zip(keys, values)) + config_kwargs = dict(zip(keys, values)) + + if name == 'default': + kwargs = defaults['pool_config'].copy() + kwargs.update(keys) + else: + kwargs = config_kwargs + return klass(vm, **kwargs) diff --git a/core/storage/xen.py b/core/storage/xen.py index 01ed0277..06bd0186 100644 --- a/core/storage/xen.py +++ b/core/storage/xen.py @@ -252,8 +252,12 @@ class XenStorage(QubesVmStorage): class XenPool(Pool): - def __init__(self, vm): + def __init__(self, vm, dir): + assert vm is not None + assert dir is not None + self.vm = vm + self.dir = dir def getStorage(self): return defaults['storage_class'](self.vm) diff --git a/tests/storage.py b/tests/storage.py index 3115535d..b5a0defc 100644 --- a/tests/storage.py +++ b/tests/storage.py @@ -30,8 +30,9 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase): """ Dumps storage instance to a storage string """ vmname = self.make_vm_name('appvm') template = self.qc.get_default_template() - storage = self.qc.add_new_vm('QubesAppVm', name=vmname, pool_name='default', - template=template).storage + vm = self.qc.add_new_vm('QubesAppVm', name=vmname, + pool_name='default', template=template) + storage = vm.storage result = qubes.storage.dump(storage) expected = 'qubes.storage.xen.XenStorage' self.assertEquals(result, expected) @@ -62,11 +63,30 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase): qubes.storage.pool_exists('asdh312096r832598213iudhas')) -class TC_01_Storage(SystemTestsMixin, QubesTestCase): +class TC_00_Pool(SystemTestsMixin, QubesTestCase): - def test_000_vm_use_default_pool(self): + def test000_no_pool_dir(self): + """ If no pool dir ist configured for a ``XenPool`` assume the default + `/var/lib/qubes/`. + """ + vm = self._init_app_vm() + result = qubes.storage.get_pool("default", vm).dir + expected = '/var/lib/qubes/' + self.assertEquals(result, expected) + + def test001_default_storage_class(self): + """ Check if when using default pool the Storage is ``XenStorage``. """ + result = self._init_app_vm().storage + self.assertIsInstance(result, XenStorage) + + def test_002_pool_name(self): + """ Default pool_name is 'default'. """ + vm = self._init_app_vm() + self.assertEquals(vm.pool_name, "default") + + def _init_app_vm(self): + """ Return initalised, but not created, AppVm. """ vmname = self.make_vm_name('appvm') template = self.qc.get_default_template() - vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template, - pool_name='default') - self.assertIsInstance(vm.storage, XenStorage) + return self.qc.add_new_vm('QubesAppVm', name=vmname, template=template, + pool_name='default') From 16d480cf4c7dce683e48739bd581638e791eb8c1 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sat, 7 Nov 2015 20:53:50 +0100 Subject: [PATCH 08/19] Add storage add_pool & remove_pool --- core/storage/__init__.py | 34 ++++++++++++++++++++++++++-------- core/storage/storage.conf | 5 +++-- tests/storage.py | 21 +++++++++++++++++---- 3 files changed, 46 insertions(+), 14 deletions(-) diff --git a/core/storage/__init__.py b/core/storage/__init__.py index 39f623d5..ef8b27e0 100644 --- a/core/storage/__init__.py +++ b/core/storage/__init__.py @@ -258,6 +258,32 @@ def get_pool(name, vm): return klass(vm, **kwargs) +def pool_exists(name): + """ Check if the specified pool exists """ + try: + _get_pool_klass(name) + return True + except StoragePoolException: + return False + +def add_pool(name, **kwargs): + """ Add a storage pool to config.""" + config = _get_storage_config_parser() + config.add_section(name) + for key, value in kwargs.iteritems(): + config.set(name, key, value) + _write_config(config) + +def remove_pool(name): + """ Remove a storage pool from config file. """ + config = _get_storage_config_parser() + config.remove_section(name) + _write_config(config) + +def _write_config(config): + with open(CONFIG_FILE, 'w') as configfile: + config.write(configfile) + def _get_storage_config_parser(): """ Instantiates a `ConfigParaser` for specified storage config file. @@ -297,14 +323,6 @@ def _get_pool_klass(name, config=None): return klass -def pool_exists(name): - """ Check if the specified pool exists """ - try: - _get_pool_klass(name) - return True - except StoragePoolException: - return False - class StoragePoolException(QubesException): pass diff --git a/core/storage/storage.conf b/core/storage/storage.conf index 8fa44c7e..45b6bda0 100644 --- a/core/storage/storage.conf +++ b/core/storage/storage.conf @@ -6,6 +6,7 @@ type=xen ; the default xen storage ; class name ; [pool-b] ; class = foo.bar.MyStorage -; - +; +; [test-dummy] +; type=dummy diff --git a/tests/storage.py b/tests/storage.py index b5a0defc..d016fb1a 100644 --- a/tests/storage.py +++ b/tests/storage.py @@ -17,11 +17,11 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -from qubes.tests import QubesTestCase, SystemTestsMixin -from qubes.qubes import defaults - import qubes.storage -from qubes.storage.xen import XenStorage, XenPool +from qubes.qubes import defaults +from qubes.tests import QubesTestCase, SystemTestsMixin + +from qubes.storage.xen import XenPool, XenStorage class TC_00_Storage(SystemTestsMixin, QubesTestCase): @@ -62,6 +62,19 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase): self.assertFalse( qubes.storage.pool_exists('asdh312096r832598213iudhas')) + def test_006_add_remove_pool(self): + """ Tries to adding and removing a pool. """ + pool_name = 'asdjhrp89132' + + # make sure it's really does not exist + qubes.storage.remove_pool(pool_name) + + qubes.storage.add_pool(pool_name, type='xen') + self.assertTrue(qubes.storage.pool_exists(pool_name)) + + qubes.storage.remove_pool(pool_name) + self.assertFalse(qubes.storage.pool_exists(pool_name)) + class TC_00_Pool(SystemTestsMixin, QubesTestCase): From 58f23ca3923e6521c278b0a20f95a9d2464e3dd4 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sat, 7 Nov 2015 22:31:50 +0100 Subject: [PATCH 09/19] Add configurable pool_dir to XenPool --- core/storage/xen.py | 61 ++++++++++++++++++++-- tests/storage.py | 121 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 176 insertions(+), 6 deletions(-) diff --git a/core/storage/xen.py b/core/storage/xen.py index 06bd0186..853f0268 100644 --- a/core/storage/xen.py +++ b/core/storage/xen.py @@ -28,7 +28,9 @@ import re import subprocess import sys -from qubes.qubes import QubesException, defaults, vm_files +from qubes.qubes import (QubesAdminVm, QubesAppVm, QubesException, QubesHVm, + QubesNetVm, QubesProxyVm, QubesTemplateHVm, + QubesTemplateVm, defaults, vm_files) from qubes.storage import Pool, QubesVmStorage @@ -37,7 +39,16 @@ class XenStorage(QubesVmStorage): Class for VM storage of Xen VMs. """ - def __init__(self, vm, **kwargs): + def __init__(self, vm, vmdir, **kwargs): + """ Instantiate the storage. + + Args: + vm: a QubesVM + vmdir: the root directory of the pool + """ + assert vm is not None + assert vmdir is not None + super(XenStorage, self).__init__(vm, **kwargs) self.root_dev = "xvda" @@ -45,8 +56,11 @@ class XenStorage(QubesVmStorage): self.volatile_dev = "xvdc" self.modules_dev = "xvdd" + self.vmdir = vmdir + if self.vm.is_template(): - self.rootcow_img = os.path.join(self.vmdir, vm_files["rootcow_img"]) + self.rootcow_img = os.path.join(self.vmdir, + vm_files["rootcow_img"]) else: self.rootcow_img = None @@ -252,12 +266,51 @@ class XenStorage(QubesVmStorage): class XenPool(Pool): + def __init__(self, vm, dir): assert vm is not None assert dir is not None + if not os.path.exists(dir): + os.mkdir(dir) + + self.vmdir = self._vmdir_path(vm, dir) self.vm = vm self.dir = dir def getStorage(self): - return defaults['storage_class'](self.vm) + """ Returns an instantiated ``XenStorage``. """ + return defaults['storage_class'](self.vm, vmdir=self.vmdir) + + def _vmdir_path(self, vm, pool_dir): + """ Get the vm dir depending on the type of the VM. + + The default QubesOS file storage saves the vm images in three + different directories depending on the ``QubesVM`` type: + + * ``appvms`` for ``QubesAppVm`` or ``QubesHvm`` + * ``vm-templates`` for ``QubesTemplateVm`` or ``QubesTemplateHvm`` + * ``servicevms`` for ``QubesProxyVm``, ``QubesNetVm`` or + ``QubesAdminVm`` + + Args: + vm: a QubesVM + pool_dir: the root directory of the pool + + Returns: + string (str) absolute path to the directory where the vm files + are stored + """ + vm_type = type(vm) + + if vm_type in [QubesAppVm, QubesHVm]: + subdir = 'appvms' + elif vm_type in [QubesTemplateVm, QubesTemplateHVm]: + subdir = 'vm-templates' + elif vm_type in [QubesAdminVm, QubesNetVm, QubesProxyVm]: + subdir = 'servicevms' + else: + raise QubesException(str(vm_type) + ' unknown vm type') + + return os.path.join(pool_dir, subdir, vm.name) + diff --git a/tests/storage.py b/tests/storage.py index d016fb1a..e1b408f7 100644 --- a/tests/storage.py +++ b/tests/storage.py @@ -17,11 +17,13 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import os +import shutil + import qubes.storage from qubes.qubes import defaults -from qubes.tests import QubesTestCase, SystemTestsMixin - from qubes.storage.xen import XenPool, XenStorage +from qubes.tests import QubesTestCase, SystemTestsMixin class TC_00_Storage(SystemTestsMixin, QubesTestCase): @@ -103,3 +105,118 @@ class TC_00_Pool(SystemTestsMixin, QubesTestCase): template = self.qc.get_default_template() return self.qc.add_new_vm('QubesAppVm', name=vmname, template=template, pool_name='default') + + +class TC_01_Pool(SystemTestsMixin, QubesTestCase): + + """ Test the paths for the default Xen file based storage + (``QubesXenVmStorage``). + """ + + POOL_DIR = '/var/lib/qubes/test-pool' + APPVMS_DIR = '/var/lib/qubes/test-pool/appvms' + TEMPLATES_DIR = '/var/lib/qubes/test-pool/vm-templates' + SERVICE_DIR = '/var/lib/qubes/test-pool/servicevms' + + def setUp(self): + """ Add a test file based storage pool """ + super(TC_01_Pool, self).setUp() + qubes.storage.add_pool('test-pool', type='xen', dir=self.POOL_DIR) + + def tearDown(self): + """ Remove the file based storage pool after testing """ + super(TC_01_Pool, self).tearDown() + qubes.storage.remove_pool("test-pool") + shutil.rmtree(self.POOL_DIR, ignore_errors=True) + + def test_001_pool_exists(self): + """ Check if the storage pool was added to the storage pool config """ + self.assertTrue(qubes.storage.pool_exists('test-pool')) + + def test_002_pool_dir_create(self): + """ Check if the storage pool dir was created """ + + # The dir should not exists before + self.assertFalse(os.path.exists(self.POOL_DIR)) + + vmname = self.make_vm_name('appvm') + template = self.qc.get_default_template() + self.qc.add_new_vm('QubesAppVm', name=vmname, template=template, + pool_name='test-pool') + + self.assertTrue(os.path.exists(self.POOL_DIR)) + + def test_003_pool_dir(self): + """ Check if the vm storage pool_dir is the same as specified """ + vmname = self.make_vm_name('appvm') + template = self.qc.get_default_template() + vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template, + pool_name='test-pool') + result = qubes.storage.get_pool('test-pool', vm).dir + self.assertEquals(self.POOL_DIR, result) + + def test_004_app_vmdir(self): + """ Check the vm storage dir for an AppVm""" + vmname = self.make_vm_name('appvm') + template = self.qc.get_default_template() + vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template, + pool_name='test-pool') + + expected = os.path.join(self.APPVMS_DIR, vm.name) + result = vm.storage.vmdir + self.assertEquals(expected, result) + + def test_005_hvm_vmdir(self): + """ Check the vm storage dir for a HVM""" + vmname = self.make_vm_name('hvm') + vm = self.qc.add_new_vm('QubesHVm', name=vmname, + pool_name='test-pool') + + expected = os.path.join(self.APPVMS_DIR, vm.name) + result = vm.storage.vmdir + self.assertEquals(expected, result) + + def test_006_net_vmdir(self): + """ Check the vm storage dir for a Netvm""" + vmname = self.make_vm_name('hvm') + vm = self.qc.add_new_vm('QubesNetVm', name=vmname, + pool_name='test-pool') + + expected = os.path.join(self.SERVICE_DIR, vm.name) + result = vm.storage.vmdir + self.assertEquals(expected, result) + + def test_007_proxy_vmdir(self): + """ Check the vm storage dir for a ProxyVm""" + vmname = self.make_vm_name('proxyvm') + vm = self.qc.add_new_vm('QubesProxyVm', name=vmname, + pool_name='test-pool') + + expected = os.path.join(self.SERVICE_DIR, vm.name) + result = vm.storage.vmdir + self.assertEquals(expected, result) + + def test_008_admin_vmdir(self): + """ Check the vm storage dir for a AdminVm""" + # TODO How to test AdminVm? + pass + + def test_009_template_vmdir(self): + """ Check the vm storage dir for a TemplateVm""" + vmname = self.make_vm_name('templatevm') + vm = self.qc.add_new_vm('QubesTemplateVm', name=vmname, + pool_name='test-pool') + + expected = os.path.join(self.TEMPLATES_DIR, vm.name) + result = vm.storage.vmdir + self.assertEquals(expected, result) + + def test_010_template_hvm_vmdir(self): + """ Check the vm storage dir for a TemplateHVm""" + vmname = self.make_vm_name('templatehvm') + vm = self.qc.add_new_vm('QubesTemplateHVm', name=vmname, + pool_name='test-pool') + + expected = os.path.join(self.TEMPLATES_DIR, vm.name) + result = vm.storage.vmdir + self.assertEquals(expected, result) From 0bc0bc90454ad70c9ba15dc1ec89c60bb3bb6916 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sat, 7 Nov 2015 22:38:34 +0100 Subject: [PATCH 10/19] XenStorage make sure subdirs exist in pool dir --- core/storage/xen.py | 17 +++++++++++++++-- tests/storage.py | 5 ++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/core/storage/xen.py b/core/storage/xen.py index 853f0268..1647eeb8 100644 --- a/core/storage/xen.py +++ b/core/storage/xen.py @@ -271,8 +271,14 @@ class XenPool(Pool): assert vm is not None assert dir is not None - if not os.path.exists(dir): - os.mkdir(dir) + appvms_path = os.path.join(dir, 'appvms') + servicevms_path = os.path.join(dir, 'servicevms') + vm_templates_path = os.path.join(dir, 'vm-templates') + + self._create_dir_if_not_exists(dir) + self._create_dir_if_not_exists(appvms_path) + self._create_dir_if_not_exists(servicevms_path) + self._create_dir_if_not_exists(vm_templates_path) self.vmdir = self._vmdir_path(vm, dir) self.vm = vm @@ -314,3 +320,10 @@ class XenPool(Pool): return os.path.join(pool_dir, subdir, vm.name) + def _create_dir_if_not_exists(self, path): + """ Check if a directory exists in if not it is created. + + This method does not create any parent directories. + """ + if not os.path.exists(path): + os.mkdir(path) diff --git a/tests/storage.py b/tests/storage.py index e1b408f7..8a5f949f 100644 --- a/tests/storage.py +++ b/tests/storage.py @@ -134,7 +134,7 @@ class TC_01_Pool(SystemTestsMixin, QubesTestCase): self.assertTrue(qubes.storage.pool_exists('test-pool')) def test_002_pool_dir_create(self): - """ Check if the storage pool dir was created """ + """ Check if the storage pool dir and subdirs were created """ # The dir should not exists before self.assertFalse(os.path.exists(self.POOL_DIR)) @@ -145,6 +145,9 @@ class TC_01_Pool(SystemTestsMixin, QubesTestCase): pool_name='test-pool') self.assertTrue(os.path.exists(self.POOL_DIR)) + self.assertTrue(os.path.exists(self.APPVMS_DIR)) + self.assertTrue(os.path.exists(self.SERVICE_DIR)) + self.assertTrue(os.path.exists(self.TEMPLATES_DIR)) def test_003_pool_dir(self): """ Check if the vm storage pool_dir is the same as specified """ From 348030bf9c7cd5685e6d462973c9f7b35b1fc6cd Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sat, 7 Nov 2015 19:10:55 +0100 Subject: [PATCH 11/19] Add --storage-pool option to qvm-create --- qvm-tools/qvm-create | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/qvm-tools/qvm-create b/qvm-tools/qvm-create index a014febf..6ae002e5 100755 --- a/qvm-tools/qvm-create +++ b/qvm-tools/qvm-create @@ -41,6 +41,8 @@ def main(): help="Specify the label to use for the new VM (e.g. red, yellow, green, ...)") parser.add_option ("-p", "--proxy", action="store_true", dest="proxyvm", default=False, help="Create ProxyVM") + parser.add_option ("-P", "--pool", dest="pool_name", + help="Specify which storage pool to use") parser.add_option ("-H", "--hvm", action="store_true", dest="hvm", default=False, help="Create HVM (standalone unless --template option used)") parser.add_option ("--hvm-template", action="store_true", dest="hvm_template", default=False, @@ -71,6 +73,11 @@ def main(): parser.error ("You must specify VM name!") vmname = args[0] + if options.pool_name is None: + pool_name = "default" + else : + pool_name = options.pool_name + if (options.netvm + options.proxyvm + options.hvm + options.hvm_template) > 1: parser.error ("You must specify at most one VM type switch") @@ -170,7 +177,9 @@ def main(): vmtype = "QubesAppVm" try: - vm = qvm_collection.add_new_vm(vmtype, name=vmname, template=new_vm_template, label = label) + vm=qvm_collection.add_new_vm(vmtype, name=vmname, + template=new_vm_template, label=label, + pool_name=pool_name) except QubesException as err: print >> sys.stderr, "ERROR: {0}".format(err) exit (1) From 76224dac8622d85a28a39113d88d5f387a936b72 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sun, 8 Nov 2015 19:29:41 +0100 Subject: [PATCH 12/19] Path to the Vm images is set by the storage - This moves the logic for setting the path to the storage specific class like XenStore --- core/storage/__init__.py | 6 ------ core/storage/xen.py | 7 +++++++ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/core/storage/__init__.py b/core/storage/__init__.py index ef8b27e0..f835d964 100644 --- a/core/storage/__init__.py +++ b/core/storage/__init__.py @@ -57,12 +57,6 @@ class QubesVmStorage(object): else: self.root_img_size = defaults['root_img_size'] - self.private_img = vm.absolute_path(vm_files["private_img"], None) - if self.vm.template: - self.root_img = self.vm.template.root_img - else: - self.root_img = vm.absolute_path(vm_files["root_img"], None) - self.volatile_img = vm.absolute_path(vm_files["volatile_img"], None) # For now compute this path still in QubesVm self.modules_img = modules_img diff --git a/core/storage/xen.py b/core/storage/xen.py index 1647eeb8..63d856e9 100644 --- a/core/storage/xen.py +++ b/core/storage/xen.py @@ -64,6 +64,13 @@ class XenStorage(QubesVmStorage): else: self.rootcow_img = None + self.private_img = os.path.join(vmdir, 'private.img') + if self.vm.template: + self.root_img = self.vm.template.root_img + else: + self.root_img = os.path.join(vmdir, 'root.img') + self.volatile_img = os.path.join(vmdir, 'volatile.img') + def _format_disk_dev(self, path, script, vdev, rw=True, type="disk", domain=None): if path is None: return '' From 710b95a6ac203af85ff5f95dcbf81109f3d602cb Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sun, 8 Nov 2015 21:22:44 +0100 Subject: [PATCH 13/19] Fix app icons errors --- core-modules/000QubesVm.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py index 615652db..eef88fbf 100644 --- a/core-modules/000QubesVm.py +++ b/core-modules/000QubesVm.py @@ -347,6 +347,13 @@ class QubesVm(object): # Initialize VM image storage class self.storage = get_pool(self.pool_name, self).getStorage() + self.dir_path = self.storage.vmdir + self.icon_path = os.path.join(self.storage.vmdir, 'icon.png') + self.conf_file = os.path.join(self.storage.vmdir, self.name + '.conf') + self.appmenus_templates_dir = os.path.join(self.storage.vmdir, 'apps.templates') + self.appmenus_icons_dir = os.path.join(self.storage.vmdir, 'apps.icons') + + if hasattr(self, 'kernels_dir'): modules_path = os.path.join(self.kernels_dir, "modules.img") From 0ba0259d1a357cf379ccb251242543a42cece20b Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sun, 8 Nov 2015 21:23:07 +0100 Subject: [PATCH 14/19] Extend documentation test/storage.py --- tests/storage.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/storage.py b/tests/storage.py index 8a5f949f..763d1f1a 100644 --- a/tests/storage.py +++ b/tests/storage.py @@ -28,6 +28,8 @@ from qubes.tests import QubesTestCase, SystemTestsMixin class TC_00_Storage(SystemTestsMixin, QubesTestCase): + """ This class tests the utility methods from :mod:``qubes.storage`` """ + def test_000_dump(self): """ Dumps storage instance to a storage string """ vmname = self.make_vm_name('appvm') @@ -80,9 +82,13 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase): class TC_00_Pool(SystemTestsMixin, QubesTestCase): - def test000_no_pool_dir(self): - """ If no pool dir ist configured for a ``XenPool`` assume the default - `/var/lib/qubes/`. + """ This class tests some properties of the 'default' pool. """ + + def test000_default_pool_dir(self): + """ The predefined dir for the default pool should be ``/var/lib/qubes`` + + .. sealso:: + Data :data:``qubes.qubes.defaults['pool_config']``. """ vm = self._init_app_vm() result = qubes.storage.get_pool("default", vm).dir @@ -90,11 +96,11 @@ class TC_00_Pool(SystemTestsMixin, QubesTestCase): self.assertEquals(result, expected) def test001_default_storage_class(self): - """ Check if when using default pool the Storage is ``XenStorage``. """ + """ Check when using default pool the Storage is ``XenStorage``. """ result = self._init_app_vm().storage self.assertIsInstance(result, XenStorage) - def test_002_pool_name(self): + def test_002_default_pool_name(self): """ Default pool_name is 'default'. """ vm = self._init_app_vm() self.assertEquals(vm.pool_name, "default") @@ -109,8 +115,7 @@ class TC_00_Pool(SystemTestsMixin, QubesTestCase): class TC_01_Pool(SystemTestsMixin, QubesTestCase): - """ Test the paths for the default Xen file based storage - (``QubesXenVmStorage``). + """ Test the paths for the default Xen file based storage (``XenStorage``). """ POOL_DIR = '/var/lib/qubes/test-pool' From 989eab1d72fab8e6ee51acc1231f241501a2af65 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sun, 8 Nov 2015 21:29:57 +0100 Subject: [PATCH 15/19] Add file image tests to tests/storage.py --- tests/storage.py | 54 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/storage.py b/tests/storage.py index 763d1f1a..873d6e71 100644 --- a/tests/storage.py +++ b/tests/storage.py @@ -228,3 +228,57 @@ class TC_01_Pool(SystemTestsMixin, QubesTestCase): expected = os.path.join(self.TEMPLATES_DIR, vm.name) result = vm.storage.vmdir self.assertEquals(expected, result) + + def test_011_appvm_file_images(self): + """ Check if all the needed image files are created for an AppVm""" + + vmname = self.make_vm_name('appvm') + template = self.qc.get_default_template() + vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template, + pool_name='test-pool') + vm.create_on_disk(verbose=False) + + expected_vmdir = os.path.join(self.APPVMS_DIR, vm.name) + self.assertEqualsAndExists(vm.storage.vmdir, expected_vmdir) + + expected_private_path = os.path.join(expected_vmdir, 'private.img') + self.assertEqualsAndExists(vm.storage.private_img, + expected_private_path) + + expected_volatile_path = os.path.join(expected_vmdir, 'volatile.img') + self.assertEqualsAndExists(vm.storage.volatile_img, + expected_volatile_path) + + def test_012_hvm_file_images(self): + """ Check if all the needed image files are created for a HVm""" + + vmname = self.make_vm_name('hvm') + vm = self.qc.add_new_vm('QubesHVm', name=vmname, + pool_name='test-pool') + vm.create_on_disk(verbose=False) + + expected_vmdir = os.path.join(self.APPVMS_DIR, vm.name) + self.assertEqualsAndExists(vm.storage.vmdir, expected_vmdir) + + expected_private_path = os.path.join(expected_vmdir, 'private.img') + self.assertEqualsAndExists(vm.storage.private_img, + expected_private_path) + + expected_root_path = os.path.join(expected_vmdir, 'root.img') + self.assertEqualsAndExists(vm.storage.root_img, expected_root_path) + + expected_volatile_path = os.path.join(expected_vmdir, 'volatile.img') + self.assertEqualsAndExists(vm.storage.volatile_img, + expected_volatile_path) + + def assertEqualsAndExists(self, result_path, expected_path): + """ Check if the ``result_path``, matches ``expected_path`` and exists. + + See also: :meth:``assertExist`` + """ + self.assertEquals(result_path, expected_path) + self.assertExist(result_path) + + def assertExist(self, path): + """ Assert that the given path exists. """ + self.assertTrue(os.path.exists(path)) From 1934f06869fc280892db593321e90737c9bbb1e3 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Tue, 17 Nov 2015 21:15:13 +0100 Subject: [PATCH 16/19] XenStorage add DisposableVm handling --- core/storage/xen.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/core/storage/xen.py b/core/storage/xen.py index 63d856e9..21c15f8c 100644 --- a/core/storage/xen.py +++ b/core/storage/xen.py @@ -28,9 +28,9 @@ import re import subprocess import sys -from qubes.qubes import (QubesAdminVm, QubesAppVm, QubesException, QubesHVm, - QubesNetVm, QubesProxyVm, QubesTemplateHVm, - QubesTemplateVm, defaults, vm_files) +from qubes.qubes import (QubesAdminVm, QubesAppVm, QubesDisposableVm, + QubesException, QubesHVm, QubesNetVm, QubesProxyVm, + QubesTemplateHVm, QubesTemplateVm, defaults, vm_files) from qubes.storage import Pool, QubesVmStorage @@ -322,6 +322,9 @@ class XenPool(Pool): subdir = 'vm-templates' elif vm_type in [QubesAdminVm, QubesNetVm, QubesProxyVm]: subdir = 'servicevms' + elif vm_type is QubesDisposableVm: + subdir = 'appvms' + return os.path.join(pool_dir, subdir, vm.template.name + '-dvm') else: raise QubesException(str(vm_type) + ' unknown vm type') From 9eee00c6d790c9513c7c92681f73987d2beeb537 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Tue, 17 Nov 2015 21:19:15 +0100 Subject: [PATCH 17/19] QubesNetVm and subclasses use servicevm/ as vmdir --- core/storage/xen.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/core/storage/xen.py b/core/storage/xen.py index 21c15f8c..e2b8482c 100644 --- a/core/storage/xen.py +++ b/core/storage/xen.py @@ -28,9 +28,9 @@ import re import subprocess import sys -from qubes.qubes import (QubesAdminVm, QubesAppVm, QubesDisposableVm, - QubesException, QubesHVm, QubesNetVm, QubesProxyVm, - QubesTemplateHVm, QubesTemplateVm, defaults, vm_files) +from qubes.qubes import (QubesAppVm, QubesDisposableVm, QubesException, + QubesHVm, QubesNetVm, QubesTemplateHVm, + QubesTemplateVm, defaults, vm_files) from qubes.storage import Pool, QubesVmStorage @@ -303,8 +303,7 @@ class XenPool(Pool): * ``appvms`` for ``QubesAppVm`` or ``QubesHvm`` * ``vm-templates`` for ``QubesTemplateVm`` or ``QubesTemplateHvm`` - * ``servicevms`` for ``QubesProxyVm``, ``QubesNetVm`` or - ``QubesAdminVm`` + * ``servicevms`` for any subclass of ``QubesNetVm`` Args: vm: a QubesVM @@ -320,7 +319,7 @@ class XenPool(Pool): subdir = 'appvms' elif vm_type in [QubesTemplateVm, QubesTemplateHVm]: subdir = 'vm-templates' - elif vm_type in [QubesAdminVm, QubesNetVm, QubesProxyVm]: + elif issubclass(vm_type, QubesNetVm): subdir = 'servicevms' elif vm_type is QubesDisposableVm: subdir = 'appvms' From 885dc5cd81ca28d1354a266746d60aeb72b14ced Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sat, 21 Nov 2015 14:46:23 +0100 Subject: [PATCH 18/19] Rename storage dir to dir_path, type to driver Bad idea to use attributes which are named after built in functions. --- core/settings-xen-Linux.py | 4 ++-- core/storage/__init__.py | 16 +++++++--------- core/storage/storage.conf | 6 +++--- core/storage/xen.py | 16 ++++++++-------- tests/storage.py | 17 +++++++++-------- 5 files changed, 29 insertions(+), 30 deletions(-) diff --git a/core/settings-xen-Linux.py b/core/settings-xen-Linux.py index 74503524..c413e8ae 100644 --- a/core/settings-xen-Linux.py +++ b/core/settings-xen-Linux.py @@ -7,5 +7,5 @@ from qubes.storage.xen import XenStorage, XenPool def apply(system_path, vm_files, defaults): defaults['storage_class'] = XenStorage - defaults['pool_types'] = {'xen': XenPool} - defaults['pool_config'] = {'dir': '/var/lib/qubes/'} + defaults['pool_drivers'] = {'xen': XenPool} + defaults['pool_config'] = {'dir_path': '/var/lib/qubes/'} diff --git a/core/storage/__init__.py b/core/storage/__init__.py index f835d964..8e2ac720 100644 --- a/core/storage/__init__.py +++ b/core/storage/__init__.py @@ -239,7 +239,7 @@ def get_pool(name, vm): klass = _get_pool_klass(name, config) - keys = [k for k in config.options(name) if k != 'type' and k != 'class'] + keys = [k for k in config.options(name) if k != 'driver' and k != 'class'] values = [config.get(name, o) for o in keys] config_kwargs = dict(zip(keys, values)) @@ -308,19 +308,17 @@ def _get_pool_klass(name, config=None): if config.has_option(name, 'class'): klass = load(config.get(name, 'class')) - elif config.has_option(name, 'type'): - pool_type = config.get(name, 'type') - klass = defaults['pool_types'][pool_type] - - if klass is None: - raise StoragePoolException('Uknown storage pool type ' + name) + elif config.has_option(name, 'driver'): + pool_driver = config.get(name, 'driver') + klass = defaults['pool_drivers'][pool_driver] + else: + raise StoragePoolException('Uknown storage pool driver ' + name) return klass - class StoragePoolException(QubesException): pass + class Pool(object): pass - diff --git a/core/storage/storage.conf b/core/storage/storage.conf index 45b6bda0..e9d067e5 100644 --- a/core/storage/storage.conf +++ b/core/storage/storage.conf @@ -1,6 +1,6 @@ [default] ; poolname -type=xen ; the default xen storage -; class = qubes.storage.xen.XenStorage ; class always overwrites type +driver=xen ; the default xen storage +; class = qubes.storage.xen.XenStorage ; class always overwrites the driver ; ; To use our own storage adapter, you need just to specify the module path and ; class name @@ -8,5 +8,5 @@ type=xen ; the default xen storage ; class = foo.bar.MyStorage ; ; [test-dummy] -; type=dummy +; driver=dummy diff --git a/core/storage/xen.py b/core/storage/xen.py index e2b8482c..1cd5dc97 100644 --- a/core/storage/xen.py +++ b/core/storage/xen.py @@ -274,22 +274,22 @@ class XenStorage(QubesVmStorage): class XenPool(Pool): - def __init__(self, vm, dir): + def __init__(self, vm, dir_path): assert vm is not None - assert dir is not None + assert dir_path is not None - appvms_path = os.path.join(dir, 'appvms') - servicevms_path = os.path.join(dir, 'servicevms') - vm_templates_path = os.path.join(dir, 'vm-templates') + appvms_path = os.path.join(dir_path, 'appvms') + servicevms_path = os.path.join(dir_path, 'servicevms') + vm_templates_path = os.path.join(dir_path, 'vm-templates') - self._create_dir_if_not_exists(dir) + self._create_dir_if_not_exists(dir_path) self._create_dir_if_not_exists(appvms_path) self._create_dir_if_not_exists(servicevms_path) self._create_dir_if_not_exists(vm_templates_path) - self.vmdir = self._vmdir_path(vm, dir) + self.vmdir = self._vmdir_path(vm, dir_path) self.vm = vm - self.dir = dir + self.dir_path = dir_path def getStorage(self): """ Returns an instantiated ``XenStorage``. """ diff --git a/tests/storage.py b/tests/storage.py index 873d6e71..5513d383 100644 --- a/tests/storage.py +++ b/tests/storage.py @@ -42,13 +42,13 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase): self.assertEquals(result, expected) def test_001_load(self): - """ Loads storage type from a storage string """ + """ Loads storage driver from a storage string """ result = qubes.storage.load('qubes.storage.xen.XenStorage') self.assertTrue(result is XenStorage) - def test_002_default_pool_types(self): - """ The only predifined pool type is xen """ - result = defaults['pool_types'].keys() + def test_002_default_pool_drivers(self): + """ The only predifined pool driver is xen """ + result = defaults['pool_drivers'].keys() expected = ["xen"] self.assertEquals(result, expected) @@ -73,7 +73,7 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase): # make sure it's really does not exist qubes.storage.remove_pool(pool_name) - qubes.storage.add_pool(pool_name, type='xen') + qubes.storage.add_pool(pool_name, driver='xen') self.assertTrue(qubes.storage.pool_exists(pool_name)) qubes.storage.remove_pool(pool_name) @@ -91,7 +91,7 @@ class TC_00_Pool(SystemTestsMixin, QubesTestCase): Data :data:``qubes.qubes.defaults['pool_config']``. """ vm = self._init_app_vm() - result = qubes.storage.get_pool("default", vm).dir + result = qubes.storage.get_pool("default", vm).dir_path expected = '/var/lib/qubes/' self.assertEquals(result, expected) @@ -126,7 +126,8 @@ class TC_01_Pool(SystemTestsMixin, QubesTestCase): def setUp(self): """ Add a test file based storage pool """ super(TC_01_Pool, self).setUp() - qubes.storage.add_pool('test-pool', type='xen', dir=self.POOL_DIR) + qubes.storage.add_pool('test-pool', driver='xen', + dir_path=self.POOL_DIR) def tearDown(self): """ Remove the file based storage pool after testing """ @@ -160,7 +161,7 @@ class TC_01_Pool(SystemTestsMixin, QubesTestCase): template = self.qc.get_default_template() vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template, pool_name='test-pool') - result = qubes.storage.get_pool('test-pool', vm).dir + result = qubes.storage.get_pool('test-pool', vm).dir_path self.assertEquals(self.POOL_DIR, result) def test_004_app_vmdir(self): From 2ebd20049d9193b7dcc4117095c2ce8892434813 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sun, 22 Nov 2015 12:21:40 +0100 Subject: [PATCH 19/19] Refactor appmenus paths to core-admin-linux --- core-modules/000QubesVm.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py index eef88fbf..de4f8851 100644 --- a/core-modules/000QubesVm.py +++ b/core-modules/000QubesVm.py @@ -350,9 +350,6 @@ class QubesVm(object): self.dir_path = self.storage.vmdir self.icon_path = os.path.join(self.storage.vmdir, 'icon.png') self.conf_file = os.path.join(self.storage.vmdir, self.name + '.conf') - self.appmenus_templates_dir = os.path.join(self.storage.vmdir, 'apps.templates') - self.appmenus_icons_dir = os.path.join(self.storage.vmdir, 'apps.icons') - if hasattr(self, 'kernels_dir'): modules_path = os.path.join(self.kernels_dir,