From 51826decec8a3480a0869f08d1ce1512088b4887 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 25 Jul 2015 03:55:50 +0200 Subject: [PATCH 001/270] core/hvm: start guid before qrexec, even when gui agent is installed This way even when qrexec agent would timeout on connection, guid will be already running. Also use new -K guid option to terminate stubdom guid when the real guid is connected (unless in debug mode - then both guid will be running). --- core-modules/01QubesHVm.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/core-modules/01QubesHVm.py b/core-modules/01QubesHVm.py index 16e33376..8217fbbc 100644 --- a/core-modules/01QubesHVm.py +++ b/core-modules/01QubesHVm.py @@ -362,25 +362,28 @@ class QubesHVm(QubesVm): if (retcode != 0) : raise QubesException("Cannot start qubes-guid!") - def start_guid(self, verbose = True, notify_function = None, - before_qrexec=False, **kwargs): - # If user force the guiagent, start_guid will mimic a standard QubesVM - if not before_qrexec and self.guiagent_installed: - kwargs['extra_guid_args'] = kwargs.get('extra_guid_args', []) + \ - ['-Q'] - super(QubesHVm, self).start_guid(verbose, notify_function, **kwargs) - stubdom_guid_pidfile = '/var/run/qubes/guid-running.%d' % self.stubdom_xid - if os.path.exists(stubdom_guid_pidfile) and not self.debug: - try: - stubdom_guid_pid = int(open(stubdom_guid_pidfile, 'r').read()) - os.kill(stubdom_guid_pid, signal.SIGTERM) - except Exception as ex: - print >> sys.stderr, "WARNING: Failed to kill stubdom gui daemon: %s" % str(ex) - elif before_qrexec and (not self.guiagent_installed or self.debug): + def start_guid(self, verbose=True, notify_function=None, + before_qrexec=False, **kwargs): + if not before_qrexec: + return + + if not self.guiagent_installed or self.debug: if verbose: print >> sys.stderr, "--> Starting Qubes GUId (full screen)..." self.start_stubdom_guid(verbose=verbose) + kwargs['extra_guid_args'] = kwargs.get('extra_guid_args', []) + \ + ['-Q', '-n'] + + stubdom_guid_pidfile = \ + '/var/run/qubes/guid-running.%d' % self.stubdom_xid + if not self.debug and os.path.exists(stubdom_guid_pidfile): + # Terminate stubdom guid once "real" gui agent connects + stubdom_guid_pid = int(open(stubdom_guid_pidfile, 'r').read()) + kwargs['extra_guid_args'] += ['-K', str(stubdom_guid_pid)] + + super(QubesHVm, self).start_guid(verbose, notify_function, **kwargs) + def start_qrexec_daemon(self, **kwargs): if not self.qrexec_installed: if kwargs.get('verbose', False): From 7550fccf940822440cde1cdb7950ee051a17f118 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sat, 7 Nov 2015 16:54:03 +0100 Subject: [PATCH 002/270] Removed WNI --- core/settings-wni-Windows_NT.py | 45 ----------- core/storage/README.md | 3 + core/storage/wni.py | 138 -------------------------------- 3 files changed, 3 insertions(+), 183 deletions(-) delete mode 100644 core/settings-wni-Windows_NT.py create mode 100644 core/storage/README.md delete mode 100644 core/storage/wni.py diff --git a/core/settings-wni-Windows_NT.py b/core/settings-wni-Windows_NT.py deleted file mode 100644 index 6e646c9d..00000000 --- a/core/settings-wni-Windows_NT.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/python2 - -from __future__ import absolute_import -import _winreg -import os -import sys - -from qubes.storage.wni import QubesWniVmStorage - -DEFAULT_INSTALLDIR = 'c:\\program files\\Invisible Things Lab\\Qubes WNI' -DEFAULT_STOREDIR = 'c:\\qubes' - -def apply(system_path, vm_files, defaults): - system_path['qubes_base_dir'] = DEFAULT_STOREDIR - installdir = DEFAULT_INSTALLDIR - try: - reg_key = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, - "Software\\Invisible Things Lab\\Qubes WNI") - installdir = _winreg.QueryValueEx(reg_key, "InstallDir")[0] - system_path['qubes_base_dir'] = \ - _winreg.QueryValueEx(reg_key, "StoreDir")[0] - except WindowsError as e: - print >>sys.stderr, \ - "WARNING: invalid installation: missing registry entries (%s)" \ - % str(e) - - system_path['config_template_pv'] = \ - os.path.join(installdir, 'vm-template.xml') - system_path['config_template_hvm'] = \ - os.path.join(installdir, 'vm-template-hvm.xml') - system_path['qubes_icon_dir'] = os.path.join(installdir, 'icons') - system_path['qubesdb_daemon_path'] = \ - os.path.join(installdir, 'bin\\qubesdb-daemon.exe') - system_path['qrexec_daemon_path'] = \ - os.path.join(installdir, 'bin\\qrexec-daemon.exe') - system_path['qrexec_client_path'] = \ - os.path.join(installdir, 'bin\\qrexec-client.exe') - system_path['qrexec_policy_dir'] = \ - os.path.join(installdir, 'qubes-rpc\\policy') - # Specific to WNI - normally VM have this file - system_path['qrexec_agent_path'] = \ - os.path.join(installdir, 'bin\\qrexec-agent.exe') - - defaults['libvirt_uri'] = 'wni:///' - defaults['storage_class'] = QubesWniVmStorage diff --git a/core/storage/README.md b/core/storage/README.md new file mode 100644 index 00000000..7258512f --- /dev/null +++ b/core/storage/README.md @@ -0,0 +1,3 @@ +# WNI File storage +Before v3.1 there existed a draft wni storage. You can find it in the git +history diff --git a/core/storage/wni.py b/core/storage/wni.py deleted file mode 100644 index a3571765..00000000 --- a/core/storage/wni.py +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/python2 -# -# The Qubes OS Project, http://www.qubes-os.org -# -# Copyright (C) 2013 Marek Marczykowski -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# - -from __future__ import absolute_import - -import sys -import os -import os.path -import win32api -import win32net -import win32netcon -import win32security -import win32profile -import pywintypes -import random - -from qubes.storage import QubesVmStorage -from qubes.qubes import QubesException,system_path - -class QubesWniVmStorage(QubesVmStorage): - """ - Class for VM storage of WNI VMs. - """ - - def __init__(self, *args, **kwargs): - super(QubesWniVmStorage, self).__init__(*args, **kwargs) - # Use the user profile as "private.img" - self.home_root = win32profile.GetProfilesDirectory() - # FIXME: the assignment below may not always be correct, - # but GetUserProfileDirectory needs a user token... - self.private_img = os.path.join(self.home_root, self._get_username()) - - # Pass paths for WNI libvirt driver - os.putenv("WNI_DRIVER_QUBESDB_PATH", system_path['qubesdb_daemon_path']) - os.putenv("WNI_DRIVER_QREXEC_AGENT_PATH", system_path['qrexec_agent_path']) - - def _get_username(self, vmname = None): - if vmname is None: - vmname = self.vm.name - return "qubes-vm-%s" % vmname - - def _get_random_password(self, vmname = None): - if vmname is None: - vmname = self.vm.name - return '%x' % random.SystemRandom().getrandombits(256) - - def get_config_params(self): - return {} - - def create_on_disk_private_img(self, verbose, source_template = None): - # FIXME: this may not always be correct - home_dir = os.path.join(self.home_root, self._get_username()) - # Create user data in information level 1 (PyUSER_INFO_1) format. - user_data = {} - user_data['name'] = self._get_username() - user_data['full_name'] = self._get_username() - # libvirt driver doesn't need to know the password anymore - user_data['password'] = self._get_random_password() - user_data['flags'] = ( - win32netcon.UF_NORMAL_ACCOUNT | - win32netcon.UF_SCRIPT | - win32netcon.UF_DONT_EXPIRE_PASSWD | - win32netcon.UF_PASSWD_CANT_CHANGE - ) - user_data['priv'] = win32netcon.USER_PRIV_USER - user_data['home_dir'] = home_dir - user_data['max_storage'] = win32netcon.USER_MAXSTORAGE_UNLIMITED - # TODO: catch possible exception - win32net.NetUserAdd(None, 1, user_data) - - def create_on_disk_root_img(self, verbose, source_template = None): - pass - - def remove_from_disk(self): - try: - sid = win32security.LookupAccountName(None, self._get_username())[0] - string_sid = win32security.ConvertSidToStringSid(sid) - win32profile.DeleteProfile(string_sid) - win32net.NetUserDel(None, self._get_username()) - except pywintypes.error, details: - if details[0] == 2221: - # "The user name cannot be found." - raise IOError("User %s doesn't exist" % self._get_username()) - else: - raise - - super(QubesWniVmStorage, self).remove_from_disk() - - def rename(self, old_name, new_name): - super(QubesWniVmStorage, self).rename(old_name, new_name) - user_data = {} - user_data['name'] = self._get_username(new_name) - win32net.NetUserSetInfo(None, - self._get_username(old_name), 0, user_data) - #TODO: rename user profile - - def verify_files(self): - if not os.path.exists (self.vmdir): - raise QubesException ( - "VM directory doesn't exist: {0}".\ - format(self.vmdir)) - - try: - # TemplateVm in WNI is quite virtual, so do not require the user - if not self.vm.is_template(): - win32net.NetUserGetInfo(None, self._get_username(), 0) - except pywintypes.error, details: - if details[0] == 2221: - # "The user name cannot be found." - raise QubesException("User %s doesn't exist" % self._get_username()) - else: - raise - - def reset_volatile_storage(self, verbose = False, source_template = None): - pass - - def prepare_for_vm_startup(self, verbose = False): - if self.vm.is_template(): - raise QubesException("Starting TemplateVM is not supported") From 15d5e6edbb1222d79b2deb3fa68f299ad2c76c2a Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sat, 7 Nov 2015 18:58:49 +0100 Subject: [PATCH 003/270] Add Pool & XenPool --- core/settings-xen-Linux.py | 4 +++- core/storage/__init__.py | 13 ++++++++----- core/storage/xen.py | 12 ++++++++---- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/core/settings-xen-Linux.py b/core/settings-xen-Linux.py index de5084f5..8b23b447 100644 --- a/core/settings-xen-Linux.py +++ b/core/settings-xen-Linux.py @@ -2,7 +2,9 @@ from __future__ import absolute_import -from qubes.storage.xen import QubesXenVmStorage +from qubes.storage.xen import QubesXenVmStorage, XenPool + def apply(system_path, vm_files, defaults): defaults['storage_class'] = QubesXenVmStorage + defaults['pool_types'] = {'xen': XenPool} diff --git a/core/storage/__init__.py b/core/storage/__init__.py index 29cf8654..795aa37a 100644 --- a/core/storage/__init__.py +++ b/core/storage/__init__.py @@ -16,22 +16,21 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. # from __future__ import absolute_import import os import os.path -import re import shutil import subprocess import sys -from qubes.qubes import vm_files,system_path,defaults -from qubes.qubes import QubesException import qubes.qubesutils +from qubes.qubes import QubesException, defaults, system_path, vm_files + class QubesVmStorage(object): """ @@ -199,3 +198,7 @@ class QubesVmStorage(object): print >>sys.stderr, "WARNING: Creating empty VM private image file: {0}".\ format(self.private_img) self.create_on_disk_private_img(verbose=False) + + +class Pool(object): + pass diff --git a/core/storage/xen.py b/core/storage/xen.py index 00c3b3da..f66126a6 100644 --- a/core/storage/xen.py +++ b/core/storage/xen.py @@ -16,20 +16,20 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. # from __future__ import absolute_import import os import os.path +import re import subprocess import sys -import re -from qubes.storage import QubesVmStorage from qubes.qubes import QubesException, vm_files +from qubes.storage import Pool, QubesVmStorage class QubesXenVmStorage(QubesVmStorage): @@ -249,3 +249,7 @@ class QubesXenVmStorage(QubesVmStorage): raise QubesException( "VM '{}' holding '{}' does not exists".format( drive_domain, drive_path)) + + +class XenPool(Pool): + pass From bfaf37dae534d0266c4e3a56808cd11ba0c8c0ef Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sat, 7 Nov 2015 19:01:25 +0100 Subject: [PATCH 004/270] Add pool config parsing --- core/storage/Makefile | 3 ++ core/storage/__init__.py | 95 +++++++++++++++++++++++++++++++++++++++ core/storage/storage.conf | 11 +++++ rpm_spec/core-dom0.spec | 1 + tests/Makefile | 2 + tests/__init__.py | 1 + tests/storage.py | 63 ++++++++++++++++++++++++++ 7 files changed, 176 insertions(+) create mode 100644 core/storage/storage.conf create mode 100644 tests/storage.py diff --git a/core/storage/Makefile b/core/storage/Makefile index ec59cc64..7c7af60e 100644 --- a/core/storage/Makefile +++ b/core/storage/Makefile @@ -1,5 +1,6 @@ OS ?= Linux +SYSCONFDIR ?= /etc PYTHON_QUBESPATH = $(PYTHON_SITEPATH)/qubes all: @@ -13,6 +14,8 @@ endif mkdir -p $(DESTDIR)$(PYTHON_QUBESPATH)/storage cp __init__.py $(DESTDIR)$(PYTHON_QUBESPATH)/storage cp __init__.py[co] $(DESTDIR)$(PYTHON_QUBESPATH)/storage + mkdir -p $(DESTDIR)$(SYSCONFDIR)/qubes + cp storage.conf $(DESTDIR)$(SYSCONFDIR)/qubes/ ifneq ($(BACKEND_VMM),) if [ -r $(BACKEND_VMM).py ]; then \ cp $(BACKEND_VMM).py $(DESTDIR)$(PYTHON_QUBESPATH)/storage && \ diff --git a/core/storage/__init__.py b/core/storage/__init__.py index 795aa37a..b9810881 100644 --- a/core/storage/__init__.py +++ b/core/storage/__init__.py @@ -22,6 +22,7 @@ from __future__ import absolute_import +import ConfigParser import os import os.path import shutil @@ -31,6 +32,8 @@ import sys import qubes.qubesutils from qubes.qubes import QubesException, defaults, system_path, vm_files +CONFIG_FILE = '/etc/qubes/storage.conf' + class QubesVmStorage(object): """ @@ -200,5 +203,97 @@ class QubesVmStorage(object): self.create_on_disk_private_img(verbose=False) +def dump(o): + """ Returns a string represention of the given object + + Args: + o (object): anything that response to `__module__` and `__class__` + + Given the class :class:`qubes.storage.QubesVmStorage` it returns + 'qubes.storage.QubesVmStorage' as string + """ + return o.__module__ + '.' + o.__class__.__name__ + + +def load(string): + """ Given a dotted full module string representation of a class it loads it + + Args: + string (str) i.e. 'qubes.storage.xen.QubesXenVmStorage' + + Returns: + type + + See also: + :func:`qubes.storage.dump` + """ + if not type(string) is str: + # This is a hack which allows giving a real class to a vm instead of a + # string as string_class parameter. + return string + + components = string.split(".") + module_path = ".".join(components[:-1]) + klass = components[-1:][0] + module = __import__(module_path, fromlist=[klass]) + return getattr(module, klass) + + +def get_pool(vm): + """ Instantiates the storage for the specified vm """ + config = _get_storage_config_parser() + + name = vm.storage_pool + klass = _get_pool_klass(name, config) + + keys = [k for k in config.options(name) if k != 'type' and k != 'class'] + values = [config.get(name, o) for o in keys] + kwargs = dict(zip(keys, values)) + return klass(vm, **kwargs) + + +def _get_storage_config_parser(): + """ Instantiates a `ConfigParaser` for specified storage config file. + + Returns: + RawConfigParser + """ + config = ConfigParser.RawConfigParser() + config.read(CONFIG_FILE) + return config + + +def _get_pool_klass(name, config=None): + """ Returns the storage klass for the specified pool. + + Args: + name: The pool name. + config: If ``config`` is not specified + `_get_storage_config_parser()` is called. + + Returns: + type: A class inheriting from `QubesVmStorage` + """ + if config is None: + config = _get_storage_config_parser() + + if not config.has_section(name): + raise StoragePoolException('Uknown storage pool ' + name) + + if config.has_option(name, 'class'): + klass = load(config.get(name, 'class')) + elif config.has_option(name, 'type'): + pool_type = config.get(name, 'type') + klass = defaults['pool_types'][pool_type] + + if klass is None: + raise StoragePoolException('Uknown storage pool type ' + name) + return klass + + +class StoragePoolException(QubesException): + pass + class Pool(object): pass + diff --git a/core/storage/storage.conf b/core/storage/storage.conf new file mode 100644 index 00000000..8fa44c7e --- /dev/null +++ b/core/storage/storage.conf @@ -0,0 +1,11 @@ +[default] ; poolname +type=xen ; the default xen storage +; class = qubes.storage.xen.XenStorage ; class always overwrites type +; +; To use our own storage adapter, you need just to specify the module path and +; class name +; [pool-b] +; class = foo.bar.MyStorage +; + + diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec index 86612c48..84e106ba 100644 --- a/rpm_spec/core-dom0.spec +++ b/rpm_spec/core-dom0.spec @@ -176,6 +176,7 @@ fi %files %defattr(-,root,root,-) %config(noreplace) %attr(0664,root,qubes) %{_sysconfdir}/qubes/qmemman.conf +%config(noreplace) %attr(0664,root,qubes) %{_sysconfdir}/qubes/storage.conf /usr/bin/qvm-* /usr/bin/qubes-* %dir %{python_sitearch}/qubes diff --git a/tests/Makefile b/tests/Makefile index 8523adde..f2dff357 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -27,3 +27,5 @@ endif cp regressions.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) cp run.py $(DESTDIR)$(PYTHON_TESTSPATH) cp run.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) + cp storage.py $(DESTDIR)$(PYTHON_TESTSPATH) + cp storage.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) diff --git a/tests/__init__.py b/tests/__init__.py index 8abea4b8..8af1afbf 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -547,6 +547,7 @@ def load_tests(loader, tests, pattern): 'qubes.tests.backup', 'qubes.tests.backupcompatibility', 'qubes.tests.regressions', + 'qubes.tests.storage', ): tests.addTests(loader.loadTestsFromName(modname)) diff --git a/tests/storage.py b/tests/storage.py new file mode 100644 index 00000000..30b9cd2d --- /dev/null +++ b/tests/storage.py @@ -0,0 +1,63 @@ +# The Qubes OS Project, https://www.qubes-os.org/ +# +# Copyright (C) 2015 Bahtiar `kalkin-` Gadimov +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +from qubes.tests import QubesTestCase, SystemTestsMixin +from qubes.qubes import defaults + +import qubes.storage +from qubes.storage.xen import QubesXenVmStorage, XenPool + + +class TC_00_Storage(SystemTestsMixin, QubesTestCase): + + def test_000_dump(self): + """ Dumps storage instance to a storage string """ + vmname = self.make_vm_name('appvm') + template = self.qc.get_default_template() + storage = self.qc.add_new_vm('QubesAppVm', name=vmname, pool='default', + template=template).storage + result = qubes.storage.dump(storage) + expected = 'qubes.storage.xen.QubesXenVmStorage' + self.assertEquals(result, expected) + + def test_001_load(self): + """ Loads storage type from a storage string """ + result = qubes.storage.load('qubes.storage.xen.QubesXenVmStorage') + self.assertTrue(result is QubesXenVmStorage) + + def test_002_default_pool_types(self): + """ The only predifined pool type is xen """ + result = defaults['pool_types'].keys() + expected = ["xen"] + self.assertEquals(result, expected) + + def test_003_get_pool_klass(self): + """ Expect the default pool to be `XenPool` """ + result = qubes.storage._get_pool_klass('default') + self.assertTrue(result is XenPool) + + +class TC_01_Storage(SystemTestsMixin, QubesTestCase): + + def test_000_vm_use_default_pool(self): + vmname = self.make_vm_name('appvm') + template = self.qc.get_default_template() + vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template, + pool='default') + self.assertIsInstance(vm.storage, QubesXenVmStorage) From 8e0207a199f9e7a3503646acef2fac4f21e1a945 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sat, 7 Nov 2015 19:04:51 +0100 Subject: [PATCH 005/270] Rename QubesXenVmStorage to XenStorage --- core/settings-xen-Linux.py | 4 ++-- core/storage/xen.py | 10 +++++----- tests/storage.py | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/core/settings-xen-Linux.py b/core/settings-xen-Linux.py index 8b23b447..30669407 100644 --- a/core/settings-xen-Linux.py +++ b/core/settings-xen-Linux.py @@ -2,9 +2,9 @@ from __future__ import absolute_import -from qubes.storage.xen import QubesXenVmStorage, XenPool +from qubes.storage.xen import XenStorage, XenPool def apply(system_path, vm_files, defaults): - defaults['storage_class'] = QubesXenVmStorage + defaults['storage_class'] = XenStorage defaults['pool_types'] = {'xen': XenPool} diff --git a/core/storage/xen.py b/core/storage/xen.py index f66126a6..9c1a75cc 100644 --- a/core/storage/xen.py +++ b/core/storage/xen.py @@ -32,13 +32,13 @@ from qubes.qubes import QubesException, vm_files from qubes.storage import Pool, QubesVmStorage -class QubesXenVmStorage(QubesVmStorage): +class XenStorage(QubesVmStorage): """ Class for VM storage of Xen VMs. """ def __init__(self, vm, **kwargs): - super(QubesXenVmStorage, self).__init__(vm, **kwargs) + super(XenStorage, self).__init__(vm, **kwargs) self.root_dev = "xvda" self.private_dev = "xvdb" @@ -158,7 +158,7 @@ class QubesXenVmStorage(QubesVmStorage): self.commit_template_changes() def rename(self, old_name, new_name): - super(QubesXenVmStorage, self).rename(old_name, new_name) + super(XenStorage, self).rename(old_name, new_name) old_dirpath = os.path.join(os.path.dirname(self.vmdir), old_name) if self.rootcow_img: @@ -226,11 +226,11 @@ class QubesXenVmStorage(QubesVmStorage): f_volatile.close() f_root.close() return - super(QubesXenVmStorage, self).reset_volatile_storage( + super(XenStorage, self).reset_volatile_storage( verbose=verbose, source_template=source_template) def prepare_for_vm_startup(self, verbose): - super(QubesXenVmStorage, self).prepare_for_vm_startup(verbose=verbose) + super(XenStorage, self).prepare_for_vm_startup(verbose=verbose) if self.drive is not None: (drive_type, drive_domain, drive_path) = self.drive.split(":") diff --git a/tests/storage.py b/tests/storage.py index 30b9cd2d..a52b62b4 100644 --- a/tests/storage.py +++ b/tests/storage.py @@ -21,7 +21,7 @@ from qubes.tests import QubesTestCase, SystemTestsMixin from qubes.qubes import defaults import qubes.storage -from qubes.storage.xen import QubesXenVmStorage, XenPool +from qubes.storage.xen import XenStorage, XenPool class TC_00_Storage(SystemTestsMixin, QubesTestCase): @@ -33,13 +33,13 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase): storage = self.qc.add_new_vm('QubesAppVm', name=vmname, pool='default', template=template).storage result = qubes.storage.dump(storage) - expected = 'qubes.storage.xen.QubesXenVmStorage' + expected = 'qubes.storage.xen.XenStorage' self.assertEquals(result, expected) def test_001_load(self): """ Loads storage type from a storage string """ - result = qubes.storage.load('qubes.storage.xen.QubesXenVmStorage') - self.assertTrue(result is QubesXenVmStorage) + result = qubes.storage.load('qubes.storage.xen.XenStorage') + self.assertTrue(result is XenStorage) def test_002_default_pool_types(self): """ The only predifined pool type is xen """ @@ -60,4 +60,4 @@ class TC_01_Storage(SystemTestsMixin, QubesTestCase): template = self.qc.get_default_template() vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template, pool='default') - self.assertIsInstance(vm.storage, QubesXenVmStorage) + self.assertIsInstance(vm.storage, XenStorage) From d1685a13df6a9114c883f35c3908db9f1e11e511 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sat, 7 Nov 2015 19:32:03 +0100 Subject: [PATCH 006/270] Add storage.pool_exists --- core/storage/__init__.py | 9 +++++++++ tests/storage.py | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/core/storage/__init__.py b/core/storage/__init__.py index b9810881..aabb2f65 100644 --- a/core/storage/__init__.py +++ b/core/storage/__init__.py @@ -291,6 +291,15 @@ def _get_pool_klass(name, config=None): return klass +def pool_exists(name): + """ Check if the specified pool exists """ + try: + _get_pool_klass(name) + return True + except StoragePoolException: + return False + + class StoragePoolException(QubesException): pass diff --git a/tests/storage.py b/tests/storage.py index a52b62b4..e3fb4b29 100644 --- a/tests/storage.py +++ b/tests/storage.py @@ -52,6 +52,15 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase): result = qubes.storage._get_pool_klass('default') self.assertTrue(result is XenPool) + def test_004_pool_exists_default(self): + """ Expect the default pool to exists """ + self.assertTrue(qubes.storage.pool_exists('default')) + + def test_005_pool_exists_random(self): + """ Expect this pool to not a exist """ + self.assertFalse( + qubes.storage.pool_exists('asdh312096r832598213iudhas')) + class TC_01_Storage(SystemTestsMixin, QubesTestCase): From 78891dd70f344bcdca2faa5f7145d642e620cc88 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sat, 7 Nov 2015 19:32:38 +0100 Subject: [PATCH 007/270] QubesVm save the name of the storage pool used --- core-modules/000QubesVm.py | 7 ++++--- core/storage/__init__.py | 3 +-- core/storage/xen.py | 8 ++++++-- tests/storage.py | 4 ++-- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py index f8fdd76c..615652db 100644 --- a/core-modules/000QubesVm.py +++ b/core-modules/000QubesVm.py @@ -40,12 +40,12 @@ import signal from qubes import qmemman from qubes import qmemman_algo import libvirt -import warnings from qubes.qubes import dry_run,vmm from qubes.qubes import register_qubes_vm_class from qubes.qubes import QubesVmCollection,QubesException,QubesHost,QubesVmLabels from qubes.qubes import defaults,system_path,vm_files,qubes_max_qid +from qubes.storage import get_pool qmemman_present = False try: @@ -109,6 +109,7 @@ class QubesVm(object): "name": { "order": 1 }, "uuid": { "order": 0, "eval": 'uuid.UUID(value) if value else None' }, "dir_path": { "default": None, "order": 2 }, + "pool_name": { "default":"default" }, "conf_file": { "func": lambda value: self.absolute_path(value, self.name + ".conf"), @@ -198,7 +199,7 @@ class QubesVm(object): 'kernelopts', 'services', 'installed_by_rpm',\ 'uses_default_netvm', 'include_in_backups', 'debug',\ 'qrexec_timeout', 'autostart', 'uses_default_dispvm_netvm', - 'backup_content', 'backup_size', 'backup_path' ]: + 'backup_content', 'backup_size', 'backup_path', 'pool_name' ]: attrs[prop]['save'] = lambda prop=prop: str(getattr(self, prop)) # Simple paths for prop in ['conf_file', 'firewall_conf']: @@ -345,7 +346,7 @@ class QubesVm(object): self.services['qubes-update-check'] = False # Initialize VM image storage class - self.storage = defaults["storage_class"](self) + self.storage = get_pool(self.pool_name, self).getStorage() if hasattr(self, 'kernels_dir'): modules_path = os.path.join(self.kernels_dir, "modules.img") diff --git a/core/storage/__init__.py b/core/storage/__init__.py index aabb2f65..4636dc47 100644 --- a/core/storage/__init__.py +++ b/core/storage/__init__.py @@ -239,11 +239,10 @@ def load(string): return getattr(module, klass) -def get_pool(vm): +def get_pool(name, vm): """ Instantiates the storage for the specified vm """ config = _get_storage_config_parser() - name = vm.storage_pool klass = _get_pool_klass(name, config) keys = [k for k in config.options(name) if k != 'type' and k != 'class'] diff --git a/core/storage/xen.py b/core/storage/xen.py index 9c1a75cc..01ed0277 100644 --- a/core/storage/xen.py +++ b/core/storage/xen.py @@ -28,7 +28,7 @@ import re import subprocess import sys -from qubes.qubes import QubesException, vm_files +from qubes.qubes import QubesException, defaults, vm_files from qubes.storage import Pool, QubesVmStorage @@ -252,4 +252,8 @@ class XenStorage(QubesVmStorage): class XenPool(Pool): - pass + def __init__(self, vm): + self.vm = vm + + def getStorage(self): + return defaults['storage_class'](self.vm) diff --git a/tests/storage.py b/tests/storage.py index e3fb4b29..3115535d 100644 --- a/tests/storage.py +++ b/tests/storage.py @@ -30,7 +30,7 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase): """ Dumps storage instance to a storage string """ vmname = self.make_vm_name('appvm') template = self.qc.get_default_template() - storage = self.qc.add_new_vm('QubesAppVm', name=vmname, pool='default', + storage = self.qc.add_new_vm('QubesAppVm', name=vmname, pool_name='default', template=template).storage result = qubes.storage.dump(storage) expected = 'qubes.storage.xen.XenStorage' @@ -68,5 +68,5 @@ class TC_01_Storage(SystemTestsMixin, QubesTestCase): vmname = self.make_vm_name('appvm') template = self.qc.get_default_template() vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template, - pool='default') + pool_name='default') self.assertIsInstance(vm.storage, XenStorage) From 26711e7e9aa64b18d9b8e00e567c6dc4e2f86992 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sat, 7 Nov 2015 20:19:17 +0100 Subject: [PATCH 008/270] Use default pool_config if pool name is 'default' --- core/settings-xen-Linux.py | 1 + core/storage/__init__.py | 9 ++++++++- core/storage/xen.py | 6 +++++- tests/storage.py | 34 +++++++++++++++++++++++++++------- 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/core/settings-xen-Linux.py b/core/settings-xen-Linux.py index 30669407..74503524 100644 --- a/core/settings-xen-Linux.py +++ b/core/settings-xen-Linux.py @@ -8,3 +8,4 @@ from qubes.storage.xen import XenStorage, XenPool def apply(system_path, vm_files, defaults): defaults['storage_class'] = XenStorage defaults['pool_types'] = {'xen': XenPool} + defaults['pool_config'] = {'dir': '/var/lib/qubes/'} diff --git a/core/storage/__init__.py b/core/storage/__init__.py index 4636dc47..39f623d5 100644 --- a/core/storage/__init__.py +++ b/core/storage/__init__.py @@ -247,7 +247,14 @@ def get_pool(name, vm): keys = [k for k in config.options(name) if k != 'type' and k != 'class'] values = [config.get(name, o) for o in keys] - kwargs = dict(zip(keys, values)) + config_kwargs = dict(zip(keys, values)) + + if name == 'default': + kwargs = defaults['pool_config'].copy() + kwargs.update(keys) + else: + kwargs = config_kwargs + return klass(vm, **kwargs) diff --git a/core/storage/xen.py b/core/storage/xen.py index 01ed0277..06bd0186 100644 --- a/core/storage/xen.py +++ b/core/storage/xen.py @@ -252,8 +252,12 @@ class XenStorage(QubesVmStorage): class XenPool(Pool): - def __init__(self, vm): + def __init__(self, vm, dir): + assert vm is not None + assert dir is not None + self.vm = vm + self.dir = dir def getStorage(self): return defaults['storage_class'](self.vm) diff --git a/tests/storage.py b/tests/storage.py index 3115535d..b5a0defc 100644 --- a/tests/storage.py +++ b/tests/storage.py @@ -30,8 +30,9 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase): """ Dumps storage instance to a storage string """ vmname = self.make_vm_name('appvm') template = self.qc.get_default_template() - storage = self.qc.add_new_vm('QubesAppVm', name=vmname, pool_name='default', - template=template).storage + vm = self.qc.add_new_vm('QubesAppVm', name=vmname, + pool_name='default', template=template) + storage = vm.storage result = qubes.storage.dump(storage) expected = 'qubes.storage.xen.XenStorage' self.assertEquals(result, expected) @@ -62,11 +63,30 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase): qubes.storage.pool_exists('asdh312096r832598213iudhas')) -class TC_01_Storage(SystemTestsMixin, QubesTestCase): +class TC_00_Pool(SystemTestsMixin, QubesTestCase): - def test_000_vm_use_default_pool(self): + def test000_no_pool_dir(self): + """ If no pool dir ist configured for a ``XenPool`` assume the default + `/var/lib/qubes/`. + """ + vm = self._init_app_vm() + result = qubes.storage.get_pool("default", vm).dir + expected = '/var/lib/qubes/' + self.assertEquals(result, expected) + + def test001_default_storage_class(self): + """ Check if when using default pool the Storage is ``XenStorage``. """ + result = self._init_app_vm().storage + self.assertIsInstance(result, XenStorage) + + def test_002_pool_name(self): + """ Default pool_name is 'default'. """ + vm = self._init_app_vm() + self.assertEquals(vm.pool_name, "default") + + def _init_app_vm(self): + """ Return initalised, but not created, AppVm. """ vmname = self.make_vm_name('appvm') template = self.qc.get_default_template() - vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template, - pool_name='default') - self.assertIsInstance(vm.storage, XenStorage) + return self.qc.add_new_vm('QubesAppVm', name=vmname, template=template, + pool_name='default') From 16d480cf4c7dce683e48739bd581638e791eb8c1 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sat, 7 Nov 2015 20:53:50 +0100 Subject: [PATCH 009/270] Add storage add_pool & remove_pool --- core/storage/__init__.py | 34 ++++++++++++++++++++++++++-------- core/storage/storage.conf | 5 +++-- tests/storage.py | 21 +++++++++++++++++---- 3 files changed, 46 insertions(+), 14 deletions(-) diff --git a/core/storage/__init__.py b/core/storage/__init__.py index 39f623d5..ef8b27e0 100644 --- a/core/storage/__init__.py +++ b/core/storage/__init__.py @@ -258,6 +258,32 @@ def get_pool(name, vm): return klass(vm, **kwargs) +def pool_exists(name): + """ Check if the specified pool exists """ + try: + _get_pool_klass(name) + return True + except StoragePoolException: + return False + +def add_pool(name, **kwargs): + """ Add a storage pool to config.""" + config = _get_storage_config_parser() + config.add_section(name) + for key, value in kwargs.iteritems(): + config.set(name, key, value) + _write_config(config) + +def remove_pool(name): + """ Remove a storage pool from config file. """ + config = _get_storage_config_parser() + config.remove_section(name) + _write_config(config) + +def _write_config(config): + with open(CONFIG_FILE, 'w') as configfile: + config.write(configfile) + def _get_storage_config_parser(): """ Instantiates a `ConfigParaser` for specified storage config file. @@ -297,14 +323,6 @@ def _get_pool_klass(name, config=None): return klass -def pool_exists(name): - """ Check if the specified pool exists """ - try: - _get_pool_klass(name) - return True - except StoragePoolException: - return False - class StoragePoolException(QubesException): pass diff --git a/core/storage/storage.conf b/core/storage/storage.conf index 8fa44c7e..45b6bda0 100644 --- a/core/storage/storage.conf +++ b/core/storage/storage.conf @@ -6,6 +6,7 @@ type=xen ; the default xen storage ; class name ; [pool-b] ; class = foo.bar.MyStorage -; - +; +; [test-dummy] +; type=dummy diff --git a/tests/storage.py b/tests/storage.py index b5a0defc..d016fb1a 100644 --- a/tests/storage.py +++ b/tests/storage.py @@ -17,11 +17,11 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -from qubes.tests import QubesTestCase, SystemTestsMixin -from qubes.qubes import defaults - import qubes.storage -from qubes.storage.xen import XenStorage, XenPool +from qubes.qubes import defaults +from qubes.tests import QubesTestCase, SystemTestsMixin + +from qubes.storage.xen import XenPool, XenStorage class TC_00_Storage(SystemTestsMixin, QubesTestCase): @@ -62,6 +62,19 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase): self.assertFalse( qubes.storage.pool_exists('asdh312096r832598213iudhas')) + def test_006_add_remove_pool(self): + """ Tries to adding and removing a pool. """ + pool_name = 'asdjhrp89132' + + # make sure it's really does not exist + qubes.storage.remove_pool(pool_name) + + qubes.storage.add_pool(pool_name, type='xen') + self.assertTrue(qubes.storage.pool_exists(pool_name)) + + qubes.storage.remove_pool(pool_name) + self.assertFalse(qubes.storage.pool_exists(pool_name)) + class TC_00_Pool(SystemTestsMixin, QubesTestCase): From 58f23ca3923e6521c278b0a20f95a9d2464e3dd4 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sat, 7 Nov 2015 22:31:50 +0100 Subject: [PATCH 010/270] Add configurable pool_dir to XenPool --- core/storage/xen.py | 61 ++++++++++++++++++++-- tests/storage.py | 121 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 176 insertions(+), 6 deletions(-) diff --git a/core/storage/xen.py b/core/storage/xen.py index 06bd0186..853f0268 100644 --- a/core/storage/xen.py +++ b/core/storage/xen.py @@ -28,7 +28,9 @@ import re import subprocess import sys -from qubes.qubes import QubesException, defaults, vm_files +from qubes.qubes import (QubesAdminVm, QubesAppVm, QubesException, QubesHVm, + QubesNetVm, QubesProxyVm, QubesTemplateHVm, + QubesTemplateVm, defaults, vm_files) from qubes.storage import Pool, QubesVmStorage @@ -37,7 +39,16 @@ class XenStorage(QubesVmStorage): Class for VM storage of Xen VMs. """ - def __init__(self, vm, **kwargs): + def __init__(self, vm, vmdir, **kwargs): + """ Instantiate the storage. + + Args: + vm: a QubesVM + vmdir: the root directory of the pool + """ + assert vm is not None + assert vmdir is not None + super(XenStorage, self).__init__(vm, **kwargs) self.root_dev = "xvda" @@ -45,8 +56,11 @@ class XenStorage(QubesVmStorage): self.volatile_dev = "xvdc" self.modules_dev = "xvdd" + self.vmdir = vmdir + if self.vm.is_template(): - self.rootcow_img = os.path.join(self.vmdir, vm_files["rootcow_img"]) + self.rootcow_img = os.path.join(self.vmdir, + vm_files["rootcow_img"]) else: self.rootcow_img = None @@ -252,12 +266,51 @@ class XenStorage(QubesVmStorage): class XenPool(Pool): + def __init__(self, vm, dir): assert vm is not None assert dir is not None + if not os.path.exists(dir): + os.mkdir(dir) + + self.vmdir = self._vmdir_path(vm, dir) self.vm = vm self.dir = dir def getStorage(self): - return defaults['storage_class'](self.vm) + """ Returns an instantiated ``XenStorage``. """ + return defaults['storage_class'](self.vm, vmdir=self.vmdir) + + def _vmdir_path(self, vm, pool_dir): + """ Get the vm dir depending on the type of the VM. + + The default QubesOS file storage saves the vm images in three + different directories depending on the ``QubesVM`` type: + + * ``appvms`` for ``QubesAppVm`` or ``QubesHvm`` + * ``vm-templates`` for ``QubesTemplateVm`` or ``QubesTemplateHvm`` + * ``servicevms`` for ``QubesProxyVm``, ``QubesNetVm`` or + ``QubesAdminVm`` + + Args: + vm: a QubesVM + pool_dir: the root directory of the pool + + Returns: + string (str) absolute path to the directory where the vm files + are stored + """ + vm_type = type(vm) + + if vm_type in [QubesAppVm, QubesHVm]: + subdir = 'appvms' + elif vm_type in [QubesTemplateVm, QubesTemplateHVm]: + subdir = 'vm-templates' + elif vm_type in [QubesAdminVm, QubesNetVm, QubesProxyVm]: + subdir = 'servicevms' + else: + raise QubesException(str(vm_type) + ' unknown vm type') + + return os.path.join(pool_dir, subdir, vm.name) + diff --git a/tests/storage.py b/tests/storage.py index d016fb1a..e1b408f7 100644 --- a/tests/storage.py +++ b/tests/storage.py @@ -17,11 +17,13 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import os +import shutil + import qubes.storage from qubes.qubes import defaults -from qubes.tests import QubesTestCase, SystemTestsMixin - from qubes.storage.xen import XenPool, XenStorage +from qubes.tests import QubesTestCase, SystemTestsMixin class TC_00_Storage(SystemTestsMixin, QubesTestCase): @@ -103,3 +105,118 @@ class TC_00_Pool(SystemTestsMixin, QubesTestCase): template = self.qc.get_default_template() return self.qc.add_new_vm('QubesAppVm', name=vmname, template=template, pool_name='default') + + +class TC_01_Pool(SystemTestsMixin, QubesTestCase): + + """ Test the paths for the default Xen file based storage + (``QubesXenVmStorage``). + """ + + POOL_DIR = '/var/lib/qubes/test-pool' + APPVMS_DIR = '/var/lib/qubes/test-pool/appvms' + TEMPLATES_DIR = '/var/lib/qubes/test-pool/vm-templates' + SERVICE_DIR = '/var/lib/qubes/test-pool/servicevms' + + def setUp(self): + """ Add a test file based storage pool """ + super(TC_01_Pool, self).setUp() + qubes.storage.add_pool('test-pool', type='xen', dir=self.POOL_DIR) + + def tearDown(self): + """ Remove the file based storage pool after testing """ + super(TC_01_Pool, self).tearDown() + qubes.storage.remove_pool("test-pool") + shutil.rmtree(self.POOL_DIR, ignore_errors=True) + + def test_001_pool_exists(self): + """ Check if the storage pool was added to the storage pool config """ + self.assertTrue(qubes.storage.pool_exists('test-pool')) + + def test_002_pool_dir_create(self): + """ Check if the storage pool dir was created """ + + # The dir should not exists before + self.assertFalse(os.path.exists(self.POOL_DIR)) + + vmname = self.make_vm_name('appvm') + template = self.qc.get_default_template() + self.qc.add_new_vm('QubesAppVm', name=vmname, template=template, + pool_name='test-pool') + + self.assertTrue(os.path.exists(self.POOL_DIR)) + + def test_003_pool_dir(self): + """ Check if the vm storage pool_dir is the same as specified """ + vmname = self.make_vm_name('appvm') + template = self.qc.get_default_template() + vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template, + pool_name='test-pool') + result = qubes.storage.get_pool('test-pool', vm).dir + self.assertEquals(self.POOL_DIR, result) + + def test_004_app_vmdir(self): + """ Check the vm storage dir for an AppVm""" + vmname = self.make_vm_name('appvm') + template = self.qc.get_default_template() + vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template, + pool_name='test-pool') + + expected = os.path.join(self.APPVMS_DIR, vm.name) + result = vm.storage.vmdir + self.assertEquals(expected, result) + + def test_005_hvm_vmdir(self): + """ Check the vm storage dir for a HVM""" + vmname = self.make_vm_name('hvm') + vm = self.qc.add_new_vm('QubesHVm', name=vmname, + pool_name='test-pool') + + expected = os.path.join(self.APPVMS_DIR, vm.name) + result = vm.storage.vmdir + self.assertEquals(expected, result) + + def test_006_net_vmdir(self): + """ Check the vm storage dir for a Netvm""" + vmname = self.make_vm_name('hvm') + vm = self.qc.add_new_vm('QubesNetVm', name=vmname, + pool_name='test-pool') + + expected = os.path.join(self.SERVICE_DIR, vm.name) + result = vm.storage.vmdir + self.assertEquals(expected, result) + + def test_007_proxy_vmdir(self): + """ Check the vm storage dir for a ProxyVm""" + vmname = self.make_vm_name('proxyvm') + vm = self.qc.add_new_vm('QubesProxyVm', name=vmname, + pool_name='test-pool') + + expected = os.path.join(self.SERVICE_DIR, vm.name) + result = vm.storage.vmdir + self.assertEquals(expected, result) + + def test_008_admin_vmdir(self): + """ Check the vm storage dir for a AdminVm""" + # TODO How to test AdminVm? + pass + + def test_009_template_vmdir(self): + """ Check the vm storage dir for a TemplateVm""" + vmname = self.make_vm_name('templatevm') + vm = self.qc.add_new_vm('QubesTemplateVm', name=vmname, + pool_name='test-pool') + + expected = os.path.join(self.TEMPLATES_DIR, vm.name) + result = vm.storage.vmdir + self.assertEquals(expected, result) + + def test_010_template_hvm_vmdir(self): + """ Check the vm storage dir for a TemplateHVm""" + vmname = self.make_vm_name('templatehvm') + vm = self.qc.add_new_vm('QubesTemplateHVm', name=vmname, + pool_name='test-pool') + + expected = os.path.join(self.TEMPLATES_DIR, vm.name) + result = vm.storage.vmdir + self.assertEquals(expected, result) From 0bc0bc90454ad70c9ba15dc1ec89c60bb3bb6916 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sat, 7 Nov 2015 22:38:34 +0100 Subject: [PATCH 011/270] XenStorage make sure subdirs exist in pool dir --- core/storage/xen.py | 17 +++++++++++++++-- tests/storage.py | 5 ++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/core/storage/xen.py b/core/storage/xen.py index 853f0268..1647eeb8 100644 --- a/core/storage/xen.py +++ b/core/storage/xen.py @@ -271,8 +271,14 @@ class XenPool(Pool): assert vm is not None assert dir is not None - if not os.path.exists(dir): - os.mkdir(dir) + appvms_path = os.path.join(dir, 'appvms') + servicevms_path = os.path.join(dir, 'servicevms') + vm_templates_path = os.path.join(dir, 'vm-templates') + + self._create_dir_if_not_exists(dir) + self._create_dir_if_not_exists(appvms_path) + self._create_dir_if_not_exists(servicevms_path) + self._create_dir_if_not_exists(vm_templates_path) self.vmdir = self._vmdir_path(vm, dir) self.vm = vm @@ -314,3 +320,10 @@ class XenPool(Pool): return os.path.join(pool_dir, subdir, vm.name) + def _create_dir_if_not_exists(self, path): + """ Check if a directory exists in if not it is created. + + This method does not create any parent directories. + """ + if not os.path.exists(path): + os.mkdir(path) diff --git a/tests/storage.py b/tests/storage.py index e1b408f7..8a5f949f 100644 --- a/tests/storage.py +++ b/tests/storage.py @@ -134,7 +134,7 @@ class TC_01_Pool(SystemTestsMixin, QubesTestCase): self.assertTrue(qubes.storage.pool_exists('test-pool')) def test_002_pool_dir_create(self): - """ Check if the storage pool dir was created """ + """ Check if the storage pool dir and subdirs were created """ # The dir should not exists before self.assertFalse(os.path.exists(self.POOL_DIR)) @@ -145,6 +145,9 @@ class TC_01_Pool(SystemTestsMixin, QubesTestCase): pool_name='test-pool') self.assertTrue(os.path.exists(self.POOL_DIR)) + self.assertTrue(os.path.exists(self.APPVMS_DIR)) + self.assertTrue(os.path.exists(self.SERVICE_DIR)) + self.assertTrue(os.path.exists(self.TEMPLATES_DIR)) def test_003_pool_dir(self): """ Check if the vm storage pool_dir is the same as specified """ From 348030bf9c7cd5685e6d462973c9f7b35b1fc6cd Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sat, 7 Nov 2015 19:10:55 +0100 Subject: [PATCH 012/270] Add --storage-pool option to qvm-create --- qvm-tools/qvm-create | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/qvm-tools/qvm-create b/qvm-tools/qvm-create index a014febf..6ae002e5 100755 --- a/qvm-tools/qvm-create +++ b/qvm-tools/qvm-create @@ -41,6 +41,8 @@ def main(): help="Specify the label to use for the new VM (e.g. red, yellow, green, ...)") parser.add_option ("-p", "--proxy", action="store_true", dest="proxyvm", default=False, help="Create ProxyVM") + parser.add_option ("-P", "--pool", dest="pool_name", + help="Specify which storage pool to use") parser.add_option ("-H", "--hvm", action="store_true", dest="hvm", default=False, help="Create HVM (standalone unless --template option used)") parser.add_option ("--hvm-template", action="store_true", dest="hvm_template", default=False, @@ -71,6 +73,11 @@ def main(): parser.error ("You must specify VM name!") vmname = args[0] + if options.pool_name is None: + pool_name = "default" + else : + pool_name = options.pool_name + if (options.netvm + options.proxyvm + options.hvm + options.hvm_template) > 1: parser.error ("You must specify at most one VM type switch") @@ -170,7 +177,9 @@ def main(): vmtype = "QubesAppVm" try: - vm = qvm_collection.add_new_vm(vmtype, name=vmname, template=new_vm_template, label = label) + vm=qvm_collection.add_new_vm(vmtype, name=vmname, + template=new_vm_template, label=label, + pool_name=pool_name) except QubesException as err: print >> sys.stderr, "ERROR: {0}".format(err) exit (1) From 76224dac8622d85a28a39113d88d5f387a936b72 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sun, 8 Nov 2015 19:29:41 +0100 Subject: [PATCH 013/270] Path to the Vm images is set by the storage - This moves the logic for setting the path to the storage specific class like XenStore --- core/storage/__init__.py | 6 ------ core/storage/xen.py | 7 +++++++ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/core/storage/__init__.py b/core/storage/__init__.py index ef8b27e0..f835d964 100644 --- a/core/storage/__init__.py +++ b/core/storage/__init__.py @@ -57,12 +57,6 @@ class QubesVmStorage(object): else: self.root_img_size = defaults['root_img_size'] - self.private_img = vm.absolute_path(vm_files["private_img"], None) - if self.vm.template: - self.root_img = self.vm.template.root_img - else: - self.root_img = vm.absolute_path(vm_files["root_img"], None) - self.volatile_img = vm.absolute_path(vm_files["volatile_img"], None) # For now compute this path still in QubesVm self.modules_img = modules_img diff --git a/core/storage/xen.py b/core/storage/xen.py index 1647eeb8..63d856e9 100644 --- a/core/storage/xen.py +++ b/core/storage/xen.py @@ -64,6 +64,13 @@ class XenStorage(QubesVmStorage): else: self.rootcow_img = None + self.private_img = os.path.join(vmdir, 'private.img') + if self.vm.template: + self.root_img = self.vm.template.root_img + else: + self.root_img = os.path.join(vmdir, 'root.img') + self.volatile_img = os.path.join(vmdir, 'volatile.img') + def _format_disk_dev(self, path, script, vdev, rw=True, type="disk", domain=None): if path is None: return '' From 710b95a6ac203af85ff5f95dcbf81109f3d602cb Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sun, 8 Nov 2015 21:22:44 +0100 Subject: [PATCH 014/270] Fix app icons errors --- core-modules/000QubesVm.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py index 615652db..eef88fbf 100644 --- a/core-modules/000QubesVm.py +++ b/core-modules/000QubesVm.py @@ -347,6 +347,13 @@ class QubesVm(object): # Initialize VM image storage class self.storage = get_pool(self.pool_name, self).getStorage() + self.dir_path = self.storage.vmdir + self.icon_path = os.path.join(self.storage.vmdir, 'icon.png') + self.conf_file = os.path.join(self.storage.vmdir, self.name + '.conf') + self.appmenus_templates_dir = os.path.join(self.storage.vmdir, 'apps.templates') + self.appmenus_icons_dir = os.path.join(self.storage.vmdir, 'apps.icons') + + if hasattr(self, 'kernels_dir'): modules_path = os.path.join(self.kernels_dir, "modules.img") From 0ba0259d1a357cf379ccb251242543a42cece20b Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sun, 8 Nov 2015 21:23:07 +0100 Subject: [PATCH 015/270] Extend documentation test/storage.py --- tests/storage.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/storage.py b/tests/storage.py index 8a5f949f..763d1f1a 100644 --- a/tests/storage.py +++ b/tests/storage.py @@ -28,6 +28,8 @@ from qubes.tests import QubesTestCase, SystemTestsMixin class TC_00_Storage(SystemTestsMixin, QubesTestCase): + """ This class tests the utility methods from :mod:``qubes.storage`` """ + def test_000_dump(self): """ Dumps storage instance to a storage string """ vmname = self.make_vm_name('appvm') @@ -80,9 +82,13 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase): class TC_00_Pool(SystemTestsMixin, QubesTestCase): - def test000_no_pool_dir(self): - """ If no pool dir ist configured for a ``XenPool`` assume the default - `/var/lib/qubes/`. + """ This class tests some properties of the 'default' pool. """ + + def test000_default_pool_dir(self): + """ The predefined dir for the default pool should be ``/var/lib/qubes`` + + .. sealso:: + Data :data:``qubes.qubes.defaults['pool_config']``. """ vm = self._init_app_vm() result = qubes.storage.get_pool("default", vm).dir @@ -90,11 +96,11 @@ class TC_00_Pool(SystemTestsMixin, QubesTestCase): self.assertEquals(result, expected) def test001_default_storage_class(self): - """ Check if when using default pool the Storage is ``XenStorage``. """ + """ Check when using default pool the Storage is ``XenStorage``. """ result = self._init_app_vm().storage self.assertIsInstance(result, XenStorage) - def test_002_pool_name(self): + def test_002_default_pool_name(self): """ Default pool_name is 'default'. """ vm = self._init_app_vm() self.assertEquals(vm.pool_name, "default") @@ -109,8 +115,7 @@ class TC_00_Pool(SystemTestsMixin, QubesTestCase): class TC_01_Pool(SystemTestsMixin, QubesTestCase): - """ Test the paths for the default Xen file based storage - (``QubesXenVmStorage``). + """ Test the paths for the default Xen file based storage (``XenStorage``). """ POOL_DIR = '/var/lib/qubes/test-pool' From 989eab1d72fab8e6ee51acc1231f241501a2af65 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sun, 8 Nov 2015 21:29:57 +0100 Subject: [PATCH 016/270] Add file image tests to tests/storage.py --- tests/storage.py | 54 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/storage.py b/tests/storage.py index 763d1f1a..873d6e71 100644 --- a/tests/storage.py +++ b/tests/storage.py @@ -228,3 +228,57 @@ class TC_01_Pool(SystemTestsMixin, QubesTestCase): expected = os.path.join(self.TEMPLATES_DIR, vm.name) result = vm.storage.vmdir self.assertEquals(expected, result) + + def test_011_appvm_file_images(self): + """ Check if all the needed image files are created for an AppVm""" + + vmname = self.make_vm_name('appvm') + template = self.qc.get_default_template() + vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template, + pool_name='test-pool') + vm.create_on_disk(verbose=False) + + expected_vmdir = os.path.join(self.APPVMS_DIR, vm.name) + self.assertEqualsAndExists(vm.storage.vmdir, expected_vmdir) + + expected_private_path = os.path.join(expected_vmdir, 'private.img') + self.assertEqualsAndExists(vm.storage.private_img, + expected_private_path) + + expected_volatile_path = os.path.join(expected_vmdir, 'volatile.img') + self.assertEqualsAndExists(vm.storage.volatile_img, + expected_volatile_path) + + def test_012_hvm_file_images(self): + """ Check if all the needed image files are created for a HVm""" + + vmname = self.make_vm_name('hvm') + vm = self.qc.add_new_vm('QubesHVm', name=vmname, + pool_name='test-pool') + vm.create_on_disk(verbose=False) + + expected_vmdir = os.path.join(self.APPVMS_DIR, vm.name) + self.assertEqualsAndExists(vm.storage.vmdir, expected_vmdir) + + expected_private_path = os.path.join(expected_vmdir, 'private.img') + self.assertEqualsAndExists(vm.storage.private_img, + expected_private_path) + + expected_root_path = os.path.join(expected_vmdir, 'root.img') + self.assertEqualsAndExists(vm.storage.root_img, expected_root_path) + + expected_volatile_path = os.path.join(expected_vmdir, 'volatile.img') + self.assertEqualsAndExists(vm.storage.volatile_img, + expected_volatile_path) + + def assertEqualsAndExists(self, result_path, expected_path): + """ Check if the ``result_path``, matches ``expected_path`` and exists. + + See also: :meth:``assertExist`` + """ + self.assertEquals(result_path, expected_path) + self.assertExist(result_path) + + def assertExist(self, path): + """ Assert that the given path exists. """ + self.assertTrue(os.path.exists(path)) From 1934f06869fc280892db593321e90737c9bbb1e3 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Tue, 17 Nov 2015 21:15:13 +0100 Subject: [PATCH 017/270] XenStorage add DisposableVm handling --- core/storage/xen.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/core/storage/xen.py b/core/storage/xen.py index 63d856e9..21c15f8c 100644 --- a/core/storage/xen.py +++ b/core/storage/xen.py @@ -28,9 +28,9 @@ import re import subprocess import sys -from qubes.qubes import (QubesAdminVm, QubesAppVm, QubesException, QubesHVm, - QubesNetVm, QubesProxyVm, QubesTemplateHVm, - QubesTemplateVm, defaults, vm_files) +from qubes.qubes import (QubesAdminVm, QubesAppVm, QubesDisposableVm, + QubesException, QubesHVm, QubesNetVm, QubesProxyVm, + QubesTemplateHVm, QubesTemplateVm, defaults, vm_files) from qubes.storage import Pool, QubesVmStorage @@ -322,6 +322,9 @@ class XenPool(Pool): subdir = 'vm-templates' elif vm_type in [QubesAdminVm, QubesNetVm, QubesProxyVm]: subdir = 'servicevms' + elif vm_type is QubesDisposableVm: + subdir = 'appvms' + return os.path.join(pool_dir, subdir, vm.template.name + '-dvm') else: raise QubesException(str(vm_type) + ' unknown vm type') From 9eee00c6d790c9513c7c92681f73987d2beeb537 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Tue, 17 Nov 2015 21:19:15 +0100 Subject: [PATCH 018/270] QubesNetVm and subclasses use servicevm/ as vmdir --- core/storage/xen.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/core/storage/xen.py b/core/storage/xen.py index 21c15f8c..e2b8482c 100644 --- a/core/storage/xen.py +++ b/core/storage/xen.py @@ -28,9 +28,9 @@ import re import subprocess import sys -from qubes.qubes import (QubesAdminVm, QubesAppVm, QubesDisposableVm, - QubesException, QubesHVm, QubesNetVm, QubesProxyVm, - QubesTemplateHVm, QubesTemplateVm, defaults, vm_files) +from qubes.qubes import (QubesAppVm, QubesDisposableVm, QubesException, + QubesHVm, QubesNetVm, QubesTemplateHVm, + QubesTemplateVm, defaults, vm_files) from qubes.storage import Pool, QubesVmStorage @@ -303,8 +303,7 @@ class XenPool(Pool): * ``appvms`` for ``QubesAppVm`` or ``QubesHvm`` * ``vm-templates`` for ``QubesTemplateVm`` or ``QubesTemplateHvm`` - * ``servicevms`` for ``QubesProxyVm``, ``QubesNetVm`` or - ``QubesAdminVm`` + * ``servicevms`` for any subclass of ``QubesNetVm`` Args: vm: a QubesVM @@ -320,7 +319,7 @@ class XenPool(Pool): subdir = 'appvms' elif vm_type in [QubesTemplateVm, QubesTemplateHVm]: subdir = 'vm-templates' - elif vm_type in [QubesAdminVm, QubesNetVm, QubesProxyVm]: + elif issubclass(vm_type, QubesNetVm): subdir = 'servicevms' elif vm_type is QubesDisposableVm: subdir = 'appvms' From 885dc5cd81ca28d1354a266746d60aeb72b14ced Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sat, 21 Nov 2015 14:46:23 +0100 Subject: [PATCH 019/270] Rename storage dir to dir_path, type to driver Bad idea to use attributes which are named after built in functions. --- core/settings-xen-Linux.py | 4 ++-- core/storage/__init__.py | 16 +++++++--------- core/storage/storage.conf | 6 +++--- core/storage/xen.py | 16 ++++++++-------- tests/storage.py | 17 +++++++++-------- 5 files changed, 29 insertions(+), 30 deletions(-) diff --git a/core/settings-xen-Linux.py b/core/settings-xen-Linux.py index 74503524..c413e8ae 100644 --- a/core/settings-xen-Linux.py +++ b/core/settings-xen-Linux.py @@ -7,5 +7,5 @@ from qubes.storage.xen import XenStorage, XenPool def apply(system_path, vm_files, defaults): defaults['storage_class'] = XenStorage - defaults['pool_types'] = {'xen': XenPool} - defaults['pool_config'] = {'dir': '/var/lib/qubes/'} + defaults['pool_drivers'] = {'xen': XenPool} + defaults['pool_config'] = {'dir_path': '/var/lib/qubes/'} diff --git a/core/storage/__init__.py b/core/storage/__init__.py index f835d964..8e2ac720 100644 --- a/core/storage/__init__.py +++ b/core/storage/__init__.py @@ -239,7 +239,7 @@ def get_pool(name, vm): klass = _get_pool_klass(name, config) - keys = [k for k in config.options(name) if k != 'type' and k != 'class'] + keys = [k for k in config.options(name) if k != 'driver' and k != 'class'] values = [config.get(name, o) for o in keys] config_kwargs = dict(zip(keys, values)) @@ -308,19 +308,17 @@ def _get_pool_klass(name, config=None): if config.has_option(name, 'class'): klass = load(config.get(name, 'class')) - elif config.has_option(name, 'type'): - pool_type = config.get(name, 'type') - klass = defaults['pool_types'][pool_type] - - if klass is None: - raise StoragePoolException('Uknown storage pool type ' + name) + elif config.has_option(name, 'driver'): + pool_driver = config.get(name, 'driver') + klass = defaults['pool_drivers'][pool_driver] + else: + raise StoragePoolException('Uknown storage pool driver ' + name) return klass - class StoragePoolException(QubesException): pass + class Pool(object): pass - diff --git a/core/storage/storage.conf b/core/storage/storage.conf index 45b6bda0..e9d067e5 100644 --- a/core/storage/storage.conf +++ b/core/storage/storage.conf @@ -1,6 +1,6 @@ [default] ; poolname -type=xen ; the default xen storage -; class = qubes.storage.xen.XenStorage ; class always overwrites type +driver=xen ; the default xen storage +; class = qubes.storage.xen.XenStorage ; class always overwrites the driver ; ; To use our own storage adapter, you need just to specify the module path and ; class name @@ -8,5 +8,5 @@ type=xen ; the default xen storage ; class = foo.bar.MyStorage ; ; [test-dummy] -; type=dummy +; driver=dummy diff --git a/core/storage/xen.py b/core/storage/xen.py index e2b8482c..1cd5dc97 100644 --- a/core/storage/xen.py +++ b/core/storage/xen.py @@ -274,22 +274,22 @@ class XenStorage(QubesVmStorage): class XenPool(Pool): - def __init__(self, vm, dir): + def __init__(self, vm, dir_path): assert vm is not None - assert dir is not None + assert dir_path is not None - appvms_path = os.path.join(dir, 'appvms') - servicevms_path = os.path.join(dir, 'servicevms') - vm_templates_path = os.path.join(dir, 'vm-templates') + appvms_path = os.path.join(dir_path, 'appvms') + servicevms_path = os.path.join(dir_path, 'servicevms') + vm_templates_path = os.path.join(dir_path, 'vm-templates') - self._create_dir_if_not_exists(dir) + self._create_dir_if_not_exists(dir_path) self._create_dir_if_not_exists(appvms_path) self._create_dir_if_not_exists(servicevms_path) self._create_dir_if_not_exists(vm_templates_path) - self.vmdir = self._vmdir_path(vm, dir) + self.vmdir = self._vmdir_path(vm, dir_path) self.vm = vm - self.dir = dir + self.dir_path = dir_path def getStorage(self): """ Returns an instantiated ``XenStorage``. """ diff --git a/tests/storage.py b/tests/storage.py index 873d6e71..5513d383 100644 --- a/tests/storage.py +++ b/tests/storage.py @@ -42,13 +42,13 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase): self.assertEquals(result, expected) def test_001_load(self): - """ Loads storage type from a storage string """ + """ Loads storage driver from a storage string """ result = qubes.storage.load('qubes.storage.xen.XenStorage') self.assertTrue(result is XenStorage) - def test_002_default_pool_types(self): - """ The only predifined pool type is xen """ - result = defaults['pool_types'].keys() + def test_002_default_pool_drivers(self): + """ The only predifined pool driver is xen """ + result = defaults['pool_drivers'].keys() expected = ["xen"] self.assertEquals(result, expected) @@ -73,7 +73,7 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase): # make sure it's really does not exist qubes.storage.remove_pool(pool_name) - qubes.storage.add_pool(pool_name, type='xen') + qubes.storage.add_pool(pool_name, driver='xen') self.assertTrue(qubes.storage.pool_exists(pool_name)) qubes.storage.remove_pool(pool_name) @@ -91,7 +91,7 @@ class TC_00_Pool(SystemTestsMixin, QubesTestCase): Data :data:``qubes.qubes.defaults['pool_config']``. """ vm = self._init_app_vm() - result = qubes.storage.get_pool("default", vm).dir + result = qubes.storage.get_pool("default", vm).dir_path expected = '/var/lib/qubes/' self.assertEquals(result, expected) @@ -126,7 +126,8 @@ class TC_01_Pool(SystemTestsMixin, QubesTestCase): def setUp(self): """ Add a test file based storage pool """ super(TC_01_Pool, self).setUp() - qubes.storage.add_pool('test-pool', type='xen', dir=self.POOL_DIR) + qubes.storage.add_pool('test-pool', driver='xen', + dir_path=self.POOL_DIR) def tearDown(self): """ Remove the file based storage pool after testing """ @@ -160,7 +161,7 @@ class TC_01_Pool(SystemTestsMixin, QubesTestCase): template = self.qc.get_default_template() vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template, pool_name='test-pool') - result = qubes.storage.get_pool('test-pool', vm).dir + result = qubes.storage.get_pool('test-pool', vm).dir_path self.assertEquals(self.POOL_DIR, result) def test_004_app_vmdir(self): From 2ebd20049d9193b7dcc4117095c2ce8892434813 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sun, 22 Nov 2015 12:21:40 +0100 Subject: [PATCH 020/270] Refactor appmenus paths to core-admin-linux --- core-modules/000QubesVm.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py index eef88fbf..de4f8851 100644 --- a/core-modules/000QubesVm.py +++ b/core-modules/000QubesVm.py @@ -350,9 +350,6 @@ class QubesVm(object): self.dir_path = self.storage.vmdir self.icon_path = os.path.join(self.storage.vmdir, 'icon.png') self.conf_file = os.path.join(self.storage.vmdir, self.name + '.conf') - self.appmenus_templates_dir = os.path.join(self.storage.vmdir, 'apps.templates') - self.appmenus_icons_dir = os.path.join(self.storage.vmdir, 'apps.icons') - if hasattr(self, 'kernels_dir'): modules_path = os.path.join(self.kernels_dir, From c337494ba675933773e89a4431c0dc87d7027de4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 23 Nov 2015 16:26:44 +0100 Subject: [PATCH 021/270] Add qvm-pci --add-class option for adding all devices of given class This should reduce code duplication in most of initial setup like places - currently firstboot and live image setup. --- doc/qvm-tools/qvm-pci.rst | 4 ++++ qvm-tools/qvm-pci | 49 +++++++++++++++++++++++++++++++++++---- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/doc/qvm-tools/qvm-pci.rst b/doc/qvm-tools/qvm-pci.rst index c2e667f8..80e5272f 100644 --- a/doc/qvm-tools/qvm-pci.rst +++ b/doc/qvm-tools/qvm-pci.rst @@ -23,6 +23,10 @@ OPTIONS List VM PCI devices -a, --add Add a PCI device to specified VM +-C, --add-class + Add all devices of given class: + net - network interfaces, + usb - USB controllers -d, --delete Remove a PCI device from specified VM diff --git a/qvm-tools/qvm-pci b/qvm-tools/qvm-pci index 80ca4a1f..c9c715ec 100755 --- a/qvm-tools/qvm-pci +++ b/qvm-tools/qvm-pci @@ -27,6 +27,25 @@ import subprocess import os import sys from qubes.qubes import vmm +import re + + +def find_devices_of_class(klass): + p = subprocess.Popen(["/sbin/lspci", "-mm", "-n"], stdout=subprocess.PIPE) + result = p.communicate() + retcode = p.returncode + if retcode != 0: + print "ERROR when executing lspci!" + raise IOError + + rx_netdev = re.compile(r"^([0-9][0-9]:[0-9][0-9].[0-9]) \"{}".format( + klass)) + for dev in str(result[0]).splitlines(): + match = rx_netdev.match(dev) + if match is not None: + dev_bdf = match.group(1) + assert dev_bdf is not None + yield dev_bdf def main(): @@ -39,6 +58,9 @@ def main(): parser.add_option ("-l", "--list", action="store_true", dest="do_list", default=False) parser.add_option ("-a", "--add", action="store_true", dest="do_add", default=False) parser.add_option ("-d", "--delete", action="store_true", dest="do_delete", default=False) + parser.add_option("-C", "--add-class", action="store_true", + dest="do_add_class", default=False, + help="Add all devices of given class (net, usb)") parser.add_option ("--offline-mode", dest="offline_mode", action="store_true", default=False, help="Offline mode") @@ -49,14 +71,15 @@ def main(): vmname = args[0] - if options.do_list + options.do_add + options.do_delete > 1: - print >> sys.stderr, "Only one of -l -a -d is allowed!" - exit (1) + if options.do_list + options.do_add + options.do_delete + \ + options.do_add_class > 1: + print >> sys.stderr, "Only one of -l -a -d -C is allowed!" + exit(1) if options.offline_mode: vmm.offline_mode = True - if options.do_add or options.do_delete: + if options.do_add or options.do_delete or options.add_class: qvm_collection = QubesVmCollection() qvm_collection.lock_db_for_writing() qvm_collection.load() @@ -81,6 +104,24 @@ def main(): qvm_collection.save() qvm_collection.unlock_db() + elif options.do_add_class: + if len(args) < 2: + print >> sys.stderr, "You must specify the PCI device class to add" + exit(1) + + klass = args[1] + + if klass == 'net': + devs = find_devices_of_class("02") + elif klass == 'usb': + devs = find_devices_of_class("0c03") + else: + print >> sys.stderr, "Supported classes: net, usb" + exit(1) + + for dev in devs: + vm.pci_add(dev) + elif options.do_delete: if len (args) < 2: print >> sys.stderr, "You must specify the PCI device to delete" From 0c476f014d7014d93d7d2a12fce086f19a311a7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 25 Nov 2015 00:59:54 +0100 Subject: [PATCH 022/270] backup: avoid deadlock when VM process fails If SendWorker queue is full, check if that thread is still alive. Otherwise it would deadlock on putting an entry to that queue. This also requires that SendWorker must ensure that the main thread isn't currently waiting for queue space when it fails. We can do this by simply removing an entry from a queue - so on the next iteration SendWorker would be already dead and main thread would notice it. Getting an entry from queue in such (error) situation is harmless, because other checks will notice it's an error condition. Fixes QubesOS/qubes-issues#1359 --- core/backup.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/core/backup.py b/core/backup.py index 6b20212c..c84d7b39 100644 --- a/core/backup.py +++ b/core/backup.py @@ -400,6 +400,10 @@ class SendWorker(Process): stdin=subprocess.PIPE, stdout=self.backup_stdout) if final_proc.wait() >= 2: + if self.queue.full(): + # if queue is already full, remove some entry to wake up + # main thread, so it will be able to notice error + self.queue.get() # handle only exit code 2 (tar fatal error) or # greater (call failed?) raise QubesException( @@ -448,6 +452,17 @@ def backup_do(base_backup_dir, files_to_backup, passphrase, crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM): global running_backup_operation + def queue_put_with_check(proc, vmproc, queue, element): + if queue.full(): + if not proc.is_alive(): + if vmproc: + message = ("Failed to write the backup, VM output:\n" + + vmproc.stderr.read()) + else: + message = "Failed to write the backup. Out of disk space?" + raise QubesException(message) + queue.put(element) + total_backup_sz = 0 passphrase = passphrase.encode('utf-8') for f in files_to_backup: @@ -650,7 +665,9 @@ def backup_do(base_backup_dir, files_to_backup, passphrase, run_error) # Send the chunk to the backup target - to_send.put(os.path.relpath(chunkfile, backup_tmpdir)) + queue_put_with_check( + send_proc, vmproc, to_send, + os.path.relpath(chunkfile, backup_tmpdir)) # Close HMAC hmac.stdin.close() @@ -668,7 +685,9 @@ def backup_do(base_backup_dir, files_to_backup, passphrase, hmac_file.close() # Send the HMAC to the backup target - to_send.put(os.path.relpath(chunkfile, backup_tmpdir) + ".hmac") + queue_put_with_check( + send_proc, vmproc, to_send, + os.path.relpath(chunkfile, backup_tmpdir) + ".hmac") if tar_sparse.poll() is None or run_error == "size_limit": run_error = "paused" @@ -680,7 +699,7 @@ def backup_do(base_backup_dir, files_to_backup, passphrase, .poll() pipe.close() - to_send.put("FINISHED") + queue_put_with_check(send_proc, vmproc, to_send, "FINISHED") send_proc.join() shutil.rmtree(backup_tmpdir) From 760786d999127ca226d78953edd76c69be9fb133 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 25 Nov 2015 01:08:15 +0100 Subject: [PATCH 023/270] tests: minor formating --- tests/__init__.py | 65 ++++++++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 35 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 8af1afbf..563ddf28 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -133,14 +133,12 @@ class QubesTestCase(unittest.TestCase): self.__class__.__name__, self._testMethodName)) - def __str__(self): return '{}/{}/{}'.format( '.'.join(self.__class__.__module__.split('.')[2:]), self.__class__.__name__, self._testMethodName) - def tearDown(self): super(QubesTestCase, self).tearDown() @@ -153,7 +151,6 @@ class QubesTestCase(unittest.TestCase): and filter((lambda (tc, exc): tc is self), l): raise BeforeCleanExit() - def assertNotRaises(self, excClass, callableObj=None, *args, **kwargs): """Fail if an exception of class excClass is raised by callableObj when invoked with arguments args and keyword @@ -183,15 +180,14 @@ class QubesTestCase(unittest.TestCase): with context: callableObj(*args, **kwargs) - def assertXMLEqual(self, xml1, xml2): - '''Check for equality of two XML objects. + """Check for equality of two XML objects. :param xml1: first element :param xml2: second element :type xml1: :py:class:`lxml.etree._Element` :type xml2: :py:class:`lxml.etree._Element` - ''' # pylint: disable=invalid-name + """ # pylint: disable=invalid-name self.assertEqual(xml1.tag, xml2.tag) self.assertEqual(xml1.text, xml2.text) @@ -202,13 +198,13 @@ class QubesTestCase(unittest.TestCase): class SystemTestsMixin(object): def setUp(self): - '''Set up the test. + """Set up the test. .. warning:: This method instantiates QubesVmCollection acquires write lock for it. You can use is as :py:attr:`qc`. You can (and probably should) release the lock at the end of setUp in subclass - ''' + """ super(SystemTestsMixin, self).setUp() @@ -220,12 +216,13 @@ class SystemTestsMixin(object): self.remove_test_vms() - def tearDown(self): super(SystemTestsMixin, self).tearDown() - try: self.qc.lock_db_for_writing() - except qubes.qubes.QubesException: pass + try: + self.qc.lock_db_for_writing() + except qubes.qubes.QubesException: + pass self.qc.load() self.remove_test_vms() @@ -236,7 +233,6 @@ class SystemTestsMixin(object): self.conn.close() - def make_vm_name(self, name): return VMPREFIX + name @@ -253,13 +249,18 @@ class SystemTestsMixin(object): # XXX .is_running() may throw libvirtError if undefined if vm.is_running(): vm.force_shutdown() - except: pass + except: + pass - try: vm.remove_from_disk() - except: pass + try: + vm.remove_from_disk() + except: + pass - try: vm.libvirt_domain.undefine() - except libvirt.libvirtError: pass + try: + vm.libvirt_domain.undefine() + except libvirt.libvirtError: + pass self.qc.pop(vm.qid) del vm @@ -268,13 +269,13 @@ class SystemTestsMixin(object): # for example if vm.libvirtDomain malfunctioned. try: dom = self.conn.lookupByName(vmname) - except: pass + except: + pass else: self._remove_vm_libvirt(dom) self._remove_vm_disk(vmname) - def _remove_vm_libvirt(self, dom): try: dom.destroy() @@ -282,7 +283,6 @@ class SystemTestsMixin(object): pass dom.undefine() - def _remove_vm_disk(self, vmname): for dirspec in ( 'qubes_appvms_dir', @@ -296,21 +296,19 @@ class SystemTestsMixin(object): else: os.unlink(dirpath) - def remove_vms(self, vms): for vm in vms: self._remove_vm_qubes(vm) self.save_and_reload_db() - def remove_test_vms(self): - '''Aggresively remove any domain that has name in testing namespace. + """Aggresively remove any domain that has name in testing namespace. .. warning:: The test suite hereby claims any domain whose name starts with :py:data:`VMPREFIX` as fair game. This is needed to enforce sane test executing environment. If you have domains named ``test-*``, don't run the tests. - ''' + """ # first, remove them Qubes-way something_removed = False @@ -392,28 +390,23 @@ class BackupTestsMixin(SystemTestsMixin): shutil.rmtree(self.backupdir) os.mkdir(self.backupdir) - def tearDown(self): super(BackupTestsMixin, self).tearDown() shutil.rmtree(self.backupdir) - def print_progress(self, progress): if self.verbose: print >> sys.stderr, "\r-> Backing up files: {0}%...".format(progress) - def error_callback(self, message): self.error_detected.put(message) if self.verbose: print >> sys.stderr, "ERROR: {0}".format(message) - def print_callback(self, msg): if self.verbose: print msg - def fill_image(self, path, size=None, sparse=False): block_size = 4096 @@ -432,7 +425,6 @@ class BackupTestsMixin(SystemTestsMixin): f.close() - # NOTE: this was create_basic_vms def create_backup_vms(self): template=self.qc.get_default_template() @@ -459,7 +451,6 @@ class BackupTestsMixin(SystemTestsMixin): return vms - def make_backup(self, vms, prepare_kwargs=dict(), do_kwargs=dict(), target=None): # XXX: bakup_prepare and backup_do don't support host_collection @@ -472,19 +463,24 @@ class BackupTestsMixin(SystemTestsMixin): print_callback=self.print_callback, **prepare_kwargs) except qubes.qubes.QubesException as e: - self.fail("QubesException during backup_prepare: %s" % str(e)) + if not expect_failure: + self.fail("QubesException during backup_prepare: %s" % str(e)) + else: + raise try: qubes.backup.backup_do(target, files_to_backup, "qubes", progress_callback=self.print_progress, **do_kwargs) except qubes.qubes.QubesException as e: - self.fail("QubesException during backup_do: %s" % str(e)) + if not expect_failure: + self.fail("QubesException during backup_do: %s" % str(e)) + else: + raise self.qc.lock_db_for_writing() self.qc.load() - def restore_backup(self, source=None, appvm=None, options=None, expect_errors=None): if source is None: @@ -528,7 +524,6 @@ class BackupTestsMixin(SystemTestsMixin): if not appvm and not os.path.isdir(backupfile): os.unlink(backupfile) - def create_sparse(self, path, size): f = open(path, "w") f.truncate(size) From 736e2788658eb13075d3935af998f5e6a00e0e29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 25 Nov 2015 01:08:57 +0100 Subject: [PATCH 024/270] tests: fix wait_for_window function The `show` parameter wasn't handled (show=False was assumed...) --- tests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/__init__.py b/tests/__init__.py index 563ddf28..8a254cff 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -353,7 +353,7 @@ class SystemTestsMixin(object): wait_count = 0 while subprocess.call(['xdotool', 'search', '--name', title], stdout=open(os.path.devnull, 'w'), - stderr=subprocess.STDOUT) == 0: + stderr=subprocess.STDOUT) == int(show): wait_count += 1 if wait_count > timeout*10: self.fail("Timeout while waiting for {} window to {}".format( From db35e03aa76a6a381a25396c76e6805a59da84b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 25 Nov 2015 01:14:53 +0100 Subject: [PATCH 025/270] tests: backup to VM tests Regression tests for QubesOS/qubes-issues#1371 and QubesOS/qubes-issues#1359 Fixes QubesOS/qubes-issues#1435 --- tests/__init__.py | 8 +--- tests/backup.py | 110 +++++++++++++++++++++++++++++++++++++++------- 2 files changed, 94 insertions(+), 24 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 8a254cff..b46a83c6 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -379,12 +379,6 @@ class BackupTestsMixin(SystemTestsMixin): if self.verbose: print >>sys.stderr, "-> Creating backupvm" - # TODO: allow non-default template - self.backupvm = self.qc.add_new_vm("QubesAppVm", - name=self.make_vm_name('backupvm'), - template=self.qc.get_default_template()) - self.backupvm.create_on_disk(verbose=self.verbose) - self.backupdir = os.path.join(os.environ["HOME"], "test-backup") if os.path.exists(self.backupdir): shutil.rmtree(self.backupdir) @@ -452,7 +446,7 @@ class BackupTestsMixin(SystemTestsMixin): return vms def make_backup(self, vms, prepare_kwargs=dict(), do_kwargs=dict(), - target=None): + target=None, expect_failure=False): # XXX: bakup_prepare and backup_do don't support host_collection self.qc.unlock_db() if target is None: diff --git a/tests/backup.py b/tests/backup.py index 15f63014..4ed6bfe0 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -28,7 +28,7 @@ import os import unittest import sys - +from qubes.qubes import QubesException, QubesTemplateVm import qubes.tests class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase): @@ -85,22 +85,6 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase): self.restore_backup() self.remove_vms(vms) - - # TODO: iterate over templates - def test_100_send_to_vm(self): - vms = self.create_backup_vms() - self.backupvm.start() - self.make_backup(vms, - do_kwargs={ - 'appvm': self.backupvm, - 'compressed': True, - 'encrypted': True}, - target='dd of=/var/tmp/backup-test') - self.remove_vms(vms) - self.restore_backup(source='dd if=/var/tmp/backup-test', - appvm=self.backupvm) - self.remove_vms(vms) - def test_200_restore_over_existing_directory(self): """ Regression test for #1386 @@ -119,3 +103,95 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase): test_dir) ]) self.remove_vms(vms) + +class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin): + def setUp(self): + super(TC_10_BackupVMMixin, self).setUp() + self.backupvm = self.qc.add_new_vm( + "QubesAppVm", + name=self.make_vm_name('backupvm'), + template=self.qc.get_vm_by_name(self.template) + ) + self.backupvm.create_on_disk(verbose=self.verbose) + + def test_100_send_to_vm_file_with_spaces(self): + vms = self.create_backup_vms() + self.backupvm.start() + self.backupvm.run("mkdir '/var/tmp/backup directory'", wait=True) + self.make_backup(vms, + do_kwargs={ + 'appvm': self.backupvm, + 'compressed': True, + 'encrypted': True}, + target='/var/tmp/backup directory') + self.remove_vms(vms) + p = self.backupvm.run("ls /var/tmp/backup*/qubes-backup*", + passio_popen=True) + (backup_path, _) = p.communicate() + backup_path = backup_path.strip() + self.restore_backup(source=backup_path, + appvm=self.backupvm) + self.remove_vms(vms) + + def test_110_send_to_vm_command(self): + vms = self.create_backup_vms() + self.backupvm.start() + self.make_backup(vms, + do_kwargs={ + 'appvm': self.backupvm, + 'compressed': True, + 'encrypted': True}, + target='dd of=/var/tmp/backup-test') + self.remove_vms(vms) + self.restore_backup(source='dd if=/var/tmp/backup-test', + appvm=self.backupvm) + self.remove_vms(vms) + + def test_110_send_to_vm_no_space(self): + """ + Check whether backup properly report failure when no enough space is + available + :return: + """ + vms = self.create_backup_vms() + self.backupvm.start() + retcode = self.backupvm.run( + "truncate -s 50M /home/user/backup.img && " + "mkfs.ext4 -F /home/user/backup.img && " + "mkdir /home/user/backup && " + "mount /home/user/backup.img /home/user/backup -o loop &&" + "chmod 777 /home/user/backup", + user="root", wait=True) + if retcode != 0: + raise RuntimeError("Failed to prepare backup directory") + with self.assertRaises(QubesException): + self.make_backup(vms, + do_kwargs={ + 'appvm': self.backupvm, + 'compressed': False, + 'encrypted': True}, + target='/home/user/backup', + expect_failure=True) + self.qc.lock_db_for_writing() + self.qc.load() + self.remove_vms(vms) + + +def load_tests(loader, tests, pattern): + try: + qc = qubes.qubes.QubesVmCollection() + qc.lock_db_for_reading() + qc.load() + qc.unlock_db() + templates = [vm.name for vm in qc.values() if + isinstance(vm, QubesTemplateVm)] + except OSError: + templates = [] + for template in templates: + tests.addTests(loader.loadTestsFromTestCase( + type( + 'TC_10_BackupVM_' + template, + (TC_10_BackupVMMixin, qubes.tests.QubesTestCase), + {'template': template}))) + + return tests From d7ab2d985dda391a85a379d477537f93781859a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 25 Nov 2015 01:18:05 +0100 Subject: [PATCH 026/270] tests: use wait_for_window/enter_keys_in_window wrappers when applicable Reduce code duplication. --- tests/vm_qrexec_gui.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py index 6d62f6f8..5b7661c2 100644 --- a/tests/vm_qrexec_gui.py +++ b/tests/vm_qrexec_gui.py @@ -555,8 +555,7 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin): self.testvm2.name, passio_popen=True, passio_stderr=True) # Confirm transfer - subprocess.check_call( - ['xdotool', 'search', '--sync', '--name', 'Question', 'key', 'y']) + self.enter_keys_in_window('Question', ['y']) p.wait() self.assertEqual(p.returncode, 0, "qvm-copy-to-vm failed: %s" % p.stderr.read()) @@ -856,23 +855,14 @@ class TC_20_DispVMMixin(qubes.tests.SystemTestsMixin): try: window_title = 'user@%s' % (dispvm.template.name + "-dvm") p.stdin.write("xterm -e " - "\"sh -s -c 'echo \\\"\033]0;{}\007\\\";read;'\"\n". + "\"sh -s -c 'echo \\\"\033]0;{}\007\\\";read x;'\"\n". format(window_title)) self.wait_for_window(window_title) time.sleep(0.5) - subprocess.check_call(['xdotool', 'search', '--name', window_title, - 'windowactivate', 'key', 'Return']) - - wait_count = 0 - while subprocess.call(['xdotool', 'search', '--name', window_title], - stdout=open(os.path.devnull, 'w'), - stderr=subprocess.STDOUT) == 0: - wait_count += 1 - if wait_count > 100: - self.fail("Timeout while waiting for gnome-terminal " - "termination") - time.sleep(0.1) + self.enter_keys_in_window(window_title, ['Return']) + # Wait for window to close + self.wait_for_window(window_title, show=False) finally: p.stdin.close() From 564ea5d64c7444cb7e836fdbe3356c8d62dc9928 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 25 Nov 2015 01:19:42 +0100 Subject: [PATCH 027/270] tests: check if qvm-move-to-vm properly fails in case of out of disk space Regression test for QubesOS/qubes-issues#1355 --- tests/vm_qrexec_gui.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py index 5b7661c2..d7d442d9 100644 --- a/tests/vm_qrexec_gui.py +++ b/tests/vm_qrexec_gui.py @@ -603,6 +603,41 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin): wait=True) self.assertEqual(retcode, 0, "file differs") + @unittest.skipUnless(spawn.find_executable('xdotool'), + "xdotool not installed") + def test_130_qrexec_filemove_disk_full(self): + self.testvm1.start() + self.testvm2.start() + # Prepare test file + prepare_cmd = ("yes teststring | dd of=testfile bs=1M " + "count=50 iflag=fullblock") + retcode = self.testvm1.run(prepare_cmd, wait=True) + if retcode != 0: + raise RuntimeError("Failed '{}' in {}".format(prepare_cmd, + self.testvm1.name)) + # Prepare target directory with limited size + prepare_cmd = ( + "mkdir -p /home/user/QubesIncoming && " + "chown user /home/user/QubesIncoming && " + "mount -t tmpfs none /home/user/QubesIncoming -o size=48M" + ) + retcode = self.testvm2.run(prepare_cmd, user="root", wait=True) + if retcode != 0: + raise RuntimeError("Failed '{}' in {}".format(prepare_cmd, + self.testvm2.name)) + p = self.testvm1.run("qvm-move-to-vm %s testfile" % + self.testvm2.name, passio_popen=True, + passio_stderr=True) + # Confirm transfer + self.enter_keys_in_window('Question', ['y']) + # Close GUI error message + self.enter_keys_in_window('Error', ['Return']) + p.wait() + self.assertEqual(p.returncode, 255, "qvm-move-to-vm should fail") + retcode = self.testvm1.run("test -f testfile", wait=True) + self.assertEqual(retcode, 0, "testfile should not be deleted in " + "source VM") + def test_200_timezone(self): """Test whether timezone setting is properly propagated to the VM""" if "whonix" in self.template: From de275cc917771b3a5a417ffd85060c547e7a710d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 25 Nov 2015 01:24:00 +0100 Subject: [PATCH 028/270] core: leave volatile.img structure creation for the VM This is already handled by a script in initramfs. Fixes QubesOS/qubes-issues#1308 --- linux/aux-tools/prepare-volatile-img.sh | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/linux/aux-tools/prepare-volatile-img.sh b/linux/aux-tools/prepare-volatile-img.sh index 33dfc9b9..40e22f5d 100755 --- a/linux/aux-tools/prepare-volatile-img.sh +++ b/linux/aux-tools/prepare-volatile-img.sh @@ -26,28 +26,3 @@ fi TOTAL_SIZE=$[ $ROOT_SIZE + $SWAP_SIZE + 512 ] truncate -s ${TOTAL_SIZE}M "$FILENAME" -sfdisk --no-reread -u M "$FILENAME" > /dev/null 2> /dev/null < /dev/null - if [ "$created" = "yes" ]; then - rm -f ${loopdev}p* - fi - losetup -d ${loopdev} || : - chown --reference `dirname "$FILENAME"` "$FILENAME" -) 200>"/var/run/qubes/prepare-volatile-img.lock" From a14e1cc86ff838829e2da955ce9c225010fbe7e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 25 Nov 2015 03:14:23 +0100 Subject: [PATCH 029/270] tests: move enter_keys_in_window to base SystemTestsMixin class This ways it will be available also in DispVM tests (already referenced there), and others. --- tests/__init__.py | 18 ++++++++++++++++++ tests/vm_qrexec_gui.py | 18 ------------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index b46a83c6..f75be979 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -361,6 +361,24 @@ class SystemTestsMixin(object): ) time.sleep(0.1) + def enter_keys_in_window(self, title, keys): + """ + Search for window with given title, then enter listed keys there. + The function will wait for said window to appear. + + :param title: title of window + :param keys: list of keys to enter, as for `xdotool key` + :return: None + """ + + # 'xdotool search --sync' sometimes crashes on some race when + # accessing window properties + self.wait_for_window(title) + command = ['xdotool', 'search', '--name', title, + 'windowactivate', + 'key'] + keys + subprocess.check_call(command) + def shutdown_and_wait(self, vm, timeout=60): vm.shutdown() while timeout > 0: diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py index d7d442d9..f601d3d3 100644 --- a/tests/vm_qrexec_gui.py +++ b/tests/vm_qrexec_gui.py @@ -54,24 +54,6 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin): self.testvm1 = self.qc[self.testvm1.qid] self.testvm2 = self.qc[self.testvm2.qid] - def enter_keys_in_window(self, title, keys): - """ - Search for window with given title, then enter listed keys there. - The function will wait for said window to appear. - - :param title: title of window - :param keys: list of keys to enter, as for `xdotool key` - :return: None - """ - - # 'xdotool search --sync' sometimes crashes on some race when - # accessing window properties - self.wait_for_window(title) - command = ['xdotool', 'search', '--name', title, - 'windowactivate', - 'key'] + keys - subprocess.check_call(command) - def test_000_start_shutdown(self): self.testvm1.start() self.assertEquals(self.testvm1.get_power_state(), "Running") From 64343572c1c1555d4ae44c978dc7d5302e6f2c10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 25 Nov 2015 03:15:23 +0100 Subject: [PATCH 030/270] tests: handle older `df` version in resize private.img tests df --output=size isn't available in Debian (7, 8). --- tests/vm_qrexec_gui.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py index f601d3d3..5d9a9f9f 100644 --- a/tests/vm_qrexec_gui.py +++ b/tests/vm_qrexec_gui.py @@ -693,7 +693,9 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin): # First offline test self.testvm1.resize_private_img(4*1024**3) self.testvm1.start() - p = self.testvm1.run('df --output=size /rw|tail -n 1', + df_cmd = '( df --output=size /rw || df /rw | awk \'{print $2}\' )|' \ + 'tail -n 1' + p = self.testvm1.run(df_cmd, passio_popen=True) # new_size in 1k-blocks (new_size, _) = p.communicate() @@ -701,7 +703,7 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin): self.assertGreater(int(new_size.strip()), 3.8*1024**2) # Then online test self.testvm1.resize_private_img(6*1024**3) - p = self.testvm1.run('df --output=size /rw|tail -n 1', + p = self.testvm1.run(df_cmd, passio_popen=True) # new_size in 1k-blocks (new_size, _) = p.communicate() From aeec42dacf12186e67fed9f85d57ab6bb42f8b1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 25 Nov 2015 03:16:24 +0100 Subject: [PATCH 031/270] tests: PV Grub: ensure that latest packages are installed Flush yum case before that. --- tests/vm_qrexec_gui.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py index 5d9a9f9f..c64da4d4 100644 --- a/tests/vm_qrexec_gui.py +++ b/tests/vm_qrexec_gui.py @@ -1070,7 +1070,8 @@ class TC_40_PVGrub(qubes.tests.SystemTestsMixin): def install_packages(self, vm): if self.template.startswith('fedora-'): - cmd_install1 = 'yum install -y qubes-kernel-vm-support grub2-tools' + cmd_install1 = 'yum clean expire-cache && ' \ + 'yum install -y qubes-kernel-vm-support grub2-tools' cmd_install2 = 'yum install -y kernel kernel-devel' cmd_update_grub = 'grub2-mkconfig -o /boot/grub2/grub.cfg' elif self.template.startswith('debian-'): From feaaaa75fad902f05204a3e632318d4150432492 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sat, 21 Nov 2015 13:22:27 +0100 Subject: [PATCH 032/270] Move the vmdir logic from XenPool to Pool Any storage implementation needs this logic for saving the vm config and `*.desktop` files. --- core/storage/__init__.py | 67 ++++++++++++++++++++++++++++++++++++++-- core/storage/xen.py | 64 ++------------------------------------ 2 files changed, 68 insertions(+), 63 deletions(-) diff --git a/core/storage/__init__.py b/core/storage/__init__.py index 8e2ac720..0c302082 100644 --- a/core/storage/__init__.py +++ b/core/storage/__init__.py @@ -30,7 +30,7 @@ import subprocess import sys import qubes.qubesutils -from qubes.qubes import QubesException, defaults, system_path, vm_files +from qubes.qubes import QubesException, defaults, system_path CONFIG_FILE = '/etc/qubes/storage.conf' @@ -321,4 +321,67 @@ class StoragePoolException(QubesException): class Pool(object): - pass + def __init__(self, vm, dir_path): + assert vm is not None + assert dir_path is not None + + self.vm = vm + self.dir_path = dir_path + + self.create_dir_if_not_exists(self.dir_path) + + self.vmdir = self.vmdir_path(vm, self.dir_path) + + appvms_path = os.path.join(self.dir_path, 'appvms') + self.create_dir_if_not_exists(appvms_path) + + servicevms_path = os.path.join(self.dir_path, 'servicevms') + self.create_dir_if_not_exists(servicevms_path) + + vm_templates_path = os.path.join(self.dir_path, 'vm-templates') + self.create_dir_if_not_exists(vm_templates_path) + + def vmdir_path(self, vm, pool_dir): + """ Returns the path to vmdir depending on the type of the VM. + + The default QubesOS file storage saves the vm images in three + different directories depending on the ``QubesVM`` type: + + * ``appvms`` for ``QubesAppVm`` or ``QubesHvm`` + * ``vm-templates`` for ``QubesTemplateVm`` or ``QubesTemplateHvm`` + * ``servicevms`` for any subclass of ``QubesNetVm`` + + Args: + vm: a QubesVM + pool_dir: the root directory of the pool + + Returns: + string (str) absolute path to the directory where the vm files + are stored + """ + # TODO: This is a hack, circular dependencies problem? + from qubes.qubes import (QubesAppVm, QubesDisposableVm, QubesHVm, + QubesNetVm, QubesTemplateHVm, QubesTemplateVm) + vm_type = type(vm) + + if vm_type in [QubesAppVm, QubesHVm]: + subdir = 'appvms' + elif vm_type in [QubesTemplateVm, QubesTemplateHVm]: + subdir = 'vm-templates' + elif issubclass(vm_type, QubesNetVm): + subdir = 'servicevms' + elif vm_type is QubesDisposableVm: + subdir = 'appvms' + return os.path.join(pool_dir, subdir, vm.template.name + '-dvm') + else: + raise QubesException(str(vm_type) + ' unknown vm type') + + return os.path.join(pool_dir, subdir, vm.name) + + def create_dir_if_not_exists(self, path): + """ Check if a directory exists in if not create it. + + This method does not create any parent directories. + """ + if not os.path.exists(path): + os.mkdir(path) diff --git a/core/storage/xen.py b/core/storage/xen.py index 1cd5dc97..11ff9b67 100644 --- a/core/storage/xen.py +++ b/core/storage/xen.py @@ -28,9 +28,7 @@ import re import subprocess import sys -from qubes.qubes import (QubesAppVm, QubesDisposableVm, QubesException, - QubesHVm, QubesNetVm, QubesTemplateHVm, - QubesTemplateVm, defaults, vm_files) +from qubes.qubes import QubesException, vm_files from qubes.storage import Pool, QubesVmStorage @@ -275,64 +273,8 @@ class XenStorage(QubesVmStorage): class XenPool(Pool): def __init__(self, vm, dir_path): - assert vm is not None - assert dir_path is not None - - appvms_path = os.path.join(dir_path, 'appvms') - servicevms_path = os.path.join(dir_path, 'servicevms') - vm_templates_path = os.path.join(dir_path, 'vm-templates') - - self._create_dir_if_not_exists(dir_path) - self._create_dir_if_not_exists(appvms_path) - self._create_dir_if_not_exists(servicevms_path) - self._create_dir_if_not_exists(vm_templates_path) - - self.vmdir = self._vmdir_path(vm, dir_path) - self.vm = vm - self.dir_path = dir_path + super(XenPool, self).__init__(vm, dir_path) def getStorage(self): """ Returns an instantiated ``XenStorage``. """ - return defaults['storage_class'](self.vm, vmdir=self.vmdir) - - def _vmdir_path(self, vm, pool_dir): - """ Get the vm dir depending on the type of the VM. - - The default QubesOS file storage saves the vm images in three - different directories depending on the ``QubesVM`` type: - - * ``appvms`` for ``QubesAppVm`` or ``QubesHvm`` - * ``vm-templates`` for ``QubesTemplateVm`` or ``QubesTemplateHvm`` - * ``servicevms`` for any subclass of ``QubesNetVm`` - - Args: - vm: a QubesVM - pool_dir: the root directory of the pool - - Returns: - string (str) absolute path to the directory where the vm files - are stored - """ - vm_type = type(vm) - - if vm_type in [QubesAppVm, QubesHVm]: - subdir = 'appvms' - elif vm_type in [QubesTemplateVm, QubesTemplateHVm]: - subdir = 'vm-templates' - elif issubclass(vm_type, QubesNetVm): - subdir = 'servicevms' - elif vm_type is QubesDisposableVm: - subdir = 'appvms' - return os.path.join(pool_dir, subdir, vm.template.name + '-dvm') - else: - raise QubesException(str(vm_type) + ' unknown vm type') - - return os.path.join(pool_dir, subdir, vm.name) - - def _create_dir_if_not_exists(self, path): - """ Check if a directory exists in if not it is created. - - This method does not create any parent directories. - """ - if not os.path.exists(path): - os.mkdir(path) + return XenStorage(self.vm, vmdir=self.vmdir) From 9b23576ff6a93a5ac2e90772e0af1bbdda117dc2 Mon Sep 17 00:00:00 2001 From: Bahtiar `kalkin-` Gadimov Date: Sun, 22 Nov 2015 17:18:42 +0100 Subject: [PATCH 033/270] Provide method format_disk_dev() to all storages The method XenStorage._format_disk_dev() generates the xml config for a device. It is not specific to the Xen file storage implementation. It can and must be reused by other storage implementations --- core/storage/__init__.py | 19 +++++++++++++++++++ core/storage/xen.py | 35 ++++++++--------------------------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/core/storage/__init__.py b/core/storage/__init__.py index 0c302082..530198e3 100644 --- a/core/storage/__init__.py +++ b/core/storage/__init__.py @@ -65,6 +65,25 @@ class QubesVmStorage(object): # Additional drive (currently used only by HVM) self.drive = None + def format_disk_dev(self, path, script, vdev, rw=True, type="disk", + domain=None): + if path is None: + return '' + template = " \n" \ + " \n" \ + " \n" \ + " \n" \ + "{params}" \ + " \n" + params = "" + if not rw: + params += " \n" + if domain: + params += " \n" % domain + if script: + params += "