diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..b99d8d48 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,9 @@ +sudo: required +dist: trusty +language: generic +install: git clone https://github.com/QubesOS/qubes-builder ~/qubes-builder +# debootstrap in trusty is old... +before_script: sudo ln -s sid /usr/share/debootstrap/scripts/stretch +script: ~/qubes-builder/scripts/travis-build +env: + - DIST_DOM0=fc23 USE_QUBES_REPO_VERSION=3.2 USE_QUBES_REPO_TESTING=1 diff --git a/Makefile b/Makefile index 2282173e..545e8089 100644 --- a/Makefile +++ b/Makefile @@ -72,12 +72,15 @@ endif mkdir -p $(DESTDIR)/usr/libexec/qubes cp qubes-rpc-policy/qubes.Filecopy.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.Filecopy cp qubes-rpc-policy/qubes.OpenInVM.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.OpenInVM + cp qubes-rpc-policy/qubes.OpenURL.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.OpenURL cp qubes-rpc-policy/qubes.VMShell.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.VMShell cp qubes-rpc-policy/qubes.NotifyUpdates.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.NotifyUpdates cp qubes-rpc-policy/qubes.NotifyTools.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.NotifyTools cp qubes-rpc-policy/qubes.GetImageRGBA.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.GetImageRGBA + cp qubes-rpc-policy/qubes.GetRandomizedTime.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.GetRandomizedTime cp qubes-rpc/qubes.NotifyUpdates $(DESTDIR)/etc/qubes-rpc/ cp qubes-rpc/qubes.NotifyTools $(DESTDIR)/etc/qubes-rpc/ + cp qubes-rpc/qubes.GetRandomizedTime $(DESTDIR)/etc/qubes-rpc/ cp qubes-rpc/qubes-notify-updates $(DESTDIR)/usr/libexec/qubes/ cp qubes-rpc/qubes-notify-tools $(DESTDIR)/usr/libexec/qubes/ mkdir -p "$(DESTDIR)$(FILESDIR)" diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py index f7411a97..3c0760a6 100644 --- a/core-modules/000QubesVm.py +++ b/core-modules/000QubesVm.py @@ -26,6 +26,7 @@ import datetime import base64 import hashlib import logging +import grp import lxml.etree import os import os.path @@ -37,15 +38,16 @@ import time import uuid import xml.parsers.expat import signal +import pwd from qubes import qmemman from qubes import qmemman_algo import libvirt -import warnings from qubes.qubes import dry_run,vmm from qubes.qubes import register_qubes_vm_class from qubes.qubes import QubesVmCollection,QubesException,QubesHost,QubesVmLabels from qubes.qubes import defaults,system_path,vm_files,qubes_max_qid +from qubes.storage import get_pool qmemman_present = False try: @@ -109,6 +111,7 @@ class QubesVm(object): "name": { "order": 1 }, "uuid": { "order": 0, "eval": 'uuid.UUID(value) if value else None' }, "dir_path": { "default": None, "order": 2 }, + "pool_name": { "default":"default" }, "conf_file": { "func": lambda value: self.absolute_path(value, self.name + ".conf"), @@ -133,9 +136,10 @@ class QubesVm(object): eval(value) if value.find("[") >= 0 else eval("[" + value + "]") }, "pci_strictreset": {"default": True}, + "pci_e820_host": {"default": True}, # Internal VM (not shown in qubes-manager, doesn't create appmenus entries "internal": { "default": False, 'attr': '_internal' }, - "vcpus": { "default": None }, + "vcpus": { "default": 2 }, "uses_default_kernel": { "default": True, 'order': 30 }, "uses_default_kernelopts": { "default": True, 'order': 30 }, "kernel": { @@ -198,7 +202,8 @@ class QubesVm(object): 'kernelopts', 'services', 'installed_by_rpm',\ 'uses_default_netvm', 'include_in_backups', 'debug',\ 'qrexec_timeout', 'autostart', 'uses_default_dispvm_netvm', - 'backup_content', 'backup_size', 'backup_path' ]: + 'backup_content', 'backup_size', 'backup_path', 'pool_name',\ + 'pci_e820_host']: attrs[prop]['save'] = lambda prop=prop: str(getattr(self, prop)) # Simple paths for prop in ['conf_file', 'firewall_conf']: @@ -326,11 +331,6 @@ class QubesVm(object): if self.maxmem > self.memory * 10: self.maxmem = self.memory * 10 - # By default allow use all VCPUs - if self.vcpus is None and not vmm.offline_mode: - qubes_host = QubesHost() - self.vcpus = qubes_host.no_cpus - # Always set if meminfo-writer should be active or not if 'meminfo-writer' not in self.services: self.services['meminfo-writer'] = not (len(self.pcidevs) > 0) @@ -345,7 +345,11 @@ class QubesVm(object): self.services['qubes-update-check'] = False # Initialize VM image storage class - self.storage = defaults["storage_class"](self) + self.storage = get_pool(self.pool_name, self).getStorage() + self.dir_path = self.storage.vmdir + self.icon_path = os.path.join(self.storage.vmdir, 'icon.png') + self.conf_file = os.path.join(self.storage.vmdir, self.name + '.conf') + if hasattr(self, 'kernels_dir'): modules_path = os.path.join(self.kernels_dir, "modules.img") @@ -561,6 +565,10 @@ class QubesVm(object): return False if len(name) > 31: return False + if name == 'lost+found': + # avoid conflict when /var/lib/qubes/appvms is mounted on + # separate partition + return False return re.match(r"^[a-zA-Z][a-zA-Z0-9_.-]*$", name) is not None def pre_rename(self, new_name): @@ -582,9 +590,18 @@ class QubesVm(object): if self.installed_by_rpm: raise QubesException("Cannot rename VM installed by RPM -- first clone VM and then use yum to remove package.") + assert self._collection is not None + if self._collection.get_vm_by_name(name): + raise QubesException("VM with this name already exists") + self.pre_rename(name) - if self.libvirt_domain: + try: self.libvirt_domain.undefine() + except libvirt.libvirtError as e: + if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN: + pass + else: + raise if self._qdb_connection: self._qdb_connection.close() self._qdb_connection = None @@ -714,6 +731,8 @@ class QubesVm(object): if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN: return -1 else: + print >>sys.stderr, "libvirt error code: {!r}".format( + e.get_error_code()) raise @@ -766,9 +785,13 @@ class QubesVm(object): if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN: return 0 # libxl_domain_info failed - domain no longer exists - elif e.get_error_code() == libvirt.VIR_INTERNAL_ERROR: + elif e.get_error_code() == libvirt.VIR_ERR_INTERNAL_ERROR: + return 0 + elif e.get_error_code() is None: # unknown... return 0 else: + print >>sys.stderr, "libvirt error code: {!r}".format( + e.get_error_code()) raise def get_cputime(self): @@ -783,9 +806,13 @@ class QubesVm(object): if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN: return 0 # libxl_domain_info failed - domain no longer exists - elif e.get_error_code() == libvirt.VIR_INTERNAL_ERROR: + elif e.get_error_code() == libvirt.VIR_ERR_INTERNAL_ERROR: + return 0 + elif e.get_error_code() is None: # unknown... return 0 else: + print >>sys.stderr, "libvirt error code: {!r}".format( + e.get_error_code()) raise def get_mem_static_max(self): @@ -827,6 +854,8 @@ class QubesVm(object): if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN: return 0 else: + print >>sys.stderr, "libvirt error code: {!r}".format( + e.get_error_code()) raise def get_disk_utilization_root_img(self): @@ -901,7 +930,14 @@ class QubesVm(object): except libvirt.libvirtError as e: if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN: return False + # libxl_domain_info failed - domain no longer exists + elif e.get_error_code() == libvirt.VIR_ERR_INTERNAL_ERROR: + return False + elif e.get_error_code() is None: # unknown... + return False else: + print >>sys.stderr, "libvirt error code: {!r}".format( + e.get_error_code()) raise def is_paused(self): @@ -913,7 +949,14 @@ class QubesVm(object): except libvirt.libvirtError as e: if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN: return False + # libxl_domain_info failed - domain no longer exists + elif e.get_error_code() == libvirt.VIR_ERR_INTERNAL_ERROR: + return False + elif e.get_error_code() is None: # unknown... + return False else: + print >>sys.stderr, "libvirt error code: {!r}".format( + e.get_error_code()) raise def get_start_time(self): @@ -924,7 +967,7 @@ class QubesVm(object): uuid = self.uuid start_time = vmm.xs.read('', "/vm/%s/start_time" % str(uuid)) - if start_time != '': + if start_time: return datetime.datetime.fromtimestamp(float(start_time)) else: return None @@ -1061,6 +1104,7 @@ class QubesVm(object): if self.is_netvm(): self.qdb.write("/qubes-netvm-gateway", self.gateway) + self.qdb.write("/qubes-netvm-primary-dns", self.gateway) self.qdb.write("/qubes-netvm-secondary-dns", self.secondary_dns) self.qdb.write("/qubes-netvm-netmask", self.netmask) self.qdb.write("/qubes-netvm-network", self.network) @@ -1069,6 +1113,7 @@ class QubesVm(object): self.qdb.write("/qubes-ip", self.ip) self.qdb.write("/qubes-netmask", self.netvm.netmask) self.qdb.write("/qubes-gateway", self.netvm.gateway) + self.qdb.write("/qubes-primary-dns", self.netvm.gateway) self.qdb.write("/qubes-secondary-dns", self.netvm.secondary_dns) tzname = self.get_timezone() @@ -1148,6 +1193,7 @@ class QubesVm(object): # If dynamic memory management disabled, set maxmem=mem args['maxmem'] = args['mem'] args['vcpus'] = str(self.vcpus) + args['features'] = '' if self.netvm is not None: args['ip'] = self.ip args['mac'] = self.mac @@ -1156,8 +1202,10 @@ class QubesVm(object): args['dns2'] = self.secondary_dns args['netmask'] = self.netmask args['netdev'] = self._format_net_dev(self.ip, self.mac, self.netvm.name) - args['disable_network1'] = ''; - args['disable_network2'] = ''; + args['network_begin'] = '' + args['network_end'] = '' + args['no_network_begin'] = '' else: args['ip'] = '' args['mac'] = '' @@ -1166,8 +1214,12 @@ class QubesVm(object): args['dns2'] = '' args['netmask'] = '' args['netdev'] = '' - args['disable_network1'] = ''; + args['network_begin'] = '' + args['no_network_begin'] = '' + args['no_network_end'] = '' + if len(self.pcidevs) and self.pci_e820_host: + args['features'] = '' args.update(self.storage.get_config_params()) if hasattr(self, 'kernelopts'): args['kernelopts'] = self.kernelopts @@ -1262,9 +1314,10 @@ class QubesVm(object): hook(self, verbose, source_template=source_template) def get_clone_attrs(self): - attrs = ['kernel', 'uses_default_kernel', 'netvm', 'uses_default_netvm', \ - 'memory', 'maxmem', 'kernelopts', 'uses_default_kernelopts', 'services', 'vcpus', \ - '_mac', 'pcidevs', 'include_in_backups', '_label', 'default_user'] + attrs = ['kernel', 'uses_default_kernel', 'netvm', 'uses_default_netvm', + 'memory', 'maxmem', 'kernelopts', 'uses_default_kernelopts', + 'services', 'vcpus', '_mac', 'pcidevs', 'include_in_backups', + '_label', 'default_user', 'qrexec_timeout'] # fire hooks for hook in self.hooks_get_clone_attrs: @@ -1357,8 +1410,16 @@ class QubesVm(object): # already undefined pass else: + print >>sys.stderr, "libvirt error code: {!r}".format( + e.get_error_code()) raise + if os.path.exists("/etc/systemd/system/multi-user.target.wants/qubes-vm@" + self.name + ".service"): + retcode = subprocess.call(["sudo", "systemctl", "-q", "disable", + "qubes-vm@" + self.name + ".service"]) + if retcode != 0: + raise QubesException("Failed to delete autostart entry for VM") + self.storage.remove_from_disk() def write_firewall_conf(self, conf): @@ -1623,17 +1684,22 @@ class QubesVm(object): if bool(input) + bool(passio_popen) + bool(localcmd) > 1: raise ValueError("'input', 'passio_popen', 'localcmd' cannot be " "used together") + if not wait and (localcmd or input): + raise ValueError("Cannot use wait=False with input or " + "localcmd specified") if localcmd: return self.run("QUBESRPC %s %s" % (service, source), localcmd=localcmd, user=user, wait=wait, gui=gui) elif input: - return self.run("QUBESRPC %s %s" % (service, source), - localcmd="echo %s" % input, user=user, wait=wait, - gui=gui) + p = self.run("QUBESRPC %s %s" % (service, source), + user=user, wait=wait, gui=gui, passio_popen=True, + passio_stderr=True) + p.communicate(input) + return p.returncode else: return self.run("QUBESRPC %s %s" % (service, source), passio_popen=passio_popen, user=user, wait=wait, - gui=gui) + gui=gui, passio_stderr=passio_popen) def attach_network(self, verbose = False, wait = True, netvm = None): self.log.debug('attach_network(netvm={!r})'.format(netvm)) @@ -1695,7 +1761,15 @@ class QubesVm(object): if verbose: print >> sys.stderr, "--> Starting Qubes GUId..." - guid_cmd = [system_path["qubes_guid_path"], + guid_cmd = [] + if os.getuid() == 0: + # try to always have guid running as normal user, otherwise + # clipboard file may be created as root and other permission + # problems + qubes_group = grp.getgrnam('qubes') + guid_cmd = ['runuser', '-u', qubes_group.gr_mem[0], '--'] + + guid_cmd += [system_path["qubes_guid_path"], "-d", str(self.xid), "-N", self.name, "-c", self.label.color, "-i", self.label.icon_path, @@ -1706,6 +1780,33 @@ class QubesVm(object): guid_cmd += ['-v', '-v'] elif not verbose: guid_cmd += ['-q'] + # Avoid using environment variables for checking the current session, + # because this script may be called with cleared env (like with sudo). + if subprocess.check_output( + ['xprop', '-root', '-notype', 'KDE_SESSION_VERSION']) == \ + 'KDE_SESSION_VERSION = 5\n': + # native decoration plugins is used, so adjust window properties + # accordingly + guid_cmd += ['-T'] # prefix window titles with VM name + # get owner of X11 session + session_owner = None + for line in subprocess.check_output(['xhost']).splitlines(): + if line == 'SI:localuser:root': + pass + elif line.startswith('SI:localuser:'): + session_owner = line.split(":")[2] + if session_owner is not None: + data_dir = os.path.expanduser( + '~{}/.local/share'.format(session_owner)) + else: + # fallback to current user + data_dir = os.path.expanduser('~/.local/share') + + guid_cmd += ['-p', + '_KDE_NET_WM_COLOR_SCHEME=s:{}'.format( + os.path.join(data_dir, + 'qubes-kde', self.label.name + '.colors'))] + retcode = subprocess.call (guid_cmd) if (retcode != 0) : raise QubesException("Cannot start qubes-guid!") @@ -1734,13 +1835,21 @@ class QubesVm(object): self.log.debug('start_qrexec_daemon()') if verbose: print >> sys.stderr, "--> Starting the qrexec daemon..." + qrexec = [] + if os.getuid() == 0: + # try to always have qrexec running as normal user, otherwise + # many qrexec services would need to deal with root/user + # permission problems + qubes_group = grp.getgrnam('qubes') + qrexec = ['runuser', '-u', qubes_group.gr_mem[0], '--'] + + qrexec += ['env', 'QREXEC_STARTUP_TIMEOUT=' + str(self.qrexec_timeout), + system_path["qrexec_daemon_path"]] + qrexec_args = [str(self.xid), self.name, self.default_user] if not verbose: qrexec_args.insert(0, "-q") - qrexec_env = os.environ - qrexec_env['QREXEC_STARTUP_TIMEOUT'] = str(self.qrexec_timeout) - retcode = subprocess.call ([system_path["qrexec_daemon_path"]] + - qrexec_args, env=qrexec_env) + retcode = subprocess.call(qrexec + qrexec_args) if (retcode != 0) : raise OSError ("Cannot execute qrexec-daemon!") @@ -1769,10 +1878,19 @@ class QubesVm(object): # force connection to a new daemon self._qdb_connection = None - retcode = subprocess.call ([ + qubesdb_cmd = [] + if os.getuid() == 0: + # try to always have qubesdb running as normal user, otherwise + # killing it at VM restart (see above) will always fail + qubes_group = grp.getgrnam('qubes') + qubesdb_cmd = ['runuser', '-u', qubes_group.gr_mem[0], '--'] + + qubesdb_cmd += [ system_path["qubesdb_daemon_path"], str(self.xid), - self.name]) + self.name] + + retcode = subprocess.call (qubesdb_cmd) if retcode != 0: raise OSError("ERROR: Cannot execute qubesdb-daemon!") diff --git a/core-modules/001QubesResizableVm.py b/core-modules/001QubesResizableVm.py index 88467ed1..f50ead70 100644 --- a/core-modules/001QubesResizableVm.py +++ b/core-modules/001QubesResizableVm.py @@ -31,11 +31,12 @@ from qubes.qubes import ( QubesException, QubesVm, ) +from time import sleep class QubesResizableVm(QubesVm): - def resize_root_img(self, size): + def resize_root_img(self, size, allow_start=False): if self.template: raise QubesException("Cannot resize root.img of template-based VM" ". Resize the root.img of the template " @@ -56,12 +57,20 @@ class QubesResizableVm(QubesVm): class QubesResizableVmWithResize2fs(QubesResizableVm): - def resize_root_img(self, size): - super(QubesResizableVmWithResize2fs, self).resize_root_img(size) + def resize_root_img(self, size, allow_start=False): + super(QubesResizableVmWithResize2fs, self).\ + resize_root_img(size, allow_start=allow_start) + if not allow_start: + raise QubesException("VM start required to complete the " + "operation, but not allowed. Either run the " + "operation again allowing VM start this " + "time, or run resize2fs in the VM manually.") self.start(start_guid=False) self.run("resize2fs /dev/mapper/dmroot", user="root", wait=True, gui=False) self.shutdown() + while self.is_running(): + sleep(1) register_qubes_vm_class(QubesResizableVm) diff --git a/core-modules/005QubesNetVm.py b/core-modules/005QubesNetVm.py index 33934f4b..904e74f1 100644 --- a/core-modules/005QubesNetVm.py +++ b/core-modules/005QubesNetVm.py @@ -42,6 +42,7 @@ class QubesNetVm(QubesVm): attrs_config['dir_path']['func'] = \ lambda value: value if value is not None else \ os.path.join(system_path["qubes_servicevms_dir"], self.name) + attrs_config['uses_default_netvm']['func'] = lambda x: False attrs_config['label']['default'] = defaults["servicevm_label"] attrs_config['memory']['default'] = 300 diff --git a/core-modules/01QubesDisposableVm.py b/core-modules/01QubesDisposableVm.py index 8fedb742..4e3ebdd8 100644 --- a/core-modules/01QubesDisposableVm.py +++ b/core-modules/01QubesDisposableVm.py @@ -168,8 +168,13 @@ class QubesDisposableVm(QubesVm): if self.get_power_state() != "Halted": raise QubesException ("VM is already running!") - # skip netvm state checking - calling VM have the same netvm, so it - # must be already running + if self.netvm is not None: + if self.netvm.qid != 0: + if not self.netvm.is_running(): + if verbose: + print >> sys.stderr, "--> Starting NetVM {0}...".\ + format(self.netvm.name) + self.netvm.start(verbose=verbose, **kwargs) if verbose: print >> sys.stderr, "--> Loading the VM (type = {0})...".format(self.type) @@ -232,5 +237,9 @@ class QubesDisposableVm(QubesVm): return self.xid + def remove_from_disk(self): + # nothing to remove + pass + # register classes register_qubes_vm_class(QubesDisposableVm) diff --git a/core-modules/01QubesHVm.py b/core-modules/01QubesHVm.py index c3b692a8..c98b67b5 100644 --- a/core-modules/01QubesHVm.py +++ b/core-modules/01QubesHVm.py @@ -100,8 +100,6 @@ class QubesHVm(QubesResizableVm): (not 'xml_element' in kwargs or kwargs['xml_element'].get('guiagent_installed') is None): self.services['meminfo-writer'] = False - self.storage.rootcow_img = None - @property def type(self): return "HVM" @@ -314,7 +312,16 @@ class QubesHVm(QubesResizableVm): else: return -1 + def validate_drive_path(self, drive): + drive_type, drive_domain, drive_path = drive.split(':', 2) + if drive_domain == 'dom0': + if not os.path.exists(drive_path): + raise QubesException("Invalid drive path '{}'".format( + drive_path)) + def start(self, *args, **kwargs): + if self.drive: + self.validate_drive_path(self.drive) # make it available to storage.prepare_for_vm_startup, which is # called before actually building VM libvirt configuration self.storage.drive = self.drive @@ -352,25 +359,28 @@ class QubesHVm(QubesResizableVm): if (retcode != 0) : raise QubesException("Cannot start qubes-guid!") - def start_guid(self, verbose = True, notify_function = None, - before_qrexec=False, **kwargs): - # If user force the guiagent, start_guid will mimic a standard QubesVM - if not before_qrexec and self.guiagent_installed: - kwargs['extra_guid_args'] = kwargs.get('extra_guid_args', []) + \ - ['-Q'] - super(QubesHVm, self).start_guid(verbose, notify_function, **kwargs) - stubdom_guid_pidfile = '/var/run/qubes/guid-running.%d' % self.stubdom_xid - if os.path.exists(stubdom_guid_pidfile) and not self.debug: - try: - stubdom_guid_pid = int(open(stubdom_guid_pidfile, 'r').read()) - os.kill(stubdom_guid_pid, signal.SIGTERM) - except Exception as ex: - print >> sys.stderr, "WARNING: Failed to kill stubdom gui daemon: %s" % str(ex) - elif before_qrexec and (not self.guiagent_installed or self.debug): + def start_guid(self, verbose=True, notify_function=None, + before_qrexec=False, **kwargs): + if not before_qrexec: + return + + if not self.guiagent_installed or self.debug: if verbose: print >> sys.stderr, "--> Starting Qubes GUId (full screen)..." self.start_stubdom_guid(verbose=verbose) + kwargs['extra_guid_args'] = kwargs.get('extra_guid_args', []) + \ + ['-Q', '-n'] + + stubdom_guid_pidfile = \ + '/var/run/qubes/guid-running.%d' % self.stubdom_xid + if not self.debug and os.path.exists(stubdom_guid_pidfile): + # Terminate stubdom guid once "real" gui agent connects + stubdom_guid_pid = int(open(stubdom_guid_pidfile, 'r').read()) + kwargs['extra_guid_args'] += ['-K', str(stubdom_guid_pid)] + + super(QubesHVm, self).start_guid(verbose, notify_function, **kwargs) + def start_qrexec_daemon(self, **kwargs): if not self.qrexec_installed: if kwargs.get('verbose', False): diff --git a/core-modules/02QubesTemplateHVm.py b/core-modules/02QubesTemplateHVm.py index eb5b309e..6452a8eb 100644 --- a/core-modules/02QubesTemplateHVm.py +++ b/core-modules/02QubesTemplateHVm.py @@ -29,7 +29,7 @@ import stat import sys import re -from qubes.qubes import QubesHVm,register_qubes_vm_class,dry_run +from qubes.qubes import QubesHVm,register_qubes_vm_class,dry_run,vmm from qubes.qubes import QubesException,QubesVmCollection from qubes.qubes import system_path,defaults @@ -70,6 +70,7 @@ class QubesTemplateHVm(QubesHVm): def is_appvm(self): return False + @property def rootcow_img(self): return self.storage.rootcow_img @@ -95,7 +96,15 @@ class QubesTemplateHVm(QubesHVm): def commit_changes (self, verbose = False): self.log.debug('commit_changes()') - # nothing to do as long as root-cow.img is unused - pass + if not vmm.offline_mode: + assert not self.is_running(), "Attempt to commit changes on running Template VM!" + + if verbose: + print >> sys.stderr, "--> Commiting template updates... COW: {0}...".format (self.rootcow_img) + + if dry_run: + return + + self.storage.commit_template_changes() register_qubes_vm_class(QubesTemplateHVm) diff --git a/core/backup.py b/core/backup.py index 6b20212c..e9915d56 100644 --- a/core/backup.py +++ b/core/backup.py @@ -357,8 +357,8 @@ def backup_prepare(vms_list=None, exclude_list=None, vms_not_for_backup = [vm.name for vm in qvm_collection.values() if not vm.backup_content] - print_callback("VMs not selected for backup: %s" % " ".join( - vms_not_for_backup)) + print_callback("VMs not selected for backup:\n%s" % "\n".join(sorted( + vms_not_for_backup))) if there_are_running_vms: raise QubesException("Please shutdown all VMs before proceeding.") @@ -400,6 +400,10 @@ class SendWorker(Process): stdin=subprocess.PIPE, stdout=self.backup_stdout) if final_proc.wait() >= 2: + if self.queue.full(): + # if queue is already full, remove some entry to wake up + # main thread, so it will be able to notice error + self.queue.get() # handle only exit code 2 (tar fatal error) or # greater (call failed?) raise QubesException( @@ -445,9 +449,21 @@ def prepare_backup_header(target_directory, passphrase, compressed=False, def backup_do(base_backup_dir, files_to_backup, passphrase, progress_callback=None, encrypted=False, appvm=None, compressed=False, hmac_algorithm=DEFAULT_HMAC_ALGORITHM, - crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM): + crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM, + tmpdir=None): global running_backup_operation + def queue_put_with_check(proc, vmproc, queue, element): + if queue.full(): + if not proc.is_alive(): + if vmproc: + message = ("Failed to write the backup, VM output:\n" + + vmproc.stderr.read()) + else: + message = "Failed to write the backup. Out of disk space?" + raise QubesException(message) + queue.put(element) + total_backup_sz = 0 passphrase = passphrase.encode('utf-8') for f in files_to_backup: @@ -495,7 +511,7 @@ def backup_do(base_backup_dir, files_to_backup, passphrase, progress = blocks_backedup * 11 / total_backup_sz progress_callback(progress) - backup_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/backup_") + backup_tmpdir = tempfile.mkdtemp(prefix="backup_", dir=tmpdir) running_backup_operation.tmpdir_to_remove = backup_tmpdir # Tar with tape length does not deals well with stdout (close stdout between @@ -552,15 +568,16 @@ def backup_do(base_backup_dir, files_to_backup, passphrase, # be verified before untaring this. # Prefix the path in archive with filename["subdir"] to have it # verified during untar - tar_cmdline = ["tar", "-Pc", '--sparse', + tar_cmdline = (["tar", "-Pc", '--sparse', "-f", backup_pipe, - '-C', os.path.dirname(filename["path"]), - '--dereference', - '--xform', 's:^%s:%s\\0:' % ( + '-C', os.path.dirname(filename["path"])] + + (['--dereference'] if filename["subdir"] != "dom0-home/" + else []) + + ['--xform', 's:^%s:%s\\0:' % ( os.path.basename(filename["path"]), filename["subdir"]), os.path.basename(filename["path"]) - ] + ]) if compressed: tar_cmdline.insert(-1, "--use-compress-program=%s" % compression_filter) @@ -650,7 +667,9 @@ def backup_do(base_backup_dir, files_to_backup, passphrase, run_error) # Send the chunk to the backup target - to_send.put(os.path.relpath(chunkfile, backup_tmpdir)) + queue_put_with_check( + send_proc, vmproc, to_send, + os.path.relpath(chunkfile, backup_tmpdir)) # Close HMAC hmac.stdin.close() @@ -668,7 +687,9 @@ def backup_do(base_backup_dir, files_to_backup, passphrase, hmac_file.close() # Send the HMAC to the backup target - to_send.put(os.path.relpath(chunkfile, backup_tmpdir) + ".hmac") + queue_put_with_check( + send_proc, vmproc, to_send, + os.path.relpath(chunkfile, backup_tmpdir) + ".hmac") if tar_sparse.poll() is None or run_error == "size_limit": run_error = "paused" @@ -680,7 +701,7 @@ def backup_do(base_backup_dir, files_to_backup, passphrase, .poll() pipe.close() - to_send.put("FINISHED") + queue_put_with_check(send_proc, vmproc, to_send, "FINISHED") send_proc.join() shutil.rmtree(backup_tmpdir) @@ -1553,6 +1574,8 @@ def backup_restore_set_defaults(options): options['ignore-username-mismatch'] = False if 'verify-only' not in options: options['verify-only'] = False + if 'rename-conflicting' not in options: + options['rename-conflicting'] = False return options @@ -1620,6 +1643,22 @@ def backup_restore_header(source, passphrase, return (restore_tmpdir, os.path.join(restore_tmpdir, "qubes.xml"), header_data) +def generate_new_name_for_conflicting_vm(orig_name, host_collection, + restore_info): + number = 1 + if len(orig_name) > 29: + orig_name = orig_name[0:29] + new_name = orig_name + while (new_name in restore_info.keys() or + new_name in map(lambda x: x.get('rename_to', None), + restore_info.values()) or + host_collection.get_vm_by_name(new_name)): + new_name = str('{}{}'.format(orig_name, number)) + number += 1 + if number == 100: + # give up + return None + return new_name def restore_info_verify(restore_info, host_collection): options = restore_info['$OPTIONS$'] @@ -1637,7 +1676,16 @@ def restore_info_verify(restore_info, host_collection): vm_info.pop('already-exists', None) if not options['verify-only'] and \ host_collection.get_vm_by_name(vm) is not None: - vm_info['already-exists'] = True + if options['rename-conflicting']: + new_name = generate_new_name_for_conflicting_vm( + vm, host_collection, restore_info + ) + if new_name is not None: + vm_info['rename-to'] = new_name + else: + vm_info['already-exists'] = True + else: + vm_info['already-exists'] = True # check template vm_info.pop('missing-template', None) @@ -1658,7 +1706,11 @@ def restore_info_verify(restore_info, host_collection): # check netvm vm_info.pop('missing-netvm', None) - if vm_info['netvm']: + if vm_info['vm'].uses_default_netvm: + default_netvm = host_collection.get_default_netvm() + vm_info['netvm'] = default_netvm.name if \ + default_netvm else None + elif vm_info['netvm']: netvm_name = vm_info['netvm'] netvm_on_host = host_collection.get_vm_by_name(netvm_name) @@ -1670,8 +1722,9 @@ def restore_info_verify(restore_info, host_collection): if not (netvm_name in restore_info.keys() and restore_info[netvm_name]['vm'].is_netvm()): if options['use-default-netvm']: - vm_info['netvm'] = host_collection \ - .get_default_netvm().name + default_netvm = host_collection.get_default_netvm() + vm_info['netvm'] = default_netvm.name if \ + default_netvm else None vm_info['vm'].uses_default_netvm = True elif options['use-none-netvm']: vm_info['netvm'] = None @@ -1684,6 +1737,22 @@ def restore_info_verify(restore_info, host_collection): 'already-exists', 'excluded']]) + # update references to renamed VMs: + for vm in restore_info.keys(): + if vm in ['$OPTIONS$', 'dom0']: + continue + vm_info = restore_info[vm] + template_name = vm_info['template'] + if (template_name in restore_info and + restore_info[template_name]['good-to-go'] and + 'rename-to' in restore_info[template_name]): + vm_info['template'] = restore_info[template_name]['rename-to'] + netvm_name = vm_info['netvm'] + if (netvm_name in restore_info and + restore_info[netvm_name]['good-to-go'] and + 'rename-to' in restore_info[netvm_name]): + vm_info['netvm'] = restore_info[netvm_name]['rename-to'] + return restore_info @@ -1956,8 +2025,11 @@ def backup_restore_print_summary(restore_info, print_callback=print_stdout): s += " <-- No matching template on the host or in the backup found!" elif 'missing-netvm' in vm_info: s += " <-- No matching netvm on the host or in the backup found!" - elif 'orig-template' in vm_info: - s += " <-- Original template was '%s'" % (vm_info['orig-template']) + else: + if 'orig-template' in vm_info: + s += " <-- Original template was '%s'" % (vm_info['orig-template']) + if 'rename-to' in vm_info: + s += " <-- Will be renamed to '%s'" % vm_info['rename-to'] print_callback(s) @@ -2105,33 +2177,35 @@ def backup_restore_do(restore_info, error_callback("Skipping...") continue - if os.path.exists(vm.dir_path): - move_to_path = tempfile.mkdtemp('', os.path.basename( - vm.dir_path), os.path.dirname(vm.dir_path)) - try: - os.rename(vm.dir_path, move_to_path) - error_callback("*** Directory {} already exists! It has " - "been moved to {}".format(vm.dir_path, - move_to_path)) - except OSError: - error_callback("*** Directory {} already exists and " - "cannot be moved!".format(vm.dir_path)) - error_callback("Skipping...") - continue - template = None if vm.template is not None: template_name = restore_info[vm.name]['template'] template = host_collection.get_vm_by_name(template_name) new_vm = None + vm_name = vm.name + if 'rename-to' in restore_info[vm.name]: + vm_name = restore_info[vm.name]['rename-to'] try: - new_vm = host_collection.add_new_vm(vm_class_name, name=vm.name, - conf_file=vm.conf_file, - dir_path=vm.dir_path, + new_vm = host_collection.add_new_vm(vm_class_name, name=vm_name, template=template, installed_by_rpm=False) + if os.path.exists(new_vm.dir_path): + move_to_path = tempfile.mkdtemp('', os.path.basename( + new_vm.dir_path), os.path.dirname(new_vm.dir_path)) + try: + os.rename(new_vm.dir_path, move_to_path) + error_callback( + "*** Directory {} already exists! It has " + "been moved to {}".format(new_vm.dir_path, + move_to_path)) + except OSError: + error_callback( + "*** Directory {} already exists and " + "cannot be moved!".format(new_vm.dir_path)) + error_callback("Skipping...") + continue if format_version == 1: restore_vm_dir_v1(backup_location, @@ -2166,6 +2240,13 @@ def backup_restore_do(restore_info, error_callback("ERROR: {0}".format(err)) error_callback("*** Some VM property will not be restored") + try: + for service, value in vm.services.items(): + new_vm.services[service] = value + except Exception as err: + error_callback("ERROR: {0}".format(err)) + error_callback("*** Some VM property will not be restored") + try: new_vm.appmenus_create(verbose=callable(print_callback)) except Exception as err: @@ -2175,7 +2256,11 @@ def backup_restore_do(restore_info, # Set network dependencies - only non-default netvm setting for vm in vms.values(): - host_vm = host_collection.get_vm_by_name(vm.name) + vm_name = vm.name + if 'rename-to' in restore_info[vm.name]: + vm_name = restore_info[vm.name]['rename-to'] + host_vm = host_collection.get_vm_by_name(vm_name) + if host_vm is None: # Failed/skipped VM continue @@ -2231,6 +2316,10 @@ def backup_restore_do(restore_info, if retcode != 0: error_callback("*** Error while setting home directory owner") + if callable(print_callback): + print_callback("-> Done. Please install updates for all the restored " + "templates.") + shutil.rmtree(restore_tmpdir) # vim:sw=4:et: diff --git a/core/qubes.py b/core/qubes.py index e910b66b..f3be12d3 100755 --- a/core/qubes.py +++ b/core/qubes.py @@ -224,7 +224,7 @@ class QubesHost(object): cputime = vm.get_cputime() previous[vm.xid] = {} previous[vm.xid]['cpu_time'] = ( - cputime / vm.vcpus) + cputime / max(vm.vcpus, 1)) previous[vm.xid]['cpu_usage'] = 0 time.sleep(wait_time) @@ -651,6 +651,8 @@ class QubesVmCollection(dict): self.qubes_store_file.close() def unlock_db(self): + if self.qubes_store_file is None: + return # intentionally do not call explicit unlock to not unlock the file # before all buffers are flushed self.log.debug('unlock_db()') @@ -711,18 +713,13 @@ class QubesVmCollection(dict): def set_netvm_dependency(self, element): kwargs = {} - attr_list = ("qid", "uses_default_netvm", "netvm_qid") + attr_list = ("qid", "netvm_qid") for attribute in attr_list: kwargs[attribute] = element.get(attribute) vm = self[int(kwargs["qid"])] - if "uses_default_netvm" not in kwargs: - vm.uses_default_netvm = True - else: - vm.uses_default_netvm = ( - True if kwargs["uses_default_netvm"] == "True" else False) if vm.uses_default_netvm is True: if vm.is_proxyvm(): netvm = self.get_default_fw_netvm() diff --git a/core/qubesutils.py b/core/qubesutils.py index 689138a3..ce15dc10 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -25,6 +25,7 @@ from __future__ import absolute_import import string +import errno from lxml import etree from lxml.etree import ElementTree, SubElement, Element @@ -50,6 +51,9 @@ BLKSIZE = 512 AVAILABLE_FRONTENDS = ['xvd'+c for c in string.lowercase[8:]+string.lowercase[:8]] +class USBProxyNotInstalled(QubesException): + pass + def mbytes_to_kmg(size): if size > 1024: return "%d GiB" % (size/1024) @@ -420,6 +424,8 @@ def block_attach(qvmc, vm, device, frontend=None, mode="w", auto_detach=False, w SubElement(disk, 'target').set('dev', frontend) if backend_vm.qid != 0: SubElement(disk, 'backenddomain').set('name', device['vm']) + if mode == "r": + SubElement(disk, 'readonly') vm.libvirt_domain.attachDevice(etree.tostring(disk, encoding='utf-8')) try: # trigger watches to update device status @@ -463,263 +469,205 @@ def block_detach_all(vm): usb_ver_re = re.compile(r"^(1|2)$") usb_device_re = re.compile(r"^[0-9]+-[0-9]+(_[0-9]+)?$") usb_port_re = re.compile(r"^$|^[0-9]+-[0-9]+(\.[0-9]+)?$") +usb_desc_re = re.compile(r"^[ -~]{1,255}$") +# should match valid VM name +usb_connected_to_re = re.compile(r"^[a-zA-Z][a-zA-Z0-9_.-]*$") -def usb_setup(backend_vm_xid, vm_xid, devid, usb_ver): - """ - Attach frontend to the backend. - backend_vm_xid - id of the backend domain - vm_xid - id of the frontend domain - devid - id of the pvusb controller - """ - num_ports = 8 - trans = vmm.xs.transaction_start() - - be_path = "/local/domain/%d/backend/vusb/%d/%d" % (backend_vm_xid, vm_xid, devid) - fe_path = "/local/domain/%d/device/vusb/%d" % (vm_xid, devid) - - be_perm = [{'dom': backend_vm_xid}, {'dom': vm_xid, 'read': True} ] - fe_perm = [{'dom': vm_xid}, {'dom': backend_vm_xid, 'read': True} ] - - # Create directories and set permissions - vmm.xs.write(trans, be_path, "") - vmm.xs.set_permissions(trans, be_path, be_perm) - - vmm.xs.write(trans, fe_path, "") - vmm.xs.set_permissions(trans, fe_path, fe_perm) - - # Write backend information into the location that frontend looks for - vmm.xs.write(trans, "%s/backend-id" % fe_path, str(backend_vm_xid)) - vmm.xs.write(trans, "%s/backend" % fe_path, be_path) - - # Write frontend information into the location that backend looks for - vmm.xs.write(trans, "%s/frontend-id" % be_path, str(vm_xid)) - vmm.xs.write(trans, "%s/frontend" % be_path, fe_path) - - # Write USB Spec version field. - vmm.xs.write(trans, "%s/usb-ver" % be_path, usb_ver) - - # Write virtual root hub field. - vmm.xs.write(trans, "%s/num-ports" % be_path, str(num_ports)) - for port in range(1, num_ports+1): - # Set all port to disconnected state - vmm.xs.write(trans, "%s/port/%d" % (be_path, port), "") - - # Set state to XenbusStateInitialising - vmm.xs.write(trans, "%s/state" % fe_path, "1") - vmm.xs.write(trans, "%s/state" % be_path, "1") - vmm.xs.write(trans, "%s/online" % be_path, "1") - - vmm.xs.transaction_end(trans) - -def usb_decode_device_from_xs(xs_encoded_device): +def usb_decode_device_from_qdb(qdb_encoded_device): """ recover actual device name (xenstore doesn't allow dot in key names, so it was translated to underscore) """ - return xs_encoded_device.replace('_', '.') + return qdb_encoded_device.replace('_', '.') -def usb_encode_device_for_xs(device): +def usb_encode_device_for_qdb(device): """ encode actual device name (xenstore doesn't allow dot in key names, so translated it into underscore) """ return device.replace('.', '_') -def usb_list(): +def usb_list_vm(qvmc, vm): + if not vm.is_running(): + return {} + + try: + untrusted_devices = vm.qdb.multiread('/qubes-usb-devices/') + except Error: + vm.refresh() + return {} + + def get_dev_item(dev, item): + return untrusted_devices.get( + '/qubes-usb-devices/%s/%s' % (dev, item), + None) + + devices = {} + + untrusted_devices_names = list(set(map(lambda x: x.split("/")[2], + untrusted_devices.keys()))) + for untrusted_dev_name in untrusted_devices_names: + if usb_device_re.match(untrusted_dev_name): + dev_name = untrusted_dev_name + untrusted_device_desc = get_dev_item(dev_name, 'desc') + if not usb_desc_re.match(untrusted_device_desc): + print >> sys.stderr, "Invalid %s device desc in VM '%s'" % ( + dev_name, vm.name) + continue + device_desc = untrusted_device_desc + + untrusted_connected_to = get_dev_item(dev_name, 'connected-to') + if untrusted_connected_to: + if not usb_connected_to_re.match(untrusted_connected_to): + print >>sys.stderr, \ + "Invalid %s device 'connected-to' in VM '%s'" % ( + dev_name, vm.name) + continue + connected_to = qvmc.get_vm_by_name(untrusted_connected_to) + if connected_to is None: + print >>sys.stderr, \ + "Device {} appears to be connected to {}, " \ + "but such VM doesn't exist".format( + dev_name, untrusted_connected_to) + else: + connected_to = None + + device = usb_decode_device_from_qdb(dev_name) + + full_name = vm.name + ':' + device + + devices[full_name] = { + 'vm': vm, + 'device': device, + 'qdb_path': '/qubes-usb-devices/' + dev_name, + 'name': full_name, + 'desc': device_desc, + 'connected-to': connected_to, + } + return devices + + +def usb_list(qvmc, vm=None): """ Returns a dictionary of USB devices (for PVUSB backends running in all VM). The dictionary is keyed by 'name' (see below), each element is a dictionary itself: - vm = name of the backend domain - xid = xid of the backend domain - device = - - name = :- + vm = backend domain object + device = device ID + name = : desc = description """ - # FIXME: any better idea of desc_re? - desc_re = re.compile(r"^.{1,255}$") + if vm is not None: + if not vm.is_running(): + return {} + else: + vm_list = [vm] + else: + vm_list = qvmc.values() devices_list = {} - - xs_trans = vmm.xs.transaction_start() - vm_list = vmm.xs.ls(xs_trans, '/local/domain') - - for xid in vm_list: - vm_name = vmm.xs.read(xs_trans, '/local/domain/%s/name' % xid) - vm_devices = vmm.xs.ls(xs_trans, '/local/domain/%s/qubes-usb-devices' % xid) - if vm_devices is None: - continue - # when listing devices in xenstore we get encoded names - for xs_encoded_device in vm_devices: - # Sanitize device id - if not usb_device_re.match(xs_encoded_device): - print >> sys.stderr, "Invalid device id in backend VM '%s'" % vm_name - continue - device = usb_decode_device_from_xs(xs_encoded_device) - device_desc = vmm.xs.read(xs_trans, '/local/domain/%s/qubes-usb-devices/%s/desc' % (xid, xs_encoded_device)) - if not desc_re.match(device_desc): - print >> sys.stderr, "Invalid %s device desc in VM '%s'" % (device, vm_name) - continue - visible_name = "%s:%s" % (vm_name, device) - # grab version - usb_ver = vmm.xs.read(xs_trans, '/local/domain/%s/qubes-usb-devices/%s/usb-ver' % (xid, xs_encoded_device)) - if usb_ver is None or not usb_ver_re.match(usb_ver): - print >> sys.stderr, "Invalid %s device USB version in VM '%s'" % (device, vm_name) - continue - devices_list[visible_name] = {"name": visible_name, "xid":int(xid), - "vm": vm_name, "device":device, - "desc":device_desc, - "usb_ver":usb_ver} - - vmm.xs.transaction_end(xs_trans) + for vm in vm_list: + devices_list.update(usb_list_vm(qvmc, vm)) return devices_list -def usb_check_attached(xs_trans, backend_vm, device): - """ - Checks if the given device in the given backend attached to any frontend. - Parameters: - backend_vm - xid of the backend domain - device - device name in the backend domain - Returns None or a dictionary: - vm - the name of the frontend domain - xid - xid of the frontend domain - frontend - frontend device number FIXME - devid - frontend port number FIXME - """ - # sample xs content: /local/domain/0/backend/vusb/4/0/port/1 = "7-5" - attached_dev = None - vms = vmm.xs.ls(xs_trans, '/local/domain/%d/backend/vusb' % backend_vm) - if vms is None: - return None - for vm in vms: - if not vm.isdigit(): - print >> sys.stderr, "Invalid VM id" - continue - frontend_devs = vmm.xs.ls(xs_trans, '/local/domain/%d/backend/vusb/%s' % (backend_vm, vm)) - if frontend_devs is None: - continue - for frontend_dev in frontend_devs: - if not frontend_dev.isdigit(): - print >> sys.stderr, "Invalid frontend in VM %s" % vm - continue - ports = vmm.xs.ls(xs_trans, '/local/domain/%d/backend/vusb/%s/%s/port' % (backend_vm, vm, frontend_dev)) - if ports is None: - continue - for port in ports: - # FIXME: refactor, see similar loop in usb_find_unused_frontend(), use usb_list() instead? - if not port.isdigit(): - print >> sys.stderr, "Invalid port in VM %s frontend %s" % (vm, frontend) - continue - dev = vmm.xs.read(xs_trans, '/local/domain/%d/backend/vusb/%s/%s/port/%s' % (backend_vm, vm, frontend_dev, port)) - if dev == "": - continue - # Sanitize device id - if not usb_port_re.match(dev): - print >> sys.stderr, "Invalid device id in backend VM %d @ %s/%s/port/%s" % \ - (backend_vm, vm, frontend_dev, port) - continue - if dev == device: - frontend = "%s-%s" % (frontend_dev, port) - #TODO - vm_name = xl_ctx.domid_to_name(int(vm)) - if vm_name is None: - # FIXME: should we wipe references to frontends running on nonexistent VMs? - continue - attached_dev = {"xid":int(vm), "frontend": frontend, "devid": device, "vm": vm_name} - break - return attached_dev - -#def usb_check_frontend_busy(vm, front_dev, port): -# devport = frontend.split("-") -# if len(devport) != 2: -# raise QubesException("Malformed frontend syntax, must be in device-port format") -# # FIXME: -# # return vmm.xs.read('', '/local/domain/%d/device/vusb/%d/state' % (vm.xid, frontend)) == '4' -# return False - -def usb_find_unused_frontend(xs_trans, backend_vm_xid, vm_xid, usb_ver): - """ - Find an unused frontend/port to link the given backend with the given frontend. - Creates new frontend if needed. - Returns frontend specification in - format. - """ - - # This variable holds an index of last frontend scanned by the loop below. - # If nothing found, this value will be used to derive the index of a new frontend. - last_frontend_dev = -1 - - frontend_devs = vmm.xs.ls(xs_trans, "/local/domain/%d/device/vusb" % vm_xid) - if frontend_devs is not None: - for frontend_dev in frontend_devs: - if not frontend_dev.isdigit(): - print >> sys.stderr, "Invalid frontend_dev in VM %d" % vm_xid - continue - frontend_dev = int(frontend_dev) - fe_path = "/local/domain/%d/device/vusb/%d" % (vm_xid, frontend_dev) - if vmm.xs.read(xs_trans, "%s/backend-id" % fe_path) == str(backend_vm_xid): - if vmm.xs.read(xs_trans, '/local/domain/%d/backend/vusb/%d/%d/usb-ver' % (backend_vm_xid, vm_xid, frontend_dev)) != usb_ver: - last_frontend_dev = frontend_dev - continue - # here: found an existing frontend already connected to right backend using an appropriate USB version - ports = vmm.xs.ls(xs_trans, '/local/domain/%d/backend/vusb/%d/%d/port' % (backend_vm_xid, vm_xid, frontend_dev)) - if ports is None: - print >> sys.stderr, "No ports in VM %d frontend_dev %d?" % (vm_xid, frontend_dev) - last_frontend_dev = frontend_dev - continue - for port in ports: - # FIXME: refactor, see similar loop in usb_check_attached(), use usb_list() instead? - if not port.isdigit(): - print >> sys.stderr, "Invalid port in VM %d frontend_dev %d" % (vm_xid, frontend_dev) - continue - port = int(port) - dev = vmm.xs.read(xs_trans, '/local/domain/%d/backend/vusb/%d/%s/port/%s' % (backend_vm_xid, vm_xid, frontend_dev, port)) - # Sanitize device id - if not usb_port_re.match(dev): - print >> sys.stderr, "Invalid device id in backend VM %d @ %d/%d/port/%d" % \ - (backend_vm_xid, vm_xid, frontend_dev, port) - continue - if dev == "": - return '%d-%d' % (frontend_dev, port) - last_frontend_dev = frontend_dev - - # create a new frontend_dev and link it to the backend - frontend_dev = last_frontend_dev + 1 - usb_setup(backend_vm_xid, vm_xid, frontend_dev, usb_ver) - return '%d-%d' % (frontend_dev, 1) - -def usb_attach(vm, backend_vm, device, frontend=None, auto_detach=False, wait=True): - device_attach_check(vm, backend_vm, device, frontend) - - xs_trans = vmm.xs.transaction_start() - - xs_encoded_device = usb_encode_device_for_xs(device) - usb_ver = vmm.xs.read(xs_trans, '/local/domain/%s/qubes-usb-devices/%s/usb-ver' % (backend_vm.xid, xs_encoded_device)) - if usb_ver is None or not usb_ver_re.match(usb_ver): - vmm.xs.transaction_end(xs_trans) - raise QubesException("Invalid %s device USB version in VM '%s'" % (device, backend_vm.name)) - - if frontend is None: - frontend = usb_find_unused_frontend(xs_trans, backend_vm.xid, vm.xid, usb_ver) +def usb_check_attached(qvmc, device): + """Reread device attachment status""" + vm = device['vm'] + untrusted_connected_to = vm.qdb.read( + '{}/connected-to'.format(device['qdb_path'])) + if untrusted_connected_to: + if not usb_connected_to_re.match(untrusted_connected_to): + raise QubesException( + "Invalid %s device 'connected-to' in VM '%s'" % ( + device['device'], vm.name)) + connected_to = qvmc.get_vm_by_name(untrusted_connected_to) + if connected_to is None: + print >>sys.stderr, \ + "Device {} appears to be connected to {}, " \ + "but such VM doesn't exist".format( + device['device'], untrusted_connected_to) else: - # Check if any device attached at this frontend - #if usb_check_frontend_busy(vm, frontend): - # raise QubesException("Frontend %s busy in VM %s, detach it first" % (frontend, vm.name)) - vmm.xs.transaction_end(xs_trans) - raise NotImplementedError("Explicit USB frontend specification is not implemented yet") + connected_to = None + return connected_to - # Check if this device is attached to some domain - attached_vm = usb_check_attached(xs_trans, backend_vm.xid, device) - vmm.xs.transaction_end(xs_trans) +def usb_attach(qvmc, vm, device, auto_detach=False, wait=True): + if not vm.is_running(): + raise QubesException("VM {} not running".format(vm.name)) - if attached_vm: + if not device['vm'].is_running(): + raise QubesException("VM {} not running".format(device['vm'].name)) + + connected_to = usb_check_attached(qvmc, device) + if connected_to: if auto_detach: - usb_detach(backend_vm, attached_vm) + usb_detach(qvmc, device) else: - raise QubesException("Device %s from %s already connected to VM %s as %s" % (device, backend_vm.name, attached_vm['vm'], attached_vm['frontend'])) + raise QubesException("Device {} already connected, to {}".format( + device['name'], connected_to + )) - # Run helper script - xl_cmd = [ '/usr/lib/qubes/xl-qvm-usb-attach.py', str(vm.xid), device, frontend, str(backend_vm.xid) ] - subprocess.check_call(xl_cmd) + # set qrexec policy to allow this device + policy_line = '{} {} allow\n'.format(vm.name, device['vm'].name) + policy_path = '/etc/qubes-rpc/policy/qubes.USB+{}'.format(device['device']) + policy_exists = os.path.exists(policy_path) + if not policy_exists: + try: + fd = os.open(policy_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY) + with os.fdopen(fd, 'w') as f: + f.write(policy_line) + except OSError as e: + if e.errno == errno.EEXIST: + pass + else: + raise + else: + with open(policy_path, 'r+') as f: + policy = f.readlines() + policy.insert(0, policy_line) + f.truncate(0) + f.seek(0) + f.write(''.join(policy)) + try: + # and actual attach + p = vm.run_service('qubes.USBAttach', passio_popen=True, user='root') + (stdout, stderr) = p.communicate( + '{} {}\n'.format(device['vm'].name, device['device'])) + if p.returncode == 127: + raise USBProxyNotInstalled( + "qubes-usb-proxy not installed in the VM") + elif p.returncode != 0: + # TODO: sanitize and include stdout + sanitized_stderr = ''.join([c for c in stderr if ord(c) >= 0x20]) + raise QubesException('Device attach failed: {}'.format( + sanitized_stderr)) + finally: + # FIXME: there is a race condition here - some other process might + # modify the file in the meantime. This may result in unexpected + # denials, but will not allow too much + if not policy_exists: + os.unlink(policy_path) + else: + with open(policy_path, 'r+') as f: + policy = f.readlines() + policy.remove('{} {} allow\n'.format(vm.name, device['vm'].name)) + f.truncate(0) + f.seek(0) + f.write(''.join(policy)) -def usb_detach(backend_vm, attachment): - xl_cmd = [ '/usr/lib/qubes/xl-qvm-usb-detach.py', str(attachment['xid']), attachment['devid'], attachment['frontend'], str(backend_vm.xid) ] - subprocess.check_call(xl_cmd) +def usb_detach(qvmc, vm, device): + connected_to = usb_check_attached(qvmc, device) + # detect race conditions; there is still race here, but much smaller + if connected_to is None or connected_to.qid != vm.qid: + raise QubesException( + "Device {} not connected to VM {}".format( + device['name'], vm.name)) -def usb_detach_all(vm): - raise NotImplementedError("Detaching all devices from a given VM is not implemented yet") + p = device['vm'].run_service('qubes.USBDetach', passio_popen=True, + user='root') + (stdout, stderr) = p.communicate( + '{}\n'.format(device['device'])) + if p.returncode != 0: + # TODO: sanitize and include stdout + raise QubesException('Device detach failed') + +def usb_detach_all(qvmc, vm): + for dev in usb_list(qvmc).values(): + connected_to = dev['connected-to'] + if connected_to is not None and connected_to.qid == vm.qid: + usb_detach(qvmc, connected_to, dev) ####### QubesWatch ###### @@ -760,7 +708,8 @@ class QubesWatch(object): # which can just remove the domain if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN: pass - raise + else: + raise # and for dom0 self._register_watches(None) @@ -791,6 +740,10 @@ class QubesWatch(object): return '/local/domain/%s/memory/meminfo' % xid def _register_watches(self, libvirt_domain): + if libvirt_domain and libvirt_domain.ID() == 0: + # don't use libvirt object for dom0, to always have the same + # hardcoded "dom0" name + libvirt_domain = None if libvirt_domain: name = libvirt_domain.name() if name in self._qdb: @@ -811,6 +764,8 @@ class QubesWatch(object): return else: name = "dom0" + if name in self._qdb: + return self._qdb[name] = QubesDB(name) try: self._qdb[name].watch('/qubes-block-devices') @@ -831,7 +786,10 @@ class QubesWatch(object): self._register_watches(libvirt_domain) def _unregister_watches(self, libvirt_domain): - name = libvirt_domain.name() + if libvirt_domain and libvirt_domain.ID() == 0: + name = "dom0" + else: + name = libvirt_domain.name() if name in self._qdb_events: libvirt.virEventRemoveHandle(self._qdb_events[name]) del(self._qdb_events[name]) diff --git a/core/settings-wni-Windows_NT.py b/core/settings-wni-Windows_NT.py deleted file mode 100644 index 6e646c9d..00000000 --- a/core/settings-wni-Windows_NT.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/python2 - -from __future__ import absolute_import -import _winreg -import os -import sys - -from qubes.storage.wni import QubesWniVmStorage - -DEFAULT_INSTALLDIR = 'c:\\program files\\Invisible Things Lab\\Qubes WNI' -DEFAULT_STOREDIR = 'c:\\qubes' - -def apply(system_path, vm_files, defaults): - system_path['qubes_base_dir'] = DEFAULT_STOREDIR - installdir = DEFAULT_INSTALLDIR - try: - reg_key = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, - "Software\\Invisible Things Lab\\Qubes WNI") - installdir = _winreg.QueryValueEx(reg_key, "InstallDir")[0] - system_path['qubes_base_dir'] = \ - _winreg.QueryValueEx(reg_key, "StoreDir")[0] - except WindowsError as e: - print >>sys.stderr, \ - "WARNING: invalid installation: missing registry entries (%s)" \ - % str(e) - - system_path['config_template_pv'] = \ - os.path.join(installdir, 'vm-template.xml') - system_path['config_template_hvm'] = \ - os.path.join(installdir, 'vm-template-hvm.xml') - system_path['qubes_icon_dir'] = os.path.join(installdir, 'icons') - system_path['qubesdb_daemon_path'] = \ - os.path.join(installdir, 'bin\\qubesdb-daemon.exe') - system_path['qrexec_daemon_path'] = \ - os.path.join(installdir, 'bin\\qrexec-daemon.exe') - system_path['qrexec_client_path'] = \ - os.path.join(installdir, 'bin\\qrexec-client.exe') - system_path['qrexec_policy_dir'] = \ - os.path.join(installdir, 'qubes-rpc\\policy') - # Specific to WNI - normally VM have this file - system_path['qrexec_agent_path'] = \ - os.path.join(installdir, 'bin\\qrexec-agent.exe') - - defaults['libvirt_uri'] = 'wni:///' - defaults['storage_class'] = QubesWniVmStorage diff --git a/core/settings-xen-Linux.py b/core/settings-xen-Linux.py index de5084f5..c413e8ae 100644 --- a/core/settings-xen-Linux.py +++ b/core/settings-xen-Linux.py @@ -2,7 +2,10 @@ from __future__ import absolute_import -from qubes.storage.xen import QubesXenVmStorage +from qubes.storage.xen import XenStorage, XenPool + def apply(system_path, vm_files, defaults): - defaults['storage_class'] = QubesXenVmStorage + defaults['storage_class'] = XenStorage + defaults['pool_drivers'] = {'xen': XenPool} + defaults['pool_config'] = {'dir_path': '/var/lib/qubes/'} diff --git a/core/storage/Makefile b/core/storage/Makefile index ec59cc64..7c7af60e 100644 --- a/core/storage/Makefile +++ b/core/storage/Makefile @@ -1,5 +1,6 @@ OS ?= Linux +SYSCONFDIR ?= /etc PYTHON_QUBESPATH = $(PYTHON_SITEPATH)/qubes all: @@ -13,6 +14,8 @@ endif mkdir -p $(DESTDIR)$(PYTHON_QUBESPATH)/storage cp __init__.py $(DESTDIR)$(PYTHON_QUBESPATH)/storage cp __init__.py[co] $(DESTDIR)$(PYTHON_QUBESPATH)/storage + mkdir -p $(DESTDIR)$(SYSCONFDIR)/qubes + cp storage.conf $(DESTDIR)$(SYSCONFDIR)/qubes/ ifneq ($(BACKEND_VMM),) if [ -r $(BACKEND_VMM).py ]; then \ cp $(BACKEND_VMM).py $(DESTDIR)$(PYTHON_QUBESPATH)/storage && \ diff --git a/core/storage/README.md b/core/storage/README.md new file mode 100644 index 00000000..7258512f --- /dev/null +++ b/core/storage/README.md @@ -0,0 +1,3 @@ +# WNI File storage +Before v3.1 there existed a draft wni storage. You can find it in the git +history diff --git a/core/storage/__init__.py b/core/storage/__init__.py index 29cf8654..b5a744aa 100644 --- a/core/storage/__init__.py +++ b/core/storage/__init__.py @@ -16,22 +16,24 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. # from __future__ import absolute_import +import ConfigParser import os import os.path -import re import shutil import subprocess import sys -from qubes.qubes import vm_files,system_path,defaults -from qubes.qubes import QubesException import qubes.qubesutils +from qubes.qubes import QubesException, defaults, system_path + +CONFIG_FILE = '/etc/qubes/storage.conf' + class QubesVmStorage(object): """ @@ -55,12 +57,10 @@ class QubesVmStorage(object): else: self.root_img_size = defaults['root_img_size'] - self.private_img = vm.absolute_path(vm_files["private_img"], None) - if self.vm.template: - self.root_img = self.vm.template.root_img - else: - self.root_img = vm.absolute_path(vm_files["root_img"], None) - self.volatile_img = vm.absolute_path(vm_files["volatile_img"], None) + self.root_dev = "xvda" + self.private_dev = "xvdb" + self.volatile_dev = "xvdc" + self.modules_dev = "xvdd" # For now compute this path still in QubesVm self.modules_img = modules_img @@ -69,9 +69,69 @@ class QubesVmStorage(object): # Additional drive (currently used only by HVM) self.drive = None + def format_disk_dev(self, path, script, vdev, rw=True, type="disk", + domain=None): + if path is None: + return '' + template = " \n" \ + " \n" \ + " \n" \ + " \n" \ + "{params}" \ + " \n" + params = "" + if not rw: + params += " \n" + if domain: + params += " \n" % domain + if script: + params += "