From d2640b517fc3e0ee4659bfce126a8fe9a771d5c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 20 Jan 2016 03:31:11 +0100 Subject: [PATCH 01/38] backup: Allow to specify custom temporary directory Using tmpfs-backed directory may speed up the backup process. QubesOS/qubes-issues#1652 --- core/backup.py | 5 +++-- qvm-tools/qvm-backup | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/core/backup.py b/core/backup.py index 7ba8808f..f329b083 100644 --- a/core/backup.py +++ b/core/backup.py @@ -449,7 +449,8 @@ def prepare_backup_header(target_directory, passphrase, compressed=False, def backup_do(base_backup_dir, files_to_backup, passphrase, progress_callback=None, encrypted=False, appvm=None, compressed=False, hmac_algorithm=DEFAULT_HMAC_ALGORITHM, - crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM): + crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM, + tmpdir="/var/tmp"): global running_backup_operation def queue_put_with_check(proc, vmproc, queue, element): @@ -510,7 +511,7 @@ def backup_do(base_backup_dir, files_to_backup, passphrase, progress = blocks_backedup * 11 / total_backup_sz progress_callback(progress) - backup_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/backup_") + backup_tmpdir = tempfile.mkdtemp(prefix="{}/backup_".format(tmpdir)) running_backup_operation.tmpdir_to_remove = backup_tmpdir # Tar with tape length does not deals well with stdout (close stdout between diff --git a/qvm-tools/qvm-backup b/qvm-tools/qvm-backup index aa69f9c6..5270864c 100755 --- a/qvm-tools/qvm-backup +++ b/qvm-tools/qvm-backup @@ -67,6 +67,10 @@ def main(): dest="compress_filter", default=False, help="Compress the backup using specified filter " "program (default: gzip)") + parser.add_option("--tmpdir", action="store", dest="tmpdir", default=None, + help="Custom temporary directory (if you have at least " + "1GB free RAM in dom0, use of /tmp is advised) (" + "default: /var/tmp)") parser.add_option ("--debug", action="store_true", dest="debug", default=False, help="Enable (a lot of) debug output") @@ -180,6 +184,8 @@ def main(): kwargs['hmac_algorithm'] = options.hmac_algorithm if options.crypto_algorithm: kwargs['crypto_algorithm'] = options.crypto_algorithm + if options.tmpdir: + kwargs['tmpdir'] = options.tmpdir try: backup_do(base_backup_dir, files_to_backup, passphrase, From 958c292607a16f3cd72120accd2535a62c055240 Mon Sep 17 00:00:00 2001 From: o Date: Mon, 4 Apr 2016 15:29:47 +0200 Subject: [PATCH 02/38] fix qvm-ls display of cpu_time qhost.measure_cpu_usage expects the qvm_collection as parameter. Also the number of vcpus of dom0 seems to be 0, leading to a div by 0. A more complete fix would probably involve e.g. a new num_cores property which would contain number of vcpu for vhosts and number of actual cores for dom0. For now this is a partial solution. --- core/qubes.py | 2 +- qvm-tools/qvm-ls | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/qubes.py b/core/qubes.py index 34d007f5..2fc9c140 100755 --- a/core/qubes.py +++ b/core/qubes.py @@ -224,7 +224,7 @@ class QubesHost(object): cputime = vm.get_cputime() previous[vm.xid] = {} previous[vm.xid]['cpu_time'] = ( - cputime / vm.vcpus) + cputime / max(vm.vcpus, 1)) previous[vm.xid]['cpu_usage'] = 0 time.sleep(wait_time) diff --git a/qvm-tools/qvm-ls b/qvm-tools/qvm-ls index 42e6f4c7..e53d2a3e 100755 --- a/qvm-tools/qvm-ls +++ b/qvm-tools/qvm-ls @@ -148,7 +148,7 @@ def main(): if (options.cpu): qhost = QubesHost() - (measure_time, cpu_usages) = qhost.measure_cpu_usage() + (measure_time, cpu_usages) = qhost.measure_cpu_usage(qvm_collection) fields_to_display += ["cpu"] if (options.mem): From b04d1ce00589b550ce608a1a5cd3cea004b992b0 Mon Sep 17 00:00:00 2001 From: o Date: Mon, 4 Apr 2016 15:48:40 +0200 Subject: [PATCH 03/38] new qvm-top utility Display cpu and mem similar to qvm-ls but ordered by cpu time. Also a one line summary switch which includes the top n cpu consuming vms and total memory consumption. Intended usage is to e.g. embed in a window manager widget. --- qvm-tools/qvm-top | 128 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100755 qvm-tools/qvm-top diff --git a/qvm-tools/qvm-top b/qvm-tools/qvm-top new file mode 100755 index 00000000..eb54379a --- /dev/null +++ b/qvm-tools/qvm-top @@ -0,0 +1,128 @@ +#!/usr/bin/python2 +# -*- encoding: utf8 -*- +# +# The Qubes OS Project, http://www.qubes-os.org +# +# Copyright (C) 2010 Joanna Rutkowska +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# + +from qubes.qubes import QubesVmCollection +from qubes.qubes import QubesHost +from qubes.qubes import QubesException +from optparse import OptionParser +import sys + +def main(): + usage = "usage: %prog [options]" + parser = OptionParser (usage) + + parser.add_option("--list", dest="list_top", + action="store_true", default=False, + help="n m : One line summary of top n vms with more than m cpu_time %") + + + (options, args) = parser.parse_args () + + + qvm_collection = QubesVmCollection() + qvm_collection.lock_db_for_reading() + qvm_collection.load() + qvm_collection.unlock_db() + + fields_to_display = ["name", "cpu", "mem"] + + cpu_usages = None + + qhost = QubesHost() + + (measure_time, cpu_usages) = qhost.measure_cpu_usage(qvm_collection) + + vms_list = [vm for vm in qvm_collection.values() if vm.is_running()] + vms_list = sorted(vms_list, key= lambda vm: 1-cpu_usages[vm.get_xid()]['cpu_usage']) + + no_vms = len (vms_list) + vms_to_display = vms_list + + if options.list_top: + any_shown = False + ndisp = 3 + cputh = 0 + if len(args) > 0: + ndisp = int(args[0]) + if len(args) > 1: + cputh = int(args[1]) + + for vm in vms_to_display[:ndisp]: + cpu = cpu_usages[vm.get_xid()]['cpu_usage'] + if cpu > cputh: + any_shown = True + sys.stdout.write("%d %s, " % (cpu, vm.name)) + + if any_shown: + sys.stdout.write(" ... | ") + + totalMem = 0 + dom0mem = 0 + for vm in vms_to_display: + if not vm.name == "dom0": + totalMem += vm.get_mem() + else: + dom0mem = vm.get_mem() + totalMem /= 1024.0 * 1024.0 + dom0mem /= 1024.0 * 1024.0 + sys.stdout.write("%.1f G + %.1f G" % (totalMem, dom0mem)) + return + + max_width = { 'name': 0, 'cpu': 0, 'mem': 0 } + data_to_display = [] + for vm in vms_to_display: + data_row = {} + data_row['name'] = vm.name + max_width['name'] = max(max_width['name'], len(data_row['name'])) + data_row['cpu'] = "%.1f" % (cpu_usages[vm.get_xid()]['cpu_usage']) + max_width['cpu'] = max(max_width['cpu'], len(data_row['cpu'])) + data_row['mem'] = "%d" % (vm.get_mem() / (1024.0)) + max_width['mem'] = max(max_width['mem'], len(data_row['mem'])) + data_to_display.append(data_row) + + # Display the header + s = "" + for f in fields_to_display: + fmt="{{0:-^{0}}}-+".format(max_width[f] + 1) + s += fmt.format('-') + print s + s = "" + for f in fields_to_display: + fmt="{{0:>{0}}} |".format(max_width[f] + 1) + s += fmt.format(f) + print s + s = "" + for f in fields_to_display: + fmt="{{0:-^{0}}}-+".format(max_width[f] + 1) + s += fmt.format('-') + print s + + # ... and the actual data + for row in data_to_display: + s = "" + for f in fields_to_display: + fmt="{{0:>{0}}} |".format(max_width[f] + 1) + s += fmt.format(row[f]) + print s + +main() From 01bc25726532be0bae30fa8529653ccd88dce94a Mon Sep 17 00:00:00 2001 From: o Date: Mon, 4 Apr 2016 15:57:29 +0200 Subject: [PATCH 04/38] cleanup whitespace mess --- qvm-tools/qvm-top | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/qvm-tools/qvm-top b/qvm-tools/qvm-top index eb54379a..16ebdb8d 100755 --- a/qvm-tools/qvm-top +++ b/qvm-tools/qvm-top @@ -35,10 +35,8 @@ def main(): action="store_true", default=False, help="n m : One line summary of top n vms with more than m cpu_time %") - (options, args) = parser.parse_args () - qvm_collection = QubesVmCollection() qvm_collection.lock_db_for_reading() qvm_collection.load() @@ -47,9 +45,7 @@ def main(): fields_to_display = ["name", "cpu", "mem"] cpu_usages = None - qhost = QubesHost() - (measure_time, cpu_usages) = qhost.measure_cpu_usage(qvm_collection) vms_list = [vm for vm in qvm_collection.values() if vm.is_running()] @@ -62,33 +58,33 @@ def main(): any_shown = False ndisp = 3 cputh = 0 - if len(args) > 0: - ndisp = int(args[0]) - if len(args) > 1: - cputh = int(args[1]) + if len(args) > 0: + ndisp = int(args[0]) + if len(args) > 1: + cputh = int(args[1]) for vm in vms_to_display[:ndisp]: cpu = cpu_usages[vm.get_xid()]['cpu_usage'] if cpu > cputh: any_shown = True - sys.stdout.write("%d %s, " % (cpu, vm.name)) + sys.stdout.write("%d %s, " % (cpu, vm.name)) - if any_shown: + if any_shown: sys.stdout.write(" ... | ") totalMem = 0 dom0mem = 0 - for vm in vms_to_display: + for vm in vms_to_display: if not vm.name == "dom0": - totalMem += vm.get_mem() + totalMem += vm.get_mem() else: dom0mem = vm.get_mem() totalMem /= 1024.0 * 1024.0 dom0mem /= 1024.0 * 1024.0 sys.stdout.write("%.1f G + %.1f G" % (totalMem, dom0mem)) - return + return - max_width = { 'name': 0, 'cpu': 0, 'mem': 0 } + max_width = { 'name': 0, 'cpu': 0, 'mem': 0 } data_to_display = [] for vm in vms_to_display: data_row = {} @@ -99,7 +95,7 @@ def main(): data_row['mem'] = "%d" % (vm.get_mem() / (1024.0)) max_width['mem'] = max(max_width['mem'], len(data_row['mem'])) data_to_display.append(data_row) - + # Display the header s = "" for f in fields_to_display: @@ -109,7 +105,7 @@ def main(): s = "" for f in fields_to_display: fmt="{{0:>{0}}} |".format(max_width[f] + 1) - s += fmt.format(f) + s += fmt.format(f) print s s = "" for f in fields_to_display: @@ -122,7 +118,7 @@ def main(): s = "" for f in fields_to_display: fmt="{{0:>{0}}} |".format(max_width[f] + 1) - s += fmt.format(row[f]) + s += fmt.format(row[f]) print s main() From d9b8516b6d574caba68ef4516de2fd00c982d3dc Mon Sep 17 00:00:00 2001 From: Mario Geckler Date: Wed, 27 Apr 2016 12:46:05 +0200 Subject: [PATCH 05/38] Fixed #1930: Delete autostart Symlink when deleting a VM --- core-modules/000QubesVm.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py index 227ab8ac..fbbe8b40 100644 --- a/core-modules/000QubesVm.py +++ b/core-modules/000QubesVm.py @@ -1412,6 +1412,11 @@ class QubesVm(object): e.get_error_code()) raise + if os.path.exists("/etc/systemd/system/multi-user.target.wants/qubes-vm@" + self.name + ".service") == True: + retcode = subprocess.call(["sudo", "rm", "/etc/systemd/system/multi-user.target.wants/qubes-vm@" + self.name + ".service"]) + if retcode != 0: + raise QubesException("Failed to delete autostart entry for VM") + self.storage.remove_from_disk() def write_firewall_conf(self, conf): From 1fe625aa536ca16d6672efeefd58759a37ee99a6 Mon Sep 17 00:00:00 2001 From: Patrick Schleizer Date: Wed, 27 Apr 2016 17:43:31 +0000 Subject: [PATCH 06/38] allow sys-whonix, whonix-gw and whonix-ws by default to open links in anon-whonix https://phabricator.whonix.org/T452 --- qubes-rpc-policy/qubes.OpenInVM.policy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qubes-rpc-policy/qubes.OpenInVM.policy b/qubes-rpc-policy/qubes.OpenInVM.policy index 41217337..27303cc9 100644 --- a/qubes-rpc-policy/qubes.OpenInVM.policy +++ b/qubes-rpc-policy/qubes.OpenInVM.policy @@ -3,5 +3,8 @@ ## Please use a single # to start your custom comments +sys-whonix anon-whonix allow +whonix-gw anon-whonix allow +whonix-ws anon-whonix allow $anyvm $dispvm allow $anyvm $anyvm ask From 23ec9e92bbe5e4944e7b94f2a9a47bb265b108f5 Mon Sep 17 00:00:00 2001 From: Mario Geckler Date: Thu, 28 Apr 2016 08:30:57 +0200 Subject: [PATCH 07/38] removed unnecessary comparison with True and changed to sysctl instead of remove --- core-modules/000QubesVm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py index fbbe8b40..2755a90a 100644 --- a/core-modules/000QubesVm.py +++ b/core-modules/000QubesVm.py @@ -1412,8 +1412,8 @@ class QubesVm(object): e.get_error_code()) raise - if os.path.exists("/etc/systemd/system/multi-user.target.wants/qubes-vm@" + self.name + ".service") == True: - retcode = subprocess.call(["sudo", "rm", "/etc/systemd/system/multi-user.target.wants/qubes-vm@" + self.name + ".service"]) + if os.path.exists("/etc/systemd/system/multi-user.target.wants/qubes-vm@" + self.name + ".service"): + subprocess.call(["sudo", "systemctl", "-q", "disable","qubes-vm@" + self.name + ".service"]) if retcode != 0: raise QubesException("Failed to delete autostart entry for VM") From edd473c4be2ccd7eaff32b17e554f84e6d1c3693 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 2 May 2016 00:18:37 +0200 Subject: [PATCH 08/38] tests: fix 'extra' tests loader QubesOS/qubes-issues#1800 --- tests/extra.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/extra.py b/tests/extra.py index 31943b3c..91efade8 100644 --- a/tests/extra.py +++ b/tests/extra.py @@ -73,7 +73,7 @@ class ExtraTestMixin(qubes.tests.SystemTestsMixin): def load_tests(loader, tests, pattern): for entry in pkg_resources.iter_entry_points('qubes.tests.extra'): - for test_case in entry(): + for test_case in entry.load()(): tests.addTests(loader.loadTestsFromTestCase( type( entry.name + '_' + test_case.__name__, From d88ff935d02ec82ed024bb3acf7e35e6fba035af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 2 May 2016 00:19:18 +0200 Subject: [PATCH 09/38] tests: adjust 'extra' tests API to better design from core3 Do not force inheritance of ExtraTestMixin and QubesTestCase. Instead provide 'qubes.tests.extra.ExtraTestCase' for external tests. This makes the API less "magic", easier to understand and apply static analysis tools on it. QubesOS/qubes-issues#1800 --- tests/__init__.py | 2 +- tests/extra.py | 15 ++++----------- tests/run.py | 2 +- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index e30200fa..c9d26b49 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -175,7 +175,7 @@ class QubesTestCase(unittest.TestCase): def __str__(self): return '{}/{}/{}'.format( - '.'.join(self.__class__.__module__.split('.')[2:]), + self.__class__.__module__, self.__class__.__name__, self._testMethodName) diff --git a/tests/extra.py b/tests/extra.py index 91efade8..c46a8a81 100644 --- a/tests/extra.py +++ b/tests/extra.py @@ -27,12 +27,12 @@ import qubes.tests import qubes.qubes -class ExtraTestMixin(qubes.tests.SystemTestsMixin): +class ExtraTestCase(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): template = None def setUp(self): - super(ExtraTestMixin, self).setUp() + super(ExtraTestCase, self).setUp() self.qc.unlock_db() def create_vms(self, names): @@ -74,13 +74,7 @@ class ExtraTestMixin(qubes.tests.SystemTestsMixin): def load_tests(loader, tests, pattern): for entry in pkg_resources.iter_entry_points('qubes.tests.extra'): for test_case in entry.load()(): - tests.addTests(loader.loadTestsFromTestCase( - type( - entry.name + '_' + test_case.__name__, - (test_case, ExtraTestMixin, qubes.tests.QubesTestCase), - {} - ) - )) + tests.addTests(loader.loadTestsFromTestCase(test_case)) try: qc = qubes.qubes.QubesVmCollection() @@ -100,8 +94,7 @@ def load_tests(loader, tests, pattern): type( '{}_{}_{}'.format( entry.name, test_case.__name__, template), - (test_case, ExtraTestMixin, - qubes.tests.QubesTestCase), + (test_case,), {'template': template} ) )) diff --git a/tests/run.py b/tests/run.py index dae4c0b7..78441eb3 100755 --- a/tests/run.py +++ b/tests/run.py @@ -314,7 +314,7 @@ def main(): for name in args.names: suite.addTests( [test for test in list_test_cases(alltests) - if (str(test)+'/').startswith(name.replace('.', '/')+'/')]) + if str(test).startswith(name)]) else: suite.addTests(loader.loadTestsFromName('qubes.tests')) From 7e76342919c8f9123a64a4838afdcf3dd5b116a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 5 May 2016 00:00:25 +0200 Subject: [PATCH 10/38] backup: default tmpdir to /tmp, respect TMPDIR This requires having at least 1GB free on /tmp, but it is fair assumption - it's tmpfs in dom0 and while performing the backup most of the VMs aren't running, so shouldn't be a problem. Anyway it is always possible to set TMPDIR variable or pass --tmpdir cmdline option. Using tmpfs based temporary directory should speedup the backup. QubesOS/qubes-issues#1652 --- core/backup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/backup.py b/core/backup.py index 49618598..1f774122 100644 --- a/core/backup.py +++ b/core/backup.py @@ -450,7 +450,7 @@ def backup_do(base_backup_dir, files_to_backup, passphrase, progress_callback=None, encrypted=False, appvm=None, compressed=False, hmac_algorithm=DEFAULT_HMAC_ALGORITHM, crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM, - tmpdir="/var/tmp"): + tmpdir=None): global running_backup_operation def queue_put_with_check(proc, vmproc, queue, element): @@ -511,7 +511,7 @@ def backup_do(base_backup_dir, files_to_backup, passphrase, progress = blocks_backedup * 11 / total_backup_sz progress_callback(progress) - backup_tmpdir = tempfile.mkdtemp(prefix="{}/backup_".format(tmpdir)) + backup_tmpdir = tempfile.mkdtemp(prefix="backup_", dir=tmpdir) running_backup_operation.tmpdir_to_remove = backup_tmpdir # Tar with tape length does not deals well with stdout (close stdout between From db8e79a9034e080016c7e44a3e27152a14ee6ec5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 5 May 2016 00:03:45 +0200 Subject: [PATCH 11/38] version 3.2.1 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 944880fa..e4604e3a 100644 --- a/version +++ b/version @@ -1 +1 @@ -3.2.0 +3.2.1 From 16fbb33ce3b45a111e53c16eb22ff7ad09957fde Mon Sep 17 00:00:00 2001 From: Jeppler Date: Fri, 13 May 2016 20:50:02 -0500 Subject: [PATCH 12/38] Tool to create bug reports. A bug report is a collection of system information and log files for a specific qube. --- qubes-bug-report | 213 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 qubes-bug-report diff --git a/qubes-bug-report b/qubes-bug-report new file mode 100644 index 00000000..f4127d4a --- /dev/null +++ b/qubes-bug-report @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 + +import subprocess as sub +import shlex +import argparse +import time +import sys + +from os.path import expanduser + +#the term qube refers to a qubes vm + +def is_program_installed_in_qube( qube_name, command ): + is_installed = True + + try: + shell_cmd = "qvm-run " + shlex.quote( qube_name ) + " --pass-io --no-color-output 'command -v " + command + "' &> /dev/null" + sub.check_call( shell_cmd, shell = True ) + + except sub.CalledProcessError: + is_installed = False + + return is_installed + + +#this function requires virsh +#domstate only works for Xen domU (guests) +def is_qube_running( qube_name ): + runs = False + + out = sub.check_output([ "virsh", "-c", "xen:///", "domstate", shlex.quote( qube_name ) ]) + out = out.decode('utf-8').replace('\n', '') + + if 'running' == out: + runs = True + + return runs + + +def get_qube_packages( qube_name ): + content = "## Qubes Packages\n\n" + + #a qube can have more than one package manager installed (only one is functional) + pkg_cmd = { 'dpkg' : 'dpkg -l qubes-*', 'pacman' : 'pacman -Qs', 'rpm' : 'rpm -qa qubes-*' } + + if is_qube_running( qube_name ): + + for package_manager in pkg_cmd.keys(): + if is_program_installed_in_qube( qube_name, package_manager ): + pkg_list_cmd = pkg_cmd[package_manager] + + try: + shell_cmd = "qvm-run " + shlex.quote( qube_name ) + " --pass-io --no-color-output '" + pkg_list_cmd + "' 2> /dev/null" + out = sub.check_output( shell_cmd, shell = True ) + out = out.decode('utf-8') + content = content + "### Package Manager: " + package_manager + "\n\n" + content = content + wrap_code( out ) + except sub.CalledProcessError: + True #do nothing + + else: + content = content + "**No packages listed, because Qube " + qube_name + " was not running**\n\n" + + return content + +def get_dom0_packages(): + content = "## Dom0 Packages\n\n" + out = sub.check_output([ "rpm", "-qa", "qubes-*" ]) + out = out.decode('utf-8') + content = content + wrap_code( out ) + + return content + + +def wrap_code( text ): + code = "~~~\n" + text + "~~~\n\n" + + return code + + +def get_log_file_content( qube_name ): + content = "## Log Files\n\n" + qubes_os_log = "/var/log/qubes/" + ext = ".log" + + log_prefix = [ "guid", "pacat", "qubesdb", "qrexec" ] + + for prefix in log_prefix: + log_file = prefix + "." + qube_name + ext + content = content + "### Log File: " + log_file + "\n\n" + content = content + wrap_code( get_log_file( qubes_os_log + log_file ) ) + + return content + + +def get_qube_prefs( qube_name ): + qube_prefs = sub.check_output([ "qvm-prefs", shlex.quote( qube_name ) ]) + qube_prefs = qube_prefs.decode('utf-8') + + content = "### Qube Prefs\n\n" + content = content + wrap_code( qube_prefs ) + + return content + + +def report( qube_name ): + template = '''# {title} + +{content} +''' + + title = "Bug report: " + qube_name + + content = get_qube_prefs( qube_name ) + content = content + get_dom0_packages() + content = content + get_log_file_content( qube_name ) + content = content + get_qube_packages( qube_name ) + + + report = template.format( **locals() ) + + return report + + +def write_report( report_content, file_path ): + with open( file_path, 'w' ) as report_file: + report_file.write( report_content ) + + +def send_report( dest_qube, file_path): + #if dest_qube is not running -> start dest_qube + if not is_qube_running( dest_qube ): + try: + sub.check_call([ "qvm-start", shlex.quote( dest_qube ) ]) + except sub.CalledProcessError: + print( "Error while starting: " + dest_qube, file = sys.stderr ) + + try: + sub.check_call([ "qvm-move-to-vm", shlex.quote( dest_qube ), file_path ]) + except sub.calledProcessError: + print( "Moving file bug-report failed", file = sys.stderr ) + + +def get_log_file( log_file ): + data = "" + + #open and close the file + with open( log_file ) as log: + data = log.read() + + return data + + +def qube_exist( qube_name ): + exists = True + + try: + #calls: qvm-check --quiet vmanme + sub.check_call([ "qvm-check", "--quiet", shlex.quote( qube_name ) ]) + + except sub.CalledProcessError: + exists = False + + return exists + + +def get_report_file_path( qube_name ): + #exapanduser sub.CalledProcessError:-> works corss platform + home_dir = expanduser("~") + date = time.strftime("%H%M%S") + file_path = home_dir + "/" + qube_name + "_bug-report_" + date + ".md" + + return file_path + + +def main(): + parser = argparse.ArgumentParser( description = 'Generates a bug report for a specific qube (Qubes VM)' ) + parser.add_argument( 'vmname', metavar = '', type = str ) + parser.add_argument( '-d', '--dest-vm', dest = "destvm", type = str, default = 'dom0', help = "destination vm" ) + parser.add_argument( '-p', '--print-report', action = 'store_const', const = "print_report", required = False, help = "prints the report without writing it or sending it to a destination VM" ) + args = parser.parse_args() + + if qube_exist( args.vmname ): + + if qube_exist( args.destvm ): + report_content = report( args.vmname ) + + if args.print_report: + print( report_content ) + + else: + file_path = get_report_file_path( args.vmname ) + write_report( report_content, file_path ) + print( "Report written to: " + file_path ) + + if 'dom0' != args.destvm: + send_report( args.destvm, file_path ) + print( "Report send to VM: " + args.destvm ) + + exit(0) + + else: + print ( "Destination VM does not exist" ) + exit(1) + + else: + print( "VM does not exist" ) + exit(1) + + print( args ) + + +main() From 92b49fe9a49281b216ff890a4736c79c93ee9c34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 15 May 2016 14:00:37 +0200 Subject: [PATCH 13/38] core: Change default vCPUS to 2 This behaves better when running multiple VMs. If one need full CPU power in some VM, it is always possible to increase vCPUs for this particular VM. Fixes QubesOS/qubes-issues#1891 --- core-modules/000QubesVm.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py index 2755a90a..fec8a7b7 100644 --- a/core-modules/000QubesVm.py +++ b/core-modules/000QubesVm.py @@ -136,7 +136,7 @@ class QubesVm(object): "pci_strictreset": {"default": True}, # Internal VM (not shown in qubes-manager, doesn't create appmenus entries "internal": { "default": False, 'attr': '_internal' }, - "vcpus": { "default": None }, + "vcpus": { "default": 2 }, "uses_default_kernel": { "default": True, 'order': 30 }, "uses_default_kernelopts": { "default": True, 'order': 30 }, "kernel": { @@ -327,11 +327,6 @@ class QubesVm(object): if self.maxmem > self.memory * 10: self.maxmem = self.memory * 10 - # By default allow use all VCPUs - if self.vcpus is None and not vmm.offline_mode: - qubes_host = QubesHost() - self.vcpus = qubes_host.no_cpus - # Always set if meminfo-writer should be active or not if 'meminfo-writer' not in self.services: self.services['meminfo-writer'] = not (len(self.pcidevs) > 0) From 10f071691102d7f0359060bd3cf4833f497904ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 15 May 2016 15:08:30 +0200 Subject: [PATCH 14/38] trim-template: Fix handling long named templates Trim template name to fit full VM name in 31 chars. At the same time, check if the VM already exists - if so - remove it first (or error out asking the user to remove it manually - if VM isn't marked as internal). Now that VM is created as internal, to skip appmenus creation. QubesOS/qubes-issues#1910 Fixes QubesOS/qubes-issues#1655 --- qvm-tools/qvm-trim-template | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/qvm-tools/qvm-trim-template b/qvm-tools/qvm-trim-template index f576822b..d5812210 100755 --- a/qvm-tools/qvm-trim-template +++ b/qvm-tools/qvm-trim-template @@ -92,10 +92,21 @@ def main(): touch_dvm_savefile = is_dvm_up_to_date(tvm, dvm_tmpl) print >> sys.stderr, "Creating temporary VM..." - fstrim_vm = qvm_collection.add_new_vm("QubesAppVm", - template=tvm, - name="{}-fstrim".format(tvm_name), - netvm=None, + trim_vmname = "trim-{}".format(tvm_name[:31 - len('trim-')]) + fstrim_vm = qvm_collection.get_vm_by_name(trim_vmname) + if fstrim_vm is not None: + if not fstrim_vm.internal: + print >>sys.stderr, \ + "ERROR: VM '{}' already exists and is not marked as internal. " \ + "Remove it manually." + fstrim_vm.remove_from_disk() + qvm_collection.pop(fstrim_vm.qid) + fstrim_vm = qvm_collection.add_new_vm( + "QubesAppVm", + template=tvm, + name=trim_vmname, + netvm=None, + internal=True, ) if not fstrim_vm: print >> sys.stderr, "ERROR: Failed to create new VM" From 55af04293b190215c4e2ecced4fbca60f6ef7a58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 16 May 2016 04:48:29 +0200 Subject: [PATCH 15/38] tests: block devices listing QubesOS/qubes-issues#1600 --- tests/Makefile | 2 + tests/__init__.py | 1 + tests/block.py | 302 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 305 insertions(+) create mode 100644 tests/block.py diff --git a/tests/Makefile b/tests/Makefile index 993c741a..8f01e2ad 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -17,6 +17,8 @@ endif cp backupcompatibility.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) cp basic.py $(DESTDIR)$(PYTHON_TESTSPATH) cp basic.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) + cp block.py $(DESTDIR)$(PYTHON_TESTSPATH) + cp block.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) cp dom0_update.py $(DESTDIR)$(PYTHON_TESTSPATH) cp dom0_update.py[co] $(DESTDIR)$(PYTHON_TESTSPATH) cp network.py $(DESTDIR)$(PYTHON_TESTSPATH) diff --git a/tests/__init__.py b/tests/__init__.py index c9d26b49..67ceeaee 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -758,6 +758,7 @@ def load_tests(loader, tests, pattern): 'qubes.tests.regressions', 'qubes.tests.storage', 'qubes.tests.storage_xen', + 'qubes.tests.block', 'qubes.tests.hardware', 'qubes.tests.extra', ): diff --git a/tests/block.py b/tests/block.py new file mode 100644 index 00000000..927c9c14 --- /dev/null +++ b/tests/block.py @@ -0,0 +1,302 @@ +# vim: fileencoding=utf-8 +# +# The Qubes OS Project, https://www.qubes-os.org/ +# +# Copyright (C) 2016 +# Marek Marczykowski-Górecki +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +import os + +import qubes.tests +import qubes.qubesutils +import subprocess + +# the same class for both dom0 and VMs +class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): + template = None + + def setUp(self): + super(TC_00_List, self).setUp() + self.img_path = '/tmp/test.img' + self.mount_point = '/tmp/test-dir' + if self.template is not None: + self.vm = self.qc.add_new_vm( + "QubesAppVm", + name=self.make_vm_name("vm"), + template=self.qc.get_vm_by_name(self.template)) + self.vm.create_on_disk(verbose=False) + self.vm.start() + else: + self.vm = self.qc[0] + + def tearDown(self): + super(TC_00_List, self).tearDown() + if self.template is None: + if os.path.exists(self.mount_point): + subprocess.call(['sudo', 'umount', self.mount_point]) + subprocess.call(['sudo', 'rmdir', self.mount_point]) + subprocess.call(['sudo', 'dmsetup', 'remove', 'test-dm']) + if os.path.exists(self.img_path): + loopdev = subprocess.check_output(['losetup', '-j', + self.img_path]) + for dev in loopdev.splitlines(): + subprocess.call( + ['sudo', 'losetup', '-d', dev.split(':')[0]]) + subprocess.call(['sudo', 'rm', '-f', self.img_path]) + + def run_script(self, script, user="user"): + if self.template is None: + if user == "user": + subprocess.check_call(script, shell=True) + elif user == "root": + subprocess.check_call(['sudo', 'sh', '-c', script]) + else: + retcode = self.vm.run(script, user=user, wait=True) + if retcode != 0: + raise subprocess.CalledProcessError + + def test_000_list_loop(self): + if self.template is None: + self.skipTest('loop devices excluded in dom0') + self.run_script( + "set -e;" + "truncate -s 128M {path}; " + "losetup -f {path}; " + "udevadm settle".format(path=self.img_path), user="root") + + dev_list = qubes.qubesutils.block_list_vm(self.vm) + found = False + for dev in dev_list.keys(): + if dev_list[dev]['desc'] == self.img_path: + self.assertTrue(dev.startswith(self.vm.name + ':loop')) + self.assertEquals(dev_list[dev]['mode'], 'w') + self.assertEquals(dev_list[dev]['size'], 1024 * 1024 * 128) + self.assertEquals( + dev_list[dev]['device'], '/dev/' + dev.split(':')[1]) + found = True + + if not found: + self.fail("Device {} not found in {!r}".format(self.img_path, dev_list)) + + def test_001_list_loop_mounted(self): + if self.template is None: + self.skipTest('loop devices excluded in dom0') + self.run_script( + "set -e;" + "truncate -s 128M {path}; " + "mkfs.ext4 -q -F {path}; " + "mkdir -p {mntdir}; " + "mount {path} {mntdir} -o loop; " + "udevadm settle".format( + path=self.img_path, + mntdir=self.mount_point), + user="root") + + dev_list = qubes.qubesutils.block_list_vm(self.vm) + for dev in dev_list.keys(): + if dev_list[dev]['desc'] == self.img_path: + self.fail( + 'Device {} ({}) should not be listed because is mounted' + .format(dev, self.img_path)) + + def test_010_list_dm(self): + self.run_script( + "set -e;" + "truncate -s 128M {path}; " + "loopdev=`losetup -f`; " + "losetup $loopdev {path}; " + "dmsetup create test-dm --table \"0 262144 linear $(cat " + "/sys/block/$(basename $loopdev)/dev) 0\";" + "udevadm settle".format(path=self.img_path), user="root") + + dev_list = qubes.qubesutils.block_list_vm(self.vm) + found = False + for dev in dev_list.keys(): + if dev.startswith(self.vm.name + ':loop'): + self.assertNotEquals(dev_list[dev]['desc'], self.img_path, + "Device {} ({}) should not be listed as it is used in " + "device-mapper".format(dev, self.img_path) + ) + elif dev_list[dev]['desc'] == 'test-dm': + self.assertEquals(dev_list[dev]['mode'], 'w') + self.assertEquals(dev_list[dev]['size'], 1024 * 1024 * 128) + self.assertEquals( + dev_list[dev]['device'], '/dev/' + dev.split(':')[1]) + found = True + + if not found: + self.fail("Device {} not found in {!r}".format('test-dm', dev_list)) + + def test_011_list_dm_mounted(self): + self.run_script( + "set -e;" + "truncate -s 128M {path}; " + "loopdev=`losetup -f`; " + "losetup $loopdev {path}; " + "dmsetup create test-dm --table \"0 262144 linear $(cat " + "/sys/block/$(basename $loopdev)/dev) 0\";" + "mkfs.ext4 -q -F /dev/mapper/test-dm;" + "mkdir -p {mntdir};" + "mount /dev/mapper/test-dm {mntdir};" + "udevadm settle".format( + path=self.img_path, + mntdir=self.mount_point), + user="root") + + dev_list = qubes.qubesutils.block_list_vm(self.vm) + for dev in dev_list.keys(): + if dev.startswith(self.vm.name + ':loop'): + self.assertNotEquals(dev_list[dev]['desc'], self.img_path, + "Device {} ({}) should not be listed as it is used in " + "device-mapper".format(dev, self.img_path) + ) + else: + self.assertNotEquals(dev_list[dev]['desc'], 'test-dm', + "Device {} ({}) should not be listed as it is " + "mounted".format(dev, 'test-dm') + ) + + def test_012_list_dm_delayed(self): + self.run_script( + "set -e;" + "truncate -s 128M {path}; " + "loopdev=`losetup -f`; " + "losetup $loopdev {path}; " + "udevadm settle; " + "dmsetup create test-dm --table \"0 262144 linear $(cat " + "/sys/block/$(basename $loopdev)/dev) 0\";" + "udevadm settle".format(path=self.img_path), user="root") + + dev_list = qubes.qubesutils.block_list_vm(self.vm) + found = False + for dev in dev_list.keys(): + if dev.startswith(self.vm.name + ':loop'): + self.assertNotEquals(dev_list[dev]['desc'], self.img_path, + "Device {} ({}) should not be listed as it is used in " + "device-mapper".format(dev, self.img_path) + ) + elif dev_list[dev]['desc'] == 'test-dm': + self.assertEquals(dev_list[dev]['mode'], 'w') + self.assertEquals(dev_list[dev]['size'], 1024 * 1024 * 128) + self.assertEquals( + dev_list[dev]['device'], '/dev/' + dev.split(':')[1]) + found = True + + if not found: + self.fail("Device {} not found in {!r}".format('test-dm', dev_list)) + + def test_013_list_dm_removed(self): + self.run_script( + "set -e;" + "truncate -s 128M {path}; " + "loopdev=`losetup -f`; " + "losetup $loopdev {path}; " + "dmsetup create test-dm --table \"0 262144 linear $(cat " + "/sys/block/$(basename $loopdev)/dev) 0\";" + "udevadm settle;" + "dmsetup remove test-dm;" + "udevadm settle".format(path=self.img_path), user="root") + + dev_list = qubes.qubesutils.block_list_vm(self.vm) + found = False + for dev in dev_list.keys(): + if dev_list[dev]['desc'] == self.img_path: + self.assertTrue(dev.startswith(self.vm.name + ':loop')) + self.assertEquals(dev_list[dev]['mode'], 'w') + self.assertEquals(dev_list[dev]['size'], 1024 * 1024 * 128) + self.assertEquals( + dev_list[dev]['device'], '/dev/' + dev.split(':')[1]) + found = True + + if not found: + self.fail("Device {} not found in {!r}".format(self.img_path, dev_list)) + + def test_020_list_loop_partition(self): + if self.template is None: + self.skipTest('loop devices excluded in dom0') + self.run_script( + "set -e;" + "truncate -s 128M {path}; " + "echo ,,L | sfdisk {path};" + "loopdev=`losetup -f`; " + "losetup -P $loopdev {path}; " + "udevadm settle".format(path=self.img_path), user="root") + + dev_list = qubes.qubesutils.block_list_vm(self.vm) + found = False + for dev in dev_list.keys(): + if dev_list[dev]['desc'] == self.img_path: + self.assertTrue(dev.startswith(self.vm.name + ':loop')) + self.assertEquals(dev_list[dev]['mode'], 'w') + self.assertEquals(dev_list[dev]['size'], 1024 * 1024 * 128) + self.assertEquals( + dev_list[dev]['device'], '/dev/' + dev.split(':')[1]) + self.assertIn(dev + 'p1', dev_list) + found = True + + if not found: + self.fail("Device {} not found in {!r}".format(self.img_path, dev_list)) + + def test_021_list_loop_partition_mounted(self): + if self.template is None: + self.skipTest('loop devices excluded in dom0') + self.run_script( + "set -e;" + "truncate -s 128M {path}; " + "echo ,,L | sfdisk {path};" + "loopdev=`losetup -f`; " + "losetup -P $loopdev {path}; " + "mkfs.ext4 -q -F ${{loopdev}}p1; " + "mkdir -p {mntdir}; " + "mount ${{loopdev}}p1 {mntdir}; " + "udevadm settle".format( + path=self.img_path, mntdir=self.mount_point), + user="root") + + dev_list = qubes.qubesutils.block_list_vm(self.vm) + for dev in dev_list.keys(): + if dev_list[dev]['desc'] == self.img_path: + self.fail( + 'Device {} ({}) should not be listed because its ' + 'partition is mounted' + .format(dev, self.img_path)) + elif dev.startswith(self.vm.name + ':loop') and dev.endswith('p1'): + # FIXME: risky assumption that only tests create partitioned + # loop devices + self.fail( + 'Device {} ({}) should not be listed because is mounted' + .format(dev, self.img_path)) + + +def load_tests(loader, tests, pattern): + try: + qc = qubes.qubes.QubesVmCollection() + qc.lock_db_for_reading() + qc.load() + qc.unlock_db() + templates = [vm.name for vm in qc.values() if + isinstance(vm, qubes.qubes.QubesTemplateVm)] + except OSError: + templates = [] + for template in templates: + tests.addTests(loader.loadTestsFromTestCase( + type( + 'TC_00_List_' + template, + (TC_00_List, qubes.tests.QubesTestCase), + {'template': template}))) + + return tests From ea7631208cacade735f1f11a6ff028920deb4e10 Mon Sep 17 00:00:00 2001 From: Jeepler Date: Mon, 16 May 2016 23:36:27 -0500 Subject: [PATCH 16/38] qubes-bug-report subprocess removed, refactored and fixed pacman command for archlinux packages --- .../qubes-bug-report | 92 +++++++++++-------- 1 file changed, 53 insertions(+), 39 deletions(-) rename qubes-bug-report => qvm-tools/qubes-bug-report (59%) mode change 100644 => 100755 diff --git a/qubes-bug-report b/qvm-tools/qubes-bug-report old mode 100644 new mode 100755 similarity index 59% rename from qubes-bug-report rename to qvm-tools/qubes-bug-report index f4127d4a..3bbab112 --- a/qubes-bug-report +++ b/qvm-tools/qubes-bug-report @@ -1,23 +1,23 @@ #!/usr/bin/env python3 -import subprocess as sub -import shlex +import subprocess import argparse import time import sys +import os from os.path import expanduser #the term qube refers to a qubes vm -def is_program_installed_in_qube( qube_name, command ): +def is_program_installed_in_qube( program, qube_name ): is_installed = True try: - shell_cmd = "qvm-run " + shlex.quote( qube_name ) + " --pass-io --no-color-output 'command -v " + command + "' &> /dev/null" - sub.check_call( shell_cmd, shell = True ) + command = 'command -v ' + program + subprocess.check_call([ 'qvm-run', qube_name, '--pass-io', '--no-color-output', command ], stdout = open( os.devnull, 'w' ) ) - except sub.CalledProcessError: + except subprocess.CalledProcessError: is_installed = False return is_installed @@ -28,7 +28,7 @@ def is_program_installed_in_qube( qube_name, command ): def is_qube_running( qube_name ): runs = False - out = sub.check_output([ "virsh", "-c", "xen:///", "domstate", shlex.quote( qube_name ) ]) + out = subprocess.check_output([ "virsh", "-c", "xen:///", "domstate", qube_name ]) out = out.decode('utf-8').replace('\n', '') if 'running' == out: @@ -41,31 +41,31 @@ def get_qube_packages( qube_name ): content = "## Qubes Packages\n\n" #a qube can have more than one package manager installed (only one is functional) - pkg_cmd = { 'dpkg' : 'dpkg -l qubes-*', 'pacman' : 'pacman -Qs', 'rpm' : 'rpm -qa qubes-*' } + pkg_cmd = { 'dpkg' : 'dpkg -l qubes-*', 'pacman' : 'pacman -Qs qubes', 'rpm' : 'rpm -qa qubes-*' } if is_qube_running( qube_name ): for package_manager in pkg_cmd.keys(): - if is_program_installed_in_qube( qube_name, package_manager ): + if is_program_installed_in_qube( package_manager, qube_name ): pkg_list_cmd = pkg_cmd[package_manager] try: - shell_cmd = "qvm-run " + shlex.quote( qube_name ) + " --pass-io --no-color-output '" + pkg_list_cmd + "' 2> /dev/null" - out = sub.check_output( shell_cmd, shell = True ) + out = subprocess.check_output([ 'qvm-run', qube_name, '--pass-io', '--no-color-output', pkg_list_cmd ], stderr = open( os.devnull, 'w' ) ) out = out.decode('utf-8') - content = content + "### Package Manager: " + package_manager + "\n\n" + content = content + create_heading( ( "Package Manager: " + package_manager ), 3 ) content = content + wrap_code( out ) - except sub.CalledProcessError: - True #do nothing + except subprocess.CalledProcessError: + pass #do nothing else: content = content + "**No packages listed, because Qube " + qube_name + " was not running**\n\n" - return content + return content + def get_dom0_packages(): - content = "## Dom0 Packages\n\n" - out = sub.check_output([ "rpm", "-qa", "qubes-*" ]) + content = create_heading( "Dom0 Packages", 2 ) + out = subprocess.check_output([ "rpm", "-qa", "qubes-*" ]) out = out.decode('utf-8') content = content + wrap_code( out ) @@ -77,6 +77,19 @@ def wrap_code( text ): return code +def create_heading( heading, level ): + heading = heading + "\n\n" + + if 1 == level: + heading = "# " + heading + elif 2 == level: + heading = "## " + heading + else: + heading = "### " + heading + + return heading + + def get_log_file_content( qube_name ): content = "## Log Files\n\n" @@ -87,37 +100,36 @@ def get_log_file_content( qube_name ): for prefix in log_prefix: log_file = prefix + "." + qube_name + ext - content = content + "### Log File: " + log_file + "\n\n" + content = content + create_heading( ( "Log File: " + log_file ), 3 ) content = content + wrap_code( get_log_file( qubes_os_log + log_file ) ) return content def get_qube_prefs( qube_name ): - qube_prefs = sub.check_output([ "qvm-prefs", shlex.quote( qube_name ) ]) + qube_prefs = subprocess.check_output([ "qvm-prefs", qube_name ]) qube_prefs = qube_prefs.decode('utf-8') - content = "### Qube Prefs\n\n" + content = create_heading( "Qube Prefs", 2 ) content = content + wrap_code( qube_prefs ) return content def report( qube_name ): - template = '''# {title} - + template = '''{title} {content} ''' - title = "Bug report: " + qube_name + title_text = create_heading( "Bug report: " + qube_name, 1 ) - content = get_qube_prefs( qube_name ) - content = content + get_dom0_packages() - content = content + get_log_file_content( qube_name ) - content = content + get_qube_packages( qube_name ) + content_text = get_qube_prefs( qube_name ) + content_text = content_text + get_dom0_packages() + content_text = content_text + get_log_file_content( qube_name ) + content_text = content_text + get_qube_packages( qube_name ) - report = template.format( **locals() ) + report = template.format( title=title_text, content=content_text ) return report @@ -131,13 +143,13 @@ def send_report( dest_qube, file_path): #if dest_qube is not running -> start dest_qube if not is_qube_running( dest_qube ): try: - sub.check_call([ "qvm-start", shlex.quote( dest_qube ) ]) - except sub.CalledProcessError: + subprocess.check_call([ "qvm-start", dest_qube ]) + except subprocess.CalledProcessError: print( "Error while starting: " + dest_qube, file = sys.stderr ) try: - sub.check_call([ "qvm-move-to-vm", shlex.quote( dest_qube ), file_path ]) - except sub.calledProcessError: + subprocess.check_call([ "qvm-move-to-vm", dest_qube, file_path ]) + except subprocess.calledProcessError: print( "Moving file bug-report failed", file = sys.stderr ) @@ -156,38 +168,41 @@ def qube_exist( qube_name ): try: #calls: qvm-check --quiet vmanme - sub.check_call([ "qvm-check", "--quiet", shlex.quote( qube_name ) ]) + subprocess.check_call([ "qvm-check", "--quiet", qube_name ]) - except sub.CalledProcessError: + except subprocess.CalledProcessError: exists = False return exists def get_report_file_path( qube_name ): - #exapanduser sub.CalledProcessError:-> works corss platform + #exapanduser -> works corss platform home_dir = expanduser("~") date = time.strftime("%H%M%S") file_path = home_dir + "/" + qube_name + "_bug-report_" + date + ".md" return file_path - + def main(): parser = argparse.ArgumentParser( description = 'Generates a bug report for a specific qube (Qubes VM)' ) parser.add_argument( 'vmname', metavar = '', type = str ) - parser.add_argument( '-d', '--dest-vm', dest = "destvm", type = str, default = 'dom0', help = "destination vm" ) + parser.add_argument( '-d', '--dest-vm', metavar = '', dest = "destvm", type = str, default = 'dom0', help = "send the report to the destination VM" ) parser.add_argument( '-p', '--print-report', action = 'store_const', const = "print_report", required = False, help = "prints the report without writing it or sending it to a destination VM" ) args = parser.parse_args() if qube_exist( args.vmname ): if qube_exist( args.destvm ): + #get the report report_content = report( args.vmname ) + #if -p or --print-report is an argument print the report if args.print_report: print( report_content ) + #write and send the report else: file_path = get_report_file_path( args.vmname ) write_report( report_content, file_path ) @@ -206,8 +221,7 @@ def main(): else: print( "VM does not exist" ) exit(1) - - print( args ) +#calls the main function -> program start point main() From 14efbb4a228567318d30291b97d091d1b9fb5e03 Mon Sep 17 00:00:00 2001 From: Jeepler Date: Tue, 17 May 2016 13:15:26 -0500 Subject: [PATCH 17/38] qubes-bug-report coding style 4 spaces instead of tabs and using += operator --- qvm-tools/qubes-bug-report | 265 +++++++++++++++++++------------------ 1 file changed, 133 insertions(+), 132 deletions(-) diff --git a/qvm-tools/qubes-bug-report b/qvm-tools/qubes-bug-report index 3bbab112..5dcb12ce 100755 --- a/qvm-tools/qubes-bug-report +++ b/qvm-tools/qubes-bug-report @@ -11,216 +11,217 @@ from os.path import expanduser #the term qube refers to a qubes vm def is_program_installed_in_qube( program, qube_name ): - is_installed = True + is_installed = True - try: - command = 'command -v ' + program - subprocess.check_call([ 'qvm-run', qube_name, '--pass-io', '--no-color-output', command ], stdout = open( os.devnull, 'w' ) ) + try: + command = 'command -v ' + program + subprocess.check_call([ 'qvm-run', qube_name, '--pass-io', '--no-color-output', command ], stdout = open( os.devnull, 'w' ) ) - except subprocess.CalledProcessError: - is_installed = False + except subprocess.CalledProcessError: + is_installed = False - return is_installed + return is_installed #this function requires virsh #domstate only works for Xen domU (guests) def is_qube_running( qube_name ): - runs = False + runs = False - out = subprocess.check_output([ "virsh", "-c", "xen:///", "domstate", qube_name ]) - out = out.decode('utf-8').replace('\n', '') + out = subprocess.check_output([ "virsh", "-c", "xen:///", "domstate", qube_name ]) + out = out.decode('utf-8').replace('\n', '') - if 'running' == out: - runs = True + if 'running' == out: + runs = True - return runs + return runs def get_qube_packages( qube_name ): - content = "## Qubes Packages\n\n" + content = "## Qubes Packages\n\n" - #a qube can have more than one package manager installed (only one is functional) - pkg_cmd = { 'dpkg' : 'dpkg -l qubes-*', 'pacman' : 'pacman -Qs qubes', 'rpm' : 'rpm -qa qubes-*' } + #a qube can have more than one package manager installed (only one is functional) + pkg_cmd = { 'dpkg' : 'dpkg -l qubes-*', 'pacman' : 'pacman -Qs qubes', 'rpm' : 'rpm -qa qubes-*' } - if is_qube_running( qube_name ): + if is_qube_running( qube_name ): - for package_manager in pkg_cmd.keys(): - if is_program_installed_in_qube( package_manager, qube_name ): - pkg_list_cmd = pkg_cmd[package_manager] + for package_manager in pkg_cmd.keys(): + if is_program_installed_in_qube( package_manager, qube_name ): + pkg_list_cmd = pkg_cmd[package_manager] - try: - out = subprocess.check_output([ 'qvm-run', qube_name, '--pass-io', '--no-color-output', pkg_list_cmd ], stderr = open( os.devnull, 'w' ) ) - out = out.decode('utf-8') - content = content + create_heading( ( "Package Manager: " + package_manager ), 3 ) - content = content + wrap_code( out ) - except subprocess.CalledProcessError: - pass #do nothing + try: + out = subprocess.check_output([ 'qvm-run', qube_name, '--pass-io', '--no-color-output', pkg_list_cmd ], stderr = open( os.devnull, 'w' ) ) + out = out.decode('utf-8') + content += create_heading( ( "Package Manager: " + package_manager ), 3 ) + content += wrap_code( out ) + except subprocess.CalledProcessError: + pass #do nothing - else: - content = content + "**No packages listed, because Qube " + qube_name + " was not running**\n\n" + else: + content += "**No packages listed, because Qube " + qube_name + " was not running**\n\n" - return content - + return content + def get_dom0_packages(): - content = create_heading( "Dom0 Packages", 2 ) - out = subprocess.check_output([ "rpm", "-qa", "qubes-*" ]) - out = out.decode('utf-8') - content = content + wrap_code( out ) - - return content + content = create_heading( "Dom0 Packages", 2 ) + out = subprocess.check_output([ "rpm", "-qa", "qubes-*" ]) + out = out.decode('utf-8') + content += wrap_code( out ) + + return content def wrap_code( text ): - code = "~~~\n" + text + "~~~\n\n" - - return code + code = "~~~\n" + text + "~~~\n\n" + + return code def create_heading( heading, level ): - heading = heading + "\n\n" - - if 1 == level: - heading = "# " + heading - elif 2 == level: - heading = "## " + heading - else: - heading = "### " + heading + heading = heading + "\n\n" + + if 1 == level: + heading = "# " + heading + elif 2 == level: + heading = "## " + heading + else: + heading = "### " + heading - return heading - + return heading + def get_log_file_content( qube_name ): - content = "## Log Files\n\n" - qubes_os_log = "/var/log/qubes/" - ext = ".log" + content = "## Log Files\n\n" + qubes_os_log = "/var/log/qubes/" + ext = ".log" - log_prefix = [ "guid", "pacat", "qubesdb", "qrexec" ] + log_prefix = [ "guid", "pacat", "qubesdb", "qrexec" ] - for prefix in log_prefix: - log_file = prefix + "." + qube_name + ext - content = content + create_heading( ( "Log File: " + log_file ), 3 ) - content = content + wrap_code( get_log_file( qubes_os_log + log_file ) ) + #constructs for each log file prefix the full path and reads the log file + for prefix in log_prefix: + log_file = prefix + "." + qube_name + ext + content += create_heading( ( "Log File: " + log_file ), 3 ) + content += wrap_code( get_log_file( qubes_os_log + log_file ) ) - return content + return content def get_qube_prefs( qube_name ): - qube_prefs = subprocess.check_output([ "qvm-prefs", qube_name ]) - qube_prefs = qube_prefs.decode('utf-8') - - content = create_heading( "Qube Prefs", 2 ) - content = content + wrap_code( qube_prefs ) - - return content + qube_prefs = subprocess.check_output([ "qvm-prefs", qube_name ]) + qube_prefs = qube_prefs.decode('utf-8') + + content = create_heading( "Qube Prefs", 2 ) + content += wrap_code( qube_prefs ) + + return content def report( qube_name ): - template = '''{title} + template = '''{title} {content} ''' - title_text = create_heading( "Bug report: " + qube_name, 1 ) + title_text = create_heading( "Bug report: " + qube_name, 1 ) - content_text = get_qube_prefs( qube_name ) - content_text = content_text + get_dom0_packages() - content_text = content_text + get_log_file_content( qube_name ) - content_text = content_text + get_qube_packages( qube_name ) - + content_text = get_qube_prefs( qube_name ) + content_text += get_dom0_packages() + content_text += get_log_file_content( qube_name ) + content_text += get_qube_packages( qube_name ) + - report = template.format( title=title_text, content=content_text ) + report = template.format( title=title_text, content=content_text ) - return report + return report def write_report( report_content, file_path ): - with open( file_path, 'w' ) as report_file: - report_file.write( report_content ) + with open( file_path, 'w' ) as report_file: + report_file.write( report_content ) def send_report( dest_qube, file_path): - #if dest_qube is not running -> start dest_qube - if not is_qube_running( dest_qube ): - try: - subprocess.check_call([ "qvm-start", dest_qube ]) - except subprocess.CalledProcessError: - print( "Error while starting: " + dest_qube, file = sys.stderr ) + #if dest_qube is not running -> start dest_qube + if not is_qube_running( dest_qube ): + try: + subprocess.check_call([ "qvm-start", dest_qube ]) + except subprocess.CalledProcessError: + print( "Error while starting: " + dest_qube, file = sys.stderr ) - try: - subprocess.check_call([ "qvm-move-to-vm", dest_qube, file_path ]) - except subprocess.calledProcessError: - print( "Moving file bug-report failed", file = sys.stderr ) + try: + subprocess.check_call([ "qvm-move-to-vm", dest_qube, file_path ]) + except subprocess.calledProcessError: + print( "Moving file bug-report failed", file = sys.stderr ) def get_log_file( log_file ): - data = "" + data = "" - #open and close the file - with open( log_file ) as log: - data = log.read() + #open and close the file + with open( log_file ) as log: + data = log.read() - return data + return data def qube_exist( qube_name ): - exists = True - - try: - #calls: qvm-check --quiet vmanme - subprocess.check_call([ "qvm-check", "--quiet", qube_name ]) + exists = True + + try: + #calls: qvm-check --quiet vmanme + subprocess.check_call([ "qvm-check", "--quiet", qube_name ]) - except subprocess.CalledProcessError: - exists = False + except subprocess.CalledProcessError: + exists = False - return exists + return exists def get_report_file_path( qube_name ): - #exapanduser -> works corss platform - home_dir = expanduser("~") - date = time.strftime("%H%M%S") - file_path = home_dir + "/" + qube_name + "_bug-report_" + date + ".md" + #exapanduser -> works corss platform + home_dir = expanduser("~") + date = time.strftime("%H%M%S") + file_path = home_dir + "/" + qube_name + "_bug-report_" + date + ".md" - return file_path + return file_path def main(): - parser = argparse.ArgumentParser( description = 'Generates a bug report for a specific qube (Qubes VM)' ) - parser.add_argument( 'vmname', metavar = '', type = str ) - parser.add_argument( '-d', '--dest-vm', metavar = '', dest = "destvm", type = str, default = 'dom0', help = "send the report to the destination VM" ) - parser.add_argument( '-p', '--print-report', action = 'store_const', const = "print_report", required = False, help = "prints the report without writing it or sending it to a destination VM" ) - args = parser.parse_args() + parser = argparse.ArgumentParser( description = 'Generates a bug report for a specific qube (Qubes VM)' ) + parser.add_argument( 'vmname', metavar = '', type = str ) + parser.add_argument( '-d', '--dest-vm', metavar = '', dest = "destvm", type = str, default = 'dom0', help = "send the report to the destination VM" ) + parser.add_argument( '-p', '--print-report', action = 'store_const', const = "print_report", required = False, help = "prints the report without writing it or sending it to a destination VM" ) + args = parser.parse_args() - if qube_exist( args.vmname ): + if qube_exist( args.vmname ): - if qube_exist( args.destvm ): - #get the report - report_content = report( args.vmname ) + if qube_exist( args.destvm ): + #get the report + report_content = report( args.vmname ) - #if -p or --print-report is an argument print the report - if args.print_report: - print( report_content ) + #if -p or --print-report is an argument print the report + if args.print_report: + print( report_content ) - #write and send the report - else: - file_path = get_report_file_path( args.vmname ) - write_report( report_content, file_path ) - print( "Report written to: " + file_path ) + #write and send the report + else: + file_path = get_report_file_path( args.vmname ) + write_report( report_content, file_path ) + print( "Report written to: " + file_path ) - if 'dom0' != args.destvm: - send_report( args.destvm, file_path ) - print( "Report send to VM: " + args.destvm ) + if 'dom0' != args.destvm: + send_report( args.destvm, file_path ) + print( "Report send to VM: " + args.destvm ) - exit(0) + exit(0) - else: - print ( "Destination VM does not exist" ) - exit(1) + else: + print ( "Destination VM does not exist" ) + exit(1) - else: - print( "VM does not exist" ) - exit(1) + else: + print( "VM does not exist" ) + exit(1) #calls the main function -> program start point From 94d52a13e79dc0274593768ad39ba68f19e5131c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Tue, 17 May 2016 20:22:13 +0200 Subject: [PATCH 18/38] core: adjust guid parameters when running on KDE5 On KDE5 native decoration plugin is used and requires special properties set (instead of `_QUBES_VMNAME` etc). Special care needs to be taken when detecting environment, because environment variables aren't good enough - this script may be running with cleared environment (through sudo, or from systemd). So check properties of X11 root window. QubesOS/qubes-issues#1784 --- core-modules/000QubesVm.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py index fec8a7b7..23d40921 100644 --- a/core-modules/000QubesVm.py +++ b/core-modules/000QubesVm.py @@ -1763,6 +1763,33 @@ class QubesVm(object): guid_cmd += ['-v', '-v'] elif not verbose: guid_cmd += ['-q'] + # Avoid using environment variables for checking the current session, + # because this script may be called with cleared env (like with sudo). + if subprocess.check_output( + ['xprop', '-root', '-notype', 'KDE_SESSION_VERSION']) == \ + 'KDE_SESSION_VERSION = 5\n': + # native decoration plugins is used, so adjust window properties + # accordingly + guid_cmd += ['-T'] # prefix window titles with VM name + # get owner of X11 session + session_owner = None + for line in subprocess.check_output(['xhost']).splitlines(): + if line == 'SI:localuser:root': + pass + elif line.startswith('SI:localuser:'): + session_owner = line.split(":")[2] + if session_owner is not None: + data_dir = os.path.expanduser( + '~{}/.local/share'.format(session_owner)) + else: + # fallback to current user + data_dir = os.path.expanduser('~/.local/share') + + guid_cmd += ['-p', + '_KDE_NET_WM_COLOR_SCHEME=s:{}'.format( + os.path.join(data_dir, + 'qubes-kde', self.label.name + '.colors'))] + retcode = subprocess.call (guid_cmd) if (retcode != 0) : raise QubesException("Cannot start qubes-guid!") From 692254fcbff51de0a708d470ccb4ce6452fd84d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Tue, 17 May 2016 20:31:24 +0200 Subject: [PATCH 19/38] qubes-bug-report: remove trailing spaces QubesOS/qubes-issues#901 --- qvm-tools/qubes-bug-report | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/qvm-tools/qubes-bug-report b/qvm-tools/qubes-bug-report index 5dcb12ce..9bb23fcc 100755 --- a/qvm-tools/qubes-bug-report +++ b/qvm-tools/qubes-bug-report @@ -23,7 +23,7 @@ def is_program_installed_in_qube( program, qube_name ): return is_installed -#this function requires virsh +#this function requires virsh #domstate only works for Xen domU (guests) def is_qube_running( qube_name ): runs = False @@ -58,37 +58,37 @@ def get_qube_packages( qube_name ): pass #do nothing else: - content += "**No packages listed, because Qube " + qube_name + " was not running**\n\n" + content += "**No packages listed, because Qube " + qube_name + " was not running**\n\n" + + return content - return content - def get_dom0_packages(): content = create_heading( "Dom0 Packages", 2 ) out = subprocess.check_output([ "rpm", "-qa", "qubes-*" ]) out = out.decode('utf-8') content += wrap_code( out ) - + return content def wrap_code( text ): code = "~~~\n" + text + "~~~\n\n" - + return code def create_heading( heading, level ): heading = heading + "\n\n" - + if 1 == level: heading = "# " + heading - elif 2 == level: + elif 2 == level: heading = "## " + heading else: heading = "### " + heading return heading - + def get_log_file_content( qube_name ): @@ -110,10 +110,10 @@ def get_log_file_content( qube_name ): def get_qube_prefs( qube_name ): qube_prefs = subprocess.check_output([ "qvm-prefs", qube_name ]) qube_prefs = qube_prefs.decode('utf-8') - + content = create_heading( "Qube Prefs", 2 ) content += wrap_code( qube_prefs ) - + return content @@ -128,7 +128,7 @@ def report( qube_name ): content_text += get_dom0_packages() content_text += get_log_file_content( qube_name ) content_text += get_qube_packages( qube_name ) - + report = template.format( title=title_text, content=content_text ) @@ -148,10 +148,10 @@ def send_report( dest_qube, file_path): except subprocess.CalledProcessError: print( "Error while starting: " + dest_qube, file = sys.stderr ) - try: + try: subprocess.check_call([ "qvm-move-to-vm", dest_qube, file_path ]) except subprocess.calledProcessError: - print( "Moving file bug-report failed", file = sys.stderr ) + print( "Moving file bug-report failed", file = sys.stderr ) def get_log_file( log_file ): @@ -161,12 +161,12 @@ def get_log_file( log_file ): with open( log_file ) as log: data = log.read() - return data + return data def qube_exist( qube_name ): exists = True - + try: #calls: qvm-check --quiet vmanme subprocess.check_call([ "qvm-check", "--quiet", qube_name ]) @@ -174,7 +174,7 @@ def qube_exist( qube_name ): except subprocess.CalledProcessError: exists = False - return exists + return exists def get_report_file_path( qube_name ): @@ -218,11 +218,11 @@ def main(): else: print ( "Destination VM does not exist" ) exit(1) - + else: print( "VM does not exist" ) exit(1) - + #calls the main function -> program start point main() From 3abf2b24b4fbc8c8b8977fb464219c51063a9555 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Tue, 17 May 2016 22:56:18 +0200 Subject: [PATCH 20/38] tests: check opening URL While at it, fix policy preparation for qvm-open-in-vm tests. QubesOS/qubes-issues#1487 --- tests/vm_qrexec_gui.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py index db514249..5994acbb 100644 --- a/tests/vm_qrexec_gui.py +++ b/tests/vm_qrexec_gui.py @@ -1441,7 +1441,9 @@ class TC_50_MimeHandlers(qubes.tests.SystemTestsMixin): passio_popen=True) vmpattern = "disp*" else: - self.qrexec_policy('qubes.Filecopy', self.source_vm.name, + self.qrexec_policy('qubes.OpenInVM', self.source_vm.name, + self.target_vmname) + self.qrexec_policy('qubes.OpenURL', self.source_vm.name, self.target_vmname) p = self.source_vm.run("qvm-open-in-vm {} {}".format( self.target_vmname, filename), passio_popen=True) @@ -1573,6 +1575,10 @@ class TC_50_MimeHandlers(qubes.tests.SystemTestsMixin): self.open_file_and_check_viewer(filename, [], ["shotwell", "eog", "display"]) + def test_010_url(self): + self.open_file_and_check_viewer("https://www.qubes-os.org/", [], + ["Firefox", "Iceweasel"]) + def test_100_txt_dispvm(self): filename = "/home/user/test_file.txt" self.prepare_txt(filename) @@ -1622,6 +1628,11 @@ class TC_50_MimeHandlers(qubes.tests.SystemTestsMixin): ["shotwell", "eog", "display"], dispvm=True) + def test_110_url_dispvm(self): + self.open_file_and_check_viewer("https://www.qubes-os.org/", [], + ["Firefox", "Iceweasel"], + dispvm=True) + def load_tests(loader, tests, pattern): try: From 405fd40aaa75d0026857ba1d99494256fc299ebe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Tue, 17 May 2016 22:59:39 +0200 Subject: [PATCH 21/38] Add policy for qubes.OpenURL service For now the same as for qubes.OpenInVM. Fixes QubesOS/qubes-issues#1487 --- Makefile | 1 + qubes-rpc-policy/qubes.OpenURL.policy | 10 ++++++++++ rpm_spec/core-dom0.spec | 1 + 3 files changed, 12 insertions(+) create mode 100644 qubes-rpc-policy/qubes.OpenURL.policy diff --git a/Makefile b/Makefile index 189989b5..545e8089 100644 --- a/Makefile +++ b/Makefile @@ -72,6 +72,7 @@ endif mkdir -p $(DESTDIR)/usr/libexec/qubes cp qubes-rpc-policy/qubes.Filecopy.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.Filecopy cp qubes-rpc-policy/qubes.OpenInVM.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.OpenInVM + cp qubes-rpc-policy/qubes.OpenURL.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.OpenURL cp qubes-rpc-policy/qubes.VMShell.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.VMShell cp qubes-rpc-policy/qubes.NotifyUpdates.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.NotifyUpdates cp qubes-rpc-policy/qubes.NotifyTools.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.NotifyTools diff --git a/qubes-rpc-policy/qubes.OpenURL.policy b/qubes-rpc-policy/qubes.OpenURL.policy new file mode 100644 index 00000000..27303cc9 --- /dev/null +++ b/qubes-rpc-policy/qubes.OpenURL.policy @@ -0,0 +1,10 @@ +## Note that policy parsing stops at the first match, +## so adding anything below "$anyvm $anyvm action" line will have no effect + +## Please use a single # to start your custom comments + +sys-whonix anon-whonix allow +whonix-gw anon-whonix allow +whonix-ws anon-whonix allow +$anyvm $dispvm allow +$anyvm $anyvm ask diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec index a9b5188b..86c531bf 100644 --- a/rpm_spec/core-dom0.spec +++ b/rpm_spec/core-dom0.spec @@ -249,6 +249,7 @@ fi %attr(0664,root,qubes) %config(noreplace) /etc/qubes-rpc/policy/qubes.Filecopy %attr(0664,root,qubes) %config(noreplace) /etc/qubes-rpc/policy/qubes.GetImageRGBA %attr(0664,root,qubes) %config(noreplace) /etc/qubes-rpc/policy/qubes.OpenInVM +%attr(0664,root,qubes) %config(noreplace) /etc/qubes-rpc/policy/qubes.OpenURL %attr(0664,root,qubes) %config(noreplace) /etc/qubes-rpc/policy/qubes.NotifyTools %attr(0664,root,qubes) %config(noreplace) /etc/qubes-rpc/policy/qubes.NotifyUpdates %attr(0664,root,qubes) %config(noreplace) /etc/qubes-rpc/policy/qubes.VMShell From 6311eec6fdac729c9f9c9dd620be13c25bd44a29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Tue, 17 May 2016 23:39:01 +0200 Subject: [PATCH 22/38] tests: force reloading partition table after setting partitioned loop dev Apparently "losetup -P" doesn't always properly read partition table. Force reload using blockdev --rereadpt. --- tests/block.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/block.py b/tests/block.py index 927c9c14..b6d9e86c 100644 --- a/tests/block.py +++ b/tests/block.py @@ -234,6 +234,7 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): "echo ,,L | sfdisk {path};" "loopdev=`losetup -f`; " "losetup -P $loopdev {path}; " + "blockdev --rereadpt $loopdev; " "udevadm settle".format(path=self.img_path), user="root") dev_list = qubes.qubesutils.block_list_vm(self.vm) @@ -260,6 +261,7 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase): "echo ,,L | sfdisk {path};" "loopdev=`losetup -f`; " "losetup -P $loopdev {path}; " + "blockdev --rereadpt $loopdev; " "mkfs.ext4 -q -F ${{loopdev}}p1; " "mkdir -p {mntdir}; " "mount ${{loopdev}}p1 {mntdir}; " From a8fcc58934443ba4e647c012437285970140c654 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 18 May 2016 03:00:41 +0200 Subject: [PATCH 23/38] version 3.2.2 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index e4604e3a..be94e6f5 100644 --- a/version +++ b/version @@ -1 +1 @@ -3.2.1 +3.2.2 From 7c0f5a4be682866670ee0124c5655e97aa3a2982 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 18 May 2016 14:21:26 +0200 Subject: [PATCH 24/38] qubes-hcl-report: filename sanitization for old bash Bash in dom0 (Fedora 20 based) doesn't properly handle "+(..)" operator. So remove it for now. Fixes QubesOS/qubes-issues#1994 --- qvm-tools/qubes-hcl-report | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qvm-tools/qubes-hcl-report b/qvm-tools/qubes-hcl-report index e26d9b86..031b16e9 100755 --- a/qvm-tools/qubes-hcl-report +++ b/qvm-tools/qubes-hcl-report @@ -103,7 +103,7 @@ XL_VTX=`cat $TEMP_DIR/xl-info |grep xen_caps | grep hvm` XL_VTD=`cat $TEMP_DIR/xl-info |grep virt_caps |grep hvm_directio` PCRS=`find /sys/devices/ -name pcrs` -FILENAME="Qubes-HCL-${BRAND//+([^[:alnum:]])/_}-${PRODUCT//+([^[:alnum:]])/_}-$DATE" +FILENAME="Qubes-HCL-${BRAND//[^[:alnum:]]/_}-${PRODUCT//[^[:alnum:]]/_}-$DATE" if [[ $XL_VTX ]] then From 3afc7b7d504c92298f2a363936c32cb5da386cea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 26 May 2016 01:34:53 +0200 Subject: [PATCH 25/38] core: start qrexec-daemon as normal user, even when VM is started by root qrexec-daemon will start new processes for called services, which include starting new DispVM, starting other required VMs (like backend GPG VM). Having those processes as root leads to many permissions problems, like the one linked below. So when VM is started by root, make sure that qrexec-daemon will be running as normal user (the first user in group 'qubes' - there should be only one). QubesOS/qubes-issues#1768 --- core-modules/000QubesVm.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py index 23d40921..45fd2df1 100644 --- a/core-modules/000QubesVm.py +++ b/core-modules/000QubesVm.py @@ -26,6 +26,7 @@ import datetime import base64 import hashlib import logging +import grp import lxml.etree import os import os.path @@ -37,6 +38,7 @@ import time import uuid import xml.parsers.expat import signal +import pwd from qubes import qmemman from qubes import qmemman_algo import libvirt @@ -1818,13 +1820,21 @@ class QubesVm(object): self.log.debug('start_qrexec_daemon()') if verbose: print >> sys.stderr, "--> Starting the qrexec daemon..." + qrexec = [] + if os.getuid() == 0: + # try to always have qrexec running as normal user, otherwise + # many qrexec services would need to deal with root/user + # permission problems + qubes_group = grp.getgrnam('qubes') + qrexec = ['sudo', '-u', qubes_group.gr_mem[0]] + + qrexec += ['env', 'QREXEC_STARTUP_TIMEOUT=' + str(self.qrexec_timeout), + system_path["qrexec_daemon_path"]] + qrexec_args = [str(self.xid), self.name, self.default_user] if not verbose: qrexec_args.insert(0, "-q") - qrexec_env = os.environ - qrexec_env['QREXEC_STARTUP_TIMEOUT'] = str(self.qrexec_timeout) - retcode = subprocess.call ([system_path["qrexec_daemon_path"]] + - qrexec_args, env=qrexec_env) + retcode = subprocess.call(qrexec + qrexec_args) if (retcode != 0) : raise OSError ("Cannot execute qrexec-daemon!") From d67636308fcf7f75db9f3f3b5473fdd5b36b4fa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 26 May 2016 01:38:08 +0200 Subject: [PATCH 26/38] qvm-usb: modify for USBIP-over-qrexec implementation QubesOS/qubes-issues#531 --- core/qubesutils.py | 389 ++++++++++++++++++--------------------------- qvm-tools/qvm-usb | 60 +++---- 2 files changed, 189 insertions(+), 260 deletions(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index 689138a3..417a609d 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -25,6 +25,7 @@ from __future__ import absolute_import import string +import errno from lxml import etree from lxml.etree import ElementTree, SubElement, Element @@ -463,260 +464,184 @@ def block_detach_all(vm): usb_ver_re = re.compile(r"^(1|2)$") usb_device_re = re.compile(r"^[0-9]+-[0-9]+(_[0-9]+)?$") usb_port_re = re.compile(r"^$|^[0-9]+-[0-9]+(\.[0-9]+)?$") +usb_desc_re = re.compile(r"^[ -~]{1,255}$") +# should match valid VM name +usb_connected_to_re = re.compile(r"^[a-zA-Z][a-zA-Z0-9_.-]*$") -def usb_setup(backend_vm_xid, vm_xid, devid, usb_ver): - """ - Attach frontend to the backend. - backend_vm_xid - id of the backend domain - vm_xid - id of the frontend domain - devid - id of the pvusb controller - """ - num_ports = 8 - trans = vmm.xs.transaction_start() - - be_path = "/local/domain/%d/backend/vusb/%d/%d" % (backend_vm_xid, vm_xid, devid) - fe_path = "/local/domain/%d/device/vusb/%d" % (vm_xid, devid) - - be_perm = [{'dom': backend_vm_xid}, {'dom': vm_xid, 'read': True} ] - fe_perm = [{'dom': vm_xid}, {'dom': backend_vm_xid, 'read': True} ] - - # Create directories and set permissions - vmm.xs.write(trans, be_path, "") - vmm.xs.set_permissions(trans, be_path, be_perm) - - vmm.xs.write(trans, fe_path, "") - vmm.xs.set_permissions(trans, fe_path, fe_perm) - - # Write backend information into the location that frontend looks for - vmm.xs.write(trans, "%s/backend-id" % fe_path, str(backend_vm_xid)) - vmm.xs.write(trans, "%s/backend" % fe_path, be_path) - - # Write frontend information into the location that backend looks for - vmm.xs.write(trans, "%s/frontend-id" % be_path, str(vm_xid)) - vmm.xs.write(trans, "%s/frontend" % be_path, fe_path) - - # Write USB Spec version field. - vmm.xs.write(trans, "%s/usb-ver" % be_path, usb_ver) - - # Write virtual root hub field. - vmm.xs.write(trans, "%s/num-ports" % be_path, str(num_ports)) - for port in range(1, num_ports+1): - # Set all port to disconnected state - vmm.xs.write(trans, "%s/port/%d" % (be_path, port), "") - - # Set state to XenbusStateInitialising - vmm.xs.write(trans, "%s/state" % fe_path, "1") - vmm.xs.write(trans, "%s/state" % be_path, "1") - vmm.xs.write(trans, "%s/online" % be_path, "1") - - vmm.xs.transaction_end(trans) - -def usb_decode_device_from_xs(xs_encoded_device): +def usb_decode_device_from_qdb(qdb_encoded_device): """ recover actual device name (xenstore doesn't allow dot in key names, so it was translated to underscore) """ - return xs_encoded_device.replace('_', '.') + return qdb_encoded_device.replace('_', '.') -def usb_encode_device_for_xs(device): +def usb_encode_device_for_qdb(device): """ encode actual device name (xenstore doesn't allow dot in key names, so translated it into underscore) """ return device.replace('.', '_') -def usb_list(): +def usb_list_vm(vm): + if not vm.is_running(): + return {} + + try: + untrusted_devices = vm.qdb.multiread('/qubes-usb-devices/') + except Error: + vm.refresh() + return {} + + def get_dev_item(dev, item): + return untrusted_devices.get( + '/qubes-usb-devices/%s/%s' % (dev, item), + None) + + devices = {} + + untrusted_devices_names = list(set(map(lambda x: x.split("/")[2], + untrusted_devices.keys()))) + for untrusted_dev_name in untrusted_devices_names: + if usb_device_re.match(untrusted_dev_name): + dev_name = untrusted_dev_name + untrusted_device_desc = get_dev_item(dev_name, 'desc') + if not usb_desc_re.match(untrusted_device_desc): + print >> sys.stderr, "Invalid %s device desc in VM '%s'" % ( + dev_name, vm.name) + continue + device_desc = untrusted_device_desc + + untrusted_connected_to = get_dev_item(dev_name, 'connected-to') + if untrusted_connected_to: + if not usb_connected_to_re.match(untrusted_connected_to): + print >>sys.stderr, \ + "Invalid %s device 'connected-to' in VM '%s'" % ( + dev_name, vm.name) + continue + connected_to = untrusted_connected_to + else: + connected_to = None + + device = usb_decode_device_from_qdb(dev_name) + + full_name = vm.name + ':' + device + + devices[full_name] = { + 'vm': vm, + 'device': device, + 'qdb_path': '/qubes-usb-devices/' + dev_name, + 'name': full_name, + 'desc': device_desc, + 'connected-to': connected_to, + } + return devices + + +def usb_list(qvmc=None, vm=None): """ Returns a dictionary of USB devices (for PVUSB backends running in all VM). The dictionary is keyed by 'name' (see below), each element is a dictionary itself: - vm = name of the backend domain - xid = xid of the backend domain - device = - - name = :- + vm = backend domain object + device = device ID + name = : desc = description """ - # FIXME: any better idea of desc_re? - desc_re = re.compile(r"^.{1,255}$") + if vm is not None: + if not vm.is_running(): + return {} + else: + vm_list = [vm] + else: + if qvmc is None: + raise QubesException("You must pass either qvm or vm argument") + vm_list = qvmc.values() devices_list = {} - - xs_trans = vmm.xs.transaction_start() - vm_list = vmm.xs.ls(xs_trans, '/local/domain') - - for xid in vm_list: - vm_name = vmm.xs.read(xs_trans, '/local/domain/%s/name' % xid) - vm_devices = vmm.xs.ls(xs_trans, '/local/domain/%s/qubes-usb-devices' % xid) - if vm_devices is None: - continue - # when listing devices in xenstore we get encoded names - for xs_encoded_device in vm_devices: - # Sanitize device id - if not usb_device_re.match(xs_encoded_device): - print >> sys.stderr, "Invalid device id in backend VM '%s'" % vm_name - continue - device = usb_decode_device_from_xs(xs_encoded_device) - device_desc = vmm.xs.read(xs_trans, '/local/domain/%s/qubes-usb-devices/%s/desc' % (xid, xs_encoded_device)) - if not desc_re.match(device_desc): - print >> sys.stderr, "Invalid %s device desc in VM '%s'" % (device, vm_name) - continue - visible_name = "%s:%s" % (vm_name, device) - # grab version - usb_ver = vmm.xs.read(xs_trans, '/local/domain/%s/qubes-usb-devices/%s/usb-ver' % (xid, xs_encoded_device)) - if usb_ver is None or not usb_ver_re.match(usb_ver): - print >> sys.stderr, "Invalid %s device USB version in VM '%s'" % (device, vm_name) - continue - devices_list[visible_name] = {"name": visible_name, "xid":int(xid), - "vm": vm_name, "device":device, - "desc":device_desc, - "usb_ver":usb_ver} - - vmm.xs.transaction_end(xs_trans) + for vm in vm_list: + devices_list.update(usb_list_vm(vm)) return devices_list -def usb_check_attached(xs_trans, backend_vm, device): - """ - Checks if the given device in the given backend attached to any frontend. - Parameters: - backend_vm - xid of the backend domain - device - device name in the backend domain - Returns None or a dictionary: - vm - the name of the frontend domain - xid - xid of the frontend domain - frontend - frontend device number FIXME - devid - frontend port number FIXME - """ - # sample xs content: /local/domain/0/backend/vusb/4/0/port/1 = "7-5" - attached_dev = None - vms = vmm.xs.ls(xs_trans, '/local/domain/%d/backend/vusb' % backend_vm) - if vms is None: - return None - for vm in vms: - if not vm.isdigit(): - print >> sys.stderr, "Invalid VM id" - continue - frontend_devs = vmm.xs.ls(xs_trans, '/local/domain/%d/backend/vusb/%s' % (backend_vm, vm)) - if frontend_devs is None: - continue - for frontend_dev in frontend_devs: - if not frontend_dev.isdigit(): - print >> sys.stderr, "Invalid frontend in VM %s" % vm - continue - ports = vmm.xs.ls(xs_trans, '/local/domain/%d/backend/vusb/%s/%s/port' % (backend_vm, vm, frontend_dev)) - if ports is None: - continue - for port in ports: - # FIXME: refactor, see similar loop in usb_find_unused_frontend(), use usb_list() instead? - if not port.isdigit(): - print >> sys.stderr, "Invalid port in VM %s frontend %s" % (vm, frontend) - continue - dev = vmm.xs.read(xs_trans, '/local/domain/%d/backend/vusb/%s/%s/port/%s' % (backend_vm, vm, frontend_dev, port)) - if dev == "": - continue - # Sanitize device id - if not usb_port_re.match(dev): - print >> sys.stderr, "Invalid device id in backend VM %d @ %s/%s/port/%s" % \ - (backend_vm, vm, frontend_dev, port) - continue - if dev == device: - frontend = "%s-%s" % (frontend_dev, port) - #TODO - vm_name = xl_ctx.domid_to_name(int(vm)) - if vm_name is None: - # FIXME: should we wipe references to frontends running on nonexistent VMs? - continue - attached_dev = {"xid":int(vm), "frontend": frontend, "devid": device, "vm": vm_name} - break - return attached_dev - -#def usb_check_frontend_busy(vm, front_dev, port): -# devport = frontend.split("-") -# if len(devport) != 2: -# raise QubesException("Malformed frontend syntax, must be in device-port format") -# # FIXME: -# # return vmm.xs.read('', '/local/domain/%d/device/vusb/%d/state' % (vm.xid, frontend)) == '4' -# return False - -def usb_find_unused_frontend(xs_trans, backend_vm_xid, vm_xid, usb_ver): - """ - Find an unused frontend/port to link the given backend with the given frontend. - Creates new frontend if needed. - Returns frontend specification in - format. - """ - - # This variable holds an index of last frontend scanned by the loop below. - # If nothing found, this value will be used to derive the index of a new frontend. - last_frontend_dev = -1 - - frontend_devs = vmm.xs.ls(xs_trans, "/local/domain/%d/device/vusb" % vm_xid) - if frontend_devs is not None: - for frontend_dev in frontend_devs: - if not frontend_dev.isdigit(): - print >> sys.stderr, "Invalid frontend_dev in VM %d" % vm_xid - continue - frontend_dev = int(frontend_dev) - fe_path = "/local/domain/%d/device/vusb/%d" % (vm_xid, frontend_dev) - if vmm.xs.read(xs_trans, "%s/backend-id" % fe_path) == str(backend_vm_xid): - if vmm.xs.read(xs_trans, '/local/domain/%d/backend/vusb/%d/%d/usb-ver' % (backend_vm_xid, vm_xid, frontend_dev)) != usb_ver: - last_frontend_dev = frontend_dev - continue - # here: found an existing frontend already connected to right backend using an appropriate USB version - ports = vmm.xs.ls(xs_trans, '/local/domain/%d/backend/vusb/%d/%d/port' % (backend_vm_xid, vm_xid, frontend_dev)) - if ports is None: - print >> sys.stderr, "No ports in VM %d frontend_dev %d?" % (vm_xid, frontend_dev) - last_frontend_dev = frontend_dev - continue - for port in ports: - # FIXME: refactor, see similar loop in usb_check_attached(), use usb_list() instead? - if not port.isdigit(): - print >> sys.stderr, "Invalid port in VM %d frontend_dev %d" % (vm_xid, frontend_dev) - continue - port = int(port) - dev = vmm.xs.read(xs_trans, '/local/domain/%d/backend/vusb/%d/%s/port/%s' % (backend_vm_xid, vm_xid, frontend_dev, port)) - # Sanitize device id - if not usb_port_re.match(dev): - print >> sys.stderr, "Invalid device id in backend VM %d @ %d/%d/port/%d" % \ - (backend_vm_xid, vm_xid, frontend_dev, port) - continue - if dev == "": - return '%d-%d' % (frontend_dev, port) - last_frontend_dev = frontend_dev - - # create a new frontend_dev and link it to the backend - frontend_dev = last_frontend_dev + 1 - usb_setup(backend_vm_xid, vm_xid, frontend_dev, usb_ver) - return '%d-%d' % (frontend_dev, 1) - -def usb_attach(vm, backend_vm, device, frontend=None, auto_detach=False, wait=True): - device_attach_check(vm, backend_vm, device, frontend) - - xs_trans = vmm.xs.transaction_start() - - xs_encoded_device = usb_encode_device_for_xs(device) - usb_ver = vmm.xs.read(xs_trans, '/local/domain/%s/qubes-usb-devices/%s/usb-ver' % (backend_vm.xid, xs_encoded_device)) - if usb_ver is None or not usb_ver_re.match(usb_ver): - vmm.xs.transaction_end(xs_trans) - raise QubesException("Invalid %s device USB version in VM '%s'" % (device, backend_vm.name)) - - if frontend is None: - frontend = usb_find_unused_frontend(xs_trans, backend_vm.xid, vm.xid, usb_ver) +def usb_check_attached(device): + """Reread device attachment status""" + vm = device['vm'] + untrusted_connected_to = vm.qdb.read( + '{}/connected-to'.format(device['qdb_path'])) + if untrusted_connected_to: + if not usb_connected_to_re.match(untrusted_connected_to): + raise QubesException( + "Invalid %s device 'connected-to' in VM '%s'" % ( + device['device'], vm.name)) + connected_to = untrusted_connected_to else: - # Check if any device attached at this frontend - #if usb_check_frontend_busy(vm, frontend): - # raise QubesException("Frontend %s busy in VM %s, detach it first" % (frontend, vm.name)) - vmm.xs.transaction_end(xs_trans) - raise NotImplementedError("Explicit USB frontend specification is not implemented yet") + connected_to = None + return connected_to - # Check if this device is attached to some domain - attached_vm = usb_check_attached(xs_trans, backend_vm.xid, device) - vmm.xs.transaction_end(xs_trans) +def usb_attach(vm, device, auto_detach=False, wait=True): + if not vm.is_running(): + raise QubesException("VM {} not running".format(vm.name)) - if attached_vm: + if not device['vm'].is_running(): + raise QubesException("VM {} not running".format(device['vm'].name)) + + connected_to = usb_check_attached(device) + if connected_to: if auto_detach: - usb_detach(backend_vm, attached_vm) + usb_detach(device) else: - raise QubesException("Device %s from %s already connected to VM %s as %s" % (device, backend_vm.name, attached_vm['vm'], attached_vm['frontend'])) + raise QubesException("Device {} already connected, to {}".format( + device['name'], connected_to + )) - # Run helper script - xl_cmd = [ '/usr/lib/qubes/xl-qvm-usb-attach.py', str(vm.xid), device, frontend, str(backend_vm.xid) ] - subprocess.check_call(xl_cmd) + # set qrexec policy to allow this device + policy_line = '{} {} allow\n'.format(vm.name, device['vm'].name) + policy_path = '/etc/qubes-rpc/policy/qubes.USB+{}'.format(device['device']) + policy_exists = os.path.exists(policy_path) + if not policy_exists: + try: + fd = os.open(policy_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY) + with os.fdopen(fd, 'w') as f: + f.write(policy_line) + except OSError as e: + if e.errno == errno.EEXIST: + pass + else: + raise + else: + with open(policy_path, 'r+') as f: + policy = f.readlines() + policy.insert(0, policy_line) + f.truncate(0) + f.seek(0) + f.write(''.join(policy)) + try: + # and actual attach + p = vm.run_service('qubes.USBAttach', passio_popen=True, user='root') + (stdout, stderr) = p.communicate( + '{} {}\n'.format(device['vm'].name, device['device'])) + if p.returncode != 0: + # TODO: sanitize and include stdout + raise QubesException('Device attach failed') + finally: + # FIXME: there is a race condition here - some other process might + # modify the file in the meantime. This may result in unexpected + # denials, but will not allow too much + if not policy_exists: + os.unlink(policy_path) + else: + with open(policy_path, 'r+') as f: + policy = f.readlines() + policy.remove('{} {} allow\n'.format(vm.name, device['vm'].name)) + f.truncate(0) + f.seek(0) + f.write(''.join(policy)) -def usb_detach(backend_vm, attachment): - xl_cmd = [ '/usr/lib/qubes/xl-qvm-usb-detach.py', str(attachment['xid']), attachment['devid'], attachment['frontend'], str(backend_vm.xid) ] - subprocess.check_call(xl_cmd) +def usb_detach(vm, device): + connected_to = usb_check_attached(device) + # detect race conditions; there is still race here, but much smaller + if vm.name != connected_to: + raise QubesException( + "Device {} not connected to VM {}".format(device['name'], vm.name)) + + p = vm.run_service('qubes.USBDetach', passio_popen=True, user='root') + (stdout, stderr) = p.communicate( + '{} {}\n'.format(device['vm'].name, device['device'])) + if p.returncode != 0: + # TODO: sanitize and include stdout + raise QubesException('Device detach failed') def usb_detach_all(vm): raise NotImplementedError("Detaching all devices from a given VM is not implemented yet") diff --git a/qvm-tools/qvm-usb b/qvm-tools/qvm-usb index 31556379..3faa7b25 100755 --- a/qvm-tools/qvm-usb +++ b/qvm-tools/qvm-usb @@ -78,11 +78,10 @@ def main(): print >> sys.stderr, "Only one of -l -a -d is allowed!" exit (1) - if options.do_attach or options.do_detach: - qvm_collection = QubesVmCollection() - qvm_collection.lock_db_for_reading() - qvm_collection.load() - qvm_collection.unlock_db() + qvm_collection = QubesVmCollection() + qvm_collection.lock_db_for_reading() + qvm_collection.load() + qvm_collection.unlock_db() if options.do_attach: if (len (args) != 2): @@ -91,14 +90,17 @@ def main(): if vm is None: parser.error ("Invalid VM name: %s" % args[0]) - # FIXME: here we assume that device is always in form "domain:dev", which can be changed in the future + # FIXME: here we assume that device is always in form "domain:dev", + # which can be changed in the future if args[1].find(":") < 0: - parser.error ("Invalid device syntax: %s" % args[1]) - dev_list = usb_list() + parser.error("Invalid device syntax: %s" % args[1]) + backend_vm = qvm_collection.get_vm_by_name(args[1].split(":")[0]) + if backend_vm is None: + parser.error("No such VM: {}".format(args[1].split(":")[0])) + dev_list = usb_list(vm=backend_vm) if not args[1] in dev_list.keys(): - parser.error ("Invalid device name: %s" % args[1]) + parser.error("Invalid device name: %s" % args[1]) dev = dev_list[args[1]] - backend_vm = qvm_collection.get_vm_by_name(dev['vm']) assert backend_vm is not None kwargs = {} @@ -106,14 +108,14 @@ def main(): # kwargs['frontend'] = options.frontend kwargs['auto_detach'] = options.auto_detach try: - usb_attach(vm, backend_vm, dev['device'], **kwargs) + usb_attach(vm, dev, **kwargs) except QubesException as e: print >> sys.stderr, "ERROR: %s" % str(e) sys.exit(1) elif options.do_detach: if (len (args) < 1): parser.error ("You must provide device or vm name!") - if len(args) > 1: + if len(args) > 1: parser.error ("Too many parameters") # Check if provided name is VM vm = qvm_collection.get_vm_by_name(args[0]) @@ -127,32 +129,34 @@ def main(): else: # Maybe usbvm:device? - # FIXME: nasty copy-paste from attach code half a page above - # FIXME: here we assume that device is always in form "domain:dev", which can be changed in the future + # FIXME: nasty copy-paste from attach code half a page above + # FIXME: here we assume that device is always in form "domain:dev", + # which can be changed in the future if args[0].find(":") < 0: - parser.error ("Invalid device syntax: %s" % args[0]) - dev_list = usb_list() + parser.error("Invalid device syntax: %s" % args[0]) + backend_vm = qvm_collection.get_vm_by_name(args[0].split(":")[0]) + if backend_vm is None: + parser.error("No such VM: {}".format(args[0].split(":")[0])) + dev_list = usb_list(vm=backend_vm) if not args[0] in dev_list.keys(): - parser.error ("Invalid device name: %s" % args[0]) + parser.error("Invalid device name: %s" % args[0]) dev = dev_list[args[0]] - backend_vm = qvm_collection.get_vm_by_name(dev['vm']) - assert backend_vm is not None - - attached_to = usb_check_attached('', backend_vm.xid, dev['device']) + attached_to = usb_check_attached(dev) if attached_to is None: print >> sys.stderr, "WARNING: Device not connected to any VM" exit(0) - usb_detach(backend_vm, attached_to) + vm = qvm_collection.get_vm_by_name(attached_to) + usb_detach(vm, dev) else: - if len(args) > 0: - parser.error ("Too many parameters") + if len(args) > 0: + parser.error("Too many parameters") # do_list - for dev in usb_list().values(): - attached_to = usb_check_attached('', dev['xid'], dev['device']) + for dev in usb_list(qvmc=qvm_collection).values(): + attached_to = usb_check_attached(dev) attached_to_str = "" if attached_to: - attached_to_str = " (attached to %s:%s)" % (attached_to['vm'], attached_to['frontend']) - print "%s\t%s%s (USBv%s)" % (dev['name'], dev['desc'], attached_to_str, dev['usb_ver']) + attached_to_str = " (attached to %s)" % (attached_to) + print "%s\t%s%s" % (dev['name'], dev['desc'], attached_to_str) exit (0) main() From 52fb410deba1a33d5e4bc78f4ea44bdd6b1d9320 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 2 Jun 2016 02:44:38 +0200 Subject: [PATCH 27/38] qvm-usb: always pass VM as object reference not a name Make the API consistent. QubesOS/qubes-issues#531 --- core/qubesutils.py | 39 ++++++++++++++++++++++++--------------- qvm-tools/qvm-usb | 19 +++++++++---------- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index 417a609d..27455bca 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -476,7 +476,7 @@ def usb_encode_device_for_qdb(device): """ encode actual device name (xenstore doesn't allow dot in key names, so translated it into underscore) """ return device.replace('.', '_') -def usb_list_vm(vm): +def usb_list_vm(qvmc, vm): if not vm.is_running(): return {} @@ -512,7 +512,12 @@ def usb_list_vm(vm): "Invalid %s device 'connected-to' in VM '%s'" % ( dev_name, vm.name) continue - connected_to = untrusted_connected_to + connected_to = qvmc.get_vm_by_name(untrusted_connected_to) + if connected_to is None: + print >>sys.stderr, \ + "Device {} appears to be connected to {}, " \ + "but such VM doesn't exist".format( + dev_name, untrusted_connected_to) else: connected_to = None @@ -531,7 +536,7 @@ def usb_list_vm(vm): return devices -def usb_list(qvmc=None, vm=None): +def usb_list(qvmc, vm=None): """ Returns a dictionary of USB devices (for PVUSB backends running in all VM). The dictionary is keyed by 'name' (see below), each element is a dictionary itself: @@ -546,16 +551,14 @@ def usb_list(qvmc=None, vm=None): else: vm_list = [vm] else: - if qvmc is None: - raise QubesException("You must pass either qvm or vm argument") vm_list = qvmc.values() devices_list = {} for vm in vm_list: - devices_list.update(usb_list_vm(vm)) + devices_list.update(usb_list_vm(qvmc, vm)) return devices_list -def usb_check_attached(device): +def usb_check_attached(qvmc, device): """Reread device attachment status""" vm = device['vm'] untrusted_connected_to = vm.qdb.read( @@ -565,22 +568,27 @@ def usb_check_attached(device): raise QubesException( "Invalid %s device 'connected-to' in VM '%s'" % ( device['device'], vm.name)) - connected_to = untrusted_connected_to + connected_to = qvmc.get_vm_by_name(untrusted_connected_to) + if connected_to is None: + print >>sys.stderr, \ + "Device {} appears to be connected to {}, " \ + "but such VM doesn't exist".format( + device['device'], untrusted_connected_to) else: connected_to = None return connected_to -def usb_attach(vm, device, auto_detach=False, wait=True): +def usb_attach(qvmc, vm, device, auto_detach=False, wait=True): if not vm.is_running(): raise QubesException("VM {} not running".format(vm.name)) if not device['vm'].is_running(): raise QubesException("VM {} not running".format(device['vm'].name)) - connected_to = usb_check_attached(device) + connected_to = usb_check_attached(qvmc, device) if connected_to: if auto_detach: - usb_detach(device) + usb_detach(qvmc, device) else: raise QubesException("Device {} already connected, to {}".format( device['name'], connected_to @@ -629,12 +637,13 @@ def usb_attach(vm, device, auto_detach=False, wait=True): f.seek(0) f.write(''.join(policy)) -def usb_detach(vm, device): - connected_to = usb_check_attached(device) +def usb_detach(qvmc, vm, device): + connected_to = usb_check_attached(qvmc, device) # detect race conditions; there is still race here, but much smaller - if vm.name != connected_to: + if connected_to is None or connected_to.qid != vm.qid: raise QubesException( - "Device {} not connected to VM {}".format(device['name'], vm.name)) + "Device {} not connected to VM {}".format( + device['name'], vm.name)) p = vm.run_service('qubes.USBDetach', passio_popen=True, user='root') (stdout, stderr) = p.communicate( diff --git a/qvm-tools/qvm-usb b/qvm-tools/qvm-usb index 3faa7b25..32abf3bd 100755 --- a/qvm-tools/qvm-usb +++ b/qvm-tools/qvm-usb @@ -97,7 +97,7 @@ def main(): backend_vm = qvm_collection.get_vm_by_name(args[1].split(":")[0]) if backend_vm is None: parser.error("No such VM: {}".format(args[1].split(":")[0])) - dev_list = usb_list(vm=backend_vm) + dev_list = usb_list(qvm_collection, vm=backend_vm) if not args[1] in dev_list.keys(): parser.error("Invalid device name: %s" % args[1]) dev = dev_list[args[1]] @@ -108,7 +108,7 @@ def main(): # kwargs['frontend'] = options.frontend kwargs['auto_detach'] = options.auto_detach try: - usb_attach(vm, dev, **kwargs) + usb_attach(qvm_collection, vm, dev, **kwargs) except QubesException as e: print >> sys.stderr, "ERROR: %s" % str(e) sys.exit(1) @@ -125,7 +125,7 @@ def main(): # kwargs['frontend'] = options.frontend # usb_detach(vm, **kwargs) #else: - usb_detach_all(vm) + usb_detach_all(qvm_collection, vm) else: # Maybe usbvm:device? @@ -137,25 +137,24 @@ def main(): backend_vm = qvm_collection.get_vm_by_name(args[0].split(":")[0]) if backend_vm is None: parser.error("No such VM: {}".format(args[0].split(":")[0])) - dev_list = usb_list(vm=backend_vm) + dev_list = usb_list(qvm_collection, vm=backend_vm) if not args[0] in dev_list.keys(): parser.error("Invalid device name: %s" % args[0]) dev = dev_list[args[0]] - attached_to = usb_check_attached(dev) + attached_to = usb_check_attached(qvm_collection, dev) if attached_to is None: print >> sys.stderr, "WARNING: Device not connected to any VM" exit(0) - vm = qvm_collection.get_vm_by_name(attached_to) - usb_detach(vm, dev) + usb_detach(qvm_collection, attached_to, dev) else: if len(args) > 0: parser.error("Too many parameters") # do_list - for dev in usb_list(qvmc=qvm_collection).values(): - attached_to = usb_check_attached(dev) + for dev in usb_list(qvm_collection).values(): + attached_to = dev['connected-to'] attached_to_str = "" if attached_to: - attached_to_str = " (attached to %s)" % (attached_to) + attached_to_str = " (attached to %s)" % (attached_to.name) print "%s\t%s%s" % (dev['name'], dev['desc'], attached_to_str) exit (0) From d5e06bfb8387ed2cc06041d18ed8e5e3a55cc765 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 2 Jun 2016 02:45:26 +0200 Subject: [PATCH 28/38] qvm-usb: issue detach call to backend domain Make sure that even compromised frontend will be cut of (possibly sensitive - like a webcam) device. On the other hand, if backend domain is already compromised, it may already compromise frontend domain too, so none of them would be better to call detach to. QubesOS/qubes-issues#531 --- core/qubesutils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index 27455bca..494ca7d4 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -645,9 +645,10 @@ def usb_detach(qvmc, vm, device): "Device {} not connected to VM {}".format( device['name'], vm.name)) - p = vm.run_service('qubes.USBDetach', passio_popen=True, user='root') + p = device['vm'].run_service('qubes.USBDetach', passio_popen=True, + user='root') (stdout, stderr) = p.communicate( - '{} {}\n'.format(device['vm'].name, device['device'])) + '{}\n'.format(device['device'])) if p.returncode != 0: # TODO: sanitize and include stdout raise QubesException('Device detach failed') From 767d1f00747050f0139f207dc9e6ef17c1173f5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 2 Jun 2016 02:49:22 +0200 Subject: [PATCH 29/38] qvm-usb: implement usb_detach_all QubesOS/qubes-issues#531 --- core/qubesutils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index 494ca7d4..b05ddbc5 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -653,8 +653,11 @@ def usb_detach(qvmc, vm, device): # TODO: sanitize and include stdout raise QubesException('Device detach failed') -def usb_detach_all(vm): - raise NotImplementedError("Detaching all devices from a given VM is not implemented yet") +def usb_detach_all(qvmc, vm): + for dev in usb_list(qvmc).values(): + connected_to = dev['connected-to'] + if connected_to is not None and connected_to.qid == vm.qid: + usb_detach(qvmc, connected_to, dev) ####### QubesWatch ###### From e87da9ec9d67315b0852ad33c710662d90c80c66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 2 Jun 2016 02:49:42 +0200 Subject: [PATCH 30/38] tests: adjust dom0_update tests for dnf in VM There is no support for 'copy_local' repository option, so setup test repository over http. Related to QubesOS/qubes-issues#1574 --- tests/dom0_update.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/dom0_update.py b/tests/dom0_update.py index c9a8a5c2..e40365c8 100644 --- a/tests/dom0_update.py +++ b/tests/dom0_update.py @@ -39,8 +39,7 @@ class TC_00_Dom0UpgradeMixin(qubes.tests.SystemTestsMixin): Tests for downloading dom0 updates using VMs based on different templates """ pkg_name = 'qubes-test-pkg' - dom0_update_common_opts = ['--disablerepo=*', '--enablerepo=test', - '--setopt=test.copy_local=1'] + dom0_update_common_opts = ['--disablerepo=*', '--enablerepo=test'] update_flag_path = '/var/lib/qubes/updates/dom0-updates-available' @classmethod @@ -85,9 +84,9 @@ Expire-Date: 0 p.stdin.write(''' [test] name = Test -baseurl = file:///tmp/repo +baseurl = http://localhost:8080/ enabled = 1 - ''') +''') p.stdin.close() p.wait() @@ -115,6 +114,7 @@ enabled = 1 subprocess.check_call(['sudo', 'rpm', '--import', os.path.join(self.tmpdir, 'pubkey.asc')]) self.updatevm.start() + self.repo_running = False def tearDown(self): self.qc.lock_db_for_writing() @@ -186,6 +186,13 @@ Test package elif retcode != 0: self.skipTest("createrepo failed with code {}, cannot perform the " "test".format(retcode)) + self.start_repo() + + def start_repo(self): + if not self.repo_running: + self.updatevm.run("cd /tmp/repo &&" + "python -m SimpleHTTPServer 8080") + self.repo_running = True def test_000_update(self): """Dom0 update tests From a534b1dd2c1b783379fda5495b1e3fa9ae778ab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 2 Jun 2016 02:52:59 +0200 Subject: [PATCH 31/38] qvm-usb: remove scary warning about PV USB stability USBIP (used for PV USB here) is considered stable by Linux maintainers, so follow their judgement. Fixes QubesOS/qubes-issues#531 --- qvm-tools/qvm-usb | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/qvm-tools/qvm-usb b/qvm-tools/qvm-usb index 32abf3bd..f2b4407e 100755 --- a/qvm-tools/qvm-usb +++ b/qvm-tools/qvm-usb @@ -27,8 +27,6 @@ from optparse import OptionParser import sys import os -pvusb_enable_flagfile = '/var/lib/qubes/pvusb-enable.flag' - def main(): usage = "usage: %prog -l [options]\n"\ "usage: %prog -a [options] :\n"\ @@ -49,24 +47,6 @@ def main(): (options, args) = parser.parse_args () - if not os.path.exists(pvusb_enable_flagfile): - print >> sys.stderr, "" - print >> sys.stderr, "******* WARNING *** WARNING *** WARNING *** WARNING *******" - print >> sys.stderr, "*** ***" - print >> sys.stderr, "*** PVUSB passthrough kernel support is still unstable. ***" - print >> sys.stderr, "*** It can CRASH your VMs. ***" - print >> sys.stderr, "*** ***" - print >> sys.stderr, "***********************************************************" - print >> sys.stderr, "" - print >> sys.stderr, "To use it, you need install kernel from \"unstable\" repository" - print >> sys.stderr, "If you still want to enable it, type capital YES" - print >> sys.stderr, "" - prompt = raw_input ("Do you want enable PV USB support? ") - if prompt == "YES": - open(pvusb_enable_flagfile, "w").close() - else: - exit(1) - if hasattr(os, "geteuid") and os.geteuid() == 0: if not options.force_root: print >> sys.stderr, "*** Running this tool as root is strongly discouraged, this will lead you in permissions problems." From daf55710b8f0416821c75dcefb166d370b5925d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 2 Jun 2016 11:29:38 +0200 Subject: [PATCH 32/38] travis: initial version QubesOS/qubes-issues#1926 --- .travis.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..b99d8d48 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,9 @@ +sudo: required +dist: trusty +language: generic +install: git clone https://github.com/QubesOS/qubes-builder ~/qubes-builder +# debootstrap in trusty is old... +before_script: sudo ln -s sid /usr/share/debootstrap/scripts/stretch +script: ~/qubes-builder/scripts/travis-build +env: + - DIST_DOM0=fc23 USE_QUBES_REPO_VERSION=3.2 USE_QUBES_REPO_TESTING=1 From 88cb62fcf627edceed577d40a1ed1a4e1d2e5e6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 4 Jun 2016 16:52:02 +0200 Subject: [PATCH 33/38] core: add pci_e820_host property Enable e820_host option for VMs with PCI devices (to allow VM kernel to deal with address space conflicts). But add a property to allow disabling it. Fixes QubesOS/qubes-issues#2019 --- core-modules/000QubesVm.py | 4 ++++ doc/qvm-tools/qvm-prefs.rst | 8 ++++++++ qvm-tools/qvm-prefs | 10 ++++++++++ rpm_spec/core-dom0.spec | 2 +- vm-config/xen-vm-template-hvm.xml | 1 + vm-config/xen-vm-template.xml | 1 + 6 files changed, 25 insertions(+), 1 deletion(-) diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py index 45fd2df1..edf9e7fa 100644 --- a/core-modules/000QubesVm.py +++ b/core-modules/000QubesVm.py @@ -136,6 +136,7 @@ class QubesVm(object): eval(value) if value.find("[") >= 0 else eval("[" + value + "]") }, "pci_strictreset": {"default": True}, + "pci_e820_host": {"default": True}, # Internal VM (not shown in qubes-manager, doesn't create appmenus entries "internal": { "default": False, 'attr': '_internal' }, "vcpus": { "default": 2 }, @@ -1191,6 +1192,7 @@ class QubesVm(object): # If dynamic memory management disabled, set maxmem=mem args['maxmem'] = args['mem'] args['vcpus'] = str(self.vcpus) + args['features'] = '' if self.netvm is not None: args['ip'] = self.ip args['mac'] = self.mac @@ -1215,6 +1217,8 @@ class QubesVm(object): args['network_end'] = '-->' args['no_network_begin'] = '' args['no_network_end'] = '' + if len(self.pcidevs) and self.pci_e820_host: + args['features'] = '' args.update(self.storage.get_config_params()) if hasattr(self, 'kernelopts'): args['kernelopts'] = self.kernelopts diff --git a/doc/qvm-tools/qvm-prefs.rst b/doc/qvm-tools/qvm-prefs.rst index 12bef70e..937b6264 100644 --- a/doc/qvm-tools/qvm-prefs.rst +++ b/doc/qvm-tools/qvm-prefs.rst @@ -49,6 +49,14 @@ pci_strictreset cases it could make sense - for example when the VM to which it is assigned is trusted one, or is running all the time. +pci_e820_host + Accepted values: ``True``, ``False`` + + Give VM with PCI devices a memory map (e820) of the host. This is + required for some devices to properly resolve conflicts in address space. + This option is enabled by default for VMs with PCI devices and have no + effect for VMs without devices. + label Accepted values: ``red``, ``orange``, ``yellow``, ``green``, ``gray``, ``blue``, ``purple``, ``black`` diff --git a/qvm-tools/qvm-prefs b/qvm-tools/qvm-prefs index 7e2225a5..a8180754 100755 --- a/qvm-tools/qvm-prefs +++ b/qvm-tools/qvm-prefs @@ -58,6 +58,7 @@ def do_list(vm): print fmt.format ("config", vm.conf_file) print fmt.format ("pcidevs", vm.pcidevs) print fmt.format ("pci_strictreset", vm.pci_strictreset) + print fmt.format ("pci_e820_host", vm.pci_e820_host) if vm.template is None: print fmt.format ("root_img", vm.root_img) if hasattr(vm, "rootcow_img") and vm.rootcow_img is not None: @@ -228,6 +229,14 @@ def set_pci_strictreset(vms, vm, args): vm.pci_strictreset = bool(eval(args[0].capitalize())) return True +def set_pci_e820_host(vms, vm, args): + if len (args) != 1: + print >> sys.stderr, "Missing value (True/False)!" + return False + + vm.pci_e820_host = bool(eval(args[0].capitalize())) + return True + def set_netvm(vms, vm, args): if len (args) != 1: print >> sys.stderr, "Missing netvm name argument!" @@ -485,6 +494,7 @@ properties = { "include_in_backups": set_include_in_backups, "pcidevs": set_pcidevs, "pci_strictreset": set_pci_strictreset, + "pci_e820_host": set_pci_e820_host, "label" : set_label, "netvm" : set_netvm, "dispvm_netvm" : set_dispvm_netvm, diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec index 86c531bf..16d5d6c2 100644 --- a/rpm_spec/core-dom0.spec +++ b/rpm_spec/core-dom0.spec @@ -62,7 +62,7 @@ Requires: libvirt-python %if x%{?backend_vmm} == xxen Requires: xen-runtime Requires: xen-hvm -Requires: libvirt-daemon-xen >= 1.2.20-4 +Requires: libvirt-daemon-xen >= 1.2.20-6 %endif Requires: createrepo Requires: gnome-packagekit diff --git a/vm-config/xen-vm-template-hvm.xml b/vm-config/xen-vm-template-hvm.xml index 1b6b5089..728b17c4 100644 --- a/vm-config/xen-vm-template-hvm.xml +++ b/vm-config/xen-vm-template-hvm.xml @@ -15,6 +15,7 @@ + {features} destroy diff --git a/vm-config/xen-vm-template.xml b/vm-config/xen-vm-template.xml index ccb89c93..bbb7ee0a 100644 --- a/vm-config/xen-vm-template.xml +++ b/vm-config/xen-vm-template.xml @@ -10,6 +10,7 @@ {kerneldir}/initramfs root=/dev/mapper/dmroot ro nomodeset console=hvc0 rd_NO_PLYMOUTH 3 {kernelopts} + {features} From a857ac3afbf4b8ee5ffaa528c4e2eb67517c88b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 4 Jun 2016 16:57:13 +0200 Subject: [PATCH 34/38] Install dom0 qvm-* tools man pages by default --- rpm_spec/core-dom0.spec | 1 + 1 file changed, 1 insertion(+) diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec index 16d5d6c2..bc3bebf0 100644 --- a/rpm_spec/core-dom0.spec +++ b/rpm_spec/core-dom0.spec @@ -53,6 +53,7 @@ Requires(preun): systemd-units Requires(postun): systemd-units Requires: python, pciutils, python-inotify, python-daemon Requires: qubes-core-dom0-linux >= 3.1.8 +Requires: qubes-core-dom0-doc Requires: qubes-db-dom0 Requires: python-lxml Requires: python-psutil From 2265fd3d52306ec206cafb578bd2afa2e6ad74f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 4 Jun 2016 17:42:24 +0200 Subject: [PATCH 35/38] core: start qubesdb as normal user, even when VM is started by root On VM start, old qubesdb-daemon is terminated (if still running). In practice it happen only at VM startart (shutdown and quickly start again). But in that case, if the VM was started by root, such operation would fail. So when VM is started by root, make sure that qubesdb-daemon will be running as normal user (the first user in group 'qubes' - there should be only one). Fixes QubesOS/qubes-issues#1745 --- core-modules/000QubesVm.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py index edf9e7fa..77685dbf 100644 --- a/core-modules/000QubesVm.py +++ b/core-modules/000QubesVm.py @@ -1867,10 +1867,19 @@ class QubesVm(object): # force connection to a new daemon self._qdb_connection = None - retcode = subprocess.call ([ + qubesdb_cmd = [] + if os.getuid() == 0: + # try to always have qubesdb running as normal user, otherwise + # killing it at VM restart (see above) will always fail + qubes_group = grp.getgrnam('qubes') + qubesdb_cmd = ['runuser', '-u', qubes_group.gr_mem[0], '--'] + + qubesdb_cmd += [ system_path["qubesdb_daemon_path"], str(self.xid), - self.name]) + self.name] + + retcode = subprocess.call (qubesdb_cmd) if retcode != 0: raise OSError("ERROR: Cannot execute qubesdb-daemon!") From 89d002a031b2b253b0bafc83d15369ecbc3efc08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 6 Jun 2016 02:19:51 +0200 Subject: [PATCH 36/38] core: use runuser instead of sudo for switching root->user There are problems with using sudo in early system startup (systemd-logind not running yet, pam_systemd timeouts). Since we don't need full session here, runuser is good enough (even better: faster). --- core-modules/000QubesVm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py index 77685dbf..535304b1 100644 --- a/core-modules/000QubesVm.py +++ b/core-modules/000QubesVm.py @@ -1830,7 +1830,7 @@ class QubesVm(object): # many qrexec services would need to deal with root/user # permission problems qubes_group = grp.getgrnam('qubes') - qrexec = ['sudo', '-u', qubes_group.gr_mem[0]] + qrexec = ['runuser', '-u', qubes_group.gr_mem[0], '--'] qrexec += ['env', 'QREXEC_STARTUP_TIMEOUT=' + str(self.qrexec_timeout), system_path["qrexec_daemon_path"]] From d0ba43f253c7c260e7552ebee9a403bdc2f4be26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 6 Jun 2016 02:21:08 +0200 Subject: [PATCH 37/38] core: start guid as normal user even when VM started by root Another attempt to avoid permissions-related problems... QubesOS/qubes-issues#1768 --- core-modules/000QubesVm.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py index 535304b1..2e1fe8f9 100644 --- a/core-modules/000QubesVm.py +++ b/core-modules/000QubesVm.py @@ -1758,7 +1758,15 @@ class QubesVm(object): if verbose: print >> sys.stderr, "--> Starting Qubes GUId..." - guid_cmd = [system_path["qubes_guid_path"], + guid_cmd = [] + if os.getuid() == 0: + # try to always have guid running as normal user, otherwise + # clipboard file may be created as root and other permission + # problems + qubes_group = grp.getgrnam('qubes') + guid_cmd = ['runuser', '-u', qubes_group.gr_mem[0], '--'] + + guid_cmd += [system_path["qubes_guid_path"], "-d", str(self.xid), "-N", self.name, "-c", self.label.color, "-i", self.label.icon_path, From 7f86782e1470a8076bb6ab6684bb6e3907dbc7aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Tue, 7 Jun 2016 06:46:30 +0200 Subject: [PATCH 38/38] version 3.2.3 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index be94e6f5..b347b11e 100644 --- a/version +++ b/version @@ -1 +1 @@ -3.2.2 +3.2.3