From 86b7849fd441c10e380b06f5f3b00eea67b0fc5e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 20 Jun 2016 12:35:22 +0200
Subject: [PATCH 01/69] tests: update PVGrub2 test for fedora-23 template

dnf doesn't want to replace packages without --allowerasing (it is
needed to have correct kernel-devel package version). Additionally
really make sure the right version is installed and force u2mfn module
compilation.
---
 tests/vm_qrexec_gui.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index 5994acbb..61520ffa 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -1221,9 +1221,12 @@ class TC_40_PVGrub(qubes.tests.SystemTestsMixin):
 
     def install_packages(self, vm):
         if self.template.startswith('fedora-'):
-            cmd_install1 = 'yum clean expire-cache && ' \
-                           'yum install -y qubes-kernel-vm-support grub2-tools'
-            cmd_install2 = 'yum install -y kernel kernel-devel'
+            cmd_install1 = 'dnf clean expire-cache && ' \
+                'dnf install -y qubes-kernel-vm-support grub2-tools'
+            cmd_install2 = 'yum install -y kernel && ' \
+                'KVER=$(rpm -q --qf %{VERSION}-%{RELEASE}.%{ARCH} kernel) && ' \
+                'dnf install --allowerasing  -y kernel-devel-$KVER && ' \
+                'dkms autoinstall -k $KVER'
             cmd_update_grub = 'grub2-mkconfig -o /boot/grub2/grub.cfg'
         elif self.template.startswith('debian-'):
             cmd_install1 = 'apt-get update && apt-get install -y ' \

From 5921dd2a1c9d57fa2712db03633ed13b128eb2ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 20 Jun 2016 22:00:57 +0200
Subject: [PATCH 02/69] core: validate dom0 drive path before starting VM

This is very easy if the file/device is in dom0, so do it to avoid
cryptic startup error (`libvirtError: internal error: libxenlight failed
to create domain`).

Fixes QubesOS/qubes-issues#1619
---
 core-modules/01QubesHVm.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/core-modules/01QubesHVm.py b/core-modules/01QubesHVm.py
index 2a6e9f04..c98b67b5 100644
--- a/core-modules/01QubesHVm.py
+++ b/core-modules/01QubesHVm.py
@@ -312,7 +312,16 @@ class QubesHVm(QubesResizableVm):
         else:
             return -1
 
+    def validate_drive_path(self, drive):
+        drive_type, drive_domain, drive_path = drive.split(':', 2)
+        if drive_domain == 'dom0':
+            if not os.path.exists(drive_path):
+                raise QubesException("Invalid drive path '{}'".format(
+                    drive_path))
+
     def start(self, *args, **kwargs):
+        if self.drive:
+            self.validate_drive_path(self.drive)
         # make it available to storage.prepare_for_vm_startup, which is
         # called before actually building VM libvirt configuration
         self.storage.drive = self.drive

From 9dc488818db2ba6fc9998e3dff0a7d2530a8f929 Mon Sep 17 00:00:00 2001
From: ttasket <tasket@openmailbox.org>
Date: Fri, 24 Jun 2016 06:09:06 -0400
Subject: [PATCH 03/69] Add VM state options

New options: --running, --paused and --template
---
 qvm-tools/qvm-check | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/qvm-tools/qvm-check b/qvm-tools/qvm-check
index 671aeda2..83556f9d 100755
--- a/qvm-tools/qvm-check
+++ b/qvm-tools/qvm-check
@@ -27,10 +27,17 @@ import sys
 import time
 
 def main():
-    usage = "usage: %prog [options] <vm-name>"
+    usage = """usage: %prog [options] <vm-name>\n
+Specify no state options to check if VM exists"""
     parser = OptionParser (usage)
 
     parser.add_option ("-q", "--quiet", action="store_false", dest="verbose", default=True)
+    parser.add_option ("--running", action="store_true", dest="running", default=False,
+    help="Determine if VM is running")
+    parser.add_option ("--paused", action="store_true", dest="paused", default=False,
+    help="Determine if VM is paused")
+    parser.add_option ("--template", action="store_true", dest="template", default=False,
+    help="Determine if VM is a template")
     
     (options, args) = parser.parse_args ()
     if (len (args) != 1):
@@ -47,6 +54,24 @@ def main():
         if options.verbose:
             print >> sys.stdout, "A VM with the name '{0}' does not exist in the system!".format(vmname)
         exit(1)
+        
+    elif options.running:
+        vm_state=not vm.is_running()
+        if options.verbose:
+            print >> sys.stdout, "A VM with the name {0} is {1}running.".format(vmname, "not " * vm_state)
+        exit(vm_state)
+
+    elif options.paused:
+        vm_state=not vm.is_paused()
+        if options.verbose:
+            print >> sys.stdout, "A VM with the name {0} is {1}paused.".format(vmname, "not " * vm_state)
+        exit(vm_state)
+
+    elif options.template:
+        vm_state=not vm.is_template()
+        if options.verbose:
+            print >> sys.stdout, "A VM with the name {0} is {1}a template.".format(vmname, "not " * vm_state)
+        exit(vm_state)
 
     else:
         if options.verbose: 

From 34fc3f33996f35fa706d517396fda16de7b852e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 20 Jun 2016 22:02:30 +0200
Subject: [PATCH 04/69] tests: regression test for #1619 - drive path
 validation

QubesOS/qubes-issues#1619
---
 tests/vm_qrexec_gui.py | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index 61520ffa..de95f09b 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -967,6 +967,36 @@ class TC_10_HVM(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
         self.assertEquals(testvm1.get_power_state(), "Running")
         # TODO: launch some OS there and check the size
 
+    def test_200_start_invalid_drive(self):
+        """Regression test for #1619"""
+        testvm1 = self.qc.add_new_vm("QubesHVm",
+                                     name=self.make_vm_name('vm1'))
+        testvm1.create_on_disk(verbose=False)
+        testvm1.drive = 'hd:dom0:/invalid'
+        self.qc.save()
+        self.qc.unlock_db()
+        try:
+            testvm1.start()
+        except Exception as e:
+            self.assertIsInstance(e, QubesException)
+        else:
+            self.fail('No exception raised')
+
+    def test_201_start_invalid_drive_cdrom(self):
+        """Regression test for #1619"""
+        testvm1 = self.qc.add_new_vm("QubesHVm",
+                                     name=self.make_vm_name('vm1'))
+        testvm1.create_on_disk(verbose=False)
+        testvm1.drive = 'cdrom:dom0:/invalid'
+        self.qc.save()
+        self.qc.unlock_db()
+        try:
+            testvm1.start()
+        except Exception as e:
+            self.assertIsInstance(e, QubesException)
+        else:
+            self.fail('No exception raised')
+
 class TC_20_DispVMMixin(qubes.tests.SystemTestsMixin):
     def test_000_prepare_dvm(self):
         self.qc.unlock_db()

From ae44869499f96ca10c9975027cfbbd96f82640c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 20 Jun 2016 22:02:55 +0200
Subject: [PATCH 05/69] qvm-start: improve error message about missing
 qubes-windows-tools.iso

Fixes QubesOS/qubes-issues#1977
---
 qvm-tools/qvm-start | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/qvm-tools/qvm-start b/qvm-tools/qvm-start
index 002c272e..eeabe68e 100755
--- a/qvm-tools/qvm-start
+++ b/qvm-tools/qvm-start
@@ -86,7 +86,12 @@ def main():
         exit(1)
 
     if options.install_windows_tools:
-        options.drive = 'cdrom:dom0:/usr/lib/qubes/qubes-windows-tools.iso'
+        windows_tools_path = '/usr/lib/qubes/qubes-windows-tools.iso'
+        if not os.path.exists(windows_tools_path):
+            print >> sys.stderr, "You need to install 'qubes-windows-tools' " \
+                                 "package in dom0 first"
+            exit(1)
+        options.drive = 'cdrom:dom0:{}'.format(windows_tools_path)
 
     if options.drive_hd:
         options.drive = 'hd:' + options.drive_hd

From 504360ba9a7b6eaa671fa14bdeac0794f223dd76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 24 Jun 2016 00:17:16 +0200
Subject: [PATCH 06/69] tests: fix clearing 'updates pending' flag test

---
 tests/dom0_update.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/dom0_update.py b/tests/dom0_update.py
index e40365c8..8bfd0045 100644
--- a/tests/dom0_update.py
+++ b/tests/dom0_update.py
@@ -266,7 +266,8 @@ Test package
         open(self.update_flag_path, 'a').close()
 
         # remove also repodata to test #1685
-        shutil.rmtree('/var/lib/qubes/updates/repodata')
+        if os.path.exists('/var/lib/qubes/updates/repodata'):
+            shutil.rmtree('/var/lib/qubes/updates/repodata')
         logpath = os.path.join(self.tmpdir, 'dom0-update-output.txt')
         try:
             subprocess.check_call(['sudo', 'qubes-dom0-update', '-y',

From 3c7915808b613a117e6da5dad1f567649297928d Mon Sep 17 00:00:00 2001
From: GammaSQ <doesnt.work@gmx.at>
Date: Tue, 21 Jun 2016 16:55:42 +0200
Subject: [PATCH 07/69] added --raw-data option

---
 qvm-tools/qvm-ls | 172 +++++++++++++++++++++++++++--------------------
 1 file changed, 98 insertions(+), 74 deletions(-)

diff --git a/qvm-tools/qvm-ls b/qvm-tools/qvm-ls
index e53d2a3e..e36680fa 100755
--- a/qvm-tools/qvm-ls
+++ b/qvm-tools/qvm-ls
@@ -24,7 +24,7 @@
 from qubes.qubes import QubesVmCollection
 from qubes.qubes import QubesHost
 from qubes.qubes import QubesException
-from optparse import OptionParser
+from argparse import ArgumentParser
 import sys
 
 
@@ -91,119 +91,136 @@ fields = {
 
 
 def main():
-    usage = "usage: %prog [options] <vm-name>"
-    parser = OptionParser (usage)
+    usage = "%(prog)s [options]"
+    parser = ArgumentParser ()
 
-    parser.add_option ("-n", "--network", dest="network",
+    parser.add_argument ("VMs", action="store", nargs="*",
+                        help="Specify VMs to be queried")
+
+    parser.add_argument ("-n", "--network", dest="network",
                        action="store_true", default=False,
                        help="Show network addresses assigned to VMs")
 
-    parser.add_option ("-c", "--cpu", dest="cpu",
+    parser.add_argument ("-c", "--cpu", dest="cpu",
                        action="store_true", default=False,
                        help="Show CPU load")
 
-    parser.add_option ("-m", "--mem", dest="mem",
+    parser.add_argument ("-m", "--mem", dest="mem",
                        action="store_true", default=False,
                        help="Show memory usage")
 
-    parser.add_option ("-d", "--disk", dest="disk",
+    parser.add_argument ("-d", "--disk", dest="disk",
                        action="store_true", default=False,
                        help="Show VM disk utilization statistics")
 
-    parser.add_option ("-k", "--kernel", dest="kernel",
+    parser.add_argument ("-k", "--kernel", dest="kernel",
                        action="store_true", default=False,
-                       help="Show VM kernel options")
+                       help="Show VM kernel arguments")
 
-    parser.add_option ("-i", "--ids", dest="ids",
+    parser.add_argument ("-i", "--ids", dest="ids",
                        action="store_true", default=False,
                        help="Show Qubes and Xen id#s")
 
-    parser.add_option("-b", "--last-backup", dest="backup",
+    parser.add_argument("-b", "--last-backup", dest="backup",
                       action="store_true", default=False,
                       help="Show date of last VM backup")
 
-    parser.add_option("--raw-list", dest="raw_list",
+    parser.add_argument("--raw-list", dest="raw_list",
                       action="store_true", default=False,
                       help="List only VM names one per line")
 
+    parser.add_argument("--raw-data", dest="raw_data",
+                        action="store", nargs="+",
+                        help="Display specify data of specified VMs.\
+                              Intended for bash-parsing.")
 
-    (options, args) = parser.parse_args ()
+
+    arguments = parser.parse_args ()
 
     qvm_collection = QubesVmCollection()
     qvm_collection.lock_db_for_reading()
     qvm_collection.load()
     qvm_collection.unlock_db()
 
-    if options.raw_list:
+    if arguments.raw_list:
         for vm in qvm_collection.values():
             print vm.name
         return
 
-    fields_to_display = ["name", "on", "state", "updbl", "type", "template", "netvm", "label" ]
-
     cpu_usages = None
 
-    if (options.ids):
+    if arguments.raw_data:
+        if ''
+    fields_to_display = ["name", "on", "state", "updbl", "type", "template", "netvm", "label" ]
+
+    if (arguments.ids):
         fields_to_display += ["qid", "xid"]
 
-    if (options.cpu):
+    if (arguments.cpu):
         qhost = QubesHost()
         (measure_time, cpu_usages) = qhost.measure_cpu_usage(qvm_collection)
         fields_to_display += ["cpu"]
 
-    if (options.mem):
+    if (arguments.mem):
         fields_to_display += ["mem"]
 
-    if options.backup:
+    if arguments.backup:
         fields_to_display += ["last backup"]
 
-    if (options.network):
+    if (arguments.network):
         if 'template' in fields_to_display:
             fields_to_display.remove ("template")
         fields_to_display += ["ip", "ip back", "gateway/DNS"]
 
-    if (options.disk):
+    if (arguments.disk):
         if 'template' in fields_to_display:
             fields_to_display.remove ("template")
         if 'netvm' in fields_to_display:
             fields_to_display.remove ("netvm")
         fields_to_display += ["priv-curr", "priv-max", "root-curr", "root-max", "disk" ]
 
-    if (options.kernel):
+    if (arguments.kernel):
         fields_to_display += ["kernel", "kernelopts" ]
 
     
     vms_list = [vm for vm in qvm_collection.values()]
-    if len(args) > 0:
-        vms_list = [vm for vm in vms_list if vm.name in args]
-    no_vms = len (vms_list)
-    vms_to_display = []
-    # Frist, the NetVMs...
-    for netvm in vms_list:
-        if netvm.is_netvm():
-            vms_to_display.append (netvm)
+    #assume VMs are presented in desired order:
+    if len(arguments.VMs) > 0:
+        vms_list = [vm for vm in vms_list if vm.name in arguments.VMs]
+    #otherwise, format them accordingly:
+    else:
+        no_vms = len (vms_list)
+        vms_to_display = []
+        # Frist, the NetVMs...
+        for netvm in vms_list:
+            if netvm.is_netvm():
+                vms_to_display.append (netvm)
 
-    # Now, the AppVMs without template (or with template not included in the list)...
-    for appvm in vms_list:
-        if appvm.is_appvm() and not appvm.is_template() and \
-            (appvm.template is None or appvm.template not in vms_list):
-            vms_to_display.append (appvm)
+        # Now, the AppVMs without template (or with template not included in the list)...
+        for appvm in vms_list:
+            if appvm.is_appvm() and not appvm.is_template() and \
+                (appvm.template is None or appvm.template not in vms_list):
+                vms_to_display.append (appvm)
 
-    # Now, the template, and all its AppVMs...
-    for tvm in vms_list:
-        if tvm.is_template():
-            vms_to_display.append (tvm)
-            for vm in vms_list:
-                if (vm.is_appvm() or vm.is_disposablevm()) and \
-		    vm.template and vm.template.qid == tvm.qid:
-                    vms_to_display.append(vm)
+        # Now, the template, and all its AppVMs...
+        for tvm in vms_list:
+            if tvm.is_template():
+                vms_to_display.append (tvm)
+                for vm in vms_list:
+                    if (vm.is_appvm() or vm.is_disposablevm()) and \
+                vm.template and vm.template.qid == tvm.qid:
+                        vms_to_display.append(vm)
+
+        assert len(vms_to_display) == no_vms
+
+        #We DON'T NEED a max_width if we devide output by pipes!
+
+        # First calculate the maximum width of each field we want to display
+        #  also collect data to display
+        for f in fields_to_display:
+            fields[f]["max_width"] = len(f)
 
-    assert len(vms_to_display) == no_vms
 
-    # First calculate the maximum width of each field we want to display
-    #  also collect data to display
-    for f in fields_to_display:
-        fields[f]["max_width"] = len(f)
     data_to_display = []
     for vm in vms_to_display:
         data_row = {}
@@ -213,42 +230,49 @@ def main():
             else:
                 data_row[f] = str(eval(fields[f]["func"]))
             l = len(data_row[f])
-            if l > fields[f]["max_width"]:
+            if 'max_width' in fields[f] and l > fields[f]["max_width"]:
                 fields[f]["max_width"] = l
         data_to_display.append(data_row)
         try:
             vm.verify_files()
         except QubesException as err:
             print >> sys.stderr, "WARNING: VM '{0}' has corrupted files!".format(vm.name)
-                
-    # XXX: For what?
-    total_width = 0;
-    for f in fields_to_display:
-        total_width += fields[f]["max_width"]
 
-    # Display the header
-    s = ""
-    for f in fields_to_display:
-        fmt="{{0:-^{0}}}-+".format(fields[f]["max_width"] + 1)
-        s += fmt.format('-')
-    print s
-    s = ""
-    for f in fields_to_display:
-        fmt="{{0:>{0}}} |".format(fields[f]["max_width"] + 1)
-        s += fmt.format(f) 
-    print s
-    s = ""
-    for f in fields_to_display:
-        fmt="{{0:-^{0}}}-+".format(fields[f]["max_width"] + 1)
-        s += fmt.format('-')
-    print s
+    #Nicely formatted header only needed for humans
+    if not arguments.raw_data:
+        # XXX: For what?
+        total_width = 0;
+        for f in fields_to_display:
+            total_width += fields[f]["max_width"]
 
-    # ... and the actual data
-    for row in data_to_display:
+        # Display the header
+        s = ""
+        for f in fields_to_display:
+            fmt="{{0:-^{0}}}-+".format(fields[f]["max_width"] + 1)
+            s += fmt.format('-')
+        print s
         s = ""
         for f in fields_to_display:
             fmt="{{0:>{0}}} |".format(fields[f]["max_width"] + 1)
-            s += fmt.format(row[f]) 
+            s += fmt.format(f) 
+        print s
+        s = ""
+        for f in fields_to_display:
+            fmt="{{0:-^{0}}}-+".format(fields[f]["max_width"] + 1)
+            s += fmt.format('-')
         print s
 
+        # ... and the actual data
+        for row in data_to_display:
+            s = ""
+            for f in fields_to_display:
+                fmt="{{0:>{0}}} |".format(fields[f]["max_width"] + 1)
+                s += fmt.format(row[f]) 
+            print s
+
+    #won't look pretty, but is easy to parse!
+    else:
+        for row in data_to_display:
+            print '|'.join([row[f] for f in fields_to_display])
+
 main()

From 3599249e2d57e71f87b8f073e1feae0f5e1972dc Mon Sep 17 00:00:00 2001
From: GammaSQ <doesnt.work@gmx.at>
Date: Tue, 21 Jun 2016 20:30:34 +0200
Subject: [PATCH 08/69] forgot if-statement in last commit

---
 qvm-tools/qvm-ls | 52 ++++++++++++++++++++++++++----------------------
 1 file changed, 28 insertions(+), 24 deletions(-)

diff --git a/qvm-tools/qvm-ls b/qvm-tools/qvm-ls
index e36680fa..03e43ab4 100755
--- a/qvm-tools/qvm-ls
+++ b/qvm-tools/qvm-ls
@@ -150,37 +150,41 @@ def main():
     cpu_usages = None
 
     if arguments.raw_data:
-        if ''
-    fields_to_display = ["name", "on", "state", "updbl", "type", "template", "netvm", "label" ]
+        fields_to_display = arguments.raw_data
+        if 'cpu' in arguments.raw_data:
+            qhost = QubesHost()
+            (measure_time, cpu_usages) = qhost.measure_cpu_usage(qvm_collection)
+    else:
+        fields_to_display = ["name", "on", "state", "updbl", "type", "template", "netvm", "label" ]
 
-    if (arguments.ids):
-        fields_to_display += ["qid", "xid"]
+        if (arguments.ids):
+            fields_to_display += ["qid", "xid"]
 
-    if (arguments.cpu):
-        qhost = QubesHost()
-        (measure_time, cpu_usages) = qhost.measure_cpu_usage(qvm_collection)
-        fields_to_display += ["cpu"]
+        if (arguments.cpu):
+            qhost = QubesHost()
+            (measure_time, cpu_usages) = qhost.measure_cpu_usage(qvm_collection)
+            fields_to_display += ["cpu"]
 
-    if (arguments.mem):
-        fields_to_display += ["mem"]
+        if (arguments.mem):
+            fields_to_display += ["mem"]
 
-    if arguments.backup:
-        fields_to_display += ["last backup"]
+        if arguments.backup:
+            fields_to_display += ["last backup"]
 
-    if (arguments.network):
-        if 'template' in fields_to_display:
-            fields_to_display.remove ("template")
-        fields_to_display += ["ip", "ip back", "gateway/DNS"]
+        if (arguments.network):
+            if 'template' in fields_to_display:
+                fields_to_display.remove ("template")
+            fields_to_display += ["ip", "ip back", "gateway/DNS"]
 
-    if (arguments.disk):
-        if 'template' in fields_to_display:
-            fields_to_display.remove ("template")
-        if 'netvm' in fields_to_display:
-            fields_to_display.remove ("netvm")
-        fields_to_display += ["priv-curr", "priv-max", "root-curr", "root-max", "disk" ]
+        if (arguments.disk):
+            if 'template' in fields_to_display:
+                fields_to_display.remove ("template")
+            if 'netvm' in fields_to_display:
+                fields_to_display.remove ("netvm")
+            fields_to_display += ["priv-curr", "priv-max", "root-curr", "root-max", "disk" ]
 
-    if (arguments.kernel):
-        fields_to_display += ["kernel", "kernelopts" ]
+        if (arguments.kernel):
+            fields_to_display += ["kernel", "kernelopts" ]
 
     
     vms_list = [vm for vm in qvm_collection.values()]

From 7eb881c6bab823a4fe1b8639aa587bd3250f816a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 24 Jun 2016 04:36:37 +0200
Subject: [PATCH 09/69] tests: skip some tests not supported on Whonix

---
 tests/block.py         | 3 +++
 tests/dom0_update.py   | 4 ++++
 tests/network.py       | 3 +++
 tests/vm_qrexec_gui.py | 2 ++
 4 files changed, 12 insertions(+)

diff --git a/tests/block.py b/tests/block.py
index b6d9e86c..a40dfea7 100644
--- a/tests/block.py
+++ b/tests/block.py
@@ -200,6 +200,9 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
             self.fail("Device {} not found in {!r}".format('test-dm', dev_list))
 
     def test_013_list_dm_removed(self):
+        if self.template is None:
+            self.skipTest('test not supported in dom0 - loop devices excluded '
+                          'in dom0')
         self.run_script(
             "set -e;"
             "truncate -s 128M {path}; "
diff --git a/tests/dom0_update.py b/tests/dom0_update.py
index 8bfd0045..164e436a 100644
--- a/tests/dom0_update.py
+++ b/tests/dom0_update.py
@@ -100,6 +100,10 @@ enabled = 1
 
     def setUp(self):
         super(TC_00_Dom0UpgradeMixin, self).setUp()
+        if self.template.startswith('whonix-'):
+            # Whonix redirect all the traffic through tor, so repository
+            # on http://localhost:8080/ is unavailable
+            self.skipTest("Test not supported for this template")
         self.updatevm = self.qc.add_new_vm(
             "QubesProxyVm",
             name=self.make_vm_name("updatevm"),
diff --git a/tests/network.py b/tests/network.py
index 838d6cf1..a2e64d34 100644
--- a/tests/network.py
+++ b/tests/network.py
@@ -54,6 +54,9 @@ class VmNetworkingMixin(qubes.tests.SystemTestsMixin):
 
     def setUp(self):
         super(VmNetworkingMixin, self).setUp()
+        if self.template.startswith('whonix-'):
+            self.skipTest("Test not supported here - Whonix uses its own "
+                          "firewall settings")
         self.testnetvm = self.qc.add_new_vm("QubesNetVm",
             name=self.make_vm_name('netvm1'),
             template=self.qc.get_vm_by_name(self.template))
diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index de95f09b..820f4483 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -785,6 +785,8 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
 
     def test_210_time_sync(self):
         """Test time synchronization mechanism"""
+        if self.template.startswith('whonix-'):
+            self.skipTest('qvm-sync-clock disabled for Whonix VMs')
         self.testvm1.start()
         self.testvm2.start()
         (start_time, _) = subprocess.Popen(["date", "-u", "+%s"],

From 4123b958669cf4217e27286d01ba955270cecd87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 24 Jun 2016 04:37:06 +0200
Subject: [PATCH 10/69] tests: make sure dnsmasq isn't already running

On Debian when dnsmasq is installed, it is automatically started. Which
prevents starting a second instance.
---
 tests/network.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/network.py b/tests/network.py
index a2e64d34..eeab8053 100644
--- a/tests/network.py
+++ b/tests/network.py
@@ -88,6 +88,8 @@ class VmNetworkingMixin(qubes.tests.SystemTestsMixin):
         run_netvm_cmd("ip link set test0 up")
         run_netvm_cmd("ip addr add {}/24 dev test0".format(self.test_ip))
         run_netvm_cmd("iptables -I INPUT -d {} -j ACCEPT".format(self.test_ip))
+        # ignore failure
+        self.run_cmd(self.testnetvm, "killall --wait dnsmasq")
         run_netvm_cmd("dnsmasq -a {ip} -A /{name}/{ip} -i test0 -z".format(
             ip=self.test_ip, name=self.test_name))
         run_netvm_cmd("echo nameserver {} > /etc/resolv.conf".format(

From 38beb9412a80748809c933c392a21de1b37c737c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 24 Jun 2016 04:40:06 +0200
Subject: [PATCH 11/69] tests: wait for editor window to settle before sending
 any keystrokes

---
 tests/vm_qrexec_gui.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index 820f4483..fbff923d 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -1153,6 +1153,7 @@ class TC_20_DispVMMixin(qubes.tests.SystemTestsMixin):
                 self.fail("Timeout while waiting for editor window")
             time.sleep(0.3)
 
+        time.sleep(0.5)
         self._handle_editor(winid)
         p.wait()
         p = testvm1.run("cat /home/user/test.txt",

From 9bc60927c5d2d29751a90fbb86c758ef98ee584e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 24 Jun 2016 12:13:16 +0200
Subject: [PATCH 12/69] tests: replace sfdisk call with verbatim partition
 table

sfdisk options and input format differs between versions (dropped MB
units support), so instead of supporting all the combinations,
simply paste its result verbatim.
---
 tests/backupcompatibility.py | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/tests/backupcompatibility.py b/tests/backupcompatibility.py
index 775db92f..826830d2 100644
--- a/tests/backupcompatibility.py
+++ b/tests/backupcompatibility.py
@@ -167,14 +167,20 @@ class TC_00_BackupCompatibility(qubes.tests.BackupTestsMixin, qubes.tests.QubesT
 
     def create_volatile_img(self, filename):
         self.create_sparse(filename, 11.5*2**30)
-        sfdisk_input="0,1024,S\n,10240,L\n"
-        p = subprocess.Popen(["/usr/sbin/sfdisk", "--no-reread", "-u",
-                                   "M",
-                               filename], stdout=open("/dev/null","w"),
-                              stderr=subprocess.STDOUT, stdin=subprocess.PIPE)
-        p.communicate(input=sfdisk_input)
-        self.assertEqual(p.returncode, 0, "sfdisk failed with code %d" % p
-                         .returncode)
+        # here used to be sfdisk call with "0,1024,S\n,10240,L\n" input,
+        # but since sfdisk folks like to change command arguments in
+        # incompatible way, have an partition table verbatim here
+        ptable = (
+            '\x00\x00\x00\x00\x00\x00\x00\x00\xab\x39\xd5\xd4\x00\x00\x20\x00'
+            '\x00\x21\xaa\x82\x82\x28\x08\x00\x00\x00\x00\x00\x00\x20\xaa\x00'
+            '\x82\x29\x15\x83\x9c\x79\x08\x00\x00\x20\x00\x00\x01\x40\x00\x00'
+            '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+            '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xaa\x55'
+        )
+        with open(filename, 'r+') as f:
+            f.seek(0x1b0)
+            f.write(ptable)
+
         # TODO: mkswap
 
     def fullpath(self, name):

From 3cb717dced517c6405ee487c21cbceed3966f419 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 24 Jun 2016 12:15:32 +0200
Subject: [PATCH 13/69] tests: add --sync to xdotool windowactivate

This may help getting less errors from xdotool...
---
 tests/vm_qrexec_gui.py | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index fbff923d..d8ed9965 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -127,7 +127,7 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
         time.sleep(0.5)
         subprocess.check_call(
             ['xdotool', 'search', '--name', title,
-             'windowactivate', 'type', 'exit\n'])
+             'windowactivate', '--sync', 'type', 'exit\n'])
 
         wait_count = 0
         while subprocess.call(['xdotool', 'search', '--name', title],
@@ -168,7 +168,7 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
         time.sleep(0.5)
         subprocess.check_call(
             ['xdotool', 'search', '--name', title,
-             'windowactivate', 'type', 'exit\n'])
+             'windowactivate', '--sync', 'type', 'exit\n'])
 
         wait_count = 0
         while subprocess.call(['xdotool', 'search', '--name', title],
@@ -1097,25 +1097,24 @@ class TC_20_DispVMMixin(qubes.tests.SystemTestsMixin):
             replace('(', '\(').replace(')', '\)')
         time.sleep(1)
         if "gedit" in window_title:
-            subprocess.check_call(['xdotool', 'search', '--name', window_title,
-                                   'windowactivate', 'type', 'test test 2\n'])
+            subprocess.check_call(['xdotool', 'windowactivate', '--sync', winid,
+                'type', 'test test 2\n'])
             time.sleep(0.5)
-            subprocess.check_call(['xdotool', 'search', '--name', window_title,
+            subprocess.check_call(['xdotool',
                                    'key', 'ctrl+s', 'ctrl+q'])
         elif "emacs" in window_title:
-            subprocess.check_call(['xdotool', 'search', '--name', window_title,
-                                   'windowactivate', 'type', 'test test 2\n'])
+            subprocess.check_call(['xdotool', 'windowactivate', '--sync', winid,
+                                   'type', 'test test 2\n'])
             time.sleep(0.5)
-            subprocess.check_call(['xdotool', 'search', '--name', window_title,
+            subprocess.check_call(['xdotool',
                                    'key', 'ctrl+x', 'ctrl+s'])
-            subprocess.check_call(['xdotool', 'search', '--name', window_title,
+            subprocess.check_call(['xdotool',
                                    'key', 'ctrl+x', 'ctrl+c'])
         elif "vim" in window_title:
-            subprocess.check_call(['xdotool', 'search', '--name', window_title,
-                                   'windowactivate', 'key', 'i',
-                                   'type', 'test test 2\n'])
+            subprocess.check_call(['xdotool', 'windowactivate', '--sync', winid,
+                                   'key', 'i', 'type', 'test test 2\n'])
             subprocess.check_call(
-                ['xdotool', 'search', '--name', window_title,
+                ['xdotool',
                  'key', 'Escape', 'colon', 'w', 'q', 'Return'])
         else:
             self.fail("Unknown editor window: {}".format(window_title))
@@ -1190,7 +1189,7 @@ class TC_30_Gui_daemon(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
 
         # Type and copy some text
         subprocess.check_call(['xdotool', 'search', '--name', window_title,
-                               'windowactivate',
+                               'windowactivate', '--sync',
                                'type', '{}'.format(test_string)])
         # second xdotool call because type --terminator do not work (SEGV)
         # additionally do not use search here, so window stack will be empty

From ba0a01afba3bc2dcd0d4c367cd021edc8cd3f6a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 24 Jun 2016 12:29:08 +0200
Subject: [PATCH 14/69] tests: fix closing xterm window

sh -s option in dash prevent shell to terminate after command from -c
ends. So remove this option.
---
 tests/vm_qrexec_gui.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index d8ed9965..52feae79 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -1057,8 +1057,8 @@ class TC_20_DispVMMixin(qubes.tests.SystemTestsMixin):
         try:
             window_title = 'user@%s' % (dispvm.template.name + "-dvm")
             p.stdin.write("xterm -e "
-                          "\"sh -s -c 'echo \\\"\033]0;{}\007\\\";read x;'\"\n".
-                          format(window_title))
+                "\"sh -c 'echo \\\"\033]0;{}\007\\\";read x;'\"\n".
+                format(window_title))
             self.wait_for_window(window_title)
 
             time.sleep(0.5)

From a30f1d3902ab160760cbdb38d078dd7302d7f843 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 24 Jun 2016 12:31:55 +0200
Subject: [PATCH 15/69] tests: firefox sets "Navigator" as window class

---
 tests/vm_qrexec_gui.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index 52feae79..29d4bf6a 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -1612,7 +1612,7 @@ class TC_50_MimeHandlers(qubes.tests.SystemTestsMixin):
 
     def test_010_url(self):
         self.open_file_and_check_viewer("https://www.qubes-os.org/", [],
-                                        ["Firefox", "Iceweasel"])
+                                        ["Firefox", "Iceweasel", "Navigator"])
 
     def test_100_txt_dispvm(self):
         filename = "/home/user/test_file.txt"
@@ -1665,7 +1665,7 @@ class TC_50_MimeHandlers(qubes.tests.SystemTestsMixin):
 
     def test_110_url_dispvm(self):
         self.open_file_and_check_viewer("https://www.qubes-os.org/", [],
-                                        ["Firefox", "Iceweasel"],
+                                        ["Firefox", "Iceweasel", "Navigator"],
                                         dispvm=True)
 
 

From 9956e4c7b3df147cc35c5adf2ff3f394fa3b53c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 24 Jun 2016 19:43:22 +0200
Subject: [PATCH 16/69] tests: handle vim opened in xterm without adjusted
 window title

On debian vim in xterm doesn't have "vim" in title, just standard
user@host.
---
 tests/vm_qrexec_gui.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index 29d4bf6a..be12ab08 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -1110,7 +1110,7 @@ class TC_20_DispVMMixin(qubes.tests.SystemTestsMixin):
                                    'key', 'ctrl+x', 'ctrl+s'])
             subprocess.check_call(['xdotool',
                                    'key', 'ctrl+x', 'ctrl+c'])
-        elif "vim" in window_title:
+        elif "vim" in window_title or "user@" in window_title:
             subprocess.check_call(['xdotool', 'windowactivate', '--sync', winid,
                                    'key', 'i', 'type', 'test test 2\n'])
             subprocess.check_call(
@@ -1571,7 +1571,7 @@ class TC_50_MimeHandlers(qubes.tests.SystemTestsMixin):
     def test_000_txt(self):
         filename = "/home/user/test_file.txt"
         self.prepare_txt(filename)
-        self.open_file_and_check_viewer(filename, ["vim"],
+        self.open_file_and_check_viewer(filename, ["vim", "user@"],
                                         ["gedit", "emacs"])
 
     def test_001_pdf(self):
@@ -1617,7 +1617,7 @@ class TC_50_MimeHandlers(qubes.tests.SystemTestsMixin):
     def test_100_txt_dispvm(self):
         filename = "/home/user/test_file.txt"
         self.prepare_txt(filename)
-        self.open_file_and_check_viewer(filename, ["vim"],
+        self.open_file_and_check_viewer(filename, ["vim", "user@"],
                                         ["gedit", "emacs"],
                                         dispvm=True)
 

From 776393e97b0fd4decc4b7624a3137e1e143681b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 24 Jun 2016 22:43:26 +0200
Subject: [PATCH 17/69] qvm-check: whitespace fixes

---
 qvm-tools/qvm-check | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/qvm-tools/qvm-check b/qvm-tools/qvm-check
index 83556f9d..cc69113b 100755
--- a/qvm-tools/qvm-check
+++ b/qvm-tools/qvm-check
@@ -38,7 +38,7 @@ Specify no state options to check if VM exists"""
     help="Determine if VM is paused")
     parser.add_option ("--template", action="store_true", dest="template", default=False,
     help="Determine if VM is a template")
-    
+
     (options, args) = parser.parse_args ()
     if (len (args) != 1):
         parser.error ("You must specify VM name!")
@@ -54,27 +54,27 @@ Specify no state options to check if VM exists"""
         if options.verbose:
             print >> sys.stdout, "A VM with the name '{0}' does not exist in the system!".format(vmname)
         exit(1)
-        
+
     elif options.running:
-        vm_state=not vm.is_running()
+        vm_state = not vm.is_running()
         if options.verbose:
             print >> sys.stdout, "A VM with the name {0} is {1}running.".format(vmname, "not " * vm_state)
         exit(vm_state)
 
     elif options.paused:
-        vm_state=not vm.is_paused()
+        vm_state = not vm.is_paused()
         if options.verbose:
             print >> sys.stdout, "A VM with the name {0} is {1}paused.".format(vmname, "not " * vm_state)
         exit(vm_state)
 
     elif options.template:
-        vm_state=not vm.is_template()
+        vm_state = not vm.is_template()
         if options.verbose:
             print >> sys.stdout, "A VM with the name {0} is {1}a template.".format(vmname, "not " * vm_state)
         exit(vm_state)
 
     else:
-        if options.verbose: 
+        if options.verbose:
             print >> sys.stdout, "A VM with the name '{0}' does exist.".format(vmname)
         exit(0)
 

From 84677fa70bbc716abf223e639e0bf6f7771465ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 24 Jun 2016 23:05:15 +0200
Subject: [PATCH 18/69] qvm-ls: fix handling VM list on command line

---
 qvm-tools/qvm-ls | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qvm-tools/qvm-ls b/qvm-tools/qvm-ls
index 03e43ab4..c4af2d7c 100755
--- a/qvm-tools/qvm-ls
+++ b/qvm-tools/qvm-ls
@@ -190,7 +190,7 @@ def main():
     vms_list = [vm for vm in qvm_collection.values()]
     #assume VMs are presented in desired order:
     if len(arguments.VMs) > 0:
-        vms_list = [vm for vm in vms_list if vm.name in arguments.VMs]
+        vms_to_display = [vm for vm in vms_list if vm.name in arguments.VMs]
     #otherwise, format them accordingly:
     else:
         no_vms = len (vms_list)

From 376dc43b907afa41a9d94c02431a5bc7f14756dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 24 Jun 2016 23:06:35 +0200
Subject: [PATCH 19/69] version 3.2.4

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index b347b11e..351227fc 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.2.3
+3.2.4

From 748a3a90a292ed79d59a71ef0f22c8a2b5e9ce09 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 25 Jun 2016 00:19:12 +0200
Subject: [PATCH 20/69] core: fix handling disabling VM autostart on VM removal

QubesOS/qubes-issues#1930
---
 core-modules/000QubesVm.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index 2e1fe8f9..a03585a2 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -1414,7 +1414,8 @@ class QubesVm(object):
                 raise
 
         if os.path.exists("/etc/systemd/system/multi-user.target.wants/qubes-vm@" + self.name + ".service"):
-            subprocess.call(["sudo", "systemctl", "-q", "disable","qubes-vm@" + self.name + ".service"])
+            retcode = subprocess.call(["sudo", "systemctl", "-q", "disable",
+                "qubes-vm@" + self.name + ".service"])
             if retcode != 0:
                 raise QubesException("Failed to delete autostart entry for VM")
 

From 13f832645aa8d6a48ae63b083ed3e0c2b7a554d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 25 Jun 2016 00:20:40 +0200
Subject: [PATCH 21/69] qvm-remove: undefine libvirt domain even when not
 removing files

Fixes QubesOS/qubes-issues#2112
---
 qvm-tools/qvm-remove | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/qvm-tools/qvm-remove b/qvm-tools/qvm-remove
index ccb10212..b574d0de 100755
--- a/qvm-tools/qvm-remove
+++ b/qvm-tools/qvm-remove
@@ -79,6 +79,10 @@ def main():
             exit (1)
 
     try:
+        if options.remove_from_db_only:
+            # normally it is done by vm.remove_from_disk(), but it isn't
+            # called in this case
+            vm.libvirt_domain.undefine()
         if vm.installed_by_rpm:
             if options.verbose:
                 print >> sys.stderr, "--> VM installed by RPM, leaving all the files on disk"

From 9d781f77ce8b65ee98612beb5b4f9cdec872858b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 25 Jun 2016 01:53:39 +0200
Subject: [PATCH 22/69] tests: VM removal

Check if everything is cleaned up.
---
 tests/basic.py | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/tests/basic.py b/tests/basic.py
index f41563a4..485809ad 100644
--- a/tests/basic.py
+++ b/tests/basic.py
@@ -31,7 +31,7 @@ import tempfile
 
 import unittest
 import time
-from qubes.qubes import QubesVmCollection, QubesException, system_path
+from qubes.qubes import QubesVmCollection, QubesException, system_path, vmm
 import libvirt
 
 import qubes.qubes
@@ -53,6 +53,26 @@ class TC_00_Basic(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
         with self.assertNotRaises(qubes.qubes.QubesException):
             vm.verify_files()
 
+    def test_010_remove(self):
+        vmname = self.make_vm_name('appvm')
+        vm = self.qc.add_new_vm('QubesAppVm',
+            name=vmname, template=self.qc.get_default_template())
+        vm.create_on_disk(verbose=False)
+        # check for QubesOS/qubes-issues#1930
+        vm.autostart = True
+        self.save_and_reload_db()
+        vm = self.qc[vm.qid]
+        vm.remove_from_disk()
+        self.qc.pop(vm.qid)
+        self.save_and_reload_db()
+        self.assertNotIn(vm.qid, self.qc)
+        self.assertFalse(os.path.exists(vm.dir_path))
+        self.assertFalse(os.path.exists(
+            '/etc/systemd/system/multi-user.target.wants/'
+            'qubes-vm@{}.service'.format(vm.name)))
+        with self.assertRaises(libvirt.libvirtError):
+            vmm.libvirt_conn.lookupByName(vm.name)
+
 
 class TC_01_Properties(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
     def setUp(self):

From e431e8bc4560737f578e8b4bf8a624d6911aa2f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sun, 26 Jun 2016 13:02:34 +0200
Subject: [PATCH 23/69] qvm-ls: fix handling explicit VMs list

---
 qvm-tools/qvm-ls | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/qvm-tools/qvm-ls b/qvm-tools/qvm-ls
index c4af2d7c..26767890 100755
--- a/qvm-tools/qvm-ls
+++ b/qvm-tools/qvm-ls
@@ -217,12 +217,12 @@ def main():
 
         assert len(vms_to_display) == no_vms
 
-        #We DON'T NEED a max_width if we devide output by pipes!
+    #We DON'T NEED a max_width if we devide output by pipes!
 
-        # First calculate the maximum width of each field we want to display
-        #  also collect data to display
-        for f in fields_to_display:
-            fields[f]["max_width"] = len(f)
+    # First calculate the maximum width of each field we want to display
+    #  also collect data to display
+    for f in fields_to_display:
+        fields[f]["max_width"] = len(f)
 
 
     data_to_display = []

From 12bf920969f0b1aaecab29c8e2d3ff280402d75b Mon Sep 17 00:00:00 2001
From: Desobediente Civil <desci@riseup.net>
Date: Mon, 27 Jun 2016 19:10:00 -0300
Subject: [PATCH 24/69] Quoting the destination as proposed in #1672

---
 qvm-tools/qubes-hcl-report | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qvm-tools/qubes-hcl-report b/qvm-tools/qubes-hcl-report
index 031b16e9..e038aa16 100755
--- a/qvm-tools/qubes-hcl-report
+++ b/qvm-tools/qubes-hcl-report
@@ -209,7 +209,7 @@ versions:
     FIXLINK
 
 ---
-" >> $HOME/$FILENAME.yml
+" >> "$HOME/$FILENAME.yml"
 
 
 if [[ "$SUPPORT_FILES" == 1 ]]

From fa83298153949456ff7d4de64fb2277394d4d182 Mon Sep 17 00:00:00 2001
From: Desobediente Civil <desci@riseup.net>
Date: Mon, 27 Jun 2016 19:15:46 -0300
Subject: [PATCH 25/69] Modifying support cpio as proposed in #1672

---
 qvm-tools/qubes-hcl-report | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/qvm-tools/qubes-hcl-report b/qvm-tools/qubes-hcl-report
index e038aa16..72ca74f7 100755
--- a/qvm-tools/qubes-hcl-report
+++ b/qvm-tools/qubes-hcl-report
@@ -217,7 +217,7 @@ if [[ "$SUPPORT_FILES" == 1 ]]
 
     # cpio
     cd $TEMP_DIR
-    find -print0 |cpio --quiet -o -H crc --null |gzip  >$HOME/$FILENAME.cpio.gz
+    find -print0 | cpio --quiet -o -H crc --null | gzip  > "$HOME/$FILENAME.cpio.gz"
     cd
 fi
 
@@ -225,7 +225,7 @@ fi
 if [[ "$COPY2VM" != "dom0" ]]
  then
     # Copy to VM
-    qvm-start -q $COPY2VM 2>/dev/null
+    qvm-start -q $COPY2VM 2> /dev/null
 
     if [[ -f "$HOME/$FILENAME.cpio.gz" ]]
       then

From 5081b58ee88d3b1d204cb90a02b408e2080b1b1c Mon Sep 17 00:00:00 2001
From: Desobediente Civil <desci@riseup.net>
Date: Mon, 27 Jun 2016 19:19:00 -0300
Subject: [PATCH 26/69] Quoting all `cat`s as proposed in #1672

---
 qvm-tools/qubes-hcl-report | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/qvm-tools/qubes-hcl-report b/qvm-tools/qubes-hcl-report
index 72ca74f7..e5e62a45 100755
--- a/qvm-tools/qubes-hcl-report
+++ b/qvm-tools/qubes-hcl-report
@@ -229,12 +229,12 @@ if [[ "$COPY2VM" != "dom0" ]]
 
     if [[ -f "$HOME/$FILENAME.cpio.gz" ]]
       then
-        cat $HOME/$FILENAME.cpio.gz | qvm-run -a -q --pass-io $COPY2VM "cat >/home/user/$FILENAME.cpio.gz"
+        cat "$HOME/$FILENAME.cpio.gz" | qvm-run -a -q --pass-io $COPY2VM "cat >/home/user/$FILENAME.cpio.gz"
     fi
 
     if [[ -f "$HOME/$FILENAME.yml" ]]
       then
-        cat $HOME/$FILENAME.yml | qvm-run -a -q --pass-io $COPY2VM "cat >/home/user/$FILENAME.yml"
+        cat "$HOME/$FILENAME.yml" | qvm-run -a -q --pass-io $COPY2VM "cat >/home/user/$FILENAME.yml"
     fi
 
 fi

From 59e687c3f3fe5ebbbb5299e87fc8fd878ba8650e Mon Sep 17 00:00:00 2001
From: Desobediente Civil <desci@riseup.net>
Date: Mon, 27 Jun 2016 19:23:53 -0300
Subject: [PATCH 27/69] And some more quoting to satisfy #1672

---
 qvm-tools/qubes-hcl-report | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/qvm-tools/qubes-hcl-report b/qvm-tools/qubes-hcl-report
index e5e62a45..671f0000 100755
--- a/qvm-tools/qubes-hcl-report
+++ b/qvm-tools/qubes-hcl-report
@@ -229,12 +229,12 @@ if [[ "$COPY2VM" != "dom0" ]]
 
     if [[ -f "$HOME/$FILENAME.cpio.gz" ]]
       then
-        cat "$HOME/$FILENAME.cpio.gz" | qvm-run -a -q --pass-io $COPY2VM "cat >/home/user/$FILENAME.cpio.gz"
+        cat "$HOME/$FILENAME.cpio.gz" | qvm-run -a -q --pass-io $COPY2VM "cat > \"/home/user/$FILENAME.cpio.gz\""
     fi
 
     if [[ -f "$HOME/$FILENAME.yml" ]]
       then
-        cat "$HOME/$FILENAME.yml" | qvm-run -a -q --pass-io $COPY2VM "cat >/home/user/$FILENAME.yml"
+        cat "$HOME/$FILENAME.yml" | qvm-run -a -q --pass-io $COPY2VM "cat > \"/home/user/$FILENAME.yml\""
     fi
 
 fi

From afb2a65744efc0dfaab34504e4f6b60779b2203b Mon Sep 17 00:00:00 2001
From: Rusty Bird <rustybird@openmailbox.org>
Date: Tue, 28 Jun 2016 13:26:10 +0000
Subject: [PATCH 28/69] qfile-daemon-dvm: Move dispVM killing into cleanup
 function

---
 dispvm/qfile-daemon-dvm | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/dispvm/qfile-daemon-dvm b/dispvm/qfile-daemon-dvm
index 423500b6..c2ae9898 100755
--- a/dispvm/qfile-daemon-dvm
+++ b/dispvm/qfile-daemon-dvm
@@ -150,7 +150,7 @@ class QfileDaemonDvm:
         return self.do_get_dvm()
 
     @staticmethod
-    def remove_disposable_from_qdb(name):
+    def finish_disposable(name):
         qvm_collection = QubesVmCollection()
         qvm_collection.lock_db_for_writing()
         qvm_collection.load()
@@ -158,6 +158,12 @@ class QfileDaemonDvm:
         if vm is None:
             qvm_collection.unlock_db()
             return False
+
+        try:
+            vm.force_shutdown()
+        except QubesException:
+            # VM already destroyed
+            pass
         qvm_collection.pop(vm.qid)
         qvm_collection.save()
         qvm_collection.unlock_db()
@@ -181,11 +187,6 @@ def main():
         subprocess.call(['/usr/lib/qubes/qrexec-client', '-d', dispvm.name,
                          user+':exec /usr/lib/qubes/qubes-rpc-multiplexer ' +
                          exec_index + " " + src_vmname])
-        try:
-            dispvm.force_shutdown()
-        except QubesException:
-            # VM already destroyed
-            pass
-        qfile.remove_disposable_from_qdb(dispvm.name)
+        qfile.finish_disposable(dispvm.name)
 
 main()

From 142cb9e2406dc3cbe6ce9ec98ff7c06bf0a2a41d Mon Sep 17 00:00:00 2001
From: Rusty Bird <rustybird@openmailbox.org>
Date: Tue, 28 Jun 2016 13:27:03 +0000
Subject: [PATCH 29/69] qfile-daemon-dvm: Call static method by class name

---
 dispvm/qfile-daemon-dvm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dispvm/qfile-daemon-dvm b/dispvm/qfile-daemon-dvm
index c2ae9898..c79e82e1 100755
--- a/dispvm/qfile-daemon-dvm
+++ b/dispvm/qfile-daemon-dvm
@@ -187,6 +187,6 @@ def main():
         subprocess.call(['/usr/lib/qubes/qrexec-client', '-d', dispvm.name,
                          user+':exec /usr/lib/qubes/qubes-rpc-multiplexer ' +
                          exec_index + " " + src_vmname])
-        qfile.finish_disposable(dispvm.name)
+        QfileDaemonDvm.finish_disposable(dispvm.name)
 
 main()

From b964e8c33fd2a23e972022bb042fc72d006f30c2 Mon Sep 17 00:00:00 2001
From: Rusty Bird <rustybird@openmailbox.org>
Date: Tue, 28 Jun 2016 13:36:16 +0000
Subject: [PATCH 30/69] qfile-daemon-dvm: Implement LAUNCH and FINISH actions

If the action is LAUNCH instead of qubes.SomeService, then just start
the dispVM, write (only) its name to stdout, and quit.

If the action is FINISH, then kill and remove the named dispVM.
---
 dispvm/qfile-daemon-dvm | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/dispvm/qfile-daemon-dvm b/dispvm/qfile-daemon-dvm
index c79e82e1..209e2228 100755
--- a/dispvm/qfile-daemon-dvm
+++ b/dispvm/qfile-daemon-dvm
@@ -171,6 +171,10 @@ class QfileDaemonDvm:
 
 def main():
     exec_index = sys.argv[1]
+    if exec_index == "FINISH":
+        QfileDaemonDvm.finish_disposable(sys.argv[2])
+        return
+
     src_vmname = sys.argv[2]
     user = sys.argv[3]
     # accessed directly by get_dvm()
@@ -183,6 +187,10 @@ def main():
     qfile = QfileDaemonDvm(src_vmname)
     dispvm = qfile.get_dvm()
     if dispvm is not None:
+        if exec_index == "LAUNCH":
+            print dispvm.name
+            return
+
         print >>sys.stderr, "time=%s, starting VM process" % (str(time.time()))
         subprocess.call(['/usr/lib/qubes/qrexec-client', '-d', dispvm.name,
                          user+':exec /usr/lib/qubes/qubes-rpc-multiplexer ' +

From ae2194da3bd4e903190501fe46152d44201c8926 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 29 Jun 2016 23:50:52 +0200
Subject: [PATCH 31/69] tests: one more place to add xdotool --sync

---
 tests/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/__init__.py b/tests/__init__.py
index 67ceeaee..e01e2502 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -489,7 +489,7 @@ class SystemTestsMixin(object):
         # accessing window properties
         self.wait_for_window(title)
         command = ['xdotool', 'search', '--name', title,
-                   'windowactivate',
+                   'windowactivate', '--sync',
                    'key'] + keys
         subprocess.check_call(command)
 

From f6bc97d65b988acefcd1f32e8b66ae5489ba640e Mon Sep 17 00:00:00 2001
From: HW42 <hw42@ipsumj.de>
Date: Wed, 29 Jun 2016 23:18:50 +0200
Subject: [PATCH 32/69] save pci_e820_host property

---
 core-modules/000QubesVm.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index a03585a2..de308075 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -202,7 +202,8 @@ class QubesVm(object):
             'kernelopts', 'services', 'installed_by_rpm',\
             'uses_default_netvm', 'include_in_backups', 'debug',\
             'qrexec_timeout', 'autostart', 'uses_default_dispvm_netvm',
-            'backup_content', 'backup_size', 'backup_path', 'pool_name' ]:
+            'backup_content', 'backup_size', 'backup_path', 'pool_name',\
+            'pci_e820_host']:
             attrs[prop]['save'] = lambda prop=prop: str(getattr(self, prop))
         # Simple paths
         for prop in ['conf_file', 'firewall_conf']:

From 8c7f072461614bf29dcba11ffe003e9dabe37018 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 1 Jul 2016 00:47:32 +0200
Subject: [PATCH 33/69] core: fix handling vm.start_time for just shutdown VM

That xenstore entry may be already removed even when libvirt still
reports the VM as running.

Fix QubesOS/qubes-issues#2127
---
 core-modules/000QubesVm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index de308075..d3b48676 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -967,7 +967,7 @@ class QubesVm(object):
         uuid = self.uuid
 
         start_time = vmm.xs.read('', "/vm/%s/start_time" % str(uuid))
-        if start_time != '':
+        if start_time:
             return datetime.datetime.fromtimestamp(float(start_time))
         else:
             return None

From 78437f7012475aa7bdb8c1e327e333b28ace20e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 1 Jul 2016 03:08:42 +0200
Subject: [PATCH 34/69] qvm-ls: remove unused code

---
 qvm-tools/qvm-ls | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/qvm-tools/qvm-ls b/qvm-tools/qvm-ls
index 26767890..ce8e8609 100755
--- a/qvm-tools/qvm-ls
+++ b/qvm-tools/qvm-ls
@@ -244,11 +244,6 @@ def main():
 
     #Nicely formatted header only needed for humans
     if not arguments.raw_data:
-        # XXX: For what?
-        total_width = 0;
-        for f in fields_to_display:
-            total_width += fields[f]["max_width"]
-
         # Display the header
         s = ""
         for f in fields_to_display:

From 7e9c816b7bb4dfeed4eae3e6f00ff76cfd596302 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 1 Jul 2016 12:11:13 +0200
Subject: [PATCH 35/69] qubeswatch: use always "dom0" name for dom0

Libvirt reports dom0 as "Domain-0". Which is incompatible with how Qubes
and libxl toolstack names it ("dom0"). So handle this as a special case.
Otherwise reconnection retries leaks event object every iteration.

Fixes QubesOS/qubes-issues#860
Thanks @alex-mazzariol for help with debugging!
---
 core/qubesutils.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/core/qubesutils.py b/core/qubesutils.py
index b05ddbc5..b858c423 100644
--- a/core/qubesutils.py
+++ b/core/qubesutils.py
@@ -729,6 +729,10 @@ class QubesWatch(object):
         return '/local/domain/%s/memory/meminfo' % xid
 
     def _register_watches(self, libvirt_domain):
+        if libvirt_domain and libvirt_domain.ID() == 0:
+            # don't use libvirt object for dom0, to always have the same
+            # hardcoded "dom0" name
+            libvirt_domain = None
         if libvirt_domain:
             name = libvirt_domain.name()
             if name in self._qdb:
@@ -769,7 +773,10 @@ class QubesWatch(object):
         self._register_watches(libvirt_domain)
 
     def _unregister_watches(self, libvirt_domain):
-        name = libvirt_domain.name()
+        if libvirt_domain and libvirt_domain.ID() == 0:
+            name = "dom0"
+        else:
+            name = libvirt_domain.name()
         if name in self._qdb_events:
             libvirt.virEventRemoveHandle(self._qdb_events[name])
             del(self._qdb_events[name])

From da74d75e6bdaece371fd4702946559d4f4f59479 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 12 Jul 2016 06:24:07 +0200
Subject: [PATCH 36/69] core: collect stderr too when vm.run_service is called
 with passio_popen

Give access to all file descriptors when requested.
---
 core-modules/000QubesVm.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index d3b48676..dda909c9 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -1692,13 +1692,14 @@ class QubesVm(object):
                             localcmd=localcmd, user=user, wait=wait, gui=gui)
         elif input:
             p = self.run("QUBESRPC %s %s" % (service, source),
-                user=user, wait=wait, gui=gui, passio_popen=True)
+                user=user, wait=wait, gui=gui, passio_popen=True,
+                passio_stderr=True)
             p.communicate(input)
             return p.returncode
         else:
             return self.run("QUBESRPC %s %s" % (service, source),
                             passio_popen=passio_popen, user=user, wait=wait,
-                            gui=gui)
+                            gui=gui, passio_stderr=passio_popen)
 
     def attach_network(self, verbose = False, wait = True, netvm = None):
         self.log.debug('attach_network(netvm={!r})'.format(netvm))

From 69e792d82c1756526643ccac96825904c3ecad48 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 12 Jul 2016 06:26:25 +0200
Subject: [PATCH 37/69] usb_attach: improve error reporting

- specifically report when qubes-usb-proxy package is not installed
- include sanitized stderr output in error message
---
 core/qubesutils.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/core/qubesutils.py b/core/qubesutils.py
index b858c423..1f1a4f45 100644
--- a/core/qubesutils.py
+++ b/core/qubesutils.py
@@ -51,6 +51,9 @@ BLKSIZE = 512
 AVAILABLE_FRONTENDS = ['xvd'+c for c in
                        string.lowercase[8:]+string.lowercase[:8]]
 
+class USBProxyNotInstalled(QubesException):
+    pass
+
 def mbytes_to_kmg(size):
     if size > 1024:
         return "%d GiB" % (size/1024)
@@ -620,9 +623,14 @@ def usb_attach(qvmc, vm, device, auto_detach=False, wait=True):
         p = vm.run_service('qubes.USBAttach', passio_popen=True, user='root')
         (stdout, stderr) = p.communicate(
             '{} {}\n'.format(device['vm'].name, device['device']))
-        if p.returncode != 0:
+        if p.returncode == 127:
+            raise USBProxyNotInstalled(
+                "qubes-usb-proxy not installed in the VM")
+        elif p.returncode != 0:
             # TODO: sanitize and include stdout
-            raise QubesException('Device attach failed')
+            sanitized_stderr = ''.join([c for c in stderr if ord(c) >= 0x20])
+            raise QubesException('Device attach failed: {}'.format(
+                sanitized_stderr))
     finally:
         # FIXME: there is a race condition here - some other process might
         # modify the file in the meantime. This may result in unexpected

From bed5f5d88c3d9008658073943c341136a988e3bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 12 Jul 2016 06:28:01 +0200
Subject: [PATCH 38/69] tests: regression test for #1990

QubesOS/qubes-issues#1990
---
 tests/network.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/tests/network.py b/tests/network.py
index eeab8053..536f6b70 100644
--- a/tests/network.py
+++ b/tests/network.py
@@ -334,6 +334,21 @@ class VmNetworkingMixin(qubes.tests.SystemTestsMixin):
         self.assertNotEqual(self.run_cmd(self.testvm1, self.ping_ip), 0,
                          "Spoofed ping should be blocked")
 
+    def test_100_late_xldevd_startup(self):
+        """Regression test for #1990"""
+        self.qc.unlock_db()
+        # Simulater late xl devd startup
+        cmd = "systemctl stop xendriverdomain"
+        if self.run_cmd(self.testnetvm, cmd) != 0:
+            self.fail("Command '%s' failed" % cmd)
+        self.testvm1.start()
+
+        cmd = "systemctl start xendriverdomain"
+        if self.run_cmd(self.testnetvm, cmd) != 0:
+            self.fail("Command '%s' failed" % cmd)
+
+        self.assertEqual(self.run_cmd(self.testvm1, self.ping_ip), 0)
+
 class VmUpdatesMixin(qubes.tests.SystemTestsMixin):
     """
     Tests for VM updates

From 4072914b401debba0b8a2ee4341e0d0591454920 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 12 Jul 2016 06:28:36 +0200
Subject: [PATCH 39/69] version 3.2.5

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index 351227fc..5ae69bd5 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.2.4
+3.2.5

From f15ddf78018a3f3ccf949a4c8981ecee6f31260b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Thu, 14 Jul 2016 02:06:08 +0200
Subject: [PATCH 40/69] backup: fix restoring on system without default netvm

Fixes QubesOS/qubes-issues#2168
---
 core/backup.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/core/backup.py b/core/backup.py
index 1f774122..9c455e24 100644
--- a/core/backup.py
+++ b/core/backup.py
@@ -1718,8 +1718,9 @@ def restore_info_verify(restore_info, host_collection):
                 if not (netvm_name in restore_info.keys() and
                         restore_info[netvm_name]['vm'].is_netvm()):
                     if options['use-default-netvm']:
-                        vm_info['netvm'] = host_collection \
-                            .get_default_netvm().name
+                        default_netvm = host_collection.get_default_netvm()
+                        vm_info['netvm'] = default_netvm.name if \
+                            default_netvm else None
                         vm_info['vm'].uses_default_netvm = True
                     elif options['use-none-netvm']:
                         vm_info['netvm'] = None

From 1d625b3570d4f3a225436ed396e7b4366bfa81f4 Mon Sep 17 00:00:00 2001
From: Andrew David Wong <adw@qubes-os.org>
Date: Fri, 15 Jul 2016 16:20:11 -0700
Subject: [PATCH 41/69] Revise help and stderr messages

---
 qvm-tools/qvm-backup | 61 ++++++++++++++++++++++++--------------------
 1 file changed, 34 insertions(+), 27 deletions(-)

diff --git a/qvm-tools/qvm-backup b/qvm-tools/qvm-backup
index 086f22b9..14791219 100755
--- a/qvm-tools/qvm-backup
+++ b/qvm-tools/qvm-backup
@@ -41,39 +41,41 @@ def main():
 
     parser.add_option ("-x", "--exclude", action="append",
                        dest="exclude_list", default=[],
-                       help="Exclude the specified VM from backup (may be "
+                       help="Exclude the specified VM from the backup (may be "
                             "repeated)")
     parser.add_option ("--force-root", action="store_true", dest="force_root", default=False,
-                       help="Force to run, even with root privileges")
+                       help="Force to run with root privileges")
     parser.add_option ("-d", "--dest-vm", action="store", dest="appvm",
-                       help="The AppVM to send backups to (implies -e)")
+                       help="Specify the destination VM to which the backup "
+                            "will be sent (implies -e)")
     parser.add_option ("-e", "--encrypt", action="store_true", dest="encrypt", default=False,
-                       help="Encrypts the backup")
+                       help="Encrypt the backup")
     parser.add_option ("--no-encrypt", action="store_true",
                        dest="no_encrypt", default=False,
-                       help="Skip encryption even if sending the backup to VM")
+                       help="Skip encryption even if sending the backup to a "
+                            "VM")
     parser.add_option ("-p", "--passphrase-file", action="store",
                        dest="pass_file", default=None,
-                       help="File containing the pass phrase to use, or '-' "
-                            "to read it from stdin")
+                       help="Read passphrase from a file, or use '-' to read "
+                            "from stdin")
     parser.add_option ("-E", "--enc-algo", action="store",
                        dest="crypto_algorithm", default=None,
-                       help="Specify non-default encryption algorithm. For "
-                            "list of supported algos execute 'openssl "
+                       help="Specify a non-default encryption algorithm. For a "
+                            "list of supported algorithms, execute 'openssl "
                             "list-cipher-algorithms' (implies -e)")
     parser.add_option ("-H", "--hmac-algo", action="store",
                        dest="hmac_algorithm", default=None,
-                       help="Specify non-default hmac algorithm. For list of "
-                            "supported algos execute 'openssl "
+                       help="Specify a non-default HMAC algorithm. For a list "
+                            "of supported algorithms, execute 'openssl "
                             "list-message-digest-algorithms'")
     parser.add_option ("-z", "--compress", action="store_true", dest="compress", default=False,
                        help="Compress the backup")
     parser.add_option ("-Z", "--compress-filter", action="store",
                        dest="compress_filter", default=False,
-                       help="Compress the backup using specified filter "
-                            "program (default: gzip)")
+                       help="Specify a non-default compression filter program "
+                            "(default: gzip)")
     parser.add_option("--tmpdir", action="store", dest="tmpdir", default=None,
-                      help="Custom temporary directory (if you have at least "
+                      help="Specify a temporary directory (if you have at least "
                            "1GB free RAM in dom0, use of /tmp is advised) ("
                            "default: /var/tmp)")
     parser.add_option ("--debug", action="store_true", dest="debug",
@@ -82,17 +84,21 @@ def main():
     (options, args) = parser.parse_args ()
 
     if (len (args) < 1):
-        print >> sys.stderr, "You must specify the target backup directory (e.g. /mnt/backup)"
-        print >> sys.stderr, "qvm-backup will create a subdirectory there for each individual backup."
+        print >> sys.stderr, "You must specify the target backup directory "\
+                             " (e.g. /mnt/backup)."
+        print >> sys.stderr, "qvm-backup will create a subdirectory there for "\
+                             " each individual backup."
         exit (0)
 
     base_backup_dir = args[0]
 
     if hasattr(os, "geteuid") and os.geteuid() == 0:
         if not options.force_root:
-            print >> sys.stderr, "*** Running this tool as root is strongly discouraged, this will lead you in permissions problems."
-            print >> sys.stderr, "Retry as unprivileged user."
-            print >> sys.stderr, "... or use --force-root to continue anyway."
+            print >> sys.stderr, "*** Running this tool as root is strongly "\
+                                 "discouraged. This will lead to permissions "\
+                                 "problems."
+            print >> sys.stderr, "Retry as an unprivileged user, or use "\
+                                 "--force-root to continue anyway."
             exit(1)
 
     # Only for locking
@@ -136,14 +142,15 @@ def main():
         backup_fs_free_sz = stat.f_bsize * stat.f_bavail
         print
         if (total_backup_sz > backup_fs_free_sz):
-            print >>sys.stderr, "ERROR: Not enough space available on the backup filesystem!"
+            print >>sys.stderr, "ERROR: Not enough space available on the "\
+                                "backup filesystem!"
             exit(1)
 
         print "-> Available space: {0}".format(size_to_human(backup_fs_free_sz))
     else:
         appvm = qvm_collection.get_vm_by_name(options.appvm)
         if appvm is None:
-            print >>sys.stderr, "ERROR: VM {0} does not exist".format(options.appvm)
+            print >>sys.stderr, "ERROR: VM {0} does not exist!".format(options.appvm)
             exit(1)
 
         stat = os.statvfs('/var/tmp')
@@ -151,19 +158,19 @@ def main():
         print
         if (backup_fs_free_sz < 1000000000):
             print >>sys.stderr, "ERROR: Not enough space available " \
-                "on the local filesystem (needs 1GB for temporary files)!"
+                "on the local filesystem (1GB required for temporary files)!"
             exit(1)
 
         if not appvm.is_running():
             appvm.start(verbose=True)
 
     if options.appvm:
-        print >>sys.stderr, ("WARNING: VM {} excluded because it's used to "
-                            "store the backup.").format(options.appvm)
+        print >>sys.stderr, ("NOTE: VM {} will be excluded because it is "
+                             "the backup destination.").format(options.appvm)
         options.exclude_list.append(options.appvm)
 
     if not options.encrypt:
-        print >>sys.stderr, "WARNING: encryption will not be used"
+        print >>sys.stderr, "WARNING: The backup will NOT be encrypted!"
 
     if options.pass_file is not None:
         f = open(options.pass_file) if options.pass_file != "-" else sys.stdin
@@ -175,12 +182,12 @@ def main():
         if raw_input("Do you want to proceed? [y/N] ").upper() != "Y":
             exit(0)
 
-        s = ("Please enter the pass phrase that will be used to {}verify "
+        s = ("Please enter the passphrase that will be used to {}verify "
              "the backup: ").format('encrypt and ' if options.encrypt else '')
         passphrase = getpass.getpass(s)
 
         if getpass.getpass("Enter again for verification: ") != passphrase:
-            print >>sys.stderr, "ERROR: Password mismatch"
+            print >>sys.stderr, "ERROR: Passphrase mismatch!"
             exit(1)
 
     encoding = sys.stdin.encoding or getpreferredencoding()

From 1cb0f384fd3a2ca20ab9ce8421272a366efb801e Mon Sep 17 00:00:00 2001
From: Andrew David Wong <adw@qubes-os.org>
Date: Fri, 15 Jul 2016 16:29:01 -0700
Subject: [PATCH 42/69] Revise help and stderr messages

---
 qvm-tools/qvm-backup-restore | 118 ++++++++++++++++++++++-------------
 1 file changed, 75 insertions(+), 43 deletions(-)

diff --git a/qvm-tools/qvm-backup-restore b/qvm-tools/qvm-backup-restore
index 00ad7ba4..4a5cab47 100755
--- a/qvm-tools/qvm-backup-restore
+++ b/qvm-tools/qvm-backup-restore
@@ -43,27 +43,31 @@ def main():
 
     parser.add_option ("--verify-only", action="store_true",
                        dest="verify_only", default=False,
-                       help="Do not restore the data, only verify backup "
-                            "integrify.")
+                       help="Verify backup integrity without restoring any "
+                            "data")
 
     parser.add_option ("--skip-broken", action="store_true", dest="skip_broken", default=False,
-                       help="Do not restore VMs that have missing templates or netvms")
+                       help="Do not restore VMs that have missing TemplateVMs "
+                            "or NetVMs")
 
     parser.add_option ("--ignore-missing", action="store_true", dest="ignore_missing", default=False,
-                       help="Ignore missing templates or netvms, restore VMs anyway")
+                       help="Restore VMs even if their associated TemplateVMs "
+                            "and NetVMs are missing")
 
     parser.add_option ("--skip-conflicting", action="store_true", dest="skip_conflicting", default=False,
-                       help="Do not restore VMs that are already present on the host")
+                       help="Do not restore VMs that are already present on "
+                            "the host")
 
     parser.add_option ("--rename-conflicting", action="store_true",
                        dest="rename_conflicting", default=False,
-                       help="Restore VMs that are already present on the host under different name")
+                       help="Restore VMs that are already present on the host "
+                            "under different names")
 
     parser.add_option ("--force-root", action="store_true", dest="force_root", default=False,
-                       help="Force to run, even with root privileges")
+                       help="Force to run with root privileges")
 
     parser.add_option ("--replace-template", action="append", dest="replace_template", default=[],
-                       help="Restore VMs using another template, syntax: "
+                       help="Restore VMs using another TemplateVM; syntax: "
                             "old-template-name:new-template-name (may be "
                             "repeated)")
 
@@ -71,20 +75,21 @@ def main():
                        help="Skip restore of specified VM (may be repeated)")
 
     parser.add_option ("--skip-dom0-home", action="store_false", dest="dom0_home", default=True,
-                       help="Do not restore dom0 user home dir")
+                       help="Do not restore dom0 user home directory")
 
     parser.add_option ("--ignore-username-mismatch", action="store_true", dest="ignore_username_mismatch", default=False,
-                       help="Ignore dom0 username mismatch while restoring homedir")
+                       help="Ignore dom0 username mismatch when restoring home "
+                            "directory")
 
     parser.add_option ("-d", "--dest-vm", action="store", dest="appvm",
-                       help="The AppVM to send backups to")
+                       help="Specify VM containing the backup to be restored")
 
     parser.add_option ("-e", "--encrypted", action="store_true", dest="decrypt", default=False,
                        help="The backup is encrypted")
 
     parser.add_option ("-p", "--passphrase-file", action="store",
                        dest="pass_file", default=None,
-                       help="File containing the pass phrase to use, or '-' to read it from stdin")
+                       help="Read passphrase from file, or use '-' to read from stdin")
 
     parser.add_option ("-z", "--compressed", action="store_true", dest="compressed", default=False,
                        help="The backup is compressed")
@@ -95,7 +100,8 @@ def main():
     (options, args) = parser.parse_args ()
 
     if (len (args) < 1):
-        print >> sys.stderr, "You must specify the backup directory (e.g. /mnt/backup/qubes-2010-12-01-235959)"
+        print >> sys.stderr, "You must specify the backup directory "\
+                             "(e.g. /mnt/backup/qubes-2010-12-01-235959)"
         exit (0)
 
     backup_dir = args[0]
@@ -141,7 +147,8 @@ def main():
         if f is not sys.stdin:
             f.close()
     else:
-        passphrase = getpass.getpass("Please enter the pass phrase to decrypt/verify the backup: ")
+        passphrase = getpass.getpass("Please enter the passphrase to verify "
+                                     "and (if encrypted) decrypt the backup: ")
 
     encoding = sys.stdin.encoding or getpreferredencoding()
     passphrase = passphrase.decode(encoding)
@@ -197,67 +204,92 @@ def main():
     print
 
     if hasattr(os, "geteuid") and os.geteuid() == 0:
-        print >> sys.stderr, "*** Running this tool as root is strongly discouraged, this will lead you in permissions problems."
+        print >> sys.stderr, "*** Running this tool as root is strongly "\
+                             "discouraged. This will lead to permissions "\
+                             "problems."
         if options.force_root:
-            print >> sys.stderr, "Continuing as commanded. You have been warned."
+            print >> sys.stderr, "Continuing as commanded. You have been "\
+                                 "warned."
         else:
-            print >> sys.stderr, "Retry as unprivileged user."
-            print >> sys.stderr, "... or use --force-root to continue anyway."
+            print >> sys.stderr, "Retry as an unprivileged user, or use "\
+                                 "--force-root to continue anyway."
             exit(1)
 
     if there_are_conflicting_vms:
-        print >> sys.stderr, "*** There VMs with conflicting names on the host! ***"
+        print >> sys.stderr, "*** There are VMs with conflicting names on the "\
+                             "host! ***"
         if options.skip_conflicting:
-            print >> sys.stderr, "Those VMs will not be restored, the host VMs will not be overwritten!"
+            print >> sys.stderr, "Those VMs will not be restored. The host "\
+                                 "VMs will NOT be overwritten."
         else:
-            print >> sys.stderr, "Remove VMs with conflicting names from the host before proceeding."
-            print >> sys.stderr, "... or use --skip-conflicting to restore only those VMs that do not exist on the host."
-            print >> sys.stderr, "... or use --rename-conflicting to " \
-                                 "restore those VMs under modified " \
-                                 "name (with number at the end)"
+            print >> sys.stderr, "Remove VMs with conflicting names from the "\
+                                 "host before proceeding."
+            print >> sys.stderr, "Or use --skip-conflicting to restore only "\
+                                 "those VMs that do not exist on the host."
+            print >> sys.stderr, "Or use --rename-conflicting to restore " \
+                                 "those VMs under modified names (with "\
+                                 "numbers at the end)."
             exit (1)
 
     print "The above VMs will be copied and added to your system."
-    print "Exisiting VMs will not be removed."
+    print "Exisiting VMs will NOT be removed."
 
     if there_are_missing_templates:
-        print >> sys.stderr, "*** One or more template VM is missing on the host! ***"
+        print >> sys.stderr, "*** One or more TemplateVMs are missing on the"\
+                             "host! ***"
         if not (options.skip_broken or options.ignore_missing):
-            print >> sys.stderr, "Install it first, before proceeding with backup restore."
-            print >> sys.stderr, "Or pass: --skip-broken or --ignore-missing switch."
+            print >> sys.stderr, "Install them before proceeding with the "\
+                                 "restore."
+            print >> sys.stderr, "Or pass: --skip-broken or --ignore-missing."
             exit (1)
         elif options.skip_broken:
-            print >> sys.stderr, "... VMs that depend on it will not be restored (--skip-broken used)"
+            print >> sys.stderr, "Skipping broken entries: VMs that depend on "\
+                                 "missing TemplateVMs will NOT be restored."
         elif options.ignore_missing:
-            print >> sys.stderr, "... VMs that depend on it will be restored anyway (--ignore-missing used)"
+            print >> sys.stderr, "Ignoring missing entries: VMs that depend "\
+                                 "on missing TemplateVMs will NOT be restored."
         else:
-            print >> sys.stderr, "INTERNAL ERROR?!"
+            print >> sys.stderr, "INTERNAL ERROR! Please report this to the "\
+                                 "Qubes OS team!"
             exit (1)
 
     if there_are_missing_netvms:
-        print >> sys.stderr, "*** One or more network VM is missing on the host! ***"
+        print >> sys.stderr, "*** One or more NetVMs are missing on the "\
+                             "host! ***"
         if not (options.skip_broken or options.ignore_missing):
-            print >> sys.stderr, "Install it first, before proceeding with backup restore."
-            print >> sys.stderr, "Or pass: --skip_broken or --ignore_missing switch."
+            print >> sys.stderr, "Install them before proceeding with the "\
+                                 "restore."
+            print >> sys.stderr, "Or pass: --skip-broken or --ignore-missing."
             exit (1)
         elif options.skip_broken:
-            print >> sys.stderr, "... VMs that depend on it will not be restored (--skip-broken used)"
+            print >> sys.stderr, "Skipping broken entries: VMs that depend on "\
+                                 "missing NetVMs will NOT be restored."
         elif options.ignore_missing:
-            print >> sys.stderr, "... VMs that depend on it be restored anyway (--ignore-missing used)"
+            print >> sys.stderr, "Ignoring missing entries: VMs that depend "\
+                                 "on missing NetVMs will NOT be restored."
         else:
-            print >> sys.stderr, "INTERNAL ERROR?!"
+            print >> sys.stderr, "INTERNAL ERROR! Please report this to the "\
+                                 "Qubes OS team!"
             exit (1)
 
     if 'dom0' in restore_info.keys() and options.dom0_home:
         if dom0_username_mismatch:
-            print >> sys.stderr, "*** Dom0 username mismatch! This can break some settings ***"
+            print >> sys.stderr, "*** Dom0 username mismatch! This can break "\
+                                 "some settings! ***"
             if not options.ignore_username_mismatch:
-                print >> sys.stderr, "Skip dom0 home restore (--skip-dom0-home)"
-                print >> sys.stderr, "Or pass: --ignore-username-mismatch to continue anyway"
+                print >> sys.stderr, "Skip restoring the dom0 home directory "\
+                                     "(--skip-dom0-home), or pass "\
+                                     "--ignore-username-mismatch to continue "\
+                                     "anyway."
                 exit(1)
             else:
-                print >> sys.stderr, "Continuing as directed"
-        print >> sys.stderr, "While restoring user homedir, existing files/dirs will be backed up in 'home-pre-restore-<current-time>' dir"
+                print >> sys.stderr, "Continuing as directed."
+        print >> sys.stderr, "NOTE: Before restoring the dom0 home directory, "
+                             "a new directory named "\
+                             "'home-pre-restore-<current-time>' will be "\
+                             "created inside the dom0 home directory, and "\
+                             "all the other existing contents of the dom0 "\
+                             "home directory will be moved there."
 
     if options.pass_file is None:
         if raw_input("Do you want to proceed? [y/N] ").upper() != "Y":

From 521e96f2c3c14aace59d19651a61556efe20d036 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 16 Jul 2016 21:10:29 +0200
Subject: [PATCH 43/69] Revert "core: detach PCI devices before shutting down
 VM"

Many drivers, including iwlwifi doesn't handle this well, resulting in
oopses etc. Also we're disabling PCI hotplug, which may be result in
more troubles here.
This reverts commit 2658c9a6e69ed272f49e71e04b5b342ea2dc388f.

QubesOS/qubes-issues#1673
---
 core-modules/000QubesVm.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index dda909c9..3c0760a6 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -2057,15 +2057,6 @@ class QubesVm(object):
         if not self.is_running():
             raise QubesException ("VM already stopped!")
 
-        # try to gracefully detach PCI devices before shutdown, to mitigate
-        # timeouts on forcible detach at domain destroy; if that fails, too bad
-        try:
-            for pcidev in self.pcidevs:
-                self.libvirt_domain.detachDevice(self._format_pci_dev(pcidev))
-        except libvirt.libvirtError as e:
-            print >>sys.stderr, "WARNING: {}, continuing VM shutdown " \
-                                "anyway".format(str(e))
-
         self.libvirt_domain.shutdown()
 
     def force_shutdown(self, xid = None):

From 3427621f435b4f0a45d8f6e3af958eee1e0fffcd Mon Sep 17 00:00:00 2001
From: Andrew David Wong <adw@qubes-os.org>
Date: Sat, 16 Jul 2016 18:26:15 -0700
Subject: [PATCH 44/69] Correct note regarding dom0 home-pre-restore directory

---
 qvm-tools/qvm-backup-restore | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/qvm-tools/qvm-backup-restore b/qvm-tools/qvm-backup-restore
index 4a5cab47..1881fa6c 100755
--- a/qvm-tools/qvm-backup-restore
+++ b/qvm-tools/qvm-backup-restore
@@ -287,9 +287,10 @@ def main():
         print >> sys.stderr, "NOTE: Before restoring the dom0 home directory, "
                              "a new directory named "\
                              "'home-pre-restore-<current-time>' will be "\
-                             "created inside the dom0 home directory, and "\
-                             "all the other existing contents of the dom0 "\
-                             "home directory will be moved there."
+                             "created inside the dom0 home directory. If any "\
+                             "restored files conflict with existing files, "\
+                             "the existing files will be moved to this new "\
+                             "directory."
 
     if options.pass_file is None:
         if raw_input("Do you want to proceed? [y/N] ").upper() != "Y":

From c5d03cc32e17cb474f13bebf51f909bbf5fe532d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 16 Jul 2016 21:23:58 +0200
Subject: [PATCH 45/69] backup: ask user to update templates after restoring
 backup

To install all the fixes and to update applications + icons.

Fixes QubesOS/qubes-issues#2150
---
 core/backup.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/core/backup.py b/core/backup.py
index 9c455e24..390457b3 100644
--- a/core/backup.py
+++ b/core/backup.py
@@ -2305,6 +2305,10 @@ def backup_restore_do(restore_info,
         if retcode != 0:
             error_callback("*** Error while setting home directory owner")
 
+    if callable(print_callback):
+        print_callback("-> Done. Please install updates for all the restored "
+                       "templates.")
+
     shutil.rmtree(restore_tmpdir)
 
 # vim:sw=4:et:

From 8222324146b948f6556d138c5bc76c9e20568fe7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 16 Jul 2016 21:49:16 +0200
Subject: [PATCH 46/69] backup: restore 'services' property

Fixes QubesOS/qubes-issues#2106
---
 core/backup.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/core/backup.py b/core/backup.py
index 390457b3..8e05c165 100644
--- a/core/backup.py
+++ b/core/backup.py
@@ -2236,6 +2236,13 @@ def backup_restore_do(restore_info,
                 error_callback("ERROR: {0}".format(err))
                 error_callback("*** Some VM property will not be restored")
 
+            try:
+                for service, value in vm.services.items():
+                    new_vm.services[service] = value
+            except Exception as err:
+                error_callback("ERROR: {0}".format(err))
+                error_callback("*** Some VM property will not be restored")
+
             try:
                 new_vm.appmenus_create(verbose=callable(print_callback))
             except Exception as err:

From 6ccae83956fcfc76e87a8486df01bfe5f98940a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 16 Jul 2016 21:50:24 +0200
Subject: [PATCH 47/69] tests/backup: verify restored VM properties

QubesOS/qubes-issues#2106
---
 tests/__init__.py |  3 +++
 tests/backup.py   | 26 ++++++++++++++++++++++++++
 2 files changed, 29 insertions(+)

diff --git a/tests/__init__.py b/tests/__init__.py
index e01e2502..0e6891d0 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -639,6 +639,7 @@ class BackupTestsMixin(SystemTestsMixin):
         testnet = self.qc.add_new_vm('QubesNetVm',
             name=vmname, template=template)
         testnet.create_on_disk(verbose=self.verbose)
+        testnet.services['ntpd'] = True
         vms.append(testnet)
         self.fill_image(testnet.private_img, 20*1024*1024)
 
@@ -657,6 +658,8 @@ class BackupTestsMixin(SystemTestsMixin):
         if self.verbose:
             print >>sys.stderr, "-> Creating %s" % vmname
         testvm2 = self.qc.add_new_vm('QubesHVm', name=vmname)
+        # fixup - uses_default_netvm=True anyway
+        testvm2.netvm = self.qc.get_default_netvm()
         testvm2.create_on_disk(verbose=self.verbose)
         self.fill_image(testvm2.root_img, 1024*1024*1024, True)
         vms.append(testvm2)
diff --git a/tests/backup.py b/tests/backup.py
index 69b5c597..554f8dc8 100644
--- a/tests/backup.py
+++ b/tests/backup.py
@@ -37,6 +37,32 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
         self.make_backup(vms)
         self.remove_vms(vms)
         self.restore_backup()
+        for vm in vms:
+            restored_vm = self.qc.get_vm_by_name(vm.name)
+            for prop in ('name', 'kernel', 'uses_default_kernel',
+                    'uses_default_netvm', 'memory', 'maxmem', 'kernelopts',
+                    'uses_default_kernelopts', 'services', 'vcpus', 'pcidevs',
+                    'include_in_backups', 'default_user', 'qrexec_timeout',
+                    'autostart', 'pci_strictreset', 'pci_e820_host', 'debug',
+                    'internal'):
+                if prop not in vm.get_attrs_config():
+                    continue
+                self.assertEquals(
+                    getattr(vm, prop), getattr(restored_vm, prop),
+                    "VM {} - property {} not properly restored".format(
+                        vm.name, prop))
+            for prop in ('netvm', 'template', 'label'):
+                orig_value = getattr(vm, prop)
+                restored_value = getattr(restored_vm, prop)
+                if orig_value and restored_value:
+                    self.assertEquals(orig_value.name, restored_value.name,
+                        "VM {} - property {} not properly restored".format(
+                            vm.name, prop))
+                else:
+                    self.assertEquals(orig_value, restored_value,
+                        "VM {} - property {} not properly restored".format(
+                            vm.name, prop))
+
         self.remove_vms(vms)
 
     def test_001_compressed_backup(self):

From 6068c7bc5025efb4d7e8e3c33971119e05bb7918 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sun, 17 Jul 2016 03:52:40 +0200
Subject: [PATCH 48/69] core: fix handling uses_default_netvm property

- for netvm it doesn't make sense, but instead of removing it (which
        surely will break some code), make it always False
- when settings VM connections, uses_default_netvm is already loaded
- handle it properly during backup restore (really use default netvm,
        istead of assuming it's the same as during backup)
---
 core-modules/005QubesNetVm.py | 1 +
 core/backup.py                | 6 +++++-
 core/qubes.py                 | 7 +------
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/core-modules/005QubesNetVm.py b/core-modules/005QubesNetVm.py
index 33934f4b..904e74f1 100644
--- a/core-modules/005QubesNetVm.py
+++ b/core-modules/005QubesNetVm.py
@@ -42,6 +42,7 @@ class QubesNetVm(QubesVm):
         attrs_config['dir_path']['func'] = \
             lambda value: value if value is not None else \
                 os.path.join(system_path["qubes_servicevms_dir"], self.name)
+        attrs_config['uses_default_netvm']['func'] = lambda x: False
         attrs_config['label']['default'] = defaults["servicevm_label"]
         attrs_config['memory']['default'] = 300
 
diff --git a/core/backup.py b/core/backup.py
index 8e05c165..e9915d56 100644
--- a/core/backup.py
+++ b/core/backup.py
@@ -1706,7 +1706,11 @@ def restore_info_verify(restore_info, host_collection):
 
         # check netvm
         vm_info.pop('missing-netvm', None)
-        if vm_info['netvm']:
+        if vm_info['vm'].uses_default_netvm:
+            default_netvm = host_collection.get_default_netvm()
+            vm_info['netvm'] = default_netvm.name if \
+                default_netvm else None
+        elif vm_info['netvm']:
             netvm_name = vm_info['netvm']
 
             netvm_on_host = host_collection.get_vm_by_name(netvm_name)
diff --git a/core/qubes.py b/core/qubes.py
index 2fc9c140..f3be12d3 100755
--- a/core/qubes.py
+++ b/core/qubes.py
@@ -713,18 +713,13 @@ class QubesVmCollection(dict):
 
     def set_netvm_dependency(self, element):
         kwargs = {}
-        attr_list = ("qid", "uses_default_netvm", "netvm_qid")
+        attr_list = ("qid", "netvm_qid")
 
         for attribute in attr_list:
             kwargs[attribute] = element.get(attribute)
 
         vm = self[int(kwargs["qid"])]
 
-        if "uses_default_netvm" not in kwargs:
-            vm.uses_default_netvm = True
-        else:
-            vm.uses_default_netvm = (
-                    True if kwargs["uses_default_netvm"] == "True" else False)
         if vm.uses_default_netvm is True:
             if vm.is_proxyvm():
                 netvm = self.get_default_fw_netvm()

From 1a1ec88d29779e253d28cd68596d9188c3a65c7f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sun, 17 Jul 2016 05:16:31 +0200
Subject: [PATCH 49/69] tests: save qubes.xml befor launching qvm-block tests

---
 tests/block.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/block.py b/tests/block.py
index a40dfea7..3d9dcbac 100644
--- a/tests/block.py
+++ b/tests/block.py
@@ -39,8 +39,11 @@ class TC_00_List(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
                 name=self.make_vm_name("vm"),
                 template=self.qc.get_vm_by_name(self.template))
             self.vm.create_on_disk(verbose=False)
+            self.save_and_reload_db()
+            self.qc.unlock_db()
             self.vm.start()
         else:
+            self.qc.unlock_db()
             self.vm = self.qc[0]
 
     def tearDown(self):

From 9751981f69d11556f5b2dabc8008ff23651e7fc5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sun, 17 Jul 2016 05:17:57 +0200
Subject: [PATCH 50/69] version 3.2.6

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index 5ae69bd5..34cde569 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.2.5
+3.2.6

From b467dd62184709b5df483b23383ed9b7630abad1 Mon Sep 17 00:00:00 2001
From: HW42 <hw42@ipsumj.de>
Date: Sun, 17 Jul 2016 21:35:16 +0200
Subject: [PATCH 51/69] QubesWatch: do not create multiple dom0 QubesDB
 connections

When calling _register_watches() multiple times for dom0 (by passing
None or since 7e9c816b by passing the corresponding libvirt domain) the
check was missing if there is already a QubesDB in _qdb. Therefore a new
QubesDB was created and the old one is destroyed by the GC. As a
consequence the watch_fd is closed but the libvirt event handle for this
fd is still registered. So when libvirt calls poll() it returns
immediately POLLNVAL with the closed fd. This is not caught in libvirt
and the callback is called as if an event happened. _qdb_handler() now
calls read_watch() on the new fd for dom0 and thereby hangs the thread.
This leads (at leads) to qubes-manager to miss VM status updates and
block device events.

Fixes QubesOS/qubes-issues#2178
---
 core/qubesutils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/core/qubesutils.py b/core/qubesutils.py
index 1f1a4f45..90c2ef88 100644
--- a/core/qubesutils.py
+++ b/core/qubesutils.py
@@ -761,6 +761,8 @@ class QubesWatch(object):
                 return
         else:
             name = "dom0"
+            if name in self._qdb:
+                return
             self._qdb[name] = QubesDB(name)
         try:
             self._qdb[name].watch('/qubes-block-devices')

From c028df3e1e90a3017d0cfce34dbc6a85a8bb7b91 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 18 Jul 2016 15:07:02 +0200
Subject: [PATCH 52/69] QubesWatch: fix handling just removed domains

Really do not throw exception in such a case.
Reported by HW42.
---
 core/qubesutils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/qubesutils.py b/core/qubesutils.py
index 90c2ef88..85dad0f1 100644
--- a/core/qubesutils.py
+++ b/core/qubesutils.py
@@ -706,7 +706,8 @@ class QubesWatch(object):
                 # which can just remove the domain
                 if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN:
                     pass
-                raise
+                else:
+                    raise
         # and for dom0
         self._register_watches(None)
 

From f0489a3d5cb5455eb6e658cfa2350b8a86c2442c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 18 Jul 2016 15:24:25 +0200
Subject: [PATCH 53/69] version 3.2.7

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index 34cde569..406ebcbd 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.2.6
+3.2.7

From 9d1b7504da4c0f5d8411ec59634e3f67deda472e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 19 Jul 2016 00:46:48 +0200
Subject: [PATCH 54/69] qvm-sync-clock: allow colon in timezone spec

`date` in debian 9 puts colon there. Since the timezone is not used here
in any way (it operates on UTC time anyway), simply allow this format
too.
---
 qvm-tools/qvm-sync-clock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qvm-tools/qvm-sync-clock b/qvm-tools/qvm-sync-clock
index ef628551..4cd6ba8d 100755
--- a/qvm-tools/qvm-sync-clock
+++ b/qvm-tools/qvm-sync-clock
@@ -101,7 +101,7 @@ def main():
                          gui=False, passio_popen=True, ignore_stderr=True)
         date_out = p.stdout.read(100)
         date_out = date_out.strip()
-        if not re.match(r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+0000$', date_out):
+        if not re.match(r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+00:?00$', date_out):
             print >> sys.stderr, 'Invalid date output, aborting!'
             sys.exit(1)
 

From 9f7668af77d91bdd894596b1a34fcfd07e1eb0e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 19 Jul 2016 02:07:14 +0200
Subject: [PATCH 55/69] tests: allow LibreOffice as txt file editor

---
 tests/vm_qrexec_gui.py | 31 +++++++++++++++++++++++++------
 1 file changed, 25 insertions(+), 6 deletions(-)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index be12ab08..99926672 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -1098,13 +1098,29 @@ class TC_20_DispVMMixin(qubes.tests.SystemTestsMixin):
         time.sleep(1)
         if "gedit" in window_title:
             subprocess.check_call(['xdotool', 'windowactivate', '--sync', winid,
-                'type', 'test test 2\n'])
+                'type', 'Test test 2\n'])
             time.sleep(0.5)
             subprocess.check_call(['xdotool',
                                    'key', 'ctrl+s', 'ctrl+q'])
+        elif "LibreOffice" in window_title:
+            # wait for actual editor (we've got splash screen)
+            search = subprocess.Popen(['xdotool', 'search', '--sync',
+                '--onlyvisible', '--all', '--name', '--class', 'disp*|Writer'],
+                stdout=subprocess.PIPE,
+                                  stderr=open(os.path.devnull, 'w'))
+            retcode = search.wait()
+            if retcode == 0:
+                winid = search.stdout.read().strip()
+            time.sleep(0.5)
+            subprocess.check_call(['xdotool', 'windowactivate', '--sync', winid,
+                'type', 'Test test 2\n'])
+            time.sleep(0.5)
+            subprocess.check_call(['xdotool',
+                                   'key', '--delay', '100', 'ctrl+s',
+                'Return', 'ctrl+q'])
         elif "emacs" in window_title:
             subprocess.check_call(['xdotool', 'windowactivate', '--sync', winid,
-                                   'type', 'test test 2\n'])
+                                   'type', 'Test test 2\n'])
             time.sleep(0.5)
             subprocess.check_call(['xdotool',
                                    'key', 'ctrl+x', 'ctrl+s'])
@@ -1112,7 +1128,7 @@ class TC_20_DispVMMixin(qubes.tests.SystemTestsMixin):
                                    'key', 'ctrl+x', 'ctrl+c'])
         elif "vim" in window_title or "user@" in window_title:
             subprocess.check_call(['xdotool', 'windowactivate', '--sync', winid,
-                                   'key', 'i', 'type', 'test test 2\n'])
+                                   'key', 'i', 'type', 'Test test 2\n'])
             subprocess.check_call(
                 ['xdotool',
                  'key', 'Escape', 'colon', 'w', 'q', 'Return'])
@@ -1158,7 +1174,10 @@ class TC_20_DispVMMixin(qubes.tests.SystemTestsMixin):
         p = testvm1.run("cat /home/user/test.txt",
                         passio_popen=True)
         (test_txt_content, _) = p.communicate()
-        self.assertEqual(test_txt_content, "test test 2\ntest1\n")
+        # Drop BOM if added by editor
+        if test_txt_content.startswith('\xef\xbb\xbf'):
+            test_txt_content = test_txt_content[3:]
+        self.assertEqual(test_txt_content, "Test test 2\ntest1\n")
 
 
 class TC_30_Gui_daemon(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
@@ -1572,7 +1591,7 @@ class TC_50_MimeHandlers(qubes.tests.SystemTestsMixin):
         filename = "/home/user/test_file.txt"
         self.prepare_txt(filename)
         self.open_file_and_check_viewer(filename, ["vim", "user@"],
-                                        ["gedit", "emacs"])
+                                        ["gedit", "emacs", "libreoffice"])
 
     def test_001_pdf(self):
         filename = "/home/user/test_file.pdf"
@@ -1618,7 +1637,7 @@ class TC_50_MimeHandlers(qubes.tests.SystemTestsMixin):
         filename = "/home/user/test_file.txt"
         self.prepare_txt(filename)
         self.open_file_and_check_viewer(filename, ["vim", "user@"],
-                                        ["gedit", "emacs"],
+                                        ["gedit", "emacs", "libreoffice"],
                                         dispvm=True)
 
     def test_101_pdf_dispvm(self):

From 0ee64c74f94d31e38cb49758f0daeed4840703c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 19 Jul 2016 02:07:52 +0200
Subject: [PATCH 56/69] tests: cleanup after backup compatibility tests

Backup compat tests use 'test-' prefix (as it was initially for all the
tests. Since changing this right not may break those backups in
non-trivial way, simply add cleanup for 'test-*' VMs.
---
 tests/backupcompatibility.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tests/backupcompatibility.py b/tests/backupcompatibility.py
index 826830d2..cb7ca133 100644
--- a/tests/backupcompatibility.py
+++ b/tests/backupcompatibility.py
@@ -146,6 +146,19 @@ compression-filter=gzip
 '''
 
 class TC_00_BackupCompatibility(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
+    def tearDown(self):
+        self.qc.unlock_db()
+        self.qc.lock_db_for_writing()
+        self.qc.load()
+
+        # Remove here as we use 'test-' prefix, instead of 'test-inst-'
+        self._remove_test_vms(self.qc, self.conn, prefix="test-")
+
+        self.qc.save()
+        self.qc.unlock_db()
+
+        super(TC_00_BackupCompatibility, self).tearDown()
+
     def create_whitelisted_appmenus(self, filename):
         f = open(filename, "w")
         f.write("gnome-terminal.desktop\n")

From 6a516caee2bb9af46fdda0df596e4d87f1292ed1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 19 Jul 2016 02:11:55 +0200
Subject: [PATCH 57/69] tests: misc minor fixes

---
 tests/basic.py         | 2 +-
 tests/network.py       | 4 ++--
 tests/vm_qrexec_gui.py | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/basic.py b/tests/basic.py
index 485809ad..2bbe2696 100644
--- a/tests/basic.py
+++ b/tests/basic.py
@@ -878,7 +878,7 @@ class TC_04_DispVM(qubes.tests.SystemTestsMixin,
                              stderr=open(os.devnull, 'w'))
         p.stdin.write("qubesdb-read /name\n")
         p.stdin.write("echo ERROR\n")
-        p.stdin.write("poweroff\n")
+        p.stdin.write("sudo poweroff\n")
         # do not close p.stdin on purpose - wait to automatic disconnect when
         #  domain is destroyed
         timeout = 30
diff --git a/tests/network.py b/tests/network.py
index 536f6b70..2317f8dc 100644
--- a/tests/network.py
+++ b/tests/network.py
@@ -165,8 +165,8 @@ class VmNetworkingMixin(qubes.tests.SystemTestsMixin):
 
         # check for nm-applet presence
         self.assertEqual(subprocess.call([
-            'xdotool', 'search', '--all', '--name',
-            '--class', '^(NetworkManager Applet|{})$'.format(self.proxy.name)],
+            'xdotool', 'search', '--class', '{}:nm-applet'.format(
+                self.proxy.name)],
             stdout=open('/dev/null', 'w')), 0, "nm-applet window not found")
         self.assertEqual(self.run_cmd(self.testvm1, self.ping_ip), 0,
                          "Ping by IP failed (after NM reconnection")
diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index 99926672..c82f43e2 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -1274,7 +1274,7 @@ class TC_40_PVGrub(qubes.tests.SystemTestsMixin):
         if self.template.startswith('fedora-'):
             cmd_install1 = 'dnf clean expire-cache && ' \
                 'dnf install -y qubes-kernel-vm-support grub2-tools'
-            cmd_install2 = 'yum install -y kernel && ' \
+            cmd_install2 = 'dnf install -y kernel && ' \
                 'KVER=$(rpm -q --qf %{VERSION}-%{RELEASE}.%{ARCH} kernel) && ' \
                 'dnf install --allowerasing  -y kernel-devel-$KVER && ' \
                 'dkms autoinstall -k $KVER'

From 86a14b53fb0b482692835a8f89b3a12017de0fdc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 29 Jul 2016 11:20:21 +0200
Subject: [PATCH 58/69] qvm-run: color untrusted stderr even when stdout is
 redirected

When stdout is redirected to some file or command two things will
happen:
 - qvm-run will not automatically color the output as stdout is not a
 TTY
 - even when coloring is forced, it will not work, as the control
 sequence (on stdout) will be redirected anyway

Fix this by handling stdout and stderr independently and output color
switching sequence to each of them.

Fixes QubesOS/qubes-issues#2190
---
 qvm-tools/qvm-run | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/qvm-tools/qvm-run b/qvm-tools/qvm-run
index 83223fe2..e255fc19 100755
--- a/qvm-tools/qvm-run
+++ b/qvm-tools/qvm-run
@@ -47,7 +47,9 @@ def vm_run_cmd(vm, cmd, options):
     if options.verbose:
         print >> sys.stderr, "Running command on VM: '{0}'...".format(vm.name)
     if options.passio and options.color_output is not None:
-        print "\033[0;%dm" % options.color_output,
+        sys.stdout.write("\033[0;{}m".format(options.color_output))
+    if options.passio and options.color_stderr is not None:
+        sys.stderr.write("\033[0;{}m".format(options.color_stderr))
 
     try:
         def tray_notify_generic(level, str):
@@ -65,6 +67,8 @@ def vm_run_cmd(vm, cmd, options):
     except QubesException as err:
         if options.passio and options.color_output is not None:
             sys.stdout.write("\033[0m")
+        if options.passio and options.color_stderr is not None:
+            sys.stderr.write("\033[0m")
         if options.tray:
             tray_notify_error(str(err))
             notify_error_qubes_manager(vm.name, str(err))
@@ -73,6 +77,8 @@ def vm_run_cmd(vm, cmd, options):
     finally:
         if options.passio and options.color_output is not None:
             sys.stdout.write("\033[0m")
+        if options.passio and options.color_stderr is not None:
+            sys.stderr.write("\033[0m")
 
 def main():
     usage = "usage: %prog [options] [<vm-name>] [<cmd>]"
@@ -122,11 +128,20 @@ def main():
                       dest="color_output", default=None,
                       help="Disable marking VM output with red color")
 
+    parser.add_option("--no-color-stderr", action="store_false",
+                      dest="color_stderr", default=None,
+                      help="Disable marking VM stderr with red color")
+
     parser.add_option("--color-output", action="store", type="int",
                       dest="color_output",
                       help="Force marking VM output with given ANSI style ("
                            "use 31 for red)")
 
+    parser.add_option("--color-stderr", action="store", type="int",
+                      dest="color_stderr",
+                      help="Force marking VM stderr with given ANSI style ("
+                           "use 31 for red)")
+
     (options, args) = parser.parse_args ()
 
     if (options.passio and not options.localcmd) and options.run_on_all_running:
@@ -145,6 +160,12 @@ def main():
     elif options.color_output is False:
         options.color_output = None
 
+    if options.color_stderr is None:
+        if os.isatty(sys.stderr.fileno()) and not options.localcmd:
+            options.color_stderr = 31
+    elif options.color_stderr is False:
+        options.color_stderr = None
+
     if (options.pause or options.unpause):
         takes_cmd_argument = False
     else:

From fd5b35723244901a8474a17d1a72d228b4d47d12 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 29 Jul 2016 12:55:25 +0200
Subject: [PATCH 59/69] tests: split vm_qrexec_gui

---
 tests/Makefile         |  20 +-
 tests/__init__.py      |   4 +
 tests/basic.py         | 275 +++++---------
 tests/dispvm.py        | 431 +++++++++++++++++++++
 tests/hvm.py           | 125 ++++++
 tests/mime.py          | 354 +++++++++++++++++
 tests/pvgrub.py        | 172 +++++++++
 tests/vm_qrexec_gui.py | 842 -----------------------------------------
 8 files changed, 1196 insertions(+), 1027 deletions(-)
 create mode 100644 tests/dispvm.py
 create mode 100644 tests/hvm.py
 create mode 100644 tests/mime.py
 create mode 100644 tests/pvgrub.py

diff --git a/tests/Makefile b/tests/Makefile
index 8f01e2ad..243fe681 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -19,12 +19,22 @@ endif
 	cp basic.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
 	cp block.py $(DESTDIR)$(PYTHON_TESTSPATH)
 	cp block.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
+	cp dispvm.py $(DESTDIR)$(PYTHON_TESTSPATH)
+	cp dispvm.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
 	cp dom0_update.py $(DESTDIR)$(PYTHON_TESTSPATH)
 	cp dom0_update.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
+	cp extra.py $(DESTDIR)$(PYTHON_TESTSPATH)
+	cp extra.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
+	cp hardware.py $(DESTDIR)$(PYTHON_TESTSPATH)
+	cp hardware.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
+	cp hvm.py $(DESTDIR)$(PYTHON_TESTSPATH)
+	cp hvm.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
+	cp mime.py $(DESTDIR)$(PYTHON_TESTSPATH)
+	cp mime.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
 	cp network.py $(DESTDIR)$(PYTHON_TESTSPATH)
 	cp network.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp vm_qrexec_gui.py $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp vm_qrexec_gui.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
+	cp pvgrub.py $(DESTDIR)$(PYTHON_TESTSPATH)
+	cp pvgrub.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
 	cp regressions.py $(DESTDIR)$(PYTHON_TESTSPATH)
 	cp regressions.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
 	cp run.py $(DESTDIR)$(PYTHON_TESTSPATH)
@@ -33,7 +43,5 @@ endif
 	cp storage.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
 	cp storage_xen.py $(DESTDIR)$(PYTHON_TESTSPATH)
 	cp storage_xen.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp hardware.py $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp hardware.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp extra.py $(DESTDIR)$(PYTHON_TESTSPATH)
-	cp extra.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
+	cp vm_qrexec_gui.py $(DESTDIR)$(PYTHON_TESTSPATH)
+	cp vm_qrexec_gui.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
diff --git a/tests/__init__.py b/tests/__init__.py
index 0e6891d0..cc100bd1 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -755,7 +755,11 @@ def load_tests(loader, tests, pattern):
             'qubes.tests.basic',
             'qubes.tests.dom0_update',
             'qubes.tests.network',
+            'qubes.tests.dispvm',
             'qubes.tests.vm_qrexec_gui',
+            'qubes.tests.mime',
+            'qubes.tests.hvm',
+            'qubes.tests.pvgrub',
             'qubes.tests.backup',
             'qubes.tests.backupcompatibility',
             'qubes.tests.regressions',
diff --git a/tests/basic.py b/tests/basic.py
index 2bbe2696..c07bd272 100644
--- a/tests/basic.py
+++ b/tests/basic.py
@@ -22,6 +22,7 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 #
+from distutils import spawn
 
 import multiprocessing
 import os
@@ -699,205 +700,121 @@ class TC_03_QvmRevertTemplateChanges(qubes.tests.SystemTestsMixin,
         self.setup_hvm_template()
         self._do_test()
 
-class TC_04_DispVM(qubes.tests.SystemTestsMixin,
-                   qubes.tests.QubesTestCase):
-
-    @staticmethod
-    def get_dispvm_template_name():
-        vmdir = os.readlink('/var/lib/qubes/dvmdata/vmdir')
-        return os.path.basename(vmdir)
-
-    def test_000_firewall_propagation(self):
-        """
-        Check firewall propagation VM->DispVM, when VM have some firewall rules
-        """
-
-        # FIXME: currently qubes.xml doesn't contain this information...
-        dispvm_template_name = self.get_dispvm_template_name()
-        dispvm_template = self.qc.get_vm_by_name(dispvm_template_name)
 
+class TC_30_Gui_daemon(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
+    @unittest.skipUnless(spawn.find_executable('xdotool'),
+                         "xdotool not installed")
+    def test_000_clipboard(self):
         testvm1 = self.qc.add_new_vm("QubesAppVm",
                                      name=self.make_vm_name('vm1'),
                                      template=self.qc.get_default_template())
         testvm1.create_on_disk(verbose=False)
-        firewall = testvm1.get_firewall_conf()
-        firewall['allowDns'] = False
-        firewall['allowYumProxy'] = False
-        firewall['rules'] = [{'address': '1.2.3.4',
-                              'netmask': 24,
-                              'proto': 'tcp',
-                              'portBegin': 22,
-                              'portEnd': 22,
-                              }]
-        testvm1.write_firewall_conf(firewall)
+        testvm2 = self.qc.add_new_vm("QubesAppVm",
+                                     name=self.make_vm_name('vm2'),
+                                     template=self.qc.get_default_template())
+        testvm2.create_on_disk(verbose=False)
         self.qc.save()
         self.qc.unlock_db()
 
         testvm1.start()
+        testvm2.start()
 
-        p = testvm1.run("qvm-run --dispvm 'qubesdb-read /name; echo ERROR;"
-                        " read x'",
-                        passio_popen=True)
+        window_title = 'user@{}'.format(testvm1.name)
+        testvm1.run('zenity --text-info --editable --title={}'.format(
+            window_title))
 
-        dispvm_name = p.stdout.readline().strip()
-        self.qc.lock_db_for_reading()
-        self.qc.load()
-        self.qc.unlock_db()
-        dispvm = self.qc.get_vm_by_name(dispvm_name)
-        self.assertIsNotNone(dispvm, "DispVM {} not found in qubes.xml".format(
-            dispvm_name))
-        # check if firewall was propagated to the DispVM
-        self.assertEquals(testvm1.get_firewall_conf(),
-                          dispvm.get_firewall_conf())
-        # and only there (#1608)
-        self.assertNotEquals(dispvm_template.get_firewall_conf(),
-                             dispvm.get_firewall_conf())
-        # then modify some rule
-        firewall = dispvm.get_firewall_conf()
-        firewall['rules'] = [{'address': '4.3.2.1',
-                              'netmask': 24,
-                              'proto': 'tcp',
-                              'portBegin': 22,
-                              'portEnd': 22,
-                              }]
-        dispvm.write_firewall_conf(firewall)
-        # and check again if wasn't saved anywhere else (#1608)
-        self.assertNotEquals(dispvm_template.get_firewall_conf(),
-                             dispvm.get_firewall_conf())
-        self.assertNotEquals(testvm1.get_firewall_conf(),
-                             dispvm.get_firewall_conf())
-        p.stdin.write('\n')
+        self.wait_for_window(window_title)
+        time.sleep(0.5)
+        test_string = "test{}".format(testvm1.xid)
+
+        # Type and copy some text
+        subprocess.check_call(['xdotool', 'search', '--name', window_title,
+                               'windowactivate', '--sync',
+                               'type', '{}'.format(test_string)])
+        # second xdotool call because type --terminator do not work (SEGV)
+        # additionally do not use search here, so window stack will be empty
+        # and xdotool will use XTEST instead of generating events manually -
+        # this will be much better - at least because events will have
+        # correct timestamp (so gui-daemon would not drop the copy request)
+        subprocess.check_call(['xdotool',
+                               'key', 'ctrl+a', 'ctrl+c', 'ctrl+shift+c',
+                               'Escape'])
+
+        clipboard_content = \
+            open('/var/run/qubes/qubes-clipboard.bin', 'r').read().strip()
+        self.assertEquals(clipboard_content, test_string,
+                          "Clipboard copy operation failed - content")
+        clipboard_source = \
+            open('/var/run/qubes/qubes-clipboard.bin.source',
+                 'r').read().strip()
+        self.assertEquals(clipboard_source, testvm1.name,
+                          "Clipboard copy operation failed - owner")
+
+        # Then paste it to the other window
+        window_title = 'user@{}'.format(testvm2.name)
+        p = testvm2.run('zenity --entry --title={} > test.txt'.format(
+                        window_title), passio_popen=True)
+        self.wait_for_window(window_title)
+
+        subprocess.check_call(['xdotool', 'key', '--delay', '100',
+                               'ctrl+shift+v', 'ctrl+v', 'Return'])
         p.wait()
 
-    def test_001_firewall_propagation(self):
-        """
-        Check firewall propagation VM->DispVM, when VM have no firewall rules
-        """
+        # And compare the result
+        (test_output, _) = testvm2.run('cat test.txt',
+                                       passio_popen=True).communicate()
+        self.assertEquals(test_string, test_output.strip())
+
+        clipboard_content = \
+            open('/var/run/qubes/qubes-clipboard.bin', 'r').read().strip()
+        self.assertEquals(clipboard_content, "",
+                          "Clipboard not wiped after paste - content")
+        clipboard_source = \
+            open('/var/run/qubes/qubes-clipboard.bin.source', 'r').read(
+
+            ).strip()
+        self.assertEquals(clipboard_source, "",
+                          "Clipboard not wiped after paste - owner")
+
+class TC_05_StandaloneVM(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
+    def test_000_create_start(self):
         testvm1 = self.qc.add_new_vm("QubesAppVm",
-                                     name=self.make_vm_name('vm1'),
-                                     template=self.qc.get_default_template())
-        testvm1.create_on_disk(verbose=False)
+                                     template=None,
+                                     name=self.make_vm_name('vm1'))
+        testvm1.create_on_disk(verbose=False,
+                               source_template=self.qc.get_default_template())
         self.qc.save()
         self.qc.unlock_db()
+        testvm1.start()
+        self.assertEquals(testvm1.get_power_state(), "Running")
 
-        # FIXME: currently qubes.xml doesn't contain this information...
-        dispvm_template_name = self.get_dispvm_template_name()
-        dispvm_template = self.qc.get_vm_by_name(dispvm_template_name)
-        original_firewall = None
-        if os.path.exists(dispvm_template.firewall_conf):
-            original_firewall = tempfile.TemporaryFile()
-            with open(dispvm_template.firewall_conf) as f:
-                original_firewall.write(f.read())
-        try:
-
-            firewall = dispvm_template.get_firewall_conf()
-            firewall['allowDns'] = False
-            firewall['allowYumProxy'] = False
-            firewall['rules'] = [{'address': '1.2.3.4',
-                                  'netmask': 24,
-                                  'proto': 'tcp',
-                                  'portBegin': 22,
-                                  'portEnd': 22,
-                                  }]
-            dispvm_template.write_firewall_conf(firewall)
-
-            testvm1.start()
-
-            p = testvm1.run("qvm-run --dispvm 'qubesdb-read /name; echo ERROR;"
-                            " read x'",
-                            passio_popen=True)
-
-            dispvm_name = p.stdout.readline().strip()
-            self.qc.lock_db_for_reading()
-            self.qc.load()
-            self.qc.unlock_db()
-            dispvm = self.qc.get_vm_by_name(dispvm_name)
-            self.assertIsNotNone(dispvm, "DispVM {} not found in qubes.xml".format(
-                dispvm_name))
-            # check if firewall was propagated to the DispVM from the right VM
-            self.assertEquals(testvm1.get_firewall_conf(),
-                              dispvm.get_firewall_conf())
-            # and only there (#1608)
-            self.assertNotEquals(dispvm_template.get_firewall_conf(),
-                                 dispvm.get_firewall_conf())
-            # then modify some rule
-            firewall = dispvm.get_firewall_conf()
-            firewall['rules'] = [{'address': '4.3.2.1',
-                                  'netmask': 24,
-                                  'proto': 'tcp',
-                                  'portBegin': 22,
-                                  'portEnd': 22,
-                                  }]
-            dispvm.write_firewall_conf(firewall)
-            # and check again if wasn't saved anywhere else (#1608)
-            self.assertNotEquals(dispvm_template.get_firewall_conf(),
-                                 dispvm.get_firewall_conf())
-            self.assertNotEquals(testvm1.get_firewall_conf(),
-                                 dispvm.get_firewall_conf())
-            p.stdin.write('\n')
-            p.wait()
-        finally:
-            if original_firewall:
-                original_firewall.seek(0)
-                with open(dispvm_template.firewall_conf, 'w') as f:
-                    f.write(original_firewall.read())
-                original_firewall.close()
-            else:
-                os.unlink(dispvm_template.firewall_conf)
-
-    def test_002_cleanup(self):
+    def test_100_resize_root_img(self):
+        testvm1 = self.qc.add_new_vm("QubesAppVm",
+                                     template=None,
+                                     name=self.make_vm_name('vm1'))
+        testvm1.create_on_disk(verbose=False,
+                               source_template=self.qc.get_default_template())
+        self.qc.save()
         self.qc.unlock_db()
-        p = subprocess.Popen(['/usr/lib/qubes/qfile-daemon-dvm',
-                              'qubes.VMShell', 'dom0', 'DEFAULT'],
-                             stdin=subprocess.PIPE,
-                             stdout=subprocess.PIPE,
-                             stderr=open(os.devnull, 'w'))
-        (stdout, _) = p.communicate(input="echo test; qubesdb-read /name; "
-                                          "echo ERROR\n")
-        self.assertEquals(p.returncode, 0)
-        lines = stdout.splitlines()
-        self.assertEqual(lines[0], "test")
-        dispvm_name = lines[1]
-        self.qc.lock_db_for_reading()
-        self.qc.load()
-        self.qc.unlock_db()
-        dispvm = self.qc.get_vm_by_name(dispvm_name)
-        self.assertIsNone(dispvm, "DispVM {} still exists in qubes.xml".format(
-            dispvm_name))
-
-    def test_003_cleanup_destroyed(self):
-        """
-        Check if DispVM is properly removed even if it terminated itself (#1660)
-        :return:
-        """
-        self.qc.unlock_db()
-        p = subprocess.Popen(['/usr/lib/qubes/qfile-daemon-dvm',
-                              'qubes.VMShell', 'dom0', 'DEFAULT'],
-                             stdin=subprocess.PIPE,
-                             stdout=subprocess.PIPE,
-                             stderr=open(os.devnull, 'w'))
-        p.stdin.write("qubesdb-read /name\n")
-        p.stdin.write("echo ERROR\n")
-        p.stdin.write("sudo poweroff\n")
-        # do not close p.stdin on purpose - wait to automatic disconnect when
-        #  domain is destroyed
-        timeout = 30
-        while timeout > 0:
-            if p.poll():
-                break
+        with self.assertRaises(QubesException):
+            testvm1.resize_root_img(20*1024**3)
+        testvm1.resize_root_img(20*1024**3, allow_start=True)
+        timeout = 60
+        while testvm1.is_running():
             time.sleep(1)
             timeout -= 1
-        # includes check for None - timeout
-        self.assertEquals(p.returncode, 0)
-        lines = p.stdout.read().splitlines()
-        dispvm_name = lines[0]
-        self.assertNotEquals(dispvm_name, "ERROR")
-        self.qc.lock_db_for_reading()
-        self.qc.load()
-        self.qc.unlock_db()
-        dispvm = self.qc.get_vm_by_name(dispvm_name)
-        self.assertIsNone(dispvm, "DispVM {} still exists in qubes.xml".format(
-            dispvm_name))
+            if timeout == 0:
+                self.fail("Timeout while waiting for VM shutdown")
+        self.assertEquals(testvm1.get_root_img_sz(), 20*1024**3)
+        testvm1.start()
+        p = testvm1.run('df --output=size /|tail -n 1',
+                        passio_popen=True)
+        # new_size in 1k-blocks
+        (new_size, _) = p.communicate()
+        # some safety margin for FS metadata
+        self.assertGreater(int(new_size.strip()), 19*1024**2)
+
+
 
 
 
diff --git a/tests/dispvm.py b/tests/dispvm.py
new file mode 100644
index 00000000..2c5c056c
--- /dev/null
+++ b/tests/dispvm.py
@@ -0,0 +1,431 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# The Qubes OS Project, http://www.qubes-os.org
+#
+# Copyright (C) 2016 Marek Marczykowski-Górecki
+#                                        <marmarek@invisiblethingslab.com>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+#
+
+from distutils import spawn
+import qubes.tests
+import subprocess
+import tempfile
+import unittest
+import os
+import time
+
+class TC_04_DispVM(qubes.tests.SystemTestsMixin,
+                   qubes.tests.QubesTestCase):
+
+    @staticmethod
+    def get_dispvm_template_name():
+        vmdir = os.readlink('/var/lib/qubes/dvmdata/vmdir')
+        return os.path.basename(vmdir)
+
+    def test_000_firewall_propagation(self):
+        """
+        Check firewall propagation VM->DispVM, when VM have some firewall rules
+        """
+
+        # FIXME: currently qubes.xml doesn't contain this information...
+        dispvm_template_name = self.get_dispvm_template_name()
+        dispvm_template = self.qc.get_vm_by_name(dispvm_template_name)
+
+        testvm1 = self.qc.add_new_vm("QubesAppVm",
+                                     name=self.make_vm_name('vm1'),
+                                     template=self.qc.get_default_template())
+        testvm1.create_on_disk(verbose=False)
+        firewall = testvm1.get_firewall_conf()
+        firewall['allowDns'] = False
+        firewall['allowYumProxy'] = False
+        firewall['rules'] = [{'address': '1.2.3.4',
+                              'netmask': 24,
+                              'proto': 'tcp',
+                              'portBegin': 22,
+                              'portEnd': 22,
+                              }]
+        testvm1.write_firewall_conf(firewall)
+        self.qc.save()
+        self.qc.unlock_db()
+
+        testvm1.start()
+
+        p = testvm1.run("qvm-run --dispvm 'qubesdb-read /name; echo ERROR;"
+                        " read x'",
+                        passio_popen=True)
+
+        dispvm_name = p.stdout.readline().strip()
+        self.qc.lock_db_for_reading()
+        self.qc.load()
+        self.qc.unlock_db()
+        dispvm = self.qc.get_vm_by_name(dispvm_name)
+        self.assertIsNotNone(dispvm, "DispVM {} not found in qubes.xml".format(
+            dispvm_name))
+        # check if firewall was propagated to the DispVM
+        self.assertEquals(testvm1.get_firewall_conf(),
+                          dispvm.get_firewall_conf())
+        # and only there (#1608)
+        self.assertNotEquals(dispvm_template.get_firewall_conf(),
+                             dispvm.get_firewall_conf())
+        # then modify some rule
+        firewall = dispvm.get_firewall_conf()
+        firewall['rules'] = [{'address': '4.3.2.1',
+                              'netmask': 24,
+                              'proto': 'tcp',
+                              'portBegin': 22,
+                              'portEnd': 22,
+                              }]
+        dispvm.write_firewall_conf(firewall)
+        # and check again if wasn't saved anywhere else (#1608)
+        self.assertNotEquals(dispvm_template.get_firewall_conf(),
+                             dispvm.get_firewall_conf())
+        self.assertNotEquals(testvm1.get_firewall_conf(),
+                             dispvm.get_firewall_conf())
+        p.stdin.write('\n')
+        p.wait()
+
+    def test_001_firewall_propagation(self):
+        """
+        Check firewall propagation VM->DispVM, when VM have no firewall rules
+        """
+        testvm1 = self.qc.add_new_vm("QubesAppVm",
+                                     name=self.make_vm_name('vm1'),
+                                     template=self.qc.get_default_template())
+        testvm1.create_on_disk(verbose=False)
+        self.qc.save()
+        self.qc.unlock_db()
+
+        # FIXME: currently qubes.xml doesn't contain this information...
+        dispvm_template_name = self.get_dispvm_template_name()
+        dispvm_template = self.qc.get_vm_by_name(dispvm_template_name)
+        original_firewall = None
+        if os.path.exists(dispvm_template.firewall_conf):
+            original_firewall = tempfile.TemporaryFile()
+            with open(dispvm_template.firewall_conf) as f:
+                original_firewall.write(f.read())
+        try:
+
+            firewall = dispvm_template.get_firewall_conf()
+            firewall['allowDns'] = False
+            firewall['allowYumProxy'] = False
+            firewall['rules'] = [{'address': '1.2.3.4',
+                                  'netmask': 24,
+                                  'proto': 'tcp',
+                                  'portBegin': 22,
+                                  'portEnd': 22,
+                                  }]
+            dispvm_template.write_firewall_conf(firewall)
+
+            testvm1.start()
+
+            p = testvm1.run("qvm-run --dispvm 'qubesdb-read /name; echo ERROR;"
+                            " read x'",
+                            passio_popen=True)
+
+            dispvm_name = p.stdout.readline().strip()
+            self.qc.lock_db_for_reading()
+            self.qc.load()
+            self.qc.unlock_db()
+            dispvm = self.qc.get_vm_by_name(dispvm_name)
+            self.assertIsNotNone(dispvm, "DispVM {} not found in qubes.xml".format(
+                dispvm_name))
+            # check if firewall was propagated to the DispVM from the right VM
+            self.assertEquals(testvm1.get_firewall_conf(),
+                              dispvm.get_firewall_conf())
+            # and only there (#1608)
+            self.assertNotEquals(dispvm_template.get_firewall_conf(),
+                                 dispvm.get_firewall_conf())
+            # then modify some rule
+            firewall = dispvm.get_firewall_conf()
+            firewall['rules'] = [{'address': '4.3.2.1',
+                                  'netmask': 24,
+                                  'proto': 'tcp',
+                                  'portBegin': 22,
+                                  'portEnd': 22,
+                                  }]
+            dispvm.write_firewall_conf(firewall)
+            # and check again if wasn't saved anywhere else (#1608)
+            self.assertNotEquals(dispvm_template.get_firewall_conf(),
+                                 dispvm.get_firewall_conf())
+            self.assertNotEquals(testvm1.get_firewall_conf(),
+                                 dispvm.get_firewall_conf())
+            p.stdin.write('\n')
+            p.wait()
+        finally:
+            if original_firewall:
+                original_firewall.seek(0)
+                with open(dispvm_template.firewall_conf, 'w') as f:
+                    f.write(original_firewall.read())
+                original_firewall.close()
+            else:
+                os.unlink(dispvm_template.firewall_conf)
+
+    def test_002_cleanup(self):
+        self.qc.unlock_db()
+        p = subprocess.Popen(['/usr/lib/qubes/qfile-daemon-dvm',
+                              'qubes.VMShell', 'dom0', 'DEFAULT'],
+                             stdin=subprocess.PIPE,
+                             stdout=subprocess.PIPE,
+                             stderr=open(os.devnull, 'w'))
+        (stdout, _) = p.communicate(input="echo test; qubesdb-read /name; "
+                                          "echo ERROR\n")
+        self.assertEquals(p.returncode, 0)
+        lines = stdout.splitlines()
+        self.assertEqual(lines[0], "test")
+        dispvm_name = lines[1]
+        self.qc.lock_db_for_reading()
+        self.qc.load()
+        self.qc.unlock_db()
+        dispvm = self.qc.get_vm_by_name(dispvm_name)
+        self.assertIsNone(dispvm, "DispVM {} still exists in qubes.xml".format(
+            dispvm_name))
+
+    def test_003_cleanup_destroyed(self):
+        """
+        Check if DispVM is properly removed even if it terminated itself (#1660)
+        :return:
+        """
+        self.qc.unlock_db()
+        p = subprocess.Popen(['/usr/lib/qubes/qfile-daemon-dvm',
+                              'qubes.VMShell', 'dom0', 'DEFAULT'],
+                             stdin=subprocess.PIPE,
+                             stdout=subprocess.PIPE,
+                             stderr=open(os.devnull, 'w'))
+        p.stdin.write("qubesdb-read /name\n")
+        p.stdin.write("echo ERROR\n")
+        p.stdin.write("sudo poweroff\n")
+        # do not close p.stdin on purpose - wait to automatic disconnect when
+        #  domain is destroyed
+        timeout = 30
+        while timeout > 0:
+            if p.poll():
+                break
+            time.sleep(1)
+            timeout -= 1
+        # includes check for None - timeout
+        self.assertEquals(p.returncode, 0)
+        lines = p.stdout.read().splitlines()
+        dispvm_name = lines[0]
+        self.assertNotEquals(dispvm_name, "ERROR")
+        self.qc.lock_db_for_reading()
+        self.qc.load()
+        self.qc.unlock_db()
+        dispvm = self.qc.get_vm_by_name(dispvm_name)
+        self.assertIsNone(dispvm, "DispVM {} still exists in qubes.xml".format(
+            dispvm_name))
+
+
+class TC_20_DispVMMixin(qubes.tests.SystemTestsMixin):
+    def test_000_prepare_dvm(self):
+        self.qc.unlock_db()
+        retcode = subprocess.call(['/usr/bin/qvm-create-default-dvm',
+                                   self.template],
+                                  stderr=open(os.devnull, 'w'))
+        self.assertEqual(retcode, 0)
+        self.qc.lock_db_for_writing()
+        self.qc.load()
+        self.assertIsNotNone(self.qc.get_vm_by_name(
+            self.template + "-dvm"))
+        # TODO: check mtime of snapshot file
+
+    def test_010_simple_dvm_run(self):
+        self.qc.unlock_db()
+        p = subprocess.Popen(['/usr/lib/qubes/qfile-daemon-dvm',
+                              'qubes.VMShell', 'dom0', 'DEFAULT'],
+                             stdin=subprocess.PIPE,
+                             stdout=subprocess.PIPE,
+                             stderr=open(os.devnull, 'w'))
+        (stdout, _) = p.communicate(input="echo test")
+        self.assertEqual(stdout, "test\n")
+        # TODO: check if DispVM is destroyed
+
+    @unittest.skipUnless(spawn.find_executable('xdotool'),
+                         "xdotool not installed")
+    def test_020_gui_app(self):
+        self.qc.unlock_db()
+        p = subprocess.Popen(['/usr/lib/qubes/qfile-daemon-dvm',
+                              'qubes.VMShell', 'dom0', 'DEFAULT'],
+                             stdin=subprocess.PIPE,
+                             stdout=subprocess.PIPE,
+                             stderr=open(os.devnull, 'w'))
+
+        # wait for DispVM startup:
+        p.stdin.write("echo test\n")
+        p.stdin.flush()
+        l = p.stdout.readline()
+        self.assertEqual(l, "test\n")
+
+        # potential race condition, but our tests are supposed to be
+        # running on dedicated machine, so should not be a problem
+        self.qc.lock_db_for_reading()
+        self.qc.load()
+        self.qc.unlock_db()
+
+        max_qid = 0
+        for vm in self.qc.values():
+            if not vm.is_disposablevm():
+                continue
+            if vm.qid > max_qid:
+                max_qid = vm.qid
+        dispvm = self.qc[max_qid]
+        self.assertNotEqual(dispvm.qid, 0, "DispVM not found in qubes.xml")
+        self.assertTrue(dispvm.is_running())
+        try:
+            window_title = 'user@%s' % (dispvm.template.name + "-dvm")
+            p.stdin.write("xterm -e "
+                "\"sh -c 'echo \\\"\033]0;{}\007\\\";read x;'\"\n".
+                format(window_title))
+            self.wait_for_window(window_title)
+
+            time.sleep(0.5)
+            self.enter_keys_in_window(window_title, ['Return'])
+            # Wait for window to close
+            self.wait_for_window(window_title, show=False)
+        finally:
+            p.stdin.close()
+
+        wait_count = 0
+        while dispvm.is_running():
+            wait_count += 1
+            if wait_count > 100:
+                self.fail("Timeout while waiting for DispVM destruction")
+            time.sleep(0.1)
+        wait_count = 0
+        while p.poll() is None:
+            wait_count += 1
+            if wait_count > 100:
+                self.fail("Timeout while waiting for qfile-daemon-dvm "
+                          "termination")
+            time.sleep(0.1)
+        self.assertEqual(p.returncode, 0)
+
+        self.qc.lock_db_for_reading()
+        self.qc.load()
+        self.qc.unlock_db()
+        self.assertIsNone(self.qc.get_vm_by_name(dispvm.name),
+                          "DispVM not removed from qubes.xml")
+
+    def _handle_editor(self, winid):
+        (window_title, _) = subprocess.Popen(
+            ['xdotool', 'getwindowname', winid], stdout=subprocess.PIPE).\
+            communicate()
+        window_title = window_title.strip().\
+            replace('(', '\(').replace(')', '\)')
+        time.sleep(1)
+        if "gedit" in window_title:
+            subprocess.check_call(['xdotool', 'windowactivate', '--sync', winid,
+                'type', 'Test test 2\n'])
+            time.sleep(0.5)
+            subprocess.check_call(['xdotool',
+                                   'key', 'ctrl+s', 'ctrl+q'])
+        elif "LibreOffice" in window_title:
+            # wait for actual editor (we've got splash screen)
+            search = subprocess.Popen(['xdotool', 'search', '--sync',
+                '--onlyvisible', '--all', '--name', '--class', 'disp*|Writer'],
+                stdout=subprocess.PIPE,
+                                  stderr=open(os.path.devnull, 'w'))
+            retcode = search.wait()
+            if retcode == 0:
+                winid = search.stdout.read().strip()
+            time.sleep(0.5)
+            subprocess.check_call(['xdotool', 'windowactivate', '--sync', winid,
+                'type', 'Test test 2\n'])
+            time.sleep(0.5)
+            subprocess.check_call(['xdotool',
+                                   'key', '--delay', '100', 'ctrl+s',
+                'Return', 'ctrl+q'])
+        elif "emacs" in window_title:
+            subprocess.check_call(['xdotool', 'windowactivate', '--sync', winid,
+                                   'type', 'Test test 2\n'])
+            time.sleep(0.5)
+            subprocess.check_call(['xdotool',
+                                   'key', 'ctrl+x', 'ctrl+s'])
+            subprocess.check_call(['xdotool',
+                                   'key', 'ctrl+x', 'ctrl+c'])
+        elif "vim" in window_title or "user@" in window_title:
+            subprocess.check_call(['xdotool', 'windowactivate', '--sync', winid,
+                                   'key', 'i', 'type', 'Test test 2\n'])
+            subprocess.check_call(
+                ['xdotool',
+                 'key', 'Escape', 'colon', 'w', 'q', 'Return'])
+        else:
+            self.fail("Unknown editor window: {}".format(window_title))
+
+    @unittest.skipUnless(spawn.find_executable('xdotool'),
+                         "xdotool not installed")
+    def test_030_edit_file(self):
+        testvm1 = self.qc.add_new_vm("QubesAppVm",
+                                     name=self.make_vm_name('vm1'),
+                                     template=self.qc.get_vm_by_name(
+                                         self.template))
+        testvm1.create_on_disk(verbose=False)
+        self.qc.save()
+
+        testvm1.start()
+        testvm1.run("echo test1 > /home/user/test.txt", wait=True)
+
+        self.qc.unlock_db()
+        p = testvm1.run("qvm-open-in-dvm /home/user/test.txt",
+                        passio_popen=True)
+
+        wait_count = 0
+        winid = None
+        while True:
+            search = subprocess.Popen(['xdotool', 'search',
+                                       '--onlyvisible', '--class', 'disp*'],
+                                      stdout=subprocess.PIPE,
+                                      stderr=open(os.path.devnull, 'w'))
+            retcode = search.wait()
+            if retcode == 0:
+                winid = search.stdout.read().strip()
+                break
+            wait_count += 1
+            if wait_count > 100:
+                self.fail("Timeout while waiting for editor window")
+            time.sleep(0.3)
+
+        time.sleep(0.5)
+        self._handle_editor(winid)
+        p.wait()
+        p = testvm1.run("cat /home/user/test.txt",
+                        passio_popen=True)
+        (test_txt_content, _) = p.communicate()
+        # Drop BOM if added by editor
+        if test_txt_content.startswith('\xef\xbb\xbf'):
+            test_txt_content = test_txt_content[3:]
+        self.assertEqual(test_txt_content, "Test test 2\ntest1\n")
+
+def load_tests(loader, tests, pattern):
+    try:
+        qc = qubes.qubes.QubesVmCollection()
+        qc.lock_db_for_reading()
+        qc.load()
+        qc.unlock_db()
+        templates = [vm.name for vm in qc.values() if
+                     isinstance(vm, qubes.qubes.QubesTemplateVm)]
+    except OSError:
+        templates = []
+    for template in templates:
+        tests.addTests(loader.loadTestsFromTestCase(
+            type(
+                'TC_20_DispVM_' + template,
+                (TC_20_DispVMMixin, qubes.tests.QubesTestCase),
+                {'template': template})))
+
+    return tests
\ No newline at end of file
diff --git a/tests/hvm.py b/tests/hvm.py
new file mode 100644
index 00000000..9cc3e132
--- /dev/null
+++ b/tests/hvm.py
@@ -0,0 +1,125 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# The Qubes OS Project, http://www.qubes-os.org
+#
+# Copyright (C) 2016 Marek Marczykowski-Górecki
+#                                        <marmarek@invisiblethingslab.com>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+#
+
+import qubes.tests
+from qubes.qubes import QubesException
+
+class TC_10_HVM(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
+    # TODO: test with some OS inside
+    # TODO: windows tools tests
+
+    def test_000_create_start(self):
+        testvm1 = self.qc.add_new_vm("QubesHVm",
+                                     name=self.make_vm_name('vm1'))
+        testvm1.create_on_disk(verbose=False)
+        self.qc.save()
+        self.qc.unlock_db()
+        testvm1.start()
+        self.assertEquals(testvm1.get_power_state(), "Running")
+
+    def test_010_create_start_template(self):
+        templatevm = self.qc.add_new_vm("QubesTemplateHVm",
+                                        name=self.make_vm_name('template'))
+        templatevm.create_on_disk(verbose=False)
+        self.qc.save()
+        self.qc.unlock_db()
+
+        templatevm.start()
+        self.assertEquals(templatevm.get_power_state(), "Running")
+
+    def test_020_create_start_template_vm(self):
+        templatevm = self.qc.add_new_vm("QubesTemplateHVm",
+                                        name=self.make_vm_name('template'))
+        templatevm.create_on_disk(verbose=False)
+        testvm2 = self.qc.add_new_vm("QubesHVm",
+                                     name=self.make_vm_name('vm2'),
+                                     template=templatevm)
+        testvm2.create_on_disk(verbose=False)
+        self.qc.save()
+        self.qc.unlock_db()
+
+        testvm2.start()
+        self.assertEquals(testvm2.get_power_state(), "Running")
+
+    def test_030_prevent_simultaneus_start(self):
+        templatevm = self.qc.add_new_vm("QubesTemplateHVm",
+                                        name=self.make_vm_name('template'))
+        templatevm.create_on_disk(verbose=False)
+        testvm2 = self.qc.add_new_vm("QubesHVm",
+                                     name=self.make_vm_name('vm2'),
+                                     template=templatevm)
+        testvm2.create_on_disk(verbose=False)
+        self.qc.save()
+        self.qc.unlock_db()
+
+        templatevm.start()
+        self.assertEquals(templatevm.get_power_state(), "Running")
+        self.assertRaises(QubesException, testvm2.start)
+        templatevm.force_shutdown()
+        testvm2.start()
+        self.assertEquals(testvm2.get_power_state(), "Running")
+        self.assertRaises(QubesException, templatevm.start)
+
+    def test_100_resize_root_img(self):
+        testvm1 = self.qc.add_new_vm("QubesHVm",
+                                     name=self.make_vm_name('vm1'))
+        testvm1.create_on_disk(verbose=False)
+        self.qc.save()
+        self.qc.unlock_db()
+        testvm1.resize_root_img(30*1024**3)
+        self.assertEquals(testvm1.get_root_img_sz(), 30*1024**3)
+        testvm1.start()
+        self.assertEquals(testvm1.get_power_state(), "Running")
+        # TODO: launch some OS there and check the size
+
+    def test_200_start_invalid_drive(self):
+        """Regression test for #1619"""
+        testvm1 = self.qc.add_new_vm("QubesHVm",
+                                     name=self.make_vm_name('vm1'))
+        testvm1.create_on_disk(verbose=False)
+        testvm1.drive = 'hd:dom0:/invalid'
+        self.qc.save()
+        self.qc.unlock_db()
+        try:
+            testvm1.start()
+        except Exception as e:
+            self.assertIsInstance(e, QubesException)
+        else:
+            self.fail('No exception raised')
+
+    def test_201_start_invalid_drive_cdrom(self):
+        """Regression test for #1619"""
+        testvm1 = self.qc.add_new_vm("QubesHVm",
+                                     name=self.make_vm_name('vm1'))
+        testvm1.create_on_disk(verbose=False)
+        testvm1.drive = 'cdrom:dom0:/invalid'
+        self.qc.save()
+        self.qc.unlock_db()
+        try:
+            testvm1.start()
+        except Exception as e:
+            self.assertIsInstance(e, QubesException)
+        else:
+            self.fail('No exception raised')
+
diff --git a/tests/mime.py b/tests/mime.py
new file mode 100644
index 00000000..660a0338
--- /dev/null
+++ b/tests/mime.py
@@ -0,0 +1,354 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# The Qubes OS Project, http://www.qubes-os.org
+#
+# Copyright (C) 2016 Marek Marczykowski-Górecki
+#                                        <marmarek@invisiblethingslab.com>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+#
+from distutils import spawn
+import os
+import re
+import subprocess
+import time
+import unittest
+
+import qubes.tests
+import qubes.qubes
+from qubes.qubes import QubesVmCollection
+
+@unittest.skipUnless(
+    spawn.find_executable('xprop') and
+    spawn.find_executable('xdotool') and
+    spawn.find_executable('wmctrl'),
+    "xprop or xdotool or wmctrl not installed")
+class TC_50_MimeHandlers(qubes.tests.SystemTestsMixin):
+    @classmethod
+    def setUpClass(cls):
+        if cls.template == 'whonix-gw' or 'minimal' in cls.template:
+            raise unittest.SkipTest(
+                'Template {} not supported by this test'.format(cls.template))
+
+        if cls.template == 'whonix-ws':
+            # TODO remove when Whonix-based DispVMs will work (Whonix 13?)
+            raise unittest.SkipTest(
+                'Template {} not supported by this test'.format(cls.template))
+
+        qc = QubesVmCollection()
+
+        cls._kill_test_vms(qc, prefix=qubes.tests.CLSVMPREFIX)
+
+        qc.lock_db_for_writing()
+        qc.load()
+
+        cls._remove_test_vms(qc, qubes.qubes.vmm.libvirt_conn,
+                            prefix=qubes.tests.CLSVMPREFIX)
+
+        cls.source_vmname = cls.make_vm_name('source', True)
+        source_vm = qc.add_new_vm("QubesAppVm",
+                                  template=qc.get_vm_by_name(cls.template),
+                                  name=cls.source_vmname)
+        source_vm.create_on_disk(verbose=False)
+
+        cls.target_vmname = cls.make_vm_name('target', True)
+        target_vm = qc.add_new_vm("QubesAppVm",
+                                  template=qc.get_vm_by_name(cls.template),
+                                  name=cls.target_vmname)
+        target_vm.create_on_disk(verbose=False)
+
+        qc.save()
+        qc.unlock_db()
+        source_vm.start()
+        target_vm.start()
+
+        # make sure that DispVMs will be started of the same template
+        retcode = subprocess.call(['/usr/bin/qvm-create-default-dvm',
+                                   cls.template],
+                                  stderr=open(os.devnull, 'w'))
+        assert retcode == 0, "Error preparing DispVM"
+
+    def setUp(self):
+        super(TC_50_MimeHandlers, self).setUp()
+        self.source_vm = self.qc.get_vm_by_name(self.source_vmname)
+        self.target_vm = self.qc.get_vm_by_name(self.target_vmname)
+
+    def get_window_class(self, winid, dispvm=False):
+        (vm_winid, _) = subprocess.Popen(
+            ['xprop', '-id', winid, '_QUBES_VMWINDOWID'],
+            stdout=subprocess.PIPE
+        ).communicate()
+        vm_winid = vm_winid.split("#")[1].strip('\n" ')
+        if dispvm:
+            (vmname, _) = subprocess.Popen(
+                ['xprop', '-id', winid, '_QUBES_VMNAME'],
+                stdout=subprocess.PIPE
+            ).communicate()
+            vmname = vmname.split("=")[1].strip('\n" ')
+            window_class = None
+            while window_class is None:
+                # XXX to use self.qc.get_vm_by_name would require reloading
+                # qubes.xml, so use qvm-run instead
+                xprop = subprocess.Popen(
+                    ['qvm-run', '-p', vmname, 'xprop -id {} WM_CLASS'.format(
+                        vm_winid)], stdout=subprocess.PIPE)
+                (window_class, _) = xprop.communicate()
+                if xprop.returncode != 0:
+                    self.skipTest("xprop failed, not installed?")
+                if 'not found' in window_class:
+                    # WM_CLASS not set yet, wait a little
+                    time.sleep(0.1)
+                    window_class = None
+        else:
+            window_class = None
+            while window_class is None:
+                xprop = self.target_vm.run(
+                    'xprop -id {} WM_CLASS'.format(vm_winid),
+                    passio_popen=True)
+                (window_class, _) = xprop.communicate()
+                if xprop.returncode != 0:
+                    self.skipTest("xprop failed, not installed?")
+                if 'not found' in window_class:
+                    # WM_CLASS not set yet, wait a little
+                    time.sleep(0.1)
+                    window_class = None
+        # output: WM_CLASS(STRING) = "gnome-terminal-server", "Gnome-terminal"
+        try:
+            window_class = window_class.split("=")[1].split(",")[0].strip('\n" ')
+        except IndexError:
+            raise Exception(
+                "Unexpected output from xprop: '{}'".format(window_class))
+
+        return window_class
+
+    def open_file_and_check_viewer(self, filename, expected_app_titles,
+                                   expected_app_classes, dispvm=False):
+        self.qc.unlock_db()
+        if dispvm:
+            p = self.source_vm.run("qvm-open-in-dvm {}".format(filename),
+                                   passio_popen=True)
+            vmpattern = "disp*"
+        else:
+            self.qrexec_policy('qubes.OpenInVM', self.source_vm.name,
+                self.target_vmname)
+            self.qrexec_policy('qubes.OpenURL', self.source_vm.name,
+                self.target_vmname)
+            p = self.source_vm.run("qvm-open-in-vm {} {}".format(
+                self.target_vmname, filename), passio_popen=True)
+            vmpattern = self.target_vmname
+        wait_count = 0
+        winid = None
+        window_title = None
+        while True:
+            search = subprocess.Popen(['xdotool', 'search',
+                                       '--onlyvisible', '--class', vmpattern],
+                                      stdout=subprocess.PIPE,
+                                      stderr=open(os.path.devnull, 'w'))
+            retcode = search.wait()
+            if retcode == 0:
+                winid = search.stdout.read().strip()
+                # get window title
+                (window_title, _) = subprocess.Popen(
+                    ['xdotool', 'getwindowname', winid], stdout=subprocess.PIPE). \
+                    communicate()
+                window_title = window_title.strip()
+                # ignore LibreOffice splash screen and window with no title
+                # set yet
+                if window_title and not window_title.startswith("LibreOffice")\
+                        and not window_title == 'VMapp command':
+                    break
+            wait_count += 1
+            if wait_count > 100:
+                self.fail("Timeout while waiting for editor window")
+            time.sleep(0.3)
+
+        # get window class
+        window_class = self.get_window_class(winid, dispvm)
+        # close the window - we've got the window class, it is no longer needed
+        subprocess.check_call(['wmctrl', '-i', '-c', winid])
+        p.wait()
+        self.wait_for_window(window_title, show=False)
+
+        def check_matches(obj, patterns):
+            return any((pat.search(obj) if isinstance(pat, type(re.compile('')))
+                        else pat in obj) for pat in patterns)
+
+        if not check_matches(window_title, expected_app_titles) and \
+                not check_matches(window_class, expected_app_classes):
+            self.fail("Opening file {} resulted in window '{} ({})', which is "
+                      "none of {!r} ({!r})".format(
+                          filename, window_title, window_class,
+                          expected_app_titles, expected_app_classes))
+
+    def prepare_txt(self, filename):
+        p = self.source_vm.run("cat > {}".format(filename), passio_popen=True)
+        p.stdin.write("This is test\n")
+        p.stdin.close()
+        retcode = p.wait()
+        assert retcode == 0, "Failed to write {} file".format(filename)
+
+    def prepare_pdf(self, filename):
+        self.prepare_txt("/tmp/source.txt")
+        cmd = "convert /tmp/source.txt {}".format(filename)
+        retcode = self.source_vm.run(cmd, wait=True)
+        assert retcode == 0, "Failed to run '{}'".format(cmd)
+
+    def prepare_doc(self, filename):
+        self.prepare_txt("/tmp/source.txt")
+        cmd = "unoconv -f doc -o {} /tmp/source.txt".format(filename)
+        retcode = self.source_vm.run(cmd, wait=True)
+        if retcode != 0:
+            self.skipTest("Failed to run '{}', not installed?".format(cmd))
+
+    def prepare_pptx(self, filename):
+        self.prepare_txt("/tmp/source.txt")
+        cmd = "unoconv -f pptx -o {} /tmp/source.txt".format(filename)
+        retcode = self.source_vm.run(cmd, wait=True)
+        if retcode != 0:
+            self.skipTest("Failed to run '{}', not installed?".format(cmd))
+
+    def prepare_png(self, filename):
+        self.prepare_txt("/tmp/source.txt")
+        cmd = "convert /tmp/source.txt {}".format(filename)
+        retcode = self.source_vm.run(cmd, wait=True)
+        if retcode != 0:
+            self.skipTest("Failed to run '{}', not installed?".format(cmd))
+
+    def prepare_jpg(self, filename):
+        self.prepare_txt("/tmp/source.txt")
+        cmd = "convert /tmp/source.txt {}".format(filename)
+        retcode = self.source_vm.run(cmd, wait=True)
+        if retcode != 0:
+            self.skipTest("Failed to run '{}', not installed?".format(cmd))
+
+    def test_000_txt(self):
+        filename = "/home/user/test_file.txt"
+        self.prepare_txt(filename)
+        self.open_file_and_check_viewer(filename, ["vim", "user@"],
+                                        ["gedit", "emacs", "libreoffice"])
+
+    def test_001_pdf(self):
+        filename = "/home/user/test_file.pdf"
+        self.prepare_pdf(filename)
+        self.open_file_and_check_viewer(filename, [],
+                                        ["evince"])
+
+    def test_002_doc(self):
+        filename = "/home/user/test_file.doc"
+        self.prepare_doc(filename)
+        self.open_file_and_check_viewer(filename, [],
+                                        ["libreoffice", "abiword"])
+
+    def test_003_pptx(self):
+        filename = "/home/user/test_file.pptx"
+        self.prepare_pptx(filename)
+        self.open_file_and_check_viewer(filename, [],
+                                        ["libreoffice"])
+
+    def test_004_png(self):
+        filename = "/home/user/test_file.png"
+        self.prepare_png(filename)
+        self.open_file_and_check_viewer(filename, [],
+                                        ["shotwell", "eog", "display"])
+
+    def test_005_jpg(self):
+        filename = "/home/user/test_file.jpg"
+        self.prepare_jpg(filename)
+        self.open_file_and_check_viewer(filename, [],
+                                        ["shotwell", "eog", "display"])
+
+    def test_006_jpeg(self):
+        filename = "/home/user/test_file.jpeg"
+        self.prepare_jpg(filename)
+        self.open_file_and_check_viewer(filename, [],
+                                        ["shotwell", "eog", "display"])
+
+    def test_010_url(self):
+        self.open_file_and_check_viewer("https://www.qubes-os.org/", [],
+                                        ["Firefox", "Iceweasel", "Navigator"])
+
+    def test_100_txt_dispvm(self):
+        filename = "/home/user/test_file.txt"
+        self.prepare_txt(filename)
+        self.open_file_and_check_viewer(filename, ["vim", "user@"],
+                                        ["gedit", "emacs", "libreoffice"],
+                                        dispvm=True)
+
+    def test_101_pdf_dispvm(self):
+        filename = "/home/user/test_file.pdf"
+        self.prepare_pdf(filename)
+        self.open_file_and_check_viewer(filename, [],
+                                        ["evince"],
+                                        dispvm=True)
+
+    def test_102_doc_dispvm(self):
+        filename = "/home/user/test_file.doc"
+        self.prepare_doc(filename)
+        self.open_file_and_check_viewer(filename, [],
+                                        ["libreoffice", "abiword"],
+                                        dispvm=True)
+
+    def test_103_pptx_dispvm(self):
+        filename = "/home/user/test_file.pptx"
+        self.prepare_pptx(filename)
+        self.open_file_and_check_viewer(filename, [],
+                                        ["libreoffice"],
+                                        dispvm=True)
+
+    def test_104_png_dispvm(self):
+        filename = "/home/user/test_file.png"
+        self.prepare_png(filename)
+        self.open_file_and_check_viewer(filename, [],
+                                        ["shotwell", "eog", "display"],
+                                        dispvm=True)
+
+    def test_105_jpg_dispvm(self):
+        filename = "/home/user/test_file.jpg"
+        self.prepare_jpg(filename)
+        self.open_file_and_check_viewer(filename, [],
+                                        ["shotwell", "eog", "display"],
+                                        dispvm=True)
+
+    def test_106_jpeg_dispvm(self):
+        filename = "/home/user/test_file.jpeg"
+        self.prepare_jpg(filename)
+        self.open_file_and_check_viewer(filename, [],
+                                        ["shotwell", "eog", "display"],
+                                        dispvm=True)
+
+    def test_110_url_dispvm(self):
+        self.open_file_and_check_viewer("https://www.qubes-os.org/", [],
+                                        ["Firefox", "Iceweasel", "Navigator"],
+                                        dispvm=True)
+
+def load_tests(loader, tests, pattern):
+    try:
+        qc = qubes.qubes.QubesVmCollection()
+        qc.lock_db_for_reading()
+        qc.load()
+        qc.unlock_db()
+        templates = [vm.name for vm in qc.values() if
+                     isinstance(vm, qubes.qubes.QubesTemplateVm)]
+    except OSError:
+        templates = []
+    for template in templates:
+        tests.addTests(loader.loadTestsFromTestCase(
+            type(
+                'TC_50_MimeHandlers_' + template,
+                (TC_50_MimeHandlers, qubes.tests.QubesTestCase),
+                {'template': template})))
+    return tests
\ No newline at end of file
diff --git a/tests/pvgrub.py b/tests/pvgrub.py
new file mode 100644
index 00000000..5467d2a3
--- /dev/null
+++ b/tests/pvgrub.py
@@ -0,0 +1,172 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# The Qubes OS Project, http://www.qubes-os.org
+#
+# Copyright (C) 2016 Marek Marczykowski-Górecki
+#                                        <marmarek@invisiblethingslab.com>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+#
+
+import os
+import unittest
+import qubes.tests
+@unittest.skipUnless(os.path.exists('/var/lib/qubes/vm-kernels/pvgrub2'),
+                     'grub-xen package not installed')
+class TC_40_PVGrub(qubes.tests.SystemTestsMixin):
+    def setUp(self):
+        super(TC_40_PVGrub, self).setUp()
+        supported = False
+        if self.template.startswith('fedora-'):
+            supported = True
+        elif self.template.startswith('debian-'):
+            supported = True
+        if not supported:
+            self.skipTest("Template {} not supported by this test".format(
+                self.template))
+
+    def install_packages(self, vm):
+        if self.template.startswith('fedora-'):
+            cmd_install1 = 'dnf clean expire-cache && ' \
+                'dnf install -y qubes-kernel-vm-support grub2-tools'
+            cmd_install2 = 'dnf install -y kernel && ' \
+                'KVER=$(rpm -q --qf %{VERSION}-%{RELEASE}.%{ARCH} kernel) && ' \
+                'dnf install --allowerasing  -y kernel-devel-$KVER && ' \
+                'dkms autoinstall -k $KVER'
+            cmd_update_grub = 'grub2-mkconfig -o /boot/grub2/grub.cfg'
+        elif self.template.startswith('debian-'):
+            cmd_install1 = 'apt-get update && apt-get install -y ' \
+                           'qubes-kernel-vm-support grub2-common'
+            cmd_install2 = 'apt-get install -y linux-image-amd64'
+            cmd_update_grub = 'mkdir /boot/grub && update-grub2'
+        else:
+            assert False, "Unsupported template?!"
+
+        for cmd in [cmd_install1, cmd_install2, cmd_update_grub]:
+            p = vm.run(cmd, user="root", passio_popen=True, passio_stderr=True)
+            (stdout, stderr) = p.communicate()
+            self.assertEquals(p.returncode, 0,
+                              "Failed command: {}\nSTDOUT: {}\nSTDERR: {}"
+                              .format(cmd, stdout, stderr))
+
+    def get_kernel_version(self, vm):
+        if self.template.startswith('fedora-'):
+            cmd_get_kernel_version = 'rpm -q kernel|sort -n|tail -1|' \
+                                     'cut -d - -f 2-'
+        elif self.template.startswith('debian-'):
+            cmd_get_kernel_version = \
+                'dpkg-query --showformat=\'${Package}\\n\' --show ' \
+                '\'linux-image-*-amd64\'|sort -n|tail -1|cut -d - -f 3-'
+        else:
+            raise RuntimeError("Unsupported template?!")
+
+        p = vm.run(cmd_get_kernel_version, user="root", passio_popen=True)
+        (kver, _) = p.communicate()
+        self.assertEquals(p.returncode, 0,
+                          "Failed command: {}".format(cmd_get_kernel_version))
+        return kver.strip()
+
+    def test_000_standalone_vm(self):
+        testvm1 = self.qc.add_new_vm("QubesAppVm",
+                                     template=None,
+                                     name=self.make_vm_name('vm1'))
+        testvm1.create_on_disk(verbose=False,
+                               source_template=self.qc.get_vm_by_name(
+                                   self.template))
+        self.save_and_reload_db()
+        self.qc.unlock_db()
+        testvm1 = self.qc[testvm1.qid]
+        testvm1.start()
+        self.install_packages(testvm1)
+        kver = self.get_kernel_version(testvm1)
+        self.shutdown_and_wait(testvm1)
+
+        self.qc.lock_db_for_writing()
+        self.qc.load()
+        testvm1 = self.qc[testvm1.qid]
+        testvm1.kernel = 'pvgrub2'
+        self.save_and_reload_db()
+        self.qc.unlock_db()
+        testvm1 = self.qc[testvm1.qid]
+        testvm1.start()
+        p = testvm1.run('uname -r', passio_popen=True)
+        (actual_kver, _) = p.communicate()
+        self.assertEquals(actual_kver.strip(), kver)
+
+    def test_010_template_based_vm(self):
+        test_template = self.qc.add_new_vm("QubesTemplateVm",
+                                           template=None,
+                                           name=self.make_vm_name('template'))
+        test_template.clone_attrs(self.qc.get_vm_by_name(self.template))
+        test_template.clone_disk_files(
+            src_vm=self.qc.get_vm_by_name(self.template),
+            verbose=False)
+
+        testvm1 = self.qc.add_new_vm("QubesAppVm",
+                                     template=test_template,
+                                     name=self.make_vm_name('vm1'))
+        testvm1.create_on_disk(verbose=False,
+                               source_template=test_template)
+        self.save_and_reload_db()
+        self.qc.unlock_db()
+        test_template = self.qc[test_template.qid]
+        testvm1 = self.qc[testvm1.qid]
+        test_template.start()
+        self.install_packages(test_template)
+        kver = self.get_kernel_version(test_template)
+        self.shutdown_and_wait(test_template)
+
+        self.qc.lock_db_for_writing()
+        self.qc.load()
+        test_template = self.qc[test_template.qid]
+        test_template.kernel = 'pvgrub2'
+        testvm1 = self.qc[testvm1.qid]
+        testvm1.kernel = 'pvgrub2'
+        self.save_and_reload_db()
+        self.qc.unlock_db()
+
+        # Check if TemplateBasedVM boots and has the right kernel
+        testvm1 = self.qc[testvm1.qid]
+        testvm1.start()
+        p = testvm1.run('uname -r', passio_popen=True)
+        (actual_kver, _) = p.communicate()
+        self.assertEquals(actual_kver.strip(), kver)
+
+        # And the same for the TemplateVM itself
+        test_template = self.qc[test_template.qid]
+        test_template.start()
+        p = test_template.run('uname -r', passio_popen=True)
+        (actual_kver, _) = p.communicate()
+        self.assertEquals(actual_kver.strip(), kver)
+
+def load_tests(loader, tests, pattern):
+    try:
+        qc = qubes.qubes.QubesVmCollection()
+        qc.lock_db_for_reading()
+        qc.load()
+        qc.unlock_db()
+        templates = [vm.name for vm in qc.values() if
+                     isinstance(vm, qubes.qubes.QubesTemplateVm)]
+    except OSError:
+        templates = []
+    for template in templates:
+        tests.addTests(loader.loadTestsFromTestCase(
+            type(
+                'TC_40_PVGrub_' + template,
+                (TC_40_PVGrub, qubes.tests.QubesTestCase),
+                {'template': template})))
+    return tests
\ No newline at end of file
diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index c82f43e2..22fff2fd 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -862,831 +862,6 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
         # some safety margin for FS metadata
         self.assertGreater(int(new_size.strip()), 5.8*1024**2)
 
-class TC_05_StandaloneVM(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
-    def test_000_create_start(self):
-        testvm1 = self.qc.add_new_vm("QubesAppVm",
-                                     template=None,
-                                     name=self.make_vm_name('vm1'))
-        testvm1.create_on_disk(verbose=False,
-                               source_template=self.qc.get_default_template())
-        self.qc.save()
-        self.qc.unlock_db()
-        testvm1.start()
-        self.assertEquals(testvm1.get_power_state(), "Running")
-
-    def test_100_resize_root_img(self):
-        testvm1 = self.qc.add_new_vm("QubesAppVm",
-                                     template=None,
-                                     name=self.make_vm_name('vm1'))
-        testvm1.create_on_disk(verbose=False,
-                               source_template=self.qc.get_default_template())
-        self.qc.save()
-        self.qc.unlock_db()
-        with self.assertRaises(QubesException):
-            testvm1.resize_root_img(20*1024**3)
-        testvm1.resize_root_img(20*1024**3, allow_start=True)
-        timeout = 60
-        while testvm1.is_running():
-            time.sleep(1)
-            timeout -= 1
-            if timeout == 0:
-                self.fail("Timeout while waiting for VM shutdown")
-        self.assertEquals(testvm1.get_root_img_sz(), 20*1024**3)
-        testvm1.start()
-        p = testvm1.run('df --output=size /|tail -n 1',
-                        passio_popen=True)
-        # new_size in 1k-blocks
-        (new_size, _) = p.communicate()
-        # some safety margin for FS metadata
-        self.assertGreater(int(new_size.strip()), 19*1024**2)
-
-
-class TC_10_HVM(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
-    # TODO: test with some OS inside
-    # TODO: windows tools tests
-
-    def test_000_create_start(self):
-        testvm1 = self.qc.add_new_vm("QubesHVm",
-                                     name=self.make_vm_name('vm1'))
-        testvm1.create_on_disk(verbose=False)
-        self.qc.save()
-        self.qc.unlock_db()
-        testvm1.start()
-        self.assertEquals(testvm1.get_power_state(), "Running")
-
-    def test_010_create_start_template(self):
-        templatevm = self.qc.add_new_vm("QubesTemplateHVm",
-                                        name=self.make_vm_name('template'))
-        templatevm.create_on_disk(verbose=False)
-        self.qc.save()
-        self.qc.unlock_db()
-
-        templatevm.start()
-        self.assertEquals(templatevm.get_power_state(), "Running")
-
-    def test_020_create_start_template_vm(self):
-        templatevm = self.qc.add_new_vm("QubesTemplateHVm",
-                                        name=self.make_vm_name('template'))
-        templatevm.create_on_disk(verbose=False)
-        testvm2 = self.qc.add_new_vm("QubesHVm",
-                                     name=self.make_vm_name('vm2'),
-                                     template=templatevm)
-        testvm2.create_on_disk(verbose=False)
-        self.qc.save()
-        self.qc.unlock_db()
-
-        testvm2.start()
-        self.assertEquals(testvm2.get_power_state(), "Running")
-
-    def test_030_prevent_simultaneus_start(self):
-        templatevm = self.qc.add_new_vm("QubesTemplateHVm",
-                                        name=self.make_vm_name('template'))
-        templatevm.create_on_disk(verbose=False)
-        testvm2 = self.qc.add_new_vm("QubesHVm",
-                                     name=self.make_vm_name('vm2'),
-                                     template=templatevm)
-        testvm2.create_on_disk(verbose=False)
-        self.qc.save()
-        self.qc.unlock_db()
-
-        templatevm.start()
-        self.assertEquals(templatevm.get_power_state(), "Running")
-        self.assertRaises(QubesException, testvm2.start)
-        templatevm.force_shutdown()
-        testvm2.start()
-        self.assertEquals(testvm2.get_power_state(), "Running")
-        self.assertRaises(QubesException, templatevm.start)
-
-    def test_100_resize_root_img(self):
-        testvm1 = self.qc.add_new_vm("QubesHVm",
-                                     name=self.make_vm_name('vm1'))
-        testvm1.create_on_disk(verbose=False)
-        self.qc.save()
-        self.qc.unlock_db()
-        testvm1.resize_root_img(30*1024**3)
-        self.assertEquals(testvm1.get_root_img_sz(), 30*1024**3)
-        testvm1.start()
-        self.assertEquals(testvm1.get_power_state(), "Running")
-        # TODO: launch some OS there and check the size
-
-    def test_200_start_invalid_drive(self):
-        """Regression test for #1619"""
-        testvm1 = self.qc.add_new_vm("QubesHVm",
-                                     name=self.make_vm_name('vm1'))
-        testvm1.create_on_disk(verbose=False)
-        testvm1.drive = 'hd:dom0:/invalid'
-        self.qc.save()
-        self.qc.unlock_db()
-        try:
-            testvm1.start()
-        except Exception as e:
-            self.assertIsInstance(e, QubesException)
-        else:
-            self.fail('No exception raised')
-
-    def test_201_start_invalid_drive_cdrom(self):
-        """Regression test for #1619"""
-        testvm1 = self.qc.add_new_vm("QubesHVm",
-                                     name=self.make_vm_name('vm1'))
-        testvm1.create_on_disk(verbose=False)
-        testvm1.drive = 'cdrom:dom0:/invalid'
-        self.qc.save()
-        self.qc.unlock_db()
-        try:
-            testvm1.start()
-        except Exception as e:
-            self.assertIsInstance(e, QubesException)
-        else:
-            self.fail('No exception raised')
-
-class TC_20_DispVMMixin(qubes.tests.SystemTestsMixin):
-    def test_000_prepare_dvm(self):
-        self.qc.unlock_db()
-        retcode = subprocess.call(['/usr/bin/qvm-create-default-dvm',
-                                   self.template],
-                                  stderr=open(os.devnull, 'w'))
-        self.assertEqual(retcode, 0)
-        self.qc.lock_db_for_writing()
-        self.qc.load()
-        self.assertIsNotNone(self.qc.get_vm_by_name(
-            self.template + "-dvm"))
-        # TODO: check mtime of snapshot file
-
-    def test_010_simple_dvm_run(self):
-        self.qc.unlock_db()
-        p = subprocess.Popen(['/usr/lib/qubes/qfile-daemon-dvm',
-                              'qubes.VMShell', 'dom0', 'DEFAULT'],
-                             stdin=subprocess.PIPE,
-                             stdout=subprocess.PIPE,
-                             stderr=open(os.devnull, 'w'))
-        (stdout, _) = p.communicate(input="echo test")
-        self.assertEqual(stdout, "test\n")
-        # TODO: check if DispVM is destroyed
-
-    @unittest.skipUnless(spawn.find_executable('xdotool'),
-                         "xdotool not installed")
-    def test_020_gui_app(self):
-        self.qc.unlock_db()
-        p = subprocess.Popen(['/usr/lib/qubes/qfile-daemon-dvm',
-                              'qubes.VMShell', 'dom0', 'DEFAULT'],
-                             stdin=subprocess.PIPE,
-                             stdout=subprocess.PIPE,
-                             stderr=open(os.devnull, 'w'))
-
-        # wait for DispVM startup:
-        p.stdin.write("echo test\n")
-        p.stdin.flush()
-        l = p.stdout.readline()
-        self.assertEqual(l, "test\n")
-
-        # potential race condition, but our tests are supposed to be
-        # running on dedicated machine, so should not be a problem
-        self.qc.lock_db_for_reading()
-        self.qc.load()
-        self.qc.unlock_db()
-
-        max_qid = 0
-        for vm in self.qc.values():
-            if not vm.is_disposablevm():
-                continue
-            if vm.qid > max_qid:
-                max_qid = vm.qid
-        dispvm = self.qc[max_qid]
-        self.assertNotEqual(dispvm.qid, 0, "DispVM not found in qubes.xml")
-        self.assertTrue(dispvm.is_running())
-        try:
-            window_title = 'user@%s' % (dispvm.template.name + "-dvm")
-            p.stdin.write("xterm -e "
-                "\"sh -c 'echo \\\"\033]0;{}\007\\\";read x;'\"\n".
-                format(window_title))
-            self.wait_for_window(window_title)
-
-            time.sleep(0.5)
-            self.enter_keys_in_window(window_title, ['Return'])
-            # Wait for window to close
-            self.wait_for_window(window_title, show=False)
-        finally:
-            p.stdin.close()
-
-        wait_count = 0
-        while dispvm.is_running():
-            wait_count += 1
-            if wait_count > 100:
-                self.fail("Timeout while waiting for DispVM destruction")
-            time.sleep(0.1)
-        wait_count = 0
-        while p.poll() is None:
-            wait_count += 1
-            if wait_count > 100:
-                self.fail("Timeout while waiting for qfile-daemon-dvm "
-                          "termination")
-            time.sleep(0.1)
-        self.assertEqual(p.returncode, 0)
-
-        self.qc.lock_db_for_reading()
-        self.qc.load()
-        self.qc.unlock_db()
-        self.assertIsNone(self.qc.get_vm_by_name(dispvm.name),
-                          "DispVM not removed from qubes.xml")
-
-    def _handle_editor(self, winid):
-        (window_title, _) = subprocess.Popen(
-            ['xdotool', 'getwindowname', winid], stdout=subprocess.PIPE).\
-            communicate()
-        window_title = window_title.strip().\
-            replace('(', '\(').replace(')', '\)')
-        time.sleep(1)
-        if "gedit" in window_title:
-            subprocess.check_call(['xdotool', 'windowactivate', '--sync', winid,
-                'type', 'Test test 2\n'])
-            time.sleep(0.5)
-            subprocess.check_call(['xdotool',
-                                   'key', 'ctrl+s', 'ctrl+q'])
-        elif "LibreOffice" in window_title:
-            # wait for actual editor (we've got splash screen)
-            search = subprocess.Popen(['xdotool', 'search', '--sync',
-                '--onlyvisible', '--all', '--name', '--class', 'disp*|Writer'],
-                stdout=subprocess.PIPE,
-                                  stderr=open(os.path.devnull, 'w'))
-            retcode = search.wait()
-            if retcode == 0:
-                winid = search.stdout.read().strip()
-            time.sleep(0.5)
-            subprocess.check_call(['xdotool', 'windowactivate', '--sync', winid,
-                'type', 'Test test 2\n'])
-            time.sleep(0.5)
-            subprocess.check_call(['xdotool',
-                                   'key', '--delay', '100', 'ctrl+s',
-                'Return', 'ctrl+q'])
-        elif "emacs" in window_title:
-            subprocess.check_call(['xdotool', 'windowactivate', '--sync', winid,
-                                   'type', 'Test test 2\n'])
-            time.sleep(0.5)
-            subprocess.check_call(['xdotool',
-                                   'key', 'ctrl+x', 'ctrl+s'])
-            subprocess.check_call(['xdotool',
-                                   'key', 'ctrl+x', 'ctrl+c'])
-        elif "vim" in window_title or "user@" in window_title:
-            subprocess.check_call(['xdotool', 'windowactivate', '--sync', winid,
-                                   'key', 'i', 'type', 'Test test 2\n'])
-            subprocess.check_call(
-                ['xdotool',
-                 'key', 'Escape', 'colon', 'w', 'q', 'Return'])
-        else:
-            self.fail("Unknown editor window: {}".format(window_title))
-
-    @unittest.skipUnless(spawn.find_executable('xdotool'),
-                         "xdotool not installed")
-    def test_030_edit_file(self):
-        testvm1 = self.qc.add_new_vm("QubesAppVm",
-                                     name=self.make_vm_name('vm1'),
-                                     template=self.qc.get_vm_by_name(
-                                         self.template))
-        testvm1.create_on_disk(verbose=False)
-        self.qc.save()
-
-        testvm1.start()
-        testvm1.run("echo test1 > /home/user/test.txt", wait=True)
-
-        self.qc.unlock_db()
-        p = testvm1.run("qvm-open-in-dvm /home/user/test.txt",
-                        passio_popen=True)
-
-        wait_count = 0
-        winid = None
-        while True:
-            search = subprocess.Popen(['xdotool', 'search',
-                                       '--onlyvisible', '--class', 'disp*'],
-                                      stdout=subprocess.PIPE,
-                                      stderr=open(os.path.devnull, 'w'))
-            retcode = search.wait()
-            if retcode == 0:
-                winid = search.stdout.read().strip()
-                break
-            wait_count += 1
-            if wait_count > 100:
-                self.fail("Timeout while waiting for editor window")
-            time.sleep(0.3)
-
-        time.sleep(0.5)
-        self._handle_editor(winid)
-        p.wait()
-        p = testvm1.run("cat /home/user/test.txt",
-                        passio_popen=True)
-        (test_txt_content, _) = p.communicate()
-        # Drop BOM if added by editor
-        if test_txt_content.startswith('\xef\xbb\xbf'):
-            test_txt_content = test_txt_content[3:]
-        self.assertEqual(test_txt_content, "Test test 2\ntest1\n")
-
-
-class TC_30_Gui_daemon(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
-    @unittest.skipUnless(spawn.find_executable('xdotool'),
-                         "xdotool not installed")
-    def test_000_clipboard(self):
-        testvm1 = self.qc.add_new_vm("QubesAppVm",
-                                     name=self.make_vm_name('vm1'),
-                                     template=self.qc.get_default_template())
-        testvm1.create_on_disk(verbose=False)
-        testvm2 = self.qc.add_new_vm("QubesAppVm",
-                                     name=self.make_vm_name('vm2'),
-                                     template=self.qc.get_default_template())
-        testvm2.create_on_disk(verbose=False)
-        self.qc.save()
-        self.qc.unlock_db()
-
-        testvm1.start()
-        testvm2.start()
-
-        window_title = 'user@{}'.format(testvm1.name)
-        testvm1.run('zenity --text-info --editable --title={}'.format(
-            window_title))
-
-        self.wait_for_window(window_title)
-        time.sleep(0.5)
-        test_string = "test{}".format(testvm1.xid)
-
-        # Type and copy some text
-        subprocess.check_call(['xdotool', 'search', '--name', window_title,
-                               'windowactivate', '--sync',
-                               'type', '{}'.format(test_string)])
-        # second xdotool call because type --terminator do not work (SEGV)
-        # additionally do not use search here, so window stack will be empty
-        # and xdotool will use XTEST instead of generating events manually -
-        # this will be much better - at least because events will have
-        # correct timestamp (so gui-daemon would not drop the copy request)
-        subprocess.check_call(['xdotool',
-                               'key', 'ctrl+a', 'ctrl+c', 'ctrl+shift+c',
-                               'Escape'])
-
-        clipboard_content = \
-            open('/var/run/qubes/qubes-clipboard.bin', 'r').read().strip()
-        self.assertEquals(clipboard_content, test_string,
-                          "Clipboard copy operation failed - content")
-        clipboard_source = \
-            open('/var/run/qubes/qubes-clipboard.bin.source',
-                 'r').read().strip()
-        self.assertEquals(clipboard_source, testvm1.name,
-                          "Clipboard copy operation failed - owner")
-
-        # Then paste it to the other window
-        window_title = 'user@{}'.format(testvm2.name)
-        p = testvm2.run('zenity --entry --title={} > test.txt'.format(
-                        window_title), passio_popen=True)
-        self.wait_for_window(window_title)
-
-        subprocess.check_call(['xdotool', 'key', '--delay', '100',
-                               'ctrl+shift+v', 'ctrl+v', 'Return'])
-        p.wait()
-
-        # And compare the result
-        (test_output, _) = testvm2.run('cat test.txt',
-                                       passio_popen=True).communicate()
-        self.assertEquals(test_string, test_output.strip())
-
-        clipboard_content = \
-            open('/var/run/qubes/qubes-clipboard.bin', 'r').read().strip()
-        self.assertEquals(clipboard_content, "",
-                          "Clipboard not wiped after paste - content")
-        clipboard_source = \
-            open('/var/run/qubes/qubes-clipboard.bin.source', 'r').read(
-
-            ).strip()
-        self.assertEquals(clipboard_source, "",
-                          "Clipboard not wiped after paste - owner")
-
-
-@unittest.skipUnless(os.path.exists('/var/lib/qubes/vm-kernels/pvgrub2'),
-                     'grub-xen package not installed')
-class TC_40_PVGrub(qubes.tests.SystemTestsMixin):
-    def setUp(self):
-        super(TC_40_PVGrub, self).setUp()
-        supported = False
-        if self.template.startswith('fedora-'):
-            supported = True
-        elif self.template.startswith('debian-'):
-            supported = True
-        if not supported:
-            self.skipTest("Template {} not supported by this test".format(
-                self.template))
-
-    def install_packages(self, vm):
-        if self.template.startswith('fedora-'):
-            cmd_install1 = 'dnf clean expire-cache && ' \
-                'dnf install -y qubes-kernel-vm-support grub2-tools'
-            cmd_install2 = 'dnf install -y kernel && ' \
-                'KVER=$(rpm -q --qf %{VERSION}-%{RELEASE}.%{ARCH} kernel) && ' \
-                'dnf install --allowerasing  -y kernel-devel-$KVER && ' \
-                'dkms autoinstall -k $KVER'
-            cmd_update_grub = 'grub2-mkconfig -o /boot/grub2/grub.cfg'
-        elif self.template.startswith('debian-'):
-            cmd_install1 = 'apt-get update && apt-get install -y ' \
-                           'qubes-kernel-vm-support grub2-common'
-            cmd_install2 = 'apt-get install -y linux-image-amd64'
-            cmd_update_grub = 'mkdir /boot/grub && update-grub2'
-        else:
-            assert False, "Unsupported template?!"
-
-        for cmd in [cmd_install1, cmd_install2, cmd_update_grub]:
-            p = vm.run(cmd, user="root", passio_popen=True, passio_stderr=True)
-            (stdout, stderr) = p.communicate()
-            self.assertEquals(p.returncode, 0,
-                              "Failed command: {}\nSTDOUT: {}\nSTDERR: {}"
-                              .format(cmd, stdout, stderr))
-
-    def get_kernel_version(self, vm):
-        if self.template.startswith('fedora-'):
-            cmd_get_kernel_version = 'rpm -q kernel|sort -n|tail -1|' \
-                                     'cut -d - -f 2-'
-        elif self.template.startswith('debian-'):
-            cmd_get_kernel_version = \
-                'dpkg-query --showformat=\'${Package}\\n\' --show ' \
-                '\'linux-image-*-amd64\'|sort -n|tail -1|cut -d - -f 3-'
-        else:
-            raise RuntimeError("Unsupported template?!")
-
-        p = vm.run(cmd_get_kernel_version, user="root", passio_popen=True)
-        (kver, _) = p.communicate()
-        self.assertEquals(p.returncode, 0,
-                          "Failed command: {}".format(cmd_get_kernel_version))
-        return kver.strip()
-
-    def test_000_standalone_vm(self):
-        testvm1 = self.qc.add_new_vm("QubesAppVm",
-                                     template=None,
-                                     name=self.make_vm_name('vm1'))
-        testvm1.create_on_disk(verbose=False,
-                               source_template=self.qc.get_vm_by_name(
-                                   self.template))
-        self.save_and_reload_db()
-        self.qc.unlock_db()
-        testvm1 = self.qc[testvm1.qid]
-        testvm1.start()
-        self.install_packages(testvm1)
-        kver = self.get_kernel_version(testvm1)
-        self.shutdown_and_wait(testvm1)
-
-        self.qc.lock_db_for_writing()
-        self.qc.load()
-        testvm1 = self.qc[testvm1.qid]
-        testvm1.kernel = 'pvgrub2'
-        self.save_and_reload_db()
-        self.qc.unlock_db()
-        testvm1 = self.qc[testvm1.qid]
-        testvm1.start()
-        p = testvm1.run('uname -r', passio_popen=True)
-        (actual_kver, _) = p.communicate()
-        self.assertEquals(actual_kver.strip(), kver)
-
-    def test_010_template_based_vm(self):
-        test_template = self.qc.add_new_vm("QubesTemplateVm",
-                                           template=None,
-                                           name=self.make_vm_name('template'))
-        test_template.clone_attrs(self.qc.get_vm_by_name(self.template))
-        test_template.clone_disk_files(
-            src_vm=self.qc.get_vm_by_name(self.template),
-            verbose=False)
-
-        testvm1 = self.qc.add_new_vm("QubesAppVm",
-                                     template=test_template,
-                                     name=self.make_vm_name('vm1'))
-        testvm1.create_on_disk(verbose=False,
-                               source_template=test_template)
-        self.save_and_reload_db()
-        self.qc.unlock_db()
-        test_template = self.qc[test_template.qid]
-        testvm1 = self.qc[testvm1.qid]
-        test_template.start()
-        self.install_packages(test_template)
-        kver = self.get_kernel_version(test_template)
-        self.shutdown_and_wait(test_template)
-
-        self.qc.lock_db_for_writing()
-        self.qc.load()
-        test_template = self.qc[test_template.qid]
-        test_template.kernel = 'pvgrub2'
-        testvm1 = self.qc[testvm1.qid]
-        testvm1.kernel = 'pvgrub2'
-        self.save_and_reload_db()
-        self.qc.unlock_db()
-
-        # Check if TemplateBasedVM boots and has the right kernel
-        testvm1 = self.qc[testvm1.qid]
-        testvm1.start()
-        p = testvm1.run('uname -r', passio_popen=True)
-        (actual_kver, _) = p.communicate()
-        self.assertEquals(actual_kver.strip(), kver)
-
-        # And the same for the TemplateVM itself
-        test_template = self.qc[test_template.qid]
-        test_template.start()
-        p = test_template.run('uname -r', passio_popen=True)
-        (actual_kver, _) = p.communicate()
-        self.assertEquals(actual_kver.strip(), kver)
-
-@unittest.skipUnless(
-    spawn.find_executable('xprop') and
-    spawn.find_executable('xdotool') and
-    spawn.find_executable('wmctrl'),
-    "xprop or xdotool or wmctrl not installed")
-class TC_50_MimeHandlers(qubes.tests.SystemTestsMixin):
-    @classmethod
-    def setUpClass(cls):
-        if cls.template == 'whonix-gw' or 'minimal' in cls.template:
-            raise unittest.SkipTest(
-                'Template {} not supported by this test'.format(cls.template))
-
-        if cls.template == 'whonix-ws':
-            # TODO remove when Whonix-based DispVMs will work (Whonix 13?)
-            raise unittest.SkipTest(
-                'Template {} not supported by this test'.format(cls.template))
-
-        qc = QubesVmCollection()
-
-        cls._kill_test_vms(qc, prefix=qubes.tests.CLSVMPREFIX)
-
-        qc.lock_db_for_writing()
-        qc.load()
-
-        cls._remove_test_vms(qc, qubes.qubes.vmm.libvirt_conn,
-                            prefix=qubes.tests.CLSVMPREFIX)
-
-        cls.source_vmname = cls.make_vm_name('source', True)
-        source_vm = qc.add_new_vm("QubesAppVm",
-                                  template=qc.get_vm_by_name(cls.template),
-                                  name=cls.source_vmname)
-        source_vm.create_on_disk(verbose=False)
-
-        cls.target_vmname = cls.make_vm_name('target', True)
-        target_vm = qc.add_new_vm("QubesAppVm",
-                                  template=qc.get_vm_by_name(cls.template),
-                                  name=cls.target_vmname)
-        target_vm.create_on_disk(verbose=False)
-
-        qc.save()
-        qc.unlock_db()
-        source_vm.start()
-        target_vm.start()
-
-        # make sure that DispVMs will be started of the same template
-        retcode = subprocess.call(['/usr/bin/qvm-create-default-dvm',
-                                   cls.template],
-                                  stderr=open(os.devnull, 'w'))
-        assert retcode == 0, "Error preparing DispVM"
-
-    def setUp(self):
-        super(TC_50_MimeHandlers, self).setUp()
-        self.source_vm = self.qc.get_vm_by_name(self.source_vmname)
-        self.target_vm = self.qc.get_vm_by_name(self.target_vmname)
-
-    def get_window_class(self, winid, dispvm=False):
-        (vm_winid, _) = subprocess.Popen(
-            ['xprop', '-id', winid, '_QUBES_VMWINDOWID'],
-            stdout=subprocess.PIPE
-        ).communicate()
-        vm_winid = vm_winid.split("#")[1].strip('\n" ')
-        if dispvm:
-            (vmname, _) = subprocess.Popen(
-                ['xprop', '-id', winid, '_QUBES_VMNAME'],
-                stdout=subprocess.PIPE
-            ).communicate()
-            vmname = vmname.split("=")[1].strip('\n" ')
-            window_class = None
-            while window_class is None:
-                # XXX to use self.qc.get_vm_by_name would require reloading
-                # qubes.xml, so use qvm-run instead
-                xprop = subprocess.Popen(
-                    ['qvm-run', '-p', vmname, 'xprop -id {} WM_CLASS'.format(
-                        vm_winid)], stdout=subprocess.PIPE)
-                (window_class, _) = xprop.communicate()
-                if xprop.returncode != 0:
-                    self.skipTest("xprop failed, not installed?")
-                if 'not found' in window_class:
-                    # WM_CLASS not set yet, wait a little
-                    time.sleep(0.1)
-                    window_class = None
-        else:
-            window_class = None
-            while window_class is None:
-                xprop = self.target_vm.run(
-                    'xprop -id {} WM_CLASS'.format(vm_winid),
-                    passio_popen=True)
-                (window_class, _) = xprop.communicate()
-                if xprop.returncode != 0:
-                    self.skipTest("xprop failed, not installed?")
-                if 'not found' in window_class:
-                    # WM_CLASS not set yet, wait a little
-                    time.sleep(0.1)
-                    window_class = None
-        # output: WM_CLASS(STRING) = "gnome-terminal-server", "Gnome-terminal"
-        try:
-            window_class = window_class.split("=")[1].split(",")[0].strip('\n" ')
-        except IndexError:
-            raise Exception(
-                "Unexpected output from xprop: '{}'".format(window_class))
-
-        return window_class
-
-    def open_file_and_check_viewer(self, filename, expected_app_titles,
-                                   expected_app_classes, dispvm=False):
-        self.qc.unlock_db()
-        if dispvm:
-            p = self.source_vm.run("qvm-open-in-dvm {}".format(filename),
-                                   passio_popen=True)
-            vmpattern = "disp*"
-        else:
-            self.qrexec_policy('qubes.OpenInVM', self.source_vm.name,
-                self.target_vmname)
-            self.qrexec_policy('qubes.OpenURL', self.source_vm.name,
-                self.target_vmname)
-            p = self.source_vm.run("qvm-open-in-vm {} {}".format(
-                self.target_vmname, filename), passio_popen=True)
-            vmpattern = self.target_vmname
-        wait_count = 0
-        winid = None
-        window_title = None
-        while True:
-            search = subprocess.Popen(['xdotool', 'search',
-                                       '--onlyvisible', '--class', vmpattern],
-                                      stdout=subprocess.PIPE,
-                                      stderr=open(os.path.devnull, 'w'))
-            retcode = search.wait()
-            if retcode == 0:
-                winid = search.stdout.read().strip()
-                # get window title
-                (window_title, _) = subprocess.Popen(
-                    ['xdotool', 'getwindowname', winid], stdout=subprocess.PIPE). \
-                    communicate()
-                window_title = window_title.strip()
-                # ignore LibreOffice splash screen and window with no title
-                # set yet
-                if window_title and not window_title.startswith("LibreOffice")\
-                        and not window_title == 'VMapp command':
-                    break
-            wait_count += 1
-            if wait_count > 100:
-                self.fail("Timeout while waiting for editor window")
-            time.sleep(0.3)
-
-        # get window class
-        window_class = self.get_window_class(winid, dispvm)
-        # close the window - we've got the window class, it is no longer needed
-        subprocess.check_call(['wmctrl', '-i', '-c', winid])
-        p.wait()
-        self.wait_for_window(window_title, show=False)
-
-        def check_matches(obj, patterns):
-            return any((pat.search(obj) if isinstance(pat, type(re.compile('')))
-                        else pat in obj) for pat in patterns)
-
-        if not check_matches(window_title, expected_app_titles) and \
-                not check_matches(window_class, expected_app_classes):
-            self.fail("Opening file {} resulted in window '{} ({})', which is "
-                      "none of {!r} ({!r})".format(
-                          filename, window_title, window_class,
-                          expected_app_titles, expected_app_classes))
-
-    def prepare_txt(self, filename):
-        p = self.source_vm.run("cat > {}".format(filename), passio_popen=True)
-        p.stdin.write("This is test\n")
-        p.stdin.close()
-        retcode = p.wait()
-        assert retcode == 0, "Failed to write {} file".format(filename)
-
-    def prepare_pdf(self, filename):
-        self.prepare_txt("/tmp/source.txt")
-        cmd = "convert /tmp/source.txt {}".format(filename)
-        retcode = self.source_vm.run(cmd, wait=True)
-        assert retcode == 0, "Failed to run '{}'".format(cmd)
-
-    def prepare_doc(self, filename):
-        self.prepare_txt("/tmp/source.txt")
-        cmd = "unoconv -f doc -o {} /tmp/source.txt".format(filename)
-        retcode = self.source_vm.run(cmd, wait=True)
-        if retcode != 0:
-            self.skipTest("Failed to run '{}', not installed?".format(cmd))
-
-    def prepare_pptx(self, filename):
-        self.prepare_txt("/tmp/source.txt")
-        cmd = "unoconv -f pptx -o {} /tmp/source.txt".format(filename)
-        retcode = self.source_vm.run(cmd, wait=True)
-        if retcode != 0:
-            self.skipTest("Failed to run '{}', not installed?".format(cmd))
-
-    def prepare_png(self, filename):
-        self.prepare_txt("/tmp/source.txt")
-        cmd = "convert /tmp/source.txt {}".format(filename)
-        retcode = self.source_vm.run(cmd, wait=True)
-        if retcode != 0:
-            self.skipTest("Failed to run '{}', not installed?".format(cmd))
-
-    def prepare_jpg(self, filename):
-        self.prepare_txt("/tmp/source.txt")
-        cmd = "convert /tmp/source.txt {}".format(filename)
-        retcode = self.source_vm.run(cmd, wait=True)
-        if retcode != 0:
-            self.skipTest("Failed to run '{}', not installed?".format(cmd))
-
-    def test_000_txt(self):
-        filename = "/home/user/test_file.txt"
-        self.prepare_txt(filename)
-        self.open_file_and_check_viewer(filename, ["vim", "user@"],
-                                        ["gedit", "emacs", "libreoffice"])
-
-    def test_001_pdf(self):
-        filename = "/home/user/test_file.pdf"
-        self.prepare_pdf(filename)
-        self.open_file_and_check_viewer(filename, [],
-                                        ["evince"])
-
-    def test_002_doc(self):
-        filename = "/home/user/test_file.doc"
-        self.prepare_doc(filename)
-        self.open_file_and_check_viewer(filename, [],
-                                        ["libreoffice", "abiword"])
-
-    def test_003_pptx(self):
-        filename = "/home/user/test_file.pptx"
-        self.prepare_pptx(filename)
-        self.open_file_and_check_viewer(filename, [],
-                                        ["libreoffice"])
-
-    def test_004_png(self):
-        filename = "/home/user/test_file.png"
-        self.prepare_png(filename)
-        self.open_file_and_check_viewer(filename, [],
-                                        ["shotwell", "eog", "display"])
-
-    def test_005_jpg(self):
-        filename = "/home/user/test_file.jpg"
-        self.prepare_jpg(filename)
-        self.open_file_and_check_viewer(filename, [],
-                                        ["shotwell", "eog", "display"])
-
-    def test_006_jpeg(self):
-        filename = "/home/user/test_file.jpeg"
-        self.prepare_jpg(filename)
-        self.open_file_and_check_viewer(filename, [],
-                                        ["shotwell", "eog", "display"])
-
-    def test_010_url(self):
-        self.open_file_and_check_viewer("https://www.qubes-os.org/", [],
-                                        ["Firefox", "Iceweasel", "Navigator"])
-
-    def test_100_txt_dispvm(self):
-        filename = "/home/user/test_file.txt"
-        self.prepare_txt(filename)
-        self.open_file_and_check_viewer(filename, ["vim", "user@"],
-                                        ["gedit", "emacs", "libreoffice"],
-                                        dispvm=True)
-
-    def test_101_pdf_dispvm(self):
-        filename = "/home/user/test_file.pdf"
-        self.prepare_pdf(filename)
-        self.open_file_and_check_viewer(filename, [],
-                                        ["evince"],
-                                        dispvm=True)
-
-    def test_102_doc_dispvm(self):
-        filename = "/home/user/test_file.doc"
-        self.prepare_doc(filename)
-        self.open_file_and_check_viewer(filename, [],
-                                        ["libreoffice", "abiword"],
-                                        dispvm=True)
-
-    def test_103_pptx_dispvm(self):
-        filename = "/home/user/test_file.pptx"
-        self.prepare_pptx(filename)
-        self.open_file_and_check_viewer(filename, [],
-                                        ["libreoffice"],
-                                        dispvm=True)
-
-    def test_104_png_dispvm(self):
-        filename = "/home/user/test_file.png"
-        self.prepare_png(filename)
-        self.open_file_and_check_viewer(filename, [],
-                                        ["shotwell", "eog", "display"],
-                                        dispvm=True)
-
-    def test_105_jpg_dispvm(self):
-        filename = "/home/user/test_file.jpg"
-        self.prepare_jpg(filename)
-        self.open_file_and_check_viewer(filename, [],
-                                        ["shotwell", "eog", "display"],
-                                        dispvm=True)
-
-    def test_106_jpeg_dispvm(self):
-        filename = "/home/user/test_file.jpeg"
-        self.prepare_jpg(filename)
-        self.open_file_and_check_viewer(filename, [],
-                                        ["shotwell", "eog", "display"],
-                                        dispvm=True)
-
-    def test_110_url_dispvm(self):
-        self.open_file_and_check_viewer("https://www.qubes-os.org/", [],
-                                        ["Firefox", "Iceweasel", "Navigator"],
-                                        dispvm=True)
-
 
 def load_tests(loader, tests, pattern):
     try:
@@ -1705,21 +880,4 @@ def load_tests(loader, tests, pattern):
                 (TC_00_AppVMMixin, qubes.tests.QubesTestCase),
                 {'template': template})))
 
-        tests.addTests(loader.loadTestsFromTestCase(
-            type(
-                'TC_20_DispVM_' + template,
-                (TC_20_DispVMMixin, qubes.tests.QubesTestCase),
-                {'template': template})))
-        tests.addTests(loader.loadTestsFromTestCase(
-            type(
-                'TC_40_PVGrub_' + template,
-                (TC_40_PVGrub, qubes.tests.QubesTestCase),
-                {'template': template})))
-
-        tests.addTests(loader.loadTestsFromTestCase(
-            type(
-                'TC_50_MimeHandlers_' + template,
-                (TC_50_MimeHandlers, qubes.tests.QubesTestCase),
-                {'template': template})))
-
     return tests

From fd1b68166ce669c08c09bfceab992b13a45c4591 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 1 Aug 2016 13:45:22 +0200
Subject: [PATCH 60/69] tests: add test for GUI memory issues

QubesOS/qubes-issues#1028
---
 tests/vm_qrexec_gui.py | 145 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 145 insertions(+)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index 22fff2fd..e3d3539a 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -862,6 +862,151 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
         # some safety margin for FS metadata
         self.assertGreater(int(new_size.strip()), 5.8*1024**2)
 
+    @unittest.skipUnless(spawn.find_executable('xdotool'),
+                         "xdotool not installed")
+    def test_300_bug_1028_gui_memory_pinning(self):
+        """
+        If VM window composition buffers are relocated in memory, GUI will
+        still use old pointers and will display old pages
+        :return:
+        """
+        self.testvm1.memory = 800
+        self.testvm1.maxmem = 800
+        # exclude from memory balancing
+        self.testvm1.services['meminfo-writer'] = False
+        self.testvm1.start()
+        # and allow large map count
+        self.testvm1.run("echo 256000 > /proc/sys/vm/max_map_count",
+            user="root", wait=True)
+        allocator_c = (
+            "#include <sys/mman.h>\n"
+            "#include <stdlib.h>\n"
+            "#include <stdio.h>\n"
+            "\n"
+            "int main(int argc, char **argv) {\n"
+            "	int total_pages;\n"
+            "	char *addr, *iter;\n"
+            "\n"
+            "	total_pages = atoi(argv[1]);\n"
+            "	addr = mmap(NULL, total_pages * 0x1000, PROT_READ | "
+            "PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE | MAP_POPULATE, -1, 0);\n"
+            "	if (addr == MAP_FAILED) {\n"
+            "		perror(\"mmap\");\n"
+            "		exit(1);\n"
+            "	}\n"
+            "	printf(\"Stage1\\n\");\n"
+            "   fflush(stdout);\n"
+            "	getchar();\n"
+            "	for (iter = addr; iter < addr + total_pages*0x1000; iter += "
+            "0x2000) {\n"
+            "		if (mlock(iter, 0x1000) == -1) {\n"
+            "			perror(\"mlock\");\n"
+            "           fprintf(stderr, \"%d of %d\\n\", (iter-addr)/0x1000, "
+            "total_pages);\n"
+            "			exit(1);\n"
+            "		}\n"
+            "	}\n"
+            "	printf(\"Stage2\\n\");\n"
+            "   fflush(stdout);\n"
+            "	for (iter = addr+0x1000; iter < addr + total_pages*0x1000; "
+            "iter += 0x2000) {\n"
+            "		if (munmap(iter, 0x1000) == -1) {\n"
+            "			perror(\"munmap\");\n"
+            "			exit(1);\n"
+            "		}\n"
+            "	}\n"
+            "	printf(\"Stage3\\n\");\n"
+            "   fflush(stdout);\n"
+            "   fclose(stdout);\n"
+            "	getchar();\n"
+            "\n"
+            "	return 0;\n"
+            "}\n")
+
+        p = self.testvm1.run("cat > allocator.c", passio_popen=True)
+        p.communicate(allocator_c)
+        p = self.testvm1.run("gcc allocator.c -o allocator",
+            passio_popen=True, passio_stderr=True)
+        (stdout, stderr) = p.communicate()
+        if p.returncode != 0:
+            self.skipTest("allocator compile failed: {}".format(stderr))
+
+        # drop caches to have even more memory pressure
+        self.testvm1.run("echo 3 > /proc/sys/vm/drop_caches",
+            user="root", wait=True)
+
+        # now fragment all free memory
+        p = self.testvm1.run("grep ^MemFree: /proc/meminfo|awk '{print $2}'",
+            passio_popen=True)
+        memory_pages = int(p.communicate()[0].strip())
+        memory_pages /= 4 # 4k pages
+        alloc1 = self.testvm1.run(
+            "ulimit -l unlimited; exec /home/user/allocator {}".format(
+                memory_pages),
+            user="root", passio_popen=True, passio_stderr=True)
+        # wait for memory being allocated; can't use just .read(), because EOF
+        # passing is unreliable while the process is still running
+        alloc1.stdin.write("\n")
+        alloc1.stdin.flush()
+        alloc_out = alloc1.stdout.read(len("Stage1\nStage2\nStage3\n"))
+
+        if "Stage3" not in alloc_out:
+            # read stderr only in case of failed assert, but still have nice
+            # failure message (don't use self.fail() directly)
+            self.assertIn("Stage3", alloc_out, alloc1.stderr.read())
+
+        # now, launch some window - it should get fragmented composition buffer
+        # it is important to have some changing content there, to generate
+        # content update events (aka damage notify)
+        proc = self.testvm1.run("gnome-terminal --full-screen -e top",
+            passio_popen=True)
+
+        # help xdotool a little...
+        time.sleep(2)
+        # get window ID
+        search = subprocess.Popen(['xdotool', 'search', '--sync',
+            '--onlyvisible', '--class', self.testvm1.name + ':.*erminal'],
+            stdout=subprocess.PIPE)
+        winid = search.communicate()[0].strip()
+        xprop = subprocess.Popen(['xprop', '-notype', '-id', winid,
+            '_QUBES_VMWINDOWID'], stdout=subprocess.PIPE)
+        vm_winid = xprop.stdout.read().strip().split(' ')[4]
+
+        # now free the fragmented memory and trigger compaction
+        alloc1.stdin.write("\n")
+        alloc1.wait()
+        self.testvm1.run("echo 1 > /proc/sys/vm/compact_memory", user="root")
+
+        # now window may be already "broken"; to be sure, allocate (=zero)
+        # some memory
+        alloc2 = self.testvm1.run(
+            "ulimit -l unlimited; /home/user/allocator {}".format(memory_pages),
+            user="root", passio_popen=True, passio_stderr=True)
+        alloc2.stdout.read(len("Stage1\n"))
+
+        # wait for damage notify - top updates every 3 sec by default
+        time.sleep(6)
+
+        # now take screenshot of the window, from dom0 and VM
+        # choose pnm format, as it doesn't have any useless metadata - easy
+        # to compare
+        p = self.testvm1.run("import -window {} pnm:-".format(vm_winid),
+            passio_popen=True, passio_stderr=True)
+        (vm_image, stderr) = p.communicate()
+        if p.returncode != 0:
+            raise Exception("Failed to get VM window image: {}".format(
+                stderr))
+
+        p = subprocess.Popen(["import", "-window", winid, "pnm:-"],
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        (dom0_image, stderr) = p.communicate()
+        if p.returncode != 0:
+            raise Exception("Failed to get dom0 window image: {}".format(
+                stderr))
+
+        if vm_image != dom0_image:
+            self.fail("Dom0 window doesn't match VM window content")
+
 
 def load_tests(loader, tests, pattern):
     try:

From dca30e815b858cb5ec162298e6a43d2ba8d03714 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 2 Aug 2016 02:10:26 +0200
Subject: [PATCH 61/69] tests: enable 'use-default-netvm' while restoring old
 backup

Firewall VM was named 'firewallvm' at that time.
---
 tests/backupcompatibility.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/backupcompatibility.py b/tests/backupcompatibility.py
index cb7ca133..e9a2e84b 100644
--- a/tests/backupcompatibility.py
+++ b/tests/backupcompatibility.py
@@ -443,6 +443,7 @@ class TC_00_BackupCompatibility(qubes.tests.BackupTestsMixin, qubes.tests.QubesT
 
         self.restore_backup(self.backupdir, options={
             'use-default-template': True,
+            'use-default-netvm': True,
         })
         self.assertIsNotNone(self.qc.get_vm_by_name("test-template-clone"))
         self.assertIsNotNone(self.qc.get_vm_by_name("test-testproxy"))

From 07ac6c9ba18f8f6708bdc265b377be82f99a12a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 2 Aug 2016 02:11:56 +0200
Subject: [PATCH 62/69] tests: fix handling LibreOffice in DispVM tests

It's first window is a splash screen.
---
 tests/dispvm.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/tests/dispvm.py b/tests/dispvm.py
index 2c5c056c..d35e6f86 100644
--- a/tests/dispvm.py
+++ b/tests/dispvm.py
@@ -394,7 +394,16 @@ class TC_20_DispVMMixin(qubes.tests.SystemTestsMixin):
             retcode = search.wait()
             if retcode == 0:
                 winid = search.stdout.read().strip()
-                break
+                # get window title
+                (window_title, _) = subprocess.Popen(
+                    ['xdotool', 'getwindowname', winid], stdout=subprocess.PIPE). \
+                    communicate()
+                window_title = window_title.strip()
+                # ignore LibreOffice splash screen and window with no title
+                # set yet
+                if window_title and not window_title.startswith("LibreOffice")\
+                        and not window_title == 'VMapp command':
+                    break
             wait_count += 1
             if wait_count > 100:
                 self.fail("Timeout while waiting for editor window")

From 72d60788e447226b84528997ee74fc905ea07fb5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 2 Aug 2016 02:12:49 +0200
Subject: [PATCH 63/69] tests: fix Debian repository format

Use SHA256 instead of SHA1 - apt-get in Debian 9 rejects SHA1.
Fix date format (according to apt-get in Debian 9).
---
 tests/network.py     | 11 +++++------
 tests/regressions.py |  1 -
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/tests/network.py b/tests/network.py
index 2317f8dc..a1c57a09 100644
--- a/tests/network.py
+++ b/tests/network.py
@@ -518,10 +518,10 @@ class VmUpdatesMixin(qubes.tests.SystemTestsMixin):
         p = self.netvm_repo.run(
             "mkdir -p /tmp/apt-repo/dists/test && "
             "cd /tmp/apt-repo/dists/test && "
-            "cat > Release <<EOF && "
-            "echo '' $(sha1sum {p} | cut -f 1 -d ' ') $(stat -c %s {p}) {p}"
+            "cat > Release && "
+            "echo '' $(sha256sum {p} | cut -f 1 -d ' ') $(stat -c %s {p}) {p}"
             " >> Release && "
-            "echo '' $(sha1sum {z} | cut -f 1 -d ' ') $(stat -c %s {z}) {z}"
+            "echo '' $(sha256sum {z} | cut -f 1 -d ' ') $(stat -c %s {z}) {z}"
             " >> Release"
             .format(p="main/binary-amd64/Packages",
                     z="main/binary-amd64/Packages.gz"),
@@ -531,11 +531,10 @@ class VmUpdatesMixin(qubes.tests.SystemTestsMixin):
             "Label: Test repo\n"
             "Suite: test\n"
             "Codename: test\n"
-            "Date: Tue, 27 Oct 2015 03:22:09 +0100\n"
+            "Date: Tue, 27 Oct 2015 03:22:09 UTC\n"
             "Architectures: amd64\n"
             "Components: main\n"
-            "SHA1:\n"
-            "EOF\n"
+            "SHA256:\n"
         )
         p.stdin.close()
         if p.wait() != 0:
diff --git a/tests/regressions.py b/tests/regressions.py
index f61f3a2b..673de721 100644
--- a/tests/regressions.py
+++ b/tests/regressions.py
@@ -78,4 +78,3 @@ class TC_00_Regressions(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase)
             p.stdin.close()
             self.assertTrue(dispvm_name.startswith("disp"),
                                  "Try {} failed".format(try_no))
-

From 5c9157be05c2425d3bf2e241e974046ee45dd2f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 2 Aug 2016 02:13:58 +0200
Subject: [PATCH 64/69] tests: fix newline input in DispVM editor tests

Just '\n' isn't enough for xdotool to enter newline.
---
 tests/dispvm.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/tests/dispvm.py b/tests/dispvm.py
index d35e6f86..164a202c 100644
--- a/tests/dispvm.py
+++ b/tests/dispvm.py
@@ -330,7 +330,9 @@ class TC_20_DispVMMixin(qubes.tests.SystemTestsMixin):
         time.sleep(1)
         if "gedit" in window_title:
             subprocess.check_call(['xdotool', 'windowactivate', '--sync', winid,
-                'type', 'Test test 2\n'])
+                                   'type', 'Test test 2'])
+            subprocess.check_call(['xdotool', 'key', '--window', winid,
+                                   'key', 'Return'])
             time.sleep(0.5)
             subprocess.check_call(['xdotool',
                                    'key', 'ctrl+s', 'ctrl+q'])
@@ -345,14 +347,18 @@ class TC_20_DispVMMixin(qubes.tests.SystemTestsMixin):
                 winid = search.stdout.read().strip()
             time.sleep(0.5)
             subprocess.check_call(['xdotool', 'windowactivate', '--sync', winid,
-                'type', 'Test test 2\n'])
+                                   'type', 'Test test 2'])
+            subprocess.check_call(['xdotool', 'key', '--window', winid,
+                                   'key', 'Return'])
             time.sleep(0.5)
             subprocess.check_call(['xdotool',
                                    'key', '--delay', '100', 'ctrl+s',
                 'Return', 'ctrl+q'])
         elif "emacs" in window_title:
             subprocess.check_call(['xdotool', 'windowactivate', '--sync', winid,
-                                   'type', 'Test test 2\n'])
+                                   'type', 'Test test 2'])
+            subprocess.check_call(['xdotool', 'key', '--window', winid,
+                                   'key', 'Return'])
             time.sleep(0.5)
             subprocess.check_call(['xdotool',
                                    'key', 'ctrl+x', 'ctrl+s'])
@@ -360,7 +366,9 @@ class TC_20_DispVMMixin(qubes.tests.SystemTestsMixin):
                                    'key', 'ctrl+x', 'ctrl+c'])
         elif "vim" in window_title or "user@" in window_title:
             subprocess.check_call(['xdotool', 'windowactivate', '--sync', winid,
-                                   'key', 'i', 'type', 'Test test 2\n'])
+                                   'key', 'i', 'type', 'Test test 2'])
+            subprocess.check_call(['xdotool', 'key', '--window', winid,
+                                   'key', 'Return'])
             subprocess.check_call(
                 ['xdotool',
                  'key', 'Escape', 'colon', 'w', 'q', 'Return'])

From 0968c25486bd20612aee7f657e816a16df77eee9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 2 Aug 2016 02:15:08 +0200
Subject: [PATCH 65/69] tests: when creating AppVM based on whonix-ws, connect
 it to tor

Use sys-whonix if exists. This makes network-related tests more
realistic.
---
 tests/extra.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/tests/extra.py b/tests/extra.py
index c46a8a81..d818c503 100644
--- a/tests/extra.py
+++ b/tests/extra.py
@@ -34,6 +34,7 @@ class ExtraTestCase(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
     def setUp(self):
         super(ExtraTestCase, self).setUp()
         self.qc.unlock_db()
+        self.default_netvm = None
 
     def create_vms(self, names):
         """
@@ -52,7 +53,9 @@ class ExtraTestCase(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
         for vmname in names:
             vm = self.qc.add_new_vm("QubesAppVm",
                                     name=self.make_vm_name(vmname),
-                                    template=template)
+                                    template=template,
+                                    uses_default_netvm=False,
+                                    netvm=self.default_netvm)
             vm.create_on_disk(verbose=False)
         self.save_and_reload_db()
         self.qc.unlock_db()
@@ -67,9 +70,11 @@ class ExtraTestCase(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
         """
         Enable access to the network. Must be called before creating VMs.
         """
-        # nothing to do in core2
-        pass
-
+        self.default_netvm = self.qc.get_default_netvm()
+        if self.template.startswith('whonix-ws'):
+            whonix_netvm = self.qc.get_vm_by_name('sys-whonix')
+            if whonix_netvm:
+                self.default_netvm = whonix_netvm
 
 def load_tests(loader, tests, pattern):
     for entry in pkg_resources.iter_entry_points('qubes.tests.extra'):

From 09b49feea6fa1c5cacbbc46c73414ca926ea0ff8 Mon Sep 17 00:00:00 2001
From: HW42 <hw42@ipsumj.de>
Date: Fri, 5 Aug 2016 15:44:05 +0200
Subject: [PATCH 66/69] prepare-volatile-img.sh: don't run as root

This is no longer necessary since volatile.img is formated inside the
VM. This also fixes DispVM creation if the user sets a restrictive umask
for root. Maybe related to #2200.
---
 linux/aux-tools/prepare-volatile-img.sh | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/linux/aux-tools/prepare-volatile-img.sh b/linux/aux-tools/prepare-volatile-img.sh
index 40e22f5d..b32142b7 100755
--- a/linux/aux-tools/prepare-volatile-img.sh
+++ b/linux/aux-tools/prepare-volatile-img.sh
@@ -1,9 +1,5 @@
 #!/bin/sh
 
-if [ "`id -u`" != "0" ]; then
-	exec sudo $0 $*
-fi
-
 set -e
 
 if ! echo $PATH | grep -q sbin; then

From d0ddb3d17cb2243a5f490c8952c86cfa7572191c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sun, 7 Aug 2016 00:08:46 +0200
Subject: [PATCH 67/69] dispvm: error out on saved-cows.tar creation error

If it fails - for example because of too restrictive volatile.img
permissions, subsequent DispVM will not be really disposable.
The original permissions issue should be fixed by previous commit, this
one makes sure that such errors will not be ignored.

Fixes QubesOS/qubes-issues#2200
---
 dispvm/qubes-prepare-saved-domain.sh | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/dispvm/qubes-prepare-saved-domain.sh b/dispvm/qubes-prepare-saved-domain.sh
index 9c5c0f98..d5c3ad96 100755
--- a/dispvm/qubes-prepare-saved-domain.sh
+++ b/dispvm/qubes-prepare-saved-domain.sh
@@ -74,8 +74,13 @@ fstype=`df --output=fstype $VMDIR | tail -n 1`
 if [ "$fstype" = "tmpfs" ]; then
     # bsdtar doesn't work on tmpfs because FS_IOC_FIEMAP ioctl isn't supported
     # there
-    tar -cSf saved-cows.tar volatile.img
+    tar -cSf saved-cows.tar volatile.img || exit 1
 else
-    bsdtar -cSf saved-cows.tar volatile.img
+    errors=`bsdtar -cSf saved-cows.tar volatile.img 2>&1`
+    if [ -n "$errors" ]; then
+        echo "Failed to create saved-cows.tar: $errors" >&2
+        rm -f saved-cows.tar
+        exit 1
+    fi
 fi
 echo "DVM savefile created successfully."

From 290899274185a36102b7545f445dae00c0ef6b34 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sun, 7 Aug 2016 16:07:30 +0200
Subject: [PATCH 68/69] prepare-volatile-img.sh: create volatile.img group
 accessible

Otherwise qvm-create-default-dvm may fail to include it in
saved-cows.tar, which will lead to DispVM being not really disposable.

Fixes QubesOS/qubes-issues#2200
---
 linux/aux-tools/prepare-volatile-img.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/linux/aux-tools/prepare-volatile-img.sh b/linux/aux-tools/prepare-volatile-img.sh
index b32142b7..822affa9 100755
--- a/linux/aux-tools/prepare-volatile-img.sh
+++ b/linux/aux-tools/prepare-volatile-img.sh
@@ -20,5 +20,6 @@ if [ -e "$FILENAME" ]; then
 	exit 1
 fi
 
+umask 002
 TOTAL_SIZE=$[ $ROOT_SIZE + $SWAP_SIZE + 512 ]
 truncate -s ${TOTAL_SIZE}M "$FILENAME"

From 10c44e87221dfb5431934376648ce05855a63cda Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sun, 7 Aug 2016 16:11:49 +0200
Subject: [PATCH 69/69] version 3.2.8

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index 406ebcbd..f092941a 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.2.7
+3.2.8