From a96d49a40afdc9777a85322765bf0f3da4a8e3cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 1 Jul 2015 04:33:04 +0200
Subject: [PATCH 001/225] core: properly redirect qrexec output to /dev/null
 when necessary

/dev/null was opened in read-only mode ("rw" is invalid value), so
qrexec couldn't write there.
---
 core-modules/000QubesVm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index 18f8813b..c793d932 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -1527,7 +1527,7 @@ class QubesVm(object):
 
         call_kwargs = {}
         if ignore_stderr or not passio:
-            null = open("/dev/null", "rw")
+            null = open("/dev/null", "w+")
             call_kwargs['stderr'] = null
         if not passio:
             call_kwargs['stdin'] = null

From 522bfc427ada22498539baa579d85992c6346f7e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 1 Jul 2015 04:35:09 +0200
Subject: [PATCH 002/225] core: fix template-based HVM disk handling

We use only one device-mapper layer for HVMs, and this isn't the same as
for PV - it is that one, which PV does in initramfs.
Device-mapper layers summary for template-based VMs:
PV: root.img+root-cow.img (dom0) -> xvda, xvda+volatile.img (VM)
HVM: root.img+volatile.img (dom0)
---
 core-modules/01QubesHVm.py | 43 ++----------------------------------
 core/storage/xen.py        | 45 +++++++++++++++++++++++++++++++++++++-
 2 files changed, 46 insertions(+), 42 deletions(-)

diff --git a/core-modules/01QubesHVm.py b/core-modules/01QubesHVm.py
index 08fd0118..12e87df6 100644
--- a/core-modules/01QubesHVm.py
+++ b/core-modules/01QubesHVm.py
@@ -96,7 +96,7 @@ class QubesHVm(QubesVm):
             (not 'xml_element' in kwargs or kwargs['xml_element'].get('guiagent_installed') is None):
             self.services['meminfo-writer'] = False
 
-        self.storage.volatile_img = None
+        self.storage.rootcow_img = None
 
     @property
     def type(self):
@@ -233,9 +233,7 @@ class QubesHVm(QubesVm):
         if self.is_running():
             raise NotImplementedError("Online resize of HVM's private.img not implemented, shutdown the VM first")
 
-        f_private = open (self.private_img, "a+b")
-        f_private.truncate (size)
-        f_private.close ()
+        self.storage.resize_private_img(size)
 
     def resize_root_img(self, size):
         if self.template:
@@ -255,14 +253,6 @@ class QubesHVm(QubesVm):
         f_root.truncate (size)
         f_root.close ()
 
-    def get_rootdev(self, source_template=None):
-        if self.template:
-            return "'script:snapshot:{template_root}:{volatile},xvda,w',".format(
-                    template_root=self.template.root_img,
-                    volatile=self.volatile_img)
-        else:
-            return "'script:file:{root_img},xvda,w',".format(root_img=self.root_img)
-
     def get_config_params(self):
 
         params = super(QubesHVm, self).get_config_params()
@@ -295,34 +285,6 @@ class QubesHVm(QubesVm):
 
         return True
 
-    def reset_volatile_storage(self, **kwargs):
-        assert not self.is_running(), "Attempt to clean volatile image of running VM!"
-
-        source_template = kwargs.get("source_template", self.template)
-
-        if source_template is None:
-            # Nothing to do on non-template based VM
-            return
-
-        if os.path.exists (self.volatile_img):
-            if self.debug:
-                if os.path.getmtime(self.template.root_img) > os.path.getmtime(self.volatile_img):
-                    if kwargs.get("verbose", False):
-                        print >>sys.stderr, "--> WARNING: template have changed, resetting root.img"
-                else:
-                    if kwargs.get("verbose", False):
-                        print >>sys.stderr, "--> Debug mode: not resetting root.img"
-                        print >>sys.stderr, "--> Debug mode: if you want to force root.img reset, either update template VM, or remove volatile.img file"
-                    return
-            os.remove (self.volatile_img)
-
-        f_volatile = open (self.volatile_img, "w")
-        f_root = open (self.template.root_img, "r")
-        f_root.seek(0, os.SEEK_END)
-        f_volatile.truncate (f_root.tell()) # make empty sparse file of the same size as root.img
-        f_volatile.close ()
-        f_root.close()
-
     @property
     def vif(self):
         if self.xid < 0:
@@ -463,7 +425,6 @@ class QubesHVm(QubesVm):
                         guid_pid = open(guid_pidfile).read().strip()
                         os.kill(int(guid_pid), 15)
 
-
     def suspend(self):
         if dry_run:
             return
diff --git a/core/storage/xen.py b/core/storage/xen.py
index e026b6b3..5daa3bbd 100644
--- a/core/storage/xen.py
+++ b/core/storage/xen.py
@@ -76,7 +76,18 @@ class QubesXenVmStorage(QubesVmStorage):
                     "{dir}/root.img:{dir}/root-cow.img".format(
                         dir=self.vmdir),
                     "block-origin", self.root_dev, True)
+        elif self.vm.template and not self.vm.template.storage.rootcow_img:
+            # HVM template-based VM - template doesn't have own
+            # root-cow.img, only one device-mapper layer
+            return self._format_disk_dev(
+                    "{tpldir}/root.img:{vmdir}/volatile.img".format(
+                        tpldir=self.vm.template.dir_path,
+                        vmdir=self.vmdir),
+                    "block-snapshot", self.root_dev, True)
         elif self.vm.template:
+            # any other template-based VM - two device-mapper layers: one
+            # in dom0 (here) from root+root-cow, and another one from
+            # this+volatile.img
             return self._format_disk_dev(
                     "{dir}/root.img:{dir}/root-cow.img".format(
                         dir=self.vm.template.dir_path),
@@ -170,7 +181,8 @@ class QubesXenVmStorage(QubesVmStorage):
 
     def commit_template_changes(self):
         assert self.vm.is_template()
-        # TODO: move rootcow_img to this class; the same for vm.is_outdated()
+        if not self.rootcow_img:
+            return
         if os.path.exists (self.rootcow_img):
            os.rename (self.rootcow_img, self.rootcow_img + '.old')
 
@@ -183,3 +195,34 @@ class QubesXenVmStorage(QubesVmStorage):
         f_root.close()
         os.umask(old_umask)
 
+    def reset_volatile_storage(self, verbose = False, source_template = None):
+        if source_template is None:
+            source_template = self.vm.template
+
+        if source_template is not None:
+            # template-based VM with only one device-mapper layer -
+            # volatile.img used as upper layer on root.img, no root-cow.img
+            # intermediate layer
+            if not source_template.storage.rootcow_img:
+                if os.path.exists(self.volatile_img):
+                    if self.vm.debug:
+                        if os.path.getmtime(source_template.storage.root_img)\
+                                > os.path.getmtime(self.volatile_img):
+                            if verbose:
+                                print >>sys.stderr, "--> WARNING: template have changed, resetting root.img"
+                        else:
+                            if verbose:
+                                print >>sys.stderr, "--> Debug mode: not resetting root.img"
+                                print >>sys.stderr, "--> Debug mode: if you want to force root.img reset, either update template VM, or remove volatile.img file"
+                            return
+                    os.remove(self.volatile_img)
+
+                f_volatile = open(self.volatile_img, "w")
+                f_root = open(source_template.storage.root_img, "r")
+                f_root.seek(0, os.SEEK_END)
+                f_volatile.truncate(f_root.tell()) # make empty sparse file of the same size as root.img
+                f_volatile.close()
+                f_root.close()
+                return
+        super(QubesXenVmStorage, self).reset_volatile_storage(
+            verbose=verbose, source_template=source_template)

From 402d1b9a3dc9f2ace54db3aa2ecdf3c81b7f0d0b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 1 Jul 2015 04:40:48 +0200
Subject: [PATCH 003/225] indentation fix

---
 core-modules/01QubesHVm.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core-modules/01QubesHVm.py b/core-modules/01QubesHVm.py
index 12e87df6..16e33376 100644
--- a/core-modules/01QubesHVm.py
+++ b/core-modules/01QubesHVm.py
@@ -262,8 +262,8 @@ class QubesHVm(QubesVm):
         params['volatiledev'] = ''
 
         if self.timezone.lower() == 'localtime':
-             params['time_basis'] = 'localtime'
-             params['timeoffset'] = '0'
+            params['time_basis'] = 'localtime'
+            params['timeoffset'] = '0'
         elif self.timezone.isdigit():
             params['time_basis'] = 'UTC'
             params['timeoffset'] = self.timezone

From d9c299074701f93766da5cd4ee1b64064d53203b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 1 Jul 2015 04:41:09 +0200
Subject: [PATCH 004/225] core: fix creation of private.img at VM startup

This code is used when VM is migrated from older system, where HVM
didn't have private.img.
---
 core/storage/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/storage/__init__.py b/core/storage/__init__.py
index a67a921d..41f3e73f 100644
--- a/core/storage/__init__.py
+++ b/core/storage/__init__.py
@@ -198,4 +198,4 @@ class QubesVmStorage(object):
         if self.private_img and not os.path.exists (self.private_img):
             print >>sys.stderr, "WARNING: Creating empty VM private image file: {0}".\
                 format(self.private_img)
-            self.storage.create_on_disk_private_img(verbose=False)
+            self.create_on_disk_private_img(verbose=False)

From 310ba9f1dfd0e70eeda01e6dcf740403bd722a13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 1 Jul 2015 04:42:44 +0200
Subject: [PATCH 005/225] block: do not treat disks of not running VMs as used

There are legitimate use cases when one want to attach disk of one VM to
some other. Do not try to detach the disk from powered down VM in such
case.
---
 core/qubesutils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/core/qubesutils.py b/core/qubesutils.py
index 130faf8a..aba05a3c 100644
--- a/core/qubesutils.py
+++ b/core/qubesutils.py
@@ -337,6 +337,8 @@ def block_check_attached(qvmc, device):
         if vm.qid == 0:
             # Connecting devices to dom0 not supported
             continue
+        if not vm.is_running():
+            continue
         try:
             libvirt_domain = vm.libvirt_domain
             if libvirt_domain:

From 677a79b213ddd39729e0a282fcbff7f5cb157269 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 1 Jul 2015 04:44:41 +0200
Subject: [PATCH 006/225] hvm: change default graphics to std vga ('xen')

The resulting qemu option is -std-vga.
This apparently this is much better handled by many OSes.
---
 vm-config/xen-vm-template-hvm.xml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/vm-config/xen-vm-template-hvm.xml b/vm-config/xen-vm-template-hvm.xml
index ba2dc862..13431b33 100644
--- a/vm-config/xen-vm-template-hvm.xml
+++ b/vm-config/xen-vm-template-hvm.xml
@@ -29,7 +29,9 @@
 {netdev}
 {pcidevs}
     <input type='tablet' bus='usb'/>
-    <video type='vga'/>
+    <video type='vga'>
+      <model type='xen' vram='16384'/>
+    </video>
   </devices>
 </domain>
 

From 4bf73a5d7fd4022608c4afdfc662be3a78571aee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 1 Jul 2015 04:46:46 +0200
Subject: [PATCH 007/225] Increase default swiotlb size to 16MB (#1038)

Fixes qubesos/qubes-issues#1038
---
 core/qubes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/qubes.py b/core/qubes.py
index 8f814709..a66f3188 100755
--- a/core/qubes.py
+++ b/core/qubes.py
@@ -101,7 +101,7 @@ defaults = {
     'libvirt_uri': 'xen:///',
     'memory': 400,
     'kernelopts': "nopat",
-    'kernelopts_pcidevs': "nopat iommu=soft swiotlb=4096",
+    'kernelopts_pcidevs': "nopat iommu=soft swiotlb=8192",
 
     'dom0_update_check_interval': 6*3600,
 

From df509cb780417a1b0e8dd3c7706ed25db4f7c447 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 1 Jul 2015 04:47:52 +0200
Subject: [PATCH 008/225] Fix qubes.NotifyTools service (not import xenstore)

---
 qubes-rpc/qubes-notify-tools | 1 -
 1 file changed, 1 deletion(-)

diff --git a/qubes-rpc/qubes-notify-tools b/qubes-rpc/qubes-notify-tools
index ce85009f..bb93e116 100755
--- a/qubes-rpc/qubes-notify-tools
+++ b/qubes-rpc/qubes-notify-tools
@@ -5,7 +5,6 @@ import re
 import sys
 import subprocess
 from qubes.qubes import QubesVmCollection,QubesException,QubesHVm
-from qubes.qubes import xs
 
 def main():
 

From 6d19a9c2ccc7e7d688aa5dc4c87d2c15bb9d2644 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 1 Jul 2015 07:03:52 +0200
Subject: [PATCH 009/225] Do not hold the lock while retrieving application
 icons (qubes.NotifyTools)

This can take quite a long time, so do not block the whole system.
---
 qubes-rpc/qubes-notify-tools | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/qubes-rpc/qubes-notify-tools b/qubes-rpc/qubes-notify-tools
index bb93e116..f0ec3b5a 100755
--- a/qubes-rpc/qubes-notify-tools
+++ b/qubes-rpc/qubes-notify-tools
@@ -14,6 +14,7 @@ def main():
         print >> sys.stderr, 'This script must be called as qrexec service!'
         exit(1)
 
+    source_vm = None
     qvm_collection = QubesVmCollection()
     qvm_collection.lock_db_for_writing()
     try:
@@ -74,15 +75,17 @@ def main():
 
         qvm_collection.save()
 
-        retcode = subprocess.call(['qvm-sync-appmenus', '--force-rpc'])
-        if retcode == 0 and hasattr(source_vm, 'appmenus_recreate'):
-            # TODO: call the same for child VMs? This isn't done for Linux VMs,
-            # so probably should be ignored for Windows also
-            source_vm.appmenus_recreate()
     except Exception as e:
         print >> sys.stderr, e.message
         exit(1)
     finally:
         qvm_collection.unlock_db()
 
+    retcode = subprocess.call(['qvm-sync-appmenus', '--force-rpc'])
+    if retcode == 0 and hasattr(source_vm, 'appmenus_recreate'):
+        # TODO: call the same for child VMs? This isn't done for Linux VMs,
+        # so probably should be ignored for Windows also
+        source_vm.appmenus_recreate()
+
+
 main()

From 205a28ecc24e78a40ae12745bf70a4a739cdfa05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 1 Jul 2015 07:05:12 +0200
Subject: [PATCH 010/225] version 3.0.15

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index 9bbe6516..e265a8cb 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.0.14
+3.0.15

From 6d1f40219cf986a45da0b6382aba8fc1ab4deaca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 7 Jul 2015 21:41:54 +0200
Subject: [PATCH 011/225] tests: code style fixes, no functional change

---
 tests/vm_qrexec_gui.py | 217 +++++++++++++++++++++--------------------
 1 file changed, 111 insertions(+), 106 deletions(-)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index e9ec8ad8..5c8c80b5 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -39,18 +39,19 @@ TEST_DATA = "0123456789" * 1024
 class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
     def setUp(self):
         super(TC_00_AppVMMixin, self).setUp()
-        self.testvm1 = self.qc.add_new_vm("QubesAppVm",
+        self.testvm1 = self.qc.add_new_vm(
+            "QubesAppVm",
             name=self.make_vm_name('vm1'),
             template=self.qc.get_vm_by_name(self.template))
         self.testvm1.create_on_disk(verbose=False)
-        self.testvm2 = self.qc.add_new_vm("QubesAppVm",
+        self.testvm2 = self.qc.add_new_vm(
+            "QubesAppVm",
             name=self.make_vm_name('vm2'),
             template=self.qc.get_vm_by_name(self.template))
         self.testvm2.create_on_disk(verbose=False)
         self.qc.save()
         self.qc.unlock_db()
 
-
     def test_000_start_shutdown(self):
         self.testvm1.start()
         self.assertEquals(self.testvm1.get_power_state(), "Running")
@@ -65,7 +66,6 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
         time.sleep(1)
         self.assertEquals(self.testvm1.get_power_state(), "Halted")
 
-
     @unittest.skipUnless(spawn.find_executable('xdotool'),
                          "xdotool not installed")
     def test_010_run_gui_app(self):
@@ -73,21 +73,25 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
         self.assertEquals(self.testvm1.get_power_state(), "Running")
         self.testvm1.run("gnome-terminal")
         wait_count = 0
-        while subprocess.call(['xdotool', 'search', '--name', 'user@%s' %
-                self.testvm1.name], stdout=open(os.path.devnull, 'w'),
-                              stderr=subprocess.STDOUT) > 0:
+        while subprocess.call(
+                ['xdotool', 'search', '--name', 'user@{}'.
+                    format(self.testvm1.name)],
+                stdout=open(os.path.devnull, 'w'),
+                stderr=subprocess.STDOUT) > 0:
             wait_count += 1
             if wait_count > 100:
                 self.fail("Timeout while waiting for gnome-terminal window")
             time.sleep(0.1)
 
         time.sleep(0.5)
-        subprocess.check_call(['xdotool', 'search', '--name', 'user@%s' %
-                self.testvm1.name, 'windowactivate', 'type', 'exit\n'])
+        subprocess.check_call(
+            ['xdotool', 'search', '--name', 'user@{}'.format(self.testvm1.name),
+             'windowactivate', 'type', 'exit\n'])
 
         wait_count = 0
-        while subprocess.call(['xdotool', 'search', '--name', 'user@%s' %
-                self.testvm1.name], stdout=open(os.path.devnull, 'w'),
+        while subprocess.call(['xdotool', 'search', '--name',
+                               'user@{}'.format(self.testvm1.name)],
+                              stdout=open(os.path.devnull, 'w'),
                               stderr=subprocess.STDOUT) == 0:
             wait_count += 1
             if wait_count > 100:
@@ -95,13 +99,13 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
                           "termination")
             time.sleep(0.1)
 
-
     def test_050_qrexec_simple_eof(self):
         """Test for data and EOF transmission dom0->VM"""
         result = multiprocessing.Value('i', 0)
+
         def run(self, result):
             p = self.testvm1.run("cat", passio_popen=True,
-                            passio_stderr=True)
+                                 passio_stderr=True)
 
             (stdout, stderr) = p.communicate(TEST_DATA)
             if stdout != TEST_DATA:
@@ -123,10 +127,10 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
         elif result.value == 2:
             self.fail("Some data was printed to stderr")
 
-
     def test_051_qrexec_simple_eof_reverse(self):
         """Test for EOF transmission VM->dom0"""
         result = multiprocessing.Value('i', 0)
+
         def run(self, result):
             p = self.testvm1.run("echo test; exec >&-; cat > /dev/null",
                                  passio_popen=True, passio_stderr=True)
@@ -160,10 +164,10 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
         elif result.value == 3:
             self.fail("VM proceess didn't terminated on EOF")
 
-
     def test_052_qrexec_vm_service_eof(self):
         """Test for EOF transmission VM(src)->VM(dst)"""
         result = multiprocessing.Value('i', 0)
+
         def run(self, result):
             p = self.testvm1.run("/usr/lib/qubes/qrexec-client-vm %s test.EOF "
                                  "/bin/sh -c 'echo test; exec >&-; cat "
@@ -194,11 +198,11 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
         if result.value == 1:
             self.fail("Received data differs from what was expected")
 
-
     @unittest.expectedFailure
     def test_053_qrexec_vm_service_eof_reverse(self):
         """Test for EOF transmission VM(src)<-VM(dst)"""
         result = multiprocessing.Value('i', 0)
+
         def run(self, result):
             p = self.testvm1.run("/usr/lib/qubes/qrexec-client-vm %s test.EOF "
                                  "/bin/sh -c 'cat >&$SAVED_FD_1'"
@@ -229,7 +233,6 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
         if result.value == 1:
             self.fail("Received data differs from what was expected")
 
-
     def test_060_qrexec_exit_code_dom0(self):
         self.testvm1.start()
 
@@ -241,7 +244,6 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
         p.wait()
         self.assertEqual(3, p.returncode)
 
-
     @unittest.expectedFailure
     def test_065_qrexec_exit_code_vm(self):
         self.testvm1.start()
@@ -278,45 +280,44 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
         (stdout, stderr) = p.communicate()
         self.assertEqual(stdout, "3\n")
 
-
     @unittest.skipUnless(spawn.find_executable('xdotool'),
                          "xdotool not installed")
     def test_100_qrexec_filecopy(self):
         self.testvm1.start()
         self.testvm2.start()
         p = self.testvm1.run("qvm-copy-to-vm %s /etc/passwd" %
-                            self.testvm2.name, passio_popen=True,
-                            passio_stderr=True)
+                             self.testvm2.name, passio_popen=True,
+                             passio_stderr=True)
         # Confirm transfer
-        subprocess.check_call(['xdotool', 'search', '--sync', '--name', 'Question',
-                             'key', 'y'])
+        subprocess.check_call(
+            ['xdotool', 'search', '--sync', '--name', 'Question', 'key', 'y'])
         p.wait()
         self.assertEqual(p.returncode, 0, "qvm-copy-to-vm failed: %s" %
                          p.stderr.read())
         retcode = self.testvm2.run("diff /etc/passwd "
-                         "/home/user/QubesIncoming/%s/passwd" % self.testvm1.name, wait=True)
+                                   "/home/user/QubesIncoming/{}/passwd".format(
+                                       self.testvm1.name),
+                                   wait=True)
         self.assertEqual(retcode, 0, "file differs")
 
-
     @unittest.skipUnless(spawn.find_executable('xdotool'),
                          "xdotool not installed")
     def test_110_qrexec_filecopy_deny(self):
         self.testvm1.start()
         self.testvm2.start()
         p = self.testvm1.run("qvm-copy-to-vm %s /etc/passwd" %
-                            self.testvm2.name, passio_popen=True)
+                             self.testvm2.name, passio_popen=True)
         # Deny transfer
         subprocess.check_call(['xdotool', 'search', '--sync', '--name', 'Question',
-                             'key', 'n'])
+                              'key', 'n'])
         p.wait()
         self.assertNotEqual(p.returncode, 0, "qvm-copy-to-vm unexpectedly "
-                                          "succeeded")
+                            "succeeded")
         retcode = self.testvm1.run("ls /home/user/QubesIncoming/%s" %
-                                  self.testvm1.name, wait=True,
-                                  ignore_stderr=True)
+                                   self.testvm1.name, wait=True,
+                                   ignore_stderr=True)
         self.assertNotEqual(retcode, 0, "QubesIncoming exists although file "
-                                      "copy was "
-                                     "denied")
+                            "copy was denied")
 
     @unittest.skip("Xen gntalloc driver crashes when page is mapped in the "
                    "same domain")
@@ -325,16 +326,18 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
     def test_120_qrexec_filecopy_self(self):
         self.testvm1.start()
         p = self.testvm1.run("qvm-copy-to-vm %s /etc/passwd" %
-                            self.testvm1.name, passio_popen=True,
-                            passio_stderr=True)
+                             self.testvm1.name, passio_popen=True,
+                             passio_stderr=True)
         # Confirm transfer
         subprocess.check_call(['xdotool', 'search', '--sync', '--name', 'Question',
-                             'key', 'y'])
+                              'key', 'y'])
         p.wait()
         self.assertEqual(p.returncode, 0, "qvm-copy-to-vm failed: %s" %
                          p.stderr.read())
-        retcode = self.testvm1.run("diff /etc/passwd "
-                         "/home/user/QubesIncoming/%s/passwd" % self.testvm1.name, wait=True)
+        retcode = self.testvm1.run(
+            "diff /etc/passwd /home/user/QubesIncoming/{}/passwd".format(
+                self.testvm1.name),
+            wait=True)
         self.assertEqual(retcode, 0, "file differs")
 
     def test_200_timezone(self):
@@ -360,56 +363,56 @@ class TC_10_HVM(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
     # TODO: windows tools tests
 
     def test_000_create_start(self):
-        self.testvm1 = self.qc.add_new_vm("QubesHVm",
-            name=self.make_vm_name('vm1'))
-        self.testvm1.create_on_disk(verbose=False)
+        testvm1 = self.qc.add_new_vm("QubesHVm",
+                                     name=self.make_vm_name('vm1'))
+        testvm1.create_on_disk(verbose=False)
         self.qc.save()
         self.qc.unlock_db()
-        self.testvm1.start()
-        self.assertEquals(self.testvm1.get_power_state(), "Running")
+        testvm1.start()
+        self.assertEquals(testvm1.get_power_state(), "Running")
 
     def test_010_create_start_template(self):
-        self.templatevm = self.qc.add_new_vm("QubesTemplateHVm",
-            name=self.make_vm_name('template'))
-        self.templatevm.create_on_disk(verbose=False)
+        templatevm = self.qc.add_new_vm("QubesTemplateHVm",
+                                        name=self.make_vm_name('template'))
+        templatevm.create_on_disk(verbose=False)
         self.qc.save()
         self.qc.unlock_db()
 
-        self.templatevm.start()
-        self.assertEquals(self.templatevm.get_power_state(), "Running")
+        templatevm.start()
+        self.assertEquals(templatevm.get_power_state(), "Running")
 
     def test_020_create_start_template_vm(self):
-        self.templatevm = self.qc.add_new_vm("QubesTemplateHVm",
-            name=self.make_vm_name('template'))
-        self.templatevm.create_on_disk(verbose=False)
-        self.testvm2 = self.qc.add_new_vm("QubesHVm",
-            name=self.make_vm_name('vm2'),
-            template=self.templatevm)
-        self.testvm2.create_on_disk(verbose=False)
+        templatevm = self.qc.add_new_vm("QubesTemplateHVm",
+                                        name=self.make_vm_name('template'))
+        templatevm.create_on_disk(verbose=False)
+        testvm2 = self.qc.add_new_vm("QubesHVm",
+                                     name=self.make_vm_name('vm2'),
+                                     template=templatevm)
+        testvm2.create_on_disk(verbose=False)
         self.qc.save()
         self.qc.unlock_db()
 
-        self.testvm2.start()
-        self.assertEquals(self.testvm2.get_power_state(), "Running")
+        testvm2.start()
+        self.assertEquals(testvm2.get_power_state(), "Running")
 
     def test_030_prevent_simultaneus_start(self):
-        self.templatevm = self.qc.add_new_vm("QubesTemplateHVm",
-            name=self.make_vm_name('template'))
-        self.templatevm.create_on_disk(verbose=False)
-        self.testvm2 = self.qc.add_new_vm("QubesHVm",
-            name=self.make_vm_name('vm2'),
-            template=self.templatevm)
-        self.testvm2.create_on_disk(verbose=False)
+        templatevm = self.qc.add_new_vm("QubesTemplateHVm",
+                                        name=self.make_vm_name('template'))
+        templatevm.create_on_disk(verbose=False)
+        testvm2 = self.qc.add_new_vm("QubesHVm",
+                                     name=self.make_vm_name('vm2'),
+                                     template=templatevm)
+        testvm2.create_on_disk(verbose=False)
         self.qc.save()
         self.qc.unlock_db()
 
-        self.templatevm.start()
-        self.assertEquals(self.templatevm.get_power_state(), "Running")
-        self.assertRaises(QubesException, self.testvm2.start)
-        self.templatevm.force_shutdown()
-        self.testvm2.start()
-        self.assertEquals(self.testvm2.get_power_state(), "Running")
-        self.assertRaises(QubesException, self.templatevm.start)
+        templatevm.start()
+        self.assertEquals(templatevm.get_power_state(), "Running")
+        self.assertRaises(QubesException, testvm2.start)
+        templatevm.force_shutdown()
+        testvm2.start()
+        self.assertEquals(testvm2.get_power_state(), "Running")
+        self.assertRaises(QubesException, templatevm.start)
 
 
 class TC_20_DispVMMixin(qubes.tests.SystemTestsMixin):
@@ -436,7 +439,6 @@ class TC_20_DispVMMixin(qubes.tests.SystemTestsMixin):
         self.assertEqual(stdout, "test\n")
         # TODO: check if DispVM is destroyed
 
-
     @unittest.skipUnless(spawn.find_executable('xdotool'),
                          "xdotool not installed")
     def test_020_gui_app(self):
@@ -553,26 +555,27 @@ class TC_20_DispVMMixin(qubes.tests.SystemTestsMixin):
     @unittest.skipUnless(spawn.find_executable('xdotool'),
                          "xdotool not installed")
     def test_030_edit_file(self):
-        self.testvm1 = self.qc.add_new_vm("QubesAppVm",
-            name=self.make_vm_name('vm1'),
-            template=self.qc.get_vm_by_name(self.template))
-        self.testvm1.create_on_disk(verbose=False)
+        testvm1 = self.qc.add_new_vm("QubesAppVm",
+                                     name=self.make_vm_name('vm1'),
+                                     template=self.qc.get_vm_by_name(
+                                         self.template))
+        testvm1.create_on_disk(verbose=False)
         self.qc.save()
 
-        self.testvm1.start()
-        self.testvm1.run("echo test1 > /home/user/test.txt", wait=True)
+        testvm1.start()
+        testvm1.run("echo test1 > /home/user/test.txt", wait=True)
 
         self.qc.unlock_db()
-        p = self.testvm1.run("qvm-open-in-dvm /home/user/test.txt",
-                         passio_popen=True)
+        p = testvm1.run("qvm-open-in-dvm /home/user/test.txt",
+                        passio_popen=True)
 
         wait_count = 0
         winid = None
         while True:
             search = subprocess.Popen(['xdotool', 'search',
                                        '--onlyvisible', '--class', 'disp*'],
-                              stdout=subprocess.PIPE,
-                              stderr=open(os.path.devnull, 'w'))
+                                      stdout=subprocess.PIPE,
+                                      stderr=open(os.path.devnull, 'w'))
             retcode = search.wait()
             if retcode == 0:
                 winid = search.stdout.read().strip()
@@ -584,31 +587,32 @@ class TC_20_DispVMMixin(qubes.tests.SystemTestsMixin):
 
         self._handle_editor(winid)
         p.wait()
-        p = self.testvm1.run("cat /home/user/test.txt",
-                         passio_popen=True)
+        p = testvm1.run("cat /home/user/test.txt",
+                        passio_popen=True)
         (test_txt_content, _) = p.communicate()
         self.assertEqual(test_txt_content, "test test 2\ntest1\n")
 
+
 class TC_30_Gui_daemon(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
     @unittest.skipUnless(spawn.find_executable('xdotool'),
                          "xdotool not installed")
     def test_000_clipboard(self):
-        self.testvm1 = self.qc.add_new_vm("QubesAppVm",
-            name=self.make_vm_name('vm1'),
-            template=self.qc.get_default_template())
-        self.testvm1.create_on_disk(verbose=False)
-        self.testvm2 = self.qc.add_new_vm("QubesAppVm",
-            name=self.make_vm_name('vm2'),
-            template=self.qc.get_default_template())
-        self.testvm2.create_on_disk(verbose=False)
+        testvm1 = self.qc.add_new_vm("QubesAppVm",
+                                     name=self.make_vm_name('vm1'),
+                                     template=self.qc.get_default_template())
+        testvm1.create_on_disk(verbose=False)
+        testvm2 = self.qc.add_new_vm("QubesAppVm",
+                                     name=self.make_vm_name('vm2'),
+                                     template=self.qc.get_default_template())
+        testvm2.create_on_disk(verbose=False)
         self.qc.save()
         self.qc.unlock_db()
 
-        self.testvm1.start()
-        self.testvm2.start()
+        testvm1.start()
+        testvm2.start()
 
-        window_title = 'user@{}'.format(self.testvm1.name)
-        self.testvm1.run('zenity --text-info --editable --title={}'.format(
+        window_title = 'user@{}'.format(testvm1.name)
+        testvm1.run('zenity --text-info --editable --title={}'.format(
             window_title))
 
         wait_count = 0
@@ -621,33 +625,34 @@ class TC_30_Gui_daemon(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
             time.sleep(0.1)
 
         time.sleep(0.5)
-        test_string = "test{}".format(self.testvm1.xid)
+        test_string = "test{}".format(testvm1.xid)
 
         # Type and copy some text
         subprocess.check_call(['xdotool', 'search', '--name', window_title,
-                              'windowactivate',
-                              'type', '{}'.format(test_string)])
+                               'windowactivate',
+                               'type', '{}'.format(test_string)])
         # second xdotool call because type --terminator do not work (SEGV)
         # additionally do not use search here, so window stack will be empty
         # and xdotool will use XTEST instead of generating events manually -
         # this will be much better - at least because events will have
         # correct timestamp (so gui-daemon would not drop the copy request)
         subprocess.check_call(['xdotool',
-                              'key', 'ctrl+a', 'ctrl+c', 'ctrl+shift+c',
-                              'Escape'])
+                               'key', 'ctrl+a', 'ctrl+c', 'ctrl+shift+c',
+                               'Escape'])
 
         clipboard_content = \
             open('/var/run/qubes/qubes-clipboard.bin', 'r').read().strip()
         self.assertEquals(clipboard_content, test_string,
                           "Clipboard copy operation failed - content")
         clipboard_source = \
-            open('/var/run/qubes/qubes-clipboard.bin.source', 'r').read().strip()
-        self.assertEquals(clipboard_source, self.testvm1.name,
+            open('/var/run/qubes/qubes-clipboard.bin.source',
+                 'r').read().strip()
+        self.assertEquals(clipboard_source, testvm1.name,
                           "Clipboard copy operation failed - owner")
 
         # Then paste it to the other window
-        window_title = 'user@{}'.format(self.testvm2.name)
-        self.testvm2.run('zenity --entry --title={} > test.txt'.format(
+        window_title = 'user@{}'.format(testvm2.name)
+        testvm2.run('zenity --entry --title={} > test.txt'.format(
             window_title))
         wait_count = 0
         while subprocess.call(['xdotool', 'search', '--name', window_title],
@@ -663,8 +668,8 @@ class TC_30_Gui_daemon(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
         time.sleep(0.5)
 
         # And compare the result
-        (test_output, _) = self.testvm2.run('cat test.txt',
-                                            passio_popen=True).communicate()
+        (test_output, _) = testvm2.run('cat test.txt',
+                                       passio_popen=True).communicate()
         self.assertEquals(test_string, test_output.strip())
 
         clipboard_content = \

From 455535d0364c951b6b8b79c54820a1c4b9a7c7a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 8 Jul 2015 01:20:25 +0200
Subject: [PATCH 012/225] tests: reload qubes.xml after creating VMs

Unfortunately it is still needed. For example to load default netvm
setting.
---
 tests/vm_qrexec_gui.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index 5c8c80b5..86257295 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -49,8 +49,10 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
             name=self.make_vm_name('vm2'),
             template=self.qc.get_vm_by_name(self.template))
         self.testvm2.create_on_disk(verbose=False)
-        self.qc.save()
+        self.save_and_reload_db()
         self.qc.unlock_db()
+        self.testvm1 = self.qc[self.testvm1.qid]
+        self.testvm2 = self.qc[self.testvm2.qid]
 
     def test_000_start_shutdown(self):
         self.testvm1.start()

From 50a9c62d0e62fb4cdf0a3d29749c9dea73c9cfcc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 8 Jul 2015 01:21:13 +0200
Subject: [PATCH 013/225] tests: add test for qvm-sync-clock

---
 tests/vm_qrexec_gui.py | 44 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index 86257295..aa64bee1 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -359,6 +359,50 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
                                       passio_popen=True).communicate()
         self.assertEqual(vm_tz.strip(), "UTC")
 
+    def test_210_time_sync(self):
+        """Test time synchronization mechanism"""
+        self.testvm1.start()
+        self.testvm2.start()
+        (start_time, _) = subprocess.Popen(["date", "-u", "+%s"],
+                                           stdout=subprocess.PIPE).communicate()
+        original_clockvm_name = self.qc.get_clockvm_vm().name
+        try:
+            # use qubes-prefs to not hassle with qubes.xml locking
+            subprocess.check_call(["qubes-prefs", "-s", "clockvm",
+                                   self.testvm1.name])
+            # break vm and dom0 time, to check if qvm-sync-clock would fix it
+            subprocess.check_call(["sudo", "date", "-s",
+                                   "2001-01-01T12:34:56"],
+                                  stdout=open(os.devnull, 'w'))
+            retcode = self.testvm1.run("date -s 2001-01-01T12:34:56",
+                                       user="root", wait=True)
+            self.assertEquals(retcode, 0, "Failed to break the VM(1) time")
+            retcode = self.testvm2.run("date -s 2001-01-01T12:34:56",
+                                       user="root", wait=True)
+            self.assertEquals(retcode, 0, "Failed to break the VM(2) time")
+            retcode = subprocess.call(["qvm-sync-clock"])
+            self.assertEquals(retcode, 0,
+                              "qvm-sync-clock failed with code {}".
+                              format(retcode))
+            (vm_time, _) = self.testvm1.run("date -u +%s",
+                                            passio_popen=True).communicate()
+            self.assertAlmostEquals(int(vm_time), int(start_time), delta=10)
+            (vm_time, _) = self.testvm2.run("date -u +%s",
+                                            passio_popen=True).communicate()
+            self.assertAlmostEquals(int(vm_time), int(start_time), delta=10)
+            (dom0_time, _) = subprocess.Popen(["date", "-u", "+%s"],
+                                              stdout=subprocess.PIPE
+                                              ).communicate()
+            self.assertAlmostEquals(int(dom0_time), int(start_time), delta=10)
+
+        except:
+            # reset time to some approximation of the real time
+            subprocess.Popen(["sudo", "date", "-u", "-s", "@" + start_time])
+            raise
+        finally:
+            subprocess.call(["qubes-prefs", "-s", "clockvm",
+                             original_clockvm_name])
+
 
 class TC_10_HVM(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
     # TODO: test with some OS inside

From 284419b355f08d258faa888b560d1dce178a3f17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 8 Jul 2015 01:23:47 +0200
Subject: [PATCH 014/225] core: add "localcmd" support in QubesVm.run_service
 function

---
 core-modules/000QubesVm.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index c793d932..58f35acb 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -1552,16 +1552,19 @@ class QubesVm(object):
         return retcode
 
     def run_service(self, service, source="dom0", user=None,
-                    passio_popen =  False, input=None):
-        if input and passio_popen:
-            raise ValueError("'input' and 'passio_popen' cannot be used "
-                             "together")
-        if input:
+                    passio_popen=False, input=None, localcmd=None):
+        if bool(input) + bool(passio_popen) + bool(localcmd) > 1:
+            raise ValueError("'input', 'passio_popen', 'localcmd' cannot be "
+                             "used together")
+        if localcmd:
             return self.run("QUBESRPC %s %s" % (service, source),
-                        localcmd="echo %s" % input, user=user, wait=True)
+                            localcmd=localcmd, user=user, wait=True)
+        elif input:
+            return self.run("QUBESRPC %s %s" % (service, source),
+                            localcmd="echo %s" % input, user=user, wait=True)
         else:
             return self.run("QUBESRPC %s %s" % (service, source),
-                        passio_popen=passio_popen, user=user, wait=True)
+                            passio_popen=passio_popen, user=user, wait=True)
 
     def attach_network(self, verbose = False, wait = True, netvm = None):
         self.log.debug('attach_network(netvm={!r})'.format(netvm))

From e52f0a05664b09a25130badab03bef1dbd84ec01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 8 Jul 2015 01:24:35 +0200
Subject: [PATCH 015/225] core: by default call qrexec services without GUI

Most of them do not need GUI (especially those started from dom0), so
speed the things up a little (no need to wait for guid). But if some
service will need GUI access, there is "gui" parameter.
---
 core-modules/000QubesVm.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index 58f35acb..8cbd3b2f 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -1552,19 +1552,21 @@ class QubesVm(object):
         return retcode
 
     def run_service(self, service, source="dom0", user=None,
-                    passio_popen=False, input=None, localcmd=None):
+                    passio_popen=False, input=None, localcmd=None, gui=False):
         if bool(input) + bool(passio_popen) + bool(localcmd) > 1:
             raise ValueError("'input', 'passio_popen', 'localcmd' cannot be "
                              "used together")
         if localcmd:
             return self.run("QUBESRPC %s %s" % (service, source),
-                            localcmd=localcmd, user=user, wait=True)
+                            localcmd=localcmd, user=user, wait=True, gui=gui)
         elif input:
             return self.run("QUBESRPC %s %s" % (service, source),
-                            localcmd="echo %s" % input, user=user, wait=True)
+                            localcmd="echo %s" % input, user=user, wait=True,
+                            gui=gui)
         else:
             return self.run("QUBESRPC %s %s" % (service, source),
-                            passio_popen=passio_popen, user=user, wait=True)
+                            passio_popen=passio_popen, user=user, wait=True,
+                            gui=gui)
 
     def attach_network(self, verbose = False, wait = True, netvm = None):
         self.log.debug('attach_network(netvm={!r})'.format(netvm))

From 5f9a30d3357ef78a22db02042a061ae0d67743b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 8 Jul 2015 01:27:10 +0200
Subject: [PATCH 016/225] qvm-sync-clock: use qubes.SetDateTime service instead
 of direct "date" call

This way it gives more control over time synchronization to the VM. For
example Whonix VMs can decide to not use this mechanism. Also VM can
choose how that time will be set (chronyc call?). And finally it will be
possible to implement the same for other OS-es (Windows).

Additionally because of calling date as "localcmd" each time, instead of
once at the beginning, time synchronization is more accurrate now. If
some VM stall the time set call, other VMs time will no longer be
affected (but still synchronization will be delayed).
---
 qvm-tools/qvm-sync-clock | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/qvm-tools/qvm-sync-clock b/qvm-tools/qvm-sync-clock
index 2535b765..7ee738df 100755
--- a/qvm-tools/qvm-sync-clock
+++ b/qvm-tools/qvm-sync-clock
@@ -111,8 +111,8 @@ def main():
             if verbose:
                 print >> sys.stderr, '--> Syncing \'%s\' clock.' % vm.name
             try:
-                vm.run('date -u -R -s "%s"' % date_out, user="root",
-                       gui=False, verbose=verbose)
+                vm.run_service("qubes.SetDateTime", user="root",
+                               localcmd="date -u -Iseconds")
             except Exception as e:
                 print >> sys.stderr, "ERROR syncing time in VM '%s': %s" % (vm.name, str(e))
                 pass

From 6c167911f15467f0a0300f59cd1e1e89d41a422c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 8 Jul 2015 01:59:49 +0200
Subject: [PATCH 017/225] qvm-sync-clock: hide stdout in non-verbose mode

---
 qvm-tools/qvm-sync-clock | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/qvm-tools/qvm-sync-clock b/qvm-tools/qvm-sync-clock
index 7ee738df..36e4cdfb 100755
--- a/qvm-tools/qvm-sync-clock
+++ b/qvm-tools/qvm-sync-clock
@@ -102,8 +102,10 @@ def main():
     if verbose:
         print >> sys.stderr, '--> Syncing dom0 clock.'
 
-    subprocess.check_call(['sudo', 'date', '-u', '-Iseconds', '-s', date_out])
-    subprocess.check_call(['sudo', 'hwclock', '--systohc'])
+    subprocess.check_call(['sudo', 'date', '-u', '-Iseconds', '-s', date_out],
+                          stdout=None if verbose else open(os.devnull, 'w'))
+    subprocess.check_call(['sudo', 'hwclock', '--systohc'],
+                          stdout=None if verbose else open(os.devnull, 'w'))
 
     # Sync other VMs clock
     for vm in qvm_collection.values():

From a7e08e4584b16412df23f605709bbad7498c7ca0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 8 Jul 2015 04:26:11 +0200
Subject: [PATCH 018/225] tests: add qvm-clone test (#1032 case)

---
 tests/basic.py | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 86 insertions(+)

diff --git a/tests/basic.py b/tests/basic.py
index 7ed8151e..00144634 100644
--- a/tests/basic.py
+++ b/tests/basic.py
@@ -33,6 +33,8 @@ from qubes.qubes import QubesVmCollection, QubesException, system_path
 
 import qubes.qubes
 import qubes.tests
+from qubes.qubes import QubesVmLabels
+
 
 class TC_00_Basic(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
     def test_000_create(self):
@@ -153,6 +155,90 @@ class TC_01_Properties(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
         with self.assertRaises(ValueError):
             self.vm.dispvm_netvm = self.vm
 
+    def test_030_clone(self):
+        testvm1 = self.qc.add_new_vm(
+            "QubesAppVm",
+            name=self.make_vm_name("vm"),
+            template=self.qc.get_default_template())
+        testvm1.create_on_disk(verbose=False)
+        testvm2 = self.qc.add_new_vm(testvm1.__class__.__name__,
+                                     name=self.make_vm_name("clone"),
+                                     template=testvm1.template,
+                                     )
+        testvm2.clone_attrs(src_vm=testvm1)
+        testvm2.clone_disk_files(src_vm=testvm1, verbose=False)
 
+        # qubes.xml reload
+        self.save_and_reload_db()
+        testvm1 = self.qc[testvm1.qid]
+        testvm2 = self.qc[testvm2.qid]
+
+        self.assertEquals(testvm1.label, testvm2.label)
+        self.assertEquals(testvm1.netvm, testvm2.netvm)
+        self.assertEquals(testvm1.uses_default_netvm,
+                          testvm2.uses_default_netvm)
+        self.assertEquals(testvm1.kernel, testvm2.kernel)
+        self.assertEquals(testvm1.kernelopts, testvm2.kernelopts)
+        self.assertEquals(testvm1.uses_default_kernel,
+                          testvm2.uses_default_kernel)
+        self.assertEquals(testvm1.uses_default_kernelopts,
+                          testvm2.uses_default_kernelopts)
+        self.assertEquals(testvm1.memory, testvm2.memory)
+        self.assertEquals(testvm1.maxmem, testvm2.maxmem)
+        self.assertEquals(testvm1.pcidevs, testvm2.pcidevs)
+        self.assertEquals(testvm1.include_in_backups,
+                          testvm2.include_in_backups)
+        self.assertEquals(testvm1.default_user, testvm2.default_user)
+        self.assertEquals(testvm1.services, testvm2.services)
+        self.assertEquals(testvm1.get_firewall_conf(),
+                          testvm2.get_firewall_conf())
+
+        # now some non-default values
+        testvm1.netvm = None
+        testvm1.uses_default_netvm = False
+        testvm1.label = QubesVmLabels['orange']
+        testvm1.memory = 512
+        firewall = testvm1.get_firewall_conf()
+        firewall['allowDns'] = False
+        firewall['allowYumProxy'] = False
+        firewall['rules'] = [{'address': '1.2.3.4',
+                              'netmask': 24,
+                              'proto': 'tcp',
+                              'portBegin': 22,
+                              'portEnd': 22,
+                              }]
+        testvm1.write_firewall_conf(firewall)
+
+        testvm3 = self.qc.add_new_vm(testvm1.__class__.__name__,
+                                     name=self.make_vm_name("clone2"),
+                                     template=testvm1.template,
+                                     )
+        testvm3.clone_attrs(src_vm=testvm1)
+        testvm3.clone_disk_files(src_vm=testvm1, verbose=False)
+
+        # qubes.xml reload
+        self.save_and_reload_db()
+        testvm1 = self.qc[testvm1.qid]
+        testvm3 = self.qc[testvm3.qid]
+
+        self.assertEquals(testvm1.label, testvm3.label)
+        self.assertEquals(testvm1.netvm, testvm3.netvm)
+        self.assertEquals(testvm1.uses_default_netvm,
+                          testvm3.uses_default_netvm)
+        self.assertEquals(testvm1.kernel, testvm3.kernel)
+        self.assertEquals(testvm1.kernelopts, testvm3.kernelopts)
+        self.assertEquals(testvm1.uses_default_kernel,
+                          testvm3.uses_default_kernel)
+        self.assertEquals(testvm1.uses_default_kernelopts,
+                          testvm3.uses_default_kernelopts)
+        self.assertEquals(testvm1.memory, testvm3.memory)
+        self.assertEquals(testvm1.maxmem, testvm3.maxmem)
+        self.assertEquals(testvm1.pcidevs, testvm3.pcidevs)
+        self.assertEquals(testvm1.include_in_backups,
+                          testvm3.include_in_backups)
+        self.assertEquals(testvm1.default_user, testvm3.default_user)
+        self.assertEquals(testvm1.services, testvm3.services)
+        self.assertEquals(testvm1.get_firewall_conf(),
+                          testvm3.get_firewall_conf())
 
 # vim: ts=4 sw=4 et

From 1199806b5a3ce4c18082aa60317d0517cd8e4154 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 8 Jul 2015 04:42:58 +0200
Subject: [PATCH 019/225] core: fix handling firewall configuration for VM
 clones and DispVMs (#1032)

There were two bugs:
1. Firewall configuration wasn't copied during qvm-clone (it is in
   separate file, so now it is included in vm.clone_disk_files).
2. Non-default firewall configuration wasn't stored in qubes.xml. This
   means that initially DispVM got proper configuration (inherited from
   calling VM), but if anything caused firewall reload (for example
   starting another VM), the firewall rules was cleared to default state
   (allow all).

Fixes qubesos/qubes-issues#1032
---
 core-modules/000QubesVm.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index 8cbd3b2f..c0c52671 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -196,7 +196,7 @@ class QubesVm(object):
             'backup_content', 'backup_size', 'backup_path' ]:
             attrs[prop]['save'] = lambda prop=prop: str(getattr(self, prop))
         # Simple paths
-        for prop in ['conf_file']:
+        for prop in ['conf_file', 'firewall_conf']:
             attrs[prop]['save'] = \
                 lambda prop=prop: self.relative_path(getattr(self, prop))
             attrs[prop]['save_skip'] = \
@@ -1246,6 +1246,9 @@ class QubesVm(object):
                         print >> sys.stderr, "--> Copying icon: {0} -> {1}".format(src_vm.icon_path, self.icon_path)
                     shutil.copy(src_vm.icon_path, self.icon_path)
 
+        if src_vm.has_firewall():
+            self.write_firewall_conf(src_vm.get_firewall_conf())
+
         # Make sure that we have UUID allocated
         self._update_libvirt_domain()
 

From b47c74c3a4aab07f665953dbaa3a91efdf1bc651 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 8 Jul 2015 05:58:29 +0200
Subject: [PATCH 020/225] qvm-tools: fix usage info for qvm-template-commit

---
 qvm-tools/qvm-template-commit | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qvm-tools/qvm-template-commit b/qvm-tools/qvm-template-commit
index 158f3e02..a2e210da 100755
--- a/qvm-tools/qvm-template-commit
+++ b/qvm-tools/qvm-template-commit
@@ -32,7 +32,7 @@ def main():
     parser = OptionParser (usage)
     parser.add_option ("--offline-mode", dest="offline_mode",
                        action="store_true", default=False,
-                       help="Offline mode (ignored in this version)")
+                       help="Offline mode")
 
     (options, args) = parser.parse_args ()
     if (len (args) != 1):

From 4b0ce945b7af3b40f49a2f3f133892a20f9e534d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 8 Jul 2015 05:59:04 +0200
Subject: [PATCH 021/225] version 3.0.16

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index e265a8cb..ec187c44 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.0.15
+3.0.16

From 6df309351dc17695864cb36da450096938c964c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 13 Jul 2015 00:58:11 +0200
Subject: [PATCH 022/225] block: fix detection if the block device is attached
 anywhere

---
 core/qubesutils.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/core/qubesutils.py b/core/qubesutils.py
index aba05a3c..b0da7a7c 100644
--- a/core/qubesutils.py
+++ b/core/qubesutils.py
@@ -355,9 +355,8 @@ def block_check_attached(qvmc, device):
             disks = parsed_xml.xpath("//domain/devices/disk")
             for disk in disks:
                 backend_name = 'dom0'
-                # FIXME: move <domain/> into <source/>
-                if disk.find('domain') is not None:
-                    backend_name = disk.find('domain').get('name')
+                if disk.find('backenddomain') is not None:
+                    backend_name = disk.find('backenddomain').get('name')
                 source = disk.find('source')
                 if disk.get('type') == 'file':
                     path = source.get('file')

From 39379364c7f8eb3b4f2cfb0362de3ccee33ed83a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 13 Jul 2015 00:58:53 +0200
Subject: [PATCH 023/225] version 3.0.17

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index ec187c44..d19cf84f 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.0.16
+3.0.17

From 4a01c53787d8a97cfd3489923e49c400f3b45f87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 22 Jul 2015 05:46:12 +0200
Subject: [PATCH 024/225] qvm-tools: update qvm-trim-template for R3 block
 attach api

---
 qvm-tools/qvm-trim-template | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/qvm-tools/qvm-trim-template b/qvm-tools/qvm-trim-template
index 58e19d45..f576822b 100755
--- a/qvm-tools/qvm-trim-template
+++ b/qvm-tools/qvm-trim-template
@@ -118,7 +118,12 @@ def main():
     ''')
     fstrim_process.stdin.close()
 
-    qubesutils.block_attach(fstrim_vm, qvm_collection[0], tvm.root_img,
+    qubesutils.block_attach(qvm_collection, fstrim_vm,
+                            {
+                                'vm': 'dom0',
+                                'device': tvm.root_img,
+                                'mode': 'w',
+                            },
                             mode='w',
                             frontend='xvdi')
 

From 51826decec8a3480a0869f08d1ce1512088b4887 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 25 Jul 2015 03:55:50 +0200
Subject: [PATCH 025/225] core/hvm: start guid before qrexec, even when gui
 agent is installed

This way even when qrexec agent would timeout on connection, guid will
be already running.
Also use new -K guid option to terminate stubdom guid when the real guid
is connected (unless in debug mode - then both guid will be running).
---
 core-modules/01QubesHVm.py | 33 ++++++++++++++++++---------------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/core-modules/01QubesHVm.py b/core-modules/01QubesHVm.py
index 16e33376..8217fbbc 100644
--- a/core-modules/01QubesHVm.py
+++ b/core-modules/01QubesHVm.py
@@ -362,25 +362,28 @@ class QubesHVm(QubesVm):
         if (retcode != 0) :
             raise QubesException("Cannot start qubes-guid!")
 
-    def start_guid(self, verbose = True, notify_function = None,
-            before_qrexec=False, **kwargs):
-        # If user force the guiagent, start_guid will mimic a standard QubesVM
-        if not before_qrexec and self.guiagent_installed:
-            kwargs['extra_guid_args'] = kwargs.get('extra_guid_args', []) + \
-                                        ['-Q']
-            super(QubesHVm, self).start_guid(verbose, notify_function, **kwargs)
-            stubdom_guid_pidfile = '/var/run/qubes/guid-running.%d' % self.stubdom_xid
-            if os.path.exists(stubdom_guid_pidfile) and not self.debug:
-                try:
-                    stubdom_guid_pid = int(open(stubdom_guid_pidfile, 'r').read())
-                    os.kill(stubdom_guid_pid, signal.SIGTERM)
-                except Exception as ex:
-                    print >> sys.stderr, "WARNING: Failed to kill stubdom gui daemon: %s" % str(ex)
-        elif before_qrexec and (not self.guiagent_installed or self.debug):
+    def start_guid(self, verbose=True, notify_function=None,
+                   before_qrexec=False, **kwargs):
+        if not before_qrexec:
+            return
+
+        if not self.guiagent_installed or self.debug:
             if verbose:
                 print >> sys.stderr, "--> Starting Qubes GUId (full screen)..."
             self.start_stubdom_guid(verbose=verbose)
 
+        kwargs['extra_guid_args'] = kwargs.get('extra_guid_args', []) + \
+            ['-Q', '-n']
+
+        stubdom_guid_pidfile = \
+            '/var/run/qubes/guid-running.%d' % self.stubdom_xid
+        if not self.debug and os.path.exists(stubdom_guid_pidfile):
+            # Terminate stubdom guid once "real" gui agent connects
+            stubdom_guid_pid = int(open(stubdom_guid_pidfile, 'r').read())
+            kwargs['extra_guid_args'] += ['-K', str(stubdom_guid_pid)]
+
+        super(QubesHVm, self).start_guid(verbose, notify_function, **kwargs)
+
     def start_qrexec_daemon(self, **kwargs):
         if not self.qrexec_installed:
             if kwargs.get('verbose', False):

From 8f862cdf6923b4937fea6edc71bf45497cece6a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 25 Jul 2015 04:17:59 +0200
Subject: [PATCH 026/225] core/hvm: check if VM serving cdrom content is
 running

---
 core-modules/01QubesHVm.py |  4 ++++
 core/storage/xen.py        | 21 +++++++++++++++++++++
 2 files changed, 25 insertions(+)

diff --git a/core-modules/01QubesHVm.py b/core-modules/01QubesHVm.py
index 16e33376..f7cedc41 100644
--- a/core-modules/01QubesHVm.py
+++ b/core-modules/01QubesHVm.py
@@ -329,6 +329,10 @@ class QubesHVm(QubesVm):
             return -1
 
     def start(self, *args, **kwargs):
+        # make it available to storage.prepare_for_vm_startup, which is
+        # called before actually building VM libvirt configuration
+        self.storage.drive = self.drive
+
         if self.template and self.template.is_running():
             raise QubesException("Cannot start the HVM while its template is running")
         try:
diff --git a/core/storage/xen.py b/core/storage/xen.py
index 5daa3bbd..46cd422b 100644
--- a/core/storage/xen.py
+++ b/core/storage/xen.py
@@ -226,3 +226,24 @@ class QubesXenVmStorage(QubesVmStorage):
                 return
         super(QubesXenVmStorage, self).reset_volatile_storage(
             verbose=verbose, source_template=source_template)
+
+    def prepare_for_vm_startup(self, verbose):
+        super(QubesXenVmStorage, self).prepare_for_vm_startup(verbose=verbose)
+
+        if self.drive is not None:
+            (drive_type, drive_domain, drive_path) = self.drive.split(":")
+            if drive_domain.lower() != "dom0":
+                try:
+                    # FIXME: find a better way to access QubesVmCollection
+                    drive_vm = self.vm._collection.get_vm_by_name(drive_domain)
+                    # prepare for improved QubesVmCollection
+                    if drive_vm is None:
+                        raise KeyError
+                    if not drive_vm.is_running():
+                        raise QubesException(
+                            "VM '{}' holding '{}' isn't running".format(
+                                drive_domain, drive_path))
+                except KeyError:
+                    raise QubesException(
+                        "VM '{}' holding '{}' does not exists".format(
+                            drive_domain, drive_path))

From 472ed54e4c6cd22de226da3c0517b973584d5f95 Mon Sep 17 00:00:00 2001
From: Patrick Schleizer <adrelanos@riseup.net>
Date: Thu, 30 Jul 2015 17:46:24 +0200
Subject: [PATCH 027/225] fix VM autostart race condition

https://github.com/QubesOS/qubes-issues/issues/1075
---
 linux/aux-tools/prepare-volatile-img.sh | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/linux/aux-tools/prepare-volatile-img.sh b/linux/aux-tools/prepare-volatile-img.sh
index 25d9a600..e219ec09 100755
--- a/linux/aux-tools/prepare-volatile-img.sh
+++ b/linux/aux-tools/prepare-volatile-img.sh
@@ -27,8 +27,11 @@ sfdisk --no-reread -u M "$FILENAME" > /dev/null 2> /dev/null <<EOF
 ,${ROOT_SIZE},L
 EOF
 
-loopdev=`losetup -f --show --partscan "$FILENAME"`
-udevadm settle
-mkswap -f ${loopdev}p1 > /dev/null
-losetup -d ${loopdev} || :
-chown --reference `dirname "$FILENAME"` "$FILENAME"
+(
+	flock 200
+	loopdev=`losetup -f --show --partscan "$FILENAME"`
+	udevadm settle
+	mkswap -f ${loopdev}p1 > /dev/null
+	losetup -d ${loopdev} || :
+	chown --reference `dirname "$FILENAME"` "$FILENAME"
+) 200>"/var/run/qubes/prepare-volatile-img.lock"

From 90393c33f2c2bb7f2c8b5f41dd0ef151da4d5d61 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Thu, 30 Jul 2015 14:03:36 +0200
Subject: [PATCH 028/225] core: add offline mode to qvm-create, qvm-prefs

This is required to create VMs in process of building Live system, where
libvirt isn't running.

Additionally there is no udev in the build environment, so needs to
manually create /dev/loop*p* based on sysfs info.
---
 core-modules/000QubesVm.py              |  7 ++++++-
 core-modules/006QubesProxyVm.py         |  2 ++
 linux/aux-tools/prepare-volatile-img.sh | 12 ++++++++++++
 qvm-tools/qubes-prefs                   |  4 ++++
 qvm-tools/qvm-create                    |  8 ++++++++
 qvm-tools/qvm-prefs                     |  8 ++++++++
 6 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index c0c52671..c49e178e 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -848,6 +848,8 @@ class QubesVm(object):
         return True
 
     def is_running(self):
+        if vmm.offline_mode:
+            return False
         try:
             if self.libvirt_domain.isActive():
                 return True
@@ -1192,7 +1194,10 @@ class QubesVm(object):
             shutil.copy(self.label.icon_path, self.icon_path)
 
         # Make sure that we have UUID allocated
-        self._update_libvirt_domain()
+        if not vmm.offline_mode:
+            self._update_libvirt_domain()
+        else:
+            self.uuid = uuid.uuid4()
 
         # fire hooks
         for hook in self.hooks_create_on_disk:
diff --git a/core-modules/006QubesProxyVm.py b/core-modules/006QubesProxyVm.py
index 88a4781b..b5ae20f8 100644
--- a/core-modules/006QubesProxyVm.py
+++ b/core-modules/006QubesProxyVm.py
@@ -58,6 +58,8 @@ class QubesProxyVm(QubesNetVm):
     def _set_netvm(self, new_netvm):
         old_netvm = self.netvm
         super(QubesProxyVm, self)._set_netvm(new_netvm)
+        if vmm.offline_mode:
+            return
         if self.netvm is not None:
             self.netvm.add_external_ip_permission(self.get_xid())
         self.write_netvm_domid_entry()
diff --git a/linux/aux-tools/prepare-volatile-img.sh b/linux/aux-tools/prepare-volatile-img.sh
index 25d9a600..d41a3ce4 100755
--- a/linux/aux-tools/prepare-volatile-img.sh
+++ b/linux/aux-tools/prepare-volatile-img.sh
@@ -29,6 +29,18 @@ EOF
 
 loopdev=`losetup -f --show --partscan "$FILENAME"`
 udevadm settle
+created=
+if [ ! -e ${loopdev}p1 ]; then
+	# device wasn't created automatically, probably udev isn't running;
+	# create devs manually
+	for partdev in /sys/block/$(basename ${loopdev})/loop*p*; do
+		mknod /dev/$(basename ${partdev}) b $(cat ${partdev}/dev | tr : ' ')
+	done
+	created=yes
+fi
 mkswap -f ${loopdev}p1 > /dev/null
+if [ "$created" = "yes" ]; then
+	rm -f ${loopdev}p*
+fi
 losetup -d ${loopdev} || :
 chown --reference `dirname "$FILENAME"` "$FILENAME"
diff --git a/qvm-tools/qubes-prefs b/qvm-tools/qubes-prefs
index 41a365be..2f6a38aa 100755
--- a/qvm-tools/qubes-prefs
+++ b/qvm-tools/qubes-prefs
@@ -28,6 +28,8 @@ from optparse import OptionParser
 import subprocess
 import os
 import sys
+from qubes.qubes import vmm
+
 
 def handle_vm(vms, label, new_value = None):
     functions = { # label: [ getter, setter ],
@@ -116,6 +118,8 @@ def main():
         else:
             options.do_set = True
 
+    vmm.offline_mode = True
+
     if options.do_set:
         qvm_collection = QubesVmCollection()
         qvm_collection.lock_db_for_writing()
diff --git a/qvm-tools/qvm-create b/qvm-tools/qvm-create
index 1e2ac66a..dc414cf9 100755
--- a/qvm-tools/qvm-create
+++ b/qvm-tools/qvm-create
@@ -29,6 +29,8 @@ import subprocess
 import re
 import os
 import sys
+from qubes.qubes import vmm
+
 
 def main():
     usage = "usage: %prog [options] <vm-name>"
@@ -55,6 +57,9 @@ def main():
                         help="Initial memory size (in MB)")
     parser.add_option ("-c", "--vcpus", dest="vcpus", default=None,
                         help="VCPUs count")
+    parser.add_option ("--offline-mode", dest="offline_mode",
+                       action="store_true", default=False,
+                       help="Offline mode")
     parser.add_option ("-i", "--internal", action="store_true", dest="internal", default=False,
                         help="Create VM for internal use only (hidden in qubes-manager, no appmenus)")
     parser.add_option ("--force-root", action="store_true", dest="force_root", default=False,
@@ -118,6 +123,9 @@ def main():
         print >> sys.stderr, "File specified as root.img does not exists"
         exit (1)
 
+    if options.offline_mode:
+        vmm.offline_mode = True
+
     qvm_collection = QubesVmCollection()
     qvm_collection.lock_db_for_writing()
     qvm_collection.load()
diff --git a/qvm-tools/qvm-prefs b/qvm-tools/qvm-prefs
index 8eac355d..78d09b87 100755
--- a/qvm-tools/qvm-prefs
+++ b/qvm-tools/qvm-prefs
@@ -30,6 +30,8 @@ import subprocess
 import os
 import sys
 import re
+from qubes.qubes import vmm
+
 
 def do_list(vm):
     label_width = 19
@@ -541,6 +543,9 @@ def main():
     parser.add_option("--force-root", action="store_true", dest="force_root",
                       default=False,
                       help="Force to run, even with root privileges")
+    parser.add_option ("--offline-mode", dest="offline_mode",
+                       action="store_true", default=False,
+                       help="Offline mode")
 
     (options, args) = parser.parse_args ()
     if (len (args) < 1):
@@ -560,6 +565,9 @@ def main():
                              "the same time!"
         exit(1)
 
+    if options.offline_mode:
+        vmm.offline_mode = True
+
     if options.do_set:
         qvm_collection = QubesVmCollection()
         qvm_collection.lock_db_for_writing()

From 52d7de006b3d6f358a8600263ba06b7a4ad2c192 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Thu, 30 Jul 2015 23:08:45 +0200
Subject: [PATCH 029/225] do not print scary message when qvm-create
 --force-root is used

---
 qvm-tools/qvm-create | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/qvm-tools/qvm-create b/qvm-tools/qvm-create
index dc414cf9..a014febf 100755
--- a/qvm-tools/qvm-create
+++ b/qvm-tools/qvm-create
@@ -74,14 +74,11 @@ def main():
     if (options.netvm + options.proxyvm + options.hvm + options.hvm_template) > 1:
         parser.error ("You must specify at most one VM type switch")
         
-    if hasattr(os, "geteuid") and os.geteuid() == 0:
+    if not options.force_root and hasattr(os, "geteuid") and os.geteuid() == 0:
         print >> sys.stderr, "*** Running this tool as root is strongly discouraged, this will lead you in permissions problems."
-        if options.force_root:
-            print >> sys.stderr, "Continuing as commanded. You have been warned."
-        else:
-            print >> sys.stderr, "Retry as unprivileged user."
-            print >> sys.stderr, "... or use --force-root to continue anyway."
-            exit(1)
+        print >> sys.stderr, "Retry as unprivileged user."
+        print >> sys.stderr, "... or use --force-root to continue anyway."
+        exit(1)
 
     if options.label is None:
         print >> sys.stderr, "You must choose a label for the new VM by passing the --label option."

From c4a28a943a7fa4816175dac0675694e5eff66a90 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 31 Jul 2015 00:29:09 +0200
Subject: [PATCH 030/225] prepare-volatile-img: ensure that /usr/sbin in in the
 PATH

Build environment of LiveUSB does not contain /usr/sbin in path.
---
 linux/aux-tools/prepare-volatile-img.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/linux/aux-tools/prepare-volatile-img.sh b/linux/aux-tools/prepare-volatile-img.sh
index d41a3ce4..a31c92e4 100755
--- a/linux/aux-tools/prepare-volatile-img.sh
+++ b/linux/aux-tools/prepare-volatile-img.sh
@@ -6,6 +6,10 @@ fi
 
 set -e
 
+if ! echo $PATH | grep -q sbin; then
+	PATH=$PATH:/sbin:/usr/sbin
+fi
+
 FILENAME=$1
 ROOT_SIZE=$2
 SWAP_SIZE=$[ 1024 ]

From a425873e73a97c5b63da99652b6815540199645b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 31 Jul 2015 02:39:17 +0200
Subject: [PATCH 031/225] core: add offline mode to qvm-pci

Called on LiveUSB system before libvirtd starts.
---
 qvm-tools/qvm-pci | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/qvm-tools/qvm-pci b/qvm-tools/qvm-pci
index e8392be5..80ca4a1f 100755
--- a/qvm-tools/qvm-pci
+++ b/qvm-tools/qvm-pci
@@ -26,6 +26,8 @@ from optparse import OptionParser
 import subprocess
 import os
 import sys
+from qubes.qubes import vmm
+
 
 def main():
     usage = "usage: %prog -l [options] <vm-name>\n"\
@@ -37,6 +39,9 @@ def main():
     parser.add_option ("-l", "--list", action="store_true", dest="do_list", default=False)
     parser.add_option ("-a", "--add", action="store_true", dest="do_add", default=False)
     parser.add_option ("-d", "--delete", action="store_true", dest="do_delete", default=False)
+    parser.add_option ("--offline-mode", dest="offline_mode",
+                       action="store_true", default=False,
+                       help="Offline mode")
 
     (options, args) = parser.parse_args ()
     if (len (args) < 1):
@@ -48,6 +53,9 @@ def main():
         print >> sys.stderr, "Only one of -l -a -d is allowed!"
         exit (1)
 
+    if options.offline_mode:
+        vmm.offline_mode = True
+
     if options.do_add or options.do_delete:
         qvm_collection = QubesVmCollection()
         qvm_collection.lock_db_for_writing()

From d27d22a3cfe3f8d7ee2e65eaa497b5a340506f5f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 1 Aug 2015 21:51:02 +0200
Subject: [PATCH 032/225] Merge remote-tracking branch 'qubesos/pr/5'

* qubesos/pr/5:
  fix VM autostart race condition

Fixes qubesos/qubes-issues#1075

Conflicts:
	linux/aux-tools/prepare-volatile-img.sh
---
 linux/aux-tools/prepare-volatile-img.sh | 37 +++++++++++++------------
 1 file changed, 20 insertions(+), 17 deletions(-)

diff --git a/linux/aux-tools/prepare-volatile-img.sh b/linux/aux-tools/prepare-volatile-img.sh
index a31c92e4..33dfc9b9 100755
--- a/linux/aux-tools/prepare-volatile-img.sh
+++ b/linux/aux-tools/prepare-volatile-img.sh
@@ -31,20 +31,23 @@ sfdisk --no-reread -u M "$FILENAME" > /dev/null 2> /dev/null <<EOF
 ,${ROOT_SIZE},L
 EOF
 
-loopdev=`losetup -f --show --partscan "$FILENAME"`
-udevadm settle
-created=
-if [ ! -e ${loopdev}p1 ]; then
-	# device wasn't created automatically, probably udev isn't running;
-	# create devs manually
-	for partdev in /sys/block/$(basename ${loopdev})/loop*p*; do
-		mknod /dev/$(basename ${partdev}) b $(cat ${partdev}/dev | tr : ' ')
-	done
-	created=yes
-fi
-mkswap -f ${loopdev}p1 > /dev/null
-if [ "$created" = "yes" ]; then
-	rm -f ${loopdev}p*
-fi
-losetup -d ${loopdev} || :
-chown --reference `dirname "$FILENAME"` "$FILENAME"
+(
+	flock 200
+	loopdev=`losetup -f --show --partscan "$FILENAME"`
+	udevadm settle
+	created=
+	if [ ! -e ${loopdev}p1 ]; then
+		# device wasn't created automatically, probably udev isn't running;
+		# create devs manually
+		for partdev in /sys/block/$(basename ${loopdev})/loop*p*; do
+			mknod /dev/$(basename ${partdev}) b $(cat ${partdev}/dev | tr : ' ')
+		done
+		created=yes
+	fi
+	mkswap -f ${loopdev}p1 > /dev/null
+	if [ "$created" = "yes" ]; then
+		rm -f ${loopdev}p*
+	fi
+	losetup -d ${loopdev} || :
+	chown --reference `dirname "$FILENAME"` "$FILENAME"
+) 200>"/var/run/qubes/prepare-volatile-img.lock"

From fbfaa98b809053662ea8e09ba80ffeb205e26425 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 3 Aug 2015 20:28:25 +0200
Subject: [PATCH 033/225] Implement offline mode in qubes-set-updates tool

---
 qvm-tools/qubes-set-updates | 33 ++++++++++++++++++++++-----------
 1 file changed, 22 insertions(+), 11 deletions(-)

diff --git a/qvm-tools/qubes-set-updates b/qvm-tools/qubes-set-updates
index 4bb2d455..feca2c85 100755
--- a/qvm-tools/qubes-set-updates
+++ b/qvm-tools/qubes-set-updates
@@ -20,27 +20,38 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
 #
+from optparse import OptionParser
+import optparse
 
 import os
 import sys
 from qubes.qubes import QubesVmCollection
 from qubes.qubesutils import updates_vms_toggle,updates_dom0_toggle,\
     updates_dom0_status
+from qubes.qubes import vmm
 
-def usage():
-    print "Usage: qubes-set-updates enable|disable|status"
-    print "  Enable or disable globally checking for updates (both dom0 and VM)"
-    print "  Status option checks only dom0 updates status"
 
 def main():
-    if len(sys.argv) < 2:
-        usage()
-        return 1
-    
-    action = sys.argv[1]
+
+    usage = "%prog enable|disable|status\n"\
+            "  Enable or disable globally checking for updates (both dom0 and VM)"\
+            "  Status option checks only dom0 updates status"
+    parser = OptionParser (usage)
+    parser.add_option("--offline-mode", dest="offline_mode",
+                      action="store_true", default=False,
+                      help=optparse.SUPPRESS_HELP)
+
+    (options, args) = parser.parse_args()
+
+    if len(args) < 1:
+        parser.error("You must provide an action")
+
+    action = args[0]
     if action not in ['enable', 'disable', 'status']:
-        usage()
-        return 1
+        parser.error("Invalid action")
+
+    if options.offline_mode:
+        vmm.offline_mode = True
 
     qvm_collection = QubesVmCollection()
     if action == 'status':

From 46b94b6682d5762b15e6e389cb250162009836bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 3 Aug 2015 22:43:07 +0200
Subject: [PATCH 034/225] Make `qubes-set-updates` set also default state for
 new VMs

This makes possible to also check if the "updates check enabled" state
is consistent across VMs.

Fixes qubesos/qubes-issues#892
---
 core-modules/000QubesVm.py  |  6 ++++++
 core/qubesutils.py          | 21 +++++++++++++++++++++
 qvm-tools/qubes-set-updates | 16 +++++++++++-----
 3 files changed, 38 insertions(+), 5 deletions(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index c49e178e..df0180f8 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -43,6 +43,7 @@ from qubes.qubes import dry_run,vmm
 from qubes.qubes import register_qubes_vm_class
 from qubes.qubes import QubesVmCollection,QubesException,QubesHost,QubesVmLabels
 from qubes.qubes import defaults,system_path,vm_files,qubes_max_qid
+
 qmemman_present = False
 try:
     from qubes.qmemman_client import QMemmanClient
@@ -334,6 +335,11 @@ class QubesVm(object):
         if len(self.pcidevs) > 0:
             self.services['meminfo-writer'] = False
 
+        if 'xml_element' not in kwargs:
+            # New VM, disable updates check if requested for new VMs
+            if os.path.exists(qubes.qubesutils.UPDATES_DEFAULT_VM_DISABLE_FLAG):
+                self.services['qubes-update-check'] = False
+
         # Initialize VM image storage class
         self.storage = defaults["storage_class"](self)
         if hasattr(self, 'kernels_dir'):
diff --git a/core/qubesutils.py b/core/qubesutils.py
index b0da7a7c..041a66c8 100644
--- a/core/qubesutils.py
+++ b/core/qubesutils.py
@@ -844,8 +844,18 @@ class QubesWatch(object):
 ##### updates check #####
 
 UPDATES_DOM0_DISABLE_FLAG='/var/lib/qubes/updates/disable-updates'
+UPDATES_DEFAULT_VM_DISABLE_FLAG=\
+    '/var/lib/qubes/updates/vm-default-disable-updates'
 
 def updates_vms_toggle(qvm_collection, value):
+    # Flag for new VMs
+    if value:
+        if os.path.exists(UPDATES_DEFAULT_VM_DISABLE_FLAG):
+            os.unlink(UPDATES_DEFAULT_VM_DISABLE_FLAG)
+    else:
+        open(UPDATES_DEFAULT_VM_DISABLE_FLAG, "w").close()
+
+    # Change for existing VMs
     for vm in qvm_collection.values():
         if vm.qid == 0:
             continue
@@ -875,5 +885,16 @@ def updates_dom0_toggle(qvm_collection, value):
 def updates_dom0_status(qvm_collection):
     return not os.path.exists(UPDATES_DOM0_DISABLE_FLAG)
 
+def updates_vms_status(qvm_collection):
+    # default value:
+    status = not os.path.exists(UPDATES_DEFAULT_VM_DISABLE_FLAG)
+    # check if all the VMs uses the default value
+    for vm in qvm_collection.values():
+        if vm.qid == 0:
+            continue
+        if vm.services.get('qubes-update-check', True) != status:
+            # "mixed"
+            return None
+    return status
 
 # vim:sw=4:et:
diff --git a/qvm-tools/qubes-set-updates b/qvm-tools/qubes-set-updates
index feca2c85..c57a6ccb 100755
--- a/qvm-tools/qubes-set-updates
+++ b/qvm-tools/qubes-set-updates
@@ -27,15 +27,14 @@ import os
 import sys
 from qubes.qubes import QubesVmCollection
 from qubes.qubesutils import updates_vms_toggle,updates_dom0_toggle,\
-    updates_dom0_status
+    updates_dom0_status,updates_vms_status
 from qubes.qubes import vmm
 
 
 def main():
 
     usage = "%prog enable|disable|status\n"\
-            "  Enable or disable globally checking for updates (both dom0 and VM)"\
-            "  Status option checks only dom0 updates status"
+            "  Enable or disable globally checking for updates (both dom0 and VM)"
     parser = OptionParser (usage)
     parser.add_option("--offline-mode", dest="offline_mode",
                       action="store_true", default=False,
@@ -67,9 +66,16 @@ def main():
         updates_vms_toggle(qvm_collection, False)
     else:
         if updates_dom0_status(qvm_collection):
-            print "enabled"
+            print "dom0: enabled"
         else:
-            print "disabled"
+            print "dom0: disabled"
+        status_vms = updates_vms_status(qvm_collection)
+        if status_vms is None:
+            print "vms: mixed"
+        elif status_vms:
+            print "vms: enabled"
+        else:
+            print "vms: disabled"
 
     if action != 'status':
         qvm_collection.save()

From cc8af79397c6dfc067e15f9daf6361c7f94f8792 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 4 Aug 2015 18:11:32 +0200
Subject: [PATCH 035/225] dispvm: fallback to tar instead of bsdtar on tmpfs

---
 dispvm/qubes-prepare-saved-domain.sh | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/dispvm/qubes-prepare-saved-domain.sh b/dispvm/qubes-prepare-saved-domain.sh
index bc4710f2..9c5c0f98 100755
--- a/dispvm/qubes-prepare-saved-domain.sh
+++ b/dispvm/qubes-prepare-saved-domain.sh
@@ -70,5 +70,12 @@ else
 fi
 ln -snf $VMDIR /var/lib/qubes/dvmdata/vmdir
 cd $VMDIR
-bsdtar -cSf saved-cows.tar volatile.img
+fstype=`df --output=fstype $VMDIR | tail -n 1`
+if [ "$fstype" = "tmpfs" ]; then
+    # bsdtar doesn't work on tmpfs because FS_IOC_FIEMAP ioctl isn't supported
+    # there
+    tar -cSf saved-cows.tar volatile.img
+else
+    bsdtar -cSf saved-cows.tar volatile.img
+fi
 echo "DVM savefile created successfully."

From 2ed9008a83b760384d34af397d4897e601e4448d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 4 Aug 2015 18:12:18 +0200
Subject: [PATCH 036/225] core: handle firewall.xml path during VM rename

---
 core-modules/000QubesVm.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index df0180f8..b57aeffa 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -589,6 +589,9 @@ class QubesVm(object):
             self.icon_path = self.icon_path.replace(old_dirpath, new_dirpath)
         if hasattr(self, 'kernels_dir') and self.kernels_dir is not None:
             self.kernels_dir = self.kernels_dir.replace(old_dirpath, new_dirpath)
+        if self.firewall_conf is not None:
+            self.firewall_conf = self.firewall_conf.replace(old_dirpath,
+                                                            new_dirpath)
 
         self._update_libvirt_domain()
         self.post_rename(old_name)

From 75bbd1a76b5c2f64358811b0ff961be6da41de45 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 4 Aug 2015 18:12:36 +0200
Subject: [PATCH 037/225] tests: check firewall.xml path after VM rename

---
 tests/basic.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/basic.py b/tests/basic.py
index 00144634..4cc662ce 100644
--- a/tests/basic.py
+++ b/tests/basic.py
@@ -70,6 +70,8 @@ class TC_01_Properties(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
         newname = self.make_vm_name('newname')
 
         self.assertEqual(self.vm.name, self.vmname)
+        self.vm.write_firewall_conf({'allow': False, 'allowDns': False})
+        pre_rename_firewall = self.vm.get_firewall_conf()
 
         #TODO: change to setting property when implemented
         self.vm.set_name(newname)
@@ -95,6 +97,9 @@ class TC_01_Properties(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
         self.assertFalse(os.path.exists(
             os.path.join(os.getenv("HOME"), ".local/share/applications",
                 self.vmname + "-firefox.desktop")))
+        self.assertEquals(pre_rename_firewall, self.vm.get_firewall_conf())
+        with self.assertNotRaises(QubesException, OSError):
+            self.vm.write_firewall_conf({'allow': False})
 
     def test_010_netvm(self):
         if self.qc.get_default_netvm() is None:

From 7e79cd20a8a4c5745c7c16e0e9db3bd5f4174288 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 5 Aug 2015 01:34:59 +0200
Subject: [PATCH 038/225] doc: wrap long lines in qvm-prefs.rst, content
 unchanged

---
 doc/qvm-tools/qvm-prefs.rst | 95 +++++++++++++++++++++++++++++--------
 1 file changed, 75 insertions(+), 20 deletions(-)

diff --git a/doc/qvm-tools/qvm-prefs.rst b/doc/qvm-tools/qvm-prefs.rst
index 197ef6eb..16efd08e 100644
--- a/doc/qvm-tools/qvm-prefs.rst
+++ b/doc/qvm-tools/qvm-prefs.rst
@@ -32,7 +32,9 @@ PROPERTIES
 include_in_backups
     Accepted values: ``True``, ``False``
 
-    Control whenever this VM will be included in backups by default (for now works only in qubes-manager). You can always manually select or deselect any VM for backup.
+    Control whenever this VM will be included in backups by default (for now
+    works only in qubes-manager). You can always manually select or
+    deselect any VM for backup.
 
 pcidevs
     PCI devices assigned to the VM. Should be edited using qvm-pci tool.
@@ -48,34 +50,49 @@ pci_strictreset
     is trusted one, or is running all the time.
 
 label
-    Accepted values: ``red``, ``orange``, ``yellow``, ``green``, ``gray``, ``blue``, ``purple``, ``black``
+    Accepted values: ``red``, ``orange``, ``yellow``, ``green``, ``gray``,
+    ``blue``, ``purple``, ``black``
 
-    Color of VM label (icon, appmenus, windows border). If VM is running, change will be applied at first VM restart.
+    Color of VM label (icon, appmenus, windows border). If VM is running,
+    change will be applied at first VM restart.
 
 netvm
     Accepted values: netvm name, ``default``, ``none``
 
-    To which NetVM connect. Setting to ``default`` will follow system-global default NetVM (managed by qubes-prefs). Setting to ``none`` will disable networking in this VM.
+    To which NetVM connect. Setting to ``default`` will follow system-global
+    default NetVM (managed by qubes-prefs). Setting to ``none`` will disable
+    networking in this VM.
 
 dispvm_netvm
     Accepted values: netvm name, ``default``, ``none``
 
-    Which NetVM should be used for Disposable VMs started by this one. ``default`` is to use the same NetVM as the VM itself.
+    Which NetVM should be used for Disposable VMs started by this one.
+    ``default`` is to use the same NetVM as the VM itself.
 
 maxmem
     Accepted values: memory size in MB
 
-    Maximum memory size available for this VM. Dynamic memory management (aka qmemman) will not be able to balloon over this limit. For VMs with qmemman disabled, this will be overridden by *memory* property (at VM startup).
+    Maximum memory size available for this VM. Dynamic memory management (aka
+    qmemman) will not be able to balloon over this limit. For VMs with
+    qmemman disabled, this will be overridden by *memory* property (at VM
+    startup).
 
 memory
     Accepted values: memory size in MB
 
-    Initial memory size for VM. This should be large enough to allow VM startup - before qmemman starts managing memory for this VM. For VM with qmemman disabled, this is static memory size.
+    Initial memory size for VM. This should be large enough to allow VM startup
+    - before qmemman starts managing memory for this VM. For VM with qmemman
+    disabled, this is static memory size.
 
 kernel
     Accepted values: kernel version, ``default``, ``none``
 
-    Kernel version to use (only for PV VMs). Available kernel versions will be listed when no value given (there are in /var/lib/qubes/vm-kernels). Setting to ``default`` will follow system-global default kernel (managed via qubes-prefs). Setting to ``none`` will use "kernels" subdir in VM directory - this allows having VM-specific kernel; also this the only case when /lib/modules is writable from within VM.
+    Kernel version to use (only for PV VMs). Available kernel versions will be
+    listed when no value given (there are in /var/lib/qubes/vm-kernels).
+    Setting to ``default`` will follow system-global default kernel (managed
+    via qubes-prefs). Setting to ``none`` will use "kernels" subdir in
+    VM directory - this allows having VM-specific kernel; also this the only
+    case when /lib/modules is writable from within VM.
 
 template
     Accepted values: TemplateVM name
@@ -85,12 +102,17 @@ template
 vcpus
     Accepted values: no of CPUs
 
-    Number of CPU (cores) available to VM. Some VM types (eg DispVM) will not work properly with more than one CPU.
+    Number of CPU (cores) available to VM. Some VM types (eg DispVM) will not
+    work properly with more than one CPU.
 
 kernelopts
     Accepted values: string, ``default``
 
-    VM kernel parameters (available only for PV VMs). This can be used to workaround some hardware specific problems (eg for NetVM). Setting to ``default`` will use some reasonable defaults (currently different for VMs with PCI devices and without). Some helpful options (for debugging purposes): ``earlyprintk=xen``, ``init=/bin/bash``
+    VM kernel parameters (available only for PV VMs). This can be used to
+    workaround some hardware specific problems (eg for NetVM). Setting to
+    ``default`` will use some reasonable defaults (currently different for VMs
+    with PCI devices and without). Some helpful options (for debugging
+    purposes): ``earlyprintk=xen``, ``init=/bin/bash``
 
 name
     Accepted values: alphanumerical name
@@ -100,46 +122,79 @@ name
 drive
     Accepted values: [hd:\|cdrom:][backend-vm:]path
 
-    Additional drive for the VM (available only for HVMs). This can be used to attach installation image. ``path`` can be file or physical device (eg. /dev/sr0). The same syntax can be used in qvm-start --drive - to attach drive only temporarily.
+    Additional drive for the VM (available only for HVMs). This can be used to
+    attach installation image. ``path`` can be file or physical device (eg.
+    /dev/sr0). The same syntax can be used in qvm-start --drive - to
+    attach drive only temporarily.
 
 mac
     Accepted values: MAC address, ``auto``
 
-    Can be used to force specific of virtual ethernet card in the VM. Setting to ``auto`` will use automatic-generated MAC - based on VM id. Especially useful when some licencing depending on static MAC address.
+    Can be used to force specific of virtual ethernet card in the VM. Setting
+    to ``auto`` will use automatic-generated MAC - based on VM id. Especially
+    useful when some licencing depending on static MAC address.
     For template-based HVM ``auto`` mode means to clone template MAC.
 
 default_user
     Accepted values: username
 
-    Default user used by qvm-run. Note that it make sense only on non-standard template, as the standard one always have "user" account.
+    Default user used by qvm-run. Note that it make sense only on non-standard
+    template, as the standard one always have "user" account.
 
 debug
     Accepted values: ``on``, ``off``
 
-    Enables debug mode for VM. This can be used to turn on/off verbose logging in many qubes components at once (gui virtualization, VM kernel, some other services).
-    For template-based HVM, enabling debug mode also disables automatic reset root.img (actually volatile.img) before each VM startup, so changes made to root filesystem stays intact. To force reset root.img when debug mode enabled, either change something in the template (simple start+stop will do, even touch its root.img is enough), or remove VM's volatile.img (check the path with qvm-prefs).
+    Enables debug mode for VM. This can be used to turn on/off verbose logging
+    in many qubes components at once (gui virtualization, VM kernel, some other
+    services).
+    For template-based HVM, enabling debug mode also disables automatic reset
+    root.img (actually volatile.img) before each VM startup, so changes made to
+    root filesystem stays intact. To force reset root.img when debug mode
+    enabled, either change something in the template (simple start+stop will
+    do, even touch its root.img is enough), or remove VM's volatile.img
+    (check the path with qvm-prefs).
 
 qrexec_installed
     Accepted values: ``True``, ``False``
 
-    This HVM have qrexec agent installed. When VM have qrexec agent installed, one can use qvm-run to start VM process, VM will benefit from Qubes RPC services (like file copy, or inter-vm clipboard). This option will be automatically turned on during Qubes Windows Tools installation, but if you install qrexec agent in some other OS, you need to turn this option on manually.
+    This HVM have qrexec agent installed. When VM have qrexec agent installed,
+    one can use qvm-run to start VM process, VM will benefit from Qubes RPC
+    services (like file copy, or inter-vm clipboard). This option will be
+    automatically turned on during Qubes Windows Tools installation, but if you
+    install qrexec agent in some other OS, you need to turn this option on
+    manually.
 
 guiagent_installed
     Accepted values: ``True``, ``False``
 
-    This HVM have gui agent installed. This option disables full screen GUI virtualization and enables per-window seemless GUI mode. This option will be automatically turned on during Qubes Windows Tools installation, but if you install qubes gui agent in some other OS, you need to turn this option on manually. You can turn this option off to troubleshoot some early HVM OS boot problems (enter safe mode etc), but the option will be automatically enabled at first VM normal startup (and will take effect from the next startup).
+    This HVM have gui agent installed. This option disables full screen GUI
+    virtualization and enables per-window seemless GUI mode. This option will
+    be automatically turned on during Qubes Windows Tools installation, but if
+    you install qubes gui agent in some other OS, you need to turn this option
+    on manually. You can turn this option off to troubleshoot some early HVM OS
+    boot problems (enter safe mode etc), but the option will be automatically
+    enabled at first VM normal startup (and will take effect from the next
+    startup).
 
-    *Notice:* when Windows GUI agent is installed in the VM, SVGA device (used to full screen video) is disabled, so even if you disable this option, you will not get functional full desktop access (on normal VM startup). Use some other means for that (VNC, RDP or so).
+    *Notice:* when Windows GUI agent is installed in the VM, SVGA device (used
+    to full screen video) is disabled, so even if you disable this
+    option, you will not get functional full desktop access (on normal VM
+    startup). Use some other means for that (VNC, RDP or so).
 
 autostart
     Accepted values: ``True``, ``False``
 
-    Start the VM during system startup. The default netvm is autostarted regardless of this setting.
+    Start the VM during system startup. The default netvm is autostarted
+    regardless of this setting.
 
 timezone
     Accepted values: ``localtime``, time offset in seconds
 
-    Set emulated HVM clock timezone. Use ``localtime`` (the default) to use the same time as dom0 have. Note that HVM will get only clock value, not the timezone itself, so if you use ``localtime`` setting, OS inside of HVM should also be configured to treat hardware clock as local time (and have proper timezone set).
+    Set emulated HVM clock timezone. Use ``localtime`` (the default) to use the
+    same time as dom0 have. Note that HVM will get only clock value, not the
+    timezone itself, so if you use ``localtime`` setting, OS inside of HVM
+    should also be configured to treat hardware clock as local time (and have
+    proper timezone set).
 
 AUTHORS
 =======

From da0e5dd4babb72911dc0579b35afc16e26027b0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 5 Aug 2015 01:43:43 +0200
Subject: [PATCH 039/225] core: inherit kernelopts from the template by default

Fixes qubesos/qubes-issues#1091
---
 core-modules/000QubesVm.py  | 1 +
 doc/qvm-tools/qvm-prefs.rst | 6 ++++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index b57aeffa..86685053 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -146,6 +146,7 @@ class QubesVm(object):
                 "order": 31,
                 "func": lambda value: value if not self.uses_default_kernelopts\
                     else defaults["kernelopts_pcidevs"] if len(self.pcidevs)>0 \
+                    else self.template.kernelopts if self.template
                     else defaults["kernelopts"] },
             "mac": { "attr": "_mac", "default": None },
             "include_in_backups": {
diff --git a/doc/qvm-tools/qvm-prefs.rst b/doc/qvm-tools/qvm-prefs.rst
index 16efd08e..3120f143 100644
--- a/doc/qvm-tools/qvm-prefs.rst
+++ b/doc/qvm-tools/qvm-prefs.rst
@@ -111,8 +111,10 @@ kernelopts
     VM kernel parameters (available only for PV VMs). This can be used to
     workaround some hardware specific problems (eg for NetVM). Setting to
     ``default`` will use some reasonable defaults (currently different for VMs
-    with PCI devices and without). Some helpful options (for debugging
-    purposes): ``earlyprintk=xen``, ``init=/bin/bash``
+    with PCI devices and without). For VM without PCI devices
+    ``default`` option means inherit this value from the VM template (if any).
+    Some helpful options (for debugging purposes): ``earlyprintk=xen``,
+    ``init=/bin/bash``
 
 name
     Accepted values: alphanumerical name

From 754a411829c3e919129d96df190174cda39c5b76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 5 Aug 2015 02:02:50 +0200
Subject: [PATCH 040/225] version 3.0.18

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index d19cf84f..29cec99e 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.0.17
+3.0.18

From 51998409278999fc02c8caeea59c02c34813eb01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 8 Aug 2015 02:05:52 +0200
Subject: [PATCH 041/225] tests: typo fix

---
 tests/vm_qrexec_gui.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index aa64bee1..5e2238fa 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -208,7 +208,7 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
         def run(self, result):
             p = self.testvm1.run("/usr/lib/qubes/qrexec-client-vm %s test.EOF "
                                  "/bin/sh -c 'cat >&$SAVED_FD_1'"
-                                 % self.testvm1.name,
+                                 % self.testvm2.name,
                                  passio_popen=True)
             (stdout, stderr) = p.communicate()
             if stdout != "test\n":

From bbf22c44494470e808cbee7995c65122e03346c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 8 Aug 2015 02:06:03 +0200
Subject: [PATCH 042/225] tests: add a test for aborted service call

Details here:
https://groups.google.com/d/msgid/qubes-users/20150807213935.GN3740%40mail-itl
---
 tests/vm_qrexec_gui.py | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index 5e2238fa..dd2ea9d1 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -235,6 +235,42 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
         if result.value == 1:
             self.fail("Received data differs from what was expected")
 
+    def test_055_qrexec_dom0_service_abort(self):
+        """
+        Test if service abort (by dom0) is properly handled by source VM.
+
+        If "remote" part of the service terminates, the source part should
+        properly be notified. This includes closing its stdin (which is
+        already checked by test_053_qrexec_vm_service_eof_reverse), but also
+        its stdout - otherwise such service might hang on write(2) call.
+        """
+
+        def run (src):
+            p = src.run("/usr/lib/qubes/qrexec-client-vm dom0 "
+                                 "test.Abort /bin/cat /dev/zero",
+                                 passio_popen=True)
+
+            p.communicate()
+            p.wait()
+
+        self.testvm1.start()
+        service = open("/etc/qubes-rpc/test.Abort", "w")
+        service.write("sleep 1")
+        service.close()
+        self.addCleanup(os.unlink, "/etc/qubes-rpc/test.Abort")
+        policy = open("/etc/qubes-rpc/policy/test.Abort", "w")
+        policy.write("%s dom0 allow" % (self.testvm1.name))
+        policy.close()
+        self.addCleanup(os.unlink, "/etc/qubes-rpc/policy/test.Abort")
+
+        t = multiprocessing.Process(target=run, args=(self.testvm1,))
+        t.start()
+        t.join(timeout=10)
+        if t.is_alive():
+            t.terminate()
+            self.fail("Timeout, probably stdout wasn't closed")
+
+
     def test_060_qrexec_exit_code_dom0(self):
         self.testvm1.start()
 

From 5a19f37dc009d17ab5fb2e4f34697ac71761685c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 8 Aug 2015 04:23:49 +0200
Subject: [PATCH 043/225] version 3.0.19

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index 29cec99e..25875f01 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.0.18
+3.0.19

From 1ed4a2cbb240c2012d0da851b6812b270600dbe7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 8 Aug 2015 21:29:56 +0200
Subject: [PATCH 044/225] core: fix getting libvirt error code

---
 core-modules/000QubesVm.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index 86685053..7edbc350 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -694,7 +694,7 @@ class QubesVm(object):
         try:
             return self.libvirt_domain.ID()
         except libvirt.libvirtError as e:
-            if e.err[0] == libvirt.VIR_ERR_NO_DOMAIN:
+            if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN:
                 return -1
             else:
                 raise
@@ -736,7 +736,7 @@ class QubesVm(object):
                 return 0
             return self.libvirt_domain.info()[1]
         except libvirt.libvirtError as e:
-            if e.err[0] == libvirt.VIR_ERR_NO_DOMAIN:
+            if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN:
                 return 0
             else:
                 raise
@@ -750,7 +750,7 @@ class QubesVm(object):
                 return 0
             return self.libvirt_domain.info()[4]
         except libvirt.libvirtError as e:
-            if e.err[0] == libvirt.VIR_ERR_NO_DOMAIN:
+            if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN:
                 return 0
             else:
                 raise
@@ -762,7 +762,7 @@ class QubesVm(object):
         try:
             return self.libvirt_domain.maxMemory()
         except libvirt.libvirtError as e:
-            if e.err[0] == libvirt.VIR_ERR_NO_DOMAIN:
+            if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN:
                 return 0
             else:
                 raise
@@ -791,7 +791,7 @@ class QubesVm(object):
             else:
                 return 0
         except libvirt.libvirtError as e:
-            if e.err[0] == libvirt.VIR_ERR_NO_DOMAIN:
+            if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN:
                 return 0
             else:
                 raise
@@ -830,7 +830,7 @@ class QubesVm(object):
             else:
                 return 'Halted'
         except libvirt.libvirtError as e:
-            if e.err[0] == libvirt.VIR_ERR_NO_DOMAIN:
+            if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN:
                 return "Halted"
             else:
                 raise
@@ -866,7 +866,7 @@ class QubesVm(object):
             else:
                 return False
         except libvirt.libvirtError as e:
-            if e.err[0] == libvirt.VIR_ERR_NO_DOMAIN:
+            if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN:
                 return False
             else:
                 raise
@@ -878,7 +878,7 @@ class QubesVm(object):
             else:
                 return False
         except libvirt.libvirtError as e:
-            if e.err[0] == libvirt.VIR_ERR_NO_DOMAIN:
+            if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN:
                 return False
             else:
                 raise
@@ -1305,7 +1305,7 @@ class QubesVm(object):
         try:
             self.libvirt_domain.undefine()
         except libvirt.libvirtError as e:
-            if e.err[0] == libvirt.VIR_ERR_NO_DOMAIN:
+            if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN:
                 # already undefined
                 pass
             else:
@@ -1746,7 +1746,7 @@ class QubesVm(object):
             try:
                 nd = vmm.libvirt_conn.nodeDeviceLookupByName('pci_0000_' + pci.replace(':','_').replace('.','_'))
             except libvirt.libvirtError as e:
-                if e.err[0] == libvirt.VIR_ERR_NO_NODE_DEVICE:
+                if e.get_error_code() == libvirt.VIR_ERR_NO_NODE_DEVICE:
                     raise QubesException(
                         "PCI device {} does not exist (domain {})".
                         format(pci, self.name))
@@ -1755,8 +1755,8 @@ class QubesVm(object):
             try:
                 nd.dettach()
             except libvirt.libvirtError as e:
-                if e.err[0] == libvirt.VIR_ERR_INTERNAL_ERROR:
-                    # allready detached
+                if e.get_error_code() == libvirt.VIR_ERR_INTERNAL_ERROR:
+                    # already detached
                     pass
                 else:
                     raise

From 375c5ababdeabc03ea8338707d75b40395e22afd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 8 Aug 2015 22:09:55 +0200
Subject: [PATCH 045/225] version 3.0.20

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index 25875f01..3e4a61b7 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.0.19
+3.0.20

From 26cf723841daf3009355cfadbef9b8162cf9ce35 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sun, 9 Aug 2015 21:21:27 +0200
Subject: [PATCH 046/225] core/hvm: fix external disks handling

---
 core/storage/xen.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/core/storage/xen.py b/core/storage/xen.py
index 46cd422b..00c3b3da 100644
--- a/core/storage/xen.py
+++ b/core/storage/xen.py
@@ -112,6 +112,8 @@ class QubesXenVmStorage(QubesVmStorage):
                             None, self.modules_dev, self.modules_img_rw)
         elif self.drive is not None:
             (drive_type, drive_domain, drive_path) = self.drive.split(":")
+            if drive_type == "hd":
+                drive_type = "disk"
             if drive_domain.lower() == "dom0":
                 drive_domain = None
 

From e3ae3cdf34cbaf0ed5bca3d02ddc9b67b79bee8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sun, 9 Aug 2015 21:21:47 +0200
Subject: [PATCH 047/225] hvm: update appmenus only on the first
 qubes.NotifyTools call

Or more precisely - only when qrexec_installed value is changed from
False to True.
Previously it was called at each VM startup.
---
 qubes-rpc/qubes-notify-tools | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/qubes-rpc/qubes-notify-tools b/qubes-rpc/qubes-notify-tools
index f0ec3b5a..273df90d 100755
--- a/qubes-rpc/qubes-notify-tools
+++ b/qubes-rpc/qubes-notify-tools
@@ -14,6 +14,7 @@ def main():
         print >> sys.stderr, 'This script must be called as qrexec service!'
         exit(1)
 
+    prev_qrexec_installed = False
     source_vm = None
     qvm_collection = QubesVmCollection()
     qvm_collection.lock_db_for_writing()
@@ -66,6 +67,7 @@ def main():
         else:
             user = None
 
+        prev_qrexec_installed = source_vm.qrexec_installed
         # Let the tools to be able to enable *or disable* each particular component
         source_vm.qrexec_installed = qrexec > 0
         source_vm.guiagent_installed = gui > 0
@@ -81,11 +83,12 @@ def main():
     finally:
         qvm_collection.unlock_db()
 
-    retcode = subprocess.call(['qvm-sync-appmenus', '--force-rpc'])
-    if retcode == 0 and hasattr(source_vm, 'appmenus_recreate'):
-        # TODO: call the same for child VMs? This isn't done for Linux VMs,
-        # so probably should be ignored for Windows also
-        source_vm.appmenus_recreate()
+    if not prev_qrexec_installed and source_vm.qrexec_installed:
+        retcode = subprocess.call(['qvm-sync-appmenus', '--force-rpc'])
+        if retcode == 0 and hasattr(source_vm, 'appmenus_recreate'):
+            # TODO: call the same for child VMs? This isn't done for Linux VMs,
+            # so probably should be ignored for Windows also
+            source_vm.appmenus_recreate()
 
 
 main()

From f4a38f05409640607984671aa6261c28a3fee0b4 Mon Sep 17 00:00:00 2001
From: Patrick Schleizer <adrelanos@riseup.net>
Date: Mon, 10 Aug 2015 18:02:18 +0200
Subject: [PATCH 048/225] suggest to use 'yum remove' rather than 'rpm -e' for
 VM removal so avoid yum's warning of noticing rpm changes outside of yum

---
 qvm-tools/qvm-remove | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qvm-tools/qvm-remove b/qvm-tools/qvm-remove
index a5973d28..ccb10212 100755
--- a/qvm-tools/qvm-remove
+++ b/qvm-tools/qvm-remove
@@ -75,7 +75,7 @@ def main():
 
     if vm.installed_by_rpm and not options.remove_from_db_only:
         if options.verbose:
-            print >> sys.stderr, "This VM has been installed by RPM, use rpm -e <pkg name> to remove it!"
+            print >> sys.stderr, "This VM has been installed by RPM, use yum remove <pkg name> to remove it!"
             exit (1)
 
     try:

From 3beb3079013a9b43b29d1618b7dbb4a25777e510 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 2 Sep 2015 00:15:14 +0200
Subject: [PATCH 049/225] core: add VM persistence info to qubesdb (#1101)

Fixes qubesos/qubes-issues#1101
---
 core-modules/000QubesVm.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index 7edbc350..ccf56a4f 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -1020,6 +1020,10 @@ class QubesVm(object):
         self.qdb.write("/name", self.name)
         self.qdb.write("/qubes-vm-type", self.type)
         self.qdb.write("/qubes-vm-updateable", str(self.updateable))
+        self.qdb.write("/qubes-vm-persistence",
+                       "full" if self.updateable else "rw-only")
+        self.qdb.write("/qubes-base-template",
+                       self.template.name if self.template else '')
 
         if self.is_netvm():
             self.qdb.write("/qubes-netvm-gateway", self.gateway)

From 4ba7d5a3ccd231667ed853da690d5fe02b27446a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Thu, 3 Sep 2015 02:44:49 +0200
Subject: [PATCH 050/225] version 3.0.21

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index 3e4a61b7..70033792 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.0.20
+3.0.21

From 226a1442852c1f3274f001b4fe3e26407f3d37ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Thu, 3 Sep 2015 21:11:43 +0200
Subject: [PATCH 051/225] core: fix vm-persistence info for DispVM

(reported by @adrelanos)
qubesos/qubes-issues#1101
---
 core-modules/01QubesDisposableVm.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core-modules/01QubesDisposableVm.py b/core-modules/01QubesDisposableVm.py
index 6dcbe136..68b4319f 100644
--- a/core-modules/01QubesDisposableVm.py
+++ b/core-modules/01QubesDisposableVm.py
@@ -151,6 +151,7 @@ class QubesDisposableVm(QubesVm):
     def create_qubesdb_entries(self):
         super(QubesDisposableVm, self).create_qubesdb_entries()
 
+        self.qdb.write("/qubes-vm-persistence", "none")
         self.qdb.write('/qubes-restore-complete', '1')
 
     def start(self, verbose = False, **kwargs):

From 76bddaa28094b13bdd3d9bc4fde4da7d7a1b5fa3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 4 Sep 2015 00:19:40 +0200
Subject: [PATCH 052/225] core: use vm.absolute_path to parse paths in
 qubes.xml

This makes easier to handle some corner cases. One of them is having
entry without `dir_path` defined. This may happen when migrating from R2
(using backup+restore or in-place) while some DisposableVM was running
(even if not included in the backup itself).

Fixes qubesos/qubes-issues#1124
Reported by @doncohen, thanks @wyory for providing more details.
---
 core-modules/000QubesVm.py | 7 ++++++-
 core/storage/__init__.py   | 6 +++---
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index ccf56a4f..1cc8d212 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -376,8 +376,13 @@ class QubesVm(object):
     def absolute_path(self, arg, default):
         if arg is not None and os.path.isabs(arg):
             return arg
-        else:
+        elif self.dir_path is not None:
             return os.path.join(self.dir_path, (arg if arg is not None else default))
+        else:
+            # cannot provide any meaningful value without dir_path; this is
+            # only to import some older format of `qubes.xml` (for example
+            # during migration from older release)
+            return None
 
     def _absolute_path_gen(self, default):
         return lambda value: self.absolute_path(value, default)
diff --git a/core/storage/__init__.py b/core/storage/__init__.py
index 41f3e73f..3a12dbe6 100644
--- a/core/storage/__init__.py
+++ b/core/storage/__init__.py
@@ -55,12 +55,12 @@ class QubesVmStorage(object):
         else:
             self.root_img_size = defaults['root_img_size']
 
-        self.private_img = os.path.join(self.vmdir, vm_files["private_img"])
+        self.private_img = vm.absolute_path(vm_files["private_img"], None)
         if self.vm.template:
             self.root_img = self.vm.template.root_img
         else:
-            self.root_img = os.path.join(self.vmdir, vm_files["root_img"])
-        self.volatile_img = os.path.join(self.vmdir, vm_files["volatile_img"])
+            self.root_img = vm.absolute_path(vm_files["root_img"], None)
+        self.volatile_img = vm.absolute_path(vm_files["volatile_img"], None)
 
         # For now compute this path still in QubesVm
         self.modules_img = modules_img

From 496a74f873f9fc918f806ce93d39dab2bc9f7c82 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 5 Sep 2015 00:03:03 +0200
Subject: [PATCH 053/225] tests: add DispVM to backup compatibility test (r2b2)

Test for qubesos/qubes-issues#1124
---
 tests/backupcompatibility.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/backupcompatibility.py b/tests/backupcompatibility.py
index 71df0db8..2aa3faf4 100644
--- a/tests/backupcompatibility.py
+++ b/tests/backupcompatibility.py
@@ -49,6 +49,7 @@ QUBESXML_R2B2 = '''
   <QubesProxyVm installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="12" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="test-testproxy.conf" label="red" template_qid="1" kernelopts="" memory="200" default_user="user" netvm_qid="3" volatile_img="volatile.img" services="{'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="test-testproxy" netid="3" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/servicevms/test-testproxy"/>
   <QubesProxyVm installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="13" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="testproxy2.conf" label="red" template_qid="9" kernelopts="" memory="200" default_user="user" netvm_qid="2" volatile_img="volatile.img" services="{'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="testproxy2" netid="4" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/servicevms/testproxy2"/>
   <QubesHVm installed_by_rpm="False" netvm_qid="none" qid="14" include_in_backups="True" timezone="localtime" qrexec_timeout="60" conf_file="test-testhvm.conf" label="purple" template_qid="none" internal="False" memory="512" uses_default_netvm="True" services="{'meminfo-writer': False}" default_user="user" pcidevs="[]" name="test-testhvm" qrexec_installed="False" private_img="private.img" drive="None" vcpus="2" root_img="root.img" guiagent_installed="False" debug="False" dir_path="/var/lib/qubes/appvms/test-testhvm"/>
+  <QubesDisposableVm dispid="50" firewall_conf="firewall.xml" label="red" name="disp50" netvm_qid="2" qid="15" template_qid="1"/>
 </QubesVmCollection>
 '''
 

From c2cd84952ea912fe026d99c58faa20a66a68776a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 5 Sep 2015 00:04:04 +0200
Subject: [PATCH 054/225] tests: backup compatibility with R2 (#1124)

QubesOS/qubes-issues#1124
---
 tests/backupcompatibility.py | 185 +++++++++++++++++++++++++++++++++++
 1 file changed, 185 insertions(+)

diff --git a/tests/backupcompatibility.py b/tests/backupcompatibility.py
index 2aa3faf4..775db92f 100644
--- a/tests/backupcompatibility.py
+++ b/tests/backupcompatibility.py
@@ -28,6 +28,7 @@ import subprocess
 
 import unittest
 import sys
+import re
 from qubes.qubes import QubesVmCollection, QubesException
 from qubes import backup
 
@@ -53,6 +54,35 @@ QUBESXML_R2B2 = '''
 </QubesVmCollection>
 '''
 
+QUBESXML_R2 = '''
+<QubesVmCollection updatevm="3" default_kernel="3.7.6-2" default_netvm="3" default_fw_netvm="2" default_template="1" clockvm="2">
+  <QubesTemplateVm installed_by_rpm="True" kernel="3.7.6-2" uses_default_kernelopts="True" qid="1" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="fedora-20-x64.conf" label="black" template_qid="none" kernelopts="" memory="400" default_user="user" netvm_qid="3" uses_default_netvm="True" volatile_img="volatile.img" services="{ 'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="fedora-20-x64" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/vm-templates/fedora-20-x64"/>
+  <QubesNetVm installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="2" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="netvm.conf" label="red" template_qid="1" kernelopts="iommu=soft swiotlb=4096" memory="200" default_user="user" volatile_img="volatile.img" services="{'ntpd': False, 'meminfo-writer': False}" maxmem="1535" pcidevs="['02:00.0', '03:00.0']" name="netvm" netid="1" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/servicevms/netvm"/>
+  <QubesProxyVm installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="3" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="firewallvm.conf" label="green" template_qid="1" kernelopts="" memory="200" default_user="user" netvm_qid="2" volatile_img="volatile.img" services="{'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="firewallvm" netid="2" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/servicevms/firewallvm"/>
+  <QubesAppVm installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="4" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="True" conf_file="fedora-20-x64-dvm.conf" label="gray" template_qid="1" kernelopts="" memory="400" default_user="user" netvm_qid="3" uses_default_netvm="True" volatile_img="volatile.img" services="{ 'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="fedora-20-x64-dvm" private_img="private.img" vcpus="1" root_img="root.img" debug="False" dir_path="/var/lib/qubes/appvms/fedora-20-x64-dvm"/>
+  <QubesAppVm backup_content="True" backup_path="appvms/test-work" installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="5" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="test-work.conf" label="green" template_qid="1" kernelopts="" memory="400" default_user="user" netvm_qid="3" uses_default_netvm="True" volatile_img="volatile.img" services="{'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="test-work" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/appvms/test-work"/>
+  <QubesAppVm installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="6" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="banking.conf" label="green" template_qid="1" kernelopts="" memory="400" default_user="user" netvm_qid="3" uses_default_netvm="True" volatile_img="volatile.img" services="{'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="banking" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/appvms/banking"/>
+  <QubesAppVm installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="7" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="personal.conf" label="yellow" template_qid="1" kernelopts="" memory="400" default_user="user" netvm_qid="3" uses_default_netvm="True" volatile_img="volatile.img" services="{'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="personal" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/appvms/personal"/>
+  <QubesAppVm installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="8" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="untrusted.conf" label="red" template_qid="1" kernelopts="" memory="400" default_user="user" netvm_qid="12" uses_default_netvm="False" volatile_img="volatile.img" services="{'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="untrusted" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/appvms/untrusted"/>
+  <QubesTemplateVm backup_size="104857600" backup_content="True" backup_path="vm-templates/test-template-clone" installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="9" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="test-template-clone.conf" label="green" template_qid="none" kernelopts="" memory="400" default_user="user" netvm_qid="3" uses_default_netvm="True" volatile_img="volatile.img" services="{'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="test-template-clone" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/vm-templates/test-template-clone"/>
+  <QubesAppVm backup_size="104857600" backup_content="True" backup_path="appvms/test-custom-template-appvm" installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="10" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="test-custom-template-appvm.conf" label="yellow" template_qid="9" kernelopts="" memory="400" default_user="user" netvm_qid="3" uses_default_netvm="True" volatile_img="volatile.img" services="{'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="test-custom-template-appvm" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/appvms/test-custom-template-appvm"/>
+  <QubesAppVm backup_size="104857600" backup_content="True" backup_path="appvms/test-standalonevm" installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="11" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="test-standalonevm.conf" label="blue" template_qid="none" kernelopts="" memory="400" default_user="user" netvm_qid="3" uses_default_netvm="True" volatile_img="volatile.img" services="{'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="test-standalonevm" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/appvms/test-standalonevm"/>
+  <QubesProxyVm backup_size="104857600" backup_content="True" backup_path="servicevms/test-testproxy" installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="12" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="test-testproxy.conf" label="red" template_qid="1" kernelopts="" memory="200" default_user="user" netvm_qid="3" volatile_img="volatile.img" services="{'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="test-testproxy" netid="3" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/servicevms/test-testproxy"/>
+  <QubesProxyVm installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="13" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="testproxy2.conf" label="red" template_qid="9" kernelopts="" memory="200" default_user="user" netvm_qid="2" volatile_img="volatile.img" services="{'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="testproxy2" netid="4" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/servicevms/testproxy2"/>
+  <QubesHVm backup_size="104857600" backup_content="True" backup_path="appvms/test-testhvm" installed_by_rpm="False" netvm_qid="none" qid="14" include_in_backups="True" timezone="localtime" qrexec_timeout="60" conf_file="test-testhvm.conf" label="purple" template_qid="none" internal="False" memory="512" uses_default_netvm="True" services="{'meminfo-writer': False}" default_user="user" pcidevs="[]" name="test-testhvm" qrexec_installed="False" private_img="private.img" drive="None" vcpus="2" root_img="root.img" guiagent_installed="False" debug="False" dir_path="/var/lib/qubes/appvms/test-testhvm"/>
+  <QubesDisposableVm dispid="50" firewall_conf="firewall.xml" label="red" name="disp50" netvm_qid="2" qid="15" template_qid="1"/>
+</QubesVmCollection>
+'''
+
+MANGLED_SUBDIRS_R2 = {
+    "test-work": "vm5",
+    "test-template-clone": "vm9",
+    "test-custom-template-appvm": "vm10",
+    "test-standalonevm": "vm11",
+    "test-testproxy": "vm12",
+    "test-testhvm": "vm14",
+}
+
 APPTEMPLATE_R2B2 = '''
 [Desktop Entry]
 Name=%VMNAME%: {name}
@@ -107,6 +137,13 @@ QUBESXML_R1 = '''<?xml version='1.0' encoding='UTF-8'?>
 <QubesVmCollection clockvm="2" default_fw_netvm="2" default_kernel="3.2.7-10" default_netvm="3" default_template="1" updatevm="3"><QubesTemplateVm conf_file="fedora-17-x64.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/vm-templates/fedora-17-x64" include_in_backups="True" installed_by_rpm="True" internal="False" kernel="3.2.7-10" kernelopts="" label="gray" maxmem="4063" memory="400" name="fedora-17-x64" netvm_qid="3" pcidevs="[]" private_img="private.img" qid="1" root_img="root.img" services="{&apos;meminfo-writer&apos;: True}" template_qid="none" uses_default_kernel="True" uses_default_kernelopts="True" uses_default_netvm="True" vcpus="2" volatile_img="volatile.img" /><QubesNetVm conf_file="netvm.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/servicevms/netvm" include_in_backups="True" installed_by_rpm="False" internal="False" kernel="3.2.7-10" kernelopts="iommu=soft swiotlb=2048" label="red" maxmem="4063" memory="200" name="netvm" netid="1" pcidevs="[&apos;00:19.0&apos;, &apos;03:00.0&apos;]" private_img="private.img" qid="2" root_img="root.img" services="{&apos;ntpd&apos;: False, &apos;meminfo-writer&apos;: False}" template_qid="1" uses_default_kernel="True" uses_default_kernelopts="True" vcpus="2" volatile_img="volatile.img" /><QubesProxyVm conf_file="firewallvm.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/servicevms/firewallvm" include_in_backups="True" installed_by_rpm="False" internal="False" kernel="3.2.7-10" kernelopts="" label="green" maxmem="4063" memory="200" name="firewallvm" netid="2" netvm_qid="2" pcidevs="[]" private_img="private.img" qid="3" root_img="root.img" services="{&apos;meminfo-writer&apos;: True}" template_qid="1" uses_default_kernel="True" uses_default_kernelopts="True" vcpus="2" volatile_img="volatile.img" /><QubesAppVm conf_file="fedora-17-x64-dvm.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/appvms/fedora-17-x64-dvm" include_in_backups="True" installed_by_rpm="False" internal="True" kernel="3.2.7-10" kernelopts="" label="gray" maxmem="4063" memory="400" name="fedora-17-x64-dvm" netvm_qid="3" pcidevs="[]" private_img="private.img" qid="4" root_img="root.img" services="{&apos;meminfo-writer&apos;: True}" template_qid="1" uses_default_kernel="True" uses_default_kernelopts="True" uses_default_netvm="True" vcpus="1" volatile_img="volatile.img" /><QubesAppVm conf_file="test-work.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/appvms/test-work" include_in_backups="True" installed_by_rpm="False" internal="False" kernel="3.2.7-10" kernelopts="" label="green" maxmem="4063" memory="400" name="test-work" netvm_qid="3" pcidevs="[]" private_img="private.img" qid="5" root_img="root.img" services="{&apos;meminfo-writer&apos;: True}" template_qid="1" uses_default_kernel="True" uses_default_kernelopts="True" uses_default_netvm="True" vcpus="2" volatile_img="volatile.img" /><QubesAppVm conf_file="personal.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/appvms/personal" include_in_backups="True" installed_by_rpm="False" internal="False" kernel="3.2.7-10" kernelopts="" label="yellow" maxmem="4063" memory="400" name="personal" netvm_qid="3" pcidevs="[]" private_img="private.img" qid="6" root_img="root.img" services="{&apos;meminfo-writer&apos;: True}" template_qid="1" uses_default_kernel="True" uses_default_kernelopts="True" uses_default_netvm="True" vcpus="2" volatile_img="volatile.img" /><QubesAppVm conf_file="banking.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/appvms/banking" include_in_backups="True" installed_by_rpm="False" internal="False" kernel="3.2.7-10" kernelopts="" label="green" maxmem="4063" memory="400" name="banking" netvm_qid="3" pcidevs="[]" private_img="private.img" qid="7" root_img="root.img" services="{&apos;meminfo-writer&apos;: True}" template_qid="1" uses_default_kernel="True" uses_default_kernelopts="True" uses_default_netvm="True" vcpus="2" volatile_img="volatile.img" /><QubesAppVm conf_file="untrusted.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/appvms/untrusted" include_in_backups="True" installed_by_rpm="False" internal="False" kernel="3.2.7-10" kernelopts="" label="red" maxmem="4063" memory="400" name="untrusted" netvm_qid="3" pcidevs="[]" private_img="private.img" qid="8" root_img="root.img" services="{&apos;meminfo-writer&apos;: True}" template_qid="1" uses_default_kernel="True" uses_default_kernelopts="True" uses_default_netvm="True" vcpus="2" volatile_img="volatile.img" /><QubesAppVm conf_file="test-standalonevm.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/appvms/test-standalonevm" include_in_backups="True" installed_by_rpm="False" internal="False" kernel="None" kernelopts="" label="red" maxmem="4063" memory="400" name="test-standalonevm" netvm_qid="3" pcidevs="[]" private_img="private.img" qid="9" root_img="root.img" services="{&apos;meminfo-writer&apos;: True}" template_qid="none" uses_default_kernel="False" uses_default_kernelopts="True" uses_default_netvm="True" vcpus="2" volatile_img="volatile.img" /><QubesAppVm conf_file="test-testvm.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/appvms/test-testvm" include_in_backups="True" installed_by_rpm="False" internal="False" kernel="3.2.7-10" kernelopts="" label="red" mac="00:16:3E:5E:6C:55" maxmem="4063" memory="400" name="test-testvm" netvm_qid="3" pcidevs="[]" private_img="private.img" qid="10" root_img="root.img" services="{&apos;meminfo-writer&apos;: True}" template_qid="1" uses_default_kernel="True" uses_default_kernelopts="True" uses_default_netvm="True" vcpus="2" volatile_img="volatile.img" /><QubesTemplateVm conf_file="test-template-clone.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/vm-templates/test-template-clone" include_in_backups="True" installed_by_rpm="False" internal="False" kernel="3.2.7-10" kernelopts="" label="gray" maxmem="4063" memory="400" name="test-template-clone" netvm_qid="3" pcidevs="[]" private_img="private.img" qid="11" root_img="root.img" services="{&apos;meminfo-writer&apos;: True}" template_qid="none" uses_default_kernel="True" uses_default_kernelopts="True" uses_default_netvm="True" vcpus="2" volatile_img="volatile.img" /><QubesAppVm conf_file="test-custom-template-appvm.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/appvms/test-custom-template-appvm" include_in_backups="True" installed_by_rpm="False" internal="False" kernel="3.2.7-10" kernelopts="" label="yellow" maxmem="4063" memory="400" name="test-custom-template-appvm" netvm_qid="3" pcidevs="[]" private_img="private.img" qid="12" root_img="root.img" services="{&apos;meminfo-writer&apos;: True}" template_qid="11" uses_default_kernel="True" uses_default_kernelopts="True" uses_default_netvm="True" vcpus="2" volatile_img="volatile.img" /><QubesProxyVm conf_file="test-testproxy.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/servicevms/test-testproxy" include_in_backups="True" installed_by_rpm="False" internal="False" kernel="3.2.7-10" kernelopts="" label="yellow" maxmem="4063" memory="200" name="test-testproxy" netid="3" netvm_qid="2" pcidevs="[]" private_img="private.img" qid="13" root_img="root.img" services="{&apos;meminfo-writer&apos;: True}" template_qid="1" uses_default_kernel="True" uses_default_kernelopts="True" vcpus="2" volatile_img="volatile.img" /></QubesVmCollection>
 '''
 
+BACKUP_HEADER_R2 = '''version=3
+hmac-algorithm=SHA512
+crypto-algorithm=aes-256-cbc
+encrypted={encrypted}
+compressed={compressed}
+compression-filter=gzip
+'''
 
 class TC_00_BackupCompatibility(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
     def create_whitelisted_appmenus(self, filename):
@@ -244,6 +281,118 @@ class TC_00_BackupCompatibility(qubes.tests.BackupTestsMixin, qubes.tests.QubesT
                 "vm-templates/test-template-clone")),
             appmenus_list)
 
+    def calculate_hmac(self, f_name, algorithm="sha512", password="qubes"):
+        subprocess.check_call(["openssl", "dgst", "-"+algorithm, "-hmac",
+                               password],
+                              stdin=open(self.fullpath(f_name), "r"),
+                              stdout=open(self.fullpath(f_name+".hmac"), "w"))
+
+    def append_backup_stream(self, f_name, stream, basedir=None):
+        if not basedir:
+            basedir = self.backupdir
+        subprocess.check_call(["tar", "-cO", "--posix", "-C", basedir,
+                               f_name],
+                              stdout=stream)
+
+    def handle_v3_file(self, f_name, subdir, stream, compressed=True,
+                       encrypted=True):
+        # create inner archive
+        tar_cmdline = ["tar", "-Pc", '--sparse',
+               '-C', self.fullpath(os.path.dirname(f_name)),
+               '--xform', 's:^%s:%s\\0:' % (
+                   os.path.basename(f_name),
+                   subdir),
+               os.path.basename(f_name)
+               ]
+        if compressed:
+            tar_cmdline.insert(-1, "--use-compress-program=%s" % "gzip")
+        tar = subprocess.Popen(tar_cmdline, stdout=subprocess.PIPE)
+        if encrypted:
+            encryptor = subprocess.Popen(
+                ["openssl", "enc", "-e", "-aes-256-cbc", "-pass", "pass:qubes"],
+                stdin=tar.stdout,
+                stdout=subprocess.PIPE)
+            data = encryptor.stdout
+        else:
+            data = tar.stdout
+
+        stage1_dir = self.fullpath(os.path.join("stage1", subdir))
+        if not os.path.exists(stage1_dir):
+            os.makedirs(stage1_dir)
+        subprocess.check_call(["split", "--numeric-suffixes",
+                               "--suffix-length=3",
+                               "--bytes="+str(100*1024*1024), "-",
+                               os.path.join(stage1_dir,
+                                            os.path.basename(f_name+"."))],
+                              stdin=data)
+
+        for part in sorted(os.listdir(stage1_dir)):
+            if not re.match(
+                    r"^{}.[0-9][0-9][0-9]$".format(os.path.basename(f_name)),
+                    part):
+                continue
+            part_with_dir = os.path.join(subdir, part)
+            self.calculate_hmac(os.path.join("stage1", part_with_dir))
+            self.append_backup_stream(part_with_dir, stream,
+                                      basedir=self.fullpath("stage1"))
+            self.append_backup_stream(part_with_dir+".hmac", stream,
+                                      basedir=self.fullpath("stage1"))
+
+    def create_v3_backup(self, encrypted=True, compressed=True):
+        """
+        Create "backup format 3" backup - used in R2 and R3.0
+
+        :param encrypt: Should the backup be encrypted
+        :return:
+        """
+        output = open(self.fullpath("backup.bin"), "w")
+        f = open(self.fullpath("backup-header"), "w")
+        f.write(BACKUP_HEADER_R2.format(
+            encrypted=str(encrypted),
+            compressed=str(compressed)
+        ))
+        f.close()
+        self.calculate_hmac("backup-header")
+        self.append_backup_stream("backup-header", output)
+        self.append_backup_stream("backup-header.hmac", output)
+        f = open(self.fullpath("qubes.xml"), "w")
+        if encrypted:
+            qubesxml = QUBESXML_R2
+            for vmname, subdir in MANGLED_SUBDIRS_R2.items():
+                qubesxml = re.sub(r"[a-z-]*/{}".format(vmname),
+                                  subdir, qubesxml)
+            f.write(qubesxml)
+        else:
+            f.write(QUBESXML_R2)
+        f.close()
+
+        self.handle_v3_file("qubes.xml", "", output, encrypted=encrypted,
+                            compressed=compressed)
+
+        self.create_v1_files(r2b2=True)
+        for vm_type in ["appvms", "servicevms"]:
+            for vm_name in os.listdir(self.fullpath(vm_type)):
+                vm_dir = os.path.join(vm_type, vm_name)
+                for f_name in os.listdir(self.fullpath(vm_dir)):
+                    if encrypted:
+                        subdir = MANGLED_SUBDIRS_R2[vm_name]
+                    else:
+                        subdir = vm_dir
+                    self.handle_v3_file(
+                        os.path.join(vm_dir, f_name),
+                        subdir+'/', output, encrypted=encrypted)
+
+        for vm_name in os.listdir(self.fullpath("vm-templates")):
+            vm_dir = os.path.join("vm-templates", vm_name)
+            if encrypted:
+                subdir = MANGLED_SUBDIRS_R2[vm_name]
+            else:
+                subdir = vm_dir
+            self.handle_v3_file(
+                os.path.join(vm_dir, "."),
+                subdir+'/', output, encrypted=encrypted)
+
+        output.close()
 
     def test_100_r1(self):
         self.create_v1_files(r2b2=False)
@@ -286,3 +435,39 @@ class TC_00_BackupCompatibility(qubes.tests.BackupTestsMixin, qubes.tests.QubesT
         self.assertEqual(self.qc.get_vm_by_name("test-custom-template-appvm")
                          .template,
                          self.qc.get_vm_by_name("test-template-clone"))
+
+    def test_210_r2(self):
+        self.create_v3_backup(False)
+
+        self.restore_backup(self.fullpath("backup.bin"), options={
+            'use-default-template': True,
+            'use-default-netvm': True,
+        })
+        self.assertIsNotNone(self.qc.get_vm_by_name("test-template-clone"))
+        self.assertIsNotNone(self.qc.get_vm_by_name("test-testproxy"))
+        self.assertIsNotNone(self.qc.get_vm_by_name("test-work"))
+        self.assertIsNotNone(self.qc.get_vm_by_name("test-testhvm"))
+        self.assertIsNotNone(self.qc.get_vm_by_name("test-standalonevm"))
+        self.assertIsNotNone(self.qc.get_vm_by_name(
+            "test-custom-template-appvm"))
+        self.assertEqual(self.qc.get_vm_by_name("test-custom-template-appvm")
+                         .template,
+                         self.qc.get_vm_by_name("test-template-clone"))
+
+    def test_220_r2_encrypted(self):
+        self.create_v3_backup(True)
+
+        self.restore_backup(self.fullpath("backup.bin"), options={
+            'use-default-template': True,
+            'use-default-netvm': True,
+        })
+        self.assertIsNotNone(self.qc.get_vm_by_name("test-template-clone"))
+        self.assertIsNotNone(self.qc.get_vm_by_name("test-testproxy"))
+        self.assertIsNotNone(self.qc.get_vm_by_name("test-work"))
+        self.assertIsNotNone(self.qc.get_vm_by_name("test-testhvm"))
+        self.assertIsNotNone(self.qc.get_vm_by_name("test-standalonevm"))
+        self.assertIsNotNone(self.qc.get_vm_by_name(
+            "test-custom-template-appvm"))
+        self.assertEqual(self.qc.get_vm_by_name("test-custom-template-appvm")
+                         .template,
+                         self.qc.get_vm_by_name("test-template-clone"))

From 4a4d16feb97a66e3bd99292000faa11fcb3be749 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 5 Sep 2015 00:28:45 +0200
Subject: [PATCH 055/225] version 3.0.22

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index 70033792..6795d4dc 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.0.21
+3.0.22

From 9205fbb336345be58793fd325126fcd755a4ce14 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sun, 6 Sep 2015 00:06:09 +0200
Subject: [PATCH 056/225] backup: fix R2B3 and older backup restore (#1124)

Similar issue to qubesos/qubes-issues#1124
---
 core/backup.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/core/backup.py b/core/backup.py
index 92280a7c..2493d4a8 100644
--- a/core/backup.py
+++ b/core/backup.py
@@ -1706,6 +1706,10 @@ def backup_restore_prepare(backup_location, passphrase, options=None,
         if check_vm.qid == 0:
             return os.path.exists(os.path.join(backup_dir, 'dom0-home'))
 
+        # DisposableVM
+        if check_vm.dir_path is None:
+            return False
+
         backup_vm_dir_path = check_vm.dir_path.replace(
             system_path["qubes_base_dir"], backup_dir)
 

From 5f0d5640515e2224e2e9aa9258c3a9fd9e9556ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 25 Sep 2015 20:37:44 +0200
Subject: [PATCH 057/225] utils/QubesWatch: use timers to retry QubesDB watch
 registration

QubesWatch._register_watches is called from libvirt event callback,
asynchronously to qvm-start. This means that `qubesdb-daemon` may
not be running or populated yet.

If first QubesDB connection (or watch registration) fails, schedule next
try using timers in libvirt event API (as it is base of QubesWatch
mainloop), instead of some sleep loop. This way other events will be
processed in the meantime.

QubesOS/qubes-issues#1110
---
 core/qubesutils.py | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/core/qubesutils.py b/core/qubesutils.py
index 041a66c8..120d4a98 100644
--- a/core/qubesutils.py
+++ b/core/qubesutils.py
@@ -790,29 +790,41 @@ class QubesWatch(object):
             name = libvirt_domain.name()
             if name in self._qdb:
                 return
+            if not libvirt_domain.isActive():
+                return
             # open separate connection to Qubes DB:
             # 1. to not confuse pull() with responses to real commands sent from
             # other threads (like read, write etc) with watch events
             # 2. to not think whether QubesDB is thread-safe (it isn't)
-            while libvirt_domain.isActive() and name not in self._qdb:
-                try:
-                    self._qdb[name] = QubesDB(name)
-                except Error as e:
-                    if e.args[0] != 2:
-                        raise
-                time.sleep(0.5)
-            if name not in self._qdb:
-                # domain no longer active
+            try:
+                self._qdb[name] = QubesDB(name)
+            except Error as e:
+                if e.args[0] != 2:
+                    raise
+                libvirt.virEventAddTimeout(500, self._retry_register_watches,
+                                           libvirt_domain)
                 return
         else:
             name = "dom0"
             self._qdb[name] = QubesDB(name)
-        self._qdb[name].watch('/qubes-block-devices')
+        try:
+            self._qdb[name].watch('/qubes-block-devices')
+        except Error as e:
+            if e.args[0] == 102: # Connection reset by peer
+                # QubesDB daemon not running - most likely we've connected to
+                #  stale daemon which just exited; retry later
+                libvirt.virEventAddTimeout(500, self._retry_register_watches,
+                                           libvirt_domain)
+                return
         self._qdb_events[name] = libvirt.virEventAddHandle(
             self._qdb[name].watch_fd(),
             libvirt.VIR_EVENT_HANDLE_READABLE,
             self._qdb_handler, name)
 
+    def _retry_register_watches(self, timer, libvirt_domain):
+        libvirt.virRemoveTimeout(timer)
+        self._register_watches(libvirt_domain)
+
     def _unregister_watches(self, libvirt_domain):
         name = libvirt_domain.name()
         if name in self._qdb_events:

From ef6095662e9ddeb101c0065b86bac3602f9b8fa0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 25 Sep 2015 22:02:53 +0200
Subject: [PATCH 058/225] utils/QubesWatch: register domain watches on
 VIR_DOMAIN_EVENT_RESUMED

QubesVM.start() first creates domain as paused, completes its setup
(including starting qubesdb-daemon and creating appropriate entries),
then resumes the domain. So wait for that resume to be sure that
`qubesdb-daemon` is already running and populated.

QubesOS/qubes-issues#1110
---
 core/qubesutils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/core/qubesutils.py b/core/qubesutils.py
index 120d4a98..feaf1407 100644
--- a/core/qubesutils.py
+++ b/core/qubesutils.py
@@ -835,7 +835,9 @@ class QubesWatch(object):
             del(self._qdb[name])
 
     def _domain_list_changed(self, conn, domain, event, reason, param):
-        if event == libvirt.VIR_DOMAIN_EVENT_STARTED:
+        # use VIR_DOMAIN_EVENT_RESUMED instead of VIR_DOMAIN_EVENT_STARTED to
+        #  make sure that qubesdb daemon is already running
+        if event == libvirt.VIR_DOMAIN_EVENT_RESUMED:
             self._register_watches(domain)
         elif event == libvirt.VIR_DOMAIN_EVENT_STOPPED:
             self._unregister_watches(domain)

From dd1bea98e16f7674b25dd7b3ab29b5a638ee8e66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 25 Sep 2015 22:06:14 +0200
Subject: [PATCH 059/225] core/start: ensure that the previous QubesDB daemon
 isn't running

When restarting VM (starting it just after it was shut down), it may
happen that previous `qubesdb-daemon` instance is still running - if VM
doesn't properly terminate the connection, dom0 part will not terminate
immediately, but at next alive check (every 10s). Such `qubesdb-daemon`,
when terminating, will remove pid file and socket file. In case of new
daemon already running it would be those of the new daemon, making the
whole QubesDB of this VM inaccessible for dom0 (`qubesdb-daemon` is
running, but its socket is removed).

To prevent this race, ensure that previous instance is terminated before
starting the new one.
There is no need to manually removing socket file, because if some stale
socket exists, it will be replaced by the new one when new
`qubesdb-daemon` starts up.

QubesOS/qubes-issues#1241
---
 core-modules/000QubesVm.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index 1cc8d212..5006296f 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -34,6 +34,7 @@ import sys
 import time
 import uuid
 import xml.parsers.expat
+import signal
 from qubes import qmemman
 from qubes import qmemman_algo
 import libvirt
@@ -1703,6 +1704,18 @@ class QubesVm(object):
 
     def start_qubesdb(self):
         self.log.debug('start_qubesdb()')
+        pidfile = '/var/run/qubes/qubesdb.{}.pid'.format(self.name)
+        try:
+            if os.path.exists(pidfile):
+                old_qubesdb_pid = open(pidfile, 'r').read()
+                os.kill(int(old_qubesdb_pid), signal.SIGTERM)
+                timeout = 25
+                while os.path.exists(pidfile) and timeout:
+                    time.sleep(0.2)
+                    timeout -= 1
+        except IOError:  # ENOENT (pidfile)
+            pass
+
         retcode = subprocess.call ([
             system_path["qubesdb_daemon_path"],
             str(self.xid),

From b9e6b0a076595b14c45930d1233af8d59bb85357 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 25 Sep 2015 22:21:10 +0200
Subject: [PATCH 060/225] core/hvm: fix error message when creating HVM while
 not supported by the hardware

libvirt reports such errors at domain definition, not startup (as was in
libxl).

QubesOS/qubes-issues#1198
---
 core-modules/000QubesVm.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index 1cc8d212..038623b7 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -711,7 +711,15 @@ class QubesVm(object):
 
     def _update_libvirt_domain(self):
         domain_config = self.create_config_file()
-        self._libvirt_domain = vmm.libvirt_conn.defineXML(domain_config)
+        try:
+            self._libvirt_domain = vmm.libvirt_conn.defineXML(domain_config)
+        except libvirt.libvirtError as e:
+            # shouldn't this be in QubesHVm implementation?
+            if e.get_error_code() == libvirt.VIR_ERR_OS_TYPE and \
+                    e.get_str2() == 'hvm':
+                raise QubesException("HVM domains not supported on this "
+                                     "machine. Check BIOS settings for "
+                                     "VT-x/AMD-V extensions.")
         self.uuid = uuid.UUID(bytes=self._libvirt_domain.UUID())
 
     @property

From 7b9247efc5d1054d9827f2f5527ccf2506c5d7dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 29 Sep 2015 16:50:21 +0200
Subject: [PATCH 061/225] version 3.1.0

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index 6795d4dc..fd2a0186 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.0.22
+3.1.0

From 5043d17197b47af3364ba664a55d2bc1a266a444 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 30 Sep 2015 22:26:15 +0200
Subject: [PATCH 062/225] tests: fix misuse of self.assertNotRaises (#1163)

Fixes QubesOS/qubes-issues#1163
---
 tests/basic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/basic.py b/tests/basic.py
index 4cc662ce..6d36232d 100644
--- a/tests/basic.py
+++ b/tests/basic.py
@@ -98,7 +98,7 @@ class TC_01_Properties(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
             os.path.join(os.getenv("HOME"), ".local/share/applications",
                 self.vmname + "-firefox.desktop")))
         self.assertEquals(pre_rename_firewall, self.vm.get_firewall_conf())
-        with self.assertNotRaises(QubesException, OSError):
+        with self.assertNotRaises((QubesException, OSError)):
             self.vm.write_firewall_conf({'allow': False})
 
     def test_010_netvm(self):

From d900b21dd8f027cd0dcf90dd09b19ce610fb829a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 30 Sep 2015 22:27:15 +0200
Subject: [PATCH 063/225] tests: add test for #1256

QubesOS/qubes-issues#1256
---
 tests/basic.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tests/basic.py b/tests/basic.py
index 6d36232d..7cf2c993 100644
--- a/tests/basic.py
+++ b/tests/basic.py
@@ -71,6 +71,10 @@ class TC_01_Properties(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
 
         self.assertEqual(self.vm.name, self.vmname)
         self.vm.write_firewall_conf({'allow': False, 'allowDns': False})
+        self.vm.autostart = True
+        self.addCleanup(os.system,
+                        'sudo systemctl -q disable qubes-vm@{}.service || :'.
+                        format(self.vmname))
         pre_rename_firewall = self.vm.get_firewall_conf()
 
         #TODO: change to setting property when implemented
@@ -100,6 +104,13 @@ class TC_01_Properties(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
         self.assertEquals(pre_rename_firewall, self.vm.get_firewall_conf())
         with self.assertNotRaises((QubesException, OSError)):
             self.vm.write_firewall_conf({'allow': False})
+        self.assertTrue(self.vm.autostart)
+        self.assertTrue(os.path.exists(
+            '/etc/systemd/system/multi-user.target.wants/'
+            'qubes-vm@{}.service'.format(newname)))
+        self.assertFalse(os.path.exists(
+            '/etc/systemd/system/multi-user.target.wants/'
+            'qubes-vm@{}.service'.format(self.vmname)))
 
     def test_010_netvm(self):
         if self.qc.get_default_netvm() is None:

From 3cdae8525452deb7b3b8ec8f1c7d139bd18fdff4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 2 Oct 2015 01:57:00 +0200
Subject: [PATCH 064/225] core: preserve 'autostart' setting during VM rename

Systemd unit name needs to be renamed.

Fixes QubesOS/qubes-issues#1256
---
 core-modules/000QubesVm.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index 1cc8d212..5ebba41f 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -558,6 +558,9 @@ class QubesVm(object):
         return re.match(r"^[a-zA-Z][a-zA-Z0-9_.-]*$", name) is not None
 
     def pre_rename(self, new_name):
+        if self.autostart:
+            subprocess.check_call(['sudo', 'systemctl', '-q', 'disable',
+                                   'qubes-vm@{}.service'.format(self.name)])
         # fire hooks
         for hook in self.hooks_pre_rename:
             hook(self, new_name)
@@ -603,6 +606,9 @@ class QubesVm(object):
         self.post_rename(old_name)
 
     def post_rename(self, old_name):
+        if self.autostart:
+            # force setter to be called again
+            self.autostart = self.autostart
         # fire hooks
         for hook in self.hooks_post_rename:
             hook(self, old_name)

From 958b59cc0082a2723c6a20262a6df00adb750f2c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 3 Oct 2015 01:50:15 +0200
Subject: [PATCH 065/225] core: Show the right kernel version for DispVMs

Fixes QubesOS/qubes-issues#948
---
 core-modules/01QubesDisposableVm.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/core-modules/01QubesDisposableVm.py b/core-modules/01QubesDisposableVm.py
index 68b4319f..d27e9068 100644
--- a/core-modules/01QubesDisposableVm.py
+++ b/core-modules/01QubesDisposableVm.py
@@ -97,6 +97,11 @@ class QubesDisposableVm(QubesVm):
             disp_template = kwargs['disp_template']
             kwargs['template'] = disp_template.template
             kwargs['dir_path'] = disp_template.dir_path
+            kwargs['kernel'] = disp_template.kernel
+            kwargs['uses_default_kernel'] = disp_template.uses_default_kernel
+            kwargs['kernelopts'] = disp_template.kernelopts
+            kwargs['uses_default_kernelopts'] = \
+                disp_template.uses_default_kernelopts
         super(QubesDisposableVm, self).__init__(**kwargs)
 
         assert self.template is not None, "Missing template for DisposableVM!"

From 9ea81c50ae5c6a373c9836fcc7786ec1427ae857 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 5 Oct 2015 05:46:07 +0200
Subject: [PATCH 066/225] tests: add tests for 'qvm-prefs'

QubesOS/qubes-issues#1184
QubesOS/qubes-issues#1274
QubesOS/qubes-issues#1273
QubesOS/qubes-issues#1272
---
 tests/basic.py | 315 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 315 insertions(+)

diff --git a/tests/basic.py b/tests/basic.py
index 7cf2c993..d3a524c2 100644
--- a/tests/basic.py
+++ b/tests/basic.py
@@ -26,6 +26,7 @@
 import multiprocessing
 import os
 import shutil
+import subprocess
 
 import unittest
 import time
@@ -257,4 +258,318 @@ class TC_01_Properties(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
         self.assertEquals(testvm1.get_firewall_conf(),
                           testvm3.get_firewall_conf())
 
+class TC_02_QvmPrefs(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
+    def setup_appvm(self):
+        self.testvm = self.qc.add_new_vm(
+            "QubesAppVm",
+            name=self.make_vm_name("vm"),
+            template=self.qc.get_default_template())
+        self.testvm.create_on_disk(verbose=False)
+        self.save_and_reload_db()
+        self.qc.unlock_db()
+
+    def setup_hvm(self):
+        self.testvm = self.qc.add_new_vm(
+            "QubesHVm",
+            name=self.make_vm_name("hvm"))
+        self.testvm.create_on_disk(verbose=False)
+        self.save_and_reload_db()
+        self.qc.unlock_db()
+
+    def pref_set(self, name, value, valid=True):
+        p = subprocess.Popen(
+            ['qvm-prefs', '-s', '--', self.testvm.name, name, value],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        (stdout, stderr) = p.communicate()
+        if valid:
+            self.assertEquals(p.returncode, 0,
+                              "qvm-prefs -s .. '{}' '{}' failed: {}{}".format(
+                                  name, value, stdout, stderr
+                              ))
+        else:
+            self.assertNotEquals(p.returncode, 0,
+                                 "qvm-prefs should reject value '{}' for "
+                                 "property '{}'".format(value, name))
+
+    def pref_get(self, name):
+        p = subprocess.Popen(['qvm-prefs', '-g', self.testvm.name, name],
+                             stdout=subprocess.PIPE)
+        (stdout, _) = p.communicate()
+        self.assertEquals(p.returncode, 0)
+        return stdout.strip()
+
+    bool_test_values = [
+        ('true', 'True', True),
+        ('False', 'False', True),
+        ('0', 'False', True),
+        ('1', 'True', True),
+        ('invalid', '', False)
+    ]
+
+    def execute_tests(self, name, values):
+        """
+        Helper function, which executes tests for given property.
+        :param values: list of tuples (value, expected, valid),
+        where 'value' is what should be set and 'expected' is what should
+        qvm-prefs returns as a property value and 'valid' marks valid and
+        invalid values - if it's False, qvm-prefs should reject the value
+        :return: None
+        """
+        for (value, expected, valid) in values:
+            self.pref_set(name, value, valid)
+            if valid:
+                self.assertEquals(self.pref_get(name), expected)
+
+    def test_000_kernel(self):
+        self.setup_appvm()
+
+        default_kernel = self.qc.get_default_kernel()
+        self.execute_tests('kernel', [
+            ('default', default_kernel, True),
+            (default_kernel, default_kernel, True),
+            ('invalid', '', False),
+        ])
+
+    def test_001_include_in_backups(self):
+        self.setup_appvm()
+        self.execute_tests('include_in_backups', self.bool_test_values)
+
+    def test_002_qrexec_timeout(self):
+        self.setup_appvm()
+        self.execute_tests('qrexec_timeout', [
+            ('60', '60', True),
+            ('0', '0', True),
+            ('-10', '', False),
+            ('invalid', '', False)
+        ])
+
+    def test_003_internal(self):
+        self.setup_appvm()
+        self.execute_tests('include_in_backups', self.bool_test_values)
+
+    def test_004_label(self):
+        self.setup_appvm()
+        self.execute_tests('label', [
+            ('red', 'red', True),
+            ('blue', 'blue', True),
+            ('amber', '', False),
+        ])
+
+    def test_005_kernelopts(self):
+        self.setup_appvm()
+        self.execute_tests('kernelopts', [
+            ('option', 'option', True),
+            ('default', 'nopat', True),
+            ('', '', True),
+        ])
+
+    def test_006_template(self):
+        templates = [tpl for tpl in self.qc.values() if tpl.is_template()]
+        if not templates:
+            self.skip("No templates installed")
+        some_template = templates[0].name
+        self.setup_appvm()
+        self.execute_tests('template', [
+            (some_template, some_template, True),
+            ('invalid', '', False),
+        ])
+
+    def test_007_memory(self):
+        self.setup_appvm()
+        qh = qubes.qubes.QubesHost()
+        memory_total = qh.memory_total
+
+        self.execute_tests('memory', [
+            ('300', '300', True),
+            ('1500', '1500', True),
+            # TODO:
+            #('500M', '500', True),
+            #(str(self.testvm.maxmem+500), '', False),
+            (str(2*memory_total), '', False),
+        ])
+
+    def test_008_maxmem(self):
+        self.setup_appvm()
+        qh = qubes.qubes.QubesHost()
+        memory_total = qh.memory_total
+
+        self.execute_tests('memory', [
+            ('300', '300', True),
+            ('1500', '1500', True),
+            # TODO:
+            #('500M', '500', True),
+            #(str(self.testvm.memory-50), '', False),
+            (str(2*memory_total), '', False),
+        ])
+
+    def test_009_autostart(self):
+        self.setup_appvm()
+        self.execute_tests('autostart', self.bool_test_values)
+
+    def test_010_pci_strictreset(self):
+        self.setup_appvm()
+        self.execute_tests('pci_strictreset', self.bool_test_values)
+
+    def test_011_dispvm_netvm(self):
+        self.setup_appvm()
+
+        default_netvm = self.qc.get_default_netvm().name
+        netvms = [tpl for tpl in self.qc.values() if tpl.is_netvm()]
+        if not netvms:
+            self.skip("No netvms installed")
+        some_netvm = netvms[0].name
+        if some_netvm == default_netvm:
+            if len(netvms) <= 1:
+                self.skip("At least two NetVM/ProxyVM required")
+            some_netvm = netvms[1].name
+
+        self.execute_tests('dispvm_netvm', [
+            (some_netvm, some_netvm, True),
+            (default_netvm, default_netvm, True),
+            ('default', default_netvm, True),
+            ('none', '', True),
+            (self.testvm.name, '', False),
+            ('invalid', '', False)
+        ])
+
+    def test_012_mac(self):
+        self.setup_appvm()
+        default_mac = self.testvm.mac
+
+        self.execute_tests('mac', [
+            ('00:11:22:33:44:55', '00:11:22:33:44:55', True),
+            ('auto', default_mac, True),
+            # TODO:
+            #('00:11:22:33:44:55:66', '', False),
+            ('invalid', '', False),
+        ])
+
+    def test_013_default_user(self):
+        self.setup_appvm()
+        self.execute_tests('default_user', [
+            ('someuser', self.testvm.template.default_user, True)
+            # TODO: tests for standalone VMs
+        ])
+
+    def test_014_pcidevs(self):
+        self.setup_appvm()
+        self.execute_tests('pcidevs', [
+            ('[]', '[]', True),
+            ('[ "00:00.0" ]', "['00:00.0']", True),
+            ('invalid', '', False),
+            ('[invalid]', '', False),
+            # TODO:
+            #('["12:12.0"]', '', False)
+        ])
+
+    def test_015_name(self):
+        self.setup_appvm()
+        self.execute_tests('name', [
+            ('invalid!@#name', '', False),
+            # TODO: duplicate name test - would fail for now...
+        ])
+        newname = self.make_vm_name('newname')
+        self.pref_set('name', newname, True)
+        self.qc.lock_db_for_reading()
+        self.qc.load()
+        self.qc.unlock_db()
+        self.testvm = self.qc.get_vm_by_name(newname)
+        self.assertEquals(self.pref_get('name'), newname)
+
+    def test_016_vcpus(self):
+        self.setup_appvm()
+        self.execute_tests('vcpus', [
+            ('1', '1', True),
+            ('100', '', False),
+            ('-1', '', False),
+            ('invalid', '', False),
+        ])
+
+    def test_017_debug(self):
+        self.setup_appvm()
+        self.execute_tests('debug', [
+            ('on', 'True', True),
+            ('off', 'False', True),
+            ('true', 'True', True),
+            ('0', 'False', True),
+            ('invalid', '', False)
+        ])
+
+    def test_018_netvm(self):
+        self.setup_appvm()
+
+        default_netvm = self.qc.get_default_netvm().name
+        netvms = [tpl for tpl in self.qc.values() if tpl.is_netvm()]
+        if not netvms:
+            self.skip("No netvms installed")
+        some_netvm = netvms[0].name
+        if some_netvm == default_netvm:
+            if len(netvms) <= 1:
+                self.skip("At least two NetVM/ProxyVM required")
+            some_netvm = netvms[1].name
+
+        self.execute_tests('netvm', [
+            (some_netvm, some_netvm, True),
+            (default_netvm, default_netvm, True),
+            ('default', default_netvm, True),
+            ('none', '', True),
+            (self.testvm.name, '', False),
+            ('invalid', '', False)
+        ])
+
+    def test_019_guiagent_installed(self):
+        self.setup_hvm()
+        self.execute_tests('guiagent_installed', self.bool_test_values)
+
+    def test_020_qrexec_installed(self):
+        self.setup_hvm()
+        self.execute_tests('qrexec_installed', self.bool_test_values)
+
+    def test_021_seamless_gui_mode(self):
+        self.setup_hvm()
+        # should reject seamless mode without gui agent
+        self.execute_tests('seamless_gui_mode', [
+            ('True', '', False),
+            ('False', 'False', True),
+        ])
+        self.execute_tests('guiagent_installed', [('True', 'True', True)])
+        self.execute_tests('seamless_gui_mode', self.bool_test_values)
+
+    def test_022_drive(self):
+        self.setup_hvm()
+        self.execute_tests('drive', [
+            ('hd:dom0:/tmp/drive.img', 'hd:dom0:/tmp/drive.img', True),
+            ('hd:/tmp/drive.img', 'hd:dom0:/tmp/drive.img', True),
+            ('cdrom:dom0:/tmp/drive.img', 'cdrom:dom0:/tmp/drive.img', True),
+            ('cdrom:/tmp/drive.img', 'cdrom:dom0:/tmp/drive.img', True),
+            ('/tmp/drive.img', 'cdrom:dom0:/tmp/drive.img', True),
+            ('hd:drive.img', '', False),
+            ('drive.img', '', False),
+        ])
+
+    def test_023_timezone(self):
+        self.setup_hvm()
+        self.execute_tests('timezone', [
+            ('localtime', 'localtime', True),
+            ('0', '0', True),
+            ('3600', '3600', True),
+            ('-7200', '-7200', True),
+            ('invalid', '', False),
+        ])
+
+    def test_024_pv_reject_hvm_props(self):
+        self.setup_appvm()
+        self.execute_tests('guiagent_installed', [('False', '', False)])
+        self.execute_tests('qrexec_installed', [('False', '', False)])
+        self.execute_tests('drive', [('/tmp/drive.img', '', False)])
+        self.execute_tests('timezone', [('localtime', '', False)])
+
+    def test_025_hvm_reject_pv_props(self):
+        self.setup_hvm()
+        self.execute_tests('kernel', [('default', '', False)])
+        self.execute_tests('kernelopts', [('default', '', False)])
+
+
 # vim: ts=4 sw=4 et

From 16b8fa011b610cf53225431c1dd48ec633b66cac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 5 Oct 2015 05:52:35 +0200
Subject: [PATCH 067/225] qvm-prefs: fix display format of 'label' property

Fixes QubesOS/qubes-issues#1274
---
 qvm-tools/qvm-prefs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qvm-tools/qvm-prefs b/qvm-tools/qvm-prefs
index 78d09b87..3ef77c6d 100755
--- a/qvm-tools/qvm-prefs
+++ b/qvm-tools/qvm-prefs
@@ -128,7 +128,7 @@ def do_get(vms, vm, prop):
     if getattr(vm, prop, None) is None:
         # not set or set to None
         return
-    if prop in ['template', 'netvm', 'dispvm_netvm']:
+    if prop in ['template', 'netvm', 'dispvm_netvm', 'label']:
         print getattr(vm, prop).name
     else:
         print str(getattr(vm, prop))

From 7be7cf2c5e5859cf3dc9657fb6f053b7a41402ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 5 Oct 2015 05:53:04 +0200
Subject: [PATCH 068/225] qvm-prefs: fix setting dispvm_netvm to none

Fixes QubesOS/qubes-issues#1272
---
 qvm-tools/qvm-prefs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qvm-tools/qvm-prefs b/qvm-tools/qvm-prefs
index 3ef77c6d..476db078 100755
--- a/qvm-tools/qvm-prefs
+++ b/qvm-tools/qvm-prefs
@@ -268,7 +268,7 @@ def set_dispvm_netvm(vms, vm, args):
 
     netvm = args[0]
     if netvm == "none":
-        vm.netvm = None
+        vm.dispvm_netvm = None
         vm.uses_default_dispvm_netvm = False
     elif netvm == "default":
         vm.uses_default_dispvm_netvm = True

From 0e37ac617a76b2a50714a0fc16ac152c2e5f515b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 5 Oct 2015 05:53:48 +0200
Subject: [PATCH 069/225] qvm-prefs: unify 'seamless_gui_mode' with other
 true/false properties

---
 qvm-tools/qvm-prefs | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/qvm-tools/qvm-prefs b/qvm-tools/qvm-prefs
index 476db078..021399aa 100755
--- a/qvm-tools/qvm-prefs
+++ b/qvm-tools/qvm-prefs
@@ -445,14 +445,7 @@ def set_seamless_gui_mode(vms, vm, args):
         print >> sys.stderr, "Missing value (true/false)!"
         return False
 
-    if not args[0].lower() in ['true', 'false']:
-        print >> sys.stderr, "Invalid value, expected 'true' or 'false'"
-        return False
-
-    if args[0].lower() == 'true':
-        vm.seamless_gui_mode = True
-    else:
-        vm.seamless_gui_mode = False
+    vm.seamless_gui_mode = bool(eval(args[0].capitalize()))
     return True
 
 def set_autostart(vms, vm, args):

From d2617917be7422b60cf2b1464067242380e5481f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 5 Oct 2015 05:54:21 +0200
Subject: [PATCH 070/225] qvm-prefs: refuse negative 'qrexec_timeout' value

This should be done in QubesVm class (property setter). But since we're
going to rewrite both qvm-prefs and QubesVm, settle on smaller change
for now.

Fixes QubesOS/qubes-issues#1273
---
 qvm-tools/qvm-prefs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/qvm-tools/qvm-prefs b/qvm-tools/qvm-prefs
index 021399aa..4ee1db1e 100755
--- a/qvm-tools/qvm-prefs
+++ b/qvm-tools/qvm-prefs
@@ -461,6 +461,10 @@ def set_qrexec_timeout(vms, vm, args):
         print >> sys.stderr, "Missing timeout value (seconds)!"
         return False
 
+    if int(args[0]) < 0:
+        print >> sys.stderr, "Negative timeout value not allowed!"
+        return False
+
     vm.qrexec_timeout = int(args[0])
     return True
 

From 28cc933c4c33b3314c57858692dcba58221a79a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 5 Oct 2015 05:56:16 +0200
Subject: [PATCH 071/225] qvm-prefs: fix validation of 'timezone' property

This should be done in QubesVm class (property setter). But since we're
going to rewrite both qvm-prefs and QubesVm, settle on smaller change
for now.

QubesOS/qubes-issues#1184
---
 qvm-tools/qvm-prefs | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/qvm-tools/qvm-prefs b/qvm-tools/qvm-prefs
index 4ee1db1e..96cb164d 100755
--- a/qvm-tools/qvm-prefs
+++ b/qvm-tools/qvm-prefs
@@ -473,9 +473,12 @@ def set_timezone(vms, vm, args):
         print >> sys.stderr, "Missing value ('localtime' or timeoffset in seconds)!"
         return False
 
-    if not args[0].isdigit() and args[0].lower() == 'localtime':
-        print >> sys.stderr, "Invalid timezone value!"
-        return False
+    try:
+        int(args[0])
+    except ValueError:
+        if args[0].lower() != 'localtime':
+            print >> sys.stderr, "Invalid timezone value!"
+            return False
 
     vm.timezone = args[0]
     return True

From a55172d63ee479898a5fbe9588823c2f9132ed96 Mon Sep 17 00:00:00 2001
From: qubesuser <qubesuser@users.noreply.github.com>
Date: Fri, 9 Oct 2015 13:47:27 +0200
Subject: [PATCH 072/225] Support large VMs by removing the fixed balloon
 iteration limit

---
 qmemman/qmemman.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/qmemman/qmemman.py b/qmemman/qmemman.py
index 4a4b0d0e..3de81d75 100755
--- a/qmemman/qmemman.py
+++ b/qmemman/qmemman.py
@@ -136,21 +136,31 @@ class SystemState(object):
 #perform memory ballooning, across all domains, to add "memsize" to Xen free memory
     def do_balloon(self, memsize):
         self.log.info('do_balloon(memsize={!r})'.format(memsize))
-        MAX_TRIES = 20
+        CHECK_PERIOD_S = 3
+        CHECK_MB_S = 100
+
         niter = 0
         prev_memory_actual = None
 
         for i in self.domdict.keys():
             self.domdict[i].no_progress = False
 
+        check_period = max(1, int((CHECK_PERIOD_S + 0.0) / self.BALOON_DELAY))
+        check_delta = CHECK_PERIOD_S * CHECK_MB_S * 1024 * 1024
+        xenfree_ring = [0] * check_period
+
         while True:
-            self.log.debug('niter={:2d}/{:2d}'.format(niter, MAX_TRIES))
+            self.log.debug('niter={:2d}'.format(niter))
             self.refresh_memactual()
             xenfree = self.get_free_xen_memory()
             self.log.info('xenfree={!r}'.format(xenfree))
             if xenfree >= memsize + self.XEN_FREE_MEM_MIN:
                 self.inhibit_balloon_up()
                 return True
+            ring_slot = niter % check_period
+            if niter >= check_period and xenfree < xenfree_ring[ring_slot] + check_delta:
+                return False
+            xenfree_ring[ring_slot] = xenfree
             if prev_memory_actual is not None:
                 for i in prev_memory_actual.keys():
                     if prev_memory_actual[i] == self.domdict[i].memory_actual:
@@ -159,7 +169,7 @@ class SystemState(object):
                         self.log.info('domain {} stuck at {}'.format(i, self.domdict[i].memory_actual))
             memset_reqs = qmemman_algo.balloon(memsize + self.XEN_FREE_MEM_LEFT - xenfree, self.domdict)
             self.log.info('memset_reqs={!r}'.format(memset_reqs))
-            if niter > MAX_TRIES or len(memset_reqs) == 0:
+            if len(memset_reqs) == 0:
                 return False
             prev_memory_actual = {}
             for i in memset_reqs:

From 2d8a8094ed1724e4727da946c06d240a44782dbd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 9 Oct 2015 19:29:15 +0200
Subject: [PATCH 073/225] qmemman: add some comments about new condition for
 failing request

QubesOS/qubes-issues#1136
---
 qmemman/qmemman.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/qmemman/qmemman.py b/qmemman/qmemman.py
index 3de81d75..9bfdd52a 100755
--- a/qmemman/qmemman.py
+++ b/qmemman/qmemman.py
@@ -145,8 +145,13 @@ class SystemState(object):
         for i in self.domdict.keys():
             self.domdict[i].no_progress = False
 
+        #: number of loop iterations for CHECK_PERIOD_S seconds
         check_period = max(1, int((CHECK_PERIOD_S + 0.0) / self.BALOON_DELAY))
+        #: number of free memory bytes expected to get during CHECK_PERIOD_S
+        #: seconds
         check_delta = CHECK_PERIOD_S * CHECK_MB_S * 1024 * 1024
+        #: helper array for holding free memory size, CHECK_PERIOD_S seconds
+        #: ago, at every loop iteration
         xenfree_ring = [0] * check_period
 
         while True:
@@ -157,6 +162,8 @@ class SystemState(object):
             if xenfree >= memsize + self.XEN_FREE_MEM_MIN:
                 self.inhibit_balloon_up()
                 return True
+            # fail the request if over past CHECK_PERIOD_S seconds,
+            # we got less than CHECK_MB_S MB/s on average
             ring_slot = niter % check_period
             if niter >= check_period and xenfree < xenfree_ring[ring_slot] + check_delta:
                 return False

From 78c3f40f897f9f11aa7e537ec1fcb62890c3a8cd Mon Sep 17 00:00:00 2001
From: Ivan Konov <plushambush@gmail.com>
Date: Fri, 9 Oct 2015 22:52:28 +0300
Subject: [PATCH 074/225] Don't send screen layout to the VM if we haven't
 received it from xrandr because this breaks proper initial screen layout set
 by gui-agent Fixes QubesOS/qubes-issues#1305 QubesOS/qubes-issues#998

---
 core-modules/000QubesVm.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index 1cc8d212..4001c33c 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -1678,7 +1678,10 @@ class QubesVm(object):
             if verbose:
                 print >> sys.stderr, "--> Sending monitor layout..."
             monitor_layout = qubes.monitorlayoutnotify.get_monitor_layout()
-            qubes.monitorlayoutnotify.notify_vm(self, monitor_layout)
+            # Notify VM only if we've got a monitor_layout which is not empty
+            # or else we break proper VM resolution set by gui-agent
+            if len(monitor_layout) > 0:
+                qubes.monitorlayoutnotify.notify_vm(self, monitor_layout)
         except ImportError as e:
             print >>sys.stderr, "ERROR: %s" % e
 

From 540f4f7294ee5b87a909c210d458621a90d51a26 Mon Sep 17 00:00:00 2001
From: qubesuser <qubesuser@users.noreply.github.com>
Date: Fri, 9 Oct 2015 19:39:40 +0200
Subject: [PATCH 075/225] Properly account for Xen memory overhead to fix large
 VMs

---
 core-modules/000QubesVm.py          | 32 +++++++++++++++++++----------
 core-modules/01QubesDisposableVm.py | 12 +----------
 qmemman/qmemman.py                  |  8 ++++++--
 3 files changed, 28 insertions(+), 24 deletions(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index b9b32bdd..1e087196 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -1738,6 +1738,26 @@ class QubesVm(object):
             self.force_shutdown()
             raise OSError("ERROR: Cannot execute qubesdb-daemon!")
 
+    def request_memory(self, mem_required = None):
+        # Overhead of per-VM/per-vcpu Xen structures, taken from OpenStack nova/virt/xenapi/driver.py
+        # see https://wiki.openstack.org/wiki/XenServer/Overhead
+        # add an extra MB because Nova rounds up to MBs
+        MEM_OVERHEAD_BASE = (3 + 1) * 1024 * 1024
+        MEM_OVERHEAD_PER_VCPU = 3 * 1024 * 1024 / 2
+        if mem_required is None:
+            mem_required = int(self.memory) * 1024 * 1024
+        if qmemman_present:
+            qmemman_client = QMemmanClient()
+            try:
+                mem_required_with_overhead = mem_required + MEM_OVERHEAD_BASE + self.vcpus * MEM_OVERHEAD_PER_VCPU
+                got_memory = qmemman_client.request_memory(mem_required_with_overhead)
+            except IOError as e:
+                raise IOError("ERROR: Failed to connect to qmemman: %s" % str(e))
+            if not got_memory:
+                qmemman_client.close()
+                raise MemoryError ("ERROR: insufficient memory to start VM '%s'" % self.name)
+            return qmemman_client
+
     def start(self, verbose = False, preparing_dvm = False, start_guid = True,
             notify_function = None, mem_required = None):
         self.log.debug('start('
@@ -1765,17 +1785,7 @@ class QubesVm(object):
 
         self._update_libvirt_domain()
 
-        if mem_required is None:
-            mem_required = int(self.memory) * 1024 * 1024
-        if qmemman_present:
-            qmemman_client = QMemmanClient()
-            try:
-                got_memory = qmemman_client.request_memory(mem_required)
-            except IOError as e:
-                raise IOError("ERROR: Failed to connect to qmemman: %s" % str(e))
-            if not got_memory:
-                qmemman_client.close()
-                raise MemoryError ("ERROR: insufficient memory to start VM '%s'" % self.name)
+        qmemman_client = self.request_memory(mem_required)
 
         # Bind pci devices to pciback driver
         for pci in self.pcidevs:
diff --git a/core-modules/01QubesDisposableVm.py b/core-modules/01QubesDisposableVm.py
index 68b4319f..a758dd74 100644
--- a/core-modules/01QubesDisposableVm.py
+++ b/core-modules/01QubesDisposableVm.py
@@ -173,17 +173,7 @@ class QubesDisposableVm(QubesVm):
         # refresh config file
         domain_config = self.create_config_file()
 
-        if qmemman_present:
-            mem_required = int(self.memory) * 1024 * 1024
-            print >>sys.stderr, "time=%s, getting %d memory" % (str(time.time()), mem_required)
-            qmemman_client = QMemmanClient()
-            try:
-                got_memory = qmemman_client.request_memory(mem_required)
-            except IOError as e:
-                raise IOError("ERROR: Failed to connect to qmemman: %s" % str(e))
-            if not got_memory:
-                qmemman_client.close()
-                raise MemoryError ("ERROR: insufficient memory to start VM '%s'" % self.name)
+        qmemman_client = self.request_memory()
 
         # dispvm cannot have PCI devices
         assert (len(self.pcidevs) == 0), "DispVM cannot have PCI devices"
diff --git a/qmemman/qmemman.py b/qmemman/qmemman.py
index 3de81d75..d40c0b0a 100755
--- a/qmemman/qmemman.py
+++ b/qmemman/qmemman.py
@@ -56,7 +56,11 @@ class SystemState(object):
         self.BALOON_DELAY = 0.1
         self.XEN_FREE_MEM_LEFT = 50*1024*1024
         self.XEN_FREE_MEM_MIN = 25*1024*1024
-        self.ALL_PHYS_MEM = self.xc.physinfo()['total_memory']*1024
+        # Overhead of per-page Xen structures, taken from OpenStack nova/virt/xenapi/driver.py
+        # see https://wiki.openstack.org/wiki/XenServer/Overhead
+        # we divide total and free physical memory by this to get "assignable" memory
+        self.MEM_OVERHEAD_FACTOR = 1.0 / 1.00781
+        self.ALL_PHYS_MEM = int(self.xc.physinfo()['total_memory']*1024 * self.MEM_OVERHEAD_FACTOR)
 
     def add_domain(self, id):
         self.log.debug('add_domain(id={!r})'.format(id))
@@ -67,7 +71,7 @@ class SystemState(object):
         self.domdict.pop(id)
 
     def get_free_xen_memory(self):
-        return self.xc.physinfo()['free_memory']*1024
+        return int(self.xc.physinfo()['free_memory']*1024 * self.MEM_OVERHEAD_FACTOR)
 #        hosts = self.xend_session.session.xenapi.host.get_all()
 #        host_record = self.xend_session.session.xenapi.host.get_record(hosts[0])
 #        host_metrics_record = self.xend_session.session.xenapi.host_metrics.get_record(host_record["metrics"])

From 063b436b037f0e7e1725e5bdcca9b2bb1e692fac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 9 Oct 2015 20:22:22 +0200
Subject: [PATCH 076/225] notify-updates: ignore chown error

If running as normal user, chown will fail. This isn't a problem,
because the file is probably already owned by the correct user. The
whole point about this chown is to give access to the file for normal
user, so if the write succeeded, it isn't needed.
---
 qubes-rpc/qubes-notify-updates | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/qubes-rpc/qubes-notify-updates b/qubes-rpc/qubes-notify-updates
index 2dd851b8..88ea6ce0 100755
--- a/qubes-rpc/qubes-notify-updates
+++ b/qubes-rpc/qubes-notify-updates
@@ -61,7 +61,10 @@ def main():
         update_f = open(source_vm.dir_path + '/' + vm_files["updates_stat_file"], "w")
         update_f.write(update_count)
         update_f.close()
-        os.chown(source_vm.dir_path + '/' + vm_files["updates_stat_file"], -1, qubes_gid)
+        try:
+            os.chown(source_vm.dir_path + '/' + vm_files["updates_stat_file"], -1, qubes_gid)
+        except OSError:
+            pass
     elif source_vm.template is not None:
         # Hint about updates availability in template
         # If template is running - it will notify about updates itself
@@ -79,8 +82,11 @@ def main():
             update_f = open(stat_file, "w")
             update_f.write(update_count)
             update_f.close()
-            os.chown(stat_file, -1, qubes_gid)
+            try:
+                os.chown(stat_file, -1, qubes_gid)
+            except OSError:
+                pass
         else:
             print >> sys.stderr, 'Ignoring notification of no updates'
 
-main()
+main()
\ No newline at end of file

From 912d4c144730b361672c7dc7e6bbc0e5c8f4bccf Mon Sep 17 00:00:00 2001
From: HW42 <hw42@ipsumj.de>
Date: Wed, 1 Jul 2015 19:44:46 +0200
Subject: [PATCH 077/225] qubes-random-seed: feed kernel rng with randomness
 from dom0

---
 core-modules/000QubesVm.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index 1e087196..6e9db745 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -23,6 +23,8 @@
 #
 
 import datetime
+import base64
+import hashlib
 import logging
 import lxml.etree
 import os
@@ -1072,6 +1074,8 @@ class QubesVm(object):
 
         self.qdb.write("/qubes-debug-mode", str(int(self.debug)))
 
+        self.provide_random_seed_to_vm()
+
         # TODO: Currently the whole qmemman is quite Xen-specific, so stay with
         # xenstore for it until decided otherwise
         if qmemman_present:
@@ -1082,6 +1086,14 @@ class QubesVm(object):
         for hook in self.hooks_create_qubesdb_entries:
             hook(self)
 
+    def provide_random_seed_to_vm(self):
+        f = open('/dev/urandom', 'r')
+        s = f.read(64)
+        if len(s) != 64:
+            raise IOError("failed to read seed from /dev/urandom")
+        f.close()
+        self.qdb.write("/qubes-random-seed", base64.b64encode(hashlib.sha512(s).digest()))
+
     def _format_net_dev(self, ip, mac, backend):
         template = "    <interface type='ethernet'>\n" \
                    "      <mac address='{mac}'/>\n" \

From 820e5c0e699b298f17f0e19b4977ad2a1165e1b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 10 Oct 2015 02:40:01 +0200
Subject: [PATCH 078/225] Update stubdom memory allocation to 44MB

Since 677a79b "hvm: change default graphics to std vga ('xen')",
stubdomain uses 44MB RAM

QubesOS/qubes-issues#1136
---
 core-modules/01QubesHVm.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core-modules/01QubesHVm.py b/core-modules/01QubesHVm.py
index f7cedc41..8ce214ac 100644
--- a/core-modules/01QubesHVm.py
+++ b/core-modules/01QubesHVm.py
@@ -337,8 +337,8 @@ class QubesHVm(QubesVm):
             raise QubesException("Cannot start the HVM while its template is running")
         try:
             if 'mem_required' not in kwargs:
-                # Reserve 32MB for stubdomain
-                kwargs['mem_required'] = (self.memory + 32) * 1024 * 1024
+                # Reserve 44MB for stubdomain
+                kwargs['mem_required'] = (self.memory + 44) * 1024 * 1024
             return super(QubesHVm, self).start(*args, **kwargs)
         except QubesException as e:
             capabilities = vmm.libvirt_conn.getCapabilities()

From 18edf4946c5b1ad6650c7c92bfda75bf3bd99096 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 10 Oct 2015 03:39:43 +0200
Subject: [PATCH 079/225] dispvm: get rid of obsolete symlinks to dvm.conf

Separate config file is no longer created for DispVM - the configuration
is passed directly to libvirt.

Fixes QubesOS/qubes-issues#1314
---
 dispvm/qfile-daemon-dvm          | 3 +--
 dispvm/startup-dvm.sh            | 2 --
 qvm-tools/qvm-create-default-dvm | 6 +-----
 3 files changed, 2 insertions(+), 9 deletions(-)

diff --git a/dispvm/qfile-daemon-dvm b/dispvm/qfile-daemon-dvm
index 7a59113b..a4ccd8b6 100755
--- a/dispvm/qfile-daemon-dvm
+++ b/dispvm/qfile-daemon-dvm
@@ -34,7 +34,6 @@ from qubes.notify import tray_notify, tray_notify_error, tray_notify_init
 
 
 current_savefile = '/var/run/qubes/current-savefile'
-current_dvm_conf = '/var/run/qubes/current-dvm.conf'
 current_savefile_vmdir = '/var/lib/qubes/dvmdata/vmdir'
 
 
@@ -179,4 +178,4 @@ def main():
         dispvm.force_shutdown()
         qfile.remove_disposable_from_qdb(dispvm.name)
 
-main()
\ No newline at end of file
+main()
diff --git a/dispvm/startup-dvm.sh b/dispvm/startup-dvm.sh
index da0a7887..8f53484c 100755
--- a/dispvm/startup-dvm.sh
+++ b/dispvm/startup-dvm.sh
@@ -6,10 +6,8 @@ printf "\x00\x00\x00\x00" > /var/run/qubes/dispVM.seq
 chown root:qubes /var/run/qubes/dispVM.seq
 chmod 660 /var/run/qubes/dispVM.seq
 DEFAULT=/var/lib/qubes/dvmdata/default-savefile
-DEFAULT_CONFIG=/var/lib/qubes/dvmdata/default-dvm.conf
 # setup DispVM files only when they exists
 if [ -r $DEFAULT ]; then
-    ln -s $DEFAULT_CONFIG /var/run/qubes/current-dvm.conf
     if [ -f /var/lib/qubes/dvmdata/dont-use-shm ] ; then
 	ln -s $DEFAULT /var/run/qubes/current-savefile
     else
diff --git a/qvm-tools/qvm-create-default-dvm b/qvm-tools/qvm-create-default-dvm
index 810755aa..5e90cb79 100755
--- a/qvm-tools/qvm-create-default-dvm
+++ b/qvm-tools/qvm-create-default-dvm
@@ -46,16 +46,12 @@ if ! /usr/lib/qubes/qubes-prepare-saved-domain.sh \
 	exit 1
 fi
 DEFAULT=/var/lib/qubes/dvmdata/default-savefile
-DEFAULTCONF=/var/lib/qubes/dvmdata/default-dvm.conf
 CURRENT=/var/run/qubes/current-savefile
-CURRENTCONF=/var/run/qubes/current-dvm.conf
 SHMDIR=/dev/shm/qubes
 SHMCOPY=$SHMDIR/current-savefile
-rm -f $ROOT $DEFAULT $CURRENT $DEFAULTCONF $CURRENTCONF
+rm -f $ROOT $DEFAULT $CURRENT
 ln -s "/var/lib/qubes/appvms/$DVMTMPL/dvm-savefile" $DEFAULT
 ln -s "/var/lib/qubes/vm-templates/$TEMPLATENAME/root.img" $ROOT
-ln -s $DVMTMPLDIR/dvm.conf $DEFAULTCONF
-ln -s $DVMTMPLDIR/dvm.conf $CURRENTCONF
 if [ -f /var/lib/qubes/dvmdata/dont-use-shm ] ; then
 	ln -s $DEFAULT $CURRENT
 else

From 350e279f4f907583a43592821c1690fbbf0c3078 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 10 Oct 2015 17:30:40 +0200
Subject: [PATCH 080/225] Fix handling /etc/localtime hardlink

If /etc/localtime in dom0 is a hardlink to zoneinfo file (instead of
symlink) and more than one zoneinfo file is hardlinked to this inode,
appVMs will get invalid timezone, e.g. "Europe/Warsaw\x0aPoland".

Reported by @yaqu, fix provided by @yaqu
Fixes QubesOS/qubes-issues#1315
---
 core-modules/000QubesVm.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index 6e9db745..08b4d48e 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -1008,8 +1008,9 @@ class QubesVm(object):
                 return None
             if tz_info.st_nlink > 1:
                 p = subprocess.Popen(['find', '/usr/share/zoneinfo',
-                                       '-inum', str(tz_info.st_ino)],
-                                      stdout=subprocess.PIPE)
+                                      '-inum', str(tz_info.st_ino),
+                                      '-print', '-quit'],
+                                     stdout=subprocess.PIPE)
                 tz_path = p.communicate()[0].strip()
                 return tz_path.replace('/usr/share/zoneinfo/', '')
         return None

From b9fedb622ee0f3f6b5fe5e4510efc1030317b257 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sun, 11 Oct 2015 02:52:51 +0200
Subject: [PATCH 081/225] version 3.1.1

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index fd2a0186..94ff29cc 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.1.0
+3.1.1

From 5c549e1134d1635f6eef1abaac774e0da37d93b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 13 Oct 2015 23:46:59 +0200
Subject: [PATCH 082/225] core: Provide more meaningful error when failed to
 kill old QubesDB instance

This process may be running as root in case of default NetVM. But do not
kill it forcibly (through sudo or so), because it may also be stale
pid file. After all, QubesDB should automatically terminate when its VM
is terminated, so this code is already some error handling.

Fixes QubesOS/qubes-issues#1331
---
 core-modules/000QubesVm.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index 219690fb..fbca1f15 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -1738,7 +1738,15 @@ class QubesVm(object):
         try:
             if os.path.exists(pidfile):
                 old_qubesdb_pid = open(pidfile, 'r').read()
-                os.kill(int(old_qubesdb_pid), signal.SIGTERM)
+                try:
+                    os.kill(int(old_qubesdb_pid), signal.SIGTERM)
+                except OSError:
+                    raise QubesException(
+                        "Failed to kill old QubesDB instance (PID {}). "
+                        "Terminate it manually and retry. "
+                        "If that isn't QubesDB process, "
+                        "remove the pidfile: {}".format(old_qubesdb_pid,
+                                                        pidfile))
                 timeout = 25
                 while os.path.exists(pidfile) and timeout:
                     time.sleep(0.2)

From 75168c8aef9c3f8d50d3690f269db5d22f52d074 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 13 Oct 2015 23:49:32 +0200
Subject: [PATCH 083/225] core: improve handling QubesDB startup errors

Kill the VM when QubesDB initialization fails, in any way, not only
QubesDB daemon itself.
---
 core-modules/000QubesVm.py | 47 ++++++++++++++++++++------------------
 1 file changed, 25 insertions(+), 22 deletions(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index fbca1f15..6c5ba7f5 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -1759,7 +1759,6 @@ class QubesVm(object):
             str(self.xid),
             self.name])
         if retcode != 0:
-            self.force_shutdown()
             raise OSError("ERROR: Cannot execute qubesdb-daemon!")
 
     def request_memory(self, mem_required = None):
@@ -1833,31 +1832,35 @@ class QubesVm(object):
 
         self.libvirt_domain.createWithFlags(libvirt.VIR_DOMAIN_START_PAUSED)
 
-        if verbose:
-            print >> sys.stderr, "--> Starting Qubes DB..."
-        self.start_qubesdb()
+        try:
+            if verbose:
+                print >> sys.stderr, "--> Starting Qubes DB..."
+            self.start_qubesdb()
 
-        xid = self.xid
-        self.log.debug('xid={}'.format(xid))
+            xid = self.xid
+            self.log.debug('xid={}'.format(xid))
 
-        if preparing_dvm:
-            self.services['qubes-dvm'] = True
-        if verbose:
-            print >> sys.stderr, "--> Setting Qubes DB info for the VM..."
-        self.create_qubesdb_entries()
+            if preparing_dvm:
+                self.services['qubes-dvm'] = True
+            if verbose:
+                print >> sys.stderr, "--> Setting Qubes DB info for the VM..."
+            self.create_qubesdb_entries()
 
-        if verbose:
-            print >> sys.stderr, "--> Updating firewall rules..."
-        netvm = self.netvm
-        while netvm is not None:
-            if netvm.is_proxyvm() and netvm.is_running():
-                netvm.write_iptables_qubesdb_entry()
-            netvm = netvm.netvm
+            if verbose:
+                print >> sys.stderr, "--> Updating firewall rules..."
+            netvm = self.netvm
+            while netvm is not None:
+                if netvm.is_proxyvm() and netvm.is_running():
+                    netvm.write_iptables_qubesdb_entry()
+                netvm = netvm.netvm
 
-        # fire hooks
-        for hook in self.hooks_start:
-            hook(self, verbose = verbose, preparing_dvm =  preparing_dvm,
-                    start_guid = start_guid, notify_function = notify_function)
+            # fire hooks
+            for hook in self.hooks_start:
+                hook(self, verbose = verbose, preparing_dvm =  preparing_dvm,
+                     start_guid = start_guid, notify_function = notify_function)
+        except:
+            self.force_shutdown()
+            raise
 
         if verbose:
             print >> sys.stderr, "--> Starting the VM..."

From c2b56306374ac936b614e1a005bd5ad4ae149422 Mon Sep 17 00:00:00 2001
From: Patrick Schleizer <adrelanos@riseup.net>
Date: Wed, 21 Oct 2015 20:57:30 +0000
Subject: [PATCH 084/225] typo

---
 qubes-rpc/qubes-notify-updates | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/qubes-rpc/qubes-notify-updates b/qubes-rpc/qubes-notify-updates
index 88ea6ce0..9ece7c42 100755
--- a/qubes-rpc/qubes-notify-updates
+++ b/qubes-rpc/qubes-notify-updates
@@ -44,7 +44,7 @@ def main():
 
     source_vm = qvm_collection.get_vm_by_name(source)
     if source_vm is None:
-        print >> sys.stderr, 'Domain ' + source + ' does not exists (?!)'
+        print >> sys.stderr, 'Domain ' + source + ' does not exist (?!)'
         exit(1)
 
     os.umask(0002)
@@ -89,4 +89,4 @@ def main():
         else:
             print >> sys.stderr, 'Ignoring notification of no updates'
 
-main()
\ No newline at end of file
+main()

From 3be8bb31fbfb47b054d1a49050d2d53c255f7789 Mon Sep 17 00:00:00 2001
From: Michal Rostecki <michal.rostecki@gmail.com>
Date: Sun, 4 Oct 2015 22:07:50 +0200
Subject: [PATCH 085/225] Support more VM types in qvm-grow-root

QubesOS/qubes-issues#1268
---
 core-modules/001QubesResizableVm.py | 67 +++++++++++++++++++++++++++++
 core-modules/003QubesTemplateVm.py  | 16 ++++---
 core-modules/01QubesAppVm.py        | 10 ++++-
 core-modules/01QubesHVm.py          | 36 +++++-----------
 4 files changed, 97 insertions(+), 32 deletions(-)
 create mode 100644 core-modules/001QubesResizableVm.py

diff --git a/core-modules/001QubesResizableVm.py b/core-modules/001QubesResizableVm.py
new file mode 100644
index 00000000..cf9bcf1e
--- /dev/null
+++ b/core-modules/001QubesResizableVm.py
@@ -0,0 +1,67 @@
+#!/usr/bin/python2
+# -*- encoding: utf8 -*-
+#
+# The Qubes OS Project, http://www.qubes-os.org
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+#
+
+from __future__ import (
+    absolute_import,
+    division,
+    print_function,
+    unicode_literals,
+)
+
+from qubes.qubes import (
+    register_qubes_vm_class,
+    QubesException,
+    QubesVm,
+)
+
+
+class QubesResizableVm(QubesVm):
+
+    def resize_root_img(self, size):
+        if self.template:
+            raise QubesException("Cannot resize root.img of template-based VM"
+                                 ". Resize the root.img of the template "
+                                 "instead.")
+
+        if self.is_running():
+            raise QubesException("Cannot resize root.img of running VM")
+
+        if size < self.get_root_img_sz():
+            raise QubesException(
+                "For your own safety shringing of root.img is disabled. If "
+                "you really know what you are doing, use 'truncate' manually.")
+
+        f_root = open(self.root_img, "a+b")
+        f_root.truncate(size)
+        f_root.close()
+
+
+class QubesResizableVmWithResize2fs(QubesResizableVm):
+
+    def resize_root_img(self, size):
+        super(QubesResizableVmWithResize2fs, self).resize_root_img(size)
+        self.start()
+        self.run("sudo resize2fs /dev/mapper/dmroot")
+        self.shutdown()
+
+
+register_qubes_vm_class(QubesResizableVm)
+register_qubes_vm_class(QubesResizableVmWithResize2fs)
diff --git a/core-modules/003QubesTemplateVm.py b/core-modules/003QubesTemplateVm.py
index e5c992ce..4d9a6c79 100644
--- a/core-modules/003QubesTemplateVm.py
+++ b/core-modules/003QubesTemplateVm.py
@@ -23,14 +23,20 @@
 #
 
 import os
-import subprocess
 import sys
 
-from qubes.qubes import QubesVm,register_qubes_vm_class,dry_run
-from qubes.qubes import QubesVmCollection,QubesException,QubesVmLabels
-from qubes.qubes import defaults,system_path,vm_files,vmm
+from qubes.qubes import (
+    defaults,
+    dry_run,
+    register_qubes_vm_class,
+    system_path,
+    vmm,
+    QubesResizableVmWithResize2fs,
+    QubesVmCollection,
+)
 
-class QubesTemplateVm(QubesVm):
+
+class QubesTemplateVm(QubesResizableVmWithResize2fs):
     """
     A class that represents an TemplateVM. A child of QubesVm.
     """
diff --git a/core-modules/01QubesAppVm.py b/core-modules/01QubesAppVm.py
index dd119a0e..9a46fb47 100644
--- a/core-modules/01QubesAppVm.py
+++ b/core-modules/01QubesAppVm.py
@@ -24,9 +24,15 @@
 
 import os.path
 
-from qubes.qubes import QubesVm,QubesVmLabel,register_qubes_vm_class,system_path
+from qubes.qubes import (
+    register_qubes_vm_class,
+    system_path,
+    QubesResizableVmWithResize2fs,
+    QubesVmLabel,
+)
 
-class QubesAppVm(QubesVm):
+
+class QubesAppVm(QubesResizableVmWithResize2fs):
     """
     A class that represents an AppVM. A child of QubesVm.
     """
diff --git a/core-modules/01QubesHVm.py b/core-modules/01QubesHVm.py
index 8ce214ac..c3b692a8 100644
--- a/core-modules/01QubesHVm.py
+++ b/core-modules/01QubesHVm.py
@@ -26,16 +26,20 @@ import os
 import os.path
 import signal
 import subprocess
-import stat
 import sys
-import re
 import shutil
-import stat
 from xml.etree import ElementTree
 
-from qubes.qubes import QubesVm,register_qubes_vm_class,vmm,dry_run
-from qubes.qubes import system_path,defaults
-from qubes.qubes import QubesException
+from qubes.qubes import (
+    dry_run,
+    defaults,
+    register_qubes_vm_class,
+    system_path,
+    vmm,
+    QubesException,
+    QubesResizableVm,
+)
+
 
 system_path["config_template_hvm"] = '/usr/share/qubes/vm-template-hvm.xml'
 
@@ -44,7 +48,7 @@ defaults["hvm_private_img_size"] = 2*1024*1024*1024
 defaults["hvm_memory"] = 512
 
 
-class QubesHVm(QubesVm):
+class QubesHVm(QubesResizableVm):
     """
     A class that represents an HVM. A child of QubesVm.
     """
@@ -235,24 +239,6 @@ class QubesHVm(QubesVm):
 
         self.storage.resize_private_img(size)
 
-    def resize_root_img(self, size):
-        if self.template:
-            raise QubesException("Cannot resize root.img of template-based VM"
-                                 ". Resize the root.img of the template "
-                                 "instead.")
-
-        if self.is_running():
-            raise QubesException("Cannot resize root.img of running HVM")
-
-        if size < self.get_root_img_sz():
-            raise QubesException(
-                "For your own safety shringing of root.img is disabled. If "
-                "you really know what you are doing, use 'truncate' manually.")
-
-        f_root = open (self.root_img, "a+b")
-        f_root.truncate (size)
-        f_root.close ()
-
     def get_config_params(self):
 
         params = super(QubesHVm, self).get_config_params()

From e475e291a9cf05d453855d1d15dcc5c8cebaf6c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sun, 25 Oct 2015 15:23:12 +0100
Subject: [PATCH 086/225] tests: more qrexec tests, this time for deadlocks

Tests for recently discovered deadlocks on write(2).

QubesOS/qubes-issues#1347
---
 tests/vm_qrexec_gui.py | 132 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 132 insertions(+)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index dd2ea9d1..c1f2ed11 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -318,6 +318,138 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
         (stdout, stderr) = p.communicate()
         self.assertEqual(stdout, "3\n")
 
+    def test_070_qrexec_vm_simultaneous_write(self):
+        """Test for simultaneous write in VM(src)->VM(dst) connection
+
+            This is regression test for #1347
+
+            Check for deadlock when initially both sides writes a lot of data
+            (and not read anything). When one side starts reading, it should
+            get the data and the remote side should be possible to write then more.
+            There was a bug where remote side was waiting on write(2) and not
+            handling anything else.
+        """
+        result = multiprocessing.Value('i', -1)
+
+        def run(self):
+            p = self.testvm1.run(
+                "/usr/lib/qubes/qrexec-client-vm %s test.write "
+                "/bin/sh -c '"
+                # first write a lot of data to fill all the buffers
+                "dd if=/dev/zero bs=993 count=10000 iflag=fullblock & "
+                # then after some time start reading
+                "sleep 1; "
+                "dd of=/dev/null bs=993 count=10000 iflag=fullblock; "
+                "wait"
+                "'" % self.testvm2.name, passio_popen=True)
+            p.communicate()
+            result.value = p.returncode
+
+        self.testvm1.start()
+        self.testvm2.start()
+        p = self.testvm2.run("cat > /etc/qubes-rpc/test.write", user="root",
+                             passio_popen=True)
+        # first write a lot of data
+        p.stdin.write("dd if=/dev/zero bs=993 count=10000 iflag=fullblock\n")
+        # and only then read something
+        p.stdin.write("dd of=/dev/null bs=993 count=10000 iflag=fullblock\n")
+        p.stdin.close()
+        p.wait()
+        policy = open("/etc/qubes-rpc/policy/test.write", "w")
+        policy.write("%s %s allow" % (self.testvm1.name, self.testvm2.name))
+        policy.close()
+        self.addCleanup(os.unlink, "/etc/qubes-rpc/policy/test.write")
+
+        t = multiprocessing.Process(target=run, args=(self,))
+        t.start()
+        t.join(timeout=10)
+        if t.is_alive():
+            t.terminate()
+            self.fail("Timeout, probably deadlock")
+        self.assertEqual(result.value, 0, "Service call failed")
+
+    def test_071_qrexec_dom0_simultaneous_write(self):
+        """Test for simultaneous write in dom0(src)->VM(dst) connection
+
+            Similar to test_070_qrexec_vm_simultaneous_write, but with dom0
+            as a source.
+        """
+        result = multiprocessing.Value('i', -1)
+
+        def run(self):
+            result.value = self.testvm2.run_service(
+                "test.write", localcmd="/bin/sh -c '"
+                # first write a lot of data to fill all the buffers
+                "dd if=/dev/zero bs=993 count=10000 iflag=fullblock & "
+                # then after some time start reading
+                "sleep 1; "
+                "dd of=/dev/null bs=993 count=10000 iflag=fullblock; "
+                "wait"
+                "'")
+
+        self.testvm2.start()
+        p = self.testvm2.run("cat > /etc/qubes-rpc/test.write", user="root",
+                             passio_popen=True)
+        # first write a lot of data
+        p.stdin.write("dd if=/dev/zero bs=993 count=10000 iflag=fullblock\n")
+        # and only then read something
+        p.stdin.write("dd of=/dev/null bs=993 count=10000 iflag=fullblock\n")
+        p.stdin.close()
+        p.wait()
+        policy = open("/etc/qubes-rpc/policy/test.write", "w")
+        policy.write("%s %s allow" % (self.testvm1.name, self.testvm2.name))
+        policy.close()
+        self.addCleanup(os.unlink, "/etc/qubes-rpc/policy/test.write")
+
+        t = multiprocessing.Process(target=run, args=(self,))
+        t.start()
+        t.join(timeout=10)
+        if t.is_alive():
+            t.terminate()
+            self.fail("Timeout, probably deadlock")
+        self.assertEqual(result.value, 0, "Service call failed")
+
+    def test_072_qrexec_to_dom0_simultaneous_write(self):
+        """Test for simultaneous write in dom0(src)<-VM(dst) connection
+
+            Similar to test_071_qrexec_dom0_simultaneous_write, but with dom0
+            as a "hanging" side.
+        """
+        result = multiprocessing.Value('i', -1)
+
+        def run(self):
+            result.value = self.testvm2.run_service(
+                "test.write", localcmd="/bin/sh -c '"
+                # first write a lot of data to fill all the buffers
+                "dd if=/dev/zero bs=993 count=10000 iflag=fullblock "
+                # then, only when all written, read something
+                "dd of=/dev/null bs=993 count=10000 iflag=fullblock; "
+                "'")
+
+        self.testvm2.start()
+        p = self.testvm2.run("cat > /etc/qubes-rpc/test.write", user="root",
+                             passio_popen=True)
+        # first write a lot of data
+        p.stdin.write("dd if=/dev/zero bs=993 count=10000 iflag=fullblock &\n")
+        # and only then read something
+        p.stdin.write("dd of=/dev/null bs=993 count=10000 iflag=fullblock\n")
+        p.stdin.write("sleep 1; \n")
+        p.stdin.write("wait\n")
+        p.stdin.close()
+        p.wait()
+        policy = open("/etc/qubes-rpc/policy/test.write", "w")
+        policy.write("%s %s allow" % (self.testvm1.name, self.testvm2.name))
+        policy.close()
+        self.addCleanup(os.unlink, "/etc/qubes-rpc/policy/test.write")
+
+        t = multiprocessing.Process(target=run, args=(self,))
+        t.start()
+        t.join(timeout=10)
+        if t.is_alive():
+            t.terminate()
+            self.fail("Timeout, probably deadlock")
+        self.assertEqual(result.value, 0, "Service call failed")
+
     @unittest.skipUnless(spawn.find_executable('xdotool'),
                          "xdotool not installed")
     def test_100_qrexec_filecopy(self):

From 1c0ade6166c0341934e11d665b4eba1e3e7f82de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 27 Oct 2015 21:31:59 +0100
Subject: [PATCH 087/225] Fix race condition in QubesVm.get_cputime/get_mem

When domain is destroyed after isActive check, but before getting
actuall info, it would fail. So handle this situation.
---
 core-modules/000QubesVm.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index 6c5ba7f5..8cf8a661 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -760,6 +760,9 @@ class QubesVm(object):
         except libvirt.libvirtError as e:
             if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN:
                 return 0
+                # libxl_domain_info failed - domain no longer exists
+            elif e.get_error_code() == libvirt.VIR_INTERNAL_ERROR:
+                return 0
             else:
                 raise
 
@@ -774,6 +777,9 @@ class QubesVm(object):
         except libvirt.libvirtError as e:
             if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN:
                 return 0
+                # libxl_domain_info failed - domain no longer exists
+            elif e.get_error_code() == libvirt.VIR_INTERNAL_ERROR:
+                return 0
             else:
                 raise
 

From de295136ce772095f7a39bfa403223c92cc6f1f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 27 Oct 2015 21:47:01 +0100
Subject: [PATCH 088/225] Fix "utils/QubesWatch: use timers to retry QubesDB
 watch registration"

---
 core/qubesutils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/qubesutils.py b/core/qubesutils.py
index feaf1407..ee669d7c 100644
--- a/core/qubesutils.py
+++ b/core/qubesutils.py
@@ -822,7 +822,7 @@ class QubesWatch(object):
             self._qdb_handler, name)
 
     def _retry_register_watches(self, timer, libvirt_domain):
-        libvirt.virRemoveTimeout(timer)
+        libvirt.virEventRemoveTimeout(timer)
         self._register_watches(libvirt_domain)
 
     def _unregister_watches(self, libvirt_domain):

From 9cb62f8f4d0071e6317602eae5da63627357a337 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 27 Oct 2015 23:16:10 +0100
Subject: [PATCH 089/225] tests: VM updates (direct and through updates proxy)

---
 tests/network.py | 314 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 314 insertions(+)

diff --git a/tests/network.py b/tests/network.py
index aee14b10..fe202b9c 100644
--- a/tests/network.py
+++ b/tests/network.py
@@ -326,6 +326,315 @@ class VmNetworkingMixin(qubes.tests.SystemTestsMixin):
         self.assertNotEqual(self.run_cmd(self.testvm1, self.ping_ip), 0,
                          "Spoofed ping should be blocked")
 
+class VmUpdatesMixin(qubes.tests.SystemTestsMixin):
+    """
+    Tests for VM updates
+    """
+
+    # made this way to work also when no package build tools are installed
+    """
+    $ cat test-pkg.spec:
+    Name:		test-pkg
+    Version:	1.0
+    Release:	1%{?dist}
+    Summary:	Test package
+
+    Group:		System
+    License:	GPL
+    URL:		http://example.com/
+
+    %description
+    Test package
+
+    %files
+
+    %changelog
+    $ rpmbuild -bb test-pkg.spec
+    $ cat test-pkg-1.0-1.fc21.x86_64.rpm | gzip | base64
+    """
+    RPM_PACKAGE_GZIP_BASE64 = (
+        "H4sIAPzRLlYAA+2Y728URRjHn7ueUCkERKJVJDnTxLSxs7293o8WOER6ljYYrtKCLUSa3"
+        "bnZ64bd22VmTq8nr4wJbwxvjNHIG0x8oTHGGCHB8AcYE1/0lS80GgmQFCJU3wgB4ZjdfZ"
+        "q2xDe8NNlvMjfzmeeZH7tPbl98b35169cOUEpIJiTxT9SIrmVUs2hWh8dUAp54dOrM14s"
+        "JHK4D2DKl+j2qrVfjsuq3qEWbohjuAB2Lqk+p1o/8Z5QPmSi/YwnjezH+F8bLQZjqllW0"
+        "hvODRmFIL5hFk9JMXi/mi5ZuDleNwSEzP5wtmLnouNQnm3/6fndz7FLt9M/Hruj37gav4"
+        "tTjPnasWLFixYoVK1asWLFixYoV63+p0KNot9vnIPQc1vgYOwCSgXfxCoS+QzKHOVXVOj"
+        "Fn2ccIfI0k8nXkLuQbyJthxed4UrVnkG8i9yDfgsj3yCAv4foc8t+w1hf5B+Nl5Du43xj"
+        "yvxivIN9HpsgPkO2IU9uQfeRn8Xk/iJ4x1Y3nfxH1qecwfhH5+YgT25F7o/0SRdxvOppP"
+        "7MX9ZjB/DNnE/OOYX404uRGZIT+FbCFvQ3aQ8f0+/WF0XjJ8nyOw7H+BrmUA/a8pNZf2D"
+        "XrCqLG1cERbWHI8ajhznpBY9P0Tr8PkvJDMhTkp/Z0DA6xpuL7DNOq5A+DY9UYTmkOF2U"
+        "IO/sNt0wSnGvfdlZssD3rVIlLI9UUX37C6qXzHNntHPNfnTAhWHbUddtBwmegDjAUzZbu"
+        "m9lqZmzDmHc8Ik8WY8Tab4Myym4+Gx8V0qw8GtYyWIzrktEJwV9UHv3ktG471rAqHTmFQ"
+        "685V5uGqIalk06SWJr7tszR503Ac9cs493jJ8rhrSCIYbXBbzqt5v5+UZ0crh6bGR2dmJ"
+        "yuHD428VlLLLdakzJe2VxcKhFSFID73JKPS40RI7tXVCcQ3uOGWhPCJ2bAspiJ2i5Vy6n"
+        "jOqMerpEYpEe/Yks4xkU4Tt6BirmzUWanG6ozbFKhve9BsQRaLRTirzqk7hgUktXojKnf"
+        "n8jeg3X4QepP3i63po6oml+9t/CwJLya2Bn/ei6f7/4B3Ycdb0L3pt5Q5mNz16rWJ9fLk"
+        "vvOff/nxS7//8O2P2gvt7nDDnoV9L1du9N4+ucjl9u/8+a7dC5Nnvjlv9Ox5r+v9Cy0NE"
+        "m+c6rv60S/dZw98Gn6MNswcfQiWUvg3wBUAAA=="
+    )
+
+    """
+    Minimal package generated by running dh_make on empty directory
+    Then cat test-pkg_1.0-1_amd64.deb | gzip | base64
+    """
+    DEB_PACKAGE_GZIP_BASE64 = (
+        "H4sIACTXLlYAA1O0SSxKzrDjSklNykzM003KzEssqlRQUDA0MTG1NDQwNDVTUDBQAAEIa"
+        "WhgYGZioqBgogADCVxGegZcyfl5JUX5OXoliUV66VVE6DcwheuX7+ZgAAEW5rdXHb0PG4"
+        "iwf5j3WfMT6zWzzMuZgoE3jjYraNzbbFKWGms0SaRw/r2SV23WZ4IdP8preM4yqf0jt95"
+        "3c8qnacfNxJUkf9/w+/3X9ph2GEdgQdixrz/niHKKTnYXizf4oSC7tHOz2Zzq+/6vn8/7"
+        "ezQ7c1tmi7xZ3SGJ4yzhT2dcr7V+W3zM5ZPu/56PSv4Zdok+7Yv/V/6buWaKVlFkkV58S"
+        "N3GmLgnqzRmeZ3V3ymmurS5fGa85/LNx1bpZMin3S6dvXKqydp3ubP1vmyarJZb/qSh62"
+        "C8oIdxqm/BtvkGDza+On/Vfv2py7/0LV7VH+qR6a+bkKUbHXt5/SG187d+nps1a5PJfMO"
+        "i11dWcUe1HjwaW3Q5RHXn9LmcHy+tW9YcKf0768XVB1t3R0bKrzs5t9P+6r7rZ99svH10"
+        "+Q6F/o8tf1fO/32y+fWa14eifd+WxUy0jcxYH7N9/tUvmnUZL74pW32qLeuRU+ZwYGASa"
+        "GBgUWBgxM90ayy3VdmykkGDgYErJbEkERydFVWQmCMQo8aWZvAY/WteFRHFwMCYqXTPjI"
+        "lBkVEMGLsl+k8XP1D/z+gXyyDOvUemlnHqAVkvu0rRQ2fUFodkN3mtU9uwhqk8V+TqPEE"
+        "Nc7fzoQ4n71lqRs/7kbbT0+qOZuKH4r8mjzsc1k/YkCHN8Pjg48fbpE+teHa96LNcfu0V"
+        "5n2/Z2xa2KDvaCOx8cqBFxc514uZ3TmadXS+6cpzU7wSzq5SWfapJOD9n6wLXSwtlgxZh"
+        "xITzWW7buhx/bb291RcVlEfeC9K5hlrqunSzIMSZT7/Nqgc/qMvMNW227WI8ezB8mVuZh"
+        "0hERJSvysfburr4Dx0I9BW57UwR4+e1gxu49PcEt8sbK18Xpvt//Hj5UYm+Zc25q+T4xl"
+        "rJvxfVnh80oadq57OZxPaU1bbztv1yF365W4t45Yr+XrFzov237GVY1Zgf7NvE4+W2SuR"
+        "lQtLauR1TQ/mbOiIONYya6tU1jPGpWfk/i1+ttiXe3ZO14n0YOWggndznjGlGLyfVbBC6"
+        "MRP5aMM7aCco/s7sZqB8RlTQwADw8rnuT/sDHi7mUASjJFRAAbWwNLiAwAA"
+    )
+
+    def run_cmd(self, vm, cmd, user="root"):
+        p = vm.run(cmd, user=user, passio_popen=True, ignore_stderr=True)
+        p.stdin.close()
+        p.stdout.read()
+        return p.wait()
+
+    def setUp(self):
+        super(VmUpdatesMixin, self).setUp()
+
+        self.update_cmd = None
+        if self.template.count("debian"):
+            self.update_cmd = "set -o pipefail; apt-get update 2>&1 | " \
+                              "{ ! grep '^W:\|^E:'; }"
+            self.install_cmd = "apt-get install -y {}"
+            self.install_test_cmd = "dpkg -l {}"
+            self.exit_code_ok = [0]
+        elif self.template.count("fedora"):
+            cmd = "yum"
+            try:
+                # assume template name in form "fedora-XX-suffix"
+                if int(self.template.split("-")[1]) > 21:
+                    cmd = "dnf"
+            except ValueError:
+                pass
+            self.update_cmd = "{cmd} clean all; {cmd} check-update".format(
+                cmd=cmd)
+            self.install_cmd = cmd + " install -y {}"
+            self.install_test_cmd = "rpm -q {}"
+            self.exit_code_ok = [0, 100]
+        else:
+            self.skipTest("Template {} not supported by this test".format(
+                self.template))
+
+        self.testvm1 = self.qc.add_new_vm(
+            "QubesAppVm",
+            name=self.make_vm_name('vm1'),
+            template=self.qc.get_vm_by_name(self.template))
+        self.testvm1.create_on_disk(verbose=False)
+
+    def test_000_simple_update(self):
+        self.save_and_reload_db()
+        self.qc.unlock_db()
+        # reload the VM to have all the properties properly set (especially
+        # default netvm)
+        self.testvm1 = self.qc[self.testvm1.qid]
+        self.testvm1.start()
+        p = self.testvm1.run(self.update_cmd, wait=True, user="root",
+                             passio_popen=True, passio_stderr=True)
+        (stdout, stderr) = p.communicate()
+        self.assertIn(p.wait(), self.exit_code_ok,
+                      "{}: {}\n{}".format(self.update_cmd, stdout, stderr)
+                      )
+
+    def create_repo_apt(self):
+        pkg_file_name = "test-pkg_1.0-1_amd64.deb"
+        p = self.netvm_repo.run("mkdir /tmp/apt-repo && cd /tmp/apt-repo &&"
+                                "base64 -d | zcat > {}".format(pkg_file_name),
+                                passio_popen=True)
+        p.stdin.write(self.DEB_PACKAGE_GZIP_BASE64)
+        p.stdin.close()
+        if p.wait() != 0:
+            raise RuntimeError("Failed to write {}".format(pkg_file_name))
+        # do not assume dpkg-scanpackage installed
+        packages_path = "dists/test/main/binary-amd64/Packages"
+        p = self.netvm_repo.run(
+            "mkdir -p /tmp/apt-repo/dists/test/main/binary-amd64 && "
+            "cd /tmp/apt-repo && "
+            "cat > {packages} && "
+            "echo MD5sum: $(openssl md5 -r {pkg} | cut -f 1 -d ' ')"
+            " >> {packages} && "
+            "echo SHA1: $(openssl sha1 -r {pkg} | cut -f 1 -d ' ')"
+            " >> {packages} && "
+            "echo SHA256: $(openssl sha256 -r {pkg} | cut -f 1 -d ' ')"
+            " >> {packages} && "
+            "gzip < {packages} > {packages}.gz".format(pkg=pkg_file_name,
+                                                       packages=packages_path),
+            passio_popen=True, passio_stderr=True)
+        p.stdin.write(
+            "Package: test-pkg\n"
+            "Version: 1.0-1\n"
+            "Architecture: amd64\n"
+            "Maintainer: unknown <user@host>\n"
+            "Installed-Size: 25\n"
+            "Filename: {pkg}\n"
+            "Size: 994\n"
+            "Section: unknown\n"
+            "Priority: optional\n"
+            "Description: Test package\n".format(pkg=pkg_file_name)
+        )
+        p.stdin.close()
+        if p.wait() != 0:
+            raise RuntimeError("Failed to write Packages file: {}".format(
+                p.stderr.read()))
+
+        p = self.netvm_repo.run(
+            "mkdir -p /tmp/apt-repo/dists/test && "
+            "cd /tmp/apt-repo/dists/test && "
+            "cat > Release <<EOF && "
+            "echo '' $(sha1sum {p} | cut -f 1 -d ' ') $(stat -c %s {p}) {p}"
+            " >> Release && "
+            "echo '' $(sha1sum {z} | cut -f 1 -d ' ') $(stat -c %s {z}) {z}"
+            " >> Release"
+            .format(p="main/binary-amd64/Packages",
+                    z="main/binary-amd64/Packages.gz"),
+            passio_popen=True, passio_stderr=True
+        )
+        p.stdin.write(
+            "Label: Test repo\n"
+            "Suite: test\n"
+            "Codename: test\n"
+            "Date: Tue, 27 Oct 2015 03:22:09 +0100\n"
+            "Architectures: amd64\n"
+            "Components: main\n"
+            "SHA1:\n"
+            "EOF\n"
+        )
+        p.stdin.close()
+        if p.wait() != 0:
+            raise RuntimeError("Failed to write Release file: {}".format(
+                p.stderr.read()))
+
+    def create_repo_yum(self):
+        pkg_file_name = "test-pkg-1.0-1.fc21.x86_64.rpm"
+        p = self.netvm_repo.run("mkdir /tmp/yum-repo && cd /tmp/yum-repo &&"
+                                "base64 -d | zcat > {}".format(pkg_file_name),
+                                passio_popen=True, passio_stderr=True)
+        p.stdin.write(self.RPM_PACKAGE_GZIP_BASE64)
+        p.stdin.close()
+        if p.wait() != 0:
+            raise RuntimeError("Failed to write {}: {}".format(pkg_file_name,
+                                                               p.stderr.read()))
+
+        # createrepo is installed by default in Fedora template
+        p = self.netvm_repo.run("createrepo /tmp/yum-repo",
+                                passio_popen=True,
+                                passio_stderr=True)
+        if p.wait() != 0:
+            raise RuntimeError("Failed to create yum metadata: {}".format(
+                p.stderr.read()))
+
+    def create_repo_and_serve(self):
+        if self.template.count("debian") or self.template.count("whonix"):
+            self.create_repo_apt()
+            self.netvm_repo.run("cd /tmp/apt-repo &&"
+                                "python -m SimpleHTTPServer 8080")
+        elif self.template.count("fedora"):
+            self.create_repo_yum()
+            self.netvm_repo.run("cd /tmp/yum-repo &&"
+                                "python -m SimpleHTTPServer 8080")
+        else:
+            # not reachable...
+            self.skipTest("Template {} not supported by this test".format(
+                self.template))
+
+    def configure_test_repo(self):
+        """
+        Configure test repository in test-vm and disable rest of them.
+        The critical part is to use "localhost" - this will work only when
+        accessed through update proxy and this is exactly what we want to
+        test here.
+        """
+
+        if self.template.count("debian") or self.template.count("whonix"):
+            self.testvm1.run(
+                "rm -f /etc/apt/sources.list.d/* &&"
+                "echo 'deb [trusted=yes] http://localhost:8080 test main' "
+                "> /etc/apt/sources.list",
+                user="root")
+        elif self.template.count("fedora"):
+            self.testvm1.run(
+                "rm -f /etc/yum.repos.d/*.repo &&"
+                "echo '[test]' > /etc/yum.repos.d/test.repo &&"
+                "echo 'name=Test repo' >> /etc/yum.repos.d/test.repo &&"
+                "echo 'gpgcheck=0' >> /etc/yum.repos.d/test.repo &&"
+                "echo 'baseurl=http://localhost:8080/'"
+                " >> /etc/yum.repos.d/test.repo",
+                user="root"
+            )
+        else:
+            # not reachable...
+            self.skipTest("Template {} not supported by this test".format(
+                self.template))
+
+    def test_010_update_via_proxy(self):
+        """
+        Test both whether updates proxy works and whether is actually used by the VM
+        """
+        if self.template.count("minimal"):
+            self.skipTest("Template {} not supported by this test".format(
+                self.template))
+
+        self.netvm_repo = self.qc.add_new_vm(
+            "QubesNetVm",
+            name=self.make_vm_name('net'),
+            template=self.qc.get_vm_by_name(self.template))
+        self.netvm_repo.create_on_disk(verbose=False)
+        self.testvm1.netvm = self.netvm_repo
+        # NetVM should have qubes-updates-proxy enabled by default
+        #self.netvm_repo.services['qubes-updates-proxy'] = True
+        # TODO: consider also adding a test for the template itself
+        self.testvm1.services['updates-proxy-setup'] = True
+        self.qc.save()
+        self.qc.unlock_db()
+
+        # Setup test repo
+        self.netvm_repo.start()
+        self.create_repo_and_serve()
+
+        # Configure local repo
+        self.testvm1.start()
+        self.configure_test_repo()
+
+        # update repository metadata
+        p = self.testvm1.run(self.update_cmd, wait=True, user="root",
+                             passio_popen=True, passio_stderr=True)
+        (stdout, stderr) = p.communicate()
+        self.assertIn(p.wait(), self.exit_code_ok,
+                      "{}: {}\n{}".format(self.update_cmd, stdout, stderr)
+                      )
+
+        # install test package
+        p = self.testvm1.run(self.install_cmd.format('test-pkg'),
+                             wait=True, user="root",
+                             passio_popen=True, passio_stderr=True)
+        (stdout, stderr) = p.communicate()
+        self.assertIn(p.wait(), self.exit_code_ok,
+                      "{}: {}\n{}".format(self.update_cmd, stdout, stderr)
+                      )
+
+        # verify if it was really installed
+        p = self.testvm1.run(self.install_test_cmd.format('test-pkg'),
+                             wait=True, user="root",
+                             passio_popen=True, passio_stderr=True)
+        (stdout, stderr) = p.communicate()
+        self.assertIn(p.wait(), self.exit_code_ok,
+                      "{}: {}\n{}".format(self.update_cmd, stdout, stderr)
+                      )
 
 def load_tests(loader, tests, pattern):
     try:
@@ -343,4 +652,9 @@ def load_tests(loader, tests, pattern):
                 'VmNetworking_' + template,
                 (VmNetworkingMixin, qubes.tests.QubesTestCase),
                 {'template': template})))
+        tests.addTests(loader.loadTestsFromTestCase(
+            type(
+                'VmUpdates_' + template,
+                (VmUpdatesMixin, qubes.tests.QubesTestCase),
+                {'template': template})))
     return tests

From f064a4c6ea0cb242c6bf9d7f47dd7d8b924358d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 27 Oct 2015 23:16:37 +0100
Subject: [PATCH 090/225] tests: improve qvm-run of GUI apps tests, include
 qubes-desktop-run

---
 tests/vm_qrexec_gui.py | 77 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 76 insertions(+), 1 deletion(-)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index c1f2ed11..94a91dd9 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -70,7 +70,42 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
 
     @unittest.skipUnless(spawn.find_executable('xdotool'),
                          "xdotool not installed")
-    def test_010_run_gui_app(self):
+    def test_010_run_xterm(self):
+        self.testvm1.start()
+        self.assertEquals(self.testvm1.get_power_state(), "Running")
+        self.testvm1.run("xterm")
+        wait_count = 0
+        while subprocess.call(
+                ['xdotool', 'search', '--name', 'user@{}'.
+                    format(self.testvm1.name)],
+                stdout=open(os.path.devnull, 'w'),
+                stderr=subprocess.STDOUT) > 0:
+            wait_count += 1
+            if wait_count > 100:
+                self.fail("Timeout while waiting for gnome-terminal window")
+            time.sleep(0.1)
+
+        time.sleep(0.5)
+        subprocess.check_call(
+            ['xdotool', 'search', '--name', 'user@{}'.format(self.testvm1.name),
+             'windowactivate', 'type', 'exit\n'])
+
+        wait_count = 0
+        while subprocess.call(['xdotool', 'search', '--name',
+                               'user@{}'.format(self.testvm1.name)],
+                              stdout=open(os.path.devnull, 'w'),
+                              stderr=subprocess.STDOUT) == 0:
+            wait_count += 1
+            if wait_count > 100:
+                self.fail("Timeout while waiting for gnome-terminal "
+                          "termination")
+            time.sleep(0.1)
+
+    @unittest.skipUnless(spawn.find_executable('xdotool'),
+                         "xdotool not installed")
+    def test_011_run_gnome_terminal(self):
+        if "minimal" in self.template:
+            self.skipTest("Minimal template doesn't have 'gnome-terminal'")
         self.testvm1.start()
         self.assertEquals(self.testvm1.get_power_state(), "Running")
         self.testvm1.run("gnome-terminal")
@@ -101,6 +136,46 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
                           "termination")
             time.sleep(0.1)
 
+    @unittest.skipUnless(spawn.find_executable('xdotool'),
+                         "xdotool not installed")
+    def test_012_qubes_desktop_run(self):
+        self.testvm1.start()
+        self.assertEquals(self.testvm1.get_power_state(), "Running")
+        xterm_desktop_path = "/usr/share/applications/xterm.desktop"
+        # Debian has it different...
+        xterm_desktop_path_debian = \
+            "/usr/share/applications/debian-xterm.desktop"
+        if self.testvm1.run("test -r {}".format(xterm_desktop_path_debian),
+                            wait=True) == 0:
+            xterm_desktop_path = xterm_desktop_path_debian
+        self.testvm1.run("qubes-desktop-run {}".format(xterm_desktop_path))
+        wait_count = 0
+        while subprocess.call(
+                ['xdotool', 'search', '--name', 'user@{}'.
+                    format(self.testvm1.name)],
+                stdout=open(os.path.devnull, 'w'),
+                stderr=subprocess.STDOUT) > 0:
+            wait_count += 1
+            if wait_count > 100:
+                self.fail("Timeout while waiting for gnome-terminal window")
+            time.sleep(0.1)
+
+        time.sleep(0.5)
+        subprocess.check_call(
+            ['xdotool', 'search', '--name', 'user@{}'.format(self.testvm1.name),
+             'windowactivate', 'type', 'exit\n'])
+
+        wait_count = 0
+        while subprocess.call(['xdotool', 'search', '--name',
+                               'user@{}'.format(self.testvm1.name)],
+                              stdout=open(os.path.devnull, 'w'),
+                              stderr=subprocess.STDOUT) == 0:
+            wait_count += 1
+            if wait_count > 100:
+                self.fail("Timeout while waiting for gnome-terminal "
+                          "termination")
+            time.sleep(0.1)
+
     def test_050_qrexec_simple_eof(self):
         """Test for data and EOF transmission dom0->VM"""
         result = multiprocessing.Value('i', 0)

From 33255964c75646b3e24acdb58b53f6592f58e051 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 30 Oct 2015 14:09:52 +0100
Subject: [PATCH 091/225] tests: handle Whonix in simple GUI tests

Whonix VMs have always hostname of 'host', so handle that when searching
for a window based on its title.
---
 tests/vm_qrexec_gui.py | 41 ++++++++++++++++++++++-------------------
 1 file changed, 22 insertions(+), 19 deletions(-)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index 94a91dd9..4c2a309f 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -75,29 +75,30 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
         self.assertEquals(self.testvm1.get_power_state(), "Running")
         self.testvm1.run("xterm")
         wait_count = 0
+        title = 'user@{}'.format(self.testvm1.name)
+        if self.template.count("whonix"):
+            title = 'user@host'
         while subprocess.call(
-                ['xdotool', 'search', '--name', 'user@{}'.
-                    format(self.testvm1.name)],
+                ['xdotool', 'search', '--name', title],
                 stdout=open(os.path.devnull, 'w'),
                 stderr=subprocess.STDOUT) > 0:
             wait_count += 1
             if wait_count > 100:
-                self.fail("Timeout while waiting for gnome-terminal window")
+                self.fail("Timeout while waiting for xterm window")
             time.sleep(0.1)
 
         time.sleep(0.5)
         subprocess.check_call(
-            ['xdotool', 'search', '--name', 'user@{}'.format(self.testvm1.name),
+            ['xdotool', 'search', '--name', title,
              'windowactivate', 'type', 'exit\n'])
 
         wait_count = 0
-        while subprocess.call(['xdotool', 'search', '--name',
-                               'user@{}'.format(self.testvm1.name)],
+        while subprocess.call(['xdotool', 'search', '--name', title],
                               stdout=open(os.path.devnull, 'w'),
                               stderr=subprocess.STDOUT) == 0:
             wait_count += 1
             if wait_count > 100:
-                self.fail("Timeout while waiting for gnome-terminal "
+                self.fail("Timeout while waiting for xterm "
                           "termination")
             time.sleep(0.1)
 
@@ -109,10 +110,12 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
         self.testvm1.start()
         self.assertEquals(self.testvm1.get_power_state(), "Running")
         self.testvm1.run("gnome-terminal")
+        title = 'user@{}'.format(self.testvm1.name)
+        if self.template.count("whonix"):
+            title = 'user@host'
         wait_count = 0
         while subprocess.call(
-                ['xdotool', 'search', '--name', 'user@{}'.
-                    format(self.testvm1.name)],
+                ['xdotool', 'search', '--name', title],
                 stdout=open(os.path.devnull, 'w'),
                 stderr=subprocess.STDOUT) > 0:
             wait_count += 1
@@ -122,12 +125,11 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
 
         time.sleep(0.5)
         subprocess.check_call(
-            ['xdotool', 'search', '--name', 'user@{}'.format(self.testvm1.name),
+            ['xdotool', 'search', '--name', title,
              'windowactivate', 'type', 'exit\n'])
 
         wait_count = 0
-        while subprocess.call(['xdotool', 'search', '--name',
-                               'user@{}'.format(self.testvm1.name)],
+        while subprocess.call(['xdotool', 'search', '--name', title],
                               stdout=open(os.path.devnull, 'w'),
                               stderr=subprocess.STDOUT) == 0:
             wait_count += 1
@@ -149,30 +151,31 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
                             wait=True) == 0:
             xterm_desktop_path = xterm_desktop_path_debian
         self.testvm1.run("qubes-desktop-run {}".format(xterm_desktop_path))
+        title = 'user@{}'.format(self.testvm1.name)
+        if self.template.count("whonix"):
+            title = 'user@host'
         wait_count = 0
         while subprocess.call(
-                ['xdotool', 'search', '--name', 'user@{}'.
-                    format(self.testvm1.name)],
+                ['xdotool', 'search', '--name', title],
                 stdout=open(os.path.devnull, 'w'),
                 stderr=subprocess.STDOUT) > 0:
             wait_count += 1
             if wait_count > 100:
-                self.fail("Timeout while waiting for gnome-terminal window")
+                self.fail("Timeout while waiting for xterm window")
             time.sleep(0.1)
 
         time.sleep(0.5)
         subprocess.check_call(
-            ['xdotool', 'search', '--name', 'user@{}'.format(self.testvm1.name),
+            ['xdotool', 'search', '--name', title,
              'windowactivate', 'type', 'exit\n'])
 
         wait_count = 0
-        while subprocess.call(['xdotool', 'search', '--name',
-                               'user@{}'.format(self.testvm1.name)],
+        while subprocess.call(['xdotool', 'search', '--name', title],
                               stdout=open(os.path.devnull, 'w'),
                               stderr=subprocess.STDOUT) == 0:
             wait_count += 1
             if wait_count > 100:
-                self.fail("Timeout while waiting for gnome-terminal "
+                self.fail("Timeout while waiting for xterm "
                           "termination")
             time.sleep(0.1)
 

From 51604230657b557706fa358395264341ec02f3ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 30 Oct 2015 14:58:23 +0100
Subject: [PATCH 092/225] tests: perform dom0 update check using VMs of
 different templates

QubesOS/qubes-issues#1282
QubesOS/qubes-issues#1029
---
 tests/dom0_update.py | 68 +++++++++++++++++++++++---------------------
 1 file changed, 35 insertions(+), 33 deletions(-)

diff --git a/tests/dom0_update.py b/tests/dom0_update.py
index 6c4a20e9..41328a85 100644
--- a/tests/dom0_update.py
+++ b/tests/dom0_update.py
@@ -34,7 +34,10 @@ VM_PREFIX = "test-"
 @unittest.skipUnless(os.path.exists('/usr/bin/rpmsign') and
                      os.path.exists('/usr/bin/rpmbuild'),
                      'rpm-sign and/or rpm-build not installed')
-class TC_00_Dom0Upgrade(qubes.tests.QubesTestCase):
+class TC_00_Dom0UpgradeMixin(qubes.tests.SystemTestsMixin):
+    """
+    Tests for downloading dom0 updates using VMs based on different templates
+    """
     cleanup_paths = []
     pkg_name = 'qubes-test-pkg'
     dom0_update_common_opts = ['--disablerepo=*', '--enablerepo=test',
@@ -70,7 +73,7 @@ Expire-Date: 0
 
     @classmethod
     def setUpClass(cls):
-        super(TC_00_Dom0Upgrade, cls).setUpClass()
+        super(TC_00_Dom0UpgradeMixin, cls).setUpClass()
 
         cls.tmpdir = tempfile.mkdtemp()
         cls.cleanup_paths += [cls.tmpdir]
@@ -100,12 +103,11 @@ enabled = 1
         cls.cleanup_paths = []
 
     def setUp(self):
-        self.qc = QubesVmCollection()
-        self.qc.lock_db_for_writing()
-        self.qc.load()
-        self.updatevm = self.qc.add_new_vm("QubesProxyVm",
-                                           name="%supdatevm" % VM_PREFIX,
-                                           template=self.qc.get_default_template())
+        super(TC_00_Dom0UpgradeMixin, self).setUp()
+        self.updatevm = self.qc.add_new_vm(
+            "QubesProxyVm",
+            name=self.make_vm_name("updatevm"),
+            template=self.qc.get_vm_by_name(self.template))
         self.updatevm.create_on_disk(verbose=False)
         self.saved_updatevm = self.qc.get_updatevm_vm()
         self.qc.set_updatevm_vm(self.updatevm)
@@ -117,35 +119,13 @@ enabled = 1
                                os.path.join(self.tmpdir, 'pubkey.asc')])
         self.updatevm.start()
 
-
-    def remove_vms(self, vms):
+    def tearDown(self):
         self.qc.lock_db_for_writing()
         self.qc.load()
 
         self.qc.set_updatevm_vm(self.qc[self.saved_updatevm.qid])
-
-        for vm in vms:
-            if isinstance(vm, str):
-                vm = self.qc.get_vm_by_name(vm)
-            else:
-                vm = self.qc[vm.qid]
-            if vm.is_running():
-                try:
-                    vm.force_shutdown()
-                except:
-                    pass
-            try:
-                vm.remove_from_disk()
-            except OSError:
-                pass
-            self.qc.pop(vm.qid)
         self.qc.save()
-        self.qc.unlock_db()
-
-    def tearDown(self):
-        vmlist = [vm for vm in self.qc.values() if vm.name.startswith(
-            VM_PREFIX)]
-        self.remove_vms(vmlist)
+        super(TC_00_Dom0UpgradeMixin, self).tearDown()
 
         subprocess.call(['sudo', 'rpm', '-e', self.pkg_name], stderr=open(
             os.devnull, 'w'))
@@ -202,7 +182,9 @@ Test package
         p.stdin.write(open(filename).read())
         p.stdin.close()
         p.wait()
-        self.updatevm.run('cd /tmp/repo; createrepo .', wait=True)
+        retcode = self.updatevm.run('cd /tmp/repo; createrepo .', wait=True)
+        if retcode != 0:
+            raise RuntimeError("createrepo failed, cannot perform test")
 
     def test_000_update(self):
         filename = self.create_pkg(self.tmpdir, self.pkg_name, '1.0')
@@ -297,3 +279,23 @@ Test package
             self.pkg_name)], stdout=open('/dev/null', 'w'))
         self.assertEqual(retcode, 1,
                          'UNSIGNED package {}-1.0 installed'.format(self.pkg_name))
+
+
+def load_tests(loader, tests, pattern):
+    try:
+        qc = qubes.qubes.QubesVmCollection()
+        qc.lock_db_for_reading()
+        qc.load()
+        qc.unlock_db()
+        templates = [vm.name for vm in qc.values() if
+                     isinstance(vm, qubes.qubes.QubesTemplateVm)]
+    except OSError:
+        templates = []
+    for template in templates:
+        tests.addTests(loader.loadTestsFromTestCase(
+            type(
+                'TC_00_Dom0Upgrade_' + template,
+                (TC_00_Dom0UpgradeMixin, qubes.tests.QubesTestCase),
+                {'template': template})))
+
+    return tests

From cf110c74e21fa7f4826f4074b26455c4f6661f85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 3 Nov 2015 02:40:53 +0100
Subject: [PATCH 093/225] tests: fix cleanup in dom0 updates tests

The only directory to cleanup is cls.tmpdir. So make it simple, without
class property cleanup_paths, which for some reason didn't wasn't
emptied in tearDownClass.
---
 tests/dom0_update.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/tests/dom0_update.py b/tests/dom0_update.py
index 41328a85..cb294224 100644
--- a/tests/dom0_update.py
+++ b/tests/dom0_update.py
@@ -38,7 +38,6 @@ class TC_00_Dom0UpgradeMixin(qubes.tests.SystemTestsMixin):
     """
     Tests for downloading dom0 updates using VMs based on different templates
     """
-    cleanup_paths = []
     pkg_name = 'qubes-test-pkg'
     dom0_update_common_opts = ['--disablerepo=*', '--enablerepo=test',
                                '--setopt=test.copy_local=1']
@@ -76,7 +75,6 @@ Expire-Date: 0
         super(TC_00_Dom0UpgradeMixin, cls).setUpClass()
 
         cls.tmpdir = tempfile.mkdtemp()
-        cls.cleanup_paths += [cls.tmpdir]
 
         cls.keyid = cls.generate_key(cls.tmpdir)
 
@@ -98,9 +96,7 @@ enabled = 1
         subprocess.check_call(['sudo', 'rm', '-f',
                                '/etc/yum.repos.d/test.repo'])
 
-        for dir in cls.cleanup_paths:
-            shutil.rmtree(dir)
-        cls.cleanup_paths = []
+        shutil.rmtree(cls.tmpdir)
 
     def setUp(self):
         super(TC_00_Dom0UpgradeMixin, self).setUp()

From 44c340c046f7faa02738b5b86aba3de5bbf6fcfd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 3 Nov 2015 02:43:46 +0100
Subject: [PATCH 094/225] tests: add wait_for_window function to reduce code
 duplication

---
 tests/__init__.py      | 23 +++++++++++++++++++++++
 tests/vm_qrexec_gui.py | 19 ++-----------------
 2 files changed, 25 insertions(+), 17 deletions(-)

diff --git a/tests/__init__.py b/tests/__init__.py
index c3c37401..48035e69 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -35,6 +35,7 @@ import sys
 
 import qubes.backup
 import qubes.qubes
+import time
 
 VMPREFIX = 'test-'
 
@@ -339,6 +340,28 @@ class SystemTestsMixin(object):
         for vmname in vmnames:
             self._remove_vm_disk(vmname)
 
+    def wait_for_window(self, title, timeout=30, show=True):
+        """
+        Wait for a window with a given title. Depending on show parameter,
+        it will wait for either window to show or to disappear.
+
+        :param title: title of the window to wait for
+        :param timeout: timeout of the operation, in seconds
+        :param show: if True - wait for the window to be visible,
+            otherwise - to not be visible
+        :return: None
+        """
+
+        wait_count = 0
+        while subprocess.call(['xdotool', 'search', '--name', title],
+                              stdout=open(os.path.devnull, 'w'),
+                              stderr=subprocess.STDOUT) == 0:
+            wait_count += 1
+            if wait_count > timeout*10:
+                self.fail("Timeout while waiting for {} window to {}".format(
+                    title, "show" if show else "hide")
+                )
+            time.sleep(0.1)
 
 
 class BackupTestsMixin(SystemTestsMixin):
diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index 4c2a309f..52c97d8a 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -907,15 +907,7 @@ class TC_30_Gui_daemon(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
         testvm1.run('zenity --text-info --editable --title={}'.format(
             window_title))
 
-        wait_count = 0
-        while subprocess.call(['xdotool', 'search', '--name', window_title],
-                              stdout=open(os.path.devnull, 'w'),
-                              stderr=subprocess.STDOUT) > 0:
-            wait_count += 1
-            if wait_count > 100:
-                self.fail("Timeout while waiting for text-info window")
-            time.sleep(0.1)
-
+        self.wait_for_window(window_title)
         time.sleep(0.5)
         test_string = "test{}".format(testvm1.xid)
 
@@ -946,14 +938,7 @@ class TC_30_Gui_daemon(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
         window_title = 'user@{}'.format(testvm2.name)
         testvm2.run('zenity --entry --title={} > test.txt'.format(
             window_title))
-        wait_count = 0
-        while subprocess.call(['xdotool', 'search', '--name', window_title],
-                              stdout=open(os.path.devnull, 'w'),
-                              stderr=subprocess.STDOUT) > 0:
-            wait_count += 1
-            if wait_count > 100:
-                self.fail("Timeout while waiting for input window")
-            time.sleep(0.1)
+        self.wait_for_window(window_title)
 
         subprocess.check_call(['xdotool', 'key', '--delay', '100',
                                'ctrl+shift+v', 'ctrl+v', 'Return'])

From 31ad72aa700379607c823877e9f3d622278588f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 3 Nov 2015 02:44:25 +0100
Subject: [PATCH 095/225] tests: better handle templates with no 'createrepo'
 installed

---
 tests/dom0_update.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tests/dom0_update.py b/tests/dom0_update.py
index cb294224..12fcc95d 100644
--- a/tests/dom0_update.py
+++ b/tests/dom0_update.py
@@ -179,8 +179,12 @@ Test package
         p.stdin.close()
         p.wait()
         retcode = self.updatevm.run('cd /tmp/repo; createrepo .', wait=True)
-        if retcode != 0:
-            raise RuntimeError("createrepo failed, cannot perform test")
+        if retcode == 127:
+            self.skipTest("createrepo not installed in template {}".format(
+                self.template))
+        elif retcode != 0:
+            self.skipTest("createrepo failed with code {}, cannot perform the "
+                      "test".format(retcode))
 
     def test_000_update(self):
         filename = self.create_pkg(self.tmpdir, self.pkg_name, '1.0')

From c30c9c7d4ef560c1828ea760db08160da0465fbe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 3 Nov 2015 02:46:03 +0100
Subject: [PATCH 096/225] tests: minor fixes to time sync tests

- don't crash when ClockVM wasn't set previously
- increase allowed delta (for slow systems)
---
 tests/vm_qrexec_gui.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index 52c97d8a..8bc1a1db 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -611,7 +611,11 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
         self.testvm2.start()
         (start_time, _) = subprocess.Popen(["date", "-u", "+%s"],
                                            stdout=subprocess.PIPE).communicate()
-        original_clockvm_name = self.qc.get_clockvm_vm().name
+        original_clockvm = self.qc.get_clockvm_vm()
+        if original_clockvm:
+            original_clockvm_name = original_clockvm.name
+        else:
+            original_clockvm_name = "none"
         try:
             # use qubes-prefs to not hassle with qubes.xml locking
             subprocess.check_call(["qubes-prefs", "-s", "clockvm",
@@ -639,7 +643,7 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
             (dom0_time, _) = subprocess.Popen(["date", "-u", "+%s"],
                                               stdout=subprocess.PIPE
                                               ).communicate()
-            self.assertAlmostEquals(int(dom0_time), int(start_time), delta=10)
+            self.assertAlmostEquals(int(dom0_time), int(start_time), delta=30)
 
         except:
             # reset time to some approximation of the real time

From a7b124d6daad5c06b5572037c017363fd603343a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 3 Nov 2015 02:47:08 +0100
Subject: [PATCH 097/225] tests: fix race condition in copy-paste test

Don't wait arbitrary time for paste to finish.
---
 tests/vm_qrexec_gui.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index 8bc1a1db..08cdc715 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -940,13 +940,13 @@ class TC_30_Gui_daemon(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
 
         # Then paste it to the other window
         window_title = 'user@{}'.format(testvm2.name)
-        testvm2.run('zenity --entry --title={} > test.txt'.format(
-            window_title))
+        p = testvm2.run('zenity --entry --title={} > test.txt'.format(
+                        window_title), passio_popen=True)
         self.wait_for_window(window_title)
 
         subprocess.check_call(['xdotool', 'key', '--delay', '100',
                                'ctrl+shift+v', 'ctrl+v', 'Return'])
-        time.sleep(0.5)
+        p.wait()
 
         # And compare the result
         (test_output, _) = testvm2.run('cat test.txt',

From d38883833a9142eda4d6ccfb91df3026078736e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 6 Nov 2015 23:54:04 +0100
Subject: [PATCH 098/225] core: ensure that QubesDB connections are closed when
 disposing a VM collection

There are some circular dependencies (TemplateVM.appvms,
NetVM.connected_vms, and probably more), which prevents garbage
collector from cleaning them.

Fixes QubesOS/qubes-issues#1380
---
 core/qubes.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/core/qubes.py b/core/qubes.py
index a66f3188..c32a46fe 100755
--- a/core/qubes.py
+++ b/core/qubes.py
@@ -307,6 +307,16 @@ class QubesVmCollection(dict):
 
     def clear(self):
         self.log.debug('clear()')
+        # Hack for releasing FDs, which otherwise would be leaked because of
+        # circular dependencies on QubesVMs objects (so garbage collector
+        # doesn't handle them). See #1380 for details
+        for vm in self.values():
+            try:
+                if vm._qdb_connection:
+                    vm._qdb_connection.close()
+                    vm._qdb_connection = None
+            except AttributeError:
+                pass
         super(QubesVmCollection, self).clear()
 
     def values(self):

From 93b7b3cb726a56e278b428ae5064695197143c8d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 7 Nov 2015 00:20:24 +0100
Subject: [PATCH 099/225] utils/QubesWatch: improve error handling (minor)

---
 core/qubesutils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/qubesutils.py b/core/qubesutils.py
index ee669d7c..0ab25e34 100644
--- a/core/qubesutils.py
+++ b/core/qubesutils.py
@@ -750,10 +750,10 @@ class QubesWatch(object):
             try:
                 if vm.isActive():
                     self._register_watches(vm)
-            except libvirt.libvirtError:
+            except libvirt.libvirtError as e:
                 # this will happen if we loose a race with another tool,
                 # which can just remove the domain
-                if vmm.libvirt_conn.virConnGetLastError()[0] == libvirt.VIR_ERR_NO_DOMAIN:
+                if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN:
                     pass
                 raise
         # and for dom0

From 0695e7ba78f3095b8c085fa8bff841354808010e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 7 Nov 2015 00:21:16 +0100
Subject: [PATCH 100/225] utils/QubesWatch: register libvirt event loop only
 when really launched

Registering event implementation in libvirt and then not calling it is
harmful, because libvirt expects it working. Known drawbacks:
- keep-alives are advertised as supported but not really sent (cause
  dropping connections)
- connections are not closed (sockets remains open, effectively leaking
  file descriptors)

So call libvirt.virEventRegisterDefaultImpl only when it will be really
used (libvirt.virEventRunDefaultImpl called), which means calling it in
QubesWatch. Registering events implementation have effect only on new
libvirt connections, so start a new one for QubesWatch.

Fixes QubesOS/qubes-issues#1380
---
 core/qubes.py      |  1 -
 core/qubesutils.py | 11 ++++++++---
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/core/qubes.py b/core/qubes.py
index c32a46fe..e910b66b 100755
--- a/core/qubes.py
+++ b/core/qubes.py
@@ -159,7 +159,6 @@ class QubesVMMConnection(object):
 
         if 'xen.lowlevel.xs' in sys.modules:
             self._xs = xen.lowlevel.xs.xs()
-        libvirt.virEventRegisterDefaultImpl()
         self._libvirt_conn = libvirt.open(defaults['libvirt_uri'])
         if self._libvirt_conn == None:
             raise QubesException("Failed connect to libvirt driver")
diff --git a/core/qubesutils.py b/core/qubesutils.py
index 0ab25e34..689138a3 100644
--- a/core/qubesutils.py
+++ b/core/qubesutils.py
@@ -29,7 +29,7 @@ from lxml import etree
 from lxml.etree import ElementTree, SubElement, Element
 
 from qubes.qubes import QubesException
-from qubes.qubes import vmm
+from qubes.qubes import vmm,defaults
 from qubes.qubes import system_path,vm_files
 import sys
 import os
@@ -737,11 +737,16 @@ class QubesWatch(object):
         self.block_callback = None
         self.meminfo_callback = None
         self.domain_callback = None
-        vmm.libvirt_conn.domainEventRegisterAny(
+        libvirt.virEventRegisterDefaultImpl()
+        # open new libvirt connection because above
+        # virEventRegisterDefaultImpl is in practice effective only for new
+        # connections
+        self.libvirt_conn = libvirt.open(defaults['libvirt_uri'])
+        self.libvirt_conn.domainEventRegisterAny(
             None,
             libvirt.VIR_DOMAIN_EVENT_ID_LIFECYCLE,
             self._domain_list_changed, None)
-        vmm.libvirt_conn.domainEventRegisterAny(
+        self.libvirt_conn.domainEventRegisterAny(
             None,
             libvirt.VIR_DOMAIN_EVENT_ID_DEVICE_REMOVED,
             self._device_removed, None)

From 8275e828afb0ea857d7c16426e6c5f3b589a0819 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 7 Nov 2015 03:05:41 +0100
Subject: [PATCH 101/225] backup: move existing directories out of the way
 during restore

In most cases it would be some leftover after failed restore, or even
the reason why the user is restoring a VM in the first place. Move it to
nearby directory, but do not remove - backup tool should _never_ remove
any data.

When the pre-existing directory would not be moved, restore utility
(`shutil.move`) would place the data inside of that directory, with
additional directory level (for example `/var/lib/qubes/appvms/work/work`),
which would be wrong and would later fail on `vm.verify_files`. And more
importantly - such VM would not work.

Fixes QubesOS/qubes-issues#1386
---
 core/backup.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/core/backup.py b/core/backup.py
index 2493d4a8..08190bb8 100644
--- a/core/backup.py
+++ b/core/backup.py
@@ -2104,6 +2104,20 @@ def backup_restore_do(restore_info,
                 error_callback("Skipping...")
                 continue
 
+            if os.path.exists(vm.dir_path):
+                move_to_path = tempfile.mkdtemp('', os.path.basename(
+                    vm.dir_path), os.path.dirname(vm.dir_path))
+                try:
+                    os.rename(vm.dir_path, move_to_path)
+                    error_callback("*** Directory {} already exists! It has "
+                                   "been moved to {}".format(vm.dir_path,
+                                                             move_to_path))
+                except OSError:
+                    error_callback("*** Directory {} already exists and "
+                                   "cannot be moved!".format(vm.dir_path))
+                    error_callback("Skipping...")
+                    continue
+
             template = None
             if vm.template is not None:
                 template_name = restore_info[vm.name]['template']

From 6051a89227a826a84be41fce6dc32726f20367dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 7 Nov 2015 03:11:55 +0100
Subject: [PATCH 102/225] tests: add regression test for #1386

QubesOS/qubes-issues#1386
---
 tests/__init__.py | 10 ++++++++--
 tests/backup.py   | 19 +++++++++++++++++++
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/tests/__init__.py b/tests/__init__.py
index 48035e69..0a5e9c17 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -477,7 +477,8 @@ class BackupTestsMixin(SystemTestsMixin):
         self.qc.load()
 
 
-    def restore_backup(self, source=None, appvm=None, options=None):
+    def restore_backup(self, source=None, appvm=None, options=None,
+                       expect_errors=None):
         if source is None:
             backupfile = os.path.join(self.backupdir,
                                       sorted(os.listdir(self.backupdir))[-1])
@@ -506,8 +507,13 @@ class BackupTestsMixin(SystemTestsMixin):
         self.qc.load()
 
         errors = []
+        if expect_errors is None:
+            expect_errors = []
         while not self.error_detected.empty():
-            errors.append(self.error_detected.get())
+            current_error = self.error_detected.get()
+            if any(map(current_error.startswith, expect_errors)):
+                continue
+            errors.append(current_error)
         self.assertTrue(len(errors) == 0,
                          "Error(s) detected during backup_restore_do: %s" %
                          '\n'.join(errors))
diff --git a/tests/backup.py b/tests/backup.py
index c48c30e9..15f63014 100644
--- a/tests/backup.py
+++ b/tests/backup.py
@@ -100,3 +100,22 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
         self.restore_backup(source='dd if=/var/tmp/backup-test',
                             appvm=self.backupvm)
         self.remove_vms(vms)
+
+    def test_200_restore_over_existing_directory(self):
+        """
+        Regression test for #1386
+        :return:
+        """
+        vms = self.create_backup_vms()
+        self.make_backup(vms)
+        self.remove_vms(vms)
+        test_dir = vms[0].dir_path
+        os.mkdir(test_dir)
+        with open(os.path.join(test_dir, 'some-file.txt'), 'w') as f:
+            f.write('test file\n')
+        self.restore_backup(
+            expect_errors=[
+                '*** Directory {} already exists! It has been moved'.format(
+                    test_dir)
+            ])
+        self.remove_vms(vms)

From f714a8224b5d18cbf335b27bbe90a625b6b60529 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 7 Nov 2015 03:12:19 +0100
Subject: [PATCH 103/225] backup: follow symbolic links

For a long time Qubes backup did not include symlinked files, which
apparently is quite common practice for users with multiple disks (for
example HDD + SSD). It is covered in documentation
(https://www.qubes-os.org/doc/secondary-storage/), but better solution
would be to simply include symlinked files.

Restore of such files would (of course) not preserve the symlinks -
normal files will be restored instead. But that's fine. If the user want
to move the data to another location, he/she can do that and restore the
symlink.

The only possible breakage from this change is having a copy (instead of
symlink) to a VM icon. But storing that symlink in a backup was broken
for some time (because of --xform usage) and it is handled during
restore, so not a real problem.

This doesn't cover all the problems with symlinked VM images - the other
one is qvm-block behaviour, which would treat such images as non-system
disks, so easily detachable (which would break VM operation). But that's
another story.

Fixes QubesOS/qubes-issues#1384
---
 core/backup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core/backup.py b/core/backup.py
index 08190bb8..7852c278 100644
--- a/core/backup.py
+++ b/core/backup.py
@@ -555,6 +555,7 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
         tar_cmdline = ["tar", "-Pc", '--sparse',
                        "-f", backup_pipe,
                        '-C', os.path.dirname(filename["path"]),
+                       '--dereference',
                        '--xform', 's:^%s:%s\\0:' % (
                            os.path.basename(filename["path"]),
                            filename["subdir"]),

From cf6c6badedfb6b4d275ec93289fce3779ad94c3f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 7 Nov 2015 04:19:20 +0100
Subject: [PATCH 104/225] Improve resize2fs call for root.img resize

1. Do not start GUI session - to reduce startup time
2. Use user="root" instead of sudo - not all templates have sudo
installed (for example fedora-21-minimal)
3. Add missing wait=True - otherwise VM may be shutdown during the
operation.

QubesOS/qubes-issues#1268
---
 core-modules/001QubesResizableVm.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/core-modules/001QubesResizableVm.py b/core-modules/001QubesResizableVm.py
index cf9bcf1e..88467ed1 100644
--- a/core-modules/001QubesResizableVm.py
+++ b/core-modules/001QubesResizableVm.py
@@ -58,8 +58,9 @@ class QubesResizableVmWithResize2fs(QubesResizableVm):
 
     def resize_root_img(self, size):
         super(QubesResizableVmWithResize2fs, self).resize_root_img(size)
-        self.start()
-        self.run("sudo resize2fs /dev/mapper/dmroot")
+        self.start(start_guid=False)
+        self.run("resize2fs /dev/mapper/dmroot", user="root", wait=True,
+                 gui=False)
         self.shutdown()
 
 

From 49141d444ef39f665244eced822b45f323f4ef5f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 7 Nov 2015 04:22:03 +0100
Subject: [PATCH 105/225] tests: resize private.img/root.img

QubesOS/qubes-issues#1268
---
 tests/vm_qrexec_gui.py | 71 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index 08cdc715..47fd04b8 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -653,6 +653,66 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
             subprocess.call(["qubes-prefs", "-s", "clockvm",
                              original_clockvm_name])
 
+    def test_250_resize_private_img(self):
+        """
+        Test private.img resize, both offline and online
+        :return:
+        """
+        # First offline test
+        self.testvm1.resize_private_img(4*1024**3)
+        self.testvm1.start()
+        p = self.testvm1.run('df --output=size /rw|tail -n 1',
+                             passio_popen=True)
+        # new_size in 1k-blocks
+        (new_size, _) = p.communicate()
+        # some safety margin for FS metadata
+        self.assertGreater(int(new_size.strip()), 3.8*1024**2)
+        # Then online test
+        self.testvm1.resize_private_img(6*1024**3)
+        p = self.testvm1.run('df --output=size /rw|tail -n 1',
+                             passio_popen=True)
+        # new_size in 1k-blocks
+        (new_size, _) = p.communicate()
+        # some safety margin for FS metadata
+        self.assertGreater(int(new_size.strip()), 5.8*1024**2)
+
+
+class TC_05_StandaloneVM(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
+    def test_000_create_start(self):
+        testvm1 = self.qc.add_new_vm("QubesAppVm",
+                                     template=None,
+                                     name=self.make_vm_name('vm1'))
+        testvm1.create_on_disk(verbose=False,
+                               source_template=self.qc.get_default_template())
+        self.qc.save()
+        self.qc.unlock_db()
+        testvm1.start()
+        self.assertEquals(testvm1.get_power_state(), "Running")
+
+    def test_100_resize_root_img(self):
+        testvm1 = self.qc.add_new_vm("QubesAppVm",
+                                     template=None,
+                                     name=self.make_vm_name('vm1'))
+        testvm1.create_on_disk(verbose=False,
+                               source_template=self.qc.get_default_template())
+        self.qc.save()
+        self.qc.unlock_db()
+        testvm1.resize_root_img(20*1024**3)
+        timeout = 60
+        while testvm1.is_running():
+            time.sleep(1)
+            timeout -= 1
+            if timeout == 0:
+                self.fail("Timeout while waiting for VM shutdown")
+        self.assertEquals(testvm1.get_root_img_sz(), 20*1024**3)
+        testvm1.start()
+        p = testvm1.run('df --output=size /|tail -n 1',
+                        passio_popen=True)
+        # new_size in 1k-blocks
+        (new_size, _) = p.communicate()
+        # some safety margin for FS metadata
+        self.assertGreater(int(new_size.strip()), 19*1024**2)
+
 
 class TC_10_HVM(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
     # TODO: test with some OS inside
@@ -710,6 +770,17 @@ class TC_10_HVM(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
         self.assertEquals(testvm2.get_power_state(), "Running")
         self.assertRaises(QubesException, templatevm.start)
 
+    def test_100_resize_root_img(self):
+        testvm1 = self.qc.add_new_vm("QubesHVm",
+                                     name=self.make_vm_name('vm1'))
+        testvm1.create_on_disk(verbose=False)
+        self.qc.save()
+        self.qc.unlock_db()
+        testvm1.resize_root_img(30*1024**3)
+        self.assertEquals(testvm1.get_root_img_sz(), 30*1024**3)
+        testvm1.start()
+        self.assertEquals(testvm1.get_power_state(), "Running")
+        # TODO: launch some OS there and check the size
 
 class TC_20_DispVMMixin(qubes.tests.SystemTestsMixin):
     def test_000_prepare_dvm(self):

From 34f7ddd41ebddfb8a38aa8231c59c738c26082a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 7 Nov 2015 05:43:34 +0100
Subject: [PATCH 106/225] core: force QubesDB reconnection on VM start

---
 core-modules/000QubesVm.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index 8cf8a661..4927a400 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -1760,6 +1760,9 @@ class QubesVm(object):
         except IOError:  # ENOENT (pidfile)
             pass
 
+        # force connection to a new daemon
+        self._qdb_connection = None
+
         retcode = subprocess.call ([
             system_path["qubesdb_daemon_path"],
             str(self.xid),

From c595ac464be770c7e60a7393cb58685b13e719dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 7 Nov 2015 18:46:54 +0100
Subject: [PATCH 107/225] tests: make sure that DispVM service call receive EOF

Even when the test fails. Otherwise the DispVM will stay alive.
---
 tests/vm_qrexec_gui.py | 54 +++++++++++++++++++++---------------------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index 47fd04b8..176adb6b 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -837,35 +837,35 @@ class TC_20_DispVMMixin(qubes.tests.SystemTestsMixin):
         dispvm = self.qc[max_qid]
         self.assertNotEqual(dispvm.qid, 0, "DispVM not found in qubes.xml")
         self.assertTrue(dispvm.is_running())
+        try:
+            window_title = 'user@%s' % (dispvm.template.name + "-dvm")
+            p.stdin.write("gnome-terminal -e "
+                          "\"sh -s -c 'echo \\\"\033]0;{}\007\\\"'\"\n".
+                          format(window_title))
+            wait_count = 0
+            while subprocess.call(['xdotool', 'search', '--name', window_title],
+                                  stdout=open(os.path.devnull, 'w'),
+                                  stderr=subprocess.STDOUT) > 0:
+                wait_count += 1
+                if wait_count > 100:
+                    self.fail("Timeout while waiting for gnome-terminal window")
+                time.sleep(0.1)
 
-        window_title = 'user@%s' % (dispvm.template.name + "-dvm")
-        p.stdin.write("gnome-terminal -e "
-                      "\"sh -s -c 'echo \\\"\033]0;{}\007\\\"'\"\n".
-                      format(window_title))
-        wait_count = 0
-        while subprocess.call(['xdotool', 'search', '--name', window_title],
-                              stdout=open(os.path.devnull, 'w'),
-                              stderr=subprocess.STDOUT) > 0:
-            wait_count += 1
-            if wait_count > 100:
-                self.fail("Timeout while waiting for gnome-terminal window")
-            time.sleep(0.1)
+            time.sleep(0.5)
+            subprocess.check_call(['xdotool', 'search', '--name', window_title,
+                                  'windowactivate', 'type', 'exit\n'])
 
-        time.sleep(0.5)
-        subprocess.check_call(['xdotool', 'search', '--name', window_title,
-                              'windowactivate', 'type', 'exit\n'])
-
-        wait_count = 0
-        while subprocess.call(['xdotool', 'search', '--name', window_title],
-                              stdout=open(os.path.devnull, 'w'),
-                              stderr=subprocess.STDOUT) == 0:
-            wait_count += 1
-            if wait_count > 100:
-                self.fail("Timeout while waiting for gnome-terminal "
-                          "termination")
-            time.sleep(0.1)
-
-        p.stdin.close()
+            wait_count = 0
+            while subprocess.call(['xdotool', 'search', '--name', window_title],
+                                  stdout=open(os.path.devnull, 'w'),
+                                  stderr=subprocess.STDOUT) == 0:
+                wait_count += 1
+                if wait_count > 100:
+                    self.fail("Timeout while waiting for gnome-terminal "
+                              "termination")
+                time.sleep(0.1)
+        finally:
+            p.stdin.close()
 
         wait_count = 0
         while dispvm.is_running():

From ba54792c83dfdefa101e2e5db4f25f68be3e58a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 10 Nov 2015 17:04:50 +0100
Subject: [PATCH 108/225] core: make modules.img optional even for PV

When using PVGrub it doesn't make sense to attach modules.img, since
modules are already in the VM. Initramfs there will already handle such
situation and will not try to mount it, when VM's root filesystem
already contains appropriate /lib/modules/`uname -r`.

On the other hand, error earlier when initramfs is missing. While also
not used by PV Grub, the file is always specified in libvirt config and
when missing libvirt will throw rather cryptic error message.

QubesOS/qubes-issues#1354
---
 core-modules/000QubesVm.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index 4927a400..f5513520 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -347,9 +347,11 @@ class QubesVm(object):
         # Initialize VM image storage class
         self.storage = defaults["storage_class"](self)
         if hasattr(self, 'kernels_dir'):
-            self.storage.modules_img = os.path.join(self.kernels_dir,
+            modules_path = os.path.join(self.kernels_dir,
                     "modules.img")
-            self.storage.modules_img_rw = self.kernel is None
+            if os.path.exists(modules_path):
+                self.storage.modules_img = modules_path
+                self.storage.modules_img_rw = self.kernel is None
 
         # Some additional checks for template based VM
         if self.template is not None:
@@ -517,7 +519,7 @@ class QubesVm(object):
             if not os.path.exists(os.path.join(system_path[
                 'qubes_kernels_base_dir'], new_value)):
                 raise QubesException("Kernel '%s' not installed" % new_value)
-            for f in ('vmlinuz', 'modules.img'):
+            for f in ('vmlinuz', 'initramfs'):
                 if not os.path.exists(os.path.join(
                         system_path['qubes_kernels_base_dir'], new_value, f)):
                     raise QubesException(

From 540fe3f55286782bdc756cbe4b627bfe29bfccb2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 10 Nov 2015 17:07:57 +0100
Subject: [PATCH 109/225] core: set vm.uses_default_kernel=False when setting
 vm.kernel

Forgetting this leads to misterious errors (VM started with different
kernel than it was just set), so simplify the API.

Fixes QubesOS/qubes-issues#1400
---
 core-modules/000QubesVm.py |  1 +
 qvm-tools/qvm-prefs        | 10 ++++------
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index f5513520..90413bb2 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -526,6 +526,7 @@ class QubesVm(object):
                         "Kernel '%s' not properly installed: missing %s "
                         "file" % (new_value, f))
         self._kernel = new_value
+        self.uses_default_kernel = False
 
     @property
     def updateable(self):
diff --git a/qvm-tools/qvm-prefs b/qvm-tools/qvm-prefs
index 96cb164d..dea3aa4f 100755
--- a/qvm-tools/qvm-prefs
+++ b/qvm-tools/qvm-prefs
@@ -285,7 +285,7 @@ def set_dispvm_netvm(vms, vm, args):
     return True
 
 def set_kernel(vms, vm, args):
-    if len (args) != 1:
+    if len(args) != 1:
         print >> sys.stderr, "Missing kernel version argument!"
         print >> sys.stderr, "Possible values:"
         print >> sys.stderr, "1) default"
@@ -297,18 +297,16 @@ def set_kernel(vms, vm, args):
 
     kernel = args[0]
     if kernel == "default":
-        kernel = vms.get_default_kernel()
+        vm.kernel = vms.get_default_kernel()
         vm.uses_default_kernel = True
     elif kernel == "none":
-        kernel = None
-        vm.uses_default_kernel = False
+        vm.kernel = None
     else:
         if not os.path.exists(os.path.join(system_path["qubes_kernels_base_dir"], kernel)):
             print >> sys.stderr, "Kernel version {0} not installed.".format(kernel)
             return False
-        vm.uses_default_kernel = False
+        vm.kernel = kernel
 
-    vm.kernel = kernel
     return True
 
 def set_template(vms, vm, args):

From 3e3ac36f58e9780d0f02168bfbaa1df859120c29 Mon Sep 17 00:00:00 2001
From: Rusty Bird <rustybird@openmailbox.org>
Date: Tue, 10 Nov 2015 17:10:03 +0000
Subject: [PATCH 110/225] cp --reflink=auto to speed up clone/backup/restore on
 btrfs

---
 core/backup.py           | 4 ++--
 core/storage/__init__.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/core/backup.py b/core/backup.py
index 7852c278..6b20212c 100644
--- a/core/backup.py
+++ b/core/backup.py
@@ -1992,7 +1992,7 @@ def backup_restore_do(restore_info,
                                          backup_dir)
 
         # We prefer to use Linux's cp, because it nicely handles sparse files
-        cp_retcode = subprocess.call(["cp", "-rp", backup_src_dir, dst_dir])
+        cp_retcode = subprocess.call(["cp", "-rp", "--reflink=auto", backup_src_dir, dst_dir])
         if cp_retcode != 0:
             raise QubesException(
                 "*** Error while copying file {0} to {1}".format(backup_src_dir,
@@ -2224,7 +2224,7 @@ def backup_restore_do(restore_info,
                           home_dir + '/' + restore_home_backupdir + '/' + f)
             if format_version == 1:
                 subprocess.call(
-                    ["cp", "-nrp", backup_dom0_home_dir + '/' + f, home_file])
+                    ["cp", "-nrp", "--reflink=auto", backup_dom0_home_dir + '/' + f, home_file])
             elif format_version >= 2:
                 shutil.move(backup_dom0_home_dir + '/' + f, home_file)
         retcode = subprocess.call(['sudo', 'chown', '-R', local_user, home_dir])
diff --git a/core/storage/__init__.py b/core/storage/__init__.py
index 3a12dbe6..29cf8654 100644
--- a/core/storage/__init__.py
+++ b/core/storage/__init__.py
@@ -79,7 +79,7 @@ class QubesVmStorage(object):
         # TODO: Windows support
 
         # We prefer to use Linux's cp, because it nicely handles sparse files
-        retcode = subprocess.call (["cp", source, destination])
+        retcode = subprocess.call (["cp", "--reflink=auto", source, destination])
         if retcode != 0:
             raise IOError ("Error while copying {0} to {1}".\
                            format(source, destination))

From cdc6d74c86729a22b347290012faee8cdd96bd50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 10 Nov 2015 17:18:33 +0100
Subject: [PATCH 111/225] tests: PV Grub tests

QubesOS/qubes-issues#1354
---
 tests/__init__.py      |   8 +++
 tests/vm_qrexec_gui.py | 130 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 138 insertions(+)

diff --git a/tests/__init__.py b/tests/__init__.py
index 0a5e9c17..8abea4b8 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -363,6 +363,14 @@ class SystemTestsMixin(object):
                 )
             time.sleep(0.1)
 
+    def shutdown_and_wait(self, vm, timeout=60):
+        vm.shutdown()
+        while timeout > 0:
+            if not vm.is_running():
+                return
+            time.sleep(1)
+            timeout -= 1
+        self.fail("Timeout while waiting for VM {} shutdown".format(vm.name))
 
 class BackupTestsMixin(SystemTestsMixin):
     def setUp(self):
diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index 176adb6b..091f4890 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -1036,6 +1036,131 @@ class TC_30_Gui_daemon(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
                           "Clipboard not wiped after paste - owner")
 
 
+@unittest.skipUnless(os.path.exists('/var/lib/qubes/vm-kernels/pvgrub2'),
+                     'grub-xen package not installed')
+class TC_40_PVGrub(qubes.tests.SystemTestsMixin):
+    def setUp(self):
+        super(TC_40_PVGrub, self).setUp()
+        supported = False
+        if self.template.startswith('fedora-'):
+            supported = True
+        elif self.template.startswith('debian-'):
+            supported = True
+        if not supported:
+            self.skipTest("Template {} not supported by this test".format(
+                self.template))
+
+    def install_packages(self, vm):
+        if self.template.startswith('fedora-'):
+            cmd_install1 = 'yum install -y qubes-kernel-vm-support grub2-tools'
+            cmd_install2 = 'yum install -y kernel kernel-devel'
+            cmd_update_grub = 'grub2-mkconfig -o /boot/grub2/grub.cfg'
+        elif self.template.startswith('debian-'):
+            cmd_install1 = 'apt-get update && apt-get install -y ' \
+                           'qubes-kernel-vm-support grub2-common'
+            cmd_install2 = 'apt-get install -y linux-image-amd64'
+            cmd_update_grub = 'mkdir /boot/grub && update-grub2'
+        else:
+            assert False, "Unsupported template?!"
+
+        for cmd in [cmd_install1, cmd_install2, cmd_update_grub]:
+            p = vm.run(cmd, user="root", passio_popen=True, passio_stderr=True)
+            (stdout, stderr) = p.communicate()
+            self.assertEquals(p.returncode, 0,
+                              "Failed command: {}\nSTDOUT: {}\nSTDERR: {}"
+                              .format(cmd, stdout, stderr))
+
+    def get_kernel_version(self, vm):
+        if self.template.startswith('fedora-'):
+            cmd_get_kernel_version = 'rpm -q kernel|sort -n|tail -1|' \
+                                     'cut -d - -f 2-'
+        elif self.template.startswith('debian-'):
+            cmd_get_kernel_version = \
+                'dpkg-query --showformat=\'${Package}\\n\' --show ' \
+                '\'linux-image-*-amd64\'|sort -n|tail -1|cut -d - -f 3-'
+        else:
+            raise RuntimeError("Unsupported template?!")
+
+        p = vm.run(cmd_get_kernel_version, user="root", passio_popen=True)
+        (kver, _) = p.communicate()
+        self.assertEquals(p.returncode, 0,
+                          "Failed command: {}".format(cmd_get_kernel_version))
+        return kver.strip()
+
+    def test_000_standalone_vm(self):
+        testvm1 = self.qc.add_new_vm("QubesAppVm",
+                                     template=None,
+                                     name=self.make_vm_name('vm1'))
+        testvm1.create_on_disk(verbose=False,
+                               source_template=self.qc.get_vm_by_name(
+                                   self.template))
+        self.save_and_reload_db()
+        self.qc.unlock_db()
+        testvm1 = self.qc[testvm1.qid]
+        testvm1.start()
+        self.install_packages(testvm1)
+        kver = self.get_kernel_version(testvm1)
+        self.shutdown_and_wait(testvm1)
+
+        self.qc.lock_db_for_writing()
+        self.qc.load()
+        testvm1 = self.qc[testvm1.qid]
+        testvm1.kernel = 'pvgrub2'
+        self.save_and_reload_db()
+        self.qc.unlock_db()
+        testvm1 = self.qc[testvm1.qid]
+        testvm1.start()
+        p = testvm1.run('uname -r', passio_popen=True)
+        (actual_kver, _) = p.communicate()
+        self.assertEquals(actual_kver.strip(), kver)
+
+    def test_010_template_based_vm(self):
+        test_template = self.qc.add_new_vm("QubesTemplateVm",
+                                           template=None,
+                                           name=self.make_vm_name('template'))
+        test_template.clone_attrs(self.qc.get_vm_by_name(self.template))
+        test_template.clone_disk_files(
+            src_vm=self.qc.get_vm_by_name(self.template),
+            verbose=False)
+
+        testvm1 = self.qc.add_new_vm("QubesAppVm",
+                                     template=test_template,
+                                     name=self.make_vm_name('vm1'))
+        testvm1.create_on_disk(verbose=False,
+                               source_template=test_template)
+        self.save_and_reload_db()
+        self.qc.unlock_db()
+        test_template = self.qc[test_template.qid]
+        testvm1 = self.qc[testvm1.qid]
+        test_template.start()
+        self.install_packages(test_template)
+        kver = self.get_kernel_version(test_template)
+        self.shutdown_and_wait(test_template)
+
+        self.qc.lock_db_for_writing()
+        self.qc.load()
+        test_template = self.qc[test_template.qid]
+        test_template.kernel = 'pvgrub2'
+        testvm1 = self.qc[testvm1.qid]
+        testvm1.kernel = 'pvgrub2'
+        self.save_and_reload_db()
+        self.qc.unlock_db()
+
+        # Check if TemplateBasedVM boots and has the right kernel
+        testvm1 = self.qc[testvm1.qid]
+        testvm1.start()
+        p = testvm1.run('uname -r', passio_popen=True)
+        (actual_kver, _) = p.communicate()
+        self.assertEquals(actual_kver.strip(), kver)
+
+        # And the same for the TemplateVM itself
+        test_template = self.qc[test_template.qid]
+        test_template.start()
+        p = test_template.run('uname -r', passio_popen=True)
+        (actual_kver, _) = p.communicate()
+        self.assertEquals(actual_kver.strip(), kver)
+
+
 def load_tests(loader, tests, pattern):
     try:
         qc = qubes.qubes.QubesVmCollection()
@@ -1058,5 +1183,10 @@ def load_tests(loader, tests, pattern):
                 'TC_20_DispVM_' + template,
                 (TC_20_DispVMMixin, qubes.tests.QubesTestCase),
                 {'template': template})))
+        tests.addTests(loader.loadTestsFromTestCase(
+            type(
+                'TC_40_PVGrub_' + template,
+                (TC_40_PVGrub, qubes.tests.QubesTestCase),
+                {'template': template})))
 
     return tests

From 5acd27f61ece21d1902d13f4ead92cf92ddfa9be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 10 Nov 2015 17:19:30 +0100
Subject: [PATCH 112/225] tests: increase allowed time delta for timezone tests

One more time, for slow systems... Let it be 30 seconds.
---
 tests/vm_qrexec_gui.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index 091f4890..516cf4c6 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -636,10 +636,10 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
                               format(retcode))
             (vm_time, _) = self.testvm1.run("date -u +%s",
                                             passio_popen=True).communicate()
-            self.assertAlmostEquals(int(vm_time), int(start_time), delta=10)
+            self.assertAlmostEquals(int(vm_time), int(start_time), delta=30)
             (vm_time, _) = self.testvm2.run("date -u +%s",
                                             passio_popen=True).communicate()
-            self.assertAlmostEquals(int(vm_time), int(start_time), delta=10)
+            self.assertAlmostEquals(int(vm_time), int(start_time), delta=30)
             (dom0_time, _) = subprocess.Popen(["date", "-u", "+%s"],
                                               stdout=subprocess.PIPE
                                               ).communicate()

From 06f5922da67bb07e5e9ec510766884116b812524 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 11 Nov 2015 04:26:10 +0100
Subject: [PATCH 113/225] Adjust dom0 sched-credit weight based on default
 libvirt value for VMs

While default value for VMs is normally 256, libvirt set it to
(hardcoded) 1000. So adjust dom0 accordingly.

Fixes QubesOS/qubes-issues#1404
---
 linux/aux-tools/startup-misc.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/linux/aux-tools/startup-misc.sh b/linux/aux-tools/startup-misc.sh
index bba40c33..8b5f8081 100755
--- a/linux/aux-tools/startup-misc.sh
+++ b/linux/aux-tools/startup-misc.sh
@@ -8,7 +8,7 @@ xenstore-write domid 0
 DOM0_MAXMEM=`/usr/sbin/xl info | grep total_memory | awk '{ print $3 }'`
 xenstore-write /local/domain/0/memory/static-max $[ $DOM0_MAXMEM * 1024 ]
 
-xl sched-credit -d 0 -w 512
+xl sched-credit -d 0 -w 2000
 cp /var/lib/qubes/qubes.xml /var/lib/qubes/backup/qubes-$(date +%F-%T).xml
 
 /usr/lib/qubes/cleanup-dispvms

From 636f5449a8517c2394f05ce960a34e4bcc522022 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 11 Nov 2015 06:32:48 +0100
Subject: [PATCH 114/225] version 3.1.2

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index 94ff29cc..ef538c28 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.1.1
+3.1.2

From 74d9c620256dba766e38b6f5ddd2d467506ff26a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 13 Nov 2015 02:30:41 +0100
Subject: [PATCH 115/225] tests: improve qrexec confirmation handling

'xdotool search --sync' sometimes fails when searching for window. Fix
that by ensuring the windows is already visible when trying to enter
keys there.
---
 tests/vm_qrexec_gui.py | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index 516cf4c6..d84605e7 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -54,6 +54,24 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
         self.testvm1 = self.qc[self.testvm1.qid]
         self.testvm2 = self.qc[self.testvm2.qid]
 
+    def enter_keys_in_window(self, title, keys):
+        """
+        Search for window with given title, then enter listed keys there.
+        The function will wait for said window to appear.
+
+        :param title: title of window
+        :param keys: list of keys to enter, as for `xdotool key`
+        :return: None
+        """
+
+        # 'xdotool search --sync' sometimes crashes on some race when
+        # accessing window properties
+        self.wait_for_window(title)
+        command = ['xdotool', 'search', '--name', title,
+                   'windowactivate',
+                   'key'] + keys
+        subprocess.check_call(command)
+
     def test_000_start_shutdown(self):
         self.testvm1.start()
         self.assertEquals(self.testvm1.get_power_state(), "Running")
@@ -556,8 +574,7 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
         p = self.testvm1.run("qvm-copy-to-vm %s /etc/passwd" %
                              self.testvm2.name, passio_popen=True)
         # Deny transfer
-        subprocess.check_call(['xdotool', 'search', '--sync', '--name', 'Question',
-                              'key', 'n'])
+        self.enter_keys_in_window('Question', ['n'])
         p.wait()
         self.assertNotEqual(p.returncode, 0, "qvm-copy-to-vm unexpectedly "
                             "succeeded")
@@ -577,8 +594,7 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
                              self.testvm1.name, passio_popen=True,
                              passio_stderr=True)
         # Confirm transfer
-        subprocess.check_call(['xdotool', 'search', '--sync', '--name', 'Question',
-                              'key', 'y'])
+        self.enter_keys_in_window('Question', ['y'])
         p.wait()
         self.assertEqual(p.returncode, 0, "qvm-copy-to-vm failed: %s" %
                          p.stderr.read())

From 4316027fb109db1631981d7d0528c2a55451a47b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 13 Nov 2015 05:43:40 +0100
Subject: [PATCH 116/225] core: Add 'wait' parameter to vm.run_service function

Sometimes it makes sense to "fire and forget" a service. For example
"qubes.InstallUpdatesGUI".

QubesOS/qubes-issues#1249
---
 core-modules/000QubesVm.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index 90413bb2..f8fdd76c 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -1616,20 +1616,21 @@ class QubesVm(object):
         return retcode
 
     def run_service(self, service, source="dom0", user=None,
-                    passio_popen=False, input=None, localcmd=None, gui=False):
+                    passio_popen=False, input=None, localcmd=None, gui=False,
+                    wait=True):
         if bool(input) + bool(passio_popen) + bool(localcmd) > 1:
             raise ValueError("'input', 'passio_popen', 'localcmd' cannot be "
                              "used together")
         if localcmd:
             return self.run("QUBESRPC %s %s" % (service, source),
-                            localcmd=localcmd, user=user, wait=True, gui=gui)
+                            localcmd=localcmd, user=user, wait=wait, gui=gui)
         elif input:
             return self.run("QUBESRPC %s %s" % (service, source),
-                            localcmd="echo %s" % input, user=user, wait=True,
+                            localcmd="echo %s" % input, user=user, wait=wait,
                             gui=gui)
         else:
             return self.run("QUBESRPC %s %s" % (service, source),
-                            passio_popen=passio_popen, user=user, wait=True,
+                            passio_popen=passio_popen, user=user, wait=wait,
                             gui=gui)
 
     def attach_network(self, verbose = False, wait = True, netvm = None):

From 46cbb4a1337bd1e98a715aa28fa3ffe1a98c3468 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 14 Nov 2015 23:29:21 +0100
Subject: [PATCH 117/225] Prevent stopping xenconsoled before shutting down all
 the VMs

Otherwise still running VMs will hang, consuming 100% CPU.

QubesOS/qubes-issues#1425
---
 linux/systemd/qubes-core.service | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/linux/systemd/qubes-core.service b/linux/systemd/qubes-core.service
index 345d6757..ae526a82 100644
--- a/linux/systemd/qubes-core.service
+++ b/linux/systemd/qubes-core.service
@@ -1,6 +1,6 @@
 [Unit]
 Description=Qubes Dom0 startup setup
-After=qubes-db-dom0.service libvirtd.service
+After=qubes-db-dom0.service libvirtd.service xenconsoled.service
 # Cover legacy init.d script
 
 [Service]

From 7359e394bc130336936494580bd8473bdcadd810 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 14 Nov 2015 23:36:22 +0100
Subject: [PATCH 118/225] core: detach connected VMs when shutting down NetVM

This is workaround for missing libxl/libvirt functionality: QubesOS/qubes-issues#1426

Also it should improve system shutdown time, as this is the situation
where all the VMs are shutting down simultaneously.

Fixes QubesOS/qubes-issues#1425
---
 core-modules/005QubesNetVm.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/core-modules/005QubesNetVm.py b/core-modules/005QubesNetVm.py
index c721bbab..33934f4b 100644
--- a/core-modules/005QubesNetVm.py
+++ b/core-modules/005QubesNetVm.py
@@ -23,7 +23,7 @@
 #
 import sys
 import os.path
-import xen.lowlevel.xs
+import libvirt
 
 from qubes.qubes import QubesVm,register_qubes_vm_class,vmm,dry_run
 from qubes.qubes import defaults,system_path,vm_files
@@ -153,6 +153,18 @@ class QubesNetVm(QubesVm):
         if connected_vms and not force:
             raise QubesException("There are other VMs connected to this VM: " + str([vm.name for vm in connected_vms]))
 
+        # detach network interfaces of connected VMs before shutting down,
+        # otherwise libvirt will not notice it and will try to detach them
+        # again (which would fail, obviously).
+        # This code can be removed when #1426 got implemented
+        for vm in self.connected_vms.values():
+            if vm.is_running():
+                try:
+                    vm.detach_network()
+                except (QubesException, libvirt.libvirtError):
+                    # ignore errors
+                    pass
+
         super(QubesNetVm, self).shutdown(force=force)
 
     def add_external_ip_permission(self, xid):

From bae493d015f3b7dc8f707550824e95c847c49360 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 14 Nov 2015 23:43:38 +0100
Subject: [PATCH 119/225] Kill remaining qubesdb-daemon instances after
 shutting down all the VMs

It may happen (especially when VM doesn't close cleanly and needs to be
killed) that qubesdb-daemon will not notice VM shutdown immediately.
Normally it would stop after 60s timeout, but speed it up in case of
system shutdown

QubesOS/qubes-issues#1425
---
 linux/systemd/qubes-core.service | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/linux/systemd/qubes-core.service b/linux/systemd/qubes-core.service
index ae526a82..1b9ee8a7 100644
--- a/linux/systemd/qubes-core.service
+++ b/linux/systemd/qubes-core.service
@@ -9,6 +9,9 @@ StandardOutput=syslog
 RemainAfterExit=yes
 ExecStart=/usr/lib/qubes/startup-misc.sh
 ExecStop=/usr/bin/qvm-shutdown -q --all --wait
+# QubesDB daemons stop after 60s timeout in worst case; speed it up, since no
+# VMs are running now
+ExecStop=-/usr/bin/killall qubesdb-daemon
 
 [Install]
 WantedBy=multi-user.target

From 2ccfff438a268debeb2f5eec8756fe45f3faa097 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 14 Nov 2015 23:50:24 +0100
Subject: [PATCH 120/225] tests: use xterm instead of gnome-terminal in DispVM
 tests

There is no gnome-terminal in minimal template.
---
 tests/vm_qrexec_gui.py | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index d84605e7..6d62f6f8 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -855,21 +855,14 @@ class TC_20_DispVMMixin(qubes.tests.SystemTestsMixin):
         self.assertTrue(dispvm.is_running())
         try:
             window_title = 'user@%s' % (dispvm.template.name + "-dvm")
-            p.stdin.write("gnome-terminal -e "
-                          "\"sh -s -c 'echo \\\"\033]0;{}\007\\\"'\"\n".
+            p.stdin.write("xterm -e "
+                          "\"sh -s -c 'echo \\\"\033]0;{}\007\\\";read;'\"\n".
                           format(window_title))
-            wait_count = 0
-            while subprocess.call(['xdotool', 'search', '--name', window_title],
-                                  stdout=open(os.path.devnull, 'w'),
-                                  stderr=subprocess.STDOUT) > 0:
-                wait_count += 1
-                if wait_count > 100:
-                    self.fail("Timeout while waiting for gnome-terminal window")
-                time.sleep(0.1)
+            self.wait_for_window(window_title)
 
             time.sleep(0.5)
             subprocess.check_call(['xdotool', 'search', '--name', window_title,
-                                  'windowactivate', 'type', 'exit\n'])
+                                  'windowactivate', 'key', 'Return'])
 
             wait_count = 0
             while subprocess.call(['xdotool', 'search', '--name', window_title],

From 520b546d7581cdca8e346b205527b93db9af6ac6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sun, 15 Nov 2015 04:20:45 +0100
Subject: [PATCH 121/225] version 3.1.3

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index ef538c28..ff365e06 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.1.2
+3.1.3

From 7550fccf940822440cde1cdb7950ee051a17f118 Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Sat, 7 Nov 2015 16:54:03 +0100
Subject: [PATCH 122/225] Removed WNI

---
 core/settings-wni-Windows_NT.py |  45 -----------
 core/storage/README.md          |   3 +
 core/storage/wni.py             | 138 --------------------------------
 3 files changed, 3 insertions(+), 183 deletions(-)
 delete mode 100644 core/settings-wni-Windows_NT.py
 create mode 100644 core/storage/README.md
 delete mode 100644 core/storage/wni.py

diff --git a/core/settings-wni-Windows_NT.py b/core/settings-wni-Windows_NT.py
deleted file mode 100644
index 6e646c9d..00000000
--- a/core/settings-wni-Windows_NT.py
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/usr/bin/python2
-
-from __future__ import absolute_import
-import _winreg
-import os
-import sys
-
-from qubes.storage.wni import QubesWniVmStorage
-
-DEFAULT_INSTALLDIR = 'c:\\program files\\Invisible Things Lab\\Qubes WNI'
-DEFAULT_STOREDIR = 'c:\\qubes'
-
-def apply(system_path, vm_files, defaults):
-    system_path['qubes_base_dir'] = DEFAULT_STOREDIR
-    installdir = DEFAULT_INSTALLDIR
-    try:
-        reg_key = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE,
-                "Software\\Invisible Things Lab\\Qubes WNI")
-        installdir = _winreg.QueryValueEx(reg_key, "InstallDir")[0]
-        system_path['qubes_base_dir'] = \
-                _winreg.QueryValueEx(reg_key, "StoreDir")[0]
-    except WindowsError as e:
-        print >>sys.stderr, \
-                "WARNING: invalid installation: missing registry entries (%s)" \
-                % str(e)
-
-    system_path['config_template_pv'] = \
-            os.path.join(installdir, 'vm-template.xml')
-    system_path['config_template_hvm'] = \
-            os.path.join(installdir, 'vm-template-hvm.xml')
-    system_path['qubes_icon_dir'] = os.path.join(installdir, 'icons')
-    system_path['qubesdb_daemon_path'] = \
-            os.path.join(installdir, 'bin\\qubesdb-daemon.exe')
-    system_path['qrexec_daemon_path'] = \
-            os.path.join(installdir, 'bin\\qrexec-daemon.exe')
-    system_path['qrexec_client_path'] = \
-            os.path.join(installdir, 'bin\\qrexec-client.exe')
-    system_path['qrexec_policy_dir'] = \
-            os.path.join(installdir, 'qubes-rpc\\policy')
-    # Specific to WNI - normally VM have this file
-    system_path['qrexec_agent_path'] = \
-            os.path.join(installdir, 'bin\\qrexec-agent.exe')
-
-    defaults['libvirt_uri'] = 'wni:///'
-    defaults['storage_class'] = QubesWniVmStorage
diff --git a/core/storage/README.md b/core/storage/README.md
new file mode 100644
index 00000000..7258512f
--- /dev/null
+++ b/core/storage/README.md
@@ -0,0 +1,3 @@
+# WNI File storage
+Before v3.1 there existed a draft wni storage. You can find it in the git
+history
diff --git a/core/storage/wni.py b/core/storage/wni.py
deleted file mode 100644
index a3571765..00000000
--- a/core/storage/wni.py
+++ /dev/null
@@ -1,138 +0,0 @@
-#!/usr/bin/python2
-#
-# The Qubes OS Project, http://www.qubes-os.org
-#
-# Copyright (C) 2013  Marek Marczykowski <marmarek@invisiblethingslab.com>
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-#
-#
-
-from __future__ import absolute_import
-
-import sys
-import os
-import os.path
-import win32api
-import win32net
-import win32netcon
-import win32security
-import win32profile
-import pywintypes
-import random
-
-from qubes.storage import QubesVmStorage
-from qubes.qubes import QubesException,system_path
-
-class QubesWniVmStorage(QubesVmStorage):
-    """
-    Class for VM storage of WNI VMs.
-    """
-
-    def __init__(self, *args, **kwargs):
-        super(QubesWniVmStorage, self).__init__(*args, **kwargs)
-        # Use the user profile as "private.img"
-        self.home_root = win32profile.GetProfilesDirectory()
-        # FIXME: the assignment below may not always be correct,
-        # but GetUserProfileDirectory needs a user token...
-        self.private_img = os.path.join(self.home_root, self._get_username())
-
-        # Pass paths for WNI libvirt driver
-        os.putenv("WNI_DRIVER_QUBESDB_PATH", system_path['qubesdb_daemon_path'])
-        os.putenv("WNI_DRIVER_QREXEC_AGENT_PATH", system_path['qrexec_agent_path'])
-
-    def _get_username(self, vmname = None):
-        if vmname is None:
-            vmname = self.vm.name
-        return "qubes-vm-%s" % vmname
-
-    def _get_random_password(self, vmname = None):
-        if vmname is None:
-            vmname = self.vm.name
-        return '%x' % random.SystemRandom().getrandombits(256)
-
-    def get_config_params(self):
-        return {}
-
-    def create_on_disk_private_img(self, verbose, source_template = None):
-        # FIXME: this may not always be correct
-        home_dir = os.path.join(self.home_root, self._get_username())
-        # Create user data in information level 1 (PyUSER_INFO_1) format.
-        user_data = {}
-        user_data['name'] = self._get_username()
-        user_data['full_name'] = self._get_username()
-        # libvirt driver doesn't need to know the password anymore
-        user_data['password'] = self._get_random_password()
-        user_data['flags'] = (
-                win32netcon.UF_NORMAL_ACCOUNT |
-                win32netcon.UF_SCRIPT |
-                win32netcon.UF_DONT_EXPIRE_PASSWD |
-                win32netcon.UF_PASSWD_CANT_CHANGE
-                )
-        user_data['priv'] = win32netcon.USER_PRIV_USER
-        user_data['home_dir'] = home_dir
-        user_data['max_storage'] = win32netcon.USER_MAXSTORAGE_UNLIMITED
-        # TODO: catch possible exception
-        win32net.NetUserAdd(None, 1, user_data)
-
-    def create_on_disk_root_img(self, verbose, source_template = None):
-        pass
-
-    def remove_from_disk(self):
-        try:
-            sid = win32security.LookupAccountName(None, self._get_username())[0]
-            string_sid = win32security.ConvertSidToStringSid(sid)
-            win32profile.DeleteProfile(string_sid)
-            win32net.NetUserDel(None, self._get_username())
-        except pywintypes.error, details:
-            if details[0] == 2221:
-                # "The user name cannot be found."
-                raise IOError("User %s doesn't exist" % self._get_username())
-            else:
-                raise
-
-        super(QubesWniVmStorage, self).remove_from_disk()
-
-    def rename(self, old_name, new_name):
-        super(QubesWniVmStorage, self).rename(old_name, new_name)
-        user_data = {}
-        user_data['name'] = self._get_username(new_name)
-        win32net.NetUserSetInfo(None,
-                self._get_username(old_name), 0, user_data)
-        #TODO: rename user profile
-
-    def verify_files(self):
-        if not os.path.exists (self.vmdir):
-            raise QubesException (
-                    "VM directory doesn't exist: {0}".\
-                            format(self.vmdir))
-
-        try:
-            # TemplateVm in WNI is quite virtual, so do not require the user
-            if not self.vm.is_template():
-                win32net.NetUserGetInfo(None, self._get_username(), 0)
-        except pywintypes.error, details:
-            if details[0] == 2221:
-                # "The user name cannot be found."
-                raise QubesException("User %s doesn't exist" % self._get_username())
-            else:
-                raise
-
-    def reset_volatile_storage(self, verbose = False, source_template = None):
-        pass
-
-    def prepare_for_vm_startup(self, verbose = False):
-        if self.vm.is_template():
-            raise QubesException("Starting TemplateVM is not supported")

From 15d5e6edbb1222d79b2deb3fa68f299ad2c76c2a Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Sat, 7 Nov 2015 18:58:49 +0100
Subject: [PATCH 123/225] Add Pool & XenPool

---
 core/settings-xen-Linux.py |  4 +++-
 core/storage/__init__.py   | 13 ++++++++-----
 core/storage/xen.py        | 12 ++++++++----
 3 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/core/settings-xen-Linux.py b/core/settings-xen-Linux.py
index de5084f5..8b23b447 100644
--- a/core/settings-xen-Linux.py
+++ b/core/settings-xen-Linux.py
@@ -2,7 +2,9 @@
 
 from __future__ import absolute_import
 
-from qubes.storage.xen import QubesXenVmStorage
+from qubes.storage.xen import QubesXenVmStorage, XenPool
+
 
 def apply(system_path, vm_files, defaults):
     defaults['storage_class'] = QubesXenVmStorage
+    defaults['pool_types'] = {'xen': XenPool}
diff --git a/core/storage/__init__.py b/core/storage/__init__.py
index 29cf8654..795aa37a 100644
--- a/core/storage/__init__.py
+++ b/core/storage/__init__.py
@@ -16,22 +16,21 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-#
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
+# USA.
 #
 
 from __future__ import absolute_import
 
 import os
 import os.path
-import re
 import shutil
 import subprocess
 import sys
 
-from qubes.qubes import vm_files,system_path,defaults
-from qubes.qubes import QubesException
 import qubes.qubesutils
+from qubes.qubes import QubesException, defaults, system_path, vm_files
+
 
 class QubesVmStorage(object):
     """
@@ -199,3 +198,7 @@ class QubesVmStorage(object):
             print >>sys.stderr, "WARNING: Creating empty VM private image file: {0}".\
                 format(self.private_img)
             self.create_on_disk_private_img(verbose=False)
+
+
+class Pool(object):
+    pass
diff --git a/core/storage/xen.py b/core/storage/xen.py
index 00c3b3da..f66126a6 100644
--- a/core/storage/xen.py
+++ b/core/storage/xen.py
@@ -16,20 +16,20 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-#
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
+# USA.
 #
 
 from __future__ import absolute_import
 
 import os
 import os.path
+import re
 import subprocess
 import sys
-import re
 
-from qubes.storage import QubesVmStorage
 from qubes.qubes import QubesException, vm_files
+from qubes.storage import Pool, QubesVmStorage
 
 
 class QubesXenVmStorage(QubesVmStorage):
@@ -249,3 +249,7 @@ class QubesXenVmStorage(QubesVmStorage):
                     raise QubesException(
                         "VM '{}' holding '{}' does not exists".format(
                             drive_domain, drive_path))
+
+
+class XenPool(Pool):
+    pass

From bfaf37dae534d0266c4e3a56808cd11ba0c8c0ef Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Sat, 7 Nov 2015 19:01:25 +0100
Subject: [PATCH 124/225] Add pool config parsing

---
 core/storage/Makefile     |  3 ++
 core/storage/__init__.py  | 95 +++++++++++++++++++++++++++++++++++++++
 core/storage/storage.conf | 11 +++++
 rpm_spec/core-dom0.spec   |  1 +
 tests/Makefile            |  2 +
 tests/__init__.py         |  1 +
 tests/storage.py          | 63 ++++++++++++++++++++++++++
 7 files changed, 176 insertions(+)
 create mode 100644 core/storage/storage.conf
 create mode 100644 tests/storage.py

diff --git a/core/storage/Makefile b/core/storage/Makefile
index ec59cc64..7c7af60e 100644
--- a/core/storage/Makefile
+++ b/core/storage/Makefile
@@ -1,5 +1,6 @@
 OS ?= Linux
 
+SYSCONFDIR ?= /etc
 PYTHON_QUBESPATH = $(PYTHON_SITEPATH)/qubes
 
 all:
@@ -13,6 +14,8 @@ endif
 	mkdir -p $(DESTDIR)$(PYTHON_QUBESPATH)/storage
 	cp __init__.py $(DESTDIR)$(PYTHON_QUBESPATH)/storage
 	cp __init__.py[co] $(DESTDIR)$(PYTHON_QUBESPATH)/storage
+	mkdir -p $(DESTDIR)$(SYSCONFDIR)/qubes
+	cp storage.conf $(DESTDIR)$(SYSCONFDIR)/qubes/
 ifneq ($(BACKEND_VMM),)
 	if [ -r $(BACKEND_VMM).py ]; then \
 		cp $(BACKEND_VMM).py $(DESTDIR)$(PYTHON_QUBESPATH)/storage && \
diff --git a/core/storage/__init__.py b/core/storage/__init__.py
index 795aa37a..b9810881 100644
--- a/core/storage/__init__.py
+++ b/core/storage/__init__.py
@@ -22,6 +22,7 @@
 
 from __future__ import absolute_import
 
+import ConfigParser
 import os
 import os.path
 import shutil
@@ -31,6 +32,8 @@ import sys
 import qubes.qubesutils
 from qubes.qubes import QubesException, defaults, system_path, vm_files
 
+CONFIG_FILE = '/etc/qubes/storage.conf'
+
 
 class QubesVmStorage(object):
     """
@@ -200,5 +203,97 @@ class QubesVmStorage(object):
             self.create_on_disk_private_img(verbose=False)
 
 
+def dump(o):
+    """ Returns a string represention of the given object
+
+        Args:
+            o (object): anything that response to `__module__` and `__class__`
+
+        Given the class :class:`qubes.storage.QubesVmStorage` it returns
+        'qubes.storage.QubesVmStorage' as string
+    """
+    return o.__module__ + '.' + o.__class__.__name__
+
+
+def load(string):
+    """ Given a dotted full module string representation of a class it loads it
+
+        Args:
+            string (str) i.e. 'qubes.storage.xen.QubesXenVmStorage'
+
+        Returns:
+            type
+
+        See also:
+            :func:`qubes.storage.dump`
+    """
+    if not type(string) is str:
+        # This is a hack which allows giving a real class to a vm instead of a
+        # string as string_class parameter.
+        return string
+
+    components = string.split(".")
+    module_path = ".".join(components[:-1])
+    klass = components[-1:][0]
+    module = __import__(module_path, fromlist=[klass])
+    return getattr(module, klass)
+
+
+def get_pool(vm):
+    """ Instantiates the storage for the specified vm """
+    config = _get_storage_config_parser()
+
+    name = vm.storage_pool
+    klass = _get_pool_klass(name, config)
+
+    keys = [k for k in config.options(name) if k != 'type' and k != 'class']
+    values = [config.get(name, o) for o in keys]
+    kwargs = dict(zip(keys, values))
+    return klass(vm, **kwargs)
+
+
+def _get_storage_config_parser():
+    """ Instantiates a `ConfigParaser` for specified storage config file.
+
+        Returns:
+            RawConfigParser
+    """
+    config = ConfigParser.RawConfigParser()
+    config.read(CONFIG_FILE)
+    return config
+
+
+def _get_pool_klass(name, config=None):
+    """ Returns the storage klass for the specified pool.
+
+        Args:
+            name: The pool name.
+            config: If ``config`` is not specified
+                    `_get_storage_config_parser()` is called.
+
+        Returns:
+            type: A class inheriting from `QubesVmStorage`
+    """
+    if config is None:
+        config = _get_storage_config_parser()
+
+    if not config.has_section(name):
+        raise StoragePoolException('Uknown storage pool ' + name)
+
+    if config.has_option(name, 'class'):
+        klass = load(config.get(name, 'class'))
+    elif config.has_option(name, 'type'):
+        pool_type = config.get(name, 'type')
+        klass = defaults['pool_types'][pool_type]
+
+    if klass is None:
+        raise StoragePoolException('Uknown storage pool type ' + name)
+    return klass
+
+
+class StoragePoolException(QubesException):
+    pass
+
 class Pool(object):
     pass
+
diff --git a/core/storage/storage.conf b/core/storage/storage.conf
new file mode 100644
index 00000000..8fa44c7e
--- /dev/null
+++ b/core/storage/storage.conf
@@ -0,0 +1,11 @@
+[default] ; poolname
+type=xen ; the default xen storage
+; class = qubes.storage.xen.XenStorage ; class always overwrites type
+; 
+; To use our own storage adapter, you need just to specify the module path and
+; class name
+; [pool-b]
+; class = foo.bar.MyStorage
+; 
+
+
diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec
index 86612c48..84e106ba 100644
--- a/rpm_spec/core-dom0.spec
+++ b/rpm_spec/core-dom0.spec
@@ -176,6 +176,7 @@ fi
 %files
 %defattr(-,root,root,-)
 %config(noreplace) %attr(0664,root,qubes) %{_sysconfdir}/qubes/qmemman.conf
+%config(noreplace) %attr(0664,root,qubes) %{_sysconfdir}/qubes/storage.conf
 /usr/bin/qvm-*
 /usr/bin/qubes-*
 %dir %{python_sitearch}/qubes
diff --git a/tests/Makefile b/tests/Makefile
index 8523adde..f2dff357 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -27,3 +27,5 @@ endif
 	cp regressions.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
 	cp run.py $(DESTDIR)$(PYTHON_TESTSPATH)
 	cp run.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
+	cp storage.py $(DESTDIR)$(PYTHON_TESTSPATH)
+	cp storage.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
diff --git a/tests/__init__.py b/tests/__init__.py
index 8abea4b8..8af1afbf 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -547,6 +547,7 @@ def load_tests(loader, tests, pattern):
             'qubes.tests.backup',
             'qubes.tests.backupcompatibility',
             'qubes.tests.regressions',
+            'qubes.tests.storage',
             ):
         tests.addTests(loader.loadTestsFromName(modname))
 
diff --git a/tests/storage.py b/tests/storage.py
new file mode 100644
index 00000000..30b9cd2d
--- /dev/null
+++ b/tests/storage.py
@@ -0,0 +1,63 @@
+# The Qubes OS Project, https://www.qubes-os.org/
+#
+# Copyright (C) 2015  Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+from qubes.tests import QubesTestCase, SystemTestsMixin
+from qubes.qubes import defaults
+
+import qubes.storage
+from qubes.storage.xen import QubesXenVmStorage, XenPool
+
+
+class TC_00_Storage(SystemTestsMixin, QubesTestCase):
+
+    def test_000_dump(self):
+        """ Dumps storage instance to a storage string  """
+        vmname = self.make_vm_name('appvm')
+        template = self.qc.get_default_template()
+        storage = self.qc.add_new_vm('QubesAppVm', name=vmname, pool='default',
+                                     template=template).storage
+        result = qubes.storage.dump(storage)
+        expected = 'qubes.storage.xen.QubesXenVmStorage'
+        self.assertEquals(result, expected)
+
+    def test_001_load(self):
+        """ Loads storage type from a storage string  """
+        result = qubes.storage.load('qubes.storage.xen.QubesXenVmStorage')
+        self.assertTrue(result is QubesXenVmStorage)
+
+    def test_002_default_pool_types(self):
+        """ The only predifined pool type is xen """
+        result = defaults['pool_types'].keys()
+        expected = ["xen"]
+        self.assertEquals(result, expected)
+
+    def test_003_get_pool_klass(self):
+        """ Expect the default pool to be `XenPool` """
+        result = qubes.storage._get_pool_klass('default')
+        self.assertTrue(result is XenPool)
+
+
+class TC_01_Storage(SystemTestsMixin, QubesTestCase):
+
+    def test_000_vm_use_default_pool(self):
+        vmname = self.make_vm_name('appvm')
+        template = self.qc.get_default_template()
+        vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template,
+                                pool='default')
+        self.assertIsInstance(vm.storage, QubesXenVmStorage)

From 8e0207a199f9e7a3503646acef2fac4f21e1a945 Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Sat, 7 Nov 2015 19:04:51 +0100
Subject: [PATCH 125/225] Rename QubesXenVmStorage to XenStorage

---
 core/settings-xen-Linux.py |  4 ++--
 core/storage/xen.py        | 10 +++++-----
 tests/storage.py           | 10 +++++-----
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/core/settings-xen-Linux.py b/core/settings-xen-Linux.py
index 8b23b447..30669407 100644
--- a/core/settings-xen-Linux.py
+++ b/core/settings-xen-Linux.py
@@ -2,9 +2,9 @@
 
 from __future__ import absolute_import
 
-from qubes.storage.xen import QubesXenVmStorage, XenPool
+from qubes.storage.xen import XenStorage, XenPool
 
 
 def apply(system_path, vm_files, defaults):
-    defaults['storage_class'] = QubesXenVmStorage
+    defaults['storage_class'] = XenStorage
     defaults['pool_types'] = {'xen': XenPool}
diff --git a/core/storage/xen.py b/core/storage/xen.py
index f66126a6..9c1a75cc 100644
--- a/core/storage/xen.py
+++ b/core/storage/xen.py
@@ -32,13 +32,13 @@ from qubes.qubes import QubesException, vm_files
 from qubes.storage import Pool, QubesVmStorage
 
 
-class QubesXenVmStorage(QubesVmStorage):
+class XenStorage(QubesVmStorage):
     """
     Class for VM storage of Xen VMs.
     """
 
     def __init__(self, vm, **kwargs):
-        super(QubesXenVmStorage, self).__init__(vm, **kwargs)
+        super(XenStorage, self).__init__(vm, **kwargs)
 
         self.root_dev = "xvda"
         self.private_dev = "xvdb"
@@ -158,7 +158,7 @@ class QubesXenVmStorage(QubesVmStorage):
             self.commit_template_changes()
 
     def rename(self, old_name, new_name):
-        super(QubesXenVmStorage, self).rename(old_name, new_name)
+        super(XenStorage, self).rename(old_name, new_name)
 
         old_dirpath = os.path.join(os.path.dirname(self.vmdir), old_name)
         if self.rootcow_img:
@@ -226,11 +226,11 @@ class QubesXenVmStorage(QubesVmStorage):
                 f_volatile.close()
                 f_root.close()
                 return
-        super(QubesXenVmStorage, self).reset_volatile_storage(
+        super(XenStorage, self).reset_volatile_storage(
             verbose=verbose, source_template=source_template)
 
     def prepare_for_vm_startup(self, verbose):
-        super(QubesXenVmStorage, self).prepare_for_vm_startup(verbose=verbose)
+        super(XenStorage, self).prepare_for_vm_startup(verbose=verbose)
 
         if self.drive is not None:
             (drive_type, drive_domain, drive_path) = self.drive.split(":")
diff --git a/tests/storage.py b/tests/storage.py
index 30b9cd2d..a52b62b4 100644
--- a/tests/storage.py
+++ b/tests/storage.py
@@ -21,7 +21,7 @@ from qubes.tests import QubesTestCase, SystemTestsMixin
 from qubes.qubes import defaults
 
 import qubes.storage
-from qubes.storage.xen import QubesXenVmStorage, XenPool
+from qubes.storage.xen import XenStorage, XenPool
 
 
 class TC_00_Storage(SystemTestsMixin, QubesTestCase):
@@ -33,13 +33,13 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase):
         storage = self.qc.add_new_vm('QubesAppVm', name=vmname, pool='default',
                                      template=template).storage
         result = qubes.storage.dump(storage)
-        expected = 'qubes.storage.xen.QubesXenVmStorage'
+        expected = 'qubes.storage.xen.XenStorage'
         self.assertEquals(result, expected)
 
     def test_001_load(self):
         """ Loads storage type from a storage string  """
-        result = qubes.storage.load('qubes.storage.xen.QubesXenVmStorage')
-        self.assertTrue(result is QubesXenVmStorage)
+        result = qubes.storage.load('qubes.storage.xen.XenStorage')
+        self.assertTrue(result is XenStorage)
 
     def test_002_default_pool_types(self):
         """ The only predifined pool type is xen """
@@ -60,4 +60,4 @@ class TC_01_Storage(SystemTestsMixin, QubesTestCase):
         template = self.qc.get_default_template()
         vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template,
                                 pool='default')
-        self.assertIsInstance(vm.storage, QubesXenVmStorage)
+        self.assertIsInstance(vm.storage, XenStorage)

From d1685a13df6a9114c883f35c3908db9f1e11e511 Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Sat, 7 Nov 2015 19:32:03 +0100
Subject: [PATCH 126/225] Add storage.pool_exists

---
 core/storage/__init__.py | 9 +++++++++
 tests/storage.py         | 9 +++++++++
 2 files changed, 18 insertions(+)

diff --git a/core/storage/__init__.py b/core/storage/__init__.py
index b9810881..aabb2f65 100644
--- a/core/storage/__init__.py
+++ b/core/storage/__init__.py
@@ -291,6 +291,15 @@ def _get_pool_klass(name, config=None):
     return klass
 
 
+def pool_exists(name):
+    """ Check if the specified pool exists """
+    try:
+        _get_pool_klass(name)
+        return True
+    except StoragePoolException:
+        return False
+
+
 class StoragePoolException(QubesException):
     pass
 
diff --git a/tests/storage.py b/tests/storage.py
index a52b62b4..e3fb4b29 100644
--- a/tests/storage.py
+++ b/tests/storage.py
@@ -52,6 +52,15 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase):
         result = qubes.storage._get_pool_klass('default')
         self.assertTrue(result is XenPool)
 
+    def test_004_pool_exists_default(self):
+        """ Expect the default pool to exists """
+        self.assertTrue(qubes.storage.pool_exists('default'))
+
+    def test_005_pool_exists_random(self):
+        """ Expect this pool to not a exist """
+        self.assertFalse(
+            qubes.storage.pool_exists('asdh312096r832598213iudhas'))
+
 
 class TC_01_Storage(SystemTestsMixin, QubesTestCase):
 

From 78891dd70f344bcdca2faa5f7145d642e620cc88 Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Sat, 7 Nov 2015 19:32:38 +0100
Subject: [PATCH 127/225] QubesVm save the name of the storage pool used

---
 core-modules/000QubesVm.py | 7 ++++---
 core/storage/__init__.py   | 3 +--
 core/storage/xen.py        | 8 ++++++--
 tests/storage.py           | 4 ++--
 4 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index f8fdd76c..615652db 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -40,12 +40,12 @@ import signal
 from qubes import qmemman
 from qubes import qmemman_algo
 import libvirt
-import warnings
 
 from qubes.qubes import dry_run,vmm
 from qubes.qubes import register_qubes_vm_class
 from qubes.qubes import QubesVmCollection,QubesException,QubesHost,QubesVmLabels
 from qubes.qubes import defaults,system_path,vm_files,qubes_max_qid
+from qubes.storage import get_pool
 
 qmemman_present = False
 try:
@@ -109,6 +109,7 @@ class QubesVm(object):
             "name": { "order": 1 },
             "uuid": { "order": 0, "eval": 'uuid.UUID(value) if value else None' },
             "dir_path": { "default": None, "order": 2 },
+            "pool_name": { "default":"default" },
             "conf_file": {
                 "func": lambda value: self.absolute_path(value, self.name +
                                                                  ".conf"),
@@ -198,7 +199,7 @@ class QubesVm(object):
             'kernelopts', 'services', 'installed_by_rpm',\
             'uses_default_netvm', 'include_in_backups', 'debug',\
             'qrexec_timeout', 'autostart', 'uses_default_dispvm_netvm',
-            'backup_content', 'backup_size', 'backup_path' ]:
+            'backup_content', 'backup_size', 'backup_path', 'pool_name' ]:
             attrs[prop]['save'] = lambda prop=prop: str(getattr(self, prop))
         # Simple paths
         for prop in ['conf_file', 'firewall_conf']:
@@ -345,7 +346,7 @@ class QubesVm(object):
                 self.services['qubes-update-check'] = False
 
         # Initialize VM image storage class
-        self.storage = defaults["storage_class"](self)
+        self.storage = get_pool(self.pool_name, self).getStorage()
         if hasattr(self, 'kernels_dir'):
             modules_path = os.path.join(self.kernels_dir,
                     "modules.img")
diff --git a/core/storage/__init__.py b/core/storage/__init__.py
index aabb2f65..4636dc47 100644
--- a/core/storage/__init__.py
+++ b/core/storage/__init__.py
@@ -239,11 +239,10 @@ def load(string):
     return getattr(module, klass)
 
 
-def get_pool(vm):
+def get_pool(name, vm):
     """ Instantiates the storage for the specified vm """
     config = _get_storage_config_parser()
 
-    name = vm.storage_pool
     klass = _get_pool_klass(name, config)
 
     keys = [k for k in config.options(name) if k != 'type' and k != 'class']
diff --git a/core/storage/xen.py b/core/storage/xen.py
index 9c1a75cc..01ed0277 100644
--- a/core/storage/xen.py
+++ b/core/storage/xen.py
@@ -28,7 +28,7 @@ import re
 import subprocess
 import sys
 
-from qubes.qubes import QubesException, vm_files
+from qubes.qubes import QubesException, defaults, vm_files
 from qubes.storage import Pool, QubesVmStorage
 
 
@@ -252,4 +252,8 @@ class XenStorage(QubesVmStorage):
 
 
 class XenPool(Pool):
-    pass
+    def __init__(self, vm):
+        self.vm = vm
+
+    def getStorage(self):
+        return defaults['storage_class'](self.vm)
diff --git a/tests/storage.py b/tests/storage.py
index e3fb4b29..3115535d 100644
--- a/tests/storage.py
+++ b/tests/storage.py
@@ -30,7 +30,7 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase):
         """ Dumps storage instance to a storage string  """
         vmname = self.make_vm_name('appvm')
         template = self.qc.get_default_template()
-        storage = self.qc.add_new_vm('QubesAppVm', name=vmname, pool='default',
+        storage = self.qc.add_new_vm('QubesAppVm', name=vmname, pool_name='default',
                                      template=template).storage
         result = qubes.storage.dump(storage)
         expected = 'qubes.storage.xen.XenStorage'
@@ -68,5 +68,5 @@ class TC_01_Storage(SystemTestsMixin, QubesTestCase):
         vmname = self.make_vm_name('appvm')
         template = self.qc.get_default_template()
         vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template,
-                                pool='default')
+                                pool_name='default')
         self.assertIsInstance(vm.storage, XenStorage)

From 26711e7e9aa64b18d9b8e00e567c6dc4e2f86992 Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Sat, 7 Nov 2015 20:19:17 +0100
Subject: [PATCH 128/225] Use default pool_config if pool name is 'default'

---
 core/settings-xen-Linux.py |  1 +
 core/storage/__init__.py   |  9 ++++++++-
 core/storage/xen.py        |  6 +++++-
 tests/storage.py           | 34 +++++++++++++++++++++++++++-------
 4 files changed, 41 insertions(+), 9 deletions(-)

diff --git a/core/settings-xen-Linux.py b/core/settings-xen-Linux.py
index 30669407..74503524 100644
--- a/core/settings-xen-Linux.py
+++ b/core/settings-xen-Linux.py
@@ -8,3 +8,4 @@ from qubes.storage.xen import XenStorage, XenPool
 def apply(system_path, vm_files, defaults):
     defaults['storage_class'] = XenStorage
     defaults['pool_types'] = {'xen': XenPool}
+    defaults['pool_config'] = {'dir': '/var/lib/qubes/'}
diff --git a/core/storage/__init__.py b/core/storage/__init__.py
index 4636dc47..39f623d5 100644
--- a/core/storage/__init__.py
+++ b/core/storage/__init__.py
@@ -247,7 +247,14 @@ def get_pool(name, vm):
 
     keys = [k for k in config.options(name) if k != 'type' and k != 'class']
     values = [config.get(name, o) for o in keys]
-    kwargs = dict(zip(keys, values))
+    config_kwargs = dict(zip(keys, values))
+
+    if name == 'default':
+        kwargs = defaults['pool_config'].copy()
+        kwargs.update(keys)
+    else:
+        kwargs = config_kwargs
+
     return klass(vm, **kwargs)
 
 
diff --git a/core/storage/xen.py b/core/storage/xen.py
index 01ed0277..06bd0186 100644
--- a/core/storage/xen.py
+++ b/core/storage/xen.py
@@ -252,8 +252,12 @@ class XenStorage(QubesVmStorage):
 
 
 class XenPool(Pool):
-    def __init__(self, vm):
+    def __init__(self, vm, dir):
+        assert vm is not None
+        assert dir is not None
+
         self.vm = vm
+        self.dir = dir
 
     def getStorage(self):
         return defaults['storage_class'](self.vm)
diff --git a/tests/storage.py b/tests/storage.py
index 3115535d..b5a0defc 100644
--- a/tests/storage.py
+++ b/tests/storage.py
@@ -30,8 +30,9 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase):
         """ Dumps storage instance to a storage string  """
         vmname = self.make_vm_name('appvm')
         template = self.qc.get_default_template()
-        storage = self.qc.add_new_vm('QubesAppVm', name=vmname, pool_name='default',
-                                     template=template).storage
+        vm = self.qc.add_new_vm('QubesAppVm', name=vmname,
+                                pool_name='default', template=template)
+        storage = vm.storage
         result = qubes.storage.dump(storage)
         expected = 'qubes.storage.xen.XenStorage'
         self.assertEquals(result, expected)
@@ -62,11 +63,30 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase):
             qubes.storage.pool_exists('asdh312096r832598213iudhas'))
 
 
-class TC_01_Storage(SystemTestsMixin, QubesTestCase):
+class TC_00_Pool(SystemTestsMixin, QubesTestCase):
 
-    def test_000_vm_use_default_pool(self):
+    def test000_no_pool_dir(self):
+        """ If no pool dir ist configured for a ``XenPool`` assume the default
+            `/var/lib/qubes/`.
+        """
+        vm = self._init_app_vm()
+        result = qubes.storage.get_pool("default", vm).dir
+        expected = '/var/lib/qubes/'
+        self.assertEquals(result, expected)
+
+    def test001_default_storage_class(self):
+        """ Check if when using default pool the Storage is ``XenStorage``. """
+        result = self._init_app_vm().storage
+        self.assertIsInstance(result, XenStorage)
+
+    def test_002_pool_name(self):
+        """ Default pool_name is 'default'. """
+        vm = self._init_app_vm()
+        self.assertEquals(vm.pool_name, "default")
+
+    def _init_app_vm(self):
+        """ Return initalised, but not created, AppVm. """
         vmname = self.make_vm_name('appvm')
         template = self.qc.get_default_template()
-        vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template,
-                                pool_name='default')
-        self.assertIsInstance(vm.storage, XenStorage)
+        return self.qc.add_new_vm('QubesAppVm', name=vmname, template=template,
+                                  pool_name='default')

From 16d480cf4c7dce683e48739bd581638e791eb8c1 Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Sat, 7 Nov 2015 20:53:50 +0100
Subject: [PATCH 129/225] Add storage add_pool & remove_pool

---
 core/storage/__init__.py  | 34 ++++++++++++++++++++++++++--------
 core/storage/storage.conf |  5 +++--
 tests/storage.py          | 21 +++++++++++++++++----
 3 files changed, 46 insertions(+), 14 deletions(-)

diff --git a/core/storage/__init__.py b/core/storage/__init__.py
index 39f623d5..ef8b27e0 100644
--- a/core/storage/__init__.py
+++ b/core/storage/__init__.py
@@ -258,6 +258,32 @@ def get_pool(name, vm):
     return klass(vm, **kwargs)
 
 
+def pool_exists(name):
+    """ Check if the specified pool exists """
+    try:
+        _get_pool_klass(name)
+        return True
+    except StoragePoolException:
+        return False
+
+def add_pool(name, **kwargs):
+    """ Add a storage pool to config."""
+    config = _get_storage_config_parser()
+    config.add_section(name)
+    for key, value in kwargs.iteritems():
+        config.set(name, key, value)
+    _write_config(config)
+
+def remove_pool(name):
+    """ Remove a storage pool from config file.  """
+    config = _get_storage_config_parser()
+    config.remove_section(name)
+    _write_config(config)
+
+def _write_config(config):
+    with open(CONFIG_FILE, 'w') as configfile:
+        config.write(configfile)
+
 def _get_storage_config_parser():
     """ Instantiates a `ConfigParaser` for specified storage config file.
 
@@ -297,14 +323,6 @@ def _get_pool_klass(name, config=None):
     return klass
 
 
-def pool_exists(name):
-    """ Check if the specified pool exists """
-    try:
-        _get_pool_klass(name)
-        return True
-    except StoragePoolException:
-        return False
-
 
 class StoragePoolException(QubesException):
     pass
diff --git a/core/storage/storage.conf b/core/storage/storage.conf
index 8fa44c7e..45b6bda0 100644
--- a/core/storage/storage.conf
+++ b/core/storage/storage.conf
@@ -6,6 +6,7 @@ type=xen ; the default xen storage
 ; class name
 ; [pool-b]
 ; class = foo.bar.MyStorage
-; 
-
+;
+; [test-dummy]
+; type=dummy
 
diff --git a/tests/storage.py b/tests/storage.py
index b5a0defc..d016fb1a 100644
--- a/tests/storage.py
+++ b/tests/storage.py
@@ -17,11 +17,11 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 
-from qubes.tests import QubesTestCase, SystemTestsMixin
-from qubes.qubes import defaults
-
 import qubes.storage
-from qubes.storage.xen import XenStorage, XenPool
+from qubes.qubes import defaults
+from qubes.tests import QubesTestCase, SystemTestsMixin
+
+from qubes.storage.xen import XenPool, XenStorage
 
 
 class TC_00_Storage(SystemTestsMixin, QubesTestCase):
@@ -62,6 +62,19 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase):
         self.assertFalse(
             qubes.storage.pool_exists('asdh312096r832598213iudhas'))
 
+    def test_006_add_remove_pool(self):
+        """ Tries to adding and removing a pool. """
+        pool_name = 'asdjhrp89132'
+
+        # make sure it's really does not exist
+        qubes.storage.remove_pool(pool_name)
+
+        qubes.storage.add_pool(pool_name, type='xen')
+        self.assertTrue(qubes.storage.pool_exists(pool_name))
+
+        qubes.storage.remove_pool(pool_name)
+        self.assertFalse(qubes.storage.pool_exists(pool_name))
+
 
 class TC_00_Pool(SystemTestsMixin, QubesTestCase):
 

From 58f23ca3923e6521c278b0a20f95a9d2464e3dd4 Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Sat, 7 Nov 2015 22:31:50 +0100
Subject: [PATCH 130/225] Add configurable pool_dir to XenPool

---
 core/storage/xen.py |  61 ++++++++++++++++++++--
 tests/storage.py    | 121 +++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 176 insertions(+), 6 deletions(-)

diff --git a/core/storage/xen.py b/core/storage/xen.py
index 06bd0186..853f0268 100644
--- a/core/storage/xen.py
+++ b/core/storage/xen.py
@@ -28,7 +28,9 @@ import re
 import subprocess
 import sys
 
-from qubes.qubes import QubesException, defaults, vm_files
+from qubes.qubes import (QubesAdminVm, QubesAppVm, QubesException, QubesHVm,
+                         QubesNetVm, QubesProxyVm, QubesTemplateHVm,
+                         QubesTemplateVm, defaults, vm_files)
 from qubes.storage import Pool, QubesVmStorage
 
 
@@ -37,7 +39,16 @@ class XenStorage(QubesVmStorage):
     Class for VM storage of Xen VMs.
     """
 
-    def __init__(self, vm, **kwargs):
+    def __init__(self, vm, vmdir, **kwargs):
+        """ Instantiate the storage.
+
+            Args:
+                vm: a QubesVM
+                vmdir: the root directory of the pool
+        """
+        assert vm is not None
+        assert vmdir is not None
+
         super(XenStorage, self).__init__(vm, **kwargs)
 
         self.root_dev = "xvda"
@@ -45,8 +56,11 @@ class XenStorage(QubesVmStorage):
         self.volatile_dev = "xvdc"
         self.modules_dev = "xvdd"
 
+        self.vmdir = vmdir
+
         if self.vm.is_template():
-            self.rootcow_img = os.path.join(self.vmdir, vm_files["rootcow_img"])
+            self.rootcow_img = os.path.join(self.vmdir,
+                                            vm_files["rootcow_img"])
         else:
             self.rootcow_img = None
 
@@ -252,12 +266,51 @@ class XenStorage(QubesVmStorage):
 
 
 class XenPool(Pool):
+
     def __init__(self, vm, dir):
         assert vm is not None
         assert dir is not None
 
+        if not os.path.exists(dir):
+            os.mkdir(dir)
+
+        self.vmdir = self._vmdir_path(vm, dir)
         self.vm = vm
         self.dir = dir
 
     def getStorage(self):
-        return defaults['storage_class'](self.vm)
+        """ Returns an instantiated ``XenStorage``. """
+        return defaults['storage_class'](self.vm, vmdir=self.vmdir)
+
+    def _vmdir_path(self, vm, pool_dir):
+        """ Get the vm dir depending on the type of the VM.
+
+            The default QubesOS file storage saves the vm images in three
+            different directories depending on the ``QubesVM`` type:
+
+            * ``appvms`` for ``QubesAppVm`` or ``QubesHvm``
+            * ``vm-templates`` for ``QubesTemplateVm`` or ``QubesTemplateHvm``
+            * ``servicevms`` for ``QubesProxyVm``, ``QubesNetVm`` or
+                ``QubesAdminVm``
+
+            Args:
+                vm: a QubesVM
+                pool_dir: the root directory of the pool
+
+            Returns:
+                string (str) absolute path to the directory where the vm files
+                             are stored
+        """
+        vm_type = type(vm)
+
+        if vm_type in [QubesAppVm, QubesHVm]:
+            subdir = 'appvms'
+        elif vm_type in [QubesTemplateVm, QubesTemplateHVm]:
+            subdir = 'vm-templates'
+        elif vm_type in [QubesAdminVm, QubesNetVm, QubesProxyVm]:
+            subdir = 'servicevms'
+        else:
+            raise QubesException(str(vm_type) + ' unknown vm type')
+
+        return os.path.join(pool_dir, subdir, vm.name)
+
diff --git a/tests/storage.py b/tests/storage.py
index d016fb1a..e1b408f7 100644
--- a/tests/storage.py
+++ b/tests/storage.py
@@ -17,11 +17,13 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 
+import os
+import shutil
+
 import qubes.storage
 from qubes.qubes import defaults
-from qubes.tests import QubesTestCase, SystemTestsMixin
-
 from qubes.storage.xen import XenPool, XenStorage
+from qubes.tests import QubesTestCase, SystemTestsMixin
 
 
 class TC_00_Storage(SystemTestsMixin, QubesTestCase):
@@ -103,3 +105,118 @@ class TC_00_Pool(SystemTestsMixin, QubesTestCase):
         template = self.qc.get_default_template()
         return self.qc.add_new_vm('QubesAppVm', name=vmname, template=template,
                                   pool_name='default')
+
+
+class TC_01_Pool(SystemTestsMixin, QubesTestCase):
+
+    """ Test the paths for the default Xen file based storage
+        (``QubesXenVmStorage``).
+    """
+
+    POOL_DIR = '/var/lib/qubes/test-pool'
+    APPVMS_DIR = '/var/lib/qubes/test-pool/appvms'
+    TEMPLATES_DIR = '/var/lib/qubes/test-pool/vm-templates'
+    SERVICE_DIR = '/var/lib/qubes/test-pool/servicevms'
+
+    def setUp(self):
+        """ Add a test file based storage pool """
+        super(TC_01_Pool, self).setUp()
+        qubes.storage.add_pool('test-pool', type='xen', dir=self.POOL_DIR)
+
+    def tearDown(self):
+        """ Remove the file based storage pool after testing """
+        super(TC_01_Pool, self).tearDown()
+        qubes.storage.remove_pool("test-pool")
+        shutil.rmtree(self.POOL_DIR, ignore_errors=True)
+
+    def test_001_pool_exists(self):
+        """ Check if the storage pool was added to the storage pool config """
+        self.assertTrue(qubes.storage.pool_exists('test-pool'))
+
+    def test_002_pool_dir_create(self):
+        """ Check if the storage pool dir was created """
+
+        # The dir should not exists before
+        self.assertFalse(os.path.exists(self.POOL_DIR))
+
+        vmname = self.make_vm_name('appvm')
+        template = self.qc.get_default_template()
+        self.qc.add_new_vm('QubesAppVm', name=vmname, template=template,
+                           pool_name='test-pool')
+
+        self.assertTrue(os.path.exists(self.POOL_DIR))
+
+    def test_003_pool_dir(self):
+        """ Check if the vm storage pool_dir is the same as specified """
+        vmname = self.make_vm_name('appvm')
+        template = self.qc.get_default_template()
+        vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template,
+                                pool_name='test-pool')
+        result = qubes.storage.get_pool('test-pool', vm).dir
+        self.assertEquals(self.POOL_DIR, result)
+
+    def test_004_app_vmdir(self):
+        """ Check the vm storage dir for an AppVm"""
+        vmname = self.make_vm_name('appvm')
+        template = self.qc.get_default_template()
+        vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template,
+                                pool_name='test-pool')
+
+        expected = os.path.join(self.APPVMS_DIR, vm.name)
+        result = vm.storage.vmdir
+        self.assertEquals(expected, result)
+
+    def test_005_hvm_vmdir(self):
+        """ Check the vm storage dir for a HVM"""
+        vmname = self.make_vm_name('hvm')
+        vm = self.qc.add_new_vm('QubesHVm', name=vmname,
+                                pool_name='test-pool')
+
+        expected = os.path.join(self.APPVMS_DIR, vm.name)
+        result = vm.storage.vmdir
+        self.assertEquals(expected, result)
+
+    def test_006_net_vmdir(self):
+        """ Check the vm storage dir for a Netvm"""
+        vmname = self.make_vm_name('hvm')
+        vm = self.qc.add_new_vm('QubesNetVm', name=vmname,
+                                pool_name='test-pool')
+
+        expected = os.path.join(self.SERVICE_DIR, vm.name)
+        result = vm.storage.vmdir
+        self.assertEquals(expected, result)
+
+    def test_007_proxy_vmdir(self):
+        """ Check the vm storage dir for a ProxyVm"""
+        vmname = self.make_vm_name('proxyvm')
+        vm = self.qc.add_new_vm('QubesProxyVm', name=vmname,
+                                pool_name='test-pool')
+
+        expected = os.path.join(self.SERVICE_DIR, vm.name)
+        result = vm.storage.vmdir
+        self.assertEquals(expected, result)
+
+    def test_008_admin_vmdir(self):
+        """ Check the vm storage dir for a AdminVm"""
+        # TODO How to test AdminVm?
+        pass
+
+    def test_009_template_vmdir(self):
+        """ Check the vm storage dir for a TemplateVm"""
+        vmname = self.make_vm_name('templatevm')
+        vm = self.qc.add_new_vm('QubesTemplateVm', name=vmname,
+                                pool_name='test-pool')
+
+        expected = os.path.join(self.TEMPLATES_DIR, vm.name)
+        result = vm.storage.vmdir
+        self.assertEquals(expected, result)
+
+    def test_010_template_hvm_vmdir(self):
+        """ Check the vm storage dir for a TemplateHVm"""
+        vmname = self.make_vm_name('templatehvm')
+        vm = self.qc.add_new_vm('QubesTemplateHVm', name=vmname,
+                                pool_name='test-pool')
+
+        expected = os.path.join(self.TEMPLATES_DIR, vm.name)
+        result = vm.storage.vmdir
+        self.assertEquals(expected, result)

From 0bc0bc90454ad70c9ba15dc1ec89c60bb3bb6916 Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Sat, 7 Nov 2015 22:38:34 +0100
Subject: [PATCH 131/225] XenStorage make sure subdirs exist in pool dir

---
 core/storage/xen.py | 17 +++++++++++++++--
 tests/storage.py    |  5 ++++-
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/core/storage/xen.py b/core/storage/xen.py
index 853f0268..1647eeb8 100644
--- a/core/storage/xen.py
+++ b/core/storage/xen.py
@@ -271,8 +271,14 @@ class XenPool(Pool):
         assert vm is not None
         assert dir is not None
 
-        if not os.path.exists(dir):
-            os.mkdir(dir)
+        appvms_path = os.path.join(dir, 'appvms')
+        servicevms_path = os.path.join(dir, 'servicevms')
+        vm_templates_path = os.path.join(dir, 'vm-templates')
+
+        self._create_dir_if_not_exists(dir)
+        self._create_dir_if_not_exists(appvms_path)
+        self._create_dir_if_not_exists(servicevms_path)
+        self._create_dir_if_not_exists(vm_templates_path)
 
         self.vmdir = self._vmdir_path(vm, dir)
         self.vm = vm
@@ -314,3 +320,10 @@ class XenPool(Pool):
 
         return os.path.join(pool_dir, subdir, vm.name)
 
+    def _create_dir_if_not_exists(self, path):
+        """ Check if a directory exists in if not it is created.
+
+            This method does not create any parent directories.
+        """
+        if not os.path.exists(path):
+            os.mkdir(path)
diff --git a/tests/storage.py b/tests/storage.py
index e1b408f7..8a5f949f 100644
--- a/tests/storage.py
+++ b/tests/storage.py
@@ -134,7 +134,7 @@ class TC_01_Pool(SystemTestsMixin, QubesTestCase):
         self.assertTrue(qubes.storage.pool_exists('test-pool'))
 
     def test_002_pool_dir_create(self):
-        """ Check if the storage pool dir was created """
+        """ Check if the storage pool dir and subdirs were created """
 
         # The dir should not exists before
         self.assertFalse(os.path.exists(self.POOL_DIR))
@@ -145,6 +145,9 @@ class TC_01_Pool(SystemTestsMixin, QubesTestCase):
                            pool_name='test-pool')
 
         self.assertTrue(os.path.exists(self.POOL_DIR))
+        self.assertTrue(os.path.exists(self.APPVMS_DIR))
+        self.assertTrue(os.path.exists(self.SERVICE_DIR))
+        self.assertTrue(os.path.exists(self.TEMPLATES_DIR))
 
     def test_003_pool_dir(self):
         """ Check if the vm storage pool_dir is the same as specified """

From 348030bf9c7cd5685e6d462973c9f7b35b1fc6cd Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Sat, 7 Nov 2015 19:10:55 +0100
Subject: [PATCH 132/225] Add --storage-pool option to qvm-create

---
 qvm-tools/qvm-create | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/qvm-tools/qvm-create b/qvm-tools/qvm-create
index a014febf..6ae002e5 100755
--- a/qvm-tools/qvm-create
+++ b/qvm-tools/qvm-create
@@ -41,6 +41,8 @@ def main():
                        help="Specify the label to use for the new VM (e.g. red, yellow, green, ...)")
     parser.add_option ("-p", "--proxy", action="store_true", dest="proxyvm", default=False,
                         help="Create ProxyVM")
+    parser.add_option ("-P", "--pool", dest="pool_name",
+                        help="Specify which storage pool to use")
     parser.add_option ("-H", "--hvm", action="store_true", dest="hvm", default=False,
                         help="Create HVM (standalone unless --template option used)")
     parser.add_option ("--hvm-template", action="store_true", dest="hvm_template", default=False,
@@ -71,6 +73,11 @@ def main():
         parser.error ("You must specify VM name!")
     vmname = args[0]
 
+    if options.pool_name is None:
+        pool_name = "default"
+    else :
+        pool_name = options.pool_name
+
     if (options.netvm + options.proxyvm + options.hvm + options.hvm_template) > 1:
         parser.error ("You must specify at most one VM type switch")
         
@@ -170,7 +177,9 @@ def main():
         vmtype = "QubesAppVm"
 
     try:
-        vm = qvm_collection.add_new_vm(vmtype, name=vmname, template=new_vm_template, label = label)
+        vm=qvm_collection.add_new_vm(vmtype, name=vmname,
+                                       template=new_vm_template, label=label,
+                                       pool_name=pool_name)
     except QubesException as err:
         print >> sys.stderr, "ERROR: {0}".format(err)
         exit (1)

From 76224dac8622d85a28a39113d88d5f387a936b72 Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Sun, 8 Nov 2015 19:29:41 +0100
Subject: [PATCH 133/225] Path to the Vm images is set by the storage

- This moves the logic for setting the path to the storage specific class like
  XenStore
---
 core/storage/__init__.py | 6 ------
 core/storage/xen.py      | 7 +++++++
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/core/storage/__init__.py b/core/storage/__init__.py
index ef8b27e0..f835d964 100644
--- a/core/storage/__init__.py
+++ b/core/storage/__init__.py
@@ -57,12 +57,6 @@ class QubesVmStorage(object):
         else:
             self.root_img_size = defaults['root_img_size']
 
-        self.private_img = vm.absolute_path(vm_files["private_img"], None)
-        if self.vm.template:
-            self.root_img = self.vm.template.root_img
-        else:
-            self.root_img = vm.absolute_path(vm_files["root_img"], None)
-        self.volatile_img = vm.absolute_path(vm_files["volatile_img"], None)
 
         # For now compute this path still in QubesVm
         self.modules_img = modules_img
diff --git a/core/storage/xen.py b/core/storage/xen.py
index 1647eeb8..63d856e9 100644
--- a/core/storage/xen.py
+++ b/core/storage/xen.py
@@ -64,6 +64,13 @@ class XenStorage(QubesVmStorage):
         else:
             self.rootcow_img = None
 
+        self.private_img = os.path.join(vmdir, 'private.img')
+        if self.vm.template:
+            self.root_img = self.vm.template.root_img
+        else:
+            self.root_img = os.path.join(vmdir, 'root.img')
+        self.volatile_img = os.path.join(vmdir, 'volatile.img')
+
     def _format_disk_dev(self, path, script, vdev, rw=True, type="disk", domain=None):
         if path is None:
             return ''

From 710b95a6ac203af85ff5f95dcbf81109f3d602cb Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Sun, 8 Nov 2015 21:22:44 +0100
Subject: [PATCH 134/225] Fix app icons errors

---
 core-modules/000QubesVm.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index 615652db..eef88fbf 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -347,6 +347,13 @@ class QubesVm(object):
 
         # Initialize VM image storage class
         self.storage = get_pool(self.pool_name, self).getStorage()
+        self.dir_path = self.storage.vmdir
+        self.icon_path = os.path.join(self.storage.vmdir, 'icon.png')
+        self.conf_file = os.path.join(self.storage.vmdir, self.name + '.conf')
+        self.appmenus_templates_dir = os.path.join(self.storage.vmdir, 'apps.templates')
+        self.appmenus_icons_dir = os.path.join(self.storage.vmdir, 'apps.icons')
+
+
         if hasattr(self, 'kernels_dir'):
             modules_path = os.path.join(self.kernels_dir,
                     "modules.img")

From 0ba0259d1a357cf379ccb251242543a42cece20b Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Sun, 8 Nov 2015 21:23:07 +0100
Subject: [PATCH 135/225] Extend documentation test/storage.py

---
 tests/storage.py | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/tests/storage.py b/tests/storage.py
index 8a5f949f..763d1f1a 100644
--- a/tests/storage.py
+++ b/tests/storage.py
@@ -28,6 +28,8 @@ from qubes.tests import QubesTestCase, SystemTestsMixin
 
 class TC_00_Storage(SystemTestsMixin, QubesTestCase):
 
+    """ This class tests the utility methods from :mod:``qubes.storage`` """
+
     def test_000_dump(self):
         """ Dumps storage instance to a storage string  """
         vmname = self.make_vm_name('appvm')
@@ -80,9 +82,13 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase):
 
 class TC_00_Pool(SystemTestsMixin, QubesTestCase):
 
-    def test000_no_pool_dir(self):
-        """ If no pool dir ist configured for a ``XenPool`` assume the default
-            `/var/lib/qubes/`.
+    """ This class tests some properties of the 'default' pool. """
+
+    def test000_default_pool_dir(self):
+        """ The predefined dir for the default pool should be ``/var/lib/qubes``
+
+            .. sealso::
+               Data :data:``qubes.qubes.defaults['pool_config']``.
         """
         vm = self._init_app_vm()
         result = qubes.storage.get_pool("default", vm).dir
@@ -90,11 +96,11 @@ class TC_00_Pool(SystemTestsMixin, QubesTestCase):
         self.assertEquals(result, expected)
 
     def test001_default_storage_class(self):
-        """ Check if when using default pool the Storage is ``XenStorage``. """
+        """ Check when using default pool the Storage is ``XenStorage``. """
         result = self._init_app_vm().storage
         self.assertIsInstance(result, XenStorage)
 
-    def test_002_pool_name(self):
+    def test_002_default_pool_name(self):
         """ Default pool_name is 'default'. """
         vm = self._init_app_vm()
         self.assertEquals(vm.pool_name, "default")
@@ -109,8 +115,7 @@ class TC_00_Pool(SystemTestsMixin, QubesTestCase):
 
 class TC_01_Pool(SystemTestsMixin, QubesTestCase):
 
-    """ Test the paths for the default Xen file based storage
-        (``QubesXenVmStorage``).
+    """ Test the paths for the default Xen file based storage (``XenStorage``).
     """
 
     POOL_DIR = '/var/lib/qubes/test-pool'

From 989eab1d72fab8e6ee51acc1231f241501a2af65 Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Sun, 8 Nov 2015 21:29:57 +0100
Subject: [PATCH 136/225] Add file image tests to tests/storage.py

---
 tests/storage.py | 54 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)

diff --git a/tests/storage.py b/tests/storage.py
index 763d1f1a..873d6e71 100644
--- a/tests/storage.py
+++ b/tests/storage.py
@@ -228,3 +228,57 @@ class TC_01_Pool(SystemTestsMixin, QubesTestCase):
         expected = os.path.join(self.TEMPLATES_DIR, vm.name)
         result = vm.storage.vmdir
         self.assertEquals(expected, result)
+
+    def test_011_appvm_file_images(self):
+        """ Check if all the needed image files are created for an AppVm"""
+
+        vmname = self.make_vm_name('appvm')
+        template = self.qc.get_default_template()
+        vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template,
+                                pool_name='test-pool')
+        vm.create_on_disk(verbose=False)
+
+        expected_vmdir = os.path.join(self.APPVMS_DIR, vm.name)
+        self.assertEqualsAndExists(vm.storage.vmdir, expected_vmdir)
+
+        expected_private_path = os.path.join(expected_vmdir, 'private.img')
+        self.assertEqualsAndExists(vm.storage.private_img,
+                                   expected_private_path)
+
+        expected_volatile_path = os.path.join(expected_vmdir, 'volatile.img')
+        self.assertEqualsAndExists(vm.storage.volatile_img,
+                                   expected_volatile_path)
+
+    def test_012_hvm_file_images(self):
+        """ Check if all the needed image files are created for a HVm"""
+
+        vmname = self.make_vm_name('hvm')
+        vm = self.qc.add_new_vm('QubesHVm', name=vmname,
+                                pool_name='test-pool')
+        vm.create_on_disk(verbose=False)
+
+        expected_vmdir = os.path.join(self.APPVMS_DIR, vm.name)
+        self.assertEqualsAndExists(vm.storage.vmdir, expected_vmdir)
+
+        expected_private_path = os.path.join(expected_vmdir, 'private.img')
+        self.assertEqualsAndExists(vm.storage.private_img,
+                                   expected_private_path)
+
+        expected_root_path = os.path.join(expected_vmdir, 'root.img')
+        self.assertEqualsAndExists(vm.storage.root_img, expected_root_path)
+
+        expected_volatile_path = os.path.join(expected_vmdir, 'volatile.img')
+        self.assertEqualsAndExists(vm.storage.volatile_img,
+                                   expected_volatile_path)
+
+    def assertEqualsAndExists(self, result_path, expected_path):
+        """ Check if the ``result_path``, matches ``expected_path`` and exists.
+
+            See also: :meth:``assertExist``
+        """
+        self.assertEquals(result_path, expected_path)
+        self.assertExist(result_path)
+
+    def assertExist(self, path):
+        """ Assert that the given path exists. """
+        self.assertTrue(os.path.exists(path))

From 1934f06869fc280892db593321e90737c9bbb1e3 Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Tue, 17 Nov 2015 21:15:13 +0100
Subject: [PATCH 137/225] XenStorage add DisposableVm handling

---
 core/storage/xen.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/core/storage/xen.py b/core/storage/xen.py
index 63d856e9..21c15f8c 100644
--- a/core/storage/xen.py
+++ b/core/storage/xen.py
@@ -28,9 +28,9 @@ import re
 import subprocess
 import sys
 
-from qubes.qubes import (QubesAdminVm, QubesAppVm, QubesException, QubesHVm,
-                         QubesNetVm, QubesProxyVm, QubesTemplateHVm,
-                         QubesTemplateVm, defaults, vm_files)
+from qubes.qubes import (QubesAdminVm, QubesAppVm, QubesDisposableVm,
+                         QubesException, QubesHVm, QubesNetVm, QubesProxyVm,
+                         QubesTemplateHVm, QubesTemplateVm, defaults, vm_files)
 from qubes.storage import Pool, QubesVmStorage
 
 
@@ -322,6 +322,9 @@ class XenPool(Pool):
             subdir = 'vm-templates'
         elif vm_type in [QubesAdminVm, QubesNetVm, QubesProxyVm]:
             subdir = 'servicevms'
+        elif vm_type is QubesDisposableVm:
+            subdir = 'appvms'
+            return os.path.join(pool_dir, subdir, vm.template.name + '-dvm')
         else:
             raise QubesException(str(vm_type) + ' unknown vm type')
 

From 9eee00c6d790c9513c7c92681f73987d2beeb537 Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Tue, 17 Nov 2015 21:19:15 +0100
Subject: [PATCH 138/225] QubesNetVm and subclasses use servicevm/ as vmdir

---
 core/storage/xen.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/core/storage/xen.py b/core/storage/xen.py
index 21c15f8c..e2b8482c 100644
--- a/core/storage/xen.py
+++ b/core/storage/xen.py
@@ -28,9 +28,9 @@ import re
 import subprocess
 import sys
 
-from qubes.qubes import (QubesAdminVm, QubesAppVm, QubesDisposableVm,
-                         QubesException, QubesHVm, QubesNetVm, QubesProxyVm,
-                         QubesTemplateHVm, QubesTemplateVm, defaults, vm_files)
+from qubes.qubes import (QubesAppVm, QubesDisposableVm, QubesException,
+                         QubesHVm, QubesNetVm, QubesTemplateHVm,
+                         QubesTemplateVm, defaults, vm_files)
 from qubes.storage import Pool, QubesVmStorage
 
 
@@ -303,8 +303,7 @@ class XenPool(Pool):
 
             * ``appvms`` for ``QubesAppVm`` or ``QubesHvm``
             * ``vm-templates`` for ``QubesTemplateVm`` or ``QubesTemplateHvm``
-            * ``servicevms`` for ``QubesProxyVm``, ``QubesNetVm`` or
-                ``QubesAdminVm``
+            * ``servicevms`` for any subclass of  ``QubesNetVm``
 
             Args:
                 vm: a QubesVM
@@ -320,7 +319,7 @@ class XenPool(Pool):
             subdir = 'appvms'
         elif vm_type in [QubesTemplateVm, QubesTemplateHVm]:
             subdir = 'vm-templates'
-        elif vm_type in [QubesAdminVm, QubesNetVm, QubesProxyVm]:
+        elif issubclass(vm_type, QubesNetVm):
             subdir = 'servicevms'
         elif vm_type is QubesDisposableVm:
             subdir = 'appvms'

From 885dc5cd81ca28d1354a266746d60aeb72b14ced Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Sat, 21 Nov 2015 14:46:23 +0100
Subject: [PATCH 139/225] Rename storage dir to dir_path, type to driver

Bad idea to use attributes which are named after built in functions.
---
 core/settings-xen-Linux.py |  4 ++--
 core/storage/__init__.py   | 16 +++++++---------
 core/storage/storage.conf  |  6 +++---
 core/storage/xen.py        | 16 ++++++++--------
 tests/storage.py           | 17 +++++++++--------
 5 files changed, 29 insertions(+), 30 deletions(-)

diff --git a/core/settings-xen-Linux.py b/core/settings-xen-Linux.py
index 74503524..c413e8ae 100644
--- a/core/settings-xen-Linux.py
+++ b/core/settings-xen-Linux.py
@@ -7,5 +7,5 @@ from qubes.storage.xen import XenStorage, XenPool
 
 def apply(system_path, vm_files, defaults):
     defaults['storage_class'] = XenStorage
-    defaults['pool_types'] = {'xen': XenPool}
-    defaults['pool_config'] = {'dir': '/var/lib/qubes/'}
+    defaults['pool_drivers'] = {'xen': XenPool}
+    defaults['pool_config'] = {'dir_path': '/var/lib/qubes/'}
diff --git a/core/storage/__init__.py b/core/storage/__init__.py
index f835d964..8e2ac720 100644
--- a/core/storage/__init__.py
+++ b/core/storage/__init__.py
@@ -239,7 +239,7 @@ def get_pool(name, vm):
 
     klass = _get_pool_klass(name, config)
 
-    keys = [k for k in config.options(name) if k != 'type' and k != 'class']
+    keys = [k for k in config.options(name) if k != 'driver' and k != 'class']
     values = [config.get(name, o) for o in keys]
     config_kwargs = dict(zip(keys, values))
 
@@ -308,19 +308,17 @@ def _get_pool_klass(name, config=None):
 
     if config.has_option(name, 'class'):
         klass = load(config.get(name, 'class'))
-    elif config.has_option(name, 'type'):
-        pool_type = config.get(name, 'type')
-        klass = defaults['pool_types'][pool_type]
-
-    if klass is None:
-        raise StoragePoolException('Uknown storage pool type ' + name)
+    elif config.has_option(name, 'driver'):
+        pool_driver = config.get(name, 'driver')
+        klass = defaults['pool_drivers'][pool_driver]
+    else:
+        raise StoragePoolException('Uknown storage pool driver ' + name)
     return klass
 
 
-
 class StoragePoolException(QubesException):
     pass
 
+
 class Pool(object):
     pass
-
diff --git a/core/storage/storage.conf b/core/storage/storage.conf
index 45b6bda0..e9d067e5 100644
--- a/core/storage/storage.conf
+++ b/core/storage/storage.conf
@@ -1,6 +1,6 @@
 [default] ; poolname
-type=xen ; the default xen storage
-; class = qubes.storage.xen.XenStorage ; class always overwrites type
+driver=xen ; the default xen storage
+; class = qubes.storage.xen.XenStorage ; class always overwrites the driver
 ; 
 ; To use our own storage adapter, you need just to specify the module path and
 ; class name
@@ -8,5 +8,5 @@ type=xen ; the default xen storage
 ; class = foo.bar.MyStorage
 ;
 ; [test-dummy]
-; type=dummy
+; driver=dummy
 
diff --git a/core/storage/xen.py b/core/storage/xen.py
index e2b8482c..1cd5dc97 100644
--- a/core/storage/xen.py
+++ b/core/storage/xen.py
@@ -274,22 +274,22 @@ class XenStorage(QubesVmStorage):
 
 class XenPool(Pool):
 
-    def __init__(self, vm, dir):
+    def __init__(self, vm, dir_path):
         assert vm is not None
-        assert dir is not None
+        assert dir_path is not None
 
-        appvms_path = os.path.join(dir, 'appvms')
-        servicevms_path = os.path.join(dir, 'servicevms')
-        vm_templates_path = os.path.join(dir, 'vm-templates')
+        appvms_path = os.path.join(dir_path, 'appvms')
+        servicevms_path = os.path.join(dir_path, 'servicevms')
+        vm_templates_path = os.path.join(dir_path, 'vm-templates')
 
-        self._create_dir_if_not_exists(dir)
+        self._create_dir_if_not_exists(dir_path)
         self._create_dir_if_not_exists(appvms_path)
         self._create_dir_if_not_exists(servicevms_path)
         self._create_dir_if_not_exists(vm_templates_path)
 
-        self.vmdir = self._vmdir_path(vm, dir)
+        self.vmdir = self._vmdir_path(vm, dir_path)
         self.vm = vm
-        self.dir = dir
+        self.dir_path = dir_path
 
     def getStorage(self):
         """ Returns an instantiated ``XenStorage``. """
diff --git a/tests/storage.py b/tests/storage.py
index 873d6e71..5513d383 100644
--- a/tests/storage.py
+++ b/tests/storage.py
@@ -42,13 +42,13 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase):
         self.assertEquals(result, expected)
 
     def test_001_load(self):
-        """ Loads storage type from a storage string  """
+        """ Loads storage driver from a storage string  """
         result = qubes.storage.load('qubes.storage.xen.XenStorage')
         self.assertTrue(result is XenStorage)
 
-    def test_002_default_pool_types(self):
-        """ The only predifined pool type is xen """
-        result = defaults['pool_types'].keys()
+    def test_002_default_pool_drivers(self):
+        """ The only predifined pool driver is xen """
+        result = defaults['pool_drivers'].keys()
         expected = ["xen"]
         self.assertEquals(result, expected)
 
@@ -73,7 +73,7 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase):
         # make sure it's really does not exist
         qubes.storage.remove_pool(pool_name)
 
-        qubes.storage.add_pool(pool_name, type='xen')
+        qubes.storage.add_pool(pool_name, driver='xen')
         self.assertTrue(qubes.storage.pool_exists(pool_name))
 
         qubes.storage.remove_pool(pool_name)
@@ -91,7 +91,7 @@ class TC_00_Pool(SystemTestsMixin, QubesTestCase):
                Data :data:``qubes.qubes.defaults['pool_config']``.
         """
         vm = self._init_app_vm()
-        result = qubes.storage.get_pool("default", vm).dir
+        result = qubes.storage.get_pool("default", vm).dir_path
         expected = '/var/lib/qubes/'
         self.assertEquals(result, expected)
 
@@ -126,7 +126,8 @@ class TC_01_Pool(SystemTestsMixin, QubesTestCase):
     def setUp(self):
         """ Add a test file based storage pool """
         super(TC_01_Pool, self).setUp()
-        qubes.storage.add_pool('test-pool', type='xen', dir=self.POOL_DIR)
+        qubes.storage.add_pool('test-pool', driver='xen',
+                               dir_path=self.POOL_DIR)
 
     def tearDown(self):
         """ Remove the file based storage pool after testing """
@@ -160,7 +161,7 @@ class TC_01_Pool(SystemTestsMixin, QubesTestCase):
         template = self.qc.get_default_template()
         vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template,
                                 pool_name='test-pool')
-        result = qubes.storage.get_pool('test-pool', vm).dir
+        result = qubes.storage.get_pool('test-pool', vm).dir_path
         self.assertEquals(self.POOL_DIR, result)
 
     def test_004_app_vmdir(self):

From 2ebd20049d9193b7dcc4117095c2ce8892434813 Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Sun, 22 Nov 2015 12:21:40 +0100
Subject: [PATCH 140/225] Refactor appmenus paths to core-admin-linux

---
 core-modules/000QubesVm.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index eef88fbf..de4f8851 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -350,9 +350,6 @@ class QubesVm(object):
         self.dir_path = self.storage.vmdir
         self.icon_path = os.path.join(self.storage.vmdir, 'icon.png')
         self.conf_file = os.path.join(self.storage.vmdir, self.name + '.conf')
-        self.appmenus_templates_dir = os.path.join(self.storage.vmdir, 'apps.templates')
-        self.appmenus_icons_dir = os.path.join(self.storage.vmdir, 'apps.icons')
-
 
         if hasattr(self, 'kernels_dir'):
             modules_path = os.path.join(self.kernels_dir,

From eb97c5652f371914e281618cb24b54a4841fcefe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?V=C3=ADt=20=C5=A0est=C3=A1k?= <user@fedora-21-dvm>
Date: Tue, 24 Nov 2015 18:42:53 +0100
Subject: [PATCH 141/225] libvirtError in _update_libvirt_domain is no longer
 silently swallowed.

I had some issue with fstrim and the missing else had caused the code to continue and fail later with a non-descriptive error message. This commit makes the error message more descriptive and helpful.
---
 core-modules/000QubesVm.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index f8fdd76c..f7411a97 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -732,6 +732,8 @@ class QubesVm(object):
                 raise QubesException("HVM domains not supported on this "
                                      "machine. Check BIOS settings for "
                                      "VT-x/AMD-V extensions.")
+            else:
+                raise e
         self.uuid = uuid.UUID(bytes=self._libvirt_domain.UUID())
 
     @property

From c337494ba675933773e89a4431c0dc87d7027de4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 23 Nov 2015 16:26:44 +0100
Subject: [PATCH 142/225] Add qvm-pci --add-class option for adding all devices
 of given class

This should reduce code duplication in most of initial setup like places
- currently firstboot and live image setup.
---
 doc/qvm-tools/qvm-pci.rst |  4 ++++
 qvm-tools/qvm-pci         | 49 +++++++++++++++++++++++++++++++++++----
 2 files changed, 49 insertions(+), 4 deletions(-)

diff --git a/doc/qvm-tools/qvm-pci.rst b/doc/qvm-tools/qvm-pci.rst
index c2e667f8..80e5272f 100644
--- a/doc/qvm-tools/qvm-pci.rst
+++ b/doc/qvm-tools/qvm-pci.rst
@@ -23,6 +23,10 @@ OPTIONS
     List VM PCI devices    
 -a, --add
     Add a PCI device to specified VM
+-C, --add-class
+    Add all devices of given class:
+        net - network interfaces,
+        usb - USB controllers
 -d, --delete
     Remove a PCI device from specified VM
 
diff --git a/qvm-tools/qvm-pci b/qvm-tools/qvm-pci
index 80ca4a1f..c9c715ec 100755
--- a/qvm-tools/qvm-pci
+++ b/qvm-tools/qvm-pci
@@ -27,6 +27,25 @@ import subprocess
 import os
 import sys
 from qubes.qubes import vmm
+import re
+
+
+def find_devices_of_class(klass):
+    p = subprocess.Popen(["/sbin/lspci", "-mm", "-n"], stdout=subprocess.PIPE)
+    result = p.communicate()
+    retcode = p.returncode
+    if retcode != 0:
+        print "ERROR when executing lspci!"
+        raise IOError
+
+    rx_netdev = re.compile(r"^([0-9][0-9]:[0-9][0-9].[0-9]) \"{}".format(
+        klass))
+    for dev in str(result[0]).splitlines():
+        match = rx_netdev.match(dev)
+        if match is not None:
+            dev_bdf = match.group(1)
+            assert dev_bdf is not None
+            yield dev_bdf
 
 
 def main():
@@ -39,6 +58,9 @@ def main():
     parser.add_option ("-l", "--list", action="store_true", dest="do_list", default=False)
     parser.add_option ("-a", "--add", action="store_true", dest="do_add", default=False)
     parser.add_option ("-d", "--delete", action="store_true", dest="do_delete", default=False)
+    parser.add_option("-C", "--add-class", action="store_true",
+                      dest="do_add_class", default=False,
+                      help="Add all devices of given class (net, usb)")
     parser.add_option ("--offline-mode", dest="offline_mode",
                        action="store_true", default=False,
                        help="Offline mode")
@@ -49,14 +71,15 @@ def main():
 
     vmname = args[0]
 
-    if options.do_list + options.do_add + options.do_delete > 1:
-        print >> sys.stderr, "Only one of -l -a -d is allowed!"
-        exit (1)
+    if options.do_list + options.do_add + options.do_delete + \
+            options.do_add_class > 1:
+        print >> sys.stderr, "Only one of -l -a -d -C is allowed!"
+        exit(1)
 
     if options.offline_mode:
         vmm.offline_mode = True
 
-    if options.do_add or options.do_delete:
+    if options.do_add or options.do_delete or options.add_class:
         qvm_collection = QubesVmCollection()
         qvm_collection.lock_db_for_writing()
         qvm_collection.load()
@@ -81,6 +104,24 @@ def main():
         qvm_collection.save()
         qvm_collection.unlock_db()
 
+    elif options.do_add_class:
+        if len(args) < 2:
+            print >> sys.stderr, "You must specify the PCI device class to add"
+            exit(1)
+
+        klass = args[1]
+
+        if klass == 'net':
+            devs = find_devices_of_class("02")
+        elif klass == 'usb':
+            devs = find_devices_of_class("0c03")
+        else:
+            print >> sys.stderr, "Supported classes: net, usb"
+            exit(1)
+
+        for dev in devs:
+            vm.pci_add(dev)
+
     elif options.do_delete:
         if len (args) < 2:
             print >> sys.stderr, "You must specify the PCI device to delete"

From 0c476f014d7014d93d7d2a12fce086f19a311a7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 25 Nov 2015 00:59:54 +0100
Subject: [PATCH 143/225] backup: avoid deadlock when VM process fails

If SendWorker queue is full, check if that thread is still alive.
Otherwise it would deadlock on putting an entry to that queue.
This also requires that SendWorker must ensure that the main thread
isn't currently waiting for queue space when it fails. We can do this by
simply removing an entry from a queue - so on the next iteration
SendWorker would be already dead and main thread would notice it.
Getting an entry from queue in such (error) situation is harmless,
because other checks will notice it's an error condition.

Fixes QubesOS/qubes-issues#1359
---
 core/backup.py | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/core/backup.py b/core/backup.py
index 6b20212c..c84d7b39 100644
--- a/core/backup.py
+++ b/core/backup.py
@@ -400,6 +400,10 @@ class SendWorker(Process):
                                           stdin=subprocess.PIPE,
                                           stdout=self.backup_stdout)
             if final_proc.wait() >= 2:
+                if self.queue.full():
+                    # if queue is already full, remove some entry to wake up
+                    # main thread, so it will be able to notice error
+                    self.queue.get()
                 # handle only exit code 2 (tar fatal error) or
                 # greater (call failed?)
                 raise QubesException(
@@ -448,6 +452,17 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
               crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM):
     global running_backup_operation
 
+    def queue_put_with_check(proc, vmproc, queue, element):
+        if queue.full():
+            if not proc.is_alive():
+                if vmproc:
+                    message = ("Failed to write the backup, VM output:\n" +
+                               vmproc.stderr.read())
+                else:
+                    message = "Failed to write the backup. Out of disk space?"
+                raise QubesException(message)
+        queue.put(element)
+
     total_backup_sz = 0
     passphrase = passphrase.encode('utf-8')
     for f in files_to_backup:
@@ -650,7 +665,9 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
                                          run_error)
 
             # Send the chunk to the backup target
-            to_send.put(os.path.relpath(chunkfile, backup_tmpdir))
+            queue_put_with_check(
+                send_proc, vmproc, to_send,
+                os.path.relpath(chunkfile, backup_tmpdir))
 
             # Close HMAC
             hmac.stdin.close()
@@ -668,7 +685,9 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
             hmac_file.close()
 
             # Send the HMAC to the backup target
-            to_send.put(os.path.relpath(chunkfile, backup_tmpdir) + ".hmac")
+            queue_put_with_check(
+                send_proc, vmproc, to_send,
+                os.path.relpath(chunkfile, backup_tmpdir) + ".hmac")
 
             if tar_sparse.poll() is None or run_error == "size_limit":
                 run_error = "paused"
@@ -680,7 +699,7 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
                         .poll()
         pipe.close()
 
-    to_send.put("FINISHED")
+    queue_put_with_check(send_proc, vmproc, to_send, "FINISHED")
     send_proc.join()
     shutil.rmtree(backup_tmpdir)
 

From 760786d999127ca226d78953edd76c69be9fb133 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 25 Nov 2015 01:08:15 +0100
Subject: [PATCH 144/225] tests: minor formating

---
 tests/__init__.py | 65 ++++++++++++++++++++++-------------------------
 1 file changed, 30 insertions(+), 35 deletions(-)

diff --git a/tests/__init__.py b/tests/__init__.py
index 8af1afbf..563ddf28 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -133,14 +133,12 @@ class QubesTestCase(unittest.TestCase):
             self.__class__.__name__,
             self._testMethodName))
 
-
     def __str__(self):
         return '{}/{}/{}'.format(
             '.'.join(self.__class__.__module__.split('.')[2:]),
             self.__class__.__name__,
             self._testMethodName)
 
-
     def tearDown(self):
         super(QubesTestCase, self).tearDown()
 
@@ -153,7 +151,6 @@ class QubesTestCase(unittest.TestCase):
                 and filter((lambda (tc, exc): tc is self), l):
             raise BeforeCleanExit()
 
-
     def assertNotRaises(self, excClass, callableObj=None, *args, **kwargs):
         """Fail if an exception of class excClass is raised
            by callableObj when invoked with arguments args and keyword
@@ -183,15 +180,14 @@ class QubesTestCase(unittest.TestCase):
         with context:
             callableObj(*args, **kwargs)
 
-
     def assertXMLEqual(self, xml1, xml2):
-        '''Check for equality of two XML objects.
+        """Check for equality of two XML objects.
 
         :param xml1: first element
         :param xml2: second element
         :type xml1: :py:class:`lxml.etree._Element`
         :type xml2: :py:class:`lxml.etree._Element`
-        ''' # pylint: disable=invalid-name
+        """  # pylint: disable=invalid-name
 
         self.assertEqual(xml1.tag, xml2.tag)
         self.assertEqual(xml1.text, xml2.text)
@@ -202,13 +198,13 @@ class QubesTestCase(unittest.TestCase):
 
 class SystemTestsMixin(object):
     def setUp(self):
-        '''Set up the test.
+        """Set up the test.
 
         .. warning::
             This method instantiates QubesVmCollection acquires write lock for
             it. You can use is as :py:attr:`qc`. You can (and probably
             should) release the lock at the end of setUp in subclass
-        '''
+        """
 
         super(SystemTestsMixin, self).setUp()
 
@@ -220,12 +216,13 @@ class SystemTestsMixin(object):
 
         self.remove_test_vms()
 
-
     def tearDown(self):
         super(SystemTestsMixin, self).tearDown()
 
-        try: self.qc.lock_db_for_writing()
-        except qubes.qubes.QubesException: pass
+        try:
+            self.qc.lock_db_for_writing()
+        except qubes.qubes.QubesException:
+            pass
         self.qc.load()
 
         self.remove_test_vms()
@@ -236,7 +233,6 @@ class SystemTestsMixin(object):
 
         self.conn.close()
 
-
     def make_vm_name(self, name):
         return VMPREFIX + name
 
@@ -253,13 +249,18 @@ class SystemTestsMixin(object):
             # XXX .is_running() may throw libvirtError if undefined
             if vm.is_running():
                 vm.force_shutdown()
-        except: pass
+        except:
+            pass
 
-        try: vm.remove_from_disk()
-        except: pass
+        try:
+            vm.remove_from_disk()
+        except:
+            pass
 
-        try: vm.libvirt_domain.undefine()
-        except libvirt.libvirtError: pass
+        try:
+            vm.libvirt_domain.undefine()
+        except libvirt.libvirtError:
+            pass
 
         self.qc.pop(vm.qid)
         del vm
@@ -268,13 +269,13 @@ class SystemTestsMixin(object):
         # for example if vm.libvirtDomain malfunctioned.
         try:
             dom = self.conn.lookupByName(vmname)
-        except: pass
+        except:
+            pass
         else:
             self._remove_vm_libvirt(dom)
 
         self._remove_vm_disk(vmname)
 
-
     def _remove_vm_libvirt(self, dom):
         try:
             dom.destroy()
@@ -282,7 +283,6 @@ class SystemTestsMixin(object):
             pass
         dom.undefine()
 
-
     def _remove_vm_disk(self, vmname):
         for dirspec in (
                 'qubes_appvms_dir',
@@ -296,21 +296,19 @@ class SystemTestsMixin(object):
                 else:
                     os.unlink(dirpath)
 
-
     def remove_vms(self, vms):
         for vm in vms: self._remove_vm_qubes(vm)
         self.save_and_reload_db()
 
-
     def remove_test_vms(self):
-        '''Aggresively remove any domain that has name in testing namespace.
+        """Aggresively remove any domain that has name in testing namespace.
 
         .. warning::
             The test suite hereby claims any domain whose name starts with
             :py:data:`VMPREFIX` as fair game. This is needed to enforce sane
             test executing environment. If you have domains named ``test-*``,
             don't run the tests.
-        '''
+        """
 
         # first, remove them Qubes-way
         something_removed = False
@@ -392,28 +390,23 @@ class BackupTestsMixin(SystemTestsMixin):
             shutil.rmtree(self.backupdir)
         os.mkdir(self.backupdir)
 
-
     def tearDown(self):
         super(BackupTestsMixin, self).tearDown()
         shutil.rmtree(self.backupdir)
 
-
     def print_progress(self, progress):
         if self.verbose:
             print >> sys.stderr, "\r-> Backing up files: {0}%...".format(progress)
 
-
     def error_callback(self, message):
         self.error_detected.put(message)
         if self.verbose:
             print >> sys.stderr, "ERROR: {0}".format(message)
 
-
     def print_callback(self, msg):
         if self.verbose:
             print msg
 
-
     def fill_image(self, path, size=None, sparse=False):
         block_size = 4096
 
@@ -432,7 +425,6 @@ class BackupTestsMixin(SystemTestsMixin):
 
         f.close()
 
-
     # NOTE: this was create_basic_vms
     def create_backup_vms(self):
         template=self.qc.get_default_template()
@@ -459,7 +451,6 @@ class BackupTestsMixin(SystemTestsMixin):
 
         return vms
 
-
     def make_backup(self, vms, prepare_kwargs=dict(), do_kwargs=dict(),
             target=None):
         # XXX: bakup_prepare and backup_do don't support host_collection
@@ -472,19 +463,24 @@ class BackupTestsMixin(SystemTestsMixin):
                                       print_callback=self.print_callback,
                                       **prepare_kwargs)
         except qubes.qubes.QubesException as e:
-            self.fail("QubesException during backup_prepare: %s" % str(e))
+            if not expect_failure:
+                self.fail("QubesException during backup_prepare: %s" % str(e))
+            else:
+                raise
 
         try:
             qubes.backup.backup_do(target, files_to_backup, "qubes",
                              progress_callback=self.print_progress,
                              **do_kwargs)
         except qubes.qubes.QubesException as e:
-            self.fail("QubesException during backup_do: %s" % str(e))
+            if not expect_failure:
+                self.fail("QubesException during backup_do: %s" % str(e))
+            else:
+                raise
 
         self.qc.lock_db_for_writing()
         self.qc.load()
 
-
     def restore_backup(self, source=None, appvm=None, options=None,
                        expect_errors=None):
         if source is None:
@@ -528,7 +524,6 @@ class BackupTestsMixin(SystemTestsMixin):
         if not appvm and not os.path.isdir(backupfile):
             os.unlink(backupfile)
 
-
     def create_sparse(self, path, size):
         f = open(path, "w")
         f.truncate(size)

From 736e2788658eb13075d3935af998f5e6a00e0e29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 25 Nov 2015 01:08:57 +0100
Subject: [PATCH 145/225] tests: fix wait_for_window function

The `show` parameter wasn't handled (show=False was assumed...)
---
 tests/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/__init__.py b/tests/__init__.py
index 563ddf28..8a254cff 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -353,7 +353,7 @@ class SystemTestsMixin(object):
         wait_count = 0
         while subprocess.call(['xdotool', 'search', '--name', title],
                               stdout=open(os.path.devnull, 'w'),
-                              stderr=subprocess.STDOUT) == 0:
+                              stderr=subprocess.STDOUT) == int(show):
             wait_count += 1
             if wait_count > timeout*10:
                 self.fail("Timeout while waiting for {} window to {}".format(

From db35e03aa76a6a381a25396c76e6805a59da84b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 25 Nov 2015 01:14:53 +0100
Subject: [PATCH 146/225] tests: backup to VM tests

Regression tests for QubesOS/qubes-issues#1371 and
QubesOS/qubes-issues#1359

Fixes QubesOS/qubes-issues#1435
---
 tests/__init__.py |   8 +---
 tests/backup.py   | 110 +++++++++++++++++++++++++++++++++++++++-------
 2 files changed, 94 insertions(+), 24 deletions(-)

diff --git a/tests/__init__.py b/tests/__init__.py
index 8a254cff..b46a83c6 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -379,12 +379,6 @@ class BackupTestsMixin(SystemTestsMixin):
         if self.verbose:
             print >>sys.stderr, "-> Creating backupvm"
 
-        # TODO: allow non-default template
-        self.backupvm = self.qc.add_new_vm("QubesAppVm",
-            name=self.make_vm_name('backupvm'),
-            template=self.qc.get_default_template())
-        self.backupvm.create_on_disk(verbose=self.verbose)
-
         self.backupdir = os.path.join(os.environ["HOME"], "test-backup")
         if os.path.exists(self.backupdir):
             shutil.rmtree(self.backupdir)
@@ -452,7 +446,7 @@ class BackupTestsMixin(SystemTestsMixin):
         return vms
 
     def make_backup(self, vms, prepare_kwargs=dict(), do_kwargs=dict(),
-            target=None):
+                    target=None, expect_failure=False):
         # XXX: bakup_prepare and backup_do don't support host_collection
         self.qc.unlock_db()
         if target is None:
diff --git a/tests/backup.py b/tests/backup.py
index 15f63014..4ed6bfe0 100644
--- a/tests/backup.py
+++ b/tests/backup.py
@@ -28,7 +28,7 @@ import os
 
 import unittest
 import sys
-
+from qubes.qubes import QubesException, QubesTemplateVm
 import qubes.tests
 
 class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
@@ -85,22 +85,6 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
         self.restore_backup()
         self.remove_vms(vms)
 
-
-    # TODO: iterate over templates
-    def test_100_send_to_vm(self):
-        vms = self.create_backup_vms()
-        self.backupvm.start()
-        self.make_backup(vms,
-                         do_kwargs={
-                             'appvm': self.backupvm,
-                             'compressed': True,
-                             'encrypted': True},
-                         target='dd of=/var/tmp/backup-test')
-        self.remove_vms(vms)
-        self.restore_backup(source='dd if=/var/tmp/backup-test',
-                            appvm=self.backupvm)
-        self.remove_vms(vms)
-
     def test_200_restore_over_existing_directory(self):
         """
         Regression test for #1386
@@ -119,3 +103,95 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
                     test_dir)
             ])
         self.remove_vms(vms)
+
+class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin):
+    def setUp(self):
+        super(TC_10_BackupVMMixin, self).setUp()
+        self.backupvm = self.qc.add_new_vm(
+            "QubesAppVm",
+            name=self.make_vm_name('backupvm'),
+            template=self.qc.get_vm_by_name(self.template)
+        )
+        self.backupvm.create_on_disk(verbose=self.verbose)
+
+    def test_100_send_to_vm_file_with_spaces(self):
+        vms = self.create_backup_vms()
+        self.backupvm.start()
+        self.backupvm.run("mkdir '/var/tmp/backup directory'", wait=True)
+        self.make_backup(vms,
+                         do_kwargs={
+                             'appvm': self.backupvm,
+                             'compressed': True,
+                             'encrypted': True},
+                         target='/var/tmp/backup directory')
+        self.remove_vms(vms)
+        p = self.backupvm.run("ls /var/tmp/backup*/qubes-backup*",
+                              passio_popen=True)
+        (backup_path, _) = p.communicate()
+        backup_path = backup_path.strip()
+        self.restore_backup(source=backup_path,
+                            appvm=self.backupvm)
+        self.remove_vms(vms)
+
+    def test_110_send_to_vm_command(self):
+        vms = self.create_backup_vms()
+        self.backupvm.start()
+        self.make_backup(vms,
+                         do_kwargs={
+                             'appvm': self.backupvm,
+                             'compressed': True,
+                             'encrypted': True},
+                         target='dd of=/var/tmp/backup-test')
+        self.remove_vms(vms)
+        self.restore_backup(source='dd if=/var/tmp/backup-test',
+                            appvm=self.backupvm)
+        self.remove_vms(vms)
+
+    def test_110_send_to_vm_no_space(self):
+        """
+        Check whether backup properly report failure when no enough space is
+        available
+        :return:
+        """
+        vms = self.create_backup_vms()
+        self.backupvm.start()
+        retcode = self.backupvm.run(
+            "truncate -s 50M /home/user/backup.img && "
+            "mkfs.ext4 -F /home/user/backup.img && "
+            "mkdir /home/user/backup && "
+            "mount /home/user/backup.img /home/user/backup -o loop &&"
+            "chmod 777 /home/user/backup",
+            user="root", wait=True)
+        if retcode != 0:
+            raise RuntimeError("Failed to prepare backup directory")
+        with self.assertRaises(QubesException):
+            self.make_backup(vms,
+                             do_kwargs={
+                                 'appvm': self.backupvm,
+                                 'compressed': False,
+                                 'encrypted': True},
+                             target='/home/user/backup',
+                             expect_failure=True)
+        self.qc.lock_db_for_writing()
+        self.qc.load()
+        self.remove_vms(vms)
+
+
+def load_tests(loader, tests, pattern):
+    try:
+        qc = qubes.qubes.QubesVmCollection()
+        qc.lock_db_for_reading()
+        qc.load()
+        qc.unlock_db()
+        templates = [vm.name for vm in qc.values() if
+                     isinstance(vm, QubesTemplateVm)]
+    except OSError:
+        templates = []
+    for template in templates:
+        tests.addTests(loader.loadTestsFromTestCase(
+            type(
+                'TC_10_BackupVM_' + template,
+                (TC_10_BackupVMMixin, qubes.tests.QubesTestCase),
+                {'template': template})))
+
+    return tests

From d7ab2d985dda391a85a379d477537f93781859a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 25 Nov 2015 01:18:05 +0100
Subject: [PATCH 147/225] tests: use wait_for_window/enter_keys_in_window
 wrappers when applicable

Reduce code duplication.
---
 tests/vm_qrexec_gui.py | 20 +++++---------------
 1 file changed, 5 insertions(+), 15 deletions(-)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index 6d62f6f8..5b7661c2 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -555,8 +555,7 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
                              self.testvm2.name, passio_popen=True,
                              passio_stderr=True)
         # Confirm transfer
-        subprocess.check_call(
-            ['xdotool', 'search', '--sync', '--name', 'Question', 'key', 'y'])
+        self.enter_keys_in_window('Question', ['y'])
         p.wait()
         self.assertEqual(p.returncode, 0, "qvm-copy-to-vm failed: %s" %
                          p.stderr.read())
@@ -856,23 +855,14 @@ class TC_20_DispVMMixin(qubes.tests.SystemTestsMixin):
         try:
             window_title = 'user@%s' % (dispvm.template.name + "-dvm")
             p.stdin.write("xterm -e "
-                          "\"sh -s -c 'echo \\\"\033]0;{}\007\\\";read;'\"\n".
+                          "\"sh -s -c 'echo \\\"\033]0;{}\007\\\";read x;'\"\n".
                           format(window_title))
             self.wait_for_window(window_title)
 
             time.sleep(0.5)
-            subprocess.check_call(['xdotool', 'search', '--name', window_title,
-                                  'windowactivate', 'key', 'Return'])
-
-            wait_count = 0
-            while subprocess.call(['xdotool', 'search', '--name', window_title],
-                                  stdout=open(os.path.devnull, 'w'),
-                                  stderr=subprocess.STDOUT) == 0:
-                wait_count += 1
-                if wait_count > 100:
-                    self.fail("Timeout while waiting for gnome-terminal "
-                              "termination")
-                time.sleep(0.1)
+            self.enter_keys_in_window(window_title, ['Return'])
+            # Wait for window to close
+            self.wait_for_window(window_title, show=False)
         finally:
             p.stdin.close()
 

From 564ea5d64c7444cb7e836fdbe3356c8d62dc9928 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 25 Nov 2015 01:19:42 +0100
Subject: [PATCH 148/225] tests: check if qvm-move-to-vm properly fails in case
 of out of disk space

Regression test for QubesOS/qubes-issues#1355
---
 tests/vm_qrexec_gui.py | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index 5b7661c2..d7d442d9 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -603,6 +603,41 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
             wait=True)
         self.assertEqual(retcode, 0, "file differs")
 
+    @unittest.skipUnless(spawn.find_executable('xdotool'),
+                         "xdotool not installed")
+    def test_130_qrexec_filemove_disk_full(self):
+        self.testvm1.start()
+        self.testvm2.start()
+        # Prepare test file
+        prepare_cmd = ("yes teststring | dd of=testfile bs=1M "
+                       "count=50 iflag=fullblock")
+        retcode = self.testvm1.run(prepare_cmd, wait=True)
+        if retcode != 0:
+            raise RuntimeError("Failed '{}' in {}".format(prepare_cmd,
+                                                          self.testvm1.name))
+        # Prepare target directory with limited size
+        prepare_cmd = (
+            "mkdir -p /home/user/QubesIncoming && "
+            "chown user /home/user/QubesIncoming && "
+            "mount -t tmpfs none /home/user/QubesIncoming -o size=48M"
+        )
+        retcode = self.testvm2.run(prepare_cmd, user="root", wait=True)
+        if retcode != 0:
+            raise RuntimeError("Failed '{}' in {}".format(prepare_cmd,
+                                                          self.testvm2.name))
+        p = self.testvm1.run("qvm-move-to-vm %s testfile" %
+                             self.testvm2.name, passio_popen=True,
+                             passio_stderr=True)
+        # Confirm transfer
+        self.enter_keys_in_window('Question', ['y'])
+        # Close GUI error message
+        self.enter_keys_in_window('Error', ['Return'])
+        p.wait()
+        self.assertEqual(p.returncode, 255, "qvm-move-to-vm should fail")
+        retcode = self.testvm1.run("test -f testfile", wait=True)
+        self.assertEqual(retcode, 0, "testfile should not be deleted in "
+                                     "source VM")
+
     def test_200_timezone(self):
         """Test whether timezone setting is properly propagated to the VM"""
         if "whonix" in self.template:

From de275cc917771b3a5a417ffd85060c547e7a710d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 25 Nov 2015 01:24:00 +0100
Subject: [PATCH 149/225] core: leave volatile.img structure creation for the
 VM

This is already handled by a script in initramfs.

Fixes QubesOS/qubes-issues#1308
---
 linux/aux-tools/prepare-volatile-img.sh | 25 -------------------------
 1 file changed, 25 deletions(-)

diff --git a/linux/aux-tools/prepare-volatile-img.sh b/linux/aux-tools/prepare-volatile-img.sh
index 33dfc9b9..40e22f5d 100755
--- a/linux/aux-tools/prepare-volatile-img.sh
+++ b/linux/aux-tools/prepare-volatile-img.sh
@@ -26,28 +26,3 @@ fi
 
 TOTAL_SIZE=$[ $ROOT_SIZE + $SWAP_SIZE + 512 ]
 truncate -s ${TOTAL_SIZE}M "$FILENAME"
-sfdisk --no-reread -u M "$FILENAME" > /dev/null 2> /dev/null <<EOF
-0,${SWAP_SIZE},S
-,${ROOT_SIZE},L
-EOF
-
-(
-	flock 200
-	loopdev=`losetup -f --show --partscan "$FILENAME"`
-	udevadm settle
-	created=
-	if [ ! -e ${loopdev}p1 ]; then
-		# device wasn't created automatically, probably udev isn't running;
-		# create devs manually
-		for partdev in /sys/block/$(basename ${loopdev})/loop*p*; do
-			mknod /dev/$(basename ${partdev}) b $(cat ${partdev}/dev | tr : ' ')
-		done
-		created=yes
-	fi
-	mkswap -f ${loopdev}p1 > /dev/null
-	if [ "$created" = "yes" ]; then
-		rm -f ${loopdev}p*
-	fi
-	losetup -d ${loopdev} || :
-	chown --reference `dirname "$FILENAME"` "$FILENAME"
-) 200>"/var/run/qubes/prepare-volatile-img.lock"

From a14e1cc86ff838829e2da955ce9c225010fbe7e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 25 Nov 2015 03:14:23 +0100
Subject: [PATCH 150/225] tests: move enter_keys_in_window to base
 SystemTestsMixin class

This ways it will be available also in DispVM tests (already referenced
there), and others.
---
 tests/__init__.py      | 18 ++++++++++++++++++
 tests/vm_qrexec_gui.py | 18 ------------------
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/tests/__init__.py b/tests/__init__.py
index b46a83c6..f75be979 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -361,6 +361,24 @@ class SystemTestsMixin(object):
                 )
             time.sleep(0.1)
 
+    def enter_keys_in_window(self, title, keys):
+        """
+        Search for window with given title, then enter listed keys there.
+        The function will wait for said window to appear.
+
+        :param title: title of window
+        :param keys: list of keys to enter, as for `xdotool key`
+        :return: None
+        """
+
+        # 'xdotool search --sync' sometimes crashes on some race when
+        # accessing window properties
+        self.wait_for_window(title)
+        command = ['xdotool', 'search', '--name', title,
+                   'windowactivate',
+                   'key'] + keys
+        subprocess.check_call(command)
+
     def shutdown_and_wait(self, vm, timeout=60):
         vm.shutdown()
         while timeout > 0:
diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index d7d442d9..f601d3d3 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -54,24 +54,6 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
         self.testvm1 = self.qc[self.testvm1.qid]
         self.testvm2 = self.qc[self.testvm2.qid]
 
-    def enter_keys_in_window(self, title, keys):
-        """
-        Search for window with given title, then enter listed keys there.
-        The function will wait for said window to appear.
-
-        :param title: title of window
-        :param keys: list of keys to enter, as for `xdotool key`
-        :return: None
-        """
-
-        # 'xdotool search --sync' sometimes crashes on some race when
-        # accessing window properties
-        self.wait_for_window(title)
-        command = ['xdotool', 'search', '--name', title,
-                   'windowactivate',
-                   'key'] + keys
-        subprocess.check_call(command)
-
     def test_000_start_shutdown(self):
         self.testvm1.start()
         self.assertEquals(self.testvm1.get_power_state(), "Running")

From 64343572c1c1555d4ae44c978dc7d5302e6f2c10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 25 Nov 2015 03:15:23 +0100
Subject: [PATCH 151/225] tests: handle older `df` version in resize
 private.img tests

df --output=size isn't available in Debian (7, 8).
---
 tests/vm_qrexec_gui.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index f601d3d3..5d9a9f9f 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -693,7 +693,9 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
         # First offline test
         self.testvm1.resize_private_img(4*1024**3)
         self.testvm1.start()
-        p = self.testvm1.run('df --output=size /rw|tail -n 1',
+        df_cmd = '( df --output=size /rw || df /rw | awk \'{print $2}\' )|' \
+                 'tail -n 1'
+        p = self.testvm1.run(df_cmd,
                              passio_popen=True)
         # new_size in 1k-blocks
         (new_size, _) = p.communicate()
@@ -701,7 +703,7 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
         self.assertGreater(int(new_size.strip()), 3.8*1024**2)
         # Then online test
         self.testvm1.resize_private_img(6*1024**3)
-        p = self.testvm1.run('df --output=size /rw|tail -n 1',
+        p = self.testvm1.run(df_cmd,
                              passio_popen=True)
         # new_size in 1k-blocks
         (new_size, _) = p.communicate()

From aeec42dacf12186e67fed9f85d57ab6bb42f8b1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 25 Nov 2015 03:16:24 +0100
Subject: [PATCH 152/225] tests: PV Grub: ensure that latest packages are
 installed

Flush yum case before that.
---
 tests/vm_qrexec_gui.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index 5d9a9f9f..c64da4d4 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -1070,7 +1070,8 @@ class TC_40_PVGrub(qubes.tests.SystemTestsMixin):
 
     def install_packages(self, vm):
         if self.template.startswith('fedora-'):
-            cmd_install1 = 'yum install -y qubes-kernel-vm-support grub2-tools'
+            cmd_install1 = 'yum clean expire-cache && ' \
+                           'yum install -y qubes-kernel-vm-support grub2-tools'
             cmd_install2 = 'yum install -y kernel kernel-devel'
             cmd_update_grub = 'grub2-mkconfig -o /boot/grub2/grub.cfg'
         elif self.template.startswith('debian-'):

From feaaaa75fad902f05204a3e632318d4150432492 Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Sat, 21 Nov 2015 13:22:27 +0100
Subject: [PATCH 153/225] Move the vmdir logic from XenPool to Pool

Any storage implementation needs this logic for saving the vm config and
`*.desktop` files.
---
 core/storage/__init__.py | 67 ++++++++++++++++++++++++++++++++++++++--
 core/storage/xen.py      | 64 ++------------------------------------
 2 files changed, 68 insertions(+), 63 deletions(-)

diff --git a/core/storage/__init__.py b/core/storage/__init__.py
index 8e2ac720..0c302082 100644
--- a/core/storage/__init__.py
+++ b/core/storage/__init__.py
@@ -30,7 +30,7 @@ import subprocess
 import sys
 
 import qubes.qubesutils
-from qubes.qubes import QubesException, defaults, system_path, vm_files
+from qubes.qubes import QubesException, defaults, system_path
 
 CONFIG_FILE = '/etc/qubes/storage.conf'
 
@@ -321,4 +321,67 @@ class StoragePoolException(QubesException):
 
 
 class Pool(object):
-    pass
+    def __init__(self, vm, dir_path):
+        assert vm is not None
+        assert dir_path is not None
+
+        self.vm = vm
+        self.dir_path = dir_path
+
+        self.create_dir_if_not_exists(self.dir_path)
+
+        self.vmdir = self.vmdir_path(vm, self.dir_path)
+
+        appvms_path = os.path.join(self.dir_path, 'appvms')
+        self.create_dir_if_not_exists(appvms_path)
+
+        servicevms_path = os.path.join(self.dir_path, 'servicevms')
+        self.create_dir_if_not_exists(servicevms_path)
+
+        vm_templates_path = os.path.join(self.dir_path, 'vm-templates')
+        self.create_dir_if_not_exists(vm_templates_path)
+
+    def vmdir_path(self, vm, pool_dir):
+        """ Returns the path to vmdir depending on the type of the VM.
+
+            The default QubesOS file storage saves the vm images in three
+            different directories depending on the ``QubesVM`` type:
+
+            * ``appvms`` for ``QubesAppVm`` or ``QubesHvm``
+            * ``vm-templates`` for ``QubesTemplateVm`` or ``QubesTemplateHvm``
+            * ``servicevms`` for any subclass of  ``QubesNetVm``
+
+            Args:
+                vm: a QubesVM
+                pool_dir: the root directory of the pool
+
+            Returns:
+                string (str) absolute path to the directory where the vm files
+                             are stored
+        """
+        # TODO: This is a hack, circular dependencies problem?
+        from qubes.qubes import (QubesAppVm, QubesDisposableVm, QubesHVm, 
+                                 QubesNetVm, QubesTemplateHVm, QubesTemplateVm)
+        vm_type = type(vm)
+
+        if vm_type in [QubesAppVm, QubesHVm]:
+            subdir = 'appvms'
+        elif vm_type in [QubesTemplateVm, QubesTemplateHVm]:
+            subdir = 'vm-templates'
+        elif issubclass(vm_type, QubesNetVm):
+            subdir = 'servicevms'
+        elif vm_type is QubesDisposableVm:
+            subdir = 'appvms'
+            return os.path.join(pool_dir, subdir, vm.template.name + '-dvm')
+        else:
+            raise QubesException(str(vm_type) + ' unknown vm type')
+
+        return os.path.join(pool_dir, subdir, vm.name)
+
+    def create_dir_if_not_exists(self, path):
+        """ Check if a directory exists in if not create it.
+
+            This method does not create any parent directories.
+        """
+        if not os.path.exists(path):
+            os.mkdir(path)
diff --git a/core/storage/xen.py b/core/storage/xen.py
index 1cd5dc97..11ff9b67 100644
--- a/core/storage/xen.py
+++ b/core/storage/xen.py
@@ -28,9 +28,7 @@ import re
 import subprocess
 import sys
 
-from qubes.qubes import (QubesAppVm, QubesDisposableVm, QubesException,
-                         QubesHVm, QubesNetVm, QubesTemplateHVm,
-                         QubesTemplateVm, defaults, vm_files)
+from qubes.qubes import QubesException, vm_files
 from qubes.storage import Pool, QubesVmStorage
 
 
@@ -275,64 +273,8 @@ class XenStorage(QubesVmStorage):
 class XenPool(Pool):
 
     def __init__(self, vm, dir_path):
-        assert vm is not None
-        assert dir_path is not None
-
-        appvms_path = os.path.join(dir_path, 'appvms')
-        servicevms_path = os.path.join(dir_path, 'servicevms')
-        vm_templates_path = os.path.join(dir_path, 'vm-templates')
-
-        self._create_dir_if_not_exists(dir_path)
-        self._create_dir_if_not_exists(appvms_path)
-        self._create_dir_if_not_exists(servicevms_path)
-        self._create_dir_if_not_exists(vm_templates_path)
-
-        self.vmdir = self._vmdir_path(vm, dir_path)
-        self.vm = vm
-        self.dir_path = dir_path
+        super(XenPool, self).__init__(vm, dir_path)
 
     def getStorage(self):
         """ Returns an instantiated ``XenStorage``. """
-        return defaults['storage_class'](self.vm, vmdir=self.vmdir)
-
-    def _vmdir_path(self, vm, pool_dir):
-        """ Get the vm dir depending on the type of the VM.
-
-            The default QubesOS file storage saves the vm images in three
-            different directories depending on the ``QubesVM`` type:
-
-            * ``appvms`` for ``QubesAppVm`` or ``QubesHvm``
-            * ``vm-templates`` for ``QubesTemplateVm`` or ``QubesTemplateHvm``
-            * ``servicevms`` for any subclass of  ``QubesNetVm``
-
-            Args:
-                vm: a QubesVM
-                pool_dir: the root directory of the pool
-
-            Returns:
-                string (str) absolute path to the directory where the vm files
-                             are stored
-        """
-        vm_type = type(vm)
-
-        if vm_type in [QubesAppVm, QubesHVm]:
-            subdir = 'appvms'
-        elif vm_type in [QubesTemplateVm, QubesTemplateHVm]:
-            subdir = 'vm-templates'
-        elif issubclass(vm_type, QubesNetVm):
-            subdir = 'servicevms'
-        elif vm_type is QubesDisposableVm:
-            subdir = 'appvms'
-            return os.path.join(pool_dir, subdir, vm.template.name + '-dvm')
-        else:
-            raise QubesException(str(vm_type) + ' unknown vm type')
-
-        return os.path.join(pool_dir, subdir, vm.name)
-
-    def _create_dir_if_not_exists(self, path):
-        """ Check if a directory exists in if not it is created.
-
-            This method does not create any parent directories.
-        """
-        if not os.path.exists(path):
-            os.mkdir(path)
+        return XenStorage(self.vm, vmdir=self.vmdir)

From 9b23576ff6a93a5ac2e90772e0af1bbdda117dc2 Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Sun, 22 Nov 2015 17:18:42 +0100
Subject: [PATCH 154/225] Provide method format_disk_dev() to all storages

The method XenStorage._format_disk_dev() generates the xml config for a device.
It is not specific to the Xen file storage implementation. It can and must be
reused by other storage implementations
---
 core/storage/__init__.py | 19 +++++++++++++++++++
 core/storage/xen.py      | 35 ++++++++---------------------------
 2 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/core/storage/__init__.py b/core/storage/__init__.py
index 0c302082..530198e3 100644
--- a/core/storage/__init__.py
+++ b/core/storage/__init__.py
@@ -65,6 +65,25 @@ class QubesVmStorage(object):
         # Additional drive (currently used only by HVM)
         self.drive = None
 
+    def format_disk_dev(self, path, script, vdev, rw=True, type="disk",
+                        domain=None):
+        if path is None:
+            return ''
+        template = "    <disk type='block' device='{type}'>\n" \
+                   "      <driver name='phy'/>\n" \
+                   "      <source dev='{path}'/>\n" \
+                   "      <target dev='{vdev}' bus='xen'/>\n" \
+                   "{params}" \
+                   "    </disk>\n"
+        params = ""
+        if not rw:
+            params += "      <readonly/>\n"
+        if domain:
+            params += "      <backenddomain name='%s'/>\n" % domain
+        if script:
+            params += "      <script path='%s'/>\n" % script
+        return template.format(path=path, vdev=vdev, type=type, params=params)
+
     def get_config_params(self):
         raise NotImplementedError
 
diff --git a/core/storage/xen.py b/core/storage/xen.py
index 11ff9b67..8cb0011f 100644
--- a/core/storage/xen.py
+++ b/core/storage/xen.py
@@ -69,36 +69,17 @@ class XenStorage(QubesVmStorage):
             self.root_img = os.path.join(vmdir, 'root.img')
         self.volatile_img = os.path.join(vmdir, 'volatile.img')
 
-    def _format_disk_dev(self, path, script, vdev, rw=True, type="disk", domain=None):
-        if path is None:
-            return ''
-        template = "    <disk type='block' device='{type}'>\n" \
-                   "      <driver name='phy'/>\n" \
-                   "      <source dev='{path}'/>\n" \
-                   "      <target dev='{vdev}' bus='xen'/>\n" \
-                   "{params}" \
-                   "    </disk>\n"
-        params = ""
-        if not rw:
-            params += "      <readonly/>\n"
-        if domain:
-            params += "      <backenddomain name='%s'/>\n" % domain
-        if script:
-            params += "      <script path='%s'/>\n" % script
-        return template.format(path=path, vdev=vdev, type=type,
-            params=params)
-
     def _get_rootdev(self):
         if self.vm.is_template() and \
                 os.path.exists(os.path.join(self.vmdir, "root-cow.img")):
-            return self._format_disk_dev(
+            return self.format_disk_dev(
                     "{dir}/root.img:{dir}/root-cow.img".format(
                         dir=self.vmdir),
                     "block-origin", self.root_dev, True)
         elif self.vm.template and not self.vm.template.storage.rootcow_img:
             # HVM template-based VM - template doesn't have own
             # root-cow.img, only one device-mapper layer
-            return self._format_disk_dev(
+            return self.format_disk_dev(
                     "{tpldir}/root.img:{vmdir}/volatile.img".format(
                         tpldir=self.vm.template.dir_path,
                         vmdir=self.vmdir),
@@ -107,12 +88,12 @@ class XenStorage(QubesVmStorage):
             # any other template-based VM - two device-mapper layers: one
             # in dom0 (here) from root+root-cow, and another one from
             # this+volatile.img
-            return self._format_disk_dev(
+            return self.format_disk_dev(
                     "{dir}/root.img:{dir}/root-cow.img".format(
                         dir=self.vm.template.dir_path),
                     "block-snapshot", self.root_dev, False)
         else:
-            return self._format_disk_dev(
+            return self.format_disk_dev(
                     "{dir}/root.img".format(dir=self.vmdir),
                     None, self.root_dev, True)
 
@@ -120,14 +101,14 @@ class XenStorage(QubesVmStorage):
         args = {}
         args['rootdev'] = self._get_rootdev()
         args['privatedev'] = \
-                self._format_disk_dev(self.private_img,
+                self.format_disk_dev(self.private_img,
                         None, self.private_dev, True)
         args['volatiledev'] = \
-                self._format_disk_dev(self.volatile_img,
+                self.format_disk_dev(self.volatile_img,
                         None, self.volatile_dev, True)
         if self.modules_img is not None:
             args['otherdevs'] = \
-                    self._format_disk_dev(self.modules_img,
+                    self.format_disk_dev(self.modules_img,
                             None, self.modules_dev, self.modules_img_rw)
         elif self.drive is not None:
             (drive_type, drive_domain, drive_path) = self.drive.split(":")
@@ -136,7 +117,7 @@ class XenStorage(QubesVmStorage):
             if drive_domain.lower() == "dom0":
                 drive_domain = None
 
-            args['otherdevs'] = self._format_disk_dev(drive_path, None,
+            args['otherdevs'] = self.format_disk_dev(drive_path, None,
                     self.modules_dev,
                     rw=True if drive_type == "disk" else False, type=drive_type,
                     domain=drive_domain)

From 85421e3f48594c9662b61dc7ec35457e386a99ca Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Sun, 22 Nov 2015 18:41:55 +0100
Subject: [PATCH 155/225] Move device names from XenStorage to QubesVmStorage

---
 core/storage/__init__.py | 4 ++++
 core/storage/xen.py      | 5 -----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/core/storage/__init__.py b/core/storage/__init__.py
index 530198e3..5c5bc41c 100644
--- a/core/storage/__init__.py
+++ b/core/storage/__init__.py
@@ -57,6 +57,10 @@ class QubesVmStorage(object):
         else:
             self.root_img_size = defaults['root_img_size']
 
+        self.root_dev = "xvda"
+        self.private_dev = "xvdb"
+        self.volatile_dev = "xvdc"
+        self.modules_dev = "xvdd"
 
         # For now compute this path still in QubesVm
         self.modules_img = modules_img
diff --git a/core/storage/xen.py b/core/storage/xen.py
index 8cb0011f..51df4e3a 100644
--- a/core/storage/xen.py
+++ b/core/storage/xen.py
@@ -49,11 +49,6 @@ class XenStorage(QubesVmStorage):
 
         super(XenStorage, self).__init__(vm, **kwargs)
 
-        self.root_dev = "xvda"
-        self.private_dev = "xvdb"
-        self.volatile_dev = "xvdc"
-        self.modules_dev = "xvdd"
-
         self.vmdir = vmdir
 
         if self.vm.is_template():

From f5cef35cdf50fcdd86ec3d11425012cecf65ae51 Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Wed, 25 Nov 2015 16:01:28 +0100
Subject: [PATCH 156/225] Fix circular deps workaround in Pool.vmdir_path()

---
 core/storage/__init__.py | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/core/storage/__init__.py b/core/storage/__init__.py
index 5c5bc41c..c5d3c4a1 100644
--- a/core/storage/__init__.py
+++ b/core/storage/__init__.py
@@ -382,22 +382,17 @@ class Pool(object):
                 string (str) absolute path to the directory where the vm files
                              are stored
         """
-        # TODO: This is a hack, circular dependencies problem?
-        from qubes.qubes import (QubesAppVm, QubesDisposableVm, QubesHVm, 
-                                 QubesNetVm, QubesTemplateHVm, QubesTemplateVm)
-        vm_type = type(vm)
-
-        if vm_type in [QubesAppVm, QubesHVm]:
+        if vm.is_appvm():
             subdir = 'appvms'
-        elif vm_type in [QubesTemplateVm, QubesTemplateHVm]:
+        elif vm.is_template():
             subdir = 'vm-templates'
-        elif issubclass(vm_type, QubesNetVm):
+        elif vm.is_netvm():
             subdir = 'servicevms'
-        elif vm_type is QubesDisposableVm:
+        elif vm.is_disposablevm():
             subdir = 'appvms'
             return os.path.join(pool_dir, subdir, vm.template.name + '-dvm')
         else:
-            raise QubesException(str(vm_type) + ' unknown vm type')
+            raise QubesException(vm.type() + ' unknown vm type')
 
         return os.path.join(pool_dir, subdir, vm.name)
 

From 2fdbf51ccc8c3bd3dfa0efe57a0f2cd247d567ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 27 Nov 2015 00:07:54 +0100
Subject: [PATCH 157/225] core: have QubesVmCollection.unlock_db no-op if not
 locked

---
 core/qubes.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/core/qubes.py b/core/qubes.py
index e910b66b..34d007f5 100755
--- a/core/qubes.py
+++ b/core/qubes.py
@@ -651,6 +651,8 @@ class QubesVmCollection(dict):
             self.qubes_store_file.close()
 
     def unlock_db(self):
+        if self.qubes_store_file is None:
+            return
         # intentionally do not call explicit unlock to not unlock the file
         # before all buffers are flushed
         self.log.debug('unlock_db()')

From f525a58134674e519483d7608d30579c643df919 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 27 Nov 2015 00:09:11 +0100
Subject: [PATCH 158/225] core: adjust for updated stubdom support in libvirt

Fixes QubesOS/qubes-issues#1456
---
 core-modules/000QubesVm.py        | 12 ++++++++----
 rpm_spec/core-dom0.spec           |  2 +-
 vm-config/xen-vm-template-hvm.xml |  4 ++--
 3 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index d96f851e..99e94740 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -1161,8 +1161,10 @@ class QubesVm(object):
             args['dns2'] = self.secondary_dns
             args['netmask'] = self.netmask
             args['netdev'] = self._format_net_dev(self.ip, self.mac, self.netvm.name)
-            args['disable_network1'] = '';
-            args['disable_network2'] = '';
+            args['network_begin'] = ''
+            args['network_end'] = ''
+            args['no_network_begin'] = '<!--'
+            args['no_network_end'] = '-->'
         else:
             args['ip'] = ''
             args['mac'] = ''
@@ -1171,8 +1173,10 @@ class QubesVm(object):
             args['dns2'] = ''
             args['netmask'] = ''
             args['netdev'] = ''
-            args['disable_network1'] = '<!--';
-            args['disable_network2'] = '-->';
+            args['network_begin'] = '<!--'
+            args['network_end'] = '-->'
+            args['no_network_begin'] = ''
+            args['no_network_end'] = ''
         args.update(self.storage.get_config_params())
         if hasattr(self, 'kernelopts'):
             args['kernelopts'] = self.kernelopts
diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec
index 84e106ba..4d5ad834 100644
--- a/rpm_spec/core-dom0.spec
+++ b/rpm_spec/core-dom0.spec
@@ -57,7 +57,7 @@ Requires:       libvirt-python
 %if x%{?backend_vmm} == xxen
 Requires:       xen-runtime
 Requires:       xen-hvm
-Requires:       libvirt-daemon-xen >= 1.2.12-3
+Requires:       libvirt-daemon-xen >= 1.2.20-4
 %endif
 Requires:       createrepo
 Requires:       gnome-packagekit
diff --git a/vm-config/xen-vm-template-hvm.xml b/vm-config/xen-vm-template-hvm.xml
index 13431b33..1b6b5089 100644
--- a/vm-config/xen-vm-template-hvm.xml
+++ b/vm-config/xen-vm-template-hvm.xml
@@ -9,7 +9,6 @@
     <loader>hvmloader</loader>
     <boot dev='cdrom'/>
     <boot dev='hd'/>
-    {disable_network1}<cmdline>-net lwip,client_ip={ip},server_ip={dns2},dns={dns1},gw={gateway},netmask={netmask}</cmdline>{disable_network2}
   </os>
   <features>
     <pae/>
@@ -22,7 +21,8 @@
   <on_reboot>destroy</on_reboot>
   <on_crash>destroy</on_crash>
   <devices>
-    <emulator type='stubdom'/>
+    {no_network_begin}<emulator type='stubdom'/>{no_network_end}
+    {network_begin}<emulator type='stubdom' cmdline='-net lwip,client_ip={ip},server_ip={dns2},dns={dns1},gw={gateway},netmask={netmask}'/>{network_end}
 {rootdev}
 {privatedev}
 {otherdevs}

From fef8761f017f0136280046a49d2322bcee2dba16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 27 Nov 2015 00:09:59 +0100
Subject: [PATCH 159/225] qmemman: improve check whether VM is still running

In some cases it may happen that qmemman or other application using
xenstore will re-create VM directory in xenstore just after VM was
destroyed. For example when multiple VMs was destroyed at the same time,
but qmemman will kick off just at first @releaseDomain event - other VMs
will still be there (at xenstore-list time). This means that qmemman
will consider them when redistributing memory (of just destroyed one),
so will update memory/target entry of every "running" VM. And at this
point it may recreate VM directory of another already destroyed VM.

Generally fixing this race condition would require running all the
operations (from xenstore-ls, to setting memory/target) in a single
xenstore transaction. But this can be lengthly process. And if any other
modification happens in the meantime, transaction will rejected and
qmemman would need to redo all the changes. Not worth the effort.

Fixes QubesOS/qubes-issues#1409
---
 qmemman/qmemman_server.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/qmemman/qmemman_server.py b/qmemman/qmemman_server.py
index cf1bb2fb..ca6c32f6 100755
--- a/qmemman/qmemman_server.py
+++ b/qmemman/qmemman_server.py
@@ -74,6 +74,16 @@ class XS_Watcher:
         self.log.debug('domain_list_changed(param={!r})'.format(param))
 
         curr = self.handle.ls('', '/local/domain')
+        # check if domain is really there, it may happen that some empty
+        # directories are left in xenstore
+        curr = filter(
+            lambda x:
+            self.handle.read('',
+                             '/local/domain/{}/domid'.format(x)
+                             ) is not None,
+            curr
+        )
+
         self.log.debug('curr={!r}'.format(curr))
 
         if curr == None:

From 624c94b9d6807cb32db60318daed97cd78a2337a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 27 Nov 2015 00:25:24 +0100
Subject: [PATCH 160/225] dispvm: show error in tray when DispVM startup failed

Fixes QubesOS/qubes-issues#1457
---
 dispvm/qfile-daemon-dvm | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/dispvm/qfile-daemon-dvm b/dispvm/qfile-daemon-dvm
index a4ccd8b6..ca03940e 100755
--- a/dispvm/qfile-daemon-dvm
+++ b/dispvm/qfile-daemon-dvm
@@ -28,7 +28,7 @@ import sys
 import shutil
 import time
 
-from qubes.qubes import QubesVmCollection
+from qubes.qubes import QubesVmCollection, QubesException
 from qubes.qubes import QubesDispVmLabels
 from qubes.notify import tray_notify, tray_notify_error, tray_notify_init
 
@@ -98,7 +98,11 @@ class QfileDaemonDvm:
             qvm_collection.unlock_db()
             return None
         print >>sys.stderr, "time=%s, VM starting" % (str(time.time()))
-        dispvm.start()
+        try:
+            dispvm.start()
+        except (MemoryError, QubesException) as e:
+            tray_notify_error(e)
+            raise
         if vm.qid != 0:
             # if need to enable/disable netvm, do it while DispVM is alive
             if (dispvm.netvm is None) != (vm.dispvm_netvm is None):

From 3e79ff6a378d61a3a250c2d53f1066ae247f1353 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 27 Nov 2015 00:32:12 +0100
Subject: [PATCH 161/225] tests: release qubes.xml lock while killing VMs

Otherwise hotplug scripts may deadlock on qvm-template-commit and
consequently do not release loop and device-mapper devices. Which means
also not releasing disk space for underlying images.

Fixes QubesOS/qubes-issues#1458
---
 tests/__init__.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/tests/__init__.py b/tests/__init__.py
index f75be979..f8786038 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -219,10 +219,16 @@ class SystemTestsMixin(object):
     def tearDown(self):
         super(SystemTestsMixin, self).tearDown()
 
+        # release the lock, because we have no way to check whether it was
+        # read or write lock
         try:
-            self.qc.lock_db_for_writing()
+            self.qc.unlock_db()
         except qubes.qubes.QubesException:
             pass
+
+        self.kill_test_vms()
+
+        self.qc.lock_db_for_writing()
         self.qc.load()
 
         self.remove_test_vms()
@@ -242,6 +248,18 @@ class SystemTestsMixin(object):
         self.qc.lock_db_for_writing()
         self.qc.load()
 
+    def kill_test_vms(self):
+        # do not keep write lock while killing VMs, because that may cause a
+        # deadlock with disk hotplug scripts (namely qvm-template-commit
+        # called when shutting down TemplateVm)
+        self.qc.lock_db_for_reading()
+        self.qc.load()
+        self.qc.unlock_db()
+        for vm in self.qc.values():
+            if vm.name.startswith(VMPREFIX):
+                if vm.is_running():
+                    vm.force_shutdown()
+
     def _remove_vm_qubes(self, vm):
         vmname = vm.name
 

From 377d3ad43ac76646d400e4fe588ab74569c5549e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 27 Nov 2015 02:52:31 +0100
Subject: [PATCH 162/225] rpm: do not motify
 /etc/udev/rules.d/xen-backend.rules anymore

Xen 4.6 no longer uses udev to call hotplug scripts.

QubesOS/qubes-issues#1361
---
 rpm_spec/core-dom0.spec | 2 --
 1 file changed, 2 deletions(-)

diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec
index 4d5ad834..a9d30def 100644
--- a/rpm_spec/core-dom0.spec
+++ b/rpm_spec/core-dom0.spec
@@ -154,7 +154,6 @@ if ! grep -q ^qubes: /etc/group ; then
 fi
 
 %triggerin -- xen-runtime
-sed -i 's/\/block /\/block.qubes /' /etc/udev/rules.d/xen-backend.rules
 /usr/lib/qubes/fix-dir-perms.sh
 
 %preun
@@ -170,7 +169,6 @@ if [ "$1" = 0 ] ; then
     chgrp root /etc/xen
     chmod 700 /etc/xen
     groupdel qubes
-    sed -i 's/\/block.qubes /\/block /' /etc/udev/rules.d/xen-backend.rules
 fi
 
 %files

From 30654310584bd3588235db2b98bee74df4418f32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 27 Nov 2015 03:37:59 +0100
Subject: [PATCH 163/225] tests: include NetVM in basic backup tests

---
 tests/__init__.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tests/__init__.py b/tests/__init__.py
index f8786038..48c454cc 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -460,11 +460,22 @@ class BackupTestsMixin(SystemTestsMixin):
         template=self.qc.get_default_template()
 
         vms = []
+        vmname = self.make_vm_name('test-net')
+        if self.verbose:
+            print >>sys.stderr, "-> Creating %s" % vmname
+        testnet = self.qc.add_new_vm('QubesNetVm',
+            name=vmname, template=template)
+        testnet.create_on_disk(verbose=self.verbose)
+        vms.append(testnet)
+        self.fill_image(testnet.private_img, 20*1024*1024)
+
         vmname = self.make_vm_name('test1')
         if self.verbose:
             print >>sys.stderr, "-> Creating %s" % vmname
         testvm1 = self.qc.add_new_vm('QubesAppVm',
             name=vmname, template=template)
+        testvm1.uses_default_netvm = False
+        testvm1.netvm = testnet
         testvm1.create_on_disk(verbose=self.verbose)
         vms.append(testvm1)
         self.fill_image(testvm1.private_img, 100*1024*1024)

From 24d660d61eebcebd18fa3e4ad2ab1f4a573d5529 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 27 Nov 2015 03:39:18 +0100
Subject: [PATCH 164/225] backup: add option to on-the-fly renaming conflicting
 VMs

Fixes QubesOS/qubes-issues#869
---
 core/backup.py | 94 +++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 73 insertions(+), 21 deletions(-)

diff --git a/core/backup.py b/core/backup.py
index c84d7b39..597be233 100644
--- a/core/backup.py
+++ b/core/backup.py
@@ -1572,6 +1572,8 @@ def backup_restore_set_defaults(options):
         options['ignore-username-mismatch'] = False
     if 'verify-only' not in options:
         options['verify-only'] = False
+    if 'rename-conflicting' not in options:
+        options['rename-conflicting'] = False
 
     return options
 
@@ -1639,6 +1641,22 @@ def backup_restore_header(source, passphrase,
     return (restore_tmpdir, os.path.join(restore_tmpdir, "qubes.xml"),
             header_data)
 
+def generate_new_name_for_conflicting_vm(orig_name, host_collection,
+                                         restore_info):
+    number = 1
+    if len(orig_name) > 29:
+        orig_name = orig_name[0:29]
+    new_name = orig_name
+    while (new_name in restore_info.keys() or
+           new_name in map(lambda x: x.get('rename_to', None),
+                           restore_info.values()) or
+           host_collection.get_vm_by_name(new_name)):
+        new_name = str('{}{}'.format(orig_name, number))
+        number += 1
+        if number == 100:
+            # give up
+            return None
+    return new_name
 
 def restore_info_verify(restore_info, host_collection):
     options = restore_info['$OPTIONS$']
@@ -1656,7 +1674,16 @@ def restore_info_verify(restore_info, host_collection):
         vm_info.pop('already-exists', None)
         if not options['verify-only'] and \
                 host_collection.get_vm_by_name(vm) is not None:
-            vm_info['already-exists'] = True
+            if options['rename-conflicting']:
+                new_name = generate_new_name_for_conflicting_vm(
+                    vm, host_collection, restore_info
+                )
+                if new_name is not None:
+                    vm_info['rename-to'] = new_name
+                else:
+                    vm_info['already-exists'] = True
+            else:
+                vm_info['already-exists'] = True
 
         # check template
         vm_info.pop('missing-template', None)
@@ -1703,6 +1730,22 @@ def restore_info_verify(restore_info, host_collection):
                                                   'already-exists',
                                                   'excluded']])
 
+    # update references to renamed VMs:
+    for vm in restore_info.keys():
+        if vm in ['$OPTIONS$', 'dom0']:
+            continue
+        vm_info = restore_info[vm]
+        template_name = vm_info['template']
+        if (template_name in restore_info and
+                restore_info[template_name]['good-to-go'] and
+                'rename-to' in restore_info[template_name]):
+            vm_info['template'] = restore_info[template_name]['rename-to']
+        netvm_name = vm_info['netvm']
+        if (netvm_name in restore_info and
+                restore_info[netvm_name]['good-to-go'] and
+                'rename-to' in restore_info[netvm_name]):
+            vm_info['netvm'] = restore_info[netvm_name]['rename-to']
+
     return restore_info
 
 
@@ -1975,8 +2018,11 @@ def backup_restore_print_summary(restore_info, print_callback=print_stdout):
             s += " <-- No matching template on the host or in the backup found!"
         elif 'missing-netvm' in vm_info:
             s += " <-- No matching netvm on the host or in the backup found!"
-        elif 'orig-template' in vm_info:
-            s += " <-- Original template was '%s'" % (vm_info['orig-template'])
+        else:
+            if 'orig-template' in vm_info:
+                s += " <-- Original template was '%s'" % (vm_info['orig-template'])
+            if 'rename-to' in vm_info:
+                s += " <-- Will be renamed to '%s'" % vm_info['rename-to']
 
         print_callback(s)
 
@@ -2124,33 +2170,35 @@ def backup_restore_do(restore_info,
                 error_callback("Skipping...")
                 continue
 
-            if os.path.exists(vm.dir_path):
-                move_to_path = tempfile.mkdtemp('', os.path.basename(
-                    vm.dir_path), os.path.dirname(vm.dir_path))
-                try:
-                    os.rename(vm.dir_path, move_to_path)
-                    error_callback("*** Directory {} already exists! It has "
-                                   "been moved to {}".format(vm.dir_path,
-                                                             move_to_path))
-                except OSError:
-                    error_callback("*** Directory {} already exists and "
-                                   "cannot be moved!".format(vm.dir_path))
-                    error_callback("Skipping...")
-                    continue
-
             template = None
             if vm.template is not None:
                 template_name = restore_info[vm.name]['template']
                 template = host_collection.get_vm_by_name(template_name)
 
             new_vm = None
+            vm_name = vm.name
+            if 'rename-to' in restore_info[vm.name]:
+                vm_name = restore_info[vm.name]['rename-to']
 
             try:
-                new_vm = host_collection.add_new_vm(vm_class_name, name=vm.name,
-                                                    conf_file=vm.conf_file,
-                                                    dir_path=vm.dir_path,
+                new_vm = host_collection.add_new_vm(vm_class_name, name=vm_name,
                                                     template=template,
                                                     installed_by_rpm=False)
+                if os.path.exists(new_vm.dir_path):
+                    move_to_path = tempfile.mkdtemp('', os.path.basename(
+                        new_vm.dir_path), os.path.dirname(new_vm.dir_path))
+                    try:
+                        os.rename(new_vm.dir_path, move_to_path)
+                        error_callback(
+                            "*** Directory {} already exists! It has "
+                            "been moved to {}".format(new_vm.dir_path,
+                                                      move_to_path))
+                    except OSError:
+                        error_callback(
+                            "*** Directory {} already exists and "
+                            "cannot be moved!".format(new_vm.dir_path))
+                        error_callback("Skipping...")
+                        continue
 
                 if format_version == 1:
                     restore_vm_dir_v1(backup_location,
@@ -2194,7 +2242,11 @@ def backup_restore_do(restore_info,
 
     # Set network dependencies - only non-default netvm setting
     for vm in vms.values():
-        host_vm = host_collection.get_vm_by_name(vm.name)
+        vm_name = vm.name
+        if 'rename-to' in restore_info[vm.name]:
+            vm_name = restore_info[vm.name]['rename-to']
+        host_vm = host_collection.get_vm_by_name(vm_name)
+
         if host_vm is None:
             # Failed/skipped VM
             continue

From 99272132a66538fe505a7e3fb3979e75fbd3794e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 27 Nov 2015 03:40:28 +0100
Subject: [PATCH 165/225] tests: test case for on the fly VM rename during
 backup restore

QubesOS/qubes-issues#869
---
 tests/backup.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/tests/backup.py b/tests/backup.py
index 4ed6bfe0..2e0dea8b 100644
--- a/tests/backup.py
+++ b/tests/backup.py
@@ -104,6 +104,24 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
             ])
         self.remove_vms(vms)
 
+    def test_210_auto_rename(self):
+        """
+        Test for #869
+        :return:
+        """
+        vms = self.create_backup_vms()
+        self.make_backup(vms)
+        self.restore_backup(options={
+            'rename-conflicting': True
+        })
+        for vm in vms:
+            self.assertIsNotNone(self.qc.get_vm_by_name(vm.name+'1'))
+            restored_vm = self.qc.get_vm_by_name(vm.name+'1')
+            if vm.netvm and not vm.uses_default_netvm:
+                self.assertEqual(restored_vm.netvm.name, vm.netvm.name+'1')
+
+        self.remove_vms(vms)
+
 class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin):
     def setUp(self):
         super(TC_10_BackupVMMixin, self).setUp()

From 1e48beaf6f41a1fbad0f4a137807fb78c7e578ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 27 Nov 2015 03:44:15 +0100
Subject: [PATCH 166/225] backup: add qvm-backup-restore --rename-conflicting
 option

QubesOS/qubes-issues#869
---
 qvm-tools/qvm-backup-restore | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/qvm-tools/qvm-backup-restore b/qvm-tools/qvm-backup-restore
index c07119d6..bcfdfad0 100755
--- a/qvm-tools/qvm-backup-restore
+++ b/qvm-tools/qvm-backup-restore
@@ -54,6 +54,10 @@ def main():
     parser.add_option ("--skip-conflicting", action="store_true", dest="skip_conflicting", default=False,
                        help="Do not restore VMs that are already present on the host")
 
+    parser.add_option ("--rename-conflicting", action="store_true",
+                       dest="rename_conflicting", default=False,
+                       help="Restore VMs that are already present on the host under different name")
+
     parser.add_option ("--force-root", action="store_true", dest="force_root", default=False,
                        help="Force to run, even with root privileges")
 
@@ -193,6 +197,9 @@ def main():
         else:
             print >> sys.stderr, "Remove VMs with conflicting names from the host before proceeding."
             print >> sys.stderr, "... or use --skip-conflicting to restore only those VMs that do not exist on the host."
+            print >> sys.stderr, "... or use --rename-conflicting to " \
+                                 "restore those VMs under modified " \
+                                 "name (with number at the end)"
             exit (1)
 
     print "The above VMs will be copied and added to your system."

From 8725bdefb044d1e33973039979775cd47e9760f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 27 Nov 2015 03:47:04 +0100
Subject: [PATCH 167/225] backup: add qvm-backup --compress-filter option

Custom compression program was supported for a long time. But there was
no option to use it...

Fixes QubesOS/qubes-issues#1459
---
 qvm-tools/qvm-backup | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/qvm-tools/qvm-backup b/qvm-tools/qvm-backup
index dff4e9e5..aa69f9c6 100755
--- a/qvm-tools/qvm-backup
+++ b/qvm-tools/qvm-backup
@@ -63,6 +63,10 @@ def main():
                             "list-message-digest-algorithms'")
     parser.add_option ("-z", "--compress", action="store_true", dest="compress", default=False,
                        help="Compress the backup")
+    parser.add_option ("-Z", "--compress-filter", action="store",
+                       dest="compress_filter", default=False,
+                       help="Compress the backup using specified filter "
+                            "program (default: gzip)")
     parser.add_option ("--debug", action="store_true", dest="debug",
                        default=False, help="Enable (a lot of) debug output")
 
@@ -181,7 +185,7 @@ def main():
         backup_do(base_backup_dir, files_to_backup, passphrase,
                 progress_callback=print_progress,
                 encrypted=options.encrypt,
-                compressed=options.compress,
+                compressed=options.compress_filter or options.compress,
                 appvm=appvm, **kwargs)
     except QubesException as e:
         print >>sys.stderr, "ERROR: %s" % str(e)

From 280cb8fdfb55ec55490ac072ff11736d693fa3c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 27 Nov 2015 03:54:49 +0100
Subject: [PATCH 168/225] tests: custom backup compression filter

QubesOS/qubes-issues#1459
---
 tests/backup.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tests/backup.py b/tests/backup.py
index 2e0dea8b..fbea1cec 100644
--- a/tests/backup.py
+++ b/tests/backup.py
@@ -85,6 +85,13 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
         self.restore_backup()
         self.remove_vms(vms)
 
+    def test_005_compressed_custom(self):
+        vms = self.create_backup_vms()
+        self.make_backup(vms, do_kwargs={'compressed': "bzip2"})
+        self.remove_vms(vms)
+        self.restore_backup()
+        self.remove_vms(vms)
+
     def test_200_restore_over_existing_directory(self):
         """
         Regression test for #1386

From 67ca619ee5ba81e6c0845bcdcf60747957cf19a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 27 Nov 2015 22:56:55 +0100
Subject: [PATCH 169/225] qvm-pci: fix PCI device id (BDF) regexp

Actually BDF is in hex...

Fixes QubesOS/qubes-issues#1461
---
 qvm-tools/qvm-pci | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/qvm-tools/qvm-pci b/qvm-tools/qvm-pci
index c9c715ec..90a1aed6 100755
--- a/qvm-tools/qvm-pci
+++ b/qvm-tools/qvm-pci
@@ -38,8 +38,8 @@ def find_devices_of_class(klass):
         print "ERROR when executing lspci!"
         raise IOError
 
-    rx_netdev = re.compile(r"^([0-9][0-9]:[0-9][0-9].[0-9]) \"{}".format(
-        klass))
+    rx_netdev = re.compile(r"^([0-9a-f]{2}:[0-9a-f]{2}.[0-9a-f]) \"" +
+                           klass)
     for dev in str(result[0]).splitlines():
         match = rx_netdev.match(dev)
         if match is not None:

From cda6ae87510870b05911e8a948997249b03cc0c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 28 Nov 2015 23:03:08 +0100
Subject: [PATCH 170/225] qvm-pci: one more fix - save changes

---
 qvm-tools/qvm-pci | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/qvm-tools/qvm-pci b/qvm-tools/qvm-pci
index 90a1aed6..72c5d186 100755
--- a/qvm-tools/qvm-pci
+++ b/qvm-tools/qvm-pci
@@ -79,7 +79,7 @@ def main():
     if options.offline_mode:
         vmm.offline_mode = True
 
-    if options.do_add or options.do_delete or options.add_class:
+    if options.do_add or options.do_delete or options.do_add_class:
         qvm_collection = QubesVmCollection()
         qvm_collection.lock_db_for_writing()
         qvm_collection.load()
@@ -121,6 +121,8 @@ def main():
 
         for dev in devs:
             vm.pci_add(dev)
+        qvm_collection.save()
+        qvm_collection.unlock_db()
 
     elif options.do_delete:
         if len (args) < 2:

From f3b8fe67cc3fda2ec5562d2a1ac75c55568c5d27 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sun, 29 Nov 2015 03:47:08 +0100
Subject: [PATCH 171/225] version 3.1.4

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index ff365e06..0aec50e6 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.1.3
+3.1.4

From 9f0474338818998440dd0928dad404cff01d052b Mon Sep 17 00:00:00 2001
From: qubesuser <qubesuser@users.noreply.github.com>
Date: Sat, 5 Dec 2015 16:57:36 +0100
Subject: [PATCH 172/225] qmemman: fix null reference bug

---
 qmemman/qmemman_server.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/qmemman/qmemman_server.py b/qmemman/qmemman_server.py
index ca6c32f6..e0eb149e 100755
--- a/qmemman/qmemman_server.py
+++ b/qmemman/qmemman_server.py
@@ -74,6 +74,10 @@ class XS_Watcher:
         self.log.debug('domain_list_changed(param={!r})'.format(param))
 
         curr = self.handle.ls('', '/local/domain')
+
+        if curr == None:
+            return
+
         # check if domain is really there, it may happen that some empty
         # directories are left in xenstore
         curr = filter(
@@ -86,9 +90,6 @@ class XS_Watcher:
 
         self.log.debug('curr={!r}'.format(curr))
 
-        if curr == None:
-            return
-
         self.log.debug('acquiring global_lock')
         global_lock.acquire()
         self.log.debug('global_lock acquired')

From 2658c9a6e69ed272f49e71e04b5b342ea2dc388f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sun, 6 Dec 2015 14:31:43 +0100
Subject: [PATCH 173/225] core: detach PCI devices before shutting down VM

When VM is shutting down it doesn't disconnect PCI frontend (?), so when
VM is destroyed it ends up in timeouts in PCI backend shutdown (which
can't communicate with frontend at that stage). Prevent this by
detaching PCI devices while VM is still running.

Fixes QubesOS/qubes-issues#1494
Fixes QubesOS/qubes-issues#1425
---
 core-modules/000QubesVm.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index 99e94740..4e326080 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -1948,6 +1948,14 @@ class QubesVm(object):
         if not self.is_running():
             raise QubesException ("VM already stopped!")
 
+        # try to gracefully detach PCI devices before shutdown, to mitigate
+        # timeouts on forcible detach at domain destroy; if that fails, too bad
+        try:
+            for pcidev in self.pcidevs:
+                self.libvirt_domain.detachDevice(self._format_pci_dev(pcidev))
+        except libvirt.libvirtError:
+            pass
+
         self.libvirt_domain.shutdown()
 
     def force_shutdown(self, xid = None):

From 3fafc5dc67d3f105a314accff91c4cf4be8e516e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sun, 6 Dec 2015 14:34:52 +0100
Subject: [PATCH 174/225] tests: fix race condition in network test

Wait for network configuration being applied before testing it.
---
 tests/network.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/network.py b/tests/network.py
index fe202b9c..ce93cb9f 100644
--- a/tests/network.py
+++ b/tests/network.py
@@ -320,9 +320,11 @@ class VmNetworkingMixin(qubes.tests.SystemTestsMixin):
         self.testvm1.start()
 
         self.assertEqual(self.run_cmd(self.testvm1, self.ping_ip), 0)
-        self.testvm1.run("ip addr flush dev eth0", user="root")
-        self.testvm1.run("ip addr add 10.137.1.128/24 dev eth0", user="root")
-        self.testvm1.run("ip route add dev eth0", user="root")
+        self.testvm1.run("ip addr flush dev eth0", user="root", wait=True)
+        self.testvm1.run("ip addr add 10.137.1.128/24 dev eth0", user="root",
+                         wait=True)
+        self.testvm1.run("ip route add default dev eth0", user="root",
+                         wait=True)
         self.assertNotEqual(self.run_cmd(self.testvm1, self.ping_ip), 0,
                          "Spoofed ping should be blocked")
 

From 36134c8c43c0ff8dd45d9cd29d0f4b0539352c18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sun, 6 Dec 2015 14:37:01 +0100
Subject: [PATCH 175/225] version 3.1.5

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index 0aec50e6..3ad0595a 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.1.4
+3.1.5

From 027ffedec4c859acf0709bd7d2865276c3fc199b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 7 Dec 2015 00:03:39 +0100
Subject: [PATCH 176/225] core: typo fix in error handling

---
 core-modules/000QubesVm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index 4e326080..c1e1500b 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -771,7 +771,7 @@ class QubesVm(object):
             if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN:
                 return 0
                 # libxl_domain_info failed - domain no longer exists
-            elif e.get_error_code() == libvirt.VIR_INTERNAL_ERROR:
+            elif e.get_error_code() == libvirt.VIR_ERR_INTERNAL_ERROR:
                 return 0
             else:
                 raise

From bee84b83a1d0820554f3ab49262649061b076a83 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 7 Dec 2015 00:03:57 +0100
Subject: [PATCH 177/225] version 3.1.6

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index 3ad0595a..9cec7165 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.1.5
+3.1.6

From bb754b68c3e356df8c516e8ae0b0e1750e7e171c Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Sun, 6 Dec 2015 18:28:27 +0100
Subject: [PATCH 178/225] Add pool support to qvm-clone

---
 qvm-tools/qvm-clone | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/qvm-tools/qvm-clone b/qvm-tools/qvm-clone
index e2eba3b3..5f617098 100755
--- a/qvm-tools/qvm-clone
+++ b/qvm-tools/qvm-clone
@@ -38,6 +38,8 @@ def main():
                        help="Specify path to the template directory")
     parser.add_option ("--force-root", action="store_true", dest="force_root", default=False,
                        help="Force to run, even with root privileges")
+    parser.add_option ("-P", "--pool", dest="pool_name",
+                        help="Specify in to which storage pool to clone")
 
     (options, args) = parser.parse_args ()
     if (len (args) != 2):
@@ -45,6 +47,7 @@ def main():
     srcname = args[0]
     dstname = args[1]
 
+
     if hasattr(os, "geteuid") and os.geteuid() == 0:
         if not options.force_root:
             print >> sys.stderr, "*** Running this tool as root is strongly discouraged, this will lead you in permissions problems."
@@ -61,6 +64,11 @@ def main():
         print >> sys.stderr, "ERROR: A VM with the name '{0}' does not exist in the system.".format(srcname)
         exit(1)
 
+    if options.pool_name is None:
+        pool_name = src_vm.pool_name
+    else:
+        pool_name = options.pool_name
+
     if qvm_collection.get_vm_by_name(dstname) is not None:
         print >> sys.stderr, "ERROR: A VM with the name '{0}' already exists in the system.".format(dstname)
         exit(1)
@@ -70,7 +78,7 @@ def main():
         exit(1)
 
     dst_vm = qvm_collection.add_new_vm(src_vm.__class__.__name__,
-            name=dstname, template=src_vm.template,
+            name=dstname, template=src_vm.template, pool_name=pool_name,
             dir_path=options.dir_path, installed_by_rpm=False)
 
     try:

From 65f00d839352248d44d13e63d9ca3b17b7a008f1 Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Mon, 7 Dec 2015 10:59:44 +0100
Subject: [PATCH 179/225] Remove unnecessary imports from qvm-clone

---
 qvm-tools/qvm-clone | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/qvm-tools/qvm-clone b/qvm-tools/qvm-clone
index 5f617098..42749c94 100755
--- a/qvm-tools/qvm-clone
+++ b/qvm-tools/qvm-clone
@@ -17,16 +17,15 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-#
-#
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
+# USA.
+
+import os
+import sys
+from optparse import OptionParser
 
 from qubes.qubes import QubesVmCollection
-from qubes.qubes import QubesAppVm, QubesTemplateVm, QubesHVm
-from qubes.qubes import QubesException
-from optparse import OptionParser;
-import sys
-import os
+
 
 def main():
     usage = "usage: %prog [options] <src-name> <new-name>\n"\

From 8d3616689eb2b87e2ffa765faeba3dbede730326 Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Mon, 7 Dec 2015 11:03:51 +0100
Subject: [PATCH 180/225] PEP8 qvm-tools/qvm-clone

---
 qvm-tools/qvm-clone | 54 +++++++++++++++++++++++++++------------------
 1 file changed, 32 insertions(+), 22 deletions(-)

diff --git a/qvm-tools/qvm-clone b/qvm-tools/qvm-clone
index 42749c94..5ec22c5c 100755
--- a/qvm-tools/qvm-clone
+++ b/qvm-tools/qvm-clone
@@ -30,26 +30,30 @@ from qubes.qubes import QubesVmCollection
 def main():
     usage = "usage: %prog [options] <src-name> <new-name>\n"\
             "Clones an existing VM by copying all its disk files"
-          
-    parser = OptionParser (usage)
-    parser.add_option ("-q", "--quiet", action="store_false", dest="verbose", default=True)
-    parser.add_option ("-p", "--path", dest="dir_path",
-                       help="Specify path to the template directory")
-    parser.add_option ("--force-root", action="store_true", dest="force_root", default=False,
-                       help="Force to run, even with root privileges")
-    parser.add_option ("-P", "--pool", dest="pool_name",
-                        help="Specify in to which storage pool to clone")
 
-    (options, args) = parser.parse_args ()
-    if (len (args) != 2):
-        parser.error ("You must specify at least the src and dst TemplateVM names!")
+    parser = OptionParser(usage)
+    parser.add_option("-q", "--quiet", action="store_false", dest="verbose",
+                      default=True)
+    parser.add_option("-p", "--path", dest="dir_path",
+                      help="Specify path to the template directory")
+    parser.add_option("--force-root", action="store_true", dest="force_root",
+                      default=False,
+                      help="Force to run, even with root privileges")
+    parser.add_option("-P", "--pool", dest="pool_name",
+                      help="Specify in to which storage pool to clone")
+
+    (options, args) = parser.parse_args()
+    if (len(args) != 2):
+        parser.error(
+            "You must specify at least the src and dst TemplateVM names!")
     srcname = args[0]
     dstname = args[1]
 
-
     if hasattr(os, "geteuid") and os.geteuid() == 0:
         if not options.force_root:
-            print >> sys.stderr, "*** Running this tool as root is strongly discouraged, this will lead you in permissions problems."
+            print >> sys.stderr, "*** Running this tool as root is" + \
+                " strongly discouraged, this will lead you in permissions" + \
+                "problems."
             print >> sys.stderr, "Retry as unprivileged user."
             print >> sys.stderr, "... or use --force-root to continue anyway."
             exit(1)
@@ -59,8 +63,10 @@ def main():
     qvm_collection.load()
 
     src_vm = qvm_collection.get_vm_by_name(srcname)
-    if src_vm is  None:
-        print >> sys.stderr, "ERROR: A VM with the name '{0}' does not exist in the system.".format(srcname)
+    if src_vm is None:
+        print >> sys.stderr, \
+            "ERROR: A VM with the name '{0}' does not exist in the system." \
+            .format(srcname)
         exit(1)
 
     if options.pool_name is None:
@@ -69,7 +75,9 @@ def main():
         pool_name = options.pool_name
 
     if qvm_collection.get_vm_by_name(dstname) is not None:
-        print >> sys.stderr, "ERROR: A VM with the name '{0}' already exists in the system.".format(dstname)
+        print >> sys.stderr, \
+            "ERROR: A VM with the name '{0}' already exists in the system." \
+            .format(dstname)
         exit(1)
 
     if src_vm.is_disposablevm():
@@ -77,19 +85,21 @@ def main():
         exit(1)
 
     dst_vm = qvm_collection.add_new_vm(src_vm.__class__.__name__,
-            name=dstname, template=src_vm.template, pool_name=pool_name,
-            dir_path=options.dir_path, installed_by_rpm=False)
+                                       name=dstname, template=src_vm.template,
+                                       pool_name=pool_name,
+                                       dir_path=options.dir_path,
+                                       installed_by_rpm=False)
 
     try:
         dst_vm.clone_attrs(src_vm)
-        dst_vm.clone_disk_files (src_vm=src_vm, verbose=options.verbose)
+        dst_vm.clone_disk_files(src_vm=src_vm, verbose=options.verbose)
     except (IOError, OSError) as err:
         print >> sys.stderr, "ERROR: {0}".format(err)
         qvm_collection.pop(dst_vm.qid)
         dst_vm.remove_from_disk()
-        exit (1)
+        exit(1)
 
     qvm_collection.save()
     qvm_collection.unlock_db()
-    
+
 main()

From 56e6c01917ddb477e70b71def9bfb5be9b7f4cac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 22 Dec 2015 01:53:53 +0100
Subject: [PATCH 181/225] dispvm: convert exception object to string -
 otherwise it will not be visible

tray_notify_error can't handle arbitrary object.

Fixes QubesOS/qubes-issues#1457
---
 dispvm/qfile-daemon-dvm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dispvm/qfile-daemon-dvm b/dispvm/qfile-daemon-dvm
index ca03940e..26bea9ac 100755
--- a/dispvm/qfile-daemon-dvm
+++ b/dispvm/qfile-daemon-dvm
@@ -101,7 +101,7 @@ class QfileDaemonDvm:
         try:
             dispvm.start()
         except (MemoryError, QubesException) as e:
-            tray_notify_error(e)
+            tray_notify_error(str(e))
             raise
         if vm.qid != 0:
             # if need to enable/disable netvm, do it while DispVM is alive

From ff63c5030daa025809d4d0dd82596d9f58a16111 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 22 Dec 2015 01:59:20 +0100
Subject: [PATCH 182/225] version 3.1.7

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index 9cec7165..23887f6e 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.1.6
+3.1.7

From c8d26c253e7526ec1e3a0cda7d123ac6efdcc10a Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Sun, 22 Nov 2015 18:51:55 +0100
Subject: [PATCH 183/225] Storage refactor device config generation

- QubesVmStorage provides now a default get_config_params() method which should
  be enough for all possible Storage implementations.
- When writing a custom Storage implementation, one has just to reimplement the
  following methods:
  * root_dev_config()
  * private_dev_config()
  * volatile_dev_config()
- QubesVmStorage provides a default implementation of other_dev_config(),
  because it can be shared by all storage implementations
---
 core/storage/__init__.py | 41 ++++++++++++++++++++++++++++++++++++++++
 core/storage/xen.py      | 35 +++++++---------------------------
 2 files changed, 48 insertions(+), 28 deletions(-)

diff --git a/core/storage/__init__.py b/core/storage/__init__.py
index c5d3c4a1..b5a744aa 100644
--- a/core/storage/__init__.py
+++ b/core/storage/__init__.py
@@ -89,8 +89,49 @@ class QubesVmStorage(object):
         return template.format(path=path, vdev=vdev, type=type, params=params)
 
     def get_config_params(self):
+        args = {}
+        args['rootdev'] = self.root_dev_config()
+        args['privatedev'] = self.private_dev_config()
+        args['volatiledev'] = self.volatile_dev_config()
+        args['otherdevs'] = self.other_dev_config()
+
+        return args
+
+    def root_dev_config(self):
         raise NotImplementedError
 
+    def private_dev_config(self):
+        raise NotImplementedError
+
+    def volatile_dev_config(self):
+        raise NotImplementedError
+
+    def other_dev_config(self):
+        if self.modules_img is not None:
+            return self.format_disk_dev(self.modules_img,
+                                        None,
+                                        self.modules_dev,
+                                        self.modules_img_rw)
+        elif self.drive is not None:
+            (drive_type, drive_domain, drive_path) = self.drive.split(":")
+            if drive_type == "hd":
+                drive_type = "disk"
+
+            writable = False
+            if drive_type == "disk":
+                writable = True
+
+            if drive_domain.lower() == "dom0":
+                drive_domain = None
+
+            return self.format_disk_dev(drive_path, None,
+                                        self.modules_dev,
+                                        rw=writable,
+                                        type=drive_type,
+                                        domain=drive_domain)
+        else:
+            return ''
+
     def _copy_file(self, source, destination):
         """
         Effective file copy, preserving sparse files etc.
diff --git a/core/storage/xen.py b/core/storage/xen.py
index 51df4e3a..cdc52355 100644
--- a/core/storage/xen.py
+++ b/core/storage/xen.py
@@ -64,7 +64,7 @@ class XenStorage(QubesVmStorage):
             self.root_img = os.path.join(vmdir, 'root.img')
         self.volatile_img = os.path.join(vmdir, 'volatile.img')
 
-    def _get_rootdev(self):
+    def root_dev_config(self):
         if self.vm.is_template() and \
                 os.path.exists(os.path.join(self.vmdir, "root-cow.img")):
             return self.format_disk_dev(
@@ -92,34 +92,13 @@ class XenStorage(QubesVmStorage):
                     "{dir}/root.img".format(dir=self.vmdir),
                     None, self.root_dev, True)
 
-    def get_config_params(self):
-        args = {}
-        args['rootdev'] = self._get_rootdev()
-        args['privatedev'] = \
-                self.format_disk_dev(self.private_img,
-                        None, self.private_dev, True)
-        args['volatiledev'] = \
-                self.format_disk_dev(self.volatile_img,
-                        None, self.volatile_dev, True)
-        if self.modules_img is not None:
-            args['otherdevs'] = \
-                    self.format_disk_dev(self.modules_img,
-                            None, self.modules_dev, self.modules_img_rw)
-        elif self.drive is not None:
-            (drive_type, drive_domain, drive_path) = self.drive.split(":")
-            if drive_type == "hd":
-                drive_type = "disk"
-            if drive_domain.lower() == "dom0":
-                drive_domain = None
+    def private_dev_config(self):
+        return self.format_disk_dev(self.private_img, None,
+                                    self.private_dev, True)
 
-            args['otherdevs'] = self.format_disk_dev(drive_path, None,
-                    self.modules_dev,
-                    rw=True if drive_type == "disk" else False, type=drive_type,
-                    domain=drive_domain)
-        else:
-            args['otherdevs'] = ''
-
-        return args
+    def volatile_dev_config(self):
+        return self.format_disk_dev(self.volatile_img, None,
+                                    self.volatile_dev, True)
 
     def create_on_disk_private_img(self, verbose, source_template = None):
         if source_template:

From d539bd2be14b28eb2e6f1cd5b55790141de177b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 23 Dec 2015 19:07:18 +0100
Subject: [PATCH 184/225] core: preserve qrexec-timeout during VM clone

Fixes QubesOS/qubes-issues#1501
---
 core-modules/000QubesVm.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index c1e1500b..f72b51b2 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -1271,9 +1271,10 @@ class QubesVm(object):
             hook(self, verbose, source_template=source_template)
 
     def get_clone_attrs(self):
-        attrs = ['kernel', 'uses_default_kernel', 'netvm', 'uses_default_netvm', \
-            'memory', 'maxmem', 'kernelopts', 'uses_default_kernelopts', 'services', 'vcpus', \
-            '_mac', 'pcidevs', 'include_in_backups', '_label', 'default_user']
+        attrs = ['kernel', 'uses_default_kernel', 'netvm', 'uses_default_netvm',
+                 'memory', 'maxmem', 'kernelopts', 'uses_default_kernelopts',
+                 'services', 'vcpus', '_mac', 'pcidevs', 'include_in_backups',
+                 '_label', 'default_user', 'qrexec_timeout']
 
         # fire hooks
         for hook in self.hooks_get_clone_attrs:

From a30e8d1f33279cddfa08417683bb13d665659e0a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 23 Dec 2015 19:08:01 +0100
Subject: [PATCH 185/225] systemd: fix starting VMs _before_ allowing user
 login

systemd-user-sessions.service is specicically for that, do not use hack
(plymouth-quit.service), which doesn't work when the service is
disabled.

Fixes QubesOS/qubes-issues#1250
---
 linux/systemd/qubes-netvm.service | 2 +-
 linux/systemd/qubes-vm@.service   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/linux/systemd/qubes-netvm.service b/linux/systemd/qubes-netvm.service
index 450a9903..bf556f3c 100644
--- a/linux/systemd/qubes-netvm.service
+++ b/linux/systemd/qubes-netvm.service
@@ -1,7 +1,7 @@
 [Unit]
 Description=Qubes NetVM startup
 After=qubes-core.service qubes-qmemman.service libvirtd.service
-Before=plymouth-quit.service
+Before=systemd-user-sessions.service
 
 [Service]
 Type=oneshot
diff --git a/linux/systemd/qubes-vm@.service b/linux/systemd/qubes-vm@.service
index 07607d10..e35b63a4 100644
--- a/linux/systemd/qubes-vm@.service
+++ b/linux/systemd/qubes-vm@.service
@@ -1,7 +1,7 @@
 [Unit]
 Description=Start Qubes VM %i
 After=qubes-netvm.service
-Before=plymouth-quit.service
+Before=systemd-user-sessions.service
 
 [Service]
 Type=oneshot

From d4a42093b1900b9dbf96fc0832064446deae3de9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 23 Dec 2015 19:24:25 +0100
Subject: [PATCH 186/225] dispvm: Start required NetVM first (if necessary)

The comment "calling VM have the same netvm" doesn't apply to:
 - DispVM started from dom0 menu
 - DispVM started from a VM with `dispvm_netvm` property modified

Fixes QubesOS/qubes-issues#1334
---
 core-modules/01QubesDisposableVm.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/core-modules/01QubesDisposableVm.py b/core-modules/01QubesDisposableVm.py
index 8fedb742..03b10b93 100644
--- a/core-modules/01QubesDisposableVm.py
+++ b/core-modules/01QubesDisposableVm.py
@@ -168,8 +168,13 @@ class QubesDisposableVm(QubesVm):
         if self.get_power_state() != "Halted":
             raise QubesException ("VM is already running!")
 
-        # skip netvm state checking - calling VM have the same netvm, so it
-        # must be already running
+        if self.netvm is not None:
+            if self.netvm.qid != 0:
+                if not self.netvm.is_running():
+                    if verbose:
+                        print >> sys.stderr, "--> Starting NetVM {0}...".\
+                            format(self.netvm.name)
+                    self.netvm.start(verbose=verbose, **kwargs)
 
         if verbose:
             print >> sys.stderr, "--> Loading the VM (type = {0})...".format(self.type)

From adb282ef8eb2b514a6918208a608642a2aca4ec8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 26 Dec 2015 02:14:28 +0100
Subject: [PATCH 187/225] Implement qvm-shutdown --wait-time option

Flexible VM shutdown timeout option.

Fixes QubesOS/qubes-issues#1320
---
 qvm-tools/qvm-shutdown | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/qvm-tools/qvm-shutdown b/qvm-tools/qvm-shutdown
index 37374cba..da3eeec2 100755
--- a/qvm-tools/qvm-shutdown
+++ b/qvm-tools/qvm-shutdown
@@ -36,6 +36,10 @@ def main():
                       help="Force operation, even if may damage other VMs (eg shutdown of NetVM)")
     parser.add_option ("--wait", action="store_true", dest="wait_for_shutdown", default=False,
                       help="Wait for the VM(s) to shutdown")
+    parser.add_option("--wait-time", action="store", dest="wait_time",
+                      default=defaults["shutdown_counter_max"],
+                      help="Timout after which VM will be killed when --wait "
+                           "is used")
     parser.add_option ("--all", action="store_true", dest="shutdown_all", default=False,
                       help="Shutdown all running VMs")
     parser.add_option ("--exclude", action="append", dest="exclude_list",
@@ -107,7 +111,7 @@ def main():
                         continue
                     else:
                         halting_vms.append(vm)
-                if shutdown_counter > defaults["shutdown_counter_max"]:
+                if shutdown_counter > int(options.wait_time):
                     # kill the VM
                     if options.verbose:
                         print >> sys.stderr, "Killing the (apparently hanging) VM '{0}'...".format(vm.name)

From e3e8a55c92552d2f794ac4baaf47c56b28f4f523 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 26 Dec 2015 02:20:28 +0100
Subject: [PATCH 188/225] core: forbid 'lost+found' as VM name

When /var/lib/qubes/appvms is a mount point of ext4 filesystem, there
will be already 'lost+found' directory. Avoid this conflict.

Fixes QubesOS/qubes-issues#1440
---
 core-modules/000QubesVm.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index f72b51b2..fc15dde6 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -566,6 +566,10 @@ class QubesVm(object):
             return False
         if len(name) > 31:
             return False
+        if name == 'lost+found':
+            # avoid conflict when /var/lib/qubes/appvms is mounted on
+            # separate partition
+            return False
         return re.match(r"^[a-zA-Z][a-zA-Z0-9_.-]*$", name) is not None
 
     def pre_rename(self, new_name):

From 49c1ab2f994bf64796f8b9494ae8aadfa369b9b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 26 Dec 2015 04:08:12 +0100
Subject: [PATCH 189/225] qvm-grow-root: wait for VM to shutdown at the end

Otherwise it could lead to some race conditions, for example when the
user tries to start some application there afterwards.

QubesOS/qubes-issues#1268
---
 core-modules/001QubesResizableVm.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/core-modules/001QubesResizableVm.py b/core-modules/001QubesResizableVm.py
index 88467ed1..5e898b40 100644
--- a/core-modules/001QubesResizableVm.py
+++ b/core-modules/001QubesResizableVm.py
@@ -31,6 +31,7 @@ from qubes.qubes import (
     QubesException,
     QubesVm,
 )
+from time import sleep
 
 
 class QubesResizableVm(QubesVm):
@@ -62,6 +63,8 @@ class QubesResizableVmWithResize2fs(QubesResizableVm):
         self.run("resize2fs /dev/mapper/dmroot", user="root", wait=True,
                  gui=False)
         self.shutdown()
+        while self.is_running():
+            sleep(1)
 
 
 register_qubes_vm_class(QubesResizableVm)

From 1b5daea7711b0cd5a44ece345e86d115d783883c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 26 Dec 2015 04:10:20 +0100
Subject: [PATCH 190/225] qvm-grow-root: add --allow-start option

In some (most) cases VM needs to be started to complete resize
operation. This may be unexpected, so make it clear and do not start the
VM when the user did not explicitly allow that.

Fixes QubesOS/qubes-issues#1268
---
 core-modules/001QubesResizableVm.py | 12 +++++++++---
 qvm-tools/qvm-grow-root             |  6 +++++-
 tests/vm_qrexec_gui.py              |  4 +++-
 3 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/core-modules/001QubesResizableVm.py b/core-modules/001QubesResizableVm.py
index 5e898b40..f50ead70 100644
--- a/core-modules/001QubesResizableVm.py
+++ b/core-modules/001QubesResizableVm.py
@@ -36,7 +36,7 @@ from time import sleep
 
 class QubesResizableVm(QubesVm):
 
-    def resize_root_img(self, size):
+    def resize_root_img(self, size, allow_start=False):
         if self.template:
             raise QubesException("Cannot resize root.img of template-based VM"
                                  ". Resize the root.img of the template "
@@ -57,8 +57,14 @@ class QubesResizableVm(QubesVm):
 
 class QubesResizableVmWithResize2fs(QubesResizableVm):
 
-    def resize_root_img(self, size):
-        super(QubesResizableVmWithResize2fs, self).resize_root_img(size)
+    def resize_root_img(self, size, allow_start=False):
+        super(QubesResizableVmWithResize2fs, self).\
+            resize_root_img(size, allow_start=allow_start)
+        if not allow_start:
+            raise QubesException("VM start required to complete the "
+                                 "operation, but not allowed. Either run the "
+                                 "operation again allowing VM start this "
+                                 "time, or run resize2fs in the VM manually.")
         self.start(start_guid=False)
         self.run("resize2fs /dev/mapper/dmroot", user="root", wait=True,
                  gui=False)
diff --git a/qvm-tools/qvm-grow-root b/qvm-tools/qvm-grow-root
index 93c55bb2..5e650ab7 100755
--- a/qvm-tools/qvm-grow-root
+++ b/qvm-tools/qvm-grow-root
@@ -34,6 +34,10 @@ def main():
     usage = "usage: %prog <vm-name> <size>"
     parser = OptionParser (usage)
 
+    parser.add_option("--allow-start", action="store_true",
+                      dest="allow_start", default=False,
+                      help="Allow VM to be started to complete the operation")
+
     (options, args) = parser.parse_args ()
     if (len (args) != 2):
         parser.error ("You must specify VM name and new size!")
@@ -57,7 +61,7 @@ def main():
         exit(1)
 
     try:
-        vm.resize_root_img(size_bytes)
+        vm.resize_root_img(size_bytes, allow_start=options.allow_start)
     except (IOError, OSError, QubesException) as err:
         print >> sys.stderr, "ERROR: {0}".format(err)
         exit (1)
diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index c64da4d4..aa4e3da4 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -731,7 +731,9 @@ class TC_05_StandaloneVM(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase
                                source_template=self.qc.get_default_template())
         self.qc.save()
         self.qc.unlock_db()
-        testvm1.resize_root_img(20*1024**3)
+        with self.assertRaises(QubesException):
+            testvm1.resize_root_img(20*1024**3)
+        testvm1.resize_root_img(20*1024**3, allow_start=True)
         timeout = 60
         while testvm1.is_running():
             time.sleep(1)

From b8e40895b39d6f0f6193796520313d6559e86627 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 26 Dec 2015 11:31:56 +0100
Subject: [PATCH 191/225] Add qvm-start --skip-if-running option

QubesOS/qubes-issues#1528
---
 qvm-tools/qvm-start | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/qvm-tools/qvm-start b/qvm-tools/qvm-start
index 42deeb54..002c272e 100755
--- a/qvm-tools/qvm-start
+++ b/qvm-tools/qvm-start
@@ -57,6 +57,9 @@ def main():
                       help="Do actions necessary when preparing DVM image")
     parser.add_option ("--custom-config", action="store", dest="custom_config", default=None,
                       help="Use custom Xen config instead of Qubes-generated one")
+    parser.add_option("--skip-if-running", action="store_true",
+                      dest="skip_if_running", default=False,
+                      help="Do not fail if the VM is already running")
     parser.add_option ("--debug", action="store_true", dest="debug", default=False,
                       help="Enable debug mode for this VM (until its shutdown)")
 
@@ -104,6 +107,9 @@ def main():
     if options.debug:
         vm.debug = True
 
+    if options.skip_if_running and vm.is_running():
+        return
+
     try:
         vm.verify_files()
         xid = vm.start(verbose=options.verbose, preparing_dvm=options.preparing_dvm, start_guid=not options.noguid, notify_function=tray_notify_generic if options.tray else None)

From 169eac641290a3021f78856d0cb9894ffba3e744 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 26 Dec 2015 11:34:40 +0100
Subject: [PATCH 192/225] systemd: do not fail qubes-vm@ service when the VM is
 already running

Fixes QubesOS/qubes-issues#1528
---
 linux/systemd/qubes-vm@.service | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/linux/systemd/qubes-vm@.service b/linux/systemd/qubes-vm@.service
index e35b63a4..2eb09ec5 100644
--- a/linux/systemd/qubes-vm@.service
+++ b/linux/systemd/qubes-vm@.service
@@ -6,7 +6,7 @@ Before=systemd-user-sessions.service
 [Service]
 Type=oneshot
 Environment=DISPLAY=:0
-ExecStart=/usr/bin/qvm-start --no-guid %i
+ExecStart=/usr/bin/qvm-start --no-guid --skip-if-running %i
 Group=qubes
 RemainAfterExit=yes
 

From 9809e4168e9c162209d0e3c8470659d2bd2f3c9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 26 Dec 2015 14:05:21 +0100
Subject: [PATCH 193/225] core: log pci detach error on VM shutdown

Thanks @Rudd-O for pointing this out.
---
 core-modules/000QubesVm.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index fc15dde6..1996764f 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -1958,8 +1958,9 @@ class QubesVm(object):
         try:
             for pcidev in self.pcidevs:
                 self.libvirt_domain.detachDevice(self._format_pci_dev(pcidev))
-        except libvirt.libvirtError:
-            pass
+        except libvirt.libvirtError as e:
+            print >>sys.stderr, "WARNING: {}, continuing VM shutdown " \
+                                "anyway".format(str(e))
 
         self.libvirt_domain.shutdown()
 

From ba0d60595eab2e348d296d20840e6d366c160d3b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 26 Dec 2015 15:28:51 +0100
Subject: [PATCH 194/225] version 3.1.8

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index 23887f6e..c848fb9c 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.1.7
+3.1.8

From 70546ac21a4be4c2cb9fab2a6ea195949f8c944c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 28 Dec 2015 01:07:48 +0100
Subject: [PATCH 195/225] tests: add test for VM autostart on qrexec call

---
 tests/vm_qrexec_gui.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index aa4e3da4..3c3f12ea 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -547,6 +547,25 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
                                    wait=True)
         self.assertEqual(retcode, 0, "file differs")
 
+    @unittest.skipUnless(spawn.find_executable('xdotool'),
+                         "xdotool not installed")
+    def test_101_qrexec_filecopy_with_autostart(self):
+        self.testvm1.start()
+        p = self.testvm1.run("qvm-copy-to-vm %s /etc/passwd" %
+                             self.testvm2.name, passio_popen=True,
+                             passio_stderr=True)
+        # Confirm transfer
+        self.enter_keys_in_window('Question', ['y'])
+        p.wait()
+        self.assertEqual(p.returncode, 0, "qvm-copy-to-vm failed: %s" %
+                         p.stderr.read())
+        self.assertTrue(self.testvm2.is_running())
+        retcode = self.testvm2.run("diff /etc/passwd "
+                                   "/home/user/QubesIncoming/{}/passwd".format(
+                                       self.testvm1.name),
+                                   wait=True)
+        self.assertEqual(retcode, 0, "file differs")
+
     @unittest.skipUnless(spawn.find_executable('xdotool'),
                          "xdotool not installed")
     def test_110_qrexec_filecopy_deny(self):

From a78ec670713fe31919c29dffce046b1ef30386c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 29 Dec 2015 14:19:44 +0100
Subject: [PATCH 196/225] doc: Change qvm-service name from "cron" to "crond"

---
 doc/qvm-tools/qvm-service.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/qvm-tools/qvm-service.rst b/doc/qvm-tools/qvm-service.rst
index 3a552b07..c67c6670 100644
--- a/doc/qvm-tools/qvm-service.rst
+++ b/doc/qvm-tools/qvm-service.rst
@@ -73,7 +73,7 @@ cups
 
     Enable CUPS service. The user can disable cups in VM which do not need printing to speed up booting.
 
-cron
+crond
     Default: disabled
 
     Enable CRON service.

From e30e802903b5e94d8463acdf0756ddd80bd2d6ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 6 Jan 2016 00:39:00 +0100
Subject: [PATCH 197/225] qmemman: add --debug option

---
 qmemman/qmemman_server.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/qmemman/qmemman_server.py b/qmemman/qmemman_server.py
index e0eb149e..c2b7e0e3 100755
--- a/qmemman/qmemman_server.py
+++ b/qmemman/qmemman_server.py
@@ -205,8 +205,13 @@ class QMemmanServer:
         usage = "usage: %prog [options]"
         parser = OptionParser(usage)
         parser.add_option("-c", "--config", action="store", dest="config", default=config_path)
+        parser.add_option("-d", "--debug", action="store_true", dest="debug",
+                          default=False, help="Enable debugging")
         (options, args) = parser.parse_args()
 
+        if options.debug:
+            logging.root.setLevel(logging.DEBUG)
+
         # close io
         sys.stdin.close()
         sys.stdout.close()

From 181eb3e76414bd102be5c2004daca784a74ee010 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 6 Jan 2016 00:40:10 +0100
Subject: [PATCH 198/225] qmemman: handle memory assigned to VM but not yet
 used

When VM got some memory assigned, balloon driver may not pick it up
immediatelly and the memory will still be seen as "free" by Xen, but VM
can use (request) it at any time. Qmemman needs to take care of such
memory (exclude it from "free" pool), otherwise it would redistribute it
to other domains, allowing the original domain to drain Xen memory pool.

Do this by redefining DomainState.memory_actual - it is now amount of
memory available to the VM (currently used, or possibly used). Then
calculate free memory by subtracting memory allocated but not used
(memory_target-memory_current).

Fixes QubesOS/qubes-issues#1389
---
 qmemman/qmemman.py        | 37 ++++++++++++++++++++++++++++---------
 qmemman/qmemman_server.py |  8 ++++++++
 2 files changed, 36 insertions(+), 9 deletions(-)

diff --git a/qmemman/qmemman.py b/qmemman/qmemman.py
index 2d124ff6..01712375 100755
--- a/qmemman/qmemman.py
+++ b/qmemman/qmemman.py
@@ -37,7 +37,9 @@ slow_memset_react_msg="VM didn't give back all requested memory"
 class DomainState:
     def __init__(self, id):
         self.meminfo = None		#dictionary of memory info read from client
-        self.memory_actual = None	#the current memory size
+        self.memory_current = None	#the current memory size
+        self.memory_actual = None   # the current memory allocation (what VM
+                                    #  is using or can use at any time)
         self.memory_maximum = None	#the maximum memory size
         self.mem_used = None		#used memory, computed based on meminfo
         self.id = id			#domain id
@@ -65,25 +67,42 @@ class SystemState(object):
     def add_domain(self, id):
         self.log.debug('add_domain(id={!r})'.format(id))
         self.domdict[id] = DomainState(id)
+        # TODO: move to DomainState.__init__
+        target_str = self.xs.read('', '/local/domain/' + id + '/memory/target')
+        if target_str:
+            self.domdict[id].last_target = int(target_str) * 1024
 
     def del_domain(self, id):
         self.log.debug('del_domain(id={!r})'.format(id))
         self.domdict.pop(id)
 
     def get_free_xen_memory(self):
-        return int(self.xc.physinfo()['free_memory']*1024 * self.MEM_OVERHEAD_FACTOR)
-#        hosts = self.xend_session.session.xenapi.host.get_all()
-#        host_record = self.xend_session.session.xenapi.host.get_record(hosts[0])
-#        host_metrics_record = self.xend_session.session.xenapi.host_metrics.get_record(host_record["metrics"])
-#        ret = host_metrics_record["memory_free"]
-#        return long(ret)
+        xen_free = int(self.xc.physinfo()['free_memory']*1024 *
+                       self.MEM_OVERHEAD_FACTOR)
+        # now check for domains which have assigned more memory than really
+        # used - do not count it as "free", because domain is free to use it
+        # at any time
+        # assumption: self.refresh_memactual was called before
+        # (so domdict[id].memory_actual is up to date)
+        assigned_but_unused = reduce(
+            lambda acc, dom: acc + max(0, dom.last_target-dom.memory_current),
+            self.domdict.values(),
+            0
+        )
+        return xen_free - assigned_but_unused
 
 #refresh information on memory assigned to all domains
     def refresh_memactual(self):
         for domain in self.xc.domain_getinfo():
             id = str(domain['domid'])
             if self.domdict.has_key(id):
-                self.domdict[id].memory_actual = domain['mem_kb']*1024
+                # real memory usage
+                self.domdict[id].memory_current = domain['mem_kb']*1024
+                # what VM is using or can use
+                self.domdict[id].memory_actual = max(
+                    self.domdict[id].memory_current,
+                    self.domdict[id].last_target
+                )
                 self.domdict[id].memory_maximum = self.xs.read('', '/local/domain/%s/memory/static-max' % str(id))
                 if self.domdict[id].memory_maximum:
                     self.domdict[id].memory_maximum = int(self.domdict[id].memory_maximum)*1024
@@ -272,11 +291,11 @@ class SystemState(object):
                 self.log.debug('do_balance dom={!r} sleeping ntries={}'.format(
                     dom, ntries))
                 time.sleep(self.BALOON_DELAY)
+                self.refresh_memactual()
                 ntries -= 1
                 if ntries <= 0:
                     # Waiting haven't helped; Find which domain get stuck and
                     # abort balance (after distributing what we have)
-                    self.refresh_memactual()
                     for rq2 in memset_reqs:
                         dom2, mem2 = rq2
                         if dom2 == dom:
diff --git a/qmemman/qmemman_server.py b/qmemman/qmemman_server.py
index c2b7e0e3..6e6fa94c 100755
--- a/qmemman/qmemman_server.py
+++ b/qmemman/qmemman_server.py
@@ -157,6 +157,14 @@ class QMemmanReqHandler(SocketServer.BaseRequestHandler):
             self.log.debug('data={!r}'.format(self.data))
             if len(self.data) == 0:
                 self.log.info('EOF')
+                # FIXME: there is a race condition here: if XS_Watcher will
+                # handle meminfo event before @introduceDomain, it will use
+                # incomplete domain list for that and may redistribute memory
+                # allocated to some VM, but not yet used (see #1389).
+                # To fix that, system_state should be updated (refresh domain
+                #  list) before releasing the lock, but in the current code
+                # layout XS_Watcher instance isn't available here,
+                # so xenstore watches would not be registered
                 if got_lock:
                     global_lock.release()
                     self.log.debug('global_lock released')

From fdd618b420f312f55c57fe47e0fd39ea3c0939ab Mon Sep 17 00:00:00 2001
From: Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
Date: Fri, 13 Nov 2015 22:27:54 +0100
Subject: [PATCH 199/225] Move xen storage tests to own file

This allows to run just the general storage tests without all the slow xen
storage based tests.
---
 tests/Makefile       |   2 +
 tests/__init__.py    |   1 +
 tests/storage.py     | 208 ---------------------------------------
 tests/storage_xen.py | 228 +++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 231 insertions(+), 208 deletions(-)
 create mode 100644 tests/storage_xen.py

diff --git a/tests/Makefile b/tests/Makefile
index f2dff357..2f503a15 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -29,3 +29,5 @@ endif
 	cp run.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
 	cp storage.py $(DESTDIR)$(PYTHON_TESTSPATH)
 	cp storage.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
+	cp storage_xen.py $(DESTDIR)$(PYTHON_TESTSPATH)
+	cp storage_xen.py[co] $(DESTDIR)$(PYTHON_TESTSPATH)
diff --git a/tests/__init__.py b/tests/__init__.py
index 48c454cc..7f740e42 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -584,6 +584,7 @@ def load_tests(loader, tests, pattern):
             'qubes.tests.backupcompatibility',
             'qubes.tests.regressions',
             'qubes.tests.storage',
+            'qubes.tests.storage_xen',
             ):
         tests.addTests(loader.loadTestsFromName(modname))
 
diff --git a/tests/storage.py b/tests/storage.py
index 5513d383..2c32fa73 100644
--- a/tests/storage.py
+++ b/tests/storage.py
@@ -17,9 +17,6 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 
-import os
-import shutil
-
 import qubes.storage
 from qubes.qubes import defaults
 from qubes.storage.xen import XenPool, XenStorage
@@ -78,208 +75,3 @@ class TC_00_Storage(SystemTestsMixin, QubesTestCase):
 
         qubes.storage.remove_pool(pool_name)
         self.assertFalse(qubes.storage.pool_exists(pool_name))
-
-
-class TC_00_Pool(SystemTestsMixin, QubesTestCase):
-
-    """ This class tests some properties of the 'default' pool. """
-
-    def test000_default_pool_dir(self):
-        """ The predefined dir for the default pool should be ``/var/lib/qubes``
-
-            .. sealso::
-               Data :data:``qubes.qubes.defaults['pool_config']``.
-        """
-        vm = self._init_app_vm()
-        result = qubes.storage.get_pool("default", vm).dir_path
-        expected = '/var/lib/qubes/'
-        self.assertEquals(result, expected)
-
-    def test001_default_storage_class(self):
-        """ Check when using default pool the Storage is ``XenStorage``. """
-        result = self._init_app_vm().storage
-        self.assertIsInstance(result, XenStorage)
-
-    def test_002_default_pool_name(self):
-        """ Default pool_name is 'default'. """
-        vm = self._init_app_vm()
-        self.assertEquals(vm.pool_name, "default")
-
-    def _init_app_vm(self):
-        """ Return initalised, but not created, AppVm. """
-        vmname = self.make_vm_name('appvm')
-        template = self.qc.get_default_template()
-        return self.qc.add_new_vm('QubesAppVm', name=vmname, template=template,
-                                  pool_name='default')
-
-
-class TC_01_Pool(SystemTestsMixin, QubesTestCase):
-
-    """ Test the paths for the default Xen file based storage (``XenStorage``).
-    """
-
-    POOL_DIR = '/var/lib/qubes/test-pool'
-    APPVMS_DIR = '/var/lib/qubes/test-pool/appvms'
-    TEMPLATES_DIR = '/var/lib/qubes/test-pool/vm-templates'
-    SERVICE_DIR = '/var/lib/qubes/test-pool/servicevms'
-
-    def setUp(self):
-        """ Add a test file based storage pool """
-        super(TC_01_Pool, self).setUp()
-        qubes.storage.add_pool('test-pool', driver='xen',
-                               dir_path=self.POOL_DIR)
-
-    def tearDown(self):
-        """ Remove the file based storage pool after testing """
-        super(TC_01_Pool, self).tearDown()
-        qubes.storage.remove_pool("test-pool")
-        shutil.rmtree(self.POOL_DIR, ignore_errors=True)
-
-    def test_001_pool_exists(self):
-        """ Check if the storage pool was added to the storage pool config """
-        self.assertTrue(qubes.storage.pool_exists('test-pool'))
-
-    def test_002_pool_dir_create(self):
-        """ Check if the storage pool dir and subdirs were created """
-
-        # The dir should not exists before
-        self.assertFalse(os.path.exists(self.POOL_DIR))
-
-        vmname = self.make_vm_name('appvm')
-        template = self.qc.get_default_template()
-        self.qc.add_new_vm('QubesAppVm', name=vmname, template=template,
-                           pool_name='test-pool')
-
-        self.assertTrue(os.path.exists(self.POOL_DIR))
-        self.assertTrue(os.path.exists(self.APPVMS_DIR))
-        self.assertTrue(os.path.exists(self.SERVICE_DIR))
-        self.assertTrue(os.path.exists(self.TEMPLATES_DIR))
-
-    def test_003_pool_dir(self):
-        """ Check if the vm storage pool_dir is the same as specified """
-        vmname = self.make_vm_name('appvm')
-        template = self.qc.get_default_template()
-        vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template,
-                                pool_name='test-pool')
-        result = qubes.storage.get_pool('test-pool', vm).dir_path
-        self.assertEquals(self.POOL_DIR, result)
-
-    def test_004_app_vmdir(self):
-        """ Check the vm storage dir for an AppVm"""
-        vmname = self.make_vm_name('appvm')
-        template = self.qc.get_default_template()
-        vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template,
-                                pool_name='test-pool')
-
-        expected = os.path.join(self.APPVMS_DIR, vm.name)
-        result = vm.storage.vmdir
-        self.assertEquals(expected, result)
-
-    def test_005_hvm_vmdir(self):
-        """ Check the vm storage dir for a HVM"""
-        vmname = self.make_vm_name('hvm')
-        vm = self.qc.add_new_vm('QubesHVm', name=vmname,
-                                pool_name='test-pool')
-
-        expected = os.path.join(self.APPVMS_DIR, vm.name)
-        result = vm.storage.vmdir
-        self.assertEquals(expected, result)
-
-    def test_006_net_vmdir(self):
-        """ Check the vm storage dir for a Netvm"""
-        vmname = self.make_vm_name('hvm')
-        vm = self.qc.add_new_vm('QubesNetVm', name=vmname,
-                                pool_name='test-pool')
-
-        expected = os.path.join(self.SERVICE_DIR, vm.name)
-        result = vm.storage.vmdir
-        self.assertEquals(expected, result)
-
-    def test_007_proxy_vmdir(self):
-        """ Check the vm storage dir for a ProxyVm"""
-        vmname = self.make_vm_name('proxyvm')
-        vm = self.qc.add_new_vm('QubesProxyVm', name=vmname,
-                                pool_name='test-pool')
-
-        expected = os.path.join(self.SERVICE_DIR, vm.name)
-        result = vm.storage.vmdir
-        self.assertEquals(expected, result)
-
-    def test_008_admin_vmdir(self):
-        """ Check the vm storage dir for a AdminVm"""
-        # TODO How to test AdminVm?
-        pass
-
-    def test_009_template_vmdir(self):
-        """ Check the vm storage dir for a TemplateVm"""
-        vmname = self.make_vm_name('templatevm')
-        vm = self.qc.add_new_vm('QubesTemplateVm', name=vmname,
-                                pool_name='test-pool')
-
-        expected = os.path.join(self.TEMPLATES_DIR, vm.name)
-        result = vm.storage.vmdir
-        self.assertEquals(expected, result)
-
-    def test_010_template_hvm_vmdir(self):
-        """ Check the vm storage dir for a TemplateHVm"""
-        vmname = self.make_vm_name('templatehvm')
-        vm = self.qc.add_new_vm('QubesTemplateHVm', name=vmname,
-                                pool_name='test-pool')
-
-        expected = os.path.join(self.TEMPLATES_DIR, vm.name)
-        result = vm.storage.vmdir
-        self.assertEquals(expected, result)
-
-    def test_011_appvm_file_images(self):
-        """ Check if all the needed image files are created for an AppVm"""
-
-        vmname = self.make_vm_name('appvm')
-        template = self.qc.get_default_template()
-        vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template,
-                                pool_name='test-pool')
-        vm.create_on_disk(verbose=False)
-
-        expected_vmdir = os.path.join(self.APPVMS_DIR, vm.name)
-        self.assertEqualsAndExists(vm.storage.vmdir, expected_vmdir)
-
-        expected_private_path = os.path.join(expected_vmdir, 'private.img')
-        self.assertEqualsAndExists(vm.storage.private_img,
-                                   expected_private_path)
-
-        expected_volatile_path = os.path.join(expected_vmdir, 'volatile.img')
-        self.assertEqualsAndExists(vm.storage.volatile_img,
-                                   expected_volatile_path)
-
-    def test_012_hvm_file_images(self):
-        """ Check if all the needed image files are created for a HVm"""
-
-        vmname = self.make_vm_name('hvm')
-        vm = self.qc.add_new_vm('QubesHVm', name=vmname,
-                                pool_name='test-pool')
-        vm.create_on_disk(verbose=False)
-
-        expected_vmdir = os.path.join(self.APPVMS_DIR, vm.name)
-        self.assertEqualsAndExists(vm.storage.vmdir, expected_vmdir)
-
-        expected_private_path = os.path.join(expected_vmdir, 'private.img')
-        self.assertEqualsAndExists(vm.storage.private_img,
-                                   expected_private_path)
-
-        expected_root_path = os.path.join(expected_vmdir, 'root.img')
-        self.assertEqualsAndExists(vm.storage.root_img, expected_root_path)
-
-        expected_volatile_path = os.path.join(expected_vmdir, 'volatile.img')
-        self.assertEqualsAndExists(vm.storage.volatile_img,
-                                   expected_volatile_path)
-
-    def assertEqualsAndExists(self, result_path, expected_path):
-        """ Check if the ``result_path``, matches ``expected_path`` and exists.
-
-            See also: :meth:``assertExist``
-        """
-        self.assertEquals(result_path, expected_path)
-        self.assertExist(result_path)
-
-    def assertExist(self, path):
-        """ Assert that the given path exists. """
-        self.assertTrue(os.path.exists(path))
diff --git a/tests/storage_xen.py b/tests/storage_xen.py
new file mode 100644
index 00000000..a6503929
--- /dev/null
+++ b/tests/storage_xen.py
@@ -0,0 +1,228 @@
+# The Qubes OS Project, https://www.qubes-os.org/
+#
+# Copyright (C) 2015  Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+import shutil
+import qubes.storage
+from qubes.tests import QubesTestCase, SystemTestsMixin
+from qubes.storage.xen import XenStorage
+
+
+class TC_00_XenPool(SystemTestsMixin, QubesTestCase):
+
+    """ This class tests some properties of the 'default' pool. """
+
+    def test000_default_pool_dir(self):
+        """ The predefined dir for the default pool should be ``/var/lib/qubes``
+
+            .. sealso::
+               Data :data:``qubes.qubes.defaults['pool_config']``.
+        """
+        vm = self._init_app_vm()
+        result = qubes.storage.get_pool("default", vm).dir_path
+        expected = '/var/lib/qubes/'
+        self.assertEquals(result, expected)
+
+    def test001_default_storage_class(self):
+        """ Check when using default pool the Storage is ``XenStorage``. """
+        result = self._init_app_vm().storage
+        self.assertIsInstance(result, XenStorage)
+
+    def test_002_default_pool_name(self):
+        """ Default pool_name is 'default'. """
+        vm = self._init_app_vm()
+        self.assertEquals(vm.pool_name, "default")
+
+    def _init_app_vm(self):
+        """ Return initalised, but not created, AppVm. """
+        vmname = self.make_vm_name('appvm')
+        template = self.qc.get_default_template()
+        return self.qc.add_new_vm('QubesAppVm', name=vmname, template=template,
+                                  pool_name='default')
+
+
+class TC_01_XenPool(SystemTestsMixin, QubesTestCase):
+
+    """ Test the paths for the default Xen file based storage (``XenStorage``).
+    """
+
+    POOL_DIR = '/var/lib/qubes/test-pool'
+    APPVMS_DIR = '/var/lib/qubes/test-pool/appvms'
+    TEMPLATES_DIR = '/var/lib/qubes/test-pool/vm-templates'
+    SERVICE_DIR = '/var/lib/qubes/test-pool/servicevms'
+
+    def setUp(self):
+        """ Add a test file based storage pool """
+        super(TC_01_XenPool, self).setUp()
+        qubes.storage.add_pool('test-pool', driver='xen',
+                               dir_path=self.POOL_DIR)
+
+    def tearDown(self):
+        """ Remove the file based storage pool after testing """
+        super(TC_01_XenPool, self).tearDown()
+        qubes.storage.remove_pool("test-pool")
+        shutil.rmtree(self.POOL_DIR, ignore_errors=True)
+
+    def test_001_pool_exists(self):
+        """ Check if the storage pool was added to the storage pool config """
+        self.assertTrue(qubes.storage.pool_exists('test-pool'))
+
+    def test_002_pool_dir_create(self):
+        """ Check if the storage pool dir and subdirs were created """
+
+        # The dir should not exists before
+        self.assertFalse(os.path.exists(self.POOL_DIR))
+
+        vmname = self.make_vm_name('appvm')
+        template = self.qc.get_default_template()
+        self.qc.add_new_vm('QubesAppVm', name=vmname, template=template,
+                           pool_name='test-pool')
+
+        self.assertTrue(os.path.exists(self.POOL_DIR))
+        self.assertTrue(os.path.exists(self.APPVMS_DIR))
+        self.assertTrue(os.path.exists(self.SERVICE_DIR))
+        self.assertTrue(os.path.exists(self.TEMPLATES_DIR))
+
+    def test_003_pool_dir(self):
+        """ Check if the vm storage pool_dir is the same as specified """
+        vmname = self.make_vm_name('appvm')
+        template = self.qc.get_default_template()
+        vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template,
+                                pool_name='test-pool')
+        result = qubes.storage.get_pool('test-pool', vm).dir_path
+        self.assertEquals(self.POOL_DIR, result)
+
+    def test_004_app_vmdir(self):
+        """ Check the vm storage dir for an AppVm"""
+        vmname = self.make_vm_name('appvm')
+        template = self.qc.get_default_template()
+        vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template,
+                                pool_name='test-pool')
+
+        expected = os.path.join(self.APPVMS_DIR, vm.name)
+        result = vm.storage.vmdir
+        self.assertEquals(expected, result)
+
+    def test_005_hvm_vmdir(self):
+        """ Check the vm storage dir for a HVM"""
+        vmname = self.make_vm_name('hvm')
+        vm = self.qc.add_new_vm('QubesHVm', name=vmname,
+                                pool_name='test-pool')
+
+        expected = os.path.join(self.APPVMS_DIR, vm.name)
+        result = vm.storage.vmdir
+        self.assertEquals(expected, result)
+
+    def test_006_net_vmdir(self):
+        """ Check the vm storage dir for a Netvm"""
+        vmname = self.make_vm_name('hvm')
+        vm = self.qc.add_new_vm('QubesNetVm', name=vmname,
+                                pool_name='test-pool')
+
+        expected = os.path.join(self.SERVICE_DIR, vm.name)
+        result = vm.storage.vmdir
+        self.assertEquals(expected, result)
+
+    def test_007_proxy_vmdir(self):
+        """ Check the vm storage dir for a ProxyVm"""
+        vmname = self.make_vm_name('proxyvm')
+        vm = self.qc.add_new_vm('QubesProxyVm', name=vmname,
+                                pool_name='test-pool')
+
+        expected = os.path.join(self.SERVICE_DIR, vm.name)
+        result = vm.storage.vmdir
+        self.assertEquals(expected, result)
+
+    def test_008_admin_vmdir(self):
+        """ Check the vm storage dir for a AdminVm"""
+        # TODO How to test AdminVm?
+        pass
+
+    def test_009_template_vmdir(self):
+        """ Check the vm storage dir for a TemplateVm"""
+        vmname = self.make_vm_name('templatevm')
+        vm = self.qc.add_new_vm('QubesTemplateVm', name=vmname,
+                                pool_name='test-pool')
+
+        expected = os.path.join(self.TEMPLATES_DIR, vm.name)
+        result = vm.storage.vmdir
+        self.assertEquals(expected, result)
+
+    def test_010_template_hvm_vmdir(self):
+        """ Check the vm storage dir for a TemplateHVm"""
+        vmname = self.make_vm_name('templatehvm')
+        vm = self.qc.add_new_vm('QubesTemplateHVm', name=vmname,
+                                pool_name='test-pool')
+
+        expected = os.path.join(self.TEMPLATES_DIR, vm.name)
+        result = vm.storage.vmdir
+        self.assertEquals(expected, result)
+
+    def test_011_appvm_file_images(self):
+        """ Check if all the needed image files are created for an AppVm"""
+
+        vmname = self.make_vm_name('appvm')
+        template = self.qc.get_default_template()
+        vm = self.qc.add_new_vm('QubesAppVm', name=vmname, template=template,
+                                pool_name='test-pool')
+        vm.create_on_disk(verbose=False)
+
+        expected_vmdir = os.path.join(self.APPVMS_DIR, vm.name)
+        self.assertEqualsAndExists(vm.storage.vmdir, expected_vmdir)
+
+        expected_private_path = os.path.join(expected_vmdir, 'private.img')
+        self.assertEqualsAndExists(vm.storage.private_img,
+                                   expected_private_path)
+
+        expected_volatile_path = os.path.join(expected_vmdir, 'volatile.img')
+        self.assertEqualsAndExists(vm.storage.volatile_img,
+                                   expected_volatile_path)
+
+    def test_012_hvm_file_images(self):
+        """ Check if all the needed image files are created for a HVm"""
+
+        vmname = self.make_vm_name('hvm')
+        vm = self.qc.add_new_vm('QubesHVm', name=vmname,
+                                pool_name='test-pool')
+        vm.create_on_disk(verbose=False)
+
+        expected_vmdir = os.path.join(self.APPVMS_DIR, vm.name)
+        self.assertEqualsAndExists(vm.storage.vmdir, expected_vmdir)
+
+        expected_private_path = os.path.join(expected_vmdir, 'private.img')
+        self.assertEqualsAndExists(vm.storage.private_img,
+                                   expected_private_path)
+
+        expected_root_path = os.path.join(expected_vmdir, 'root.img')
+        self.assertEqualsAndExists(vm.storage.root_img, expected_root_path)
+
+        expected_volatile_path = os.path.join(expected_vmdir, 'volatile.img')
+        self.assertEqualsAndExists(vm.storage.volatile_img,
+                                   expected_volatile_path)
+
+    def assertEqualsAndExists(self, result_path, expected_path):
+        """ Check if the ``result_path``, matches ``expected_path`` and exists.
+
+            See also: :meth:``assertExist``
+        """
+        self.assertEquals(result_path, expected_path)
+        self.assertExist(result_path)
+
+    def assertExist(self, path):
+        """ Assert that the given path exists. """
+        self.assertTrue(os.path.exists(path))

From b26f2a73b408a334a3f7c5cbd37e7aee248972e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 6 Jan 2016 03:23:12 +0100
Subject: [PATCH 200/225] core: mark QubesTemplateHVm.rootcow_img as property

Add missing '@property'.

QubesOS/qubes-issues#1573
---
 core-modules/02QubesTemplateHVm.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core-modules/02QubesTemplateHVm.py b/core-modules/02QubesTemplateHVm.py
index eb5b309e..d1d6e459 100644
--- a/core-modules/02QubesTemplateHVm.py
+++ b/core-modules/02QubesTemplateHVm.py
@@ -70,6 +70,7 @@ class QubesTemplateHVm(QubesHVm):
     def is_appvm(self):
         return False
 
+    @property
     def rootcow_img(self):
         return self.storage.rootcow_img
 

From fa196f13503c2e690e1b7e0db3f578ab838a832d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 6 Jan 2016 03:27:50 +0100
Subject: [PATCH 201/225] storage: actually use storage.root_img and such
 properties

Actual VM root disk used hardcoded image names, instead of properties
set for that purpose...
---
 core/storage/xen.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/core/storage/xen.py b/core/storage/xen.py
index cdc52355..d2e1ebde 100644
--- a/core/storage/xen.py
+++ b/core/storage/xen.py
@@ -65,31 +65,31 @@ class XenStorage(QubesVmStorage):
         self.volatile_img = os.path.join(vmdir, 'volatile.img')
 
     def root_dev_config(self):
-        if self.vm.is_template() and \
-                os.path.exists(os.path.join(self.vmdir, "root-cow.img")):
+        if self.vm.is_template() and self.rootcow_img:
             return self.format_disk_dev(
-                    "{dir}/root.img:{dir}/root-cow.img".format(
-                        dir=self.vmdir),
+                    "{root}:{rootcow}".format(
+                        root=self.root_img, rootcow=self.rootcow_img),
                     "block-origin", self.root_dev, True)
         elif self.vm.template and not self.vm.template.storage.rootcow_img:
             # HVM template-based VM - template doesn't have own
             # root-cow.img, only one device-mapper layer
             return self.format_disk_dev(
-                    "{tpldir}/root.img:{vmdir}/volatile.img".format(
-                        tpldir=self.vm.template.dir_path,
-                        vmdir=self.vmdir),
+                    "{root}:{volatile}".format(
+                        root=self.vm.template.storage.root_img,
+                        volatile=self.volatile_img),
                     "block-snapshot", self.root_dev, True)
         elif self.vm.template:
             # any other template-based VM - two device-mapper layers: one
             # in dom0 (here) from root+root-cow, and another one from
             # this+volatile.img
             return self.format_disk_dev(
-                    "{dir}/root.img:{dir}/root-cow.img".format(
-                        dir=self.vm.template.dir_path),
+                    "{root}:{rootcow}".format(
+                        root=self.root_img,
+                        rootcow=self.vm.template.storage.rootcow_img),
                     "block-snapshot", self.root_dev, False)
         else:
             return self.format_disk_dev(
-                    "{dir}/root.img".format(dir=self.vmdir),
+                    "{root}".format(root=self.root_img),
                     None, self.root_dev, True)
 
     def private_dev_config(self):

From 35cb82cd2b03dab7fb5bdd83c65342dfa5a38a48 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 6 Jan 2016 03:43:32 +0100
Subject: [PATCH 202/225] storage: use root-cow.img for HVM template, to
 support reverting changes

We still can't support running HVM template and its VMs simultaneously
(easily), but still, have root-cow.img handled for HVM template, to
allow qvm-revert-template-changes.

Fixes QubesOS/qubes-issues#1573
---
 core-modules/01QubesHVm.py         |  2 --
 core-modules/02QubesTemplateHVm.py | 14 +++++++++++---
 core/storage/xen.py                |  9 ++++++---
 3 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/core-modules/01QubesHVm.py b/core-modules/01QubesHVm.py
index deb2deb8..2a6e9f04 100644
--- a/core-modules/01QubesHVm.py
+++ b/core-modules/01QubesHVm.py
@@ -100,8 +100,6 @@ class QubesHVm(QubesResizableVm):
             (not 'xml_element' in kwargs or kwargs['xml_element'].get('guiagent_installed') is None):
             self.services['meminfo-writer'] = False
 
-        self.storage.rootcow_img = None
-
     @property
     def type(self):
         return "HVM"
diff --git a/core-modules/02QubesTemplateHVm.py b/core-modules/02QubesTemplateHVm.py
index d1d6e459..6452a8eb 100644
--- a/core-modules/02QubesTemplateHVm.py
+++ b/core-modules/02QubesTemplateHVm.py
@@ -29,7 +29,7 @@ import stat
 import sys
 import re
 
-from qubes.qubes import QubesHVm,register_qubes_vm_class,dry_run
+from qubes.qubes import QubesHVm,register_qubes_vm_class,dry_run,vmm
 from qubes.qubes import QubesException,QubesVmCollection
 from qubes.qubes import system_path,defaults
 
@@ -96,7 +96,15 @@ class QubesTemplateHVm(QubesHVm):
     def commit_changes (self, verbose = False):
         self.log.debug('commit_changes()')
 
-        # nothing to do as long as root-cow.img is unused
-        pass
+        if not vmm.offline_mode:
+            assert not self.is_running(), "Attempt to commit changes on running Template VM!"
+
+        if verbose:
+            print >> sys.stderr, "--> Commiting template updates... COW: {0}...".format (self.rootcow_img)
+
+        if dry_run:
+            return
+
+        self.storage.commit_template_changes()
 
 register_qubes_vm_class(QubesTemplateHVm)
diff --git a/core/storage/xen.py b/core/storage/xen.py
index d2e1ebde..3edf0f22 100644
--- a/core/storage/xen.py
+++ b/core/storage/xen.py
@@ -70,9 +70,12 @@ class XenStorage(QubesVmStorage):
                     "{root}:{rootcow}".format(
                         root=self.root_img, rootcow=self.rootcow_img),
                     "block-origin", self.root_dev, True)
-        elif self.vm.template and not self.vm.template.storage.rootcow_img:
-            # HVM template-based VM - template doesn't have own
-            # root-cow.img, only one device-mapper layer
+        elif self.vm.template and not hasattr(self.vm, 'kernel'):
+            # HVM template-based VM - only one device-mapper layer, in dom0 (
+            # root+volatile)
+            # HVM detection based on 'kernel' property is massive hack,
+            # but taken from assumption that VM needs Qubes-specific kernel (
+            # actually initramfs) to assemble the second layer of device-mapper
             return self.format_disk_dev(
                     "{root}:{volatile}".format(
                         root=self.vm.template.storage.root_img,

From f977858610cda7341f7285f34861f311a56c050d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 6 Jan 2016 03:45:53 +0100
Subject: [PATCH 203/225] minor: add missing coding declaration

---
 tests/regressions.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/regressions.py b/tests/regressions.py
index 70c03586..3248e0da 100644
--- a/tests/regressions.py
+++ b/tests/regressions.py
@@ -1,4 +1,5 @@
 #!/usr/bin/python2 -O
+# coding=utf-8
 
 #
 # The Qubes OS Project, https://www.qubes-os.org/

From 4af5a0503dea600188c362e62b5ecf825adbd69b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 6 Jan 2016 03:47:08 +0100
Subject: [PATCH 204/225] tests: qvm-revert-template-changes

Regression test for:
QubesOS/qubes-issues#1573
---
 tests/basic.py | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)

diff --git a/tests/basic.py b/tests/basic.py
index d3a524c2..456934a1 100644
--- a/tests/basic.py
+++ b/tests/basic.py
@@ -571,5 +571,64 @@ class TC_02_QvmPrefs(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
         self.execute_tests('kernel', [('default', '', False)])
         self.execute_tests('kernelopts', [('default', '', False)])
 
+class TC_03_QvmRevertTemplateChanges(qubes.tests.SystemTestsMixin,
+                                     qubes.tests.QubesTestCase):
+
+    def setup_pv_template(self):
+        self.test_template = self.qc.add_new_vm(
+            "QubesTemplateVm",
+            name=self.make_vm_name("pv-clone"),
+        )
+        self.test_template.clone_attrs(src_vm=self.qc.get_default_template())
+        self.test_template.clone_disk_files(
+            src_vm=self.qc.get_default_template(),
+            verbose=False)
+        self.save_and_reload_db()
+        self.qc.unlock_db()
+
+    def setup_hvm_template(self):
+        self.test_template = self.qc.add_new_vm(
+            "QubesTemplateHVm",
+            name=self.make_vm_name("hvm"),
+        )
+        self.test_template.create_on_disk(verbose=False)
+        self.save_and_reload_db()
+        self.qc.unlock_db()
+
+    def get_rootimg_checksum(self):
+        p = subprocess.Popen(['sha1sum', self.test_template.root_img],
+                             stdout=subprocess.PIPE)
+        return p.communicate()[0]
+
+    def _do_test(self):
+        checksum_before = self.get_rootimg_checksum()
+        self.test_template.start(verbose=False)
+        self.shutdown_and_wait(self.test_template)
+        checksum_changed = self.get_rootimg_checksum()
+        if checksum_before == checksum_changed:
+            self.log.warning("template not modified, test result will be "
+                             "unreliable")
+        with self.assertNotRaises(subprocess.CalledProcessError):
+            subprocess.check_call(['sudo', 'qvm-revert-template-changes',
+                                   '--force', self.test_template.name])
+
+        checksum_after = self.get_rootimg_checksum()
+        self.assertEquals(checksum_before, checksum_after)
+
+    def test_000_revert_pv(self):
+        """
+        Test qvm-revert-template-changes for PV template
+        """
+        self.setup_pv_template()
+        self._do_test()
+
+    def test_000_revert_hvm(self):
+        """
+        Test qvm-revert-template-changes for HVM template
+        """
+        # TODO: have some system there, so the root.img will get modified
+        self.setup_hvm_template()
+        self._do_test()
+
 
 # vim: ts=4 sw=4 et

From 530a79417f27f391164c34aeef1a5a5ee523073c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 6 Jan 2016 04:32:11 +0100
Subject: [PATCH 205/225] Add some logging to ease debugging libvirt errors

Temporary add some more logging to ease debugging libvirt errors.
Unfortunatelly stack trace doesn't contain actual error.

QubesOS/qubes-issues#1537
---
 core-modules/000QubesVm.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py
index 1996764f..35f30d6b 100644
--- a/core-modules/000QubesVm.py
+++ b/core-modules/000QubesVm.py
@@ -723,6 +723,8 @@ class QubesVm(object):
             if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN:
                 return -1
             else:
+                print >>sys.stderr, "libvirt error code: {!r}".format(
+                    e.get_error_code())
                 raise
 
 
@@ -778,6 +780,8 @@ class QubesVm(object):
             elif e.get_error_code() == libvirt.VIR_ERR_INTERNAL_ERROR:
                 return 0
             else:
+                print >>sys.stderr, "libvirt error code: {!r}".format(
+                    e.get_error_code())
                 raise
 
     def get_cputime(self):
@@ -795,6 +799,8 @@ class QubesVm(object):
             elif e.get_error_code() == libvirt.VIR_INTERNAL_ERROR:
                 return 0
             else:
+                print >>sys.stderr, "libvirt error code: {!r}".format(
+                    e.get_error_code())
                 raise
 
     def get_mem_static_max(self):
@@ -836,6 +842,8 @@ class QubesVm(object):
             if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN:
                 return 0
             else:
+                print >>sys.stderr, "libvirt error code: {!r}".format(
+                    e.get_error_code())
                 raise
 
     def get_disk_utilization_root_img(self):
@@ -911,6 +919,8 @@ class QubesVm(object):
             if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN:
                 return False
             else:
+                print >>sys.stderr, "libvirt error code: {!r}".format(
+                    e.get_error_code())
                 raise
 
     def is_paused(self):
@@ -923,6 +933,8 @@ class QubesVm(object):
             if e.get_error_code() == libvirt.VIR_ERR_NO_DOMAIN:
                 return False
             else:
+                print >>sys.stderr, "libvirt error code: {!r}".format(
+                    e.get_error_code())
                 raise
 
     def get_start_time(self):
@@ -1371,6 +1383,8 @@ class QubesVm(object):
                 # already undefined
                 pass
             else:
+                print >>sys.stderr, "libvirt error code: {!r}".format(
+                    e.get_error_code())
                 raise
 
         self.storage.remove_from_disk()

From ba3b908b73190e5603bdb92c6618466b5ef66dc6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Thu, 7 Jan 2016 02:07:08 +0100
Subject: [PATCH 206/225] backup: do not dereference symlinks in dom0 user home
 backup

Symlinks there should be preserved. Additionally broken symlink would
cause the backup to fail.

Fixes QubesOS/qubes-issues#1515
---
 core/backup.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/core/backup.py b/core/backup.py
index 597be233..7ba8808f 100644
--- a/core/backup.py
+++ b/core/backup.py
@@ -567,15 +567,16 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
         # be verified before untaring this.
         # Prefix the path in archive with filename["subdir"] to have it
         # verified during untar
-        tar_cmdline = ["tar", "-Pc", '--sparse',
+        tar_cmdline = (["tar", "-Pc", '--sparse',
                        "-f", backup_pipe,
-                       '-C', os.path.dirname(filename["path"]),
-                       '--dereference',
-                       '--xform', 's:^%s:%s\\0:' % (
+                       '-C', os.path.dirname(filename["path"])] +
+                       (['--dereference'] if filename["subdir"] != "dom0-home/"
+                       else []) +
+                       ['--xform', 's:^%s:%s\\0:' % (
                            os.path.basename(filename["path"]),
                            filename["subdir"]),
                        os.path.basename(filename["path"])
-                       ]
+                       ])
         if compressed:
             tar_cmdline.insert(-1,
                                "--use-compress-program=%s" % compression_filter)

From 24ad20cfe6c636630acc217a6569ae0416e0f827 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Thu, 7 Jan 2016 02:14:33 +0100
Subject: [PATCH 207/225] tests: add a simple dom0 backup test

Regression test for:
QubesOS/qubes-issues#1515

It assumes existing broken symlink in user dom0 home. Thanks for systemd
it is the case (`~/.local/share/systemd/user`).
---
 tests/backup.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/backup.py b/tests/backup.py
index fbea1cec..72463c91 100644
--- a/tests/backup.py
+++ b/tests/backup.py
@@ -63,7 +63,6 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
         self.restore_backup()
         self.remove_vms(vms)
 
-
     def test_004_sparse_multipart(self):
         vms = []
 
@@ -92,6 +91,10 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
         self.restore_backup()
         self.remove_vms(vms)
 
+    def test_100_backup_dom0_no_restore(self):
+        self.make_backup([self.qc[0]])
+        # TODO: think of some safe way to test restore...
+
     def test_200_restore_over_existing_directory(self):
         """
         Regression test for #1386

From baffef0ef082bd6e8c98d2cbe0f3f1a6a1eeb5c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Thu, 7 Jan 2016 02:20:07 +0100
Subject: [PATCH 208/225] tests: fix qvm-move-to-vm error code checking

Currently error code is 28 (ENOSPC). But generally any non-zero is
expected result here.

Fix test for QubesOS/qubes-issues#1355
---
 tests/vm_qrexec_gui.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index 3c3f12ea..f1778090 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -634,7 +634,7 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
         # Close GUI error message
         self.enter_keys_in_window('Error', ['Return'])
         p.wait()
-        self.assertEqual(p.returncode, 255, "qvm-move-to-vm should fail")
+        self.assertNotEqual(p.returncode, 0, "qvm-move-to-vm should fail")
         retcode = self.testvm1.run("test -f testfile", wait=True)
         self.assertEqual(retcode, 0, "testfile should not be deleted in "
                                      "source VM")

From db136674a9ac8b8323a7ed9f875dae7867fa46e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Thu, 7 Jan 2016 02:55:20 +0100
Subject: [PATCH 209/225] tests: workaround libvirt bug - domain ID
 desynchronization

---
 tests/vm_qrexec_gui.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/vm_qrexec_gui.py b/tests/vm_qrexec_gui.py
index f1778090..e6e03eee 100644
--- a/tests/vm_qrexec_gui.py
+++ b/tests/vm_qrexec_gui.py
@@ -559,6 +559,10 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
         p.wait()
         self.assertEqual(p.returncode, 0, "qvm-copy-to-vm failed: %s" %
                          p.stderr.read())
+        # workaround for libvirt bug (domain ID isn't updated when is started
+        #  from other application) - details in
+        # QubesOS/qubes-core-libvirt@63ede4dfb4485c4161dd6a2cc809e8fb45ca664f
+        self.testvm2._libvirt_domain = None
         self.assertTrue(self.testvm2.is_running())
         retcode = self.testvm2.run("diff /etc/passwd "
                                    "/home/user/QubesIncoming/{}/passwd".format(

From 301eed55b31714ea500de9eb4e5c2ee799b03bf4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Thu, 7 Jan 2016 06:38:06 +0100
Subject: [PATCH 210/225] version 3.1.9

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index c848fb9c..7148b0a9 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.1.8
+3.1.9

From 78b63eb21ac9d9134afc89baf16d4822faa85b88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 11 Jan 2016 21:50:57 +0100
Subject: [PATCH 211/225] qmemman: fix crash when VM fails to start

In this case, qmemman would not manage to get any memory state for such
VM, so memory_current would be 'None'. But later it would perform
arithmetics on it, which would result in qmemman crash (TypeError
exception).

Fixes QubesOS/qubes-issues#1601
---
 qmemman/qmemman.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qmemman/qmemman.py b/qmemman/qmemman.py
index 01712375..57d27370 100755
--- a/qmemman/qmemman.py
+++ b/qmemman/qmemman.py
@@ -37,7 +37,7 @@ slow_memset_react_msg="VM didn't give back all requested memory"
 class DomainState:
     def __init__(self, id):
         self.meminfo = None		#dictionary of memory info read from client
-        self.memory_current = None	#the current memory size
+        self.memory_current = 0     # the current memory size
         self.memory_actual = None   # the current memory allocation (what VM
                                     #  is using or can use at any time)
         self.memory_maximum = None	#the maximum memory size

From 8b9049d8c66160eeb3b7ea60be3c34a752b5fce2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 11 Jan 2016 21:53:33 +0100
Subject: [PATCH 212/225] core/hvm: create `root-cow.img` if it's missing

It should be created at VM creation time (or template changes commit).
But for example for HVM templates created before implementing
QubesOS/qubes-issues#1573, there would be no such image. So create it
when needed, just before VM startup

Fixes QubesOS/qubes-issues#1602
---
 core/storage/xen.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/core/storage/xen.py b/core/storage/xen.py
index 3edf0f22..9ea7ba73 100644
--- a/core/storage/xen.py
+++ b/core/storage/xen.py
@@ -226,6 +226,8 @@ class XenStorage(QubesVmStorage):
                     raise QubesException(
                         "VM '{}' holding '{}' does not exists".format(
                             drive_domain, drive_path))
+        if self.rootcow_img and not os.path.exists(self.rootcow_img):
+            self.commit_template_changes()
 
 
 class XenPool(Pool):

From 46a338d0879410debda8e3e073024fefe87a4704 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 12 Jan 2016 12:05:19 +0100
Subject: [PATCH 213/225] version 3.1.10

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index 7148b0a9..c7a24988 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.1.9
+3.1.10

From f4d46c7855628b923fde1fa1ff28ae539504eb10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Thu, 14 Jan 2016 01:14:41 +0100
Subject: [PATCH 214/225] tests: add a test for proper saving of DispVM
 firewall rules

Test if DispVM firewall rules are:
 - inherited from a calling VM
 - saved properly (so will be effective after firewall reload)
 - saved only for that DispVM (#1608)

QubesOS/qubes-issues#1608
---
 tests/basic.py | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)

diff --git a/tests/basic.py b/tests/basic.py
index 456934a1..e53a0b55 100644
--- a/tests/basic.py
+++ b/tests/basic.py
@@ -630,5 +630,65 @@ class TC_03_QvmRevertTemplateChanges(qubes.tests.SystemTestsMixin,
         self.setup_hvm_template()
         self._do_test()
 
+class TC_04_DispVM(qubes.tests.SystemTestsMixin,
+                   qubes.tests.QubesTestCase):
+
+    def test_000_firewall_propagation(self):
+        testvm1 = self.qc.add_new_vm("QubesAppVm",
+                                     name=self.make_vm_name('vm1'),
+                                     template=self.qc.get_default_template())
+        testvm1.create_on_disk(verbose=False)
+        firewall = testvm1.get_firewall_conf()
+        firewall['allowDns'] = False
+        firewall['allowYumProxy'] = False
+        firewall['rules'] = [{'address': '1.2.3.4',
+                              'netmask': 24,
+                              'proto': 'tcp',
+                              'portBegin': 22,
+                              'portEnd': 22,
+                              }]
+        testvm1.write_firewall_conf(firewall)
+        self.qc.save()
+        self.qc.unlock_db()
+
+        testvm1.start()
+
+        p = testvm1.run("qvm-run --dispvm 'qubesdb-read /name; echo ERROR;"
+                        " read x'",
+                        passio_popen=True)
+
+        dispvm_name = p.stdout.readline().strip()
+        self.qc.lock_db_for_reading()
+        self.qc.load()
+        self.qc.unlock_db()
+        dispvm = self.qc.get_vm_by_name(dispvm_name)
+        self.assertIsNotNone(dispvm, "DispVM {} not found in qubes.xml".format(
+            dispvm_name))
+        # FIXME: currently qubes.xml doesn't contain this information...
+        dispvm_template_name = os.path.basename(dispvm.dir_path)
+        dispvm_template = self.qc.get_vm_by_name(dispvm_template_name)
+        # check if firewall was propagated to the DispVM
+        self.assertEquals(testvm1.get_firewall_conf(),
+                          dispvm.get_firewall_conf())
+        # and only there (#1608)
+        self.assertNotEquals(dispvm_template.get_firewall_conf(),
+                             dispvm.get_firewall_conf())
+        # then modify some rule
+        firewall = dispvm.get_firewall_conf()
+        firewall['rules'] = [{'address': '4.3.2.1',
+                              'netmask': 24,
+                              'proto': 'tcp',
+                              'portBegin': 22,
+                              'portEnd': 22,
+                              }]
+        dispvm.write_firewall_conf(firewall)
+        # and check again if wasn't saved anywhere else (#1608)
+        self.assertNotEquals(dispvm_template.get_firewall_conf(),
+                             dispvm.get_firewall_conf())
+        self.assertNotEquals(testvm1.get_firewall_conf(),
+                             dispvm.get_firewall_conf())
+        p.stdin.write('\n')
+        p.wait()
+
 
 # vim: ts=4 sw=4 et

From 6a99b0b2ba9ca2bd8eafbd2cb3cce61cdb133171 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Thu, 14 Jan 2016 03:15:38 +0100
Subject: [PATCH 215/225] qmemman: add some useful logging for #1389

QubesOS/qubes-issues#1389
---
 qmemman/qmemman.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/qmemman/qmemman.py b/qmemman/qmemman.py
index 57d27370..8d1d60c7 100755
--- a/qmemman/qmemman.py
+++ b/qmemman/qmemman.py
@@ -47,6 +47,9 @@ class DomainState:
         self.no_progress = False    #no react to memset
         self.slow_memset_react = False #slow react to memset (after few tries still above target)
 
+    def __repr__(self):
+        return self.__dict__.__repr__()
+
 class SystemState(object):
     def __init__(self):
         self.log = logging.getLogger('qmemman.systemstate')
@@ -89,6 +92,17 @@ class SystemState(object):
             self.domdict.values(),
             0
         )
+        # If, at any time, Xen have less memory than XEN_FREE_MEM_MIN,
+        # it is a failure of qmemman. Collect as much data as possible to
+        # debug it
+        if xen_free < self.XEN_FREE_MEM_MIN:
+            self.log.error("Xen free = {!r} below acceptable value! "
+                           "assigned_but_unused={!r}, domdict={!r}".format(
+                xen_free, assigned_but_unused, self.domdict))
+        elif xen_free < assigned_but_unused+self.XEN_FREE_MEM_MIN:
+            self.log.error("Xen free = {!r} too small for satisfy assignments! "
+                           "assigned_but_unused={!r}, domdict={!r}".format(
+                xen_free, assigned_but_unused, self.domdict))
         return xen_free - assigned_but_unused
 
 #refresh information on memory assigned to all domains

From 3eccc3a6339383b172a912d087d5f901c334ef02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Thu, 14 Jan 2016 03:32:18 +0100
Subject: [PATCH 216/225] qmemman: use try/finally to really release the lock

Currently not needed in practice, but a preparation for the next
commit(s).

QubesOS/qubes-issues#1389
---
 qmemman/qmemman_server.py | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/qmemman/qmemman_server.py b/qmemman/qmemman_server.py
index 6e6fa94c..31aeb587 100755
--- a/qmemman/qmemman_server.py
+++ b/qmemman/qmemman_server.py
@@ -93,22 +93,22 @@ class XS_Watcher:
         self.log.debug('acquiring global_lock')
         global_lock.acquire()
         self.log.debug('global_lock acquired')
+        try:
+            for i in only_in_first_list(curr, self.watch_token_dict.keys()):
+                #new domain has been created
+                watch = WatchType(XS_Watcher.meminfo_changed, i)
+                self.watch_token_dict[i] = watch
+                self.handle.watch(get_domain_meminfo_key(i), watch)
+                system_state.add_domain(i)
 
-        for i in only_in_first_list(curr, self.watch_token_dict.keys()):
-#new domain has been created
-            watch = WatchType(XS_Watcher.meminfo_changed, i)
-            self.watch_token_dict[i] = watch
-            self.handle.watch(get_domain_meminfo_key(i), watch)
-            system_state.add_domain(i)
-
-        for i in only_in_first_list(self.watch_token_dict.keys(), curr):
-#domain destroyed
-            self.handle.unwatch(get_domain_meminfo_key(i), self.watch_token_dict[i])
-            self.watch_token_dict.pop(i)
-            system_state.del_domain(i)
-
-        global_lock.release()
-        self.log.debug('global_lock released')
+            for i in only_in_first_list(self.watch_token_dict.keys(), curr):
+                #domain destroyed
+                self.handle.unwatch(get_domain_meminfo_key(i), self.watch_token_dict[i])
+                self.watch_token_dict.pop(i)
+                system_state.del_domain(i)
+        finally:
+            global_lock.release()
+            self.log.debug('global_lock released')
 
         system_state.do_balance()
 

From caa75cbc187d3345a2579119b2ba6aef91bfc264 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Thu, 14 Jan 2016 04:28:17 +0100
Subject: [PATCH 217/225] tests: regression test for #1389

QubesOS/qubes-issues#1389
---
 tests/regressions.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/tests/regressions.py b/tests/regressions.py
index 3248e0da..f61f3a2b 100644
--- a/tests/regressions.py
+++ b/tests/regressions.py
@@ -24,11 +24,14 @@
 #
 
 import multiprocessing
+import os
 import time
 import unittest
 
 import qubes.qubes
 import qubes.tests
+import subprocess
+
 
 class TC_00_Regressions(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
     # Bug: #906
@@ -57,3 +60,22 @@ class TC_00_Regressions(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase)
         self.assertIsNotNone(qc.get_vm_by_name(vmname1))
         self.assertIsNotNone(qc.get_vm_by_name(vmname2))
 
+    def test_bug_1389_dispvm_qubesdb_crash(self):
+        """
+        Sometimes QubesDB instance in DispVM crashes at startup.
+        Unfortunately we don't have reliable way to reproduce it, so try twice
+        :return:
+        """
+        self.qc.unlock_db()
+        for try_no in xrange(2):
+            p = subprocess.Popen(['/usr/lib/qubes/qfile-daemon-dvm',
+                                  'qubes.VMShell', 'dom0', 'DEFAULT'],
+                                 stdin=subprocess.PIPE,
+                                 stdout=subprocess.PIPE,
+                                 stderr=open(os.devnull, 'w'))
+            p.stdin.write("qubesdb-read /name || echo ERROR\n")
+            dispvm_name = p.stdout.readline()
+            p.stdin.close()
+            self.assertTrue(dispvm_name.startswith("disp"),
+                                 "Try {} failed".format(try_no))
+

From 411e8b5323f139e75cbf51783ea62e7308a87350 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Thu, 14 Jan 2016 04:34:53 +0100
Subject: [PATCH 218/225] qmemman: refresh domain list holding global_lock

Retrieve a domain list only after obtaining global lock. Otherwise an
outdated list may be used, when a domain was introduced in the meantime
(starting a new domain is done with global lock held), leading to #1389.

QubesOS/qubes-issues#1389
---
 qmemman/qmemman_server.py | 36 +++++++++++++++++-------------------
 1 file changed, 17 insertions(+), 19 deletions(-)

diff --git a/qmemman/qmemman_server.py b/qmemman/qmemman_server.py
index 31aeb587..2af58956 100755
--- a/qmemman/qmemman_server.py
+++ b/qmemman/qmemman_server.py
@@ -73,36 +73,34 @@ class XS_Watcher:
     def domain_list_changed(self, param):
         self.log.debug('domain_list_changed(param={!r})'.format(param))
 
-        curr = self.handle.ls('', '/local/domain')
-
-        if curr == None:
-            return
-
-        # check if domain is really there, it may happen that some empty
-        # directories are left in xenstore
-        curr = filter(
-            lambda x:
-            self.handle.read('',
-                             '/local/domain/{}/domid'.format(x)
-                             ) is not None,
-            curr
-        )
-
-        self.log.debug('curr={!r}'.format(curr))
-
         self.log.debug('acquiring global_lock')
         global_lock.acquire()
         self.log.debug('global_lock acquired')
         try:
+            curr = self.handle.ls('', '/local/domain')
+            if curr is None:
+                return
+
+            # check if domain is really there, it may happen that some empty
+            # directories are left in xenstore
+            curr = filter(
+                lambda x:
+                self.handle.read('',
+                                 '/local/domain/{}/domid'.format(x)
+                                 ) is not None,
+                curr
+            )
+            self.log.debug('curr={!r}'.format(curr))
+
             for i in only_in_first_list(curr, self.watch_token_dict.keys()):
-                #new domain has been created
+                # new domain has been created
                 watch = WatchType(XS_Watcher.meminfo_changed, i)
                 self.watch_token_dict[i] = watch
                 self.handle.watch(get_domain_meminfo_key(i), watch)
                 system_state.add_domain(i)
 
             for i in only_in_first_list(self.watch_token_dict.keys(), curr):
-                #domain destroyed
+                # domain destroyed
                 self.handle.unwatch(get_domain_meminfo_key(i), self.watch_token_dict[i])
                 self.watch_token_dict.pop(i)
                 system_state.del_domain(i)

From 5d36923170dfb074b7d38d6b4aafb9c8624544f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Thu, 14 Jan 2016 04:37:02 +0100
Subject: [PATCH 219/225] qmemman: fix race condition on starting new VM

Force refreshing domain list after starting new VM, even if the
triggering watch wasn't about domain list change. Otherwise (with
outdated domain list) memory allocated to a new VM, but not yet used may
be redistributed (leaving to little to the Xen itself).

Fixes QubesOS/qubes-issues#1389
---
 qmemman/qmemman_server.py | 56 ++++++++++++++++++++++++++-------------
 1 file changed, 38 insertions(+), 18 deletions(-)

diff --git a/qmemman/qmemman_server.py b/qmemman/qmemman_server.py
index 2af58956..7c295243 100755
--- a/qmemman/qmemman_server.py
+++ b/qmemman/qmemman_server.py
@@ -42,6 +42,15 @@ LOG_PATH='/var/log/qubes/qmemman.log'
 
 system_state = SystemState()
 global_lock = thread.allocate_lock()
+# If XS_Watcher will
+# handle meminfo event before @introduceDomain, it will use
+# incomplete domain list for that and may redistribute memory
+# allocated to some VM, but not yet used (see #1389).
+# To fix that, system_state should be updated (refresh domain
+# list) before processing other changes, every time some process requested
+# memory for a new VM, before releasing the lock. Then XS_Watcher will check
+# this flag before processing other event.
+force_refresh_domain_list = False
 
 def only_in_first_list(l1, l2):
     ret=[]
@@ -65,17 +74,30 @@ class XS_Watcher:
         self.log.debug('XS_Watcher()')
 
         self.handle = xen.lowlevel.xs.xs()
-        self.handle.watch('@introduceDomain', WatchType(XS_Watcher.domain_list_changed, None))
-        self.handle.watch('@releaseDomain', WatchType(XS_Watcher.domain_list_changed, None))
+        self.handle.watch('@introduceDomain', WatchType(
+            XS_Watcher.domain_list_changed, False))
+        self.handle.watch('@releaseDomain', WatchType(
+            XS_Watcher.domain_list_changed, False))
         self.watch_token_dict = {}
 
+    def domain_list_changed(self, refresh_only=False):
+        """
+        Check if any domain was created/destroyed. If it was, update
+        appropriate list. Then redistribute memory.
 
-    def domain_list_changed(self, param):
-        self.log.debug('domain_list_changed(param={!r})'.format(param))
+        :param refresh_only If True, only refresh domain list, do not
+        redistribute memory. In this mode, caller must already hold
+        global_lock.
+        """
+        self.log.debug('domain_list_changed(only_refresh={!r})'.format(
+            refresh_only))
 
-        self.log.debug('acquiring global_lock')
-        global_lock.acquire()
-        self.log.debug('global_lock acquired')
+        got_lock = False
+        if not refresh_only:
+            self.log.debug('acquiring global_lock')
+            global_lock.acquire()
+            got_lock = True
+            self.log.debug('global_lock acquired')
         try:
             curr = self.handle.ls('', '/local/domain')
             if curr is None:
@@ -105,10 +127,12 @@ class XS_Watcher:
                 self.watch_token_dict.pop(i)
                 system_state.del_domain(i)
         finally:
-            global_lock.release()
-            self.log.debug('global_lock released')
+            if got_lock:
+                global_lock.release()
+                self.log.debug('global_lock released')
 
-        system_state.do_balance()
+        if not refresh_only:
+            system_state.do_balance()
 
 
     def meminfo_changed(self, domain_id):
@@ -120,6 +144,8 @@ class XS_Watcher:
         self.log.debug('acquiring global_lock')
         global_lock.acquire()
         self.log.debug('global_lock acquired')
+        if force_refresh_domain_list:
+            self.domain_list_changed(refresh_only=True)
 
         system_state.refresh_meminfo(domain_id, untrusted_meminfo_key)
 
@@ -155,15 +181,9 @@ class QMemmanReqHandler(SocketServer.BaseRequestHandler):
             self.log.debug('data={!r}'.format(self.data))
             if len(self.data) == 0:
                 self.log.info('EOF')
-                # FIXME: there is a race condition here: if XS_Watcher will
-                # handle meminfo event before @introduceDomain, it will use
-                # incomplete domain list for that and may redistribute memory
-                # allocated to some VM, but not yet used (see #1389).
-                # To fix that, system_state should be updated (refresh domain
-                #  list) before releasing the lock, but in the current code
-                # layout XS_Watcher instance isn't available here,
-                # so xenstore watches would not be registered
                 if got_lock:
+                    global force_refresh_domain_list
+                    force_refresh_domain_list = True
                     global_lock.release()
                     self.log.debug('global_lock released')
                 return

From 04415e13fed0e15539419ec930ae1046e4f2fcd7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Fri, 15 Jan 2016 17:35:25 +0100
Subject: [PATCH 220/225] version 3.1.11

---
 version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version b/version
index c7a24988..efd03d13 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.1.10
+3.1.11

From ce75ba411ffd652c524fd355a4fb8f99b84fe4b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 18 Jan 2016 02:09:17 +0100
Subject: [PATCH 221/225] tests: "empty" firewall propagation

Check if "empty" firewall is also properly propagated.

QubesOS/qubes-issues#1608
---
 tests/basic.py | 93 ++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 90 insertions(+), 3 deletions(-)

diff --git a/tests/basic.py b/tests/basic.py
index e53a0b55..8293b3d9 100644
--- a/tests/basic.py
+++ b/tests/basic.py
@@ -27,6 +27,7 @@ import multiprocessing
 import os
 import shutil
 import subprocess
+import tempfile
 
 import unittest
 import time
@@ -633,7 +634,20 @@ class TC_03_QvmRevertTemplateChanges(qubes.tests.SystemTestsMixin,
 class TC_04_DispVM(qubes.tests.SystemTestsMixin,
                    qubes.tests.QubesTestCase):
 
+    @staticmethod
+    def get_dispvm_template_name():
+        vmdir = os.readlink('/var/lib/qubes/dvmdata/vmdir')
+        return os.path.basename(vmdir)
+
     def test_000_firewall_propagation(self):
+        """
+        Check firewall propagation VM->DispVM, when VM have some firewall rules
+        """
+
+        # FIXME: currently qubes.xml doesn't contain this information...
+        dispvm_template_name = self.get_dispvm_template_name()
+        dispvm_template = self.qc.get_vm_by_name(dispvm_template_name)
+
         testvm1 = self.qc.add_new_vm("QubesAppVm",
                                      name=self.make_vm_name('vm1'),
                                      template=self.qc.get_default_template())
@@ -664,9 +678,6 @@ class TC_04_DispVM(qubes.tests.SystemTestsMixin,
         dispvm = self.qc.get_vm_by_name(dispvm_name)
         self.assertIsNotNone(dispvm, "DispVM {} not found in qubes.xml".format(
             dispvm_name))
-        # FIXME: currently qubes.xml doesn't contain this information...
-        dispvm_template_name = os.path.basename(dispvm.dir_path)
-        dispvm_template = self.qc.get_vm_by_name(dispvm_template_name)
         # check if firewall was propagated to the DispVM
         self.assertEquals(testvm1.get_firewall_conf(),
                           dispvm.get_firewall_conf())
@@ -690,5 +701,81 @@ class TC_04_DispVM(qubes.tests.SystemTestsMixin,
         p.stdin.write('\n')
         p.wait()
 
+    def test_001_firewall_propagation(self):
+        """
+        Check firewall propagation VM->DispVM, when VM have no firewall rules
+        """
+        testvm1 = self.qc.add_new_vm("QubesAppVm",
+                                     name=self.make_vm_name('vm1'),
+                                     template=self.qc.get_default_template())
+        testvm1.create_on_disk(verbose=False)
+        self.qc.save()
+        self.qc.unlock_db()
+
+        # FIXME: currently qubes.xml doesn't contain this information...
+        dispvm_template_name = self.get_dispvm_template_name()
+        dispvm_template = self.qc.get_vm_by_name(dispvm_template_name)
+        original_firewall = None
+        if os.path.exists(dispvm_template.firewall_conf):
+            original_firewall = tempfile.TemporaryFile()
+            with open(dispvm_template.firewall_conf) as f:
+                original_firewall.write(f.read())
+        try:
+
+            firewall = dispvm_template.get_firewall_conf()
+            firewall['allowDns'] = False
+            firewall['allowYumProxy'] = False
+            firewall['rules'] = [{'address': '1.2.3.4',
+                                  'netmask': 24,
+                                  'proto': 'tcp',
+                                  'portBegin': 22,
+                                  'portEnd': 22,
+                                  }]
+            dispvm_template.write_firewall_conf(firewall)
+
+            testvm1.start()
+
+            p = testvm1.run("qvm-run --dispvm 'qubesdb-read /name; echo ERROR;"
+                            " read x'",
+                            passio_popen=True)
+
+            dispvm_name = p.stdout.readline().strip()
+            self.qc.lock_db_for_reading()
+            self.qc.load()
+            self.qc.unlock_db()
+            dispvm = self.qc.get_vm_by_name(dispvm_name)
+            self.assertIsNotNone(dispvm, "DispVM {} not found in qubes.xml".format(
+                dispvm_name))
+            # check if firewall was propagated to the DispVM from the right VM
+            self.assertEquals(testvm1.get_firewall_conf(),
+                              dispvm.get_firewall_conf())
+            # and only there (#1608)
+            self.assertNotEquals(dispvm_template.get_firewall_conf(),
+                                 dispvm.get_firewall_conf())
+            # then modify some rule
+            firewall = dispvm.get_firewall_conf()
+            firewall['rules'] = [{'address': '4.3.2.1',
+                                  'netmask': 24,
+                                  'proto': 'tcp',
+                                  'portBegin': 22,
+                                  'portEnd': 22,
+                                  }]
+            dispvm.write_firewall_conf(firewall)
+            # and check again if wasn't saved anywhere else (#1608)
+            self.assertNotEquals(dispvm_template.get_firewall_conf(),
+                                 dispvm.get_firewall_conf())
+            self.assertNotEquals(testvm1.get_firewall_conf(),
+                                 dispvm.get_firewall_conf())
+            p.stdin.write('\n')
+            p.wait()
+        finally:
+            if original_firewall:
+                original_firewall.seek(0)
+                with open(dispvm_template.firewall_conf, 'w') as f:
+                    f.write(original_firewall.read())
+                original_firewall.close()
+            else:
+                os.unlink(dispvm_template.firewall_conf)
+
 
 # vim: ts=4 sw=4 et

From 873706428e0d8ea2d0cb135b687c3d17962db789 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 18 Jan 2016 02:19:19 +0100
Subject: [PATCH 222/225] dispvm: fix firewall propagation when the calling VM
 has no rules set

Fixes QubesOS/qubes-issues#1608
---
 dispvm/qfile-daemon-dvm | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/dispvm/qfile-daemon-dvm b/dispvm/qfile-daemon-dvm
index 26bea9ac..0421abb0 100755
--- a/dispvm/qfile-daemon-dvm
+++ b/dispvm/qfile-daemon-dvm
@@ -79,10 +79,13 @@ class QfileDaemonDvm:
                                            label=label)
         print >>sys.stderr, "time=%s, VM created" % (str(time.time()))
         # By default inherit firewall rules from calling VM
+        disp_firewall_conf = '/var/run/qubes/%s-firewall.xml' % dispvm.name
+        dispvm.firewall_conf = disp_firewall_conf
         if os.path.exists(vm.firewall_conf):
-            disp_firewall_conf = '/var/run/qubes/%s-firewall.xml' % dispvm.name
             shutil.copy(vm.firewall_conf, disp_firewall_conf)
-            dispvm.firewall_conf = disp_firewall_conf
+        elif vm.qid == 0 and os.path.exists(vm_disptempl.firewall_conf):
+            # for DispVM called from dom0, copy use rules from DispVM template
+            shutil.copy(vm_disptempl.firewall_conf, disp_firewall_conf)
         if len(sys.argv) > 5 and len(sys.argv[5]) > 0:
             assert os.path.exists(sys.argv[5]), "Invalid firewall.conf location"
             dispvm.firewall_conf = sys.argv[5]

From 176755d9e226bff41e08b4b4907cc389b76cb6f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 20 Jan 2016 02:48:46 +0100
Subject: [PATCH 223/225] tests: DispVM cleanup

QubesOS/qubes-issues#1660
---
 tests/basic.py | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/tests/basic.py b/tests/basic.py
index 8293b3d9..8e362671 100644
--- a/tests/basic.py
+++ b/tests/basic.py
@@ -777,5 +777,60 @@ class TC_04_DispVM(qubes.tests.SystemTestsMixin,
             else:
                 os.unlink(dispvm_template.firewall_conf)
 
+    def test_002_cleanup(self):
+        self.qc.unlock_db()
+        p = subprocess.Popen(['/usr/lib/qubes/qfile-daemon-dvm',
+                              'qubes.VMShell', 'dom0', 'DEFAULT'],
+                             stdin=subprocess.PIPE,
+                             stdout=subprocess.PIPE,
+                             stderr=open(os.devnull, 'w'))
+        (stdout, _) = p.communicate(input="echo test; qubesdb-read /name; "
+                                          "echo ERROR\n")
+        self.assertEquals(p.returncode, 0)
+        lines = stdout.splitlines()
+        self.assertEqual(lines[0], "test")
+        dispvm_name = lines[1]
+        self.qc.lock_db_for_reading()
+        self.qc.load()
+        self.qc.unlock_db()
+        dispvm = self.qc.get_vm_by_name(dispvm_name)
+        self.assertIsNone(dispvm, "DispVM {} still exists in qubes.xml".format(
+            dispvm_name))
+
+    def test_003_cleanup_destroyed(self):
+        """
+        Check if DispVM is properly removed even if it terminated itself (#1660)
+        :return:
+        """
+        self.qc.unlock_db()
+        p = subprocess.Popen(['/usr/lib/qubes/qfile-daemon-dvm',
+                              'qubes.VMShell', 'dom0', 'DEFAULT'],
+                             stdin=subprocess.PIPE,
+                             stdout=subprocess.PIPE,
+                             stderr=open(os.devnull, 'w'))
+        p.stdin.write("qubesdb-read /name\n")
+        p.stdin.write("echo ERROR\n")
+        p.stdin.write("poweroff\n")
+        # do not close p.stdin on purpose - wait to automatic disconnect when
+        #  domain is destroyed
+        timeout = 30
+        while timeout > 0:
+            if p.poll():
+                break
+            time.sleep(1)
+            timeout -= 1
+        # includes check for None - timeout
+        self.assertEquals(p.returncode, 0)
+        lines = p.stdout.read().splitlines()
+        dispvm_name = lines[0]
+        self.assertNotEquals(dispvm_name, "ERROR")
+        self.qc.lock_db_for_reading()
+        self.qc.load()
+        self.qc.unlock_db()
+        dispvm = self.qc.get_vm_by_name(dispvm_name)
+        self.assertIsNone(dispvm, "DispVM {} still exists in qubes.xml".format(
+            dispvm_name))
+
+
 
 # vim: ts=4 sw=4 et

From 689df4afd365501ca9c81d81adf9861d5867d27c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 20 Jan 2016 02:50:47 +0100
Subject: [PATCH 224/225] dispvm: cleanup DispVM even if was already destroyed

dispvm.force_shutdown() throw an exception if the VM is already dead.

Fixes QubesOS/qubes-issues#1660
---
 dispvm/qfile-daemon-dvm | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/dispvm/qfile-daemon-dvm b/dispvm/qfile-daemon-dvm
index 0421abb0..c582f79b 100755
--- a/dispvm/qfile-daemon-dvm
+++ b/dispvm/qfile-daemon-dvm
@@ -182,7 +182,11 @@ def main():
         subprocess.call(['/usr/lib/qubes/qrexec-client', '-d', dispvm.name,
                          user+':exec /usr/lib/qubes/qubes-rpc-multiplexer ' +
                          exec_index + " " + src_vmname])
-        dispvm.force_shutdown()
+        try:
+            dispvm.force_shutdown()
+        except QubesException:
+            # VM already destroyed
+            pass
         qfile.remove_disposable_from_qdb(dispvm.name)
 
 main()

From 31a3796ed344d69f4fddd9e65ebc6ed7e8a1baa4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 20 Jan 2016 02:51:57 +0100
Subject: [PATCH 225/225] dispvm: do not remove innocent files when cleaning up
 DispVM

Fixes QubesOS/qubes-issues#1662
---
 core-modules/01QubesDisposableVm.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/core-modules/01QubesDisposableVm.py b/core-modules/01QubesDisposableVm.py
index 03b10b93..4e3ebdd8 100644
--- a/core-modules/01QubesDisposableVm.py
+++ b/core-modules/01QubesDisposableVm.py
@@ -237,5 +237,9 @@ class QubesDisposableVm(QubesVm):
 
         return self.xid
 
+    def remove_from_disk(self):
+        # nothing to remove
+        pass
+
 # register classes
 register_qubes_vm_class(QubesDisposableVm)