Merge branch 'devel-20181205'

* devel-20181205:
  vm/dispvm: fix /qubes-vm-presistence qubesdb entry
  vm/mix/net: prevent setting provides_network=false if qube is still used
  tests: updates-available notification
  tests/network: reduce code duplication
  tests: listen on 'misc' socket too

qubes/tests/__init__.py

@@ -55,6 +55,7 @@ import qubes
 import qubes.api
 import qubes.api.admin
 import qubes.api.internal
+import qubes.api.misc
 import qubes.backup
 import qubes.config
 import qubes.devices
@@ -685,6 +686,7 @@ class SystemTestCase(QubesTestCase):
             qubes.api.create_servers(
                 qubes.api.admin.QubesAdminAPI,
                 qubes.api.internal.QubesInternalAPI,
+                qubes.api.misc.QubesMiscAPI,
                 app=self.app, debug=True))
 
         self.addCleanup(self.cleanup_app)

qubes/tests/integ/network.py

@@ -1100,8 +1100,9 @@ class VmUpdatesMixin(object):
     %changelog
     $ rpmbuild -bb test-pkg.spec
     $ cat test-pkg-1.0-1.fc21.x86_64.rpm | gzip | base64
+    $ cat test-pkg-1.1-1.fc21.x86_64.rpm | gzip | base64
     """
-    RPM_PACKAGE_GZIP_BASE64 = (
+    RPM_PACKAGE_GZIP_BASE64 = [(
         b"H4sIAPzRLlYAA+2Y728URRjHn7ueUCkERKJVJDnTxLSxs7293o8WOER6ljYYrtKCLUSa3"
         b"bnZ64bd22VmTq8nr4wJbwxvjNHIG0x8oTHGGCHB8AcYE1/0lS80GgmQFCJU3wgB4ZjdfZ"
         b"q2xDe8NNlvMjfzmeeZH7tPbl98b35169cOUEpIJiTxT9SIrmVUs2hWh8dUAp54dOrM14s"
@@ -1120,13 +1121,37 @@ class VmUpdatesMixin(object):
         b"n8jeg3X4QepP3i63po6oml+9t/CwJLya2Bn/ei6f7/4B3Ycdb0L3pt5Q5mNz16rWJ9fLk"
         b"vvOff/nxS7//8O2P2gvt7nDDnoV9L1du9N4+ucjl9u/8+a7dC5Nnvjlv9Ox5r+v9Cy0NE"
         b"m+c6rv60S/dZw98Gn6MNswcfQiWUvg3wBUAAA=="
-    )
+    ), (
+        b"H4sIAMY1B1wCA+2Y72scRRjH537U1MZorKLRVjgJSIKdvf11u3dq0jZJ0wRLL+1VvBRrn"
+        b"J2dvVu6t7vd3bN3sS9ECr6RIkgR9JXQF5aiIk1L/gJF8EVe+KqiKLQQi03tmypojXO3zz"
+        b"Vp8YW+3y/MPvOZ55lnZthhXjw3Lqx9n0FcqYiFEfaP17AkSLxZVJbQ/1QKbbl/6Mxnqyn"
+        b"o9iE0uMTtOPTPcTvIJw1w+8DdDCj1KPBozJlVbrO8OcC/xvORH8/P3AT/2+D/DfynuTtH"
+        b"FKtANKNo6KpKJIs3jSkl3VIkvSAWSiZTlCL3akhXZCKKKmPcaRRJqURFS2MlSTMsgyqMz"
+        b"6RUp8SQFcmixZJpMlEWi0w1da3Eu3p3+1uW1saPHfpWOSvNXtruDVx4+A0+eAolSpQoUa"
+        b"JEiRIlSpQoUaJEiRJBTWR9ff191K1p3FM3ySGUEbndjbp1jUwOYkzetkJMr07SqZukgX8"
+        b"B7ge+DvwI2qijPMjbE8A3gIeB11BcVxGBb8J8FfgW+PcA3wb/FPAfkG8G+C/wl4HvAFPg"
+        b"v4HtmLOPA/vAT8J534vPmB2C9T+NbfYp8C8DPx1zagfwSJwvpUO+ajye2gP55iF+BtiA+"
+        b"Nch3ow5/TkwA74IbAFfBnaAl2N+7IN4vfQK8Ffg/w74arx++grwtTg+s7PDk6hXn0OSIC"
+        b"Gozx3hYzmf0OOkxu6F1/oKlx2PEqfuhRFckv1zB1ClHUasgepR5L+Qz7MWafgOE6jXyCP"
+        b"Hdpst1CpqC5qK/qUaKIQBFQK/sbGTXmeET8KaCgW7bZsbj3dsY2TSa/gBC0NmTtsOO0ga"
+        b"LBxF4OuMTNk1nmtjbI60HY90g8MZ8iabC5hlt+53z4bVxVGkCKKgYgmpgiaIXdv5FgS52"
+        b"5dUQY6P37kbWzcVNzd1cVnO4VoO+7bPcvhV4jj8y4LAC8YsL2iQCIeMNgM7avNxfxeeWp"
+        b"guHz4yOz2/UCm/cnhy35jcG99/YHZislpd2Fup7OMR5YOVHLZYizI/sj035BBG/BdhP/A"
+        b"iRiMvwGEUeC5fuxYw6gUmrlGKw5N2ROuMh4c+o+FYvhkGeX7wPD9/PmBmnURgcJ0EJnOZ"
+        b"iSmV/kM4cV3PsN04uqGp/BM1XTZW4zkCm/L9kbDt0jrfk9cMcdM9absmjojhsI3NU4eE9"
+        b"d4R+LG4g1qbGFHf9lBrEclwnTCs3r1iuOY2u/+jGVm4iCwiyXpJE61SkUq6RhVW0FVFpo"
+        b"ZZ0oiu6ppuFSxSFBXTUOQCFRmhhElFQ9XNgiyJhbv/dnf8hnaeETR4R1+sHuX37+c/H/o"
+        b"kjZ5Nbe88bMvv7voJvYWeOYaGBn7IGkr6xb3X5vqiExNL585/+NyPX3/5jbBzfaibcHhl"
+        b"4vny9ZHfT6wG0Y6Lfrv/pZXKmS+WyPD4O/2nLy0KKHXo1OjVs1eGPn75o+5DvW3+6D9jd"
+        b"bFaTBcAAA=="
+    )]
 
     """
     Minimal package generated by running dh_make on empty directory
     Then cat test-pkg_1.0-1_amd64.deb | gzip | base64
+    Then cat test-pkg_1.1-1_amd64.deb | gzip | base64
     """
-    DEB_PACKAGE_GZIP_BASE64 = (
+    DEB_PACKAGE_GZIP_BASE64 = [(
         b"H4sIACTXLlYAA1O0SSxKzrDjSklNykzM003KzEssqlRQUDA0MTG1NDQwNDVTUDBQAAEIa"
         b"WhgYGZioqBgogADCVxGegZcyfl5JUX5OXoliUV66VVE6DcwheuX7+ZgAAEW5rdXHb0PG4"
         b"iwf5j3WfMT6zWzzMuZgoE3jjYraNzbbFKWGms0SaRw/r2SV23WZ4IdP8preM4yqf0jt95"
@@ -1145,7 +1170,28 @@ class VmUpdatesMixin(object):
         b"rJvxfVnh80oadq57OZxPaU1bbztv1yF365W4t45Yr+XrFzov237GVY1Zgf7NvE4+W2SuR"
         b"lQtLauR1TQ/mbOiIONYya6tU1jPGpWfk/i1+ttiXe3ZO14n0YOWggndznjGlGLyfVbBC6"
         b"MRP5aMM7aCco/s7sZqB8RlTQwADw8rnuT/sDHi7mUASjJFRAAbWwNLiAwAA"
-    )
+    ), (
+        b"H4sIAL05B1wCA1O0SSxKzrDjSklNykzM003KzEssqlRQUDA0NTG2NDc3NjdTUDBQAAEIa"
+        b"WhgYGZioqBgogADCVxGegZcyfl5JUX5OXoliUV66VVE6De3gOuX7+ZgAAEW5rdXzmbdMR"
+        b"BgSJj/VeQzQ+ztT/W+EVEnFraKOTlXh6+JXB8RbTRpzgWb2qdLX0+RmTRZcYlyxJutJsk"
+        b"/pfsfq9yqWZJ4JVVS97jBPPnz1yviluw51b0q4tnrWemCU2a/17mTUBYX0XBC6nH8rvvZ"
+        b"n/WP7nu40+Jlz7drPNLvCjULQkXOv677OV9s4bPsv5+tvCzPG8s57no479qV/5V/813Kh"
+        b"Wy3Pbj4827Jq5v6W/wk7zL1/+zbfH6btVb/3Pm5EapukaJvdgfcape/JZZWe+mZ4+Grby"
+        b"7UTaroPzyv9urC1W2MT9+F2bZtWJOyXfGo5dv7DGXJUzee+p930Od0j8QNceNHJffOTr2"
+        b"kOJe93mWG+nPdLsG6fz++MV5h1OGr0N9yf3N2ydzQ5x/E9Aw/s9xzmOpULnKtsSZqc/rr"
+        b"RQdf/Lu/ckKE9xU5VRuNehbzTr6789a+P2lt2zk5cFqe3N2289+j/hfH2X39/+nvc5vTW"
+        b"a/+83pvWqY3e93JWYsmup693HzCOPBk0LI9O7PtiqawN9y8eaTV75DLLL2dNWqTLsTsOn"
+        b"7wy0fTe5oLH//7eNf89Co3dRUHJmLRh20s/xhYJkoeYdBgYEhJLEkEJ4uKKkgKIJQyjI3"
+        b"gKeOveVVEFAMDY6bSPTMmBkVGMWAqKdF/uviB+n/GwlgGce49MrWMUw/IetlVih46o7Y4"
+        b"0uZe/t9lt85aMUrdWhjueTHRd1nr1uK830feH74vcPKU2pkbP4SZnta5PhC9dfPTqvv7f"
+        b"n068XRDRDzLuv8Oa5p1L+02ZN127vp6mzSzzFqpLkmbwyl131J1xW58YlcxXSWs0PTbpT"
+        b"z28ZUnE/e+NN93weAd40a/zzJ7+Re/v+R7+f3VBVFJCyZsv523ySJ12t7Nt5b8uBu8zuJ"
+        b"2Laer//nZCkbXlxtYXvvA8+VSVsCRpo8BawtftKWyZBjkWa6/0X7qXfbF9reH/ro6S63Y"
+        b"rCj8t8cltPIOj9H/8LyIxj6bMsZVVtu+ngj6MCNV5JXhOs07RXWxrb3xsqJMDRksx/5bO"
+        b"bNtevXz2cdpzzI19Roede4NXxAyK9Dlrtp8JtELLNPWbBe9HfJlj1Hiv69erIFBnX/Pe1"
+        b"4QnzLD+p2AiTc383/P+7sW3WoxnXra49iJKJeZy7gc9Z02S57qrvWW3day501VhsbPtfK"
+        b"C5nyBG9qjr08E59KY1vUTGRg7mRsCGBimFa+3sTPg7WYCSTBGRgEAzEOeH04EAAA="
+    )]
 
     def run_cmd(self, vm, cmd, user="root"):
         '''Run a command *cmd* in a *vm* as *user*. Return its exit code.
@@ -1162,6 +1208,15 @@ class VmUpdatesMixin(object):
             return e.returncode
         return 0
 
+    def assertRunCommandReturnCode(self, vm, cmd, expected_returncode):
+        p = self.loop.run_until_complete(
+            vm.run(cmd, user='root',
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE))
+        (stdout, stderr) = self.loop.run_until_complete(p.communicate())
+        self.assertIn(
+            self.loop.run_until_complete(p.wait()), expected_returncode,
+            '{}: {}\n{}'.format(cmd, stdout, stderr))
+
     def setUp(self):
         '''
         :type self: qubes.tests.SystemTestCase | VmUpdatesMixin
@@ -1176,6 +1231,7 @@ class VmUpdatesMixin(object):
         if self.template.count("debian"):
             self.update_cmd = "set -o pipefail; apt-get update 2>&1 | " \
                               "{ ! grep '^W:\|^E:'; }"
+            self.upgrade_cmd = "apt-get -V dist-upgrade -y"
             self.install_cmd = "apt-get install -y {}"
             self.install_test_cmd = "dpkg -l {}"
             self.exit_code_ok = [0]
@@ -1189,6 +1245,7 @@ class VmUpdatesMixin(object):
                 pass
             self.update_cmd = "{cmd} clean all; {cmd} check-update".format(
                 cmd=cmd)
+            self.upgrade_cmd = "{cmd} upgrade -y".format(cmd=cmd)
             self.install_cmd = cmd + " install -y {}"
             self.install_test_cmd = "rpm -q {}"
             self.exit_code_ok = [0, 100]
@@ -1206,28 +1263,22 @@ class VmUpdatesMixin(object):
         :type self: qubes.tests.SystemTestCase | VmUpdatesMixin
         '''
         self.app.save()
-        # reload the VM to have all the properties properly set (especially
-        # default netvm)
         self.testvm1 = self.app.domains[self.testvm1.qid]
         self.loop.run_until_complete(self.testvm1.start())
-        p = self.loop.run_until_complete(
-            self.testvm1.run(self.update_cmd, user='root',
-            stdout=subprocess.PIPE, stderr=subprocess.PIPE))
-        (stdout, stderr) = self.loop.run_until_complete(p.communicate())
-        self.assertIn(p.returncode, self.exit_code_ok,
-            '{}: {}\n{}'.format(self.update_cmd, stdout, stderr))
+        self.assertRunCommandReturnCode(self.testvm1,
+            self.update_cmd, self.exit_code_ok)
 
-    def create_repo_apt(self):
+    def create_repo_apt(self, version=0):
         '''
         :type self: qubes.tests.SystemTestCase | VmUpdatesMixin
         '''
-        pkg_file_name = "test-pkg_1.0-1_amd64.deb"
+        pkg_file_name = "test-pkg_1.{}-1_amd64.deb".format(version)
         self.loop.run_until_complete(self.netvm_repo.run_for_stdio('''
-            mkdir /tmp/apt-repo \
+            mkdir -p /tmp/apt-repo \
             && cd /tmp/apt-repo \
             && base64 -d | zcat > {}
             '''.format(pkg_file_name),
-            input=self.DEB_PACKAGE_GZIP_BASE64))
+            input=self.DEB_PACKAGE_GZIP_BASE64[version]))
         # do not assume dpkg-scanpackage installed
         packages_path = "dists/test/main/binary-amd64/Packages"
         self.loop.run_until_complete(self.netvm_repo.run_for_stdio('''
@@ -1240,19 +1291,21 @@ class VmUpdatesMixin(object):
                 >> {packages} \
             && echo SHA256: $(openssl sha256 -r {pkg} | cut -f 1 -d ' ') \
                 >> {packages} \
+            && sed -i -e "s,@SIZE@,$(stat -c %s {pkg})," {packages} \
             && gzip < {packages} > {packages}.gz
             '''.format(pkg=pkg_file_name, packages=packages_path),
             input='''\
 Package: test-pkg
-Version: 1.0-1
+Version: 1.{version}-1
 Architecture: amd64
 Maintainer: unknown <user@host>
 Installed-Size: 25
 Filename: {pkg}
-Size: 994
+Size: @SIZE@
 Section: unknown
 Priority: optional
-Description: Test package'''.format(pkg=pkg_file_name).encode('utf-8')))
+Description: Test package'''.format(pkg=pkg_file_name, version=version).encode(
+                'utf-8')))
 
         self.loop.run_until_complete(self.netvm_repo.run_for_stdio('''
             mkdir -p /tmp/apt-repo/dists/test \
@@ -1274,16 +1327,17 @@ Components: main
 SHA256:
 '''))
 
-    def create_repo_yum(self):
+    def create_repo_yum(self, version=0):
         '''
         :type self: qubes.tests.SystemTestCase | VmUpdatesMixin
         '''
-        pkg_file_name = "test-pkg-1.0-1.fc21.x86_64.rpm"
+        pkg_file_name = "test-pkg-1.{}-1.fc21.x86_64.rpm".format(version)
         self.loop.run_until_complete(self.netvm_repo.run_for_stdio('''
-            mkdir /tmp/yum-repo \
+            mkdir -p /tmp/yum-repo \
             && cd /tmp/yum-repo \
             && base64 -d | zcat > {}
-            '''.format(pkg_file_name), input=self.RPM_PACKAGE_GZIP_BASE64))
+            '''.format(pkg_file_name), input=self.RPM_PACKAGE_GZIP_BASE64[
+            version]))
 
         # createrepo is installed by default in Fedora template
         self.loop.run_until_complete(self.netvm_repo.run_for_stdio(
@@ -1308,6 +1362,12 @@ SHA256:
             self.skipTest("Template {} not supported by this test".format(
                 self.template))
 
+    def add_update_to_repo(self):
+        if self.template.count("debian") or self.template.count("whonix"):
+            self.create_repo_apt(1)
+        elif self.template.count("fedora"):
+            self.create_repo_yum(1)
+
     def configure_test_repo(self):
         """
         Configure test repository in test-vm and disable rest of them.
@@ -1373,28 +1433,69 @@ SHA256:
         with self.qrexec_policy('qubes.UpdatesProxy', self.testvm1,
                 '$default', action='allow,target=' + self.netvm_repo.name):
             # update repository metadata
-            p = self.loop.run_until_complete(self.testvm1.run(
-                self.update_cmd, user='root', stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE))
-            (stdout, stderr) = self.loop.run_until_complete(p.communicate())
-            self.assertIn(self.loop.run_until_complete(p.wait()), self.exit_code_ok,
-                '{}: {}\n{}'.format(self.update_cmd, stdout, stderr))
+            self.assertRunCommandReturnCode(self.testvm1,
+                self.update_cmd, self.exit_code_ok)
 
             # install test package
-            p = self.loop.run_until_complete(self.testvm1.run(
-                self.install_cmd.format('test-pkg'), user='root',
-                stdout=subprocess.PIPE, stderr=subprocess.PIPE))
-            (stdout, stderr) = self.loop.run_until_complete(p.communicate())
-            self.assertIn(self.loop.run_until_complete(p.wait()), self.exit_code_ok,
-                '{}: {}\n{}'.format(self.update_cmd, stdout, stderr))
+            self.assertRunCommandReturnCode(self.testvm1,
+                self.install_cmd.format('test-pkg'), self.exit_code_ok)
 
             # verify if it was really installed
-            p = self.loop.run_until_complete(self.testvm1.run(
-                self.install_test_cmd.format('test-pkg'), user='root',
-                stdout=subprocess.PIPE, stderr=subprocess.PIPE))
-            (stdout, stderr) = self.loop.run_until_complete(p.communicate())
-            self.assertIn(self.loop.run_until_complete(p.wait()), self.exit_code_ok,
-                '{}: {}\n{}'.format(self.update_cmd, stdout, stderr))
+            self.assertRunCommandReturnCode(self.testvm1,
+                self.install_test_cmd.format('test-pkg'), self.exit_code_ok)
+
+    def test_020_updates_available_notification(self):
+        # override with StandaloneVM
+        self.testvm1 = self.app.add_new_vm(
+            qubes.vm.standalonevm.StandaloneVM,
+            name=self.make_vm_name('vm2'),
+            label='red')
+        tpl = self.app.domains[self.template]
+        self.testvm1.clone_properties(tpl)
+        self.testvm1.features.update(tpl.features)
+        self.loop.run_until_complete(
+            self.testvm1.clone_disk_files(tpl))
+        self.loop.run_until_complete(self.testvm1.start())
+        self.netvm_repo = self.testvm1
+
+        self.create_repo_and_serve()
+        self.configure_test_repo()
+
+        self.loop.run_until_complete(
+            self.testvm1.run_for_stdio(
+                '/usr/lib/qubes/upgrades-status-notify',
+                user='root',
+            ))
+        self.assertFalse(self.testvm1.features.get('updates-available', False))
+
+        # update repository metadata
+        self.assertRunCommandReturnCode(
+            self.testvm1, self.update_cmd, self.exit_code_ok)
+
+        # install test package
+        self.assertRunCommandReturnCode(
+            self.testvm1, self.install_cmd.format('test-pkg'), self.exit_code_ok)
+
+        self.assertFalse(self.testvm1.features.get('updates-available', False))
+
+        self.add_update_to_repo()
+        # update repository metadata
+        self.assertRunCommandReturnCode(
+            self.testvm1, self.update_cmd, self.exit_code_ok)
+
+        self.loop.run_until_complete(
+            self.testvm1.run_for_stdio(
+                '/usr/lib/qubes/upgrades-status-notify',
+                user='root',
+            ))
+        self.assertTrue(self.testvm1.features.get('updates-available', False))
+
+        # install updates
+        self.assertRunCommandReturnCode(
+            self.testvm1, self.upgrade_cmd, self.exit_code_ok)
+
+        self.assertFalse(self.testvm1.features.get('updates-available', False))
+
 
 def create_testcases_for_templates():
     yield from qubes.tests.create_testcases_for_templates('VmNetworking',

qubes/tests/vm/mix/net.py

@@ -141,3 +141,16 @@ class TC_00_NetVMMixin(
         self.assertPropertyInvalidValue(vm, 'ip', 'zzzz')
         self.assertPropertyInvalidValue(vm, 'ip',
             '1:2:3:4:5:6:7:8:0:a:b:c:d:e:f:0')
+
+    def test_170_provides_network_netvm(self):
+        vm = self.get_vm()
+        vm2 = self.get_vm('test2', qid=3)
+        self.assertPropertyDefaultValue(vm, 'provides_network', False)
+        self.assertPropertyInvalidValue(vm2, 'netvm', vm)
+        self.assertPropertyValue(vm, 'provides_network', True, True, 'True')
+        self.assertPropertyValue(vm2, 'netvm', vm, vm, 'test-inst-test')
+        # used by other vm
+        self.assertPropertyInvalidValue(vm, 'provides_network', False)
+        self.assertPropertyValue(vm2, 'netvm', None, None, '')
+        self.assertPropertyValue(vm2, 'netvm', '', None, '')
+        self.assertPropertyValue(vm, 'provides_network', False, False, 'False')

qubes/vm/dispvm.py

@@ -213,3 +213,7 @@ class DispVM(qubes.vm.qubesvm.QubesVM):
             # Cleanup also on failed startup
             yield from self._auto_cleanup()
             raise
+
+    def create_qdb_entries(self):
+        super().create_qdb_entries()
+        self.untrusted_qdb.write('/qubes-vm-persistence', 'none')

qubes/vm/mix/net.py

@@ -80,6 +80,16 @@ def _setter_netvm(self, prop, value):
                 'Loops in network are unsupported')
     return value
 
+def _setter_provides_network(self, prop, value):
+    value = qubes.property.bool(self, prop, value)
+    if not value:
+        if list(self.connected_vms):
+            raise qubes.exc.QubesValueError(
+                'The qube is still used by other qubes, change theirs '
+                '\'netvm\' first')
+
+    return value
+
 
 class NetVMMixin(qubes.events.Emitter):
     ''' Mixin containing network functionality '''
@@ -105,7 +115,7 @@ class NetVMMixin(qubes.events.Emitter):
             NetVM.''')
 
     provides_network = qubes.property('provides_network', default=False,
-        type=bool, setter=qubes.property.bool,
+        type=bool, setter=_setter_provides_network,
         doc='''If this domain can act as network provider (formerly known as
             NetVM or ProxyVM)''')