Merge branch 'tests-and-fixes-20171205'

linux/aux-tools/cleanup-dispvms

@@ -1,16 +1,12 @@
 #!/usr/bin/python
 
-from qubes.qubes import QubesVmCollection
+from qubesadmin import Qubes
 
 def main():
-    qvm_collection = QubesVmCollection()
+    app = Qubes()
-    qvm_collection.lock_db_for_writing()
+    for vm in app.domains:
-    qvm_collection.load()
+        if vm.klass == 'DispVM' and not vm.is_running():
-    for vm in qvm_collection.values():
+            if vm.auto_cleanup:
-        if vm.is_disposablevm() and not vm.is_running():
+                del app.domains[vm]
-            qvm_collection.pop(vm.qid)
-    qvm_collection.save()
-    qvm_collection.unlock_db()
-
 
 main()

qubes/api/admin.py

@@ -1067,7 +1067,7 @@ class QubesAdminAPI(qubes.api.AbstractQubesAPI):
             non_default_attrs = set(attr for attr in dir(dev) if
                 not attr.startswith('_')).difference((
                     'backend_domain', 'ident', 'frontend_domain',
-                    'description', 'options'))
+                    'description', 'options', 'regex'))
             properties_txt = ' '.join(
                 '{}={!s}'.format(prop, value) for prop, value
                 in itertools.chain(

qubes/app.py

@@ -607,6 +607,14 @@ def _default_pool(app):
     if 'default' in app.pools:
         return app.pools['default']
     else:
+        if 'DEFAULT_LVM_POOL' in os.environ:
+            thin_pool = os.environ['DEFAULT_LVM_POOL']
+            for pool in app.pools.values():
+                if pool.config.get('driver', None) != 'lvm_thin':
+                    continue
+                if pool.config['thin_pool'] == thin_pool:
+                    return pool
+        # no DEFAULT_LVM_POOL, or pool not defined
         root_volume_group = RootThinPool.volume_group()
         root_thin_pool = RootThinPool.thin_pool()
         if root_thin_pool:
@@ -634,6 +642,27 @@ def _setter_pool(app, prop, value):
         raise qubes.exc.QubesPropertyValueError(app, prop, value,
             'No such storage pool')
 
+def _setter_default_netvm(app, prop, value):
+    # skip netvm loop check while loading qubes.xml, to avoid tricky loading
+    # order
+    if not app.events_enabled:
+        return value
+
+    if value is None:
+        return value
+    # forbid setting to a value that would result in netvm loop
+    for vm in app.domains:
+        if not hasattr(vm, 'netvm'):
+            continue
+        if not vm.property_is_default('netvm'):
+            continue
+        if value == vm \
+                or value in app.domains.get_vms_connected_to(vm):
+            raise qubes.exc.QubesPropertyValueError(app, prop, value,
+                'Network loop on \'{!s}\''.format(vm))
+    return value
+
+
 class Qubes(qubes.PropertyHolder):
     '''Main Qubes application
 
@@ -692,6 +721,7 @@ class Qubes(qubes.PropertyHolder):
 
     default_netvm = qubes.VMProperty('default_netvm', load_stage=3,
         default=None, allow_none=True,
+        setter=_setter_default_netvm,
         doc='''Default NetVM for AppVMs. Initial state is `None`, which means
             that AppVMs are not connected to the Internet.''')
     default_fw_netvm = qubes.VMProperty('default_fw_netvm', load_stage=3,
@@ -843,7 +873,7 @@ class Qubes(qubes.PropertyHolder):
 
         if 0 not in self.domains:
             self.domains.add(
-                qubes.vm.adminvm.AdminVM(self, None, qid=0, name='dom0'),
+                qubes.vm.adminvm.AdminVM(self, None),
                 _enable_events=False)
 
         # stage 3: load global properties

qubes/ext/pci.py

@@ -128,13 +128,13 @@ class PCIDevice(qubes.devices.DeviceInfo):
     # pylint: disable=too-few-public-methods
     regex = re.compile(
         r'^(?P<bus>[0-9a-f]+)_(?P<device>[0-9a-f]+)\.(?P<function>[0-9a-f]+)$')
-    libvirt_regex = re.compile(
+    _libvirt_regex = re.compile(
         r'^pci_0000_(?P<bus>[0-9a-f]+)_(?P<device>[0-9a-f]+)_'
         r'(?P<function>[0-9a-f]+)$')
 
     def __init__(self, backend_domain, ident, libvirt_name=None):
         if libvirt_name:
-            dev_match = self.libvirt_regex.match(libvirt_name)
+            dev_match = self._libvirt_regex.match(libvirt_name)
             assert dev_match
             ident = '{bus}_{device}.{function}'.format(**dev_match.groupdict())
 

qubes/ext/r3compatibility.py

@@ -33,7 +33,7 @@ yum_proxy_port = '8082'
 
 class R3Compatibility(qubes.ext.Extension):
     '''Maintain VM interface compatibility with R3.0 and R3.1.
-    At lease where possible.
+    At least where possible.
     '''
 
     features_to_services = {

qubes/tests/__init__.py

@@ -959,7 +959,7 @@ class SystemTestCase(QubesTestCase):
                 self.fail("Timeout while waiting for {} window to {}".format(
                     title, "show" if show else "hide")
                 )
-            time.sleep(0.1)
+            self.loop.run_until_complete(asyncio.sleep(0.1))
 
     def enter_keys_in_window(self, title, keys):
         """

qubes/tests/app.py

@@ -321,6 +321,30 @@ class TC_90_Qubes(qubes.tests.QubesTestCase):
         self.assertIn('service.clocksync', self.template.features)
         self.assertTrue(self.template.features['service.clocksync'])
 
+    def test_110_netvm_loop(self):
+        '''Netvm loop through default_netvm'''
+        netvm = self.app.add_new_vm('AppVM', name='test-net',
+            template=self.template, label='red')
+        try:
+            self.app.default_netvm = None
+            netvm.netvm = qubes.property.DEFAULT
+            with self.assertRaises(ValueError):
+                self.app.default_netvm = netvm
+        finally:
+            del netvm
+
+    def test_111_netvm_loop(self):
+        '''Netvm loop through default_netvm'''
+        netvm = self.app.add_new_vm('AppVM', name='test-net',
+            template=self.template, label='red')
+        try:
+            netvm.netvm = None
+            self.app.default_netvm = netvm
+            with self.assertRaises(ValueError):
+                netvm.netvm = qubes.property.DEFAULT
+        finally:
+            del netvm
+
     def test_200_remove_template(self):
         appvm = self.app.add_new_vm('AppVM', name='test-vm',
             template=self.template,
@@ -345,6 +369,7 @@ class TC_90_Qubes(qubes.tests.QubesTestCase):
         netvm = self.app.add_new_vm('AppVM', name='test-netvm',
             template=self.template, provides_network=True,
             label='red')
+        netvm.netvm = None
         self.app.default_netvm = netvm
         with mock.patch.object(self.app, 'vmm'):
             with self.assertRaises(qubes.exc.QubesVMInUseError):

qubes/tests/integ/network.py

@@ -57,7 +57,7 @@ class VmNetworkingMixin(object):
 
     def setUp(self):
         super(VmNetworkingMixin, self).setUp()
-        if self.template.startswith('whonix-gw'):
+        if self.template.startswith('whonix-'):
             self.skipTest("Test not supported here - Whonix uses its own "
                           "firewall settings")
         self.init_default_template(self.template)
@@ -325,6 +325,9 @@ class VmNetworkingMixin(object):
         self.loop.run_until_complete(self.testvm1.start())
 
         self.assertEqual(self.run_cmd(self.testvm1, self.ping_ip), 0)
+        self.assertEqual(self.run_cmd(self.testnetvm,
+            'iptables -I INPUT -i vif+ ! -s {} -p icmp -j LOG'.format(
+                self.testvm1.ip)), 0)
         self.loop.run_until_complete(self.testvm1.run_for_stdio(
             'ip addr flush dev eth0 && '
             'ip addr add 10.137.1.128/24 dev eth0 && '
@@ -332,6 +335,16 @@ class VmNetworkingMixin(object):
             user='root'))
         self.assertNotEqual(self.run_cmd(self.testvm1, self.ping_ip), 0,
                          "Spoofed ping should be blocked")
+        try:
+            (output, _) = self.loop.run_until_complete(
+                self.testnetvm.run_for_stdio('iptables -nxvL INPUT',
+                    user='root'))
+        except subprocess.CalledProcessError:
+            self.fail('iptables -nxvL INPUT failed')
+
+        output = output.decode().splitlines()
+        packets = output[2].lstrip().split()[0]
+        self.assertEquals(packets, '0', 'Some packet hit the INPUT rule')
 
     def test_100_late_xldevd_startup(self):
         """Regression test for #1990"""

qubes/tests/integ/vm_qrexec_gui.py

@@ -704,13 +704,17 @@ class TC_00_AppVMMixin(object):
             user='root'))
 
         with self.qrexec_policy('qubes.Filecopy', self.testvm1, self.testvm2):
-            with self.assertRaises(subprocess.CalledProcessError):
+            p = self.loop.run_until_complete(self.testvm1.run(
-                self.loop.run_until_complete(self.testvm1.run_for_stdio(
                 'qvm-move-to-vm {} /tmp/testfile'.format(
                     self.testvm2.name)))
 
             # Close GUI error message
+            try:
                 self.enter_keys_in_window('Error', ['Return'])
+            except subprocess.CalledProcessError:
+                pass
+            self.loop.run_until_complete(p.wait())
+            self.assertNotEqual(p.returncode, 0)
 
         # the file shouldn't be removed in source vm
         self.loop.run_until_complete(self.testvm1.run_for_stdio(

qubes/tests/vm/__init__.py

@@ -38,6 +38,23 @@ class TestHost(object):
         self.memory_total = 1000 * 1024
         self.no_cpus = 4
 
+class TestVMsCollection(dict):
+    def get_vms_connected_to(self, vm):
+        return set()
+
+    def close(self):
+        self.clear()
+
+class TestVolume(object):
+    def __init__(self, pool):
+        self.pool = pool
+        self.size = 0
+        self.source = None
+
+class TestPool(object):
+    def init_volume(self, *args, **kwargs):
+        return TestVolume(self)
+
 class TestApp(qubes.tests.TestEmitter):
     labels = {1: qubes.Label(1, '0xcc0000', 'red')}
     check_updates_vm = False
@@ -58,12 +75,18 @@ class TestApp(qubes.tests.TestEmitter):
         super(TestApp, self).__init__()
         self.vmm = TestVMM()
         self.host = TestHost()
-        self.pools = {}
+        default_pool = TestPool()
+        self.pools = {
+            'default': default_pool,
+            default_pool: default_pool,
+            'linux-kernel': TestPool(),
+        }
         self.default_pool_volatile = 'default'
         self.default_pool_root = 'default'
         self.default_pool_private = 'default'
         self.default_pool_kernel = 'linux-kernel'
-        self.domains = {}
+        self.default_netvm = None
+        self.domains = TestVMsCollection()
         #: jinja2 environment for libvirt XML templates
         self.env = jinja2.Environment(
             loader=jinja2.FileSystemLoader([

qubes/tests/vm/mix/net.py

@@ -40,10 +40,10 @@ class TC_00_NetVMMixin(
         # testing properties used here
         self.netvm1 = qubes.vm.qubesvm.QubesVM(self.app, None, qid=2,
             name=qubes.tests.VMPREFIX + 'netvm1',
-            provides_network=True)
+            provides_network=True, netvm=None)
         self.netvm2 = qubes.vm.qubesvm.QubesVM(self.app, None, qid=3,
             name=qubes.tests.VMPREFIX + 'netvm2',
-            provides_network=True)
+            provides_network=True, netvm=None)
         self.nonetvm = qubes.vm.qubesvm.QubesVM(self.app, None, qid=4,
             name=qubes.tests.VMPREFIX + 'nonet')
         self.app.domains = qubes.app.VMCollection(self.app)
@@ -57,7 +57,10 @@ class TC_00_NetVMMixin(
         self.netvm1.close()
         self.netvm2.close()
         self.nonetvm.close()
+        try:
             self.app.domains.close()
+        except AttributeError:
+            pass
         del self.netvm1
         del self.netvm2
         del self.nonetvm

qubes/tests/vm/qubesvm.py

@@ -19,7 +19,7 @@
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, see <https://www.gnu.org/licenses/>.
 #
-
+import base64
 import os
 
 import unittest
@@ -54,6 +54,20 @@ class TestDeviceCollection(object):
     def persistent(self):
         return self._list
 
+class TestQubesDB(object):
+    def __init__(self):
+        self.data = {}
+
+    def write(self, path, value):
+        self.data[path] = value
+
+    def rm(self, path):
+        if path.endswith('/'):
+            for key in [x for x in self.data if x.startswith(path)]:
+                del self.data[key]
+        else:
+            self.data.pop(path, None)
+
 class TestVM(object):
     # pylint: disable=too-few-public-methods
     app = TestApp()
@@ -133,11 +147,25 @@ class QubesVMTestsMixin(object):
         super(QubesVMTestsMixin, self).setUp()
         self.app = qubes.tests.vm.TestApp()
         self.app.vmm.offline_mode = True
+        # when full test run is called, extensions are loaded by earlier
+        # tests, but if just this test class is run, load them manually here,
+        # to have the same behaviour
+        qubes.ext.get_extensions()
 
-    def get_vm(self, name='test', **kwargs):
+    def tearDown(self):
-        vm = qubes.vm.qubesvm.QubesVM(self.app, None,
+        try:
-            qid=1, name=qubes.tests.VMPREFIX + name,
+            self.app.domains.close()
+        except AttributeError:
+            pass
+        super(QubesVMTestsMixin, self).tearDown()
+
+    def get_vm(self, name='test', cls=qubes.vm.qubesvm.QubesVM, **kwargs):
+        vm = cls(self.app, None,
+            qid=kwargs.pop('qid', 1), name=qubes.tests.VMPREFIX + name,
             **kwargs)
+        self.app.domains[vm.qid] = vm
+        self.app.domains[vm.uuid] = vm
+        self.app.domains[vm] = vm
         self.addCleanup(vm.close)
         return vm
 
@@ -685,3 +713,118 @@ class TC_90_QubesVM(QubesVMTestsMixin, qubes.tests.QubesTestCase):
         libvirt_xml = vm.create_config_file()
         self.assertXMLEqual(lxml.etree.XML(libvirt_xml),
             lxml.etree.XML(expected))
+
+    @unittest.mock.patch('qubes.utils.get_timezone')
+    @unittest.mock.patch('qubes.utils.urandom')
+    @unittest.mock.patch('qubes.vm.qubesvm.QubesVM.untrusted_qdb')
+    def test_620_qdb_standalone(self, mock_qubesdb, mock_urandom,
+            mock_timezone):
+        mock_urandom.return_value = b'A' * 64
+        mock_timezone.return_value = 'UTC'
+        vm = self.get_vm(cls=qubes.vm.standalonevm.StandaloneVM)
+        vm.netvm = None
+        vm.events_enabled = True
+        test_qubesdb = TestQubesDB()
+        mock_qubesdb.write.side_effect = test_qubesdb.write
+        mock_qubesdb.rm.side_effect = test_qubesdb.rm
+        vm.create_qdb_entries()
+        self.maxDiff = None
+
+        iptables_header = (
+            '# Generated by Qubes Core on {}\n'
+            '*filter\n'
+            ':INPUT DROP [0:0]\n'
+            ':FORWARD DROP [0:0]\n'
+            ':OUTPUT ACCEPT [0:0]\n'
+            '-A INPUT -i vif+ -p udp -m udp --dport 68 -j DROP\n'
+            '-A INPUT -m conntrack --ctstate '
+            'RELATED,ESTABLISHED -j ACCEPT\n'
+            '-A INPUT -p icmp -j ACCEPT\n'
+            '-A INPUT -i lo -j ACCEPT\n'
+            '-A INPUT -j REJECT --reject-with '
+            'icmp-host-prohibited\n'
+            '-A FORWARD -m conntrack --ctstate '
+            'RELATED,ESTABLISHED -j ACCEPT\n'
+            '-A FORWARD -i vif+ -o vif+ -j DROP\n'
+            'COMMIT\n'.format(datetime.datetime.now().ctime()))
+
+        self.assertEqual(test_qubesdb.data, {
+            '/name': 'test-inst-test',
+            '/type': 'StandaloneVM',
+            '/qubes-vm-type': 'AppVM',
+            '/qubes-debug-mode': '0',
+            '/qubes-base-template': '',
+            '/qubes-timezone': 'UTC',
+            '/qubes-random-seed': base64.b64encode(b'A' * 64),
+            '/qubes-vm-persistence': 'full',
+            '/qubes-vm-updateable': 'True',
+            '/qubes-block-devices': '',
+            '/qubes-usb-devices': '',
+            '/qubes-iptables': 'reload',
+            '/qubes-iptables-error': '',
+            '/qubes-iptables-header': iptables_header,
+            '/qubes-service/qubes-update-check': '0',
+        })
+
+    @unittest.mock.patch('qubes.utils.get_timezone')
+    @unittest.mock.patch('qubes.utils.urandom')
+    @unittest.mock.patch('qubes.vm.qubesvm.QubesVM.untrusted_qdb')
+    def test_621_qdb_appvm_with_network(self, mock_qubesdb, mock_urandom,
+            mock_timezone):
+        mock_urandom.return_value = b'A' * 64
+        mock_timezone.return_value = 'UTC'
+        template = self.get_vm(cls=qubes.vm.templatevm.TemplateVM, name='template')
+        template.netvm = None
+        netvm = self.get_vm(cls=qubes.vm.standalonevm.StandaloneVM,
+            name='netvm', qid=2, provides_network=True)
+        vm = self.get_vm(cls=qubes.vm.appvm.AppVM, template=template,
+            name='appvm', qid=3)
+        vm.netvm = netvm
+        test_qubesdb = TestQubesDB()
+        mock_qubesdb.write.side_effect = test_qubesdb.write
+        mock_qubesdb.rm.side_effect = test_qubesdb.rm
+        self.maxDiff = None
+
+        iptables_header = (
+            '# Generated by Qubes Core on {}\n'
+            '*filter\n'
+            ':INPUT DROP [0:0]\n'
+            ':FORWARD DROP [0:0]\n'
+            ':OUTPUT ACCEPT [0:0]\n'
+            '-A INPUT -i vif+ -p udp -m udp --dport 68 -j DROP\n'
+            '-A INPUT -m conntrack --ctstate '
+            'RELATED,ESTABLISHED -j ACCEPT\n'
+            '-A INPUT -p icmp -j ACCEPT\n'
+            '-A INPUT -i lo -j ACCEPT\n'
+            '-A INPUT -j REJECT --reject-with '
+            'icmp-host-prohibited\n'
+            '-A FORWARD -m conntrack --ctstate '
+            'RELATED,ESTABLISHED -j ACCEPT\n'
+            '-A FORWARD -i vif+ -o vif+ -j DROP\n'
+            'COMMIT\n'.format(datetime.datetime.now().ctime()))
+
+        expected = {
+            '/name': 'test-inst-appvm',
+            '/type': 'AppVM',
+            '/qubes-vm-type': 'AppVM',
+            '/qubes-debug-mode': '0',
+            '/qubes-base-template': 'test-inst-template',
+            '/qubes-timezone': 'UTC',
+            '/qubes-random-seed': base64.b64encode(b'A' * 64),
+            '/qubes-vm-persistence': 'rw-only',
+            '/qubes-vm-updateable': 'False',
+            '/qubes-block-devices': '',
+            '/qubes-usb-devices': '',
+            '/qubes-iptables': 'reload',
+            '/qubes-iptables-error': '',
+            '/qubes-iptables-header': iptables_header,
+            '/qubes-service/qubes-update-check': '0',
+            '/qubes-ip': '10.137.0.3',
+            '/qubes-netmask': '255.255.255.255',
+            '/qubes-gateway': '10.137.0.2',
+            '/qubes-primary-dns': '10.139.1.1',
+            '/qubes-secondary-dns': '10.139.1.2',
+        }
+
+        vm.create_qdb_entries()
+        self.assertEqual(test_qubesdb.data, expected)

qubes/vm/__init__.py

@@ -354,6 +354,9 @@ class BaseVM(qubes.PropertyHolder):
         del self.tags
 
     def load_extras(self):
+        if self.xml is None:
+            return
+
         # features
         for node in self.xml.xpath('./features/feature'):
             self.features[node.get('name')] = node.text

qubes/vm/mix/net.py

@@ -70,6 +70,9 @@ def _setter_netvm(self, prop, value):
         raise qubes.exc.QubesValueError(
             'The {!s} qube does not provide network'.format(value))
 
+    # skip check for netvm loops during qubes.xml loading, to avoid tricky
+    # loading order
+    if self.events_enabled:
         if value is self \
                 or value in self.app.domains.get_vms_connected_to(self):
             raise qubes.exc.QubesValueError(
@@ -187,6 +190,22 @@ class NetVMMixin(qubes.events.Emitter):
         self._firewall = None
         super(NetVMMixin, self).__init__(*args, **kwargs)
 
+    @qubes.events.handler('domain-load')
+    def on_domain_load_netvm_loop_check(self, event):
+        # pylint: disable=unused-argument
+        # make sure there are no netvm loops - which could cause qubesd
+        # looping infinitely
+        if self is self.netvm:
+            self.log.error(
+                'vm \'%s\' network-connected to itself, breaking the '
+                'connection', self.name)
+            self.netvm = None
+        elif self.netvm in self.app.domains.get_vms_connected_to(self):
+            self.log.error(
+                'netvm loop detected on \'%s\', breaking the connection',
+                self.name)
+            self.netvm = None
+
     @qubes.events.handler('domain-start')
     def on_domain_started(self, event, **kwargs):
         '''Connect this domain to its downstream domains. Also reload firewall
@@ -323,6 +342,8 @@ class NetVMMixin(qubes.events.Emitter):
         # pylint: disable=unused-argument
         # we are changing to default netvm
         newvalue = type(self).netvm.get_default(self)
+        # check for netvm loop
+        _setter_netvm(self, type(self).netvm, newvalue)
         if newvalue == oldvalue:
             return
         self.fire_event('property-pre-set:netvm', pre_event=True,

qubes/vm/qubesvm.py

@@ -1244,7 +1244,12 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
             else:
                 stubdom_mem = 0
 
-            mem_required = int(self.memory + stubdom_mem) * 1024 * 1024
+            initial_memory = self.memory
+            if self.virt_mode == 'hvm' and self.devices['pci'].persistent():
+                # HVM with PCI devices does not support populate-on-demand on
+                #  Xen
+                initial_memory = self.maxmem
+            mem_required = int(initial_memory + stubdom_mem) * 1024 * 1024
 
         qmemman_client = qubes.qmemman.client.QMemmanClient()
         try:

qvm-tools/qubes-hcl-report

@@ -112,7 +112,7 @@ XL_VTX=`cat $TEMP_DIR/xl-info |grep xen_caps | grep hvm`
 XL_VTD=`cat $TEMP_DIR/xl-info |grep virt_caps |grep hvm_directio`
 XL_HAP=`cat $TEMP_DIR/xl-dmesg |grep "$XL_DMESG_PREFIX_REGEX"'HVM: Hardware Assisted Paging (HAP) detected\( but disabled\)\?$'`
 PCRS=`find /sys/devices/ -name pcrs`
-XL_REMAP=`cat $TEMP_DIR/xl-dmesg |grep "$XL_DMESG_PREFIX_REGEX"'Intel VT-d Interrupt Remapping enabled'`
+XL_REMAP=`cat $TEMP_DIR/xl-dmesg |grep "$XL_DMESG_PREFIX_REGEX"'\(Intel VT-d Interrupt Remapping enabled\|Interrupt remapping enabled\)'`
 
 
 FILENAME="Qubes-HCL-${BRAND//[^[:alnum:]]/_}-${PRODUCT//[^[:alnum:]]/_}-$DATE"