Qubes/core-admin
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'tests-and-fixes-20171205'

Browse Source
This commit is contained in:
Marek Marczykowski-Górecki 2017-12-07 01:39:34 +01:00
commit f2cd7fb226
No known key found for this signature in database
GPG Key ID: 063938BA42CFA724
16 changed files with 304 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
linux/aux-tools/cleanup-dispvms
View File

@ -1,16 +1,12 @@
#!/usr/bin/python
from qubes.qubes import QubesVmCollection
from qubesadmin import Qubes
def main():
qvm_collection = QubesVmCollection()
qvm_collection.lock_db_for_writing()
qvm_collection.load()
for vm in qvm_collection.values():
if vm.is_disposablevm() and not vm.is_running():
qvm_collection.pop(vm.qid)
qvm_collection.save()
qvm_collection.unlock_db()
app = Qubes()
for vm in app.domains:
if vm.klass == 'DispVM' and not vm.is_running():
if vm.auto_cleanup:
del app.domains[vm]
main()

2
qubes/api/admin.py
View File

@ -1067,7 +1067,7 @@ class QubesAdminAPI(qubes.api.AbstractQubesAPI):
non_default_attrs = set(attr for attr in dir(dev) if
not attr.startswith('_')).difference((
'backend_domain', 'ident', 'frontend_domain',
'description', 'options'))
'description', 'options', 'regex'))
properties_txt = ' '.join(
'{}={!s}'.format(prop, value) for prop, value
in itertools.chain(

32
qubes/app.py
View File

@ -607,6 +607,14 @@ def _default_pool(app):
if 'default' in app.pools:
return app.pools['default']
else:
if 'DEFAULT_LVM_POOL' in os.environ:
thin_pool = os.environ['DEFAULT_LVM_POOL']
for pool in app.pools.values():
if pool.config.get('driver', None) != 'lvm_thin':
continue
if pool.config['thin_pool'] == thin_pool:
return pool
# no DEFAULT_LVM_POOL, or pool not defined
root_volume_group = RootThinPool.volume_group()
root_thin_pool = RootThinPool.thin_pool()
if root_thin_pool:
@ -634,6 +642,27 @@ def _setter_pool(app, prop, value):
raise qubes.exc.QubesPropertyValueError(app, prop, value,
'No such storage pool')
def _setter_default_netvm(app, prop, value):
# skip netvm loop check while loading qubes.xml, to avoid tricky loading
# order
if not app.events_enabled:
return value
if value is None:
return value
# forbid setting to a value that would result in netvm loop
for vm in app.domains:
if not hasattr(vm, 'netvm'):
continue
if not vm.property_is_default('netvm'):
continue
if value == vm \
or value in app.domains.get_vms_connected_to(vm):
raise qubes.exc.QubesPropertyValueError(app, prop, value,
'Network loop on \'{!s}\''.format(vm))
return value
class Qubes(qubes.PropertyHolder):
'''Main Qubes application
@ -692,6 +721,7 @@ class Qubes(qubes.PropertyHolder):
default_netvm = qubes.VMProperty('default_netvm', load_stage=3,
default=None, allow_none=True,
setter=_setter_default_netvm,
doc='''Default NetVM for AppVMs. Initial state is `None`, which means
that AppVMs are not connected to the Internet.''')
default_fw_netvm = qubes.VMProperty('default_fw_netvm', load_stage=3,
@ -843,7 +873,7 @@ class Qubes(qubes.PropertyHolder):
if 0 not in self.domains:
self.domains.add(
qubes.vm.adminvm.AdminVM(self, None, qid=0, name='dom0'),
qubes.vm.adminvm.AdminVM(self, None),
_enable_events=False)
# stage 3: load global properties

4
qubes/ext/pci.py
View File

@ -128,13 +128,13 @@ class PCIDevice(qubes.devices.DeviceInfo):
# pylint: disable=too-few-public-methods
regex = re.compile(
r'^(?P<bus>[0-9a-f]+)_(?P<device>[0-9a-f]+)\.(?P<function>[0-9a-f]+)$')
libvirt_regex = re.compile(
_libvirt_regex = re.compile(
r'^pci_0000_(?P<bus>[0-9a-f]+)_(?P<device>[0-9a-f]+)_'
r'(?P<function>[0-9a-f]+)$')
def __init__(self, backend_domain, ident, libvirt_name=None):
if libvirt_name:
dev_match = self.libvirt_regex.match(libvirt_name)
dev_match = self._libvirt_regex.match(libvirt_name)
assert dev_match
ident = '{bus}_{device}.{function}'.format(**dev_match.groupdict())

2
qubes/ext/r3compatibility.py
View File

@ -33,7 +33,7 @@ yum_proxy_port = '8082'
class R3Compatibility(qubes.ext.Extension):
'''Maintain VM interface compatibility with R3.0 and R3.1.
At lease where possible.
At least where possible.
'''
features_to_services = {

2
qubes/tests/__init__.py
View File

@ -959,7 +959,7 @@ class SystemTestCase(QubesTestCase):
self.fail("Timeout while waiting for {} window to {}".format(
title, "show" if show else "hide")
)
time.sleep(0.1)
self.loop.run_until_complete(asyncio.sleep(0.1))
def enter_keys_in_window(self, title, keys):
"""

25
qubes/tests/app.py
View File

@ -321,6 +321,30 @@ class TC_90_Qubes(qubes.tests.QubesTestCase):
self.assertIn('service.clocksync', self.template.features)
self.assertTrue(self.template.features['service.clocksync'])
def test_110_netvm_loop(self):
'''Netvm loop through default_netvm'''
netvm = self.app.add_new_vm('AppVM', name='test-net',
template=self.template, label='red')
try:
self.app.default_netvm = None
netvm.netvm = qubes.property.DEFAULT
with self.assertRaises(ValueError):
self.app.default_netvm = netvm
finally:
del netvm
def test_111_netvm_loop(self):
'''Netvm loop through default_netvm'''
netvm = self.app.add_new_vm('AppVM', name='test-net',
template=self.template, label='red')
try:
netvm.netvm = None
self.app.default_netvm = netvm
with self.assertRaises(ValueError):
netvm.netvm = qubes.property.DEFAULT
finally:
del netvm
def test_200_remove_template(self):
appvm = self.app.add_new_vm('AppVM', name='test-vm',
template=self.template,
@ -345,6 +369,7 @@ class TC_90_Qubes(qubes.tests.QubesTestCase):
netvm = self.app.add_new_vm('AppVM', name='test-netvm',
template=self.template, provides_network=True,
label='red')
netvm.netvm = None
self.app.default_netvm = netvm
with mock.patch.object(self.app, 'vmm'):
with self.assertRaises(qubes.exc.QubesVMInUseError):

15
qubes/tests/integ/network.py
View File

@ -57,7 +57,7 @@ class VmNetworkingMixin(object):
def setUp(self):
super(VmNetworkingMixin, self).setUp()
if self.template.startswith('whonix-gw'):
if self.template.startswith('whonix-'):
self.skipTest("Test not supported here - Whonix uses its own "
"firewall settings")
self.init_default_template(self.template)
@ -325,6 +325,9 @@ class VmNetworkingMixin(object):
self.loop.run_until_complete(self.testvm1.start())
self.assertEqual(self.run_cmd(self.testvm1, self.ping_ip), 0)
self.assertEqual(self.run_cmd(self.testnetvm,
'iptables -I INPUT -i vif+ ! -s {} -p icmp -j LOG'.format(
self.testvm1.ip)), 0)
self.loop.run_until_complete(self.testvm1.run_for_stdio(
'ip addr flush dev eth0 && '
'ip addr add 10.137.1.128/24 dev eth0 && '
@ -332,6 +335,16 @@ class VmNetworkingMixin(object):
user='root'))
self.assertNotEqual(self.run_cmd(self.testvm1, self.ping_ip), 0,
"Spoofed ping should be blocked")
try:
(output, _) = self.loop.run_until_complete(
self.testnetvm.run_for_stdio('iptables -nxvL INPUT',
user='root'))
except subprocess.CalledProcessError:
self.fail('iptables -nxvL INPUT failed')
output = output.decode().splitlines()
packets = output[2].lstrip().split()[0]
self.assertEquals(packets, '0', 'Some packet hit the INPUT rule')
def test_100_late_xldevd_startup(self):
"""Regression test for #1990"""

16
qubes/tests/integ/vm_qrexec_gui.py
View File

@ -704,13 +704,17 @@ class TC_00_AppVMMixin(object):
user='root'))
with self.qrexec_policy('qubes.Filecopy', self.testvm1, self.testvm2):
with self.assertRaises(subprocess.CalledProcessError):
self.loop.run_until_complete(self.testvm1.run_for_stdio(
'qvm-move-to-vm {} /tmp/testfile'.format(
self.testvm2.name)))
p = self.loop.run_until_complete(self.testvm1.run(
'qvm-move-to-vm {} /tmp/testfile'.format(
self.testvm2.name)))
# Close GUI error message
self.enter_keys_in_window('Error', ['Return'])
# Close GUI error message
try:
self.enter_keys_in_window('Error', ['Return'])
except subprocess.CalledProcessError:
pass
self.loop.run_until_complete(p.wait())
self.assertNotEqual(p.returncode, 0)
# the file shouldn't be removed in source vm
self.loop.run_until_complete(self.testvm1.run_for_stdio(

27
qubes/tests/vm/__init__.py
View File

@ -38,6 +38,23 @@ class TestHost(object):
self.memory_total = 1000 * 1024
self.no_cpus = 4
class TestVMsCollection(dict):
def get_vms_connected_to(self, vm):
return set()
def close(self):
self.clear()
class TestVolume(object):
def __init__(self, pool):
self.pool = pool
self.size = 0
self.source = None
class TestPool(object):
def init_volume(self, *args, **kwargs):
return TestVolume(self)
class TestApp(qubes.tests.TestEmitter):
labels = {1: qubes.Label(1, '0xcc0000', 'red')}
check_updates_vm = False
@ -58,12 +75,18 @@ class TestApp(qubes.tests.TestEmitter):
super(TestApp, self).__init__()
self.vmm = TestVMM()
self.host = TestHost()
self.pools = {}
default_pool = TestPool()
self.pools = {
'default': default_pool,
default_pool: default_pool,
'linux-kernel': TestPool(),
}
self.default_pool_volatile = 'default'
self.default_pool_root = 'default'
self.default_pool_private = 'default'
self.default_pool_kernel = 'linux-kernel'
self.domains = {}
self.default_netvm = None
self.domains = TestVMsCollection()
#: jinja2 environment for libvirt XML templates
self.env = jinja2.Environment(
loader=jinja2.FileSystemLoader([

9
qubes/tests/vm/mix/net.py
View File

@ -40,10 +40,10 @@ class TC_00_NetVMMixin(
# testing properties used here
self.netvm1 = qubes.vm.qubesvm.QubesVM(self.app, None, qid=2,
name=qubes.tests.VMPREFIX + 'netvm1',
provides_network=True)
provides_network=True, netvm=None)
self.netvm2 = qubes.vm.qubesvm.QubesVM(self.app, None, qid=3,
name=qubes.tests.VMPREFIX + 'netvm2',
provides_network=True)
provides_network=True, netvm=None)
self.nonetvm = qubes.vm.qubesvm.QubesVM(self.app, None, qid=4,
name=qubes.tests.VMPREFIX + 'nonet')
self.app.domains = qubes.app.VMCollection(self.app)
@ -57,7 +57,10 @@ class TC_00_NetVMMixin(
self.netvm1.close()
self.netvm2.close()
self.nonetvm.close()
self.app.domains.close()
try:
self.app.domains.close()
except AttributeError:
pass
del self.netvm1
del self.netvm2
del self.nonetvm

151
qubes/tests/vm/qubesvm.py
View File

@ -19,7 +19,7 @@
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, see <https://www.gnu.org/licenses/>.
#
import base64
import os
import unittest
@ -54,6 +54,20 @@ class TestDeviceCollection(object):
def persistent(self):
return self._list
class TestQubesDB(object):
def __init__(self):
self.data = {}
def write(self, path, value):
self.data[path] = value
def rm(self, path):
if path.endswith('/'):
for key in [x for x in self.data if x.startswith(path)]:
del self.data[key]
else:
self.data.pop(path, None)
class TestVM(object):
# pylint: disable=too-few-public-methods
app = TestApp()
@ -133,11 +147,25 @@ class QubesVMTestsMixin(object):
super(QubesVMTestsMixin, self).setUp()
self.app = qubes.tests.vm.TestApp()
self.app.vmm.offline_mode = True
# when full test run is called, extensions are loaded by earlier
# tests, but if just this test class is run, load them manually here,
# to have the same behaviour
qubes.ext.get_extensions()
def get_vm(self, name='test', **kwargs):
vm = qubes.vm.qubesvm.QubesVM(self.app, None,
qid=1, name=qubes.tests.VMPREFIX + name,
def tearDown(self):
try:
self.app.domains.close()
except AttributeError:
pass
super(QubesVMTestsMixin, self).tearDown()
def get_vm(self, name='test', cls=qubes.vm.qubesvm.QubesVM, **kwargs):
vm = cls(self.app, None,
qid=kwargs.pop('qid', 1), name=qubes.tests.VMPREFIX + name,
**kwargs)
self.app.domains[vm.qid] = vm
self.app.domains[vm.uuid] = vm
self.app.domains[vm] = vm
self.addCleanup(vm.close)
return vm
@ -685,3 +713,118 @@ class TC_90_QubesVM(QubesVMTestsMixin, qubes.tests.QubesTestCase):
libvirt_xml = vm.create_config_file()
self.assertXMLEqual(lxml.etree.XML(libvirt_xml),
lxml.etree.XML(expected))
@unittest.mock.patch('qubes.utils.get_timezone')
@unittest.mock.patch('qubes.utils.urandom')
@unittest.mock.patch('qubes.vm.qubesvm.QubesVM.untrusted_qdb')
def test_620_qdb_standalone(self, mock_qubesdb, mock_urandom,
mock_timezone):
mock_urandom.return_value = b'A' * 64
mock_timezone.return_value = 'UTC'
vm = self.get_vm(cls=qubes.vm.standalonevm.StandaloneVM)
vm.netvm = None
vm.events_enabled = True
test_qubesdb = TestQubesDB()
mock_qubesdb.write.side_effect = test_qubesdb.write
mock_qubesdb.rm.side_effect = test_qubesdb.rm
vm.create_qdb_entries()
self.maxDiff = None
iptables_header = (
'# Generated by Qubes Core on {}\n'
'*filter\n'
':INPUT DROP [0:0]\n'
':FORWARD DROP [0:0]\n'
':OUTPUT ACCEPT [0:0]\n'
'-A INPUT -i vif+ -p udp -m udp --dport 68 -j DROP\n'
'-A INPUT -m conntrack --ctstate '
'RELATED,ESTABLISHED -j ACCEPT\n'
'-A INPUT -p icmp -j ACCEPT\n'
'-A INPUT -i lo -j ACCEPT\n'
'-A INPUT -j REJECT --reject-with '
'icmp-host-prohibited\n'
'-A FORWARD -m conntrack --ctstate '
'RELATED,ESTABLISHED -j ACCEPT\n'
'-A FORWARD -i vif+ -o vif+ -j DROP\n'
'COMMIT\n'.format(datetime.datetime.now().ctime()))
self.assertEqual(test_qubesdb.data, {
'/name': 'test-inst-test',
'/type': 'StandaloneVM',
'/qubes-vm-type': 'AppVM',
'/qubes-debug-mode': '0',
'/qubes-base-template': '',
'/qubes-timezone': 'UTC',
'/qubes-random-seed': base64.b64encode(b'A' * 64),
'/qubes-vm-persistence': 'full',
'/qubes-vm-updateable': 'True',
'/qubes-block-devices': '',
'/qubes-usb-devices': '',
'/qubes-iptables': 'reload',
'/qubes-iptables-error': '',
'/qubes-iptables-header': iptables_header,
'/qubes-service/qubes-update-check': '0',
})
@unittest.mock.patch('qubes.utils.get_timezone')
@unittest.mock.patch('qubes.utils.urandom')
@unittest.mock.patch('qubes.vm.qubesvm.QubesVM.untrusted_qdb')
def test_621_qdb_appvm_with_network(self, mock_qubesdb, mock_urandom,
mock_timezone):
mock_urandom.return_value = b'A' * 64
mock_timezone.return_value = 'UTC'
template = self.get_vm(cls=qubes.vm.templatevm.TemplateVM, name='template')
template.netvm = None
netvm = self.get_vm(cls=qubes.vm.standalonevm.StandaloneVM,
name='netvm', qid=2, provides_network=True)
vm = self.get_vm(cls=qubes.vm.appvm.AppVM, template=template,
name='appvm', qid=3)
vm.netvm = netvm
test_qubesdb = TestQubesDB()
mock_qubesdb.write.side_effect = test_qubesdb.write
mock_qubesdb.rm.side_effect = test_qubesdb.rm
self.maxDiff = None
iptables_header = (
'# Generated by Qubes Core on {}\n'
'*filter\n'
':INPUT DROP [0:0]\n'
':FORWARD DROP [0:0]\n'
':OUTPUT ACCEPT [0:0]\n'
'-A INPUT -i vif+ -p udp -m udp --dport 68 -j DROP\n'
'-A INPUT -m conntrack --ctstate '
'RELATED,ESTABLISHED -j ACCEPT\n'
'-A INPUT -p icmp -j ACCEPT\n'
'-A INPUT -i lo -j ACCEPT\n'
'-A INPUT -j REJECT --reject-with '
'icmp-host-prohibited\n'
'-A FORWARD -m conntrack --ctstate '
'RELATED,ESTABLISHED -j ACCEPT\n'
'-A FORWARD -i vif+ -o vif+ -j DROP\n'
'COMMIT\n'.format(datetime.datetime.now().ctime()))
expected = {
'/name': 'test-inst-appvm',
'/type': 'AppVM',
'/qubes-vm-type': 'AppVM',
'/qubes-debug-mode': '0',
'/qubes-base-template': 'test-inst-template',
'/qubes-timezone': 'UTC',
'/qubes-random-seed': base64.b64encode(b'A' * 64),
'/qubes-vm-persistence': 'rw-only',
'/qubes-vm-updateable': 'False',
'/qubes-block-devices': '',
'/qubes-usb-devices': '',
'/qubes-iptables': 'reload',
'/qubes-iptables-error': '',
'/qubes-iptables-header': iptables_header,
'/qubes-service/qubes-update-check': '0',
'/qubes-ip': '10.137.0.3',
'/qubes-netmask': '255.255.255.255',
'/qubes-gateway': '10.137.0.2',
'/qubes-primary-dns': '10.139.1.1',
'/qubes-secondary-dns': '10.139.1.2',
}
vm.create_qdb_entries()
self.assertEqual(test_qubesdb.data, expected)

3
qubes/vm/__init__.py
View File

@ -354,6 +354,9 @@ class BaseVM(qubes.PropertyHolder):
del self.tags
def load_extras(self):
if self.xml is None:
return
# features
for node in self.xml.xpath('./features/feature'):
self.features[node.get('name')] = node.text

29
qubes/vm/mix/net.py
View File

@ -70,10 +70,13 @@ def _setter_netvm(self, prop, value):
raise qubes.exc.QubesValueError(
'The {!s} qube does not provide network'.format(value))
if value is self \
or value in self.app.domains.get_vms_connected_to(self):
raise qubes.exc.QubesValueError(
'Loops in network are unsupported')
# skip check for netvm loops during qubes.xml loading, to avoid tricky
# loading order
if self.events_enabled:
if value is self \
or value in self.app.domains.get_vms_connected_to(self):
raise qubes.exc.QubesValueError(
'Loops in network are unsupported')
return value
@ -187,6 +190,22 @@ class NetVMMixin(qubes.events.Emitter):
self._firewall = None
super(NetVMMixin, self).__init__(*args, **kwargs)
@qubes.events.handler('domain-load')
def on_domain_load_netvm_loop_check(self, event):
# pylint: disable=unused-argument
# make sure there are no netvm loops - which could cause qubesd
# looping infinitely
if self is self.netvm:
self.log.error(
'vm \'%s\' network-connected to itself, breaking the '
'connection', self.name)
self.netvm = None
elif self.netvm in self.app.domains.get_vms_connected_to(self):
self.log.error(
'netvm loop detected on \'%s\', breaking the connection',
self.name)
self.netvm = None
@qubes.events.handler('domain-start')
def on_domain_started(self, event, **kwargs):
'''Connect this domain to its downstream domains. Also reload firewall
@ -323,6 +342,8 @@ class NetVMMixin(qubes.events.Emitter):
# pylint: disable=unused-argument
# we are changing to default netvm
newvalue = type(self).netvm.get_default(self)
# check for netvm loop
_setter_netvm(self, type(self).netvm, newvalue)
if newvalue == oldvalue:
return
self.fire_event('property-pre-set:netvm', pre_event=True,

7
qubes/vm/qubesvm.py
View File

@ -1244,7 +1244,12 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
else:
stubdom_mem = 0
mem_required = int(self.memory + stubdom_mem) * 1024 * 1024
initial_memory = self.memory
if self.virt_mode == 'hvm' and self.devices['pci'].persistent():
# HVM with PCI devices does not support populate-on-demand on
# Xen
initial_memory = self.maxmem
mem_required = int(initial_memory + stubdom_mem) * 1024 * 1024
qmemman_client = qubes.qmemman.client.QMemmanClient()
try:

2
qvm-tools/qubes-hcl-report
View File

@ -112,7 +112,7 @@ XL_VTX=`cat $TEMP_DIR/xl-info |grep xen_caps | grep hvm`
XL_VTD=`cat $TEMP_DIR/xl-info |grep virt_caps |grep hvm_directio`
XL_HAP=`cat $TEMP_DIR/xl-dmesg |grep "$XL_DMESG_PREFIX_REGEX"'HVM: Hardware Assisted Paging (HAP) detected\( but disabled\)\?$'`
PCRS=`find /sys/devices/ -name pcrs`
XL_REMAP=`cat $TEMP_DIR/xl-dmesg |grep "$XL_DMESG_PREFIX_REGEX"'Intel VT-d Interrupt Remapping enabled'`
XL_REMAP=`cat $TEMP_DIR/xl-dmesg |grep "$XL_DMESG_PREFIX_REGEX"'\(Intel VT-d Interrupt Remapping enabled\|Interrupt remapping enabled\)'`
FILENAME="Qubes-HCL-${BRAND//[^[:alnum:]]/_}-${PRODUCT//[^[:alnum:]]/_}-$DATE"