Merge branch 'qdb-watch'

* qdb-watch:
  tests: add qdb_watch test
  ext/block: make use of QubesDB watch
  vm: add API for watching changes in QubesDB
  vm: optimize imports
  api/admin: don't send internal events in admin.Events
  Add explanation why admin.vm.volume.Import is a custom script
  Follow change of qubesdb path return type
  Rename vm.qdb to vm.untrusted_qdb

qubes-rpc/admin.vm.volume.Import

@@ -1,4 +1,27 @@
 #!/bin/sh
+#
+# This Admin API call is implemented as a custom script, instead of dumb
+# passthrough to qubesd because it may get huge amount of data (whole root.img
+# for example). qubesd cannot handle it because:
+#  1. It loads the whole payload into memory, before even start looking at it
+#     (and later, do not allow to modify/append it).
+#  2. There is 64kB limit on payload size that qubesd can handle (because of
+#     point 1).
+#  3. Performance reasons (qubesd is not optimized for performance, passing
+#     such large data stream through it would take ages).
+#
+# The whole admin.vm.volume.Import consists of:
+#    1. Permissions checks, getting a path from appropriate storage pool (done
+#       by qubesd)
+#    2. Actual data import (done by this script, using dd)
+#    3. Report final result, produce final response to the caller (done by
+#       qubesd)
+#    
+#    This way we do not pass all the data through qubesd, but still can
+#    control the process from there in a meaningful way. Note that the last
+#    part (second call to qubesd) may perform all kind of verification (like
+#    a signature check on the data, or so) and can also prevent VM from
+#    starting (hooking also domain-pre-start event) from not verified image.
 
 set -e
 

qubes/api/admin.py

@@ -51,8 +51,18 @@ class QubesMgmtEventsDispatcher(object):
         self.send_event = send_event
 
     def vm_handler(self, subject, event, **kwargs):
+        # do not send internal events
         if event.startswith('admin-permission:'):
             return
+        if event.startswith('device-get:'):
+            return
+        if event.startswith('device-list:'):
+            return
+        if event.startswith('device-list-attached:'):
+            return
+        if event in ('domain-is-fully-usable',):
+            return
+
         if not list(qubes.api.apply_filters([(subject, event, kwargs)],
                 self.filters)):
             return

qubes/api/misc.py

@@ -51,10 +51,9 @@ class QubesMiscAPI(qubes.api.AbstractQubesAPI):
 
         prefix = '/features-request/'
 
-        keys = [key.decode('ascii', errors='strict')
-            for key in self.src.qdb.list(prefix)]
+        keys = self.src.untrusted_qdb.list(prefix)
         untrusted_features = {key[len(prefix):]:
-            self.src.qdb.read(key).decode('ascii', errors='strict')
+            self.src.untrusted_qdb.read(key).decode('ascii', errors='strict')
                 for key in keys}
 
         safe_set = string.ascii_letters + string.digits
@@ -79,7 +78,8 @@ class QubesMiscAPI(qubes.api.AbstractQubesAPI):
         safe_set = string.ascii_letters + string.digits
         expected_features = ('qrexec', 'gui', 'default-user')
         for feature in expected_features:
-            untrusted_value = self.src.qdb.read('/qubes-tools/' + feature)
+            untrusted_value = self.src.untrusted_qdb.read(
+                '/qubes-tools/' + feature)
             if untrusted_value:
                 untrusted_value = untrusted_value.decode('ascii',
                     errors='strict')

qubes/devices.py

@@ -44,11 +44,17 @@ Such extension should provide:
  domain of given identifier
  - handle `device-list-attached:class` event - list currently attached
  devices to this domain
+ - fire `device-list-change:class` event when device list change is detected
+ (new/removed device)
 
 Note that device-listing event handlers can not be asynchronous. This for
 example means you can not call qrexec service there. This is intentional to
 keep device listing operation cheap. You need to design the extension to take
 this into account (for example by using QubesDB).
+
+Extension may use QubesDB watch API (QubesVM.watch_qdb_path(path), then handle
+`domain-qdb-change:path`) to detect changes and fire
+`device-list-change:class` event.
 '''
 import asyncio
 

qubes/ext/block.py

@@ -56,7 +56,7 @@ class BlockDevice(qubes.devices.DeviceInfo):
                 return self.ident
             safe_set = {ord(c) for c in
                 string.ascii_letters + string.digits + '()+,-.:=_/ '}
-            untrusted_desc = self.backend_domain.qdb.read(
+            untrusted_desc = self.backend_domain.untrusted_qdb.read(
                 '/qubes-block-devices/{}/desc'.format(self.ident))
             desc = ''.join((chr(c) if c in safe_set else '_')
                 for c in untrusted_desc)
@@ -69,7 +69,7 @@ class BlockDevice(qubes.devices.DeviceInfo):
         if self._mode is None:
             if not self.backend_domain.is_running():
                 return 'w'
-            untrusted_mode = self.backend_domain.qdb.read(
+            untrusted_mode = self.backend_domain.untrusted_qdb.read(
                 '/qubes-block-devices/{}/mode'.format(self.ident))
             if untrusted_mode is None:
                 self._mode = 'w'
@@ -87,7 +87,7 @@ class BlockDevice(qubes.devices.DeviceInfo):
         if self._size is None:
             if not self.backend_domain.is_running():
                 return None
-            untrusted_size = self.backend_domain.qdb.read(
+            untrusted_size = self.backend_domain.untrusted_qdb.read(
                 '/qubes-block-devices/{}/size'.format(self.ident))
             if untrusted_size is None:
                 self._size = 0
@@ -106,6 +106,18 @@ class BlockDevice(qubes.devices.DeviceInfo):
 
 
 class BlockDeviceExtension(qubes.ext.Extension):
+    @qubes.ext.handler('domain-init', 'domain-load')
+    def on_domain_init_load(self, vm, event):
+        '''Initialize watching for changes'''
+        # pylint: disable=unused-argument,no-self-use
+        vm.watch_qdb_path('/qubes-block-devices')
+
+    @qubes.ext.handler('domain-qdb-change:/qubes-block-devices')
+    def on_qdb_change(self, vm, event, path):
+        '''A change in QubesDB means a change in device list'''
+        # pylint: disable=unused-argument,no-self-use
+        vm.fire_event('device-list-change:block')
+
     def device_get(self, vm, ident):
         # pylint: disable=no-self-use
         '''Read information about device from QubesDB
@@ -114,7 +126,7 @@ class BlockDeviceExtension(qubes.ext.Extension):
         :param ident: device identifier
         :returns BlockDevice'''
 
-        untrusted_qubes_device_attrs = vm.qdb.list(
+        untrusted_qubes_device_attrs = vm.untrusted_qdb.list(
             '/qubes-block-devices/{}/'.format(ident))
         if not untrusted_qubes_device_attrs:
             return None
@@ -124,12 +136,11 @@ class BlockDeviceExtension(qubes.ext.Extension):
     def on_device_list_block(self, vm, event):
         # pylint: disable=unused-argument,no-self-use
 
-        safe_set = {ord(c) for c in
-            string.ascii_letters + string.digits}
+        safe_set = string.ascii_letters + string.digits
         if not vm.is_running():
             return
-        untrusted_qubes_devices = vm.qdb.list('/qubes-block-devices/')
-        untrusted_idents = set(untrusted_path.split(b'/', 3)[2]
+        untrusted_qubes_devices = vm.untrusted_qdb.list('/qubes-block-devices/')
+        untrusted_idents = set(untrusted_path.split('/', 3)[2]
             for untrusted_path in untrusted_qubes_devices)
         for untrusted_ident in untrusted_idents:
             if not all(c in safe_set for c in untrusted_ident):
@@ -138,7 +149,7 @@ class BlockDeviceExtension(qubes.ext.Extension):
                 vm.log.warning(msg % vm.name)
                 continue
 
-            ident = untrusted_ident.decode('ascii', errors='strict')
+            ident = untrusted_ident
 
             device_info = self.device_get(vm, ident)
             if device_info:

qubes/ext/r3compatibility.py

@@ -60,9 +60,9 @@ class R3Compatibility(qubes.ext.Extension):
             vmtype = 'NetVM'
         else:
             vmtype = 'AppVM'
-        vm.qdb.write('/qubes-vm-type', vmtype)
+        vm.untrusted_qdb.write('/qubes-vm-type', vmtype)
 
-        vm.qdb.write("/qubes-iptables-error", '')
+        vm.untrusted_qdb.write("/qubes-iptables-error", '')
         self.write_iptables_qubesdb_entry(vm)
 
         self.write_services(vm)
@@ -81,7 +81,7 @@ class R3Compatibility(qubes.ext.Extension):
 
     def write_iptables_qubesdb_entry(self, firewallvm):
         # pylint: disable=no-self-use
-        firewallvm.qdb.rm("/qubes-iptables-domainrules/")
+        firewallvm.untrusted_qdb.rm("/qubes-iptables-domainrules/")
         iptables = "# Generated by Qubes Core on {0}\n".format(
             datetime.datetime.now().ctime())
         iptables += "*filter\n"
@@ -102,7 +102,7 @@ class R3Compatibility(qubes.ext.Extension):
         # Deny inter-VMs networking
         iptables += "-A FORWARD -i vif+ -o vif+ -j DROP\n"
         iptables += "COMMIT\n"
-        firewallvm.qdb.write("/qubes-iptables-header", iptables)
+        firewallvm.untrusted_qdb.write("/qubes-iptables-header", iptables)
 
         for vm in firewallvm.connected_vms:
             iptables = "*filter\n"
@@ -154,11 +154,12 @@ class R3Compatibility(qubes.ext.Extension):
             iptables += '-A FORWARD -s {0} -j {1}\n'.format(ip,
                 str(conf.policy).upper())
             iptables += 'COMMIT\n'
-            firewallvm.qdb.write('/qubes-iptables-domainrules/' + str(xid),
+            firewallvm.untrusted_qdb.write(
+                '/qubes-iptables-domainrules/' + str(xid),
                 iptables)
         # no need for ending -A FORWARD -j DROP, cause default action is DROP
 
-        firewallvm.qdb.write('/qubes-iptables', 'reload')
+        firewallvm.untrusted_qdb.write('/qubes-iptables', 'reload')
 
     def write_services(self, vm):
         for feature, value in vm.features.items():
@@ -166,8 +167,9 @@ class R3Compatibility(qubes.ext.Extension):
             if service is None:
                 continue
             # forcefully convert to '0' or '1'
-            vm.qdb.write('/qubes-service/{}'.format(service),
+            vm.untrusted_qdb.write('/qubes-service/{}'.format(service),
                 str(int(bool(value))))
         if 'updates-proxy-setup' in vm.features.keys():
-            vm.qdb.write('/qubes-service/{}'.format('yum-proxy-setup'),
+            vm.untrusted_qdb.write(
+                '/qubes-service/{}'.format('yum-proxy-setup'),
                 str(int(bool(vm.features['updates-proxy-setup']))))

qubes/storage/__init__.py

@@ -419,7 +419,8 @@ class Storage(object):
         # trigger watches to update device status
         # FIXME: this should be removed once libvirt will report such
         # events itself
-        # self.vm.qdb.write('/qubes-block-devices', '') ← do we need this?
+        # self.vm.untrusted_qdb.write('/qubes-block-devices', '')
+        # ← do we need this?
 
     def _is_already_attached(self, volume):
         ''' Checks if the given volume is already attached '''

qubes/tests/api_misc.py

@@ -37,9 +37,10 @@ class TC_00_API_Misc(qubes.tests.QubesTestCase):
 
     def configure_qdb(self, entries):
         self.src.configure_mock(**{
-            'qdb.read.side_effect': (lambda path: entries.get(path, None)),
-            'qdb.list.side_effect': (lambda path:
-                sorted(map(str.encode, entries.keys()))),
+            'untrusted_qdb.read.side_effect': (
+                lambda path: entries.get(path, None)),
+            'untrusted_qdb.list.side_effect': (
+                lambda path: sorted(entries.keys())),
         })
 
     def call_mgmt_func(self, method, arg=b'', payload=b''):
@@ -64,10 +65,10 @@ class TC_00_API_Misc(qubes.tests.QubesTestCase):
             mock.call.save()
         ])
         self.assertEqual(self.src.mock_calls, [
-            mock.call.qdb.list('/features-request/'),
-            mock.call.qdb.read('/features-request/feature1'),
-            mock.call.qdb.read('/features-request/feature2'),
-            mock.call.qdb.read('/features-request/feature3'),
+            mock.call.untrusted_qdb.list('/features-request/'),
+            mock.call.untrusted_qdb.read('/features-request/feature1'),
+            mock.call.untrusted_qdb.read('/features-request/feature2'),
+            mock.call.untrusted_qdb.read('/features-request/feature3'),
             mock.call.fire_event('features-request', untrusted_features={
                 'feature1': '1', 'feature2': '', 'feature3': 'other'})
         ])
@@ -80,7 +81,7 @@ class TC_00_API_Misc(qubes.tests.QubesTestCase):
             mock.call.save()
         ])
         self.assertEqual(self.src.mock_calls, [
-            mock.call.qdb.list('/features-request/'),
+            mock.call.untrusted_qdb.list('/features-request/'),
             mock.call.fire_event('features-request', untrusted_features={})
         ])
 
@@ -93,8 +94,8 @@ class TC_00_API_Misc(qubes.tests.QubesTestCase):
             self.call_mgmt_func(b'qubes.FeaturesRequest')
         self.assertEqual(self.app.mock_calls, [])
         self.assertEqual(self.src.mock_calls, [
-            mock.call.qdb.list('/features-request/'),
-            mock.call.qdb.read('/features-request/feature1'),
+            mock.call.untrusted_qdb.list('/features-request/'),
+            mock.call.untrusted_qdb.read('/features-request/feature1'),
         ])
 
     def test_003_features_request_invalid2(self):
@@ -106,8 +107,8 @@ class TC_00_API_Misc(qubes.tests.QubesTestCase):
             self.call_mgmt_func(b'qubes.FeaturesRequest')
         self.assertEqual(self.app.mock_calls, [])
         self.assertEqual(self.src.mock_calls, [
-            mock.call.qdb.list('/features-request/'),
-            mock.call.qdb.read('/features-request/feature1'),
+            mock.call.untrusted_qdb.list('/features-request/'),
+            mock.call.untrusted_qdb.read('/features-request/feature1'),
         ])
 
     def test_010_notify_tools(self):
@@ -125,9 +126,9 @@ class TC_00_API_Misc(qubes.tests.QubesTestCase):
             mock.call.save()
         ])
         self.assertEqual(self.src.mock_calls, [
-            mock.call.qdb.read('/qubes-tools/qrexec'),
-            mock.call.qdb.read('/qubes-tools/gui'),
-            mock.call.qdb.read('/qubes-tools/default-user'),
+            mock.call.untrusted_qdb.read('/qubes-tools/qrexec'),
+            mock.call.untrusted_qdb.read('/qubes-tools/gui'),
+            mock.call.untrusted_qdb.read('/qubes-tools/default-user'),
             mock.call.fire_event('features-request', untrusted_features={
                 'gui': '1',
                 'default-user': 'user',
@@ -146,9 +147,9 @@ class TC_00_API_Misc(qubes.tests.QubesTestCase):
         response = self.call_mgmt_func(b'qubes.NotifyTools')
         self.assertIsNone(response)
         self.assertEqual(self.src.mock_calls, [
-            mock.call.qdb.read('/qubes-tools/qrexec'),
-            mock.call.qdb.read('/qubes-tools/gui'),
-            mock.call.qdb.read('/qubes-tools/default-user'),
+            mock.call.untrusted_qdb.read('/qubes-tools/qrexec'),
+            mock.call.untrusted_qdb.read('/qubes-tools/gui'),
+            mock.call.untrusted_qdb.read('/qubes-tools/default-user'),
             mock.call.fire_event('features-request', untrusted_features={
                 'gui': '1',
                 'default-user': 'user',
@@ -169,7 +170,7 @@ class TC_00_API_Misc(qubes.tests.QubesTestCase):
             self.call_mgmt_func(b'qubes.NotifyTools')
         self.assertEqual(self.app.mock_calls, [])
         self.assertEqual(self.src.mock_calls, [
-            mock.call.qdb.read('/qubes-tools/qrexec'),
+            mock.call.untrusted_qdb.read('/qubes-tools/qrexec'),
         ])
 
     def test_016_notify_tools_invalid_value_gui(self):
@@ -185,8 +186,8 @@ class TC_00_API_Misc(qubes.tests.QubesTestCase):
             self.call_mgmt_func(b'qubes.NotifyTools')
         self.assertEqual(self.app.mock_calls, [])
         self.assertEqual(self.src.mock_calls, [
-            mock.call.qdb.read('/qubes-tools/qrexec'),
-            mock.call.qdb.read('/qubes-tools/gui'),
+            mock.call.untrusted_qdb.read('/qubes-tools/qrexec'),
+            mock.call.untrusted_qdb.read('/qubes-tools/gui'),
         ])
 
     def test_020_notify_updates_standalone(self):

qubes/tests/devices_block.py

@@ -94,13 +94,9 @@ class TestQubesDB(object):
         self._data = data
 
     def read(self, key):
-        if isinstance(key, str):
-            key = key.encode()
         return self._data.get(key, None)
 
     def list(self, prefix):
-        if isinstance(prefix, str):
-            prefix = prefix.encode()
         return [key for key in self._data if key.startswith(prefix)]
 
 
@@ -120,7 +116,7 @@ class TestApp(object):
 class TestVM(object):
     def __init__(self, qdb, domain_xml=None, running=True, name='test-vm'):
         self.name = name
-        self.qdb = TestQubesDB(qdb)
+        self.untrusted_qdb = TestQubesDB(qdb)
         self.libvirt_domain = mock.Mock()
         self.is_running = lambda: running
         self.log = mock.Mock()
@@ -143,10 +139,10 @@ class TC_00_Block(qubes.tests.QubesTestCase):
 
     def test_000_device_get(self):
         vm = TestVM({
-            b'/qubes-block-devices/sda': b'',
-            b'/qubes-block-devices/sda/desc': b'Test device',
-            b'/qubes-block-devices/sda/size': b'1024000',
-            b'/qubes-block-devices/sda/mode': b'w',
+            '/qubes-block-devices/sda': b'',
+            '/qubes-block-devices/sda/desc': b'Test device',
+            '/qubes-block-devices/sda/size': b'1024000',
+            '/qubes-block-devices/sda/mode': b'w',
         })
         device_info = self.ext.device_get(vm, 'sda')
         self.assertIsInstance(device_info, qubes.ext.block.BlockDevice)
@@ -161,10 +157,10 @@ class TC_00_Block(qubes.tests.QubesTestCase):
 
     def test_001_device_get_other_node(self):
         vm = TestVM({
-            b'/qubes-block-devices/mapper_dmroot': b'',
-            b'/qubes-block-devices/mapper_dmroot/desc': b'Test device',
-            b'/qubes-block-devices/mapper_dmroot/size': b'1024000',
-            b'/qubes-block-devices/mapper_dmroot/mode': b'w',
+            '/qubes-block-devices/mapper_dmroot': b'',
+            '/qubes-block-devices/mapper_dmroot/desc': b'Test device',
+            '/qubes-block-devices/mapper_dmroot/size': b'1024000',
+            '/qubes-block-devices/mapper_dmroot/mode': b'w',
         })
         device_info = self.ext.device_get(vm, 'mapper_dmroot')
         self.assertIsInstance(device_info, qubes.ext.block.BlockDevice)
@@ -179,20 +175,20 @@ class TC_00_Block(qubes.tests.QubesTestCase):
 
     def test_002_device_get_invalid_desc(self):
         vm = TestVM({
-            b'/qubes-block-devices/sda': b'',
-            b'/qubes-block-devices/sda/desc': b'Test device<>za\xc4\x87abc',
-            b'/qubes-block-devices/sda/size': b'1024000',
-            b'/qubes-block-devices/sda/mode': b'w',
+            '/qubes-block-devices/sda': b'',
+            '/qubes-block-devices/sda/desc': b'Test device<>za\xc4\x87abc',
+            '/qubes-block-devices/sda/size': b'1024000',
+            '/qubes-block-devices/sda/mode': b'w',
         })
         device_info = self.ext.device_get(vm, 'sda')
         self.assertEqual(device_info.description, 'Test device__za__abc')
 
     def test_003_device_get_invalid_size(self):
         vm = TestVM({
-            b'/qubes-block-devices/sda': b'',
-            b'/qubes-block-devices/sda/desc': b'Test device',
-            b'/qubes-block-devices/sda/size': b'1024000abc',
-            b'/qubes-block-devices/sda/mode': b'w',
+            '/qubes-block-devices/sda': b'',
+            '/qubes-block-devices/sda/desc': b'Test device',
+            '/qubes-block-devices/sda/size': b'1024000abc',
+            '/qubes-block-devices/sda/mode': b'w',
         })
         device_info = self.ext.device_get(vm, 'sda')
         self.assertEqual(device_info.size, 0)
@@ -200,10 +196,10 @@ class TC_00_Block(qubes.tests.QubesTestCase):
 
     def test_004_device_get_invalid_mode(self):
         vm = TestVM({
-            b'/qubes-block-devices/sda': b'',
-            b'/qubes-block-devices/sda/desc': b'Test device',
-            b'/qubes-block-devices/sda/size': b'1024000',
-            b'/qubes-block-devices/sda/mode': b'abc',
+            '/qubes-block-devices/sda': b'',
+            '/qubes-block-devices/sda/desc': b'Test device',
+            '/qubes-block-devices/sda/size': b'1024000',
+            '/qubes-block-devices/sda/mode': b'abc',
         })
         device_info = self.ext.device_get(vm, 'sda')
         self.assertEqual(device_info.mode, 'w')
@@ -211,24 +207,24 @@ class TC_00_Block(qubes.tests.QubesTestCase):
 
     def test_005_device_get_none(self):
         vm = TestVM({
-            b'/qubes-block-devices/sda': b'',
-            b'/qubes-block-devices/sda/desc': b'Test device',
-            b'/qubes-block-devices/sda/size': b'1024000',
-            b'/qubes-block-devices/sda/mode': b'w',
+            '/qubes-block-devices/sda': b'',
+            '/qubes-block-devices/sda/desc': b'Test device',
+            '/qubes-block-devices/sda/size': b'1024000',
+            '/qubes-block-devices/sda/mode': b'w',
         })
         device_info = self.ext.device_get(vm, 'sdb')
         self.assertIsNone(device_info)
 
     def test_010_devices_list(self):
         vm = TestVM({
-            b'/qubes-block-devices/sda': b'',
-            b'/qubes-block-devices/sda/desc': b'Test device',
-            b'/qubes-block-devices/sda/size': b'1024000',
-            b'/qubes-block-devices/sda/mode': b'w',
-            b'/qubes-block-devices/sdb': b'',
-            b'/qubes-block-devices/sdb/desc': b'Test device2',
-            b'/qubes-block-devices/sdb/size': b'2048000',
-            b'/qubes-block-devices/sdb/mode': b'r',
+            '/qubes-block-devices/sda': b'',
+            '/qubes-block-devices/sda/desc': b'Test device',
+            '/qubes-block-devices/sda/size': b'1024000',
+            '/qubes-block-devices/sda/mode': b'w',
+            '/qubes-block-devices/sdb': b'',
+            '/qubes-block-devices/sdb/desc': b'Test device2',
+            '/qubes-block-devices/sdb/size': b'2048000',
+            '/qubes-block-devices/sdb/mode': b'r',
         })
         devices = sorted(list(self.ext.on_device_list_block(vm, '')))
         self.assertEqual(len(devices), 2)
@@ -250,9 +246,9 @@ class TC_00_Block(qubes.tests.QubesTestCase):
 
     def test_012_devices_list_invalid_ident(self):
         vm = TestVM({
-            b'/qubes-block-devices/invalid ident': b'',
-            b'/qubes-block-devices/invalid+ident': b'',
-            b'/qubes-block-devices/invalid#': b'',
+            '/qubes-block-devices/invalid ident': b'',
+            '/qubes-block-devices/invalid+ident': b'',
+            '/qubes-block-devices/invalid#': b'',
         })
         devices = sorted(list(self.ext.on_device_list_block(vm, '')))
         self.assertEqual(len(devices), 0)
@@ -334,10 +330,10 @@ class TC_00_Block(qubes.tests.QubesTestCase):
 
     def test_040_attach(self):
         back_vm = TestVM(name='sys-usb', qdb={
-            b'/qubes-block-devices/sda': b'',
-            b'/qubes-block-devices/sda/desc': b'Test device',
-            b'/qubes-block-devices/sda/size': b'1024000',
-            b'/qubes-block-devices/sda/mode': b'w',
+            '/qubes-block-devices/sda': b'',
+            '/qubes-block-devices/sda/desc': b'Test device',
+            '/qubes-block-devices/sda/size': b'1024000',
+            '/qubes-block-devices/sda/mode': b'w',
         })
         vm = TestVM({}, domain_xml=domain_xml_template.format(''))
         dev = qubes.ext.block.BlockDevice(back_vm, 'sda')
@@ -353,10 +349,10 @@ class TC_00_Block(qubes.tests.QubesTestCase):
 
     def test_041_attach_frontend(self):
         back_vm = TestVM(name='sys-usb', qdb={
-            b'/qubes-block-devices/sda': b'',
-            b'/qubes-block-devices/sda/desc': b'Test device',
-            b'/qubes-block-devices/sda/size': b'1024000',
-            b'/qubes-block-devices/sda/mode': b'w',
+            '/qubes-block-devices/sda': b'',
+            '/qubes-block-devices/sda/desc': b'Test device',
+            '/qubes-block-devices/sda/size': b'1024000',
+            '/qubes-block-devices/sda/mode': b'w',
         })
         vm = TestVM({}, domain_xml=domain_xml_template.format(''))
         dev = qubes.ext.block.BlockDevice(back_vm, 'sda')
@@ -373,10 +369,10 @@ class TC_00_Block(qubes.tests.QubesTestCase):
 
     def test_042_attach_read_only(self):
         back_vm = TestVM(name='sys-usb', qdb={
-            b'/qubes-block-devices/sda': b'',
-            b'/qubes-block-devices/sda/desc': b'Test device',
-            b'/qubes-block-devices/sda/size': b'1024000',
-            b'/qubes-block-devices/sda/mode': b'w',
+            '/qubes-block-devices/sda': b'',
+            '/qubes-block-devices/sda/desc': b'Test device',
+            '/qubes-block-devices/sda/size': b'1024000',
+            '/qubes-block-devices/sda/mode': b'w',
         })
         vm = TestVM({}, domain_xml=domain_xml_template.format(''))
         dev = qubes.ext.block.BlockDevice(back_vm, 'sda')
@@ -394,10 +390,10 @@ class TC_00_Block(qubes.tests.QubesTestCase):
 
     def test_043_attach_invalid_option(self):
         back_vm = TestVM(name='sys-usb', qdb={
-            b'/qubes-block-devices/sda': b'',
-            b'/qubes-block-devices/sda/desc': b'Test device',
-            b'/qubes-block-devices/sda/size': b'1024000',
-            b'/qubes-block-devices/sda/mode': b'w',
+            '/qubes-block-devices/sda': b'',
+            '/qubes-block-devices/sda/desc': b'Test device',
+            '/qubes-block-devices/sda/size': b'1024000',
+            '/qubes-block-devices/sda/mode': b'w',
         })
         vm = TestVM({}, domain_xml=domain_xml_template.format(''))
         dev = qubes.ext.block.BlockDevice(back_vm, 'sda')
@@ -408,10 +404,10 @@ class TC_00_Block(qubes.tests.QubesTestCase):
 
     def test_044_attach_invalid_option2(self):
         back_vm = TestVM(name='sys-usb', qdb={
-            b'/qubes-block-devices/sda': b'',
-            b'/qubes-block-devices/sda/desc': b'Test device',
-            b'/qubes-block-devices/sda/size': b'1024000',
-            b'/qubes-block-devices/sda/mode': b'w',
+            '/qubes-block-devices/sda': b'',
+            '/qubes-block-devices/sda/desc': b'Test device',
+            '/qubes-block-devices/sda/size': b'1024000',
+            '/qubes-block-devices/sda/mode': b'w',
         })
         vm = TestVM({}, domain_xml=domain_xml_template.format(''))
         dev = qubes.ext.block.BlockDevice(back_vm, 'sda')
@@ -422,10 +418,10 @@ class TC_00_Block(qubes.tests.QubesTestCase):
 
     def test_045_attach_backend_not_running(self):
         back_vm = TestVM(name='sys-usb', running=False, qdb={
-            b'/qubes-block-devices/sda': b'',
-            b'/qubes-block-devices/sda/desc': b'Test device',
-            b'/qubes-block-devices/sda/size': b'1024000',
-            b'/qubes-block-devices/sda/mode': b'w',
+            '/qubes-block-devices/sda': b'',
+            '/qubes-block-devices/sda/desc': b'Test device',
+            '/qubes-block-devices/sda/size': b'1024000',
+            '/qubes-block-devices/sda/mode': b'w',
         })
         vm = TestVM({}, domain_xml=domain_xml_template.format(''))
         dev = qubes.ext.block.BlockDevice(back_vm, 'sda')
@@ -435,10 +431,10 @@ class TC_00_Block(qubes.tests.QubesTestCase):
 
     def test_046_attach_ro_dev_rw(self):
         back_vm = TestVM(name='sys-usb', qdb={
-            b'/qubes-block-devices/sda': b'',
-            b'/qubes-block-devices/sda/desc': b'Test device',
-            b'/qubes-block-devices/sda/size': b'1024000',
-            b'/qubes-block-devices/sda/mode': b'r',
+            '/qubes-block-devices/sda': b'',
+            '/qubes-block-devices/sda/desc': b'Test device',
+            '/qubes-block-devices/sda/size': b'1024000',
+            '/qubes-block-devices/sda/mode': b'r',
         })
         vm = TestVM({}, domain_xml=domain_xml_template.format(''))
         dev = qubes.ext.block.BlockDevice(back_vm, 'sda')
@@ -449,10 +445,10 @@ class TC_00_Block(qubes.tests.QubesTestCase):
 
     def test_047_attach_read_only_auto(self):
         back_vm = TestVM(name='sys-usb', qdb={
-            b'/qubes-block-devices/sda': b'',
-            b'/qubes-block-devices/sda/desc': b'Test device',
-            b'/qubes-block-devices/sda/size': b'1024000',
-            b'/qubes-block-devices/sda/mode': b'r',
+            '/qubes-block-devices/sda': b'',
+            '/qubes-block-devices/sda/desc': b'Test device',
+            '/qubes-block-devices/sda/size': b'1024000',
+            '/qubes-block-devices/sda/mode': b'r',
         })
         vm = TestVM({}, domain_xml=domain_xml_template.format(''))
         dev = qubes.ext.block.BlockDevice(back_vm, 'sda')
@@ -469,10 +465,10 @@ class TC_00_Block(qubes.tests.QubesTestCase):
 
     def test_050_detach(self):
         back_vm = TestVM(name='sys-usb', qdb={
-            b'/qubes-block-devices/sda': b'',
-            b'/qubes-block-devices/sda/desc': b'Test device',
-            b'/qubes-block-devices/sda/size': b'1024000',
-            b'/qubes-block-devices/sda/mode': b'r',
+            '/qubes-block-devices/sda': b'',
+            '/qubes-block-devices/sda/desc': b'Test device',
+            '/qubes-block-devices/sda/size': b'1024000',
+            '/qubes-block-devices/sda/mode': b'r',
         })
         device_xml = (
             '<disk type="block" device="disk">\n'
@@ -491,10 +487,10 @@ class TC_00_Block(qubes.tests.QubesTestCase):
 
     def test_051_detach_not_attached(self):
         back_vm = TestVM(name='sys-usb', qdb={
-            b'/qubes-block-devices/sda': b'',
-            b'/qubes-block-devices/sda/desc': b'Test device',
-            b'/qubes-block-devices/sda/size': b'1024000',
-            b'/qubes-block-devices/sda/mode': b'r',
+            '/qubes-block-devices/sda': b'',
+            '/qubes-block-devices/sda/desc': b'Test device',
+            '/qubes-block-devices/sda/size': b'1024000',
+            '/qubes-block-devices/sda/mode': b'r',
         })
         device_xml = (
             '<disk type="block" device="disk">\n'

qubes/tests/integ/basic.py

@@ -65,6 +65,20 @@ class TC_00_Basic(qubes.tests.SystemTestCase):
         with self.assertNotRaises(qubes.exc.QubesException):
             vm.storage.verify()
 
+    def test_040_qdb_watch(self):
+        flag = set()
+
+        def handler(vm, event, path):
+            if path == '/test-watch-path':
+                flag.add(True)
+
+        vm = self.app.domains[0]
+        vm.watch_qdb_path('/test-watch-path')
+        vm.add_handler('domain-qdb-change:/test-watch-path', handler)
+        self.assertFalse(flag)
+        vm.untrusted_qdb.write('/test-watch-path', 'test-value')
+        self.loop.run_until_complete(asyncio.sleep(0.1))
+        self.assertTrue(flag)
 
 class TC_01_Properties(qubes.tests.SystemTestCase):
     # pylint: disable=attribute-defined-outside-init

qubes/vm/__init__.py

@@ -24,14 +24,9 @@
 '''Qubes Virtual Machines
 
 '''
-
-import datetime
-import os
+import asyncio
 import re
 import string
-import subprocess
-import sys
-import xml.parsers.expat
 
 import lxml.etree
 
@@ -271,6 +266,8 @@ class BaseVM(qubes.PropertyHolder):
         # self.app must be set before super().__init__, because some property
         # setters need working .app attribute
         #: mother :py:class:`qubes.Qubes` object
+        self._qdb_watch_paths = set()
+        self._qdb_connection_watch = None
         self.app = app
 
         super(BaseVM, self).__init__(xml, **kwargs)
@@ -393,6 +390,55 @@ class BaseVM(qubes.PropertyHolder):
             ]).render(vm=self)
         return domain_config
 
+    def watch_qdb_path(self, path):
+        '''Add a QubesDB path to be watched.
+
+        Each change to the path will cause `domain-qdb-change:path` event to be
+        fired.
+        You can call this method for example in response to
+        `domain-init` and `domain-load` events.
+        '''
+
+        if path not in self._qdb_watch_paths:
+            self._qdb_watch_paths.add(path)
+            if self._qdb_connection_watch:
+                self._qdb_connection_watch.watch(path)
+
+    def _qdb_watch_reader(self, loop):
+        '''Callback when self._qdb_connection_watch.watch_fd() FD is
+        readable.
+
+        Read reported event (watched path change) and fire appropriate event.
+        '''
+        import qubesdb  # pylint: disable=import-error
+        try:
+            path = self._qdb_connection_watch.read_watch()
+            for watched_path in self._qdb_watch_paths:
+                if watched_path == path or (
+                            watched_path.endswith('/') and
+                            path.startswith(watched_path)):
+                    self.fire_event('domain-qdb-change:' + watched_path,
+                        path=path)
+        except qubesdb.DisconnectedError:
+            loop.remove_reader(self._qdb_connection_watch.watch_fd())
+            self._qdb_connection_watch.close()
+            self._qdb_connection_watch = None
+
+    def start_qdb_watch(self, name, loop=None):
+        '''Start watching QubesDB
+
+        Calling this method in appropriate time is responsibility of child
+        class.
+        '''
+        import qubesdb  # pylint: disable=import-error
+        self._qdb_connection_watch = qubesdb.QubesDB(name)
+        if loop is None:
+            loop = asyncio.get_event_loop()
+        loop.add_reader(self._qdb_connection_watch.watch_fd(),
+            self._qdb_watch_reader, loop)
+        for path in self._qdb_watch_paths:
+            self._qdb_connection_watch.watch(path)
+
 
 class VMProperty(qubes.property):
     '''Property that is referring to a VM

qubes/vm/adminvm.py

@@ -55,6 +55,9 @@ class AdminVM(qubes.vm.BaseVM):
         self._qdb_connection = None
         self._libvirt_domain = None
 
+        if not self.app.vmm.offline_mode:
+            self.start_qdb_watch('dom0')
+
     def __str__(self):
         return self.name
 
@@ -167,7 +170,7 @@ class AdminVM(qubes.vm.BaseVM):
         return None
 
     @property
-    def qdb(self):
+    def untrusted_qdb(self):
         '''QubesDB handle for this domain.'''
         if self._qdb_connection is None:
             import qubesdb  # pylint: disable=import-error

qubes/vm/mix/net.py

@@ -291,12 +291,12 @@ class NetVMMixin(qubes.events.Emitter):
         base_dir = '/qubes-firewall/' + vm.ip + '/'
         # remove old entries if any (but don't touch base empty entry - it
         # would trigger reload right away
-        self.qdb.rm(base_dir)
+        self.untrusted_qdb.rm(base_dir)
         # write new rules
         for key, value in vm.firewall.qdb_entries(addr_family=4).items():
-            self.qdb.write(base_dir + key, value)
+            self.untrusted_qdb.write(base_dir + key, value)
         # signal its done
-        self.qdb.write(base_dir[:-1], '')
+        self.untrusted_qdb.write(base_dir[:-1], '')
 
     def set_mapped_ip_info_for_vm(self, vm):
         '''
@@ -307,14 +307,15 @@ class NetVMMixin(qubes.events.Emitter):
         # add info about remapped IPs (VM IP hidden from the VM itself)
         mapped_ip_base = '/mapped-ip/{}'.format(vm.ip)
         if vm.visible_ip:
-            self.qdb.write(mapped_ip_base + '/visible-ip', vm.visible_ip)
+            self.untrusted_qdb.write(mapped_ip_base + '/visible-ip',
+                vm.visible_ip)
         else:
-            self.qdb.rm(mapped_ip_base + '/visible-ip')
+            self.untrusted_qdb.rm(mapped_ip_base + '/visible-ip')
         if vm.visible_gateway:
-            self.qdb.write(mapped_ip_base + '/visible-gateway',
+            self.untrusted_qdb.write(mapped_ip_base + '/visible-gateway',
                 vm.visible_gateway)
         else:
-            self.qdb.rm(mapped_ip_base + '/visible-gateway')
+            self.untrusted_qdb.rm(mapped_ip_base + '/visible-gateway')
 
 
     @qubes.events.handler('property-del:netvm')

qubes/vm/qubesvm.py

@@ -26,6 +26,8 @@ from __future__ import absolute_import
 import asyncio
 import base64
 import datetime
+import errno
+import grp
 import os
 import os.path
 import shutil
@@ -34,11 +36,8 @@ import subprocess
 import uuid
 import warnings
 
-import grp
-
-import errno
-import lxml
 import libvirt  # pylint: disable=import-error
+import lxml
 
 import qubes
 import qubes.config
@@ -249,6 +248,18 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
 
             This event is a good place to add your custom entries to the qdb.
 
+        .. event:: domain-qdb-change:watched-path (subject, event, path)
+
+            Fired when watched QubesDB entry is changed. See
+            :py:meth:`watch_qdb_path`. *watched-path* part of event name is
+            what path was registered for watching, *path* in event argument
+            is what actually have changed (which may be different if watching a
+            directory, i.e. a path with `/` at the end).
+
+            :param subject: Event emitter (the qube object)
+            :param event: Event name (``'domain-qdb-change'``)
+            :param path: changed QubesDB path
+
         .. event:: backup-get-files (subject, event)
 
             Collects additional file to be included in a backup.
@@ -590,7 +601,7 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
                 yield block_dev
 
     @property
-    def qdb(self):
+    def untrusted_qdb(self):
         '''QubesDB handle for this domain.'''
         if self._qdb_connection is None:
             if self.is_running():
@@ -727,6 +738,9 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
         if self.storage is None:
             self.storage = qubes.storage.Storage(self)
 
+        if not self.app.vmm.offline_mode and self.is_running():
+            self.start_qdb_watch(self.name)
+
     @qubes.events.handler('property-set:label')
     def on_property_set_label(self, event, name, newvalue, oldvalue=None):
         # pylint: disable=unused-argument
@@ -1714,53 +1728,53 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
         '''
         # pylint: disable=no-member
 
-        self.qdb.write('/name', self.name)
-        self.qdb.write('/type', self.__class__.__name__)
-        self.qdb.write('/qubes-vm-updateable', str(self.updateable))
-        self.qdb.write('/qubes-vm-persistence',
+        self.untrusted_qdb.write('/name', self.name)
+        self.untrusted_qdb.write('/type', self.__class__.__name__)
+        self.untrusted_qdb.write('/qubes-vm-updateable', str(self.updateable))
+        self.untrusted_qdb.write('/qubes-vm-persistence',
             'full' if self.updateable else 'rw-only')
-        self.qdb.write('/qubes-debug-mode', str(int(self.debug)))
+        self.untrusted_qdb.write('/qubes-debug-mode', str(int(self.debug)))
         try:
-            self.qdb.write('/qubes-base-template', self.template.name)
+            self.untrusted_qdb.write('/qubes-base-template', self.template.name)
         except AttributeError:
-            self.qdb.write('/qubes-base-template', '')
+            self.untrusted_qdb.write('/qubes-base-template', '')
 
-        self.qdb.write('/qubes-random-seed',
+        self.untrusted_qdb.write('/qubes-random-seed',
             base64.b64encode(qubes.utils.urandom(64)))
 
         if self.provides_network:
             # '/qubes-netvm-network' value is only checked for being non empty
-            self.qdb.write('/qubes-netvm-network', self.gateway)
-            self.qdb.write('/qubes-netvm-gateway', self.gateway)
-            self.qdb.write('/qubes-netvm-netmask', self.netmask)
+            self.untrusted_qdb.write('/qubes-netvm-network', self.gateway)
+            self.untrusted_qdb.write('/qubes-netvm-gateway', self.gateway)
+            self.untrusted_qdb.write('/qubes-netvm-netmask', self.netmask)
 
             for i, addr in zip(('primary', 'secondary'), self.dns):
-                self.qdb.write('/qubes-netvm-{}-dns'.format(i), addr)
+                self.untrusted_qdb.write('/qubes-netvm-{}-dns'.format(i), addr)
 
         if self.netvm is not None:
-            self.qdb.write('/qubes-ip', self.visible_ip)
-            self.qdb.write('/qubes-netmask', self.visible_netmask)
-            self.qdb.write('/qubes-gateway', self.visible_gateway)
+            self.untrusted_qdb.write('/qubes-ip', self.visible_ip)
+            self.untrusted_qdb.write('/qubes-netmask', self.visible_netmask)
+            self.untrusted_qdb.write('/qubes-gateway', self.visible_gateway)
 
             for i, addr in zip(('primary', 'secondary'), self.dns):
-                self.qdb.write('/qubes-{}-dns'.format(i), addr)
+                self.untrusted_qdb.write('/qubes-{}-dns'.format(i), addr)
 
 
         tzname = qubes.utils.get_timezone()
         if tzname:
-            self.qdb.write('/qubes-timezone', tzname)
+            self.untrusted_qdb.write('/qubes-timezone', tzname)
 
         for feature, value in self.features.items():
             if not feature.startswith('service.'):
                 continue
             service = feature[len('service.'):]
             # forcefully convert to '0' or '1'
-            self.qdb.write('/qubes-service/{}'.format(service),
+            self.untrusted_qdb.write('/qubes-service/{}'.format(service),
                 str(int(bool(value))))
 
-        self.qdb.write('/qubes-block-devices', '')
+        self.untrusted_qdb.write('/qubes-block-devices', '')
 
-        self.qdb.write('/qubes-usb-devices', '')
+        self.untrusted_qdb.write('/qubes-usb-devices', '')
 
         # TODO: Currently the whole qmemman is quite Xen-specific, so stay with
         # xenstore for it until decided otherwise
@@ -1771,6 +1785,8 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
 
         self.fire_event('domain-qdb-create')
 
+        self.start_qdb_watch(self.name)
+
     # TODO async; update this in constructor
     def _update_libvirt_domain(self):
         '''Re-initialise :py:attr:`libvirt_domain`.'''