devices: make attach/detach related events async

This will allow starting processes and calling RPC services in those
events. This if required for usb devices, which are attached using RPC
services.
Intentionally keep device listing events synchronous only - to
discourage putting long-running actions there.

This change also require some not-async attach method version for
loading devices from qubes.xml - have `load_persistent` for this.

qubes/api/admin.py

@@ -969,7 +969,7 @@ class QubesAdminAPI(qubes.api.AbstractQubesAPI):
         assignment = qubes.devices.DeviceAssignment(
             dev.backend_domain, dev.ident,
             options=options, persistent=persistent)
-        self.dest.devices[devclass].attach(assignment)
+        yield from self.dest.devices[devclass].attach(assignment)
         self.app.save()
 
     @qubes.api.method('admin.vm.device.{endpoint}.Detach', endpoints=(ep.name
@@ -991,7 +991,7 @@ class QubesAdminAPI(qubes.api.AbstractQubesAPI):
 
         assignment = qubes.devices.DeviceAssignment(
             dev.backend_domain, dev.ident)
-        self.dest.devices[devclass].detach(assignment)
+        yield from self.dest.devices[devclass].detach(assignment)
         self.app.save()
 
     @qubes.api.method('admin.vm.firewall.Get', no_payload=True)

qubes/devices.py

@@ -36,14 +36,21 @@ Such extension should provide:
  including bus-specific properties)
  - handle `device-attach:bus` and `device-detach:bus` events for
  performing the attach/detach action; events are fired even when domain isn't
- running and extension should be prepared for this
+ running and extension should be prepared for this; handlers for those events
+ can be coroutines
  - handle `device-list:bus` event - list devices exposed by particular
  domain; it should return list of appropriate DeviceInfo objects
  - handle `device-get:bus` event - get one device object exposed by this
  domain of given identifier
  - handle `device-list-attached:class` event - list currently attached
  devices to this domain
+
+Note that device-listing event handlers can not be asynchronous. This for
+example means you can not call qrexec service there. This is intentional to
+keep device listing operation cheap. You need to design the extension to take
+this into account (for example by using QubesDB).
 '''
+import asyncio
 
 import qubes.utils
 
@@ -111,24 +118,32 @@ class DeviceCollection(object):
 
             Fired when device is attached to a VM.
 
+            Handler for this event can be asynchronous (a coroutine).
+
             :param device: :py:class:`DeviceInfo` object to be attached
 
         .. event:: device-pre-attach:<class> (device)
 
             Fired before device is attached to a VM
 
+            Handler for this event can be asynchronous (a coroutine).
+
             :param device: :py:class:`DeviceInfo` object to be attached
 
         .. event:: device-detach:<class> (device)
 
             Fired when device is detached from a VM.
 
+            Handler for this event can be asynchronous (a coroutine).
+
             :param device: :py:class:`DeviceInfo` object to be attached
 
         .. event:: device-pre-detach:<class> (device)
 
             Fired before device is detached from a VM
 
+            Handler for this event can be asynchronous (a coroutine).
+
             :param device: :py:class:`DeviceInfo` object to be attached
 
         .. event:: device-list:<class>
@@ -160,6 +175,7 @@ class DeviceCollection(object):
         self.devclass = qubes.utils.get_entry_point_one(
             'qubes.devices', self._bus)
 
+    @asyncio.coroutine
     def attach(self, device_assignment: DeviceAssignment):
         '''Attach (add) device to domain.
 
@@ -180,13 +196,26 @@ class DeviceCollection(object):
             raise DeviceAlreadyAttached(
                 'device {!s} of class {} already attached to {!s}'.format(
                     device, self._bus, self._vm))
-        self._vm.fire_event('device-pre-attach:'+self._bus, pre_event=True,
+        yield from self._vm.fire_event_async('device-pre-attach:' + self._bus,
+            pre_event=True,
             device=device, options=device_assignment.options)
         if device_assignment.persistent:
             self._set.add(device_assignment)
-        self._vm.fire_event('device-attach:' + self._bus,
+        yield from self._vm.fire_event_async('device-attach:' + self._bus,
             device=device, options=device_assignment.options)
 
+    def load_persistent(self, device_assignment: DeviceAssignment):
+        '''Load DeviceAssignment retrieved from qubes.xml
+
+        This can be used only for loading qubes.xml, when VM events are not
+        enabled yet.
+        '''
+        assert not self._vm.events_enabled
+        assert device_assignment.persistent
+        device_assignment.bus = self._bus
+        self._set.add(device_assignment)
+
+    @asyncio.coroutine
     def detach(self, device_assignment: DeviceAssignment):
         '''Detach (remove) device from domain.
 
@@ -208,13 +237,14 @@ class DeviceCollection(object):
                     device_assignment.ident, self._bus, self._vm))
 
         device = device_assignment.device
-        self._vm.fire_event('device-pre-detach:' + self._bus,
+        yield from self._vm.fire_event_async('device-pre-detach:' + self._bus,
             pre_event=True, device=device)
         if device in self._set:
             device_assignment.persistent = True
             self._set.discard(device_assignment)
 
-        self._vm.fire_event('device-detach:' + self._bus, device=device)
+        yield from self._vm.fire_event_async('device-detach:' + self._bus,
+            device=device)
 
     def attached(self):
         '''List devices which are (or may be) attached to this vm '''

qubes/tests/api_admin.py

@@ -1319,7 +1319,8 @@ class TC_00_VMs(AdminAPITestCase):
     def test_470_vm_device_list_persistent(self):
         assignment = qubes.devices.DeviceAssignment(self.vm, '1234',
             persistent=True)
-        self.vm.devices['testclass'].attach(assignment)
+        self.loop.run_until_complete(
+            self.vm.devices['testclass'].attach(assignment))
         value = self.call_mgmt_func(b'admin.vm.device.testclass.List',
             b'test-vm1')
         self.assertEqual(value,
@@ -1329,10 +1330,12 @@ class TC_00_VMs(AdminAPITestCase):
     def test_471_vm_device_list_persistent_options(self):
         assignment = qubes.devices.DeviceAssignment(self.vm, '1234',
             persistent=True, options={'opt1': 'value'})
-        self.vm.devices['testclass'].attach(assignment)
+        self.loop.run_until_complete(
+            self.vm.devices['testclass'].attach(assignment))
         assignment = qubes.devices.DeviceAssignment(self.vm, '4321',
             persistent=True)
-        self.vm.devices['testclass'].attach(assignment)
+        self.loop.run_until_complete(
+            self.vm.devices['testclass'].attach(assignment))
         value = self.call_mgmt_func(b'admin.vm.device.testclass.List',
             b'test-vm1')
         self.assertEqual(value,
@@ -1360,7 +1363,8 @@ class TC_00_VMs(AdminAPITestCase):
             self.device_list_attached_testclass)
         assignment = qubes.devices.DeviceAssignment(self.vm, '4321',
             persistent=True)
-        self.vm.devices['testclass'].attach(assignment)
+        self.loop.run_until_complete(
+            self.vm.devices['testclass'].attach(assignment))
         value = self.call_mgmt_func(b'admin.vm.device.testclass.List',
             b'test-vm1')
         self.assertEqual(value,
@@ -1373,7 +1377,8 @@ class TC_00_VMs(AdminAPITestCase):
             self.device_list_attached_testclass)
         assignment = qubes.devices.DeviceAssignment(self.vm, '4321',
             persistent=True)
-        self.vm.devices['testclass'].attach(assignment)
+        self.loop.run_until_complete(
+            self.vm.devices['testclass'].attach(assignment))
         value = self.call_mgmt_func(b'admin.vm.device.testclass.List',
             b'test-vm1', b'test-vm1+1234')
         self.assertEqual(value,

qubes/tests/devices.py

@@ -91,15 +91,15 @@ class TC_00_DeviceCollection(qubes.tests.QubesTestCase):
         self.assertFalse(self.collection._set)
 
     def test_001_attach(self):
-        self.collection.attach(self.assignment)
+        self.loop.run_until_complete(self.collection.attach(self.assignment))
         self.assertEventFired(self.emitter, 'device-pre-attach:testclass')
         self.assertEventFired(self.emitter, 'device-attach:testclass')
         self.assertEventNotFired(self.emitter, 'device-pre-detach:testclass')
         self.assertEventNotFired(self.emitter, 'device-detach:testclass')
 
     def test_002_detach(self):
-        self.collection.attach(self.assignment)
+        self.loop.run_until_complete(self.collection.attach(self.assignment))
-        self.collection.detach(self.assignment)
+        self.loop.run_until_complete(self.collection.detach(self.assignment))
         self.assertEventFired(self.emitter, 'device-pre-attach:testclass')
         self.assertEventFired(self.emitter, 'device-attach:testclass')
         self.assertEventFired(self.emitter, 'device-pre-detach:testclass')
@@ -107,24 +107,27 @@ class TC_00_DeviceCollection(qubes.tests.QubesTestCase):
 
     def test_010_empty_detach(self):
         with self.assertRaises(LookupError):
-            self.collection.detach(self.assignment)
+            self.loop.run_until_complete(
+                self.collection.detach(self.assignment))
 
     def test_011_double_attach(self):
-        self.collection.attach(self.assignment)
+        self.loop.run_until_complete(self.collection.attach(self.assignment))
 
         with self.assertRaises(qubes.devices.DeviceAlreadyAttached):
-            self.collection.attach(self.assignment)
+            self.loop.run_until_complete(
+                self.collection.attach(self.assignment))
 
     def test_012_double_detach(self):
-        self.collection.attach(self.assignment)
+        self.loop.run_until_complete(self.collection.attach(self.assignment))
-        self.collection.detach(self.assignment)
+        self.loop.run_until_complete(self.collection.detach(self.assignment))
 
         with self.assertRaises(qubes.devices.DeviceNotAttached):
-            self.collection.detach(self.assignment)
+            self.loop.run_until_complete(
+                self.collection.detach(self.assignment))
 
     def test_013_list_attached_persistent(self):
         self.assertEqual(set([]), set(self.collection.persistent()))
-        self.collection.attach(self.assignment)
+        self.loop.run_until_complete(self.collection.attach(self.assignment))
         self.assertEventFired(self.emitter, 'device-list-attached:testclass')
         self.assertEqual({self.device}, set(self.collection.persistent()))
         self.assertEqual({self.device},
@@ -135,7 +138,7 @@ class TC_00_DeviceCollection(qubes.tests.QubesTestCase):
     def test_014_list_attached_non_persistent(self):
         self.assignment.persistent = False
         self.emitter.running = True
-        self.collection.attach(self.assignment)
+        self.loop.run_until_complete(self.collection.attach(self.assignment))
         # device-attach event not implemented, so manipulate object manually
         self.device.frontend_domain = self.emitter
         self.assertEqual({self.device},
@@ -167,6 +170,7 @@ class TC_01_DeviceManager(qubes.tests.QubesTestCase):
             backend_domain=device.backend_domain,
             ident=device.ident,
             persistent=True)
-        self.manager['testclass'].attach(assignment)
+        self.loop.run_until_complete(
+            self.manager['testclass'].attach(assignment))
         self.assertEventFired(self.emitter, 'device-attach:testclass')
 

qubes/vm/__init__.py

@@ -316,7 +316,7 @@ class BaseVM(qubes.PropertyHolder):
                     options,
                     persistent=True
                 )
-                self.devices[devclass].attach(device_assignment)
+                self.devices[devclass].load_persistent(device_assignment)
 
         # tags
         for node in self.xml.xpath('./tags/tag'):