vm: add shutdown_timeout property, make vm.shutdown(wait=True) use it

vm.shutdown(wait=True) waited indefinitely for the shutdown, which makes
useless without some boilerplate handling the timeout. Since the timeout
may depend on the operating system inside, add a per-VM property for it,
with value inheritance from template and then from global
default_shutdown_timeout property.

When timeout is reached, the method raises exception - whether to kill
it or not is left to the caller.

Fixes QubesOS/qubes-issues#1696

qubes/app.py

@@ -725,6 +725,12 @@ class Qubes(qubes.PropertyHolder):
         doc='''Default time in seconds after which qrexec connection attempt is
             deemed failed''')
 
+    default_shutdown_timeout = qubes.property('default_shutdown_timeout',
+        load_stage=3,
+        default=60,
+        type=int,
+        doc='''Default time in seconds for VM shutdown to complete''')
+
     stats_interval = qubes.property('stats_interval',
         default=3,
         type=int,

qubes/exc.py

@@ -101,6 +101,14 @@ class QubesVMNotHaltedError(QubesVMError):
         super(QubesVMNotHaltedError, self).__init__(vm,
             msg or 'Domain is not powered off: {!r}'.format(vm.name))
 
+class QubesVMShutdownTimeoutError(QubesVMError):
+    '''Domain shutdown timed out.
+
+    '''
+    def __init__(self, vm, msg=None):
+        super(QubesVMShutdownTimeoutError, self).__init__(vm,
+            msg or 'Domain shutdown timed out: {!r}'.format(vm.name))
+
 
 class QubesNoTemplateError(QubesVMError):
     '''Cannot start domain, because there is no template'''

qubes/vm/qubesvm.py

@@ -517,6 +517,14 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
             failed. Operating system inside VM should be able to boot in this
             time.''')
 
+    shutdown_timeout = qubes.property('shutdown_timeout', type=int,
+        default=_default_with_template('shutdown_timeout',
+            lambda self: self.app.default_shutdown_timeout),
+        setter=_setter_positive_int,
+        doc='''Time in seconds for shutdown of the VM, after which VM may be
+            forcefully powered off. Operating system inside VM should be
+            able to fully shutdown in this time.''')
+
     autostart = qubes.property('autostart', default=False,
         type=bool, setter=qubes.property.bool,
         doc='''Setting this to `True` means that VM should be autostarted on
@@ -1055,9 +1063,13 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
                 self.name)
 
     @asyncio.coroutine
-    def shutdown(self, force=False, wait=False):
+    def shutdown(self, force=False, wait=False, timeout=None):
         '''Shutdown domain.
 
+        :param force: ignored
+        :param wait: wait for shutdown to complete
+        :param timeout: shutdown wait timeout (for *wait*=True), defaults to
+        :py:attr:`shutdown_timeout`
         :raises qubes.exc.QubesVMNotStartedError: \
             when domain is already shut down.
         '''
@@ -1070,8 +1082,14 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
 
         self.libvirt_domain.shutdown()
 
-        while wait and not self.is_halted():
-            yield from asyncio.sleep(0.25)
+        if wait:
+            if timeout is None:
+                timeout = self.shutdown_timeout
+            while timeout > 0 and not self.is_halted():
+                yield from asyncio.sleep(0.25)
+                timeout -= 0.25
+            if not self.is_halted():
+                raise qubes.exc.QubesVMShutdownTimeoutError(self)
 
         return self