From d728f4b9ff4e43513bbcedebe2b9de0119f92abf Mon Sep 17 00:00:00 2001
From: Wojtek Porczyk <woju@invisiblethingslab.com>
Date: Thu, 5 May 2016 17:19:48 +0200
Subject: [PATCH] qubes/app: reconnect to libvirtd after crash

Sometimes libvirt crashes. After that the connection (and all
vm.libvirt_domain-s) were unusable.

fixes QubesOS/qubes-issues#990
---
 qubes/app.py | 67 +++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 64 insertions(+), 3 deletions(-)

diff --git a/qubes/app.py b/qubes/app.py
index 1ac2a41f..e2acb975 100644
--- a/qubes/app.py
+++ b/qubes/app.py
@@ -24,7 +24,9 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 #
 
+import collections
 import errno
+import functools
 import grp
 import logging
 import os
@@ -61,6 +63,66 @@ import qubes.vm.qubesvm
 import qubes.vm.templatevm
 
 
+class VirDomainWrapper(object):
+    def __init__(self, connection, vm):
+        self._connection = connection
+        self._vm = vm
+
+    def _reconnect_if_dead(self):
+        is_dead = not self._vm.connect().isAlive()
+        if is_dead:
+            self._connection._reconnect_if_dead()
+            self._vm = self._connection._conn.lookupByUUID(self._vm.getUUID())
+        return is_dead
+
+    def __getattr__(self, attrname):
+        attr = getattr(self._vm, attrname)
+        if not isinstance(attr, collections.Callable):
+            return attr
+
+        @functools.wraps(attr)
+        def wrapper(*args, **kwargs):
+            try:
+                return attr(*args, **kwargs)
+            except libvirt.libvirtError as e:
+                if self._reconnect_if_dead():
+                    return getattr(self._vm, attrname)(*args, **kwargs)
+                raise
+        return wrapper
+
+
+class VirConnectWrapper(object):
+    def __init__(self, uri):
+        self._conn = libvirt.open(uri)
+
+    def _reconnect_if_dead(self):
+        is_dead = not self._conn.isAlive()
+        if is_dead:
+            self._conn = libvirt.open(self._conn.getURI())
+        return is_dead
+
+    def _wrap_domain(self, ret):
+        if isinstance(ret, libvirt.virDomain):
+            ret = VirDomainWrapper(self, ret)
+        return ret
+
+    def __getattr__(self, attrname):
+        attr = getattr(self._conn, attrname)
+        if not isinstance(attr, collections.Callable):
+            return attr
+
+        @functools.wraps(attr)
+        def wrapper(*args, **kwargs):
+            try:
+                return self._wrap_domain(attr(*args, **kwargs))
+            except libvirt.libvirtError as e:
+                if self._reconnect_if_dead():
+                    return self._wrap_domain(
+                        getattr(self._conn, attrname)(*args, **kwargs))
+                raise
+        return wrapper
+
+
 class VMMConnection(object):
     '''Connection to Virtual Machine Manager (libvirt)'''
 
@@ -102,9 +164,8 @@ class VMMConnection(object):
             self._xs = xen.lowlevel.xs.xs()
         if 'xen.lowlevel.cs' in sys.modules:
             self._xc = xen.lowlevel.xc.xc()
-        self._libvirt_conn = libvirt.open(qubes.config.defaults['libvirt_uri'])
-        if self._libvirt_conn is None:
-            raise qubes.exc.QubesException('Failed connect to libvirt driver')
+        self._libvirt_conn = VirConnectWrapper(
+            qubes.config.defaults['libvirt_uri'])
         libvirt.registerErrorHandler(self._libvirt_error_handler, None)
 
     @property