backups: make the restore more defensive

Continue restore even if some fails failed to extract

core/backup.py

@@ -889,7 +889,7 @@ class ExtractWorker(Process):
                 if self.tar2_process != None:
                     if self.tar2_process.wait() != 0:
                         self.collect_tar_output()
-                        raise QubesException(
+                        self.error_callback(
                                 "ERROR: unable to extract files for {0}, tar "
                                 "output:\n  {1}".\
                                 format(self.tar2_current_file,
@@ -914,6 +914,12 @@ class ExtractWorker(Process):
                             fcntl.fcntl(self.tar2_process.stderr.fileno(),
                                         fcntl.F_GETFL) | os.O_NONBLOCK)
                 self.tar2_stderr = []
+            elif not self.tar2_process:
+                # Extracting of the current archive failed, skip to the next
+                # archive
+                if not BACKUP_DEBUG:
+                    os.remove(filename)
+                continue
             else:
                 self.collect_tar_output()
                 if BACKUP_DEBUG:
@@ -975,7 +981,10 @@ class ExtractWorker(Process):
                     details = "\n".join(self.tar2_stderr)
                 else:
                     details = "%s failed" % run_error
-                raise QubesException("Error while processing '%s': %s " % \
+                self.tar2_process.terminate()
+                self.tar2_process.wait()
+                self.tar2_process = None
+                self.error_callback("Error while processing '%s': %s " % \
                         (self.tar2_current_file, details))
 
             # Delete the file as we don't need it anymore
@@ -1747,6 +1756,7 @@ def backup_restore_do(restore_info,
             vms_dirs.append(os.path.dirname(restore_info['dom0']['subdir']))
             vms_size += restore_info['dom0']['size']
 
+        try:
             restore_vm_dirs (backup_location,
                     restore_tmpdir,
                     passphrase=passphrase,
@@ -1762,6 +1772,13 @@ def backup_restore_do(restore_info,
                     encrypted=encrypted,
                     compressed=compressed,
                     appvm=appvm)
+        except QubesException as e:
+            if verify_only:
+                raise
+            else:
+                print_callback("Some errors occurred during data extraction, "
+                               "continuing anyway to restore at least some "
+                               "VMs")
     else:
         if verify_only:
             print_callback("WARNING: Backup verification not supported for "

qvm-tools/qvm-backup-restore

@@ -20,6 +20,7 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
 #
+from multiprocessing import Event
 
 from qubes.qubes import QubesVmCollection
 from qubes.qubes import QubesException
@@ -27,6 +28,7 @@ from qubes.backup import backup_restore_header
 from qubes.backup import backup_restore_prepare
 from qubes.backup import backup_restore_print_summary
 from qubes.backup import backup_restore_do
+import qubes.backup
 import sys
 from optparse import OptionParser
 
@@ -122,6 +124,11 @@ def main():
 
     print >> sys.stderr, "Checking backup content..."
 
+    error_detected = Event()
+    def error_callback(message):
+        error_detected.set()
+        print >> sys.stderr, message
+
     restore_info = None
     try:
         restore_info = backup_restore_prepare(
@@ -131,7 +138,8 @@ def main():
                 host_collection=host_collection,
                 encrypted=options.decrypt,
                 compressed=options.compressed,
-                appvm=appvm)
+                appvm=appvm,
+                error_callback=error_callback)
     except QubesException as e:
         print >> sys.stderr, "ERROR: %s" % str(e)
         exit(1)
@@ -229,11 +237,16 @@ def main():
 
     try:
         backup_restore_do(restore_info,
-                host_collection=host_collection)
+                host_collection=host_collection,
+                error_callback=error_callback)
     except QubesException as e:
         print >> sys.stderr, "ERROR: %s" % str(e)
 
     host_collection.unlock_db()
 
+    if error_detected.is_set():
+        print "-> Completed with errors!"
+        exit(1)
+    else:
         print "-> Done."
 main()