From e0686e1e02a6362df7cd509283b8548fe573a415 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Thu, 10 Mar 2016 11:22:52 +0100
Subject: [PATCH 01/32] backup: initial conversion to core3 API

---
 {core => qubes}/backup.py            | 250 +++++++++++++++------------
 qubes/tests/__init__.py              |  34 ++--
 {tests => qubes/tests/int}/backup.py |  68 ++++----
 qubes/utils.py                       |  44 ++++-
 rpm_spec/core-dom0.spec              |   2 +
 5 files changed, 230 insertions(+), 168 deletions(-)
 rename {core => qubes}/backup.py (93%)
 rename {tests => qubes/tests/int}/backup.py (83%)

diff --git a/core/backup.py b/qubes/backup.py
similarity index 93%
rename from core/backup.py
rename to qubes/backup.py
index 7ba8808f..09ccb11f 100644
--- a/core/backup.py
+++ b/qubes/backup.py
@@ -23,10 +23,7 @@
 #
 #
 from __future__ import unicode_literals
-from qubes import QubesException, QubesVmCollection
-from qubes import QubesVmClasses
-from qubes import system_path, vm_files
-from qubesutils import size_to_human, print_stdout, print_stderr, get_disk_usage
+from qubes.utils import size_to_human
 import sys
 import os
 import fcntl
@@ -40,6 +37,7 @@ import pwd
 import errno
 import datetime
 from multiprocessing import Queue, Process
+import qubes
 
 BACKUP_DEBUG = False
 
@@ -53,9 +51,37 @@ MAX_STDERR_BYTES = 1024
 # header + qubes.xml max size
 HEADER_QUBES_XML_MAX_SIZE = 1024 * 1024
 
+BLKSIZE = 512
+
 # global state for backup_cancel()
 running_backup_operation = None
 
+def print_stdout(text):
+    print (text)
+
+def print_stderr(text):
+    print >> sys.stderr, (text)
+
+def get_disk_usage_one(st):
+    try:
+        return st.st_blocks * BLKSIZE
+    except AttributeError:
+        return st.st_size
+
+def get_disk_usage(path):
+    try:
+        st = os.lstat(path)
+    except OSError:
+        return 0
+
+    ret = get_disk_usage_one(st)
+
+    # if path is not a directory, this is skipped
+    for dirpath, dirnames, filenames in os.walk(path):
+        for name in dirnames + filenames:
+            ret += get_disk_usage_one(os.lstat(os.path.join(dirpath, name)))
+
+    return ret
 
 class BackupOperationInfo:
     def __init__(self):
@@ -64,7 +90,7 @@ class BackupOperationInfo:
         self.tmpdir_to_remove = None
 
 
-class BackupCanceledError(QubesException):
+class BackupCanceledError(qubes.exc.QubesException):
     def __init__(self, msg, tmpdir=None):
         super(BackupCanceledError, self).__init__(msg)
         self.tmpdir = tmpdir
@@ -86,7 +112,7 @@ def file_to_backup(file_path, subdir=None):
 
     if subdir is None:
         abs_file_path = os.path.abspath(file_path)
-        abs_base_dir = os.path.abspath(system_path["qubes_base_dir"]) + '/'
+        abs_base_dir = os.path.abspath(qubes.config.system_path["qubes_base_dir"]) + '/'
         abs_file_dir = os.path.dirname(abs_file_path) + '/'
         (nothing, directory, subdir) = abs_file_dir.partition(abs_base_dir)
         assert nothing == ""
@@ -115,33 +141,35 @@ def backup_cancel():
     return True
 
 
-def backup_prepare(vms_list=None, exclude_list=None,
+def backup_prepare(app, vms_list=None, exclude_list=None,
                    print_callback=print_stdout, hide_vm_names=True):
     """
     If vms = None, include all (sensible) VMs;
     exclude_list is always applied
     """
-    files_to_backup = file_to_backup(system_path["qubes_store_filename"])
+    # XXX hack for tests, where store filename is qubes-test.xml
+    qubes_xml = app._store
+    if os.path.basename(qubes_xml) != 'qubes.xml':
+        dir = tempfile.mkdtemp()
+        shutil.copy(qubes_xml, os.path.join(dir, 'qubes.xml'))
+        qubes_xml = os.path.join(dir, 'qubes.xml')
+        # FIXME cleanup tempdir later
+
+    files_to_backup = file_to_backup(qubes_xml, '')
 
     if exclude_list is None:
         exclude_list = []
 
-    qvm_collection = QubesVmCollection()
-    qvm_collection.lock_db_for_writing()
-    qvm_collection.load()
-
     if vms_list is None:
-        all_vms = [vm for vm in qvm_collection.values()]
+        all_vms = [vm for vm in app.domains]
         selected_vms = [vm for vm in all_vms if vm.include_in_backups]
         appvms_to_backup = [vm for vm in selected_vms if
                             vm.is_appvm() and not vm.internal]
-        netvms_to_backup = [vm for vm in selected_vms if
-                            vm.is_netvm() and not vm.qid == 0]
         template_vms_worth_backingup = [vm for vm in selected_vms if (
             vm.is_template() and vm.include_in_backups)]
-        dom0 = [qvm_collection[0]]
+        dom0 = [app.domains[0]]
 
-        vms_list = appvms_to_backup + netvms_to_backup + \
+        vms_list = appvms_to_backup + \
             template_vms_worth_backingup + dom0
 
     vms_for_backup = vms_list
@@ -207,11 +235,13 @@ def backup_prepare(vms_list=None, exclude_list=None,
                                                   subdir)
         if os.path.exists(vm.firewall_conf):
             files_to_backup += file_to_backup(vm.firewall_conf, subdir)
-        if 'appmenus_whitelist' in vm_files and \
+        if 'appmenus_whitelist' in qubes.config.vm_files and \
                 os.path.exists(os.path.join(vm.dir_path,
-                                            vm_files['appmenus_whitelist'])):
+                                            qubes.config.vm_files[
+                                                'appmenus_whitelist'])):
             files_to_backup += file_to_backup(
-                os.path.join(vm.dir_path, vm_files['appmenus_whitelist']),
+                os.path.join(vm.dir_path, qubes.config.vm_files[
+                    'appmenus_whitelist']),
                 subdir)
 
         if vm.updateable:
@@ -255,7 +285,7 @@ def backup_prepare(vms_list=None, exclude_list=None,
         else:
             template_subdir = os.path.relpath(
                 vm.dir_path,
-                system_path["qubes_base_dir"]) + '/'
+                qubes.config.system_path["qubes_base_dir"]) + '/'
         template_to_backup = [{"path": vm.dir_path + '/.',
                                "size": vm_sz,
                                "subdir": template_subdir}]
@@ -280,7 +310,7 @@ def backup_prepare(vms_list=None, exclude_list=None,
 
     # Initialize backup flag on all VMs
     vms_for_backup_qid = [vm.qid for vm in vms_for_backup]
-    for vm in qvm_collection.values():
+    for vm in app.domains:
         vm.backup_content = False
         if vm.qid == 0:
             # handle dom0 later
@@ -292,8 +322,9 @@ def backup_prepare(vms_list=None, exclude_list=None,
             if hide_vm_names:
                 vm.backup_path = 'vm%d' % vm.qid
             else:
-                vm.backup_path = os.path.relpath(vm.dir_path,
-                                                 system_path["qubes_base_dir"])
+                vm.backup_path = os.path.relpath(
+                    vm.dir_path,
+                    qubes.config.system_path["qubes_base_dir"])
 
     # Dom0 user home
     if 0 in vms_for_backup_qid:
@@ -309,7 +340,7 @@ def backup_prepare(vms_list=None, exclude_list=None,
             {"path": home_dir, "size": home_sz, "subdir": 'dom0-home/'}]
         files_to_backup += home_to_backup
 
-        vm = qvm_collection[0]
+        vm = app.domains[0]
         vm.backup_content = True
         vm.backup_size = home_sz
         vm.backup_path = os.path.join('dom0-home', os.path.basename(home_dir))
@@ -326,9 +357,8 @@ def backup_prepare(vms_list=None, exclude_list=None,
 
         print_callback(s)
 
-    qvm_collection.save()
+    app.save()
     # FIXME: should be after backup completed
-    qvm_collection.unlock_db()
 
     total_backup_sz = 0
     for f in files_to_backup:
@@ -355,13 +385,13 @@ def backup_prepare(vms_list=None, exclude_list=None,
         s += fmt.format('-')
     print_callback(s)
 
-    vms_not_for_backup = [vm.name for vm in qvm_collection.values()
+    vms_not_for_backup = [vm.name for vm in app.domains
                           if not vm.backup_content]
     print_callback("VMs not selected for backup: %s" % " ".join(
         vms_not_for_backup))
 
     if there_are_running_vms:
-        raise QubesException("Please shutdown all VMs before proceeding.")
+        raise qubes.exc.QubesException("Please shutdown all VMs before proceeding.")
 
     for fileinfo in files_to_backup:
         assert len(fileinfo["subdir"]) == 0 or fileinfo["subdir"][-1] == '/', \
@@ -406,7 +436,7 @@ class SendWorker(Process):
                     self.queue.get()
                 # handle only exit code 2 (tar fatal error) or
                 # greater (call failed?)
-                raise QubesException(
+                raise qubes.exc.QubesException(
                     "ERROR: Failed to write the backup, out of disk space? "
                     "Check console output or ~/.xsession-errors for details.")
 
@@ -442,11 +472,11 @@ def prepare_backup_header(target_directory, passphrase, compressed=False,
                             stdin=open(header_file_path, "r"),
                             stdout=open(header_file_path + ".hmac", "w"))
     if hmac.wait() != 0:
-        raise QubesException("Failed to compute hmac of header file")
+        raise qubes.exc.QubesException("Failed to compute hmac of header file")
     return HEADER_FILENAME, HEADER_FILENAME + ".hmac"
 
 
-def backup_do(base_backup_dir, files_to_backup, passphrase,
+def backup_do(app, base_backup_dir, files_to_backup, passphrase,
               progress_callback=None, encrypted=False, appvm=None,
               compressed=False, hmac_algorithm=DEFAULT_HMAC_ALGORITHM,
               crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM):
@@ -460,7 +490,7 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
                                vmproc.stderr.read())
                 else:
                     message = "Failed to write the backup. Out of disk space?"
-                raise QubesException(message)
+                raise qubes.exc.QubesException(message)
         queue.put(element)
 
     total_backup_sz = 0
@@ -497,7 +527,7 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
 
             # Create the target directory
             if not os.path.exists(os.path.dirname(base_backup_dir)):
-                raise QubesException(
+                raise qubes.exc.QubesException(
                     "ERROR: the backup directory for {0} does not exists".
                     format(base_backup_dir))
 
@@ -658,11 +688,11 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
             if run_error and run_error != "size_limit":
                 send_proc.terminate()
                 if run_error == "VM" and vmproc:
-                    raise QubesException(
+                    raise qubes.exc.QubesException(
                         "Failed to write the backup, VM output:\n" +
                         vmproc.stderr.read(MAX_STDERR_BYTES))
                 else:
-                    raise QubesException("Failed to perform backup: error in " +
+                    raise qubes.exc.QubesException("Failed to perform backup: error in " +
                                          run_error)
 
             # Send the chunk to the backup target
@@ -711,7 +741,7 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
     running_backup_operation = None
 
     if send_proc.exitcode != 0:
-        raise QubesException(
+        raise qubes.exc.QubesException(
             "Failed to send backup: error in the sending process")
 
     if vmproc:
@@ -722,16 +752,11 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
         vmproc.stdin.close()
 
     # Save date of last backup
-    qvm_collection = QubesVmCollection()
-    qvm_collection.lock_db_for_writing()
-    qvm_collection.load()
-
-    for vm in qvm_collection.values():
+    for vm in app.domains:
         if vm.backup_content:
             vm.backup_timestamp = datetime.datetime.now()
 
-    qvm_collection.save()
-    qvm_collection.unlock_db()
+    app.save()
 
 
 '''
@@ -829,7 +854,7 @@ def verify_hmac(filename, hmacfile, passphrase, algorithm):
         print "Verifying file " + filename
 
     if hmacfile != filename + ".hmac":
-        raise QubesException(
+        raise qubes.exc.QubesException(
             "ERROR: expected hmac for {}, but got {}".
             format(filename, hmacfile))
 
@@ -840,7 +865,7 @@ def verify_hmac(filename, hmacfile, passphrase, algorithm):
     hmac_stdout, hmac_stderr = hmac_proc.communicate()
 
     if len(hmac_stderr) > 0:
-        raise QubesException(
+        raise qubes.exc.QubesException(
             "ERROR: verify file {0}: {1}".format(filename, hmac_stderr))
     else:
         if BACKUP_DEBUG:
@@ -853,7 +878,7 @@ def verify_hmac(filename, hmacfile, passphrase, algorithm):
                 print "File verification OK -> Sending file " + filename
             return True
         else:
-            raise QubesException(
+            raise qubes.exc.QubesException(
                 "ERROR: invalid hmac for file {0}: {1}. "
                 "Is the passphrase correct?".
                 format(filename, load_hmac(hmac_stdout)))
@@ -1081,7 +1106,7 @@ class ExtractWorker2(Process):
                 self.tar2_process.wait()
             elif self.tar2_process.wait() != 0:
                 self.collect_tar_output()
-                raise QubesException(
+                raise qubes.exc.QubesException(
                     "unable to extract files for {0}.{1} Tar command "
                     "output: %s".
                     format(self.tar2_current_file,
@@ -1241,7 +1266,7 @@ class ExtractWorker3(ExtractWorker2):
                 self.tar2_process.wait()
             elif self.tar2_process.wait() != 0:
                 self.collect_tar_output()
-                raise QubesException(
+                raise qubes.exc.QubesException(
                     "unable to extract files for {0}.{1} Tar command "
                     "output: %s".
                     format(self.tar2_current_file,
@@ -1274,7 +1299,7 @@ def parse_backup_header(filename):
     with open(filename, 'r') as f:
         for line in f.readlines():
             if line.count('=') != 1:
-                raise QubesException("Invalid backup header (line %s)" % line)
+                raise qubes.exc.QubesException("Invalid backup header (line %s)" % line)
             (key, value) = line.strip().split('=')
             if not any([key == getattr(BackupHeader, attr) for attr in dir(
                     BackupHeader)]):
@@ -1394,7 +1419,7 @@ def restore_vm_dirs(backup_source, restore_tmpdir, passphrase, vms_dirs, vms,
             else:
                 command.wait()
                 proc_error_msg = command.stderr.read(MAX_STDERR_BYTES)
-            raise QubesException("Premature end of archive while receiving "
+            raise qubes.exc.QubesException("Premature end of archive while receiving "
                                  "backup header. Process output:\n" +
                                  proc_error_msg)
         filename = os.path.join(restore_tmpdir, filename)
@@ -1406,11 +1431,11 @@ def restore_vm_dirs(backup_source, restore_tmpdir, passphrase, vms_dirs, vms,
                     file_ok = True
                     hmac_algorithm = hmac_algo
                     break
-            except QubesException:
+            except qubes.exc.QubesException:
                 # Ignore exception here, try the next algo
                 pass
         if not file_ok:
-            raise QubesException("Corrupted backup header (hmac verification "
+            raise qubes.exc.QubesException("Corrupted backup header (hmac verification "
                                  "failed). Is the password correct?")
         if os.path.basename(filename) == HEADER_FILENAME:
             header_data = parse_backup_header(filename)
@@ -1524,18 +1549,18 @@ def restore_vm_dirs(backup_source, restore_tmpdir, passphrase, vms_dirs, vms,
                                       tmpdir=restore_tmpdir)
 
         if command.wait() != 0 and not expect_tar_error:
-            raise QubesException(
+            raise qubes.exc.QubesException(
                 "unable to read the qubes backup file {0} ({1}). "
                 "Is it really a backup?".format(backup_source, command.wait()))
         if vmproc:
             if vmproc.wait() != 0:
-                raise QubesException(
+                raise qubes.exc.QubesException(
                     "unable to read the qubes backup {0} "
                     "because of a VM error: {1}".format(
                         backup_source, vmproc.stderr.read(MAX_STDERR_BYTES)))
 
         if filename and filename != "EOF":
-            raise QubesException(
+            raise qubes.exc.QubesException(
                 "Premature end of archive, the last file was %s" % filename)
     except:
         to_extract.put("ERROR")
@@ -1551,7 +1576,7 @@ def restore_vm_dirs(backup_source, restore_tmpdir, passphrase, vms_dirs, vms,
         print_callback("Extraction process finished with code:" +
                        str(extract_proc.exitcode))
     if extract_proc.exitcode != 0:
-        raise QubesException(
+        raise qubes.exc.QubesException(
             "unable to extract the qubes backup. "
             "Check extracting process errors.")
 
@@ -1584,7 +1609,7 @@ def load_hmac(hmac):
     if len(hmac) > 1:
         hmac = hmac[1].strip()
     else:
-        raise QubesException("ERROR: invalid hmac file content")
+        raise qubes.exc.QubesException("ERROR: invalid hmac file content")
 
     return hmac
 
@@ -1642,6 +1667,7 @@ def backup_restore_header(source, passphrase,
     return (restore_tmpdir, os.path.join(restore_tmpdir, "qubes.xml"),
             header_data)
 
+
 def generate_new_name_for_conflicting_vm(orig_name, host_collection,
                                          restore_info):
     number = 1
@@ -1651,7 +1677,7 @@ def generate_new_name_for_conflicting_vm(orig_name, host_collection,
     while (new_name in restore_info.keys() or
            new_name in map(lambda x: x.get('rename_to', None),
                            restore_info.values()) or
-           host_collection.get_vm_by_name(new_name)):
+           new_name in host_collection.domains):
         new_name = str('{}{}'.format(orig_name, number))
         number += 1
         if number == 100:
@@ -1660,6 +1686,7 @@ def generate_new_name_for_conflicting_vm(orig_name, host_collection,
     return new_name
 
 def restore_info_verify(restore_info, host_collection):
+    assert isinstance(host_collection, qubes.Qubes)
     options = restore_info['$OPTIONS$']
     for vm in restore_info.keys():
         if vm in ['$OPTIONS$', 'dom0']:
@@ -1674,7 +1701,7 @@ def restore_info_verify(restore_info, host_collection):
 
         vm_info.pop('already-exists', None)
         if not options['verify-only'] and \
-                host_collection.get_vm_by_name(vm) is not None:
+                vm in host_collection.domains:
             if options['rename-conflicting']:
                 new_name = generate_new_name_for_conflicting_vm(
                     vm, host_collection, restore_info
@@ -1690,7 +1717,7 @@ def restore_info_verify(restore_info, host_collection):
         vm_info.pop('missing-template', None)
         if vm_info['template']:
             template_name = vm_info['template']
-            host_template = host_collection.get_vm_by_name(template_name)
+            host_template = host_collection.domains.get(template_name, None)
             if not host_template or not host_template.is_template():
                 # Maybe the (custom) template is in the backup?
                 if not (template_name in restore_info.keys() and
@@ -1699,7 +1726,7 @@ def restore_info_verify(restore_info, host_collection):
                         if 'orig-template' not in vm_info.keys():
                             vm_info['orig-template'] = template_name
                         vm_info['template'] = host_collection \
-                            .get_default_template().name
+                            .default_template.name
                     else:
                         vm_info['missing-template'] = True
 
@@ -1708,7 +1735,7 @@ def restore_info_verify(restore_info, host_collection):
         if vm_info['netvm']:
             netvm_name = vm_info['netvm']
 
-            netvm_on_host = host_collection.get_vm_by_name(netvm_name)
+            netvm_on_host = host_collection.domains.get(netvm_name, None)
 
             # No netvm on the host?
             if not ((netvm_on_host is not None) and netvm_on_host.is_netvm()):
@@ -1718,7 +1745,7 @@ def restore_info_verify(restore_info, host_collection):
                         restore_info[netvm_name]['vm'].is_netvm()):
                     if options['use-default-netvm']:
                         vm_info['netvm'] = host_collection \
-                            .get_default_netvm().name
+                            .default_netvm.name
                         vm_info['vm'].uses_default_netvm = True
                     elif options['use-none-netvm']:
                         vm_info['netvm'] = None
@@ -1775,7 +1802,7 @@ def backup_restore_prepare(backup_location, passphrase, options=None,
             return False
 
         backup_vm_dir_path = check_vm.dir_path.replace(
-            system_path["qubes_base_dir"], backup_dir)
+            qubes.config.system_path["qubes_base_dir"], backup_dir)
 
         if os.path.exists(backup_vm_dir_path):
             return True
@@ -1812,11 +1839,11 @@ def backup_restore_prepare(backup_location, passphrase, options=None,
         is_vm_included_in_backup = is_vm_included_in_backup_v2
         if not appvm:
             if not os.path.isfile(backup_location):
-                raise QubesException("Invalid backup location (not a file or "
+                raise qubes.exc.QubesException("Invalid backup location (not a file or "
                                      "directory with qubes.xml)"
                                      ": %s" % unicode(backup_location))
     else:
-        raise QubesException(
+        raise qubes.exc.QubesException(
             "Unknown backup format version: %s" % str(format_version))
 
     (restore_tmpdir, qubes_xml, header_data) = backup_restore_header(
@@ -1847,17 +1874,12 @@ def backup_restore_prepare(backup_location, passphrase, options=None,
 
     if BACKUP_DEBUG:
         print "Loading file", qubes_xml
-    backup_collection = QubesVmCollection(store_filename=qubes_xml)
-    backup_collection.lock_db_for_reading()
-    backup_collection.load()
+    backup_collection = qubes.Qubes(qubes_xml)
 
     if host_collection is None:
-        host_collection = QubesVmCollection()
-        host_collection.lock_db_for_reading()
-        host_collection.load()
-        host_collection.unlock_db()
+        host_collection = qubes.Qubes()
 
-    backup_vms_list = [vm for vm in backup_collection.values()]
+    backup_vms_list = [vm for vm in backup_collection.domains]
     vms_to_restore = {}
 
     # ... and the actual data
@@ -1907,8 +1929,9 @@ def backup_restore_prepare(backup_location, passphrase, options=None,
 
     # ...and dom0 home
     if options['dom0-home'] and \
-            is_vm_included_in_backup(backup_location, backup_collection[0]):
-        vm = backup_collection[0]
+            is_vm_included_in_backup(backup_location,
+                backup_collection.domains[0]):
+        vm = backup_collection.domains[0]
         vms_to_restore['dom0'] = {}
         if format_version == 1:
             vms_to_restore['dom0']['subdir'] = \
@@ -2054,13 +2077,13 @@ def backup_restore_do(restore_info,
     # Private functions begin
     def restore_vm_dir_v1(backup_dir, src_dir, dst_dir):
 
-        backup_src_dir = src_dir.replace(system_path["qubes_base_dir"],
-                                         backup_dir)
+        backup_src_dir = src_dir.replace(
+            qubes.config.system_path["qubes_base_dir"], backup_dir)
 
         # We prefer to use Linux's cp, because it nicely handles sparse files
         cp_retcode = subprocess.call(["cp", "-rp", "--reflink=auto", backup_src_dir, dst_dir])
         if cp_retcode != 0:
-            raise QubesException(
+            raise qubes.exc.QubesException(
                 "*** Error while copying file {0} to {1}".format(backup_src_dir,
                                                                  dst_dir))
 
@@ -2084,9 +2107,8 @@ def backup_restore_do(restore_info,
 
     lock_obtained = False
     if host_collection is None:
-        host_collection = QubesVmCollection()
-        host_collection.lock_db_for_writing()
-        host_collection.load()
+        host_collection = qubes.Qubes()
+        # FIXME
         lock_obtained = True
 
     # Perform VM restoration in backup order
@@ -2129,7 +2151,7 @@ def backup_restore_do(restore_info,
                             compressed=compressed,
                             compression_filter=compression_filter,
                             appvm=appvm)
-        except QubesException:
+        except qubes.exc.QubesException:
             if verify_only:
                 raise
             else:
@@ -2148,9 +2170,8 @@ def backup_restore_do(restore_info,
         shutil.rmtree(restore_tmpdir)
         return
 
-    # Add VM in right order
-    for (vm_class_name, vm_class) in sorted(QubesVmClasses.items(),
-                                            key=lambda _x: _x[1].load_order):
+    # First load templates, then other VMs
+    for do_templates in (True, False):
         if running_backup_operation.canceled:
             break
         for vm in vms.values():
@@ -2158,11 +2179,10 @@ def backup_restore_do(restore_info,
                 # only break the loop to save qubes.xml with already restored
                 # VMs
                 break
-            if not vm.__class__ == vm_class:
+            if vm.is_template != do_templates:
                 continue
             if callable(print_callback):
-                print_callback("-> Restoring {type} {0}...".
-                               format(vm.name, type=vm_class_name))
+                print_callback("-> Restoring {0}...".format(vm.name))
             retcode = subprocess.call(
                 ["mkdir", "-p", os.path.dirname(vm.dir_path)])
             if retcode != 0:
@@ -2171,10 +2191,12 @@ def backup_restore_do(restore_info,
                 error_callback("Skipping...")
                 continue
 
-            template = None
-            if vm.template is not None:
-                template_name = restore_info[vm.name]['template']
-                template = host_collection.get_vm_by_name(template_name)
+            kwargs = {}
+            if hasattr(vm, 'template'):
+                if vm.template is not None:
+                    kwargs['template'] = restore_info[vm.name]['template']
+                else:
+                    kwargs['template'] = None
 
             new_vm = None
             vm_name = vm.name
@@ -2182,9 +2204,13 @@ def backup_restore_do(restore_info,
                 vm_name = restore_info[vm.name]['rename-to']
 
             try:
-                new_vm = host_collection.add_new_vm(vm_class_name, name=vm_name,
-                                                    template=template,
-                                                    installed_by_rpm=False)
+                # first only minimal set, later clone_properties will be called
+                new_vm = host_collection.add_new_vm(
+                    vm.__class__,
+                    name=vm_name,
+                    label=vm.label,
+                    installed_by_rpm=False,
+                    **kwargs)
                 if os.path.exists(new_vm.dir_path):
                     move_to_path = tempfile.mkdtemp('', os.path.basename(
                         new_vm.dir_path), os.path.dirname(new_vm.dir_path))
@@ -2214,20 +2240,19 @@ def backup_restore_do(restore_info,
                 error_callback("ERROR: {0}".format(err))
                 error_callback("*** Skipping VM: {0}".format(vm.name))
                 if new_vm:
-                    host_collection.pop(new_vm.qid)
+                    del host_collection.domains[new_vm.qid]
                 continue
 
-            # FIXME: cannot check for 'kernel' property, because it is always
-            # defined - accessing it touches non-existent '_kernel'
-            if not isinstance(vm, QubesVmClasses['QubesHVm']):
+            if hasattr(vm, 'kernel'):
                 # TODO: add a setting for this?
-                if vm.kernel and vm.kernel not in \
-                        os.listdir(system_path['qubes_kernels_base_dir']):
+                if not vm.is_property_default('kernel') and \
+                        vm.kernel not in \
+                        os.listdir(qubes.config.system_path[
+                            'qubes_kernels_base_dir']):
                     if callable(print_callback):
                         print_callback("WARNING: Kernel %s not installed, "
                                        "using default one" % vm.kernel)
-                    vm.uses_default_kernel = True
-                    vm.kernel = host_collection.get_default_kernel()
+                    vm.kernel = qubes.property.DEFAULT
             try:
                 new_vm.clone_attrs(vm)
             except Exception as err:
@@ -2235,7 +2260,7 @@ def backup_restore_do(restore_info,
                 error_callback("*** Some VM property will not be restored")
 
             try:
-                new_vm.appmenus_create(verbose=callable(print_callback))
+                new_vm.appmenus_create()
             except Exception as err:
                 error_callback("ERROR during appmenu restore: {0}".format(err))
                 error_callback(
@@ -2246,22 +2271,19 @@ def backup_restore_do(restore_info,
         vm_name = vm.name
         if 'rename-to' in restore_info[vm.name]:
             vm_name = restore_info[vm.name]['rename-to']
-        host_vm = host_collection.get_vm_by_name(vm_name)
-
-        if host_vm is None:
+        try:
+            host_vm = host_collection.domains[vm_name]
+        except KeyError:
             # Failed/skipped VM
             continue
 
-        if not vm.uses_default_netvm:
+        if not vm.is_property_default('netvm'):
             if restore_info[vm.name]['netvm'] is not None:
-                host_vm.netvm = host_collection.get_vm_by_name(
-                    restore_info[vm.name]['netvm'])
+                host_vm.netvm = restore_info[vm.name]['netvm']
             else:
                 host_vm.netvm = None
 
     host_collection.save()
-    if lock_obtained:
-        host_collection.unlock_db()
 
     if running_backup_operation.canceled:
         if format_version >= 2:
diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py
index 60d54247..3c13f701 100644
--- a/qubes/tests/__init__.py
+++ b/qubes/tests/__init__.py
@@ -40,6 +40,8 @@ import time
 
 import qubes.config
 import qubes.events
+import qubes.backup
+import qubes.exc
 
 XMLPATH = '/var/lib/qubes/qubes-test.xml'
 CLASS_XMLPATH = '/var/lib/qubes/qubes-class-test.xml'
@@ -482,7 +484,7 @@ class SystemTestsMixin(object):
         except (AttributeError, libvirt.libvirtError):
             pass
 
-        del app.domains[vm]
+        del app.domains[vm.qid]
         del vm
 
         app.save()
@@ -686,8 +688,8 @@ class BackupTestsMixin(SystemTestsMixin):
         if self.verbose:
             print >>sys.stderr, "-> Creating %s" % vmname
         testnet = self.app.add_new_vm(qubes.vm.appvm.AppVM,
-            name=vmname, template=template, provides_network=True)
-        testnet.create_on_disk(verbose=self.verbose)
+            name=vmname, template=template, provides_network=True, label='red')
+        testnet.create_on_disk()
         vms.append(testnet)
         self.fill_image(testnet.private_img, 20*1024*1024)
 
@@ -695,10 +697,10 @@ class BackupTestsMixin(SystemTestsMixin):
         if self.verbose:
             print >>sys.stderr, "-> Creating %s" % vmname
         testvm1 = self.app.add_new_vm(qubes.vm.appvm.AppVM,
-            name=vmname, template=template)
+            name=vmname, template=template, label='red')
         testvm1.uses_default_netvm = False
         testvm1.netvm = testnet
-        testvm1.create_on_disk(verbose=self.verbose)
+        testvm1.create_on_disk()
         vms.append(testvm1)
         self.fill_image(testvm1.private_img, 100*1024*1024)
 
@@ -706,8 +708,8 @@ class BackupTestsMixin(SystemTestsMixin):
         if self.verbose:
             print >>sys.stderr, "-> Creating %s" % vmname
         testvm2 = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=vmname,
-            hvm=True)
-        testvm2.create_on_disk(verbose=self.verbose)
+            hvm=True, label='red')
+        testvm2.create_on_disk()
         self.fill_image(testvm2.root_img, 1024*1024*1024, True)
         vms.append(testvm2)
 
@@ -717,33 +719,31 @@ class BackupTestsMixin(SystemTestsMixin):
 
     def make_backup(self, vms, prepare_kwargs=dict(), do_kwargs=dict(),
                     target=None, expect_failure=False):
-        # XXX: bakup_prepare and backup_do don't support host_collection
-        # self.qc.unlock_db()
         if target is None:
             target = self.backupdir
         try:
             files_to_backup = \
-                qubes.backup.backup_prepare(vms,
+                qubes.backup.backup_prepare(self.app, vms,
                                       print_callback=self.print_callback,
                                       **prepare_kwargs)
-        except qubes.qubes.QubesException as e:
+        except qubes.exc.QubesException as e:
             if not expect_failure:
                 self.fail("QubesException during backup_prepare: %s" % str(e))
             else:
                 raise
 
         try:
-            qubes.backup.backup_do(target, files_to_backup, "qubes",
+            qubes.backup.backup_do(self.app, target, files_to_backup, "qubes",
                              progress_callback=self.print_progress,
                              **do_kwargs)
-        except qubes.qubes.QubesException as e:
+        except qubes.exc.QubesException as e:
             if not expect_failure:
                 self.fail("QubesException during backup_do: %s" % str(e))
             else:
                 raise
 
         # FIXME why?
-        self.reload_db()
+        #self.reload_db()
 
     def restore_backup(self, source=None, appvm=None, options=None,
                        expect_errors=None):
@@ -753,7 +753,7 @@ class BackupTestsMixin(SystemTestsMixin):
         else:
             backupfile = source
 
-        with self.assertNotRaises(qubes.qubes.QubesException):
+        with self.assertNotRaises(qubes.exc.QubesException):
             backup_info = qubes.backup.backup_restore_prepare(
                 backupfile, "qubes",
                 host_collection=self.app,
@@ -764,7 +764,7 @@ class BackupTestsMixin(SystemTestsMixin):
         if self.verbose:
             qubes.backup.backup_restore_print_summary(backup_info)
 
-        with self.assertNotRaises(qubes.qubes.QubesException):
+        with self.assertNotRaises(qubes.exc.QubesException):
             qubes.backup.backup_restore_do(
                 backup_info,
                 host_collection=self.app,
@@ -821,7 +821,7 @@ def load_tests(loader, tests, pattern): # pylint: disable=unused-argument
             'qubes.tests.int.dom0_update',
             'qubes.tests.int.network',
 #           'qubes.tests.vm_qrexec_gui',
-#           'qubes.tests.backup',
+            'qubes.tests.int.backup',
 #           'qubes.tests.backupcompatibility',
 #           'qubes.tests.regressions',
 
diff --git a/tests/backup.py b/qubes/tests/int/backup.py
similarity index 83%
rename from tests/backup.py
rename to qubes/tests/int/backup.py
index 72463c91..7f5fd096 100644
--- a/tests/backup.py
+++ b/qubes/tests/int/backup.py
@@ -28,8 +28,11 @@ import os
 
 import unittest
 import sys
-from qubes.qubes import QubesException, QubesTemplateVm
+import qubes
+import qubes.exc
 import qubes.tests
+import qubes.vm.appvm
+import qubes.vm.templatevm
 
 class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
     def test_000_basic_backup(self):
@@ -37,21 +40,24 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
         self.make_backup(vms)
         self.remove_vms(vms)
         self.restore_backup()
-        self.remove_vms(vms)
+        for vm in vms:
+            self.assertIn(vm.name, self.app.domains)
 
     def test_001_compressed_backup(self):
         vms = self.create_backup_vms()
         self.make_backup(vms, do_kwargs={'compressed': True})
         self.remove_vms(vms)
         self.restore_backup()
-        self.remove_vms(vms)
+        for vm in vms:
+            self.assertIn(vm.name, self.app.domains)
 
     def test_002_encrypted_backup(self):
         vms = self.create_backup_vms()
         self.make_backup(vms, do_kwargs={'encrypted': True})
         self.remove_vms(vms)
         self.restore_backup()
-        self.remove_vms(vms)
+        for vm in vms:
+            self.assertIn(vm.name, self.app.domains)
 
     def test_003_compressed_encrypted_backup(self):
         vms = self.create_backup_vms()
@@ -61,7 +67,8 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
                              'encrypted': True})
         self.remove_vms(vms)
         self.restore_backup()
-        self.remove_vms(vms)
+        for vm in vms:
+            self.assertIn(vm.name, self.app.domains)
 
     def test_004_sparse_multipart(self):
         vms = []
@@ -70,29 +77,32 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
         if self.verbose:
             print >>sys.stderr, "-> Creating %s" % vmname
 
-        hvmtemplate = self.qc.add_new_vm("QubesTemplateHVm", name=vmname)
-        hvmtemplate.create_on_disk(verbose=self.verbose)
+        hvmtemplate = self.app.add_new_vm(
+            qubes.vm.templatevm.TemplateVM, name=vmname, hvm=True, label='red')
+        hvmtemplate.create_on_disk()
         self.fill_image(os.path.join(hvmtemplate.dir_path, '00file'),
                         195*1024*1024-4096*3)
         self.fill_image(hvmtemplate.private_img, 195*1024*1024-4096*3)
         self.fill_image(hvmtemplate.root_img, 1024*1024*1024, sparse=True)
         vms.append(hvmtemplate)
-        self.qc.save()
+        self.app.save()
 
         self.make_backup(vms)
         self.remove_vms(vms)
         self.restore_backup()
-        self.remove_vms(vms)
+        for vm in vms:
+            self.assertIn(vm.name, self.app.domains)
 
     def test_005_compressed_custom(self):
         vms = self.create_backup_vms()
         self.make_backup(vms, do_kwargs={'compressed': "bzip2"})
         self.remove_vms(vms)
         self.restore_backup()
-        self.remove_vms(vms)
+        for vm in vms:
+            self.assertIn(vm.name, self.app.domains)
 
     def test_100_backup_dom0_no_restore(self):
-        self.make_backup([self.qc[0]])
+        self.make_backup([self.app.domains[0]])
         # TODO: think of some safe way to test restore...
 
     def test_200_restore_over_existing_directory(self):
@@ -112,7 +122,6 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
                 '*** Directory {} already exists! It has been moved'.format(
                     test_dir)
             ])
-        self.remove_vms(vms)
 
     def test_210_auto_rename(self):
         """
@@ -125,22 +134,23 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
             'rename-conflicting': True
         })
         for vm in vms:
-            self.assertIsNotNone(self.qc.get_vm_by_name(vm.name+'1'))
-            restored_vm = self.qc.get_vm_by_name(vm.name+'1')
-            if vm.netvm and not vm.uses_default_netvm:
-                self.assertEqual(restored_vm.netvm.name, vm.netvm.name+'1')
+            with self.assertNotRaises(qubes.exc.QubesVMNotFoundError):
+                restored_vm = self.app.domains[vm.name + '1']
+            if vm.netvm and not vm.property_is_default('netvm'):
+                self.assertEqual(restored_vm.netvm.name, vm.netvm.name + '1')
+
 
-        self.remove_vms(vms)
 
 class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin):
     def setUp(self):
         super(TC_10_BackupVMMixin, self).setUp()
-        self.backupvm = self.qc.add_new_vm(
-            "QubesAppVm",
+        self.backupvm = self.app.add_new_vm(
+            qubes.vm.appvm.AppVM,
+            label='red',
             name=self.make_vm_name('backupvm'),
-            template=self.qc.get_vm_by_name(self.template)
+            template=self.template
         )
-        self.backupvm.create_on_disk(verbose=self.verbose)
+        self.backupvm.create_on_disk()
 
     def test_100_send_to_vm_file_with_spaces(self):
         vms = self.create_backup_vms()
@@ -159,7 +169,6 @@ class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin):
         backup_path = backup_path.strip()
         self.restore_backup(source=backup_path,
                             appvm=self.backupvm)
-        self.remove_vms(vms)
 
     def test_110_send_to_vm_command(self):
         vms = self.create_backup_vms()
@@ -173,7 +182,6 @@ class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin):
         self.remove_vms(vms)
         self.restore_backup(source='dd if=/var/tmp/backup-test',
                             appvm=self.backupvm)
-        self.remove_vms(vms)
 
     def test_110_send_to_vm_no_space(self):
         """
@@ -192,7 +200,7 @@ class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin):
             user="root", wait=True)
         if retcode != 0:
             raise RuntimeError("Failed to prepare backup directory")
-        with self.assertRaises(QubesException):
+        with self.assertRaises(qubes.exc.QubesException):
             self.make_backup(vms,
                              do_kwargs={
                                  'appvm': self.backupvm,
@@ -200,19 +208,13 @@ class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin):
                                  'encrypted': True},
                              target='/home/user/backup',
                              expect_failure=True)
-        self.qc.lock_db_for_writing()
-        self.qc.load()
-        self.remove_vms(vms)
 
 
 def load_tests(loader, tests, pattern):
     try:
-        qc = qubes.qubes.QubesVmCollection()
-        qc.lock_db_for_reading()
-        qc.load()
-        qc.unlock_db()
-        templates = [vm.name for vm in qc.values() if
-                     isinstance(vm, QubesTemplateVm)]
+        app = qubes.Qubes()
+        templates = [vm.name for vm in app.domains if
+                     isinstance(vm, qubes.vm.templatevm.TemplateVM)]
     except OSError:
         templates = []
     for template in templates:
diff --git a/qubes/utils.py b/qubes/utils.py
index 08646d73..9b9384ef 100644
--- a/qubes/utils.py
+++ b/qubes/utils.py
@@ -90,9 +90,12 @@ def format_doc(docstring):
 # maybe adapt https://code.activestate.com/recipes/578019
 def parse_size(size):
     units = [
-        ('K', 1024), ('KB', 1024),
-        ('M', 1024*1024), ('MB', 1024*1024),
-        ('G', 1024*1024*1024), ('GB', 1024*1024*1024),
+        ('K', 1000), ('KB', 1000),
+        ('M', 1000 * 1000), ('MB', 1000 * 1000),
+        ('G', 1000 * 1000 * 1000), ('GB', 1000 * 1000 * 1000),
+        ('Ki', 1024), ('KiB', 1024),
+        ('Mi', 1024 * 1024), ('MiB', 1024 * 1024),
+        ('Gi', 1024 * 1024 * 1024), ('GiB', 1024 * 1024 * 1024),
     ]
 
     size = size.strip().upper()
@@ -102,10 +105,43 @@ def parse_size(size):
     for unit, multiplier in units:
         if size.endswith(unit):
             size = size[:-len(unit)].strip()
-            return int(size)*multiplier
+            return int(size) * multiplier
 
     raise qubes.exc.QubesException("Invalid size: {0}.".format(size))
 
+def mbytes_to_kmg(size):
+    if size > 1024:
+        return "%d GiB" % (size / 1024)
+    else:
+        return "%d MiB" % size
+
+
+def kbytes_to_kmg(size):
+    if size > 1024:
+        return mbytes_to_kmg(size / 1024)
+    else:
+        return "%d KiB" % size
+
+
+def bytes_to_kmg(size):
+    if size > 1024:
+        return kbytes_to_kmg(size / 1024)
+    else:
+        return "%d B" % size
+
+
+def size_to_human(size):
+    """Humane readable size, with 1/10 precision"""
+    if size < 1024:
+        return str(size)
+    elif size < 1024 * 1024:
+        return str(round(size / 1024.0, 1)) + ' KiB'
+    elif size < 1024 * 1024 * 1024:
+        return str(round(size / (1024.0 * 1024), 1)) + ' MiB'
+    else:
+        return str(round(size / (1024.0 * 1024 * 1024), 1)) + ' GiB'
+
+
 def urandom(size):
     rand = os.urandom(size)
     if rand is None:
diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec
index ffbe42d3..962b10ba 100644
--- a/rpm_spec/core-dom0.spec
+++ b/rpm_spec/core-dom0.spec
@@ -201,6 +201,7 @@ fi
 
 %dir %{python_sitelib}/qubes
 %{python_sitelib}/qubes/__init__.py*
+%{python_sitelib}/qubes/backup.py*
 %{python_sitelib}/qubes/config.py*
 %{python_sitelib}/qubes/devices.py*
 %{python_sitelib}/qubes/dochelpers.py*
@@ -272,6 +273,7 @@ fi
 
 %dir %{python_sitelib}/qubes/tests/int
 %{python_sitelib}/qubes/tests/int/__init__.py*
+%{python_sitelib}/qubes/tests/int/backup.py*
 %{python_sitelib}/qubes/tests/int/basic.py*
 %{python_sitelib}/qubes/tests/int/dom0_update.py*
 %{python_sitelib}/qubes/tests/int/network.py*

From fd1f8def6a5348df8278675d885ea98246401370 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sun, 13 Mar 2016 12:45:48 +0100
Subject: [PATCH 02/32] tests: do not store dom0 home backup inside of dom0
 home itself

---
 qubes/tests/int/backup.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/qubes/tests/int/backup.py b/qubes/tests/int/backup.py
index 7f5fd096..d47492b4 100644
--- a/qubes/tests/int/backup.py
+++ b/qubes/tests/int/backup.py
@@ -92,6 +92,7 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
         self.restore_backup()
         for vm in vms:
             self.assertIn(vm.name, self.app.domains)
+        # TODO check vm.backup_timestamp
 
     def test_005_compressed_custom(self):
         vms = self.create_backup_vms()
@@ -102,6 +103,9 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
             self.assertIn(vm.name, self.app.domains)
 
     def test_100_backup_dom0_no_restore(self):
+        # do not write it into dom0 home itself...
+        os.mkdir('/var/tmp/test-backup')
+        self.backupdir = '/var/tmp/test-backup'
         self.make_backup([self.app.domains[0]])
         # TODO: think of some safe way to test restore...
 

From e557fe9989cc2a91a34ddab04402157297a4ae4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sun, 13 Mar 2016 13:49:42 +0100
Subject: [PATCH 03/32] backup: restructude the code into classes

Introduce two main classes Backup and BackupRestore for storing the
state of the desired operation. Then a simple interface to adjust
parameters.

(Almost) no functional change.

QubesOS/qubes-issues#1213
QubesOS/qubes-issues#1214
---
 qubes/backup.py           | 3114 +++++++++++++++++++------------------
 qubes/tests/__init__.py   |   36 +-
 qubes/tests/int/backup.py |   46 +-
 3 files changed, 1598 insertions(+), 1598 deletions(-)

diff --git a/qubes/backup.py b/qubes/backup.py
index 09ccb11f..b201b7fb 100644
--- a/qubes/backup.py
+++ b/qubes/backup.py
@@ -23,6 +23,7 @@
 #
 #
 from __future__ import unicode_literals
+import itertools
 from qubes.utils import size_to_human
 import sys
 import os
@@ -45,6 +46,7 @@ HEADER_FILENAME = 'backup-header'
 DEFAULT_CRYPTO_ALGORITHM = 'aes-256-cbc'
 DEFAULT_HMAC_ALGORITHM = 'SHA512'
 DEFAULT_COMPRESSION_FILTER = 'gzip'
+# TODO: increase version after finishing implementation
 CURRENT_BACKUP_FORMAT_VERSION = '3'
 # Maximum size of error message get from process stderr (including VM process)
 MAX_STDERR_BYTES = 1024
@@ -53,9 +55,6 @@ HEADER_QUBES_XML_MAX_SIZE = 1024 * 1024
 
 BLKSIZE = 512
 
-# global state for backup_cancel()
-running_backup_operation = None
-
 def print_stdout(text):
     print (text)
 
@@ -83,322 +82,99 @@ def get_disk_usage(path):
 
     return ret
 
-class BackupOperationInfo:
-    def __init__(self):
-        self.canceled = False
-        self.processes_to_kill_on_cancel = []
-        self.tmpdir_to_remove = None
-
-
 class BackupCanceledError(qubes.exc.QubesException):
     def __init__(self, msg, tmpdir=None):
         super(BackupCanceledError, self).__init__(msg)
         self.tmpdir = tmpdir
 
 
-class BackupHeader:
-    version = 'version'
-    encrypted = 'encrypted'
-    compressed = 'compressed'
-    compression_filter = 'compression-filter'
-    crypto_algorithm = 'crypto-algorithm'
-    hmac_algorithm = 'hmac-algorithm'
+class BackupHeader(object):
+    header_keys = {
+        'version': 'version',
+        'encrypted': 'encrypted',
+        'compressed': 'compressed',
+        'compression-filter': 'compression_filter',
+        'crypto-algorithm': 'crypto_algorithm',
+        'hmac-algorithm': 'hmac_algorithm',
+    }
     bool_options = ['encrypted', 'compressed']
     int_options = ['version']
 
+    def __init__(self, header_data = None):
+        # repeat the list to help code completion...
+        self.version = None
+        self.encrypted = None
+        self.compressed = None
+        # Options introduced in backup format 3+, which always have a header,
+        # so no need for fallback in function parameter
+        self.compression_filter = None
+        self.hmac_algorithm = None
+        self.crypto_algorithm = None
 
-def file_to_backup(file_path, subdir=None):
-    sz = get_disk_usage(file_path)
+        if header_data is not None:
+            self.load(header_data)
 
-    if subdir is None:
-        abs_file_path = os.path.abspath(file_path)
-        abs_base_dir = os.path.abspath(qubes.config.system_path["qubes_base_dir"]) + '/'
-        abs_file_dir = os.path.dirname(abs_file_path) + '/'
-        (nothing, directory, subdir) = abs_file_dir.partition(abs_base_dir)
-        assert nothing == ""
-        assert directory == abs_base_dir
-    else:
-        if len(subdir) > 0 and not subdir.endswith('/'):
-            subdir += '/'
-    return [{"path": file_path, "size": sz, "subdir": subdir}]
+    def load(self, untrusted_header_text):
+        """Parse backup header file.
 
-
-def backup_cancel():
-    """
-    Cancel currently running backup/restore operation
-
-    @return: True if any operation was signaled
-    """
-    if running_backup_operation is None:
-        return False
-
-    running_backup_operation.canceled = True
-    for proc in running_backup_operation.processes_to_kill_on_cancel:
+        :param untrusted_header_text: header content
+        :type untrusted_header_text: basestring
+        .. warning::
+            This function may be exposed to not yet verified header,
+            so is security critical.
+        """
         try:
-            proc.terminate()
-        except:
+            untrusted_header_text = untrusted_header_text.decode('ascii')
+        except UnicodeDecodeError:
+            raise qubes.exc.QubesException(
+                "Non-ASCII characters in backup header")
+        for untrusted_line in untrusted_header_text.splitlines():
+            if untrusted_line.count('=') != 1:
+                raise qubes.exc.QubesException("Invalid backup header")
+            (key, value) = untrusted_line.strip().split('=')
+            if key not in self.header_keys.keys():
+                # Ignoring unknown option
+                continue
+            if getattr(self, self.header_keys[key]) is not None:
+                raise qubes.exc.QubesException(
+                    "Duplicated header line: {}".format(key))
+            if key in self.bool_options:
+                value = value.lower() in ["1", "true", "yes"]
+            elif key in self.int_options:
+                value = int(value)
+            setattr(self, self.header_keys[key], value)
+
+        self.validate()
+
+    def validate(self):
+        if self.version == 1:
+            # header not really present
             pass
-    return True
-
-
-def backup_prepare(app, vms_list=None, exclude_list=None,
-                   print_callback=print_stdout, hide_vm_names=True):
-    """
-    If vms = None, include all (sensible) VMs;
-    exclude_list is always applied
-    """
-    # XXX hack for tests, where store filename is qubes-test.xml
-    qubes_xml = app._store
-    if os.path.basename(qubes_xml) != 'qubes.xml':
-        dir = tempfile.mkdtemp()
-        shutil.copy(qubes_xml, os.path.join(dir, 'qubes.xml'))
-        qubes_xml = os.path.join(dir, 'qubes.xml')
-        # FIXME cleanup tempdir later
-
-    files_to_backup = file_to_backup(qubes_xml, '')
-
-    if exclude_list is None:
-        exclude_list = []
-
-    if vms_list is None:
-        all_vms = [vm for vm in app.domains]
-        selected_vms = [vm for vm in all_vms if vm.include_in_backups]
-        appvms_to_backup = [vm for vm in selected_vms if
-                            vm.is_appvm() and not vm.internal]
-        template_vms_worth_backingup = [vm for vm in selected_vms if (
-            vm.is_template() and vm.include_in_backups)]
-        dom0 = [app.domains[0]]
-
-        vms_list = appvms_to_backup + \
-            template_vms_worth_backingup + dom0
-
-    vms_for_backup = vms_list
-    # Apply exclude list
-    if exclude_list:
-        vms_for_backup = [vm for vm in vms_list if vm.name not in exclude_list]
-
-    there_are_running_vms = False
-
-    fields_to_display = [
-        {"name": "VM", "width": 16},
-        {"name": "type", "width": 12},
-        {"name": "size", "width": 12}
-    ]
-
-    # Display the header
-    s = ""
-    for f in fields_to_display:
-        fmt = "{{0:-^{0}}}-+".format(f["width"] + 1)
-        s += fmt.format('-')
-    print_callback(s)
-    s = ""
-    for f in fields_to_display:
-        fmt = "{{0:>{0}}} |".format(f["width"] + 1)
-        s += fmt.format(f["name"])
-    print_callback(s)
-    s = ""
-    for f in fields_to_display:
-        fmt = "{{0:-^{0}}}-+".format(f["width"] + 1)
-        s += fmt.format('-')
-    print_callback(s)
-
-    files_to_backup_index = 0
-    for vm in vms_for_backup:
-        if vm.is_template():
-            # handle templates later
-            continue
-        if vm.qid == 0:
-            # handle dom0 later
-            continue
-
-        if hide_vm_names:
-            subdir = 'vm%d/' % vm.qid
+        elif self.version in [2, 3]:
+            expected_attrs = ['version', 'encrypted', 'compressed',
+                'hmac_algorithm']
+            if self.encrypted:
+                expected_attrs += ['crypto_algorithm']
+            if self.version >= 3 and self.compressed:
+                expected_attrs += ['compression_filter']
+            for key in expected_attrs:
+                if getattr(self, key) is None:
+                    raise qubes.exc.QubesException(
+                        "Backup header lack '{}' info".format(key))
         else:
-            subdir = None
-
-        if vm.private_img is not None:
-            files_to_backup += file_to_backup(vm.private_img, subdir)
-
-        if vm.is_appvm():
-            files_to_backup += file_to_backup(vm.icon_path, subdir)
-        if vm.updateable:
-            if os.path.exists(vm.dir_path + "/apps.templates"):
-                # template
-                files_to_backup += file_to_backup(
-                    vm.dir_path + "/apps.templates", subdir)
-            else:
-                # standaloneVM
-                files_to_backup += file_to_backup(vm.dir_path + "/apps", subdir)
-
-            if os.path.exists(vm.dir_path + "/kernels"):
-                files_to_backup += file_to_backup(vm.dir_path + "/kernels",
-                                                  subdir)
-        if os.path.exists(vm.firewall_conf):
-            files_to_backup += file_to_backup(vm.firewall_conf, subdir)
-        if 'appmenus_whitelist' in qubes.config.vm_files and \
-                os.path.exists(os.path.join(vm.dir_path,
-                                            qubes.config.vm_files[
-                                                'appmenus_whitelist'])):
-            files_to_backup += file_to_backup(
-                os.path.join(vm.dir_path, qubes.config.vm_files[
-                    'appmenus_whitelist']),
-                subdir)
-
-        if vm.updateable:
-            files_to_backup += file_to_backup(vm.root_img, subdir)
-
-        s = ""
-        fmt = "{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1)
-        s += fmt.format(vm.name)
-
-        fmt = "{{0:>{0}}} |".format(fields_to_display[1]["width"] + 1)
-        if vm.is_netvm():
-            s += fmt.format("NetVM" + (" + Sys" if vm.updateable else ""))
-        else:
-            s += fmt.format("AppVM" + (" + Sys" if vm.updateable else ""))
-
-        vm_size = reduce(lambda x, y: x + y["size"],
-                         files_to_backup[files_to_backup_index:],
-                         0)
-        files_to_backup_index = len(files_to_backup)
-
-        fmt = "{{0:>{0}}} |".format(fields_to_display[2]["width"] + 1)
-        s += fmt.format(size_to_human(vm_size))
-
-        if vm.is_running():
-            s += " <-- The VM is running, please shut it down before proceeding " \
-                 "with the backup!"
-            there_are_running_vms = True
-
-        print_callback(s)
-
-    for vm in vms_for_backup:
-        if not vm.is_template():
-            # already handled
-            continue
-        if vm.qid == 0:
-            # handle dom0 later
-            continue
-        vm_sz = vm.get_disk_utilization()
-        if hide_vm_names:
-            template_subdir = 'vm%d/' % vm.qid
-        else:
-            template_subdir = os.path.relpath(
-                vm.dir_path,
-                qubes.config.system_path["qubes_base_dir"]) + '/'
-        template_to_backup = [{"path": vm.dir_path + '/.',
-                               "size": vm_sz,
-                               "subdir": template_subdir}]
-        files_to_backup += template_to_backup
-
-        s = ""
-        fmt = "{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1)
-        s += fmt.format(vm.name)
-
-        fmt = "{{0:>{0}}} |".format(fields_to_display[1]["width"] + 1)
-        s += fmt.format("Template VM")
-
-        fmt = "{{0:>{0}}} |".format(fields_to_display[2]["width"] + 1)
-        s += fmt.format(size_to_human(vm_sz))
-
-        if vm.is_running():
-            s += " <-- The VM is running, please shut it down before proceeding " \
-                 "with the backup!"
-            there_are_running_vms = True
-
-        print_callback(s)
-
-    # Initialize backup flag on all VMs
-    vms_for_backup_qid = [vm.qid for vm in vms_for_backup]
-    for vm in app.domains:
-        vm.backup_content = False
-        if vm.qid == 0:
-            # handle dom0 later
-            continue
-
-        if vm.qid in vms_for_backup_qid:
-            vm.backup_content = True
-            vm.backup_size = vm.get_disk_utilization()
-            if hide_vm_names:
-                vm.backup_path = 'vm%d' % vm.qid
-            else:
-                vm.backup_path = os.path.relpath(
-                    vm.dir_path,
-                    qubes.config.system_path["qubes_base_dir"])
-
-    # Dom0 user home
-    if 0 in vms_for_backup_qid:
-        local_user = grp.getgrnam('qubes').gr_mem[0]
-        home_dir = pwd.getpwnam(local_user).pw_dir
-        # Home dir should have only user-owned files, so fix it now to prevent
-        # permissions problems - some root-owned files can left after
-        # 'sudo bash' and similar commands
-        subprocess.check_call(['sudo', 'chown', '-R', local_user, home_dir])
-
-        home_sz = get_disk_usage(home_dir)
-        home_to_backup = [
-            {"path": home_dir, "size": home_sz, "subdir": 'dom0-home/'}]
-        files_to_backup += home_to_backup
-
-        vm = app.domains[0]
-        vm.backup_content = True
-        vm.backup_size = home_sz
-        vm.backup_path = os.path.join('dom0-home', os.path.basename(home_dir))
-
-        s = ""
-        fmt = "{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1)
-        s += fmt.format('Dom0')
-
-        fmt = "{{0:>{0}}} |".format(fields_to_display[1]["width"] + 1)
-        s += fmt.format("User home")
-
-        fmt = "{{0:>{0}}} |".format(fields_to_display[2]["width"] + 1)
-        s += fmt.format(size_to_human(home_sz))
-
-        print_callback(s)
-
-    app.save()
-    # FIXME: should be after backup completed
-
-    total_backup_sz = 0
-    for f in files_to_backup:
-        total_backup_sz += f["size"]
-
-    s = ""
-    for f in fields_to_display:
-        fmt = "{{0:-^{0}}}-+".format(f["width"] + 1)
-        s += fmt.format('-')
-    print_callback(s)
-
-    s = ""
-    fmt = "{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1)
-    s += fmt.format("Total size:")
-    fmt = "{{0:>{0}}} |".format(
-        fields_to_display[1]["width"] + 1 + 2 + fields_to_display[2][
-            "width"] + 1)
-    s += fmt.format(size_to_human(total_backup_sz))
-    print_callback(s)
-
-    s = ""
-    for f in fields_to_display:
-        fmt = "{{0:-^{0}}}-+".format(f["width"] + 1)
-        s += fmt.format('-')
-    print_callback(s)
-
-    vms_not_for_backup = [vm.name for vm in app.domains
-                          if not vm.backup_content]
-    print_callback("VMs not selected for backup: %s" % " ".join(
-        vms_not_for_backup))
-
-    if there_are_running_vms:
-        raise qubes.exc.QubesException("Please shutdown all VMs before proceeding.")
-
-    for fileinfo in files_to_backup:
-        assert len(fileinfo["subdir"]) == 0 or fileinfo["subdir"][-1] == '/', \
-            "'subdir' must ends with a '/': %s" % unicode(fileinfo)
-
-    return files_to_backup
+            raise qubes.exc.QubesException(
+                "Unsupported backup version {}".format(self.version))
 
+    def save(self, filename):
+        with open(filename, "w") as f:
+            # make sure 'version' is the first key
+            f.write('version={}\n'.format(self.version))
+            for key, attr in self.header_keys.iteritems():
+                if key == 'version':
+                    continue
+                if getattr(self, attr) is None:
+                    continue
+                f.write("{!s}={!s}\n".format(key, getattr(self, attr)))
 
 class SendWorker(Process):
     def __init__(self, queue, base_dir, backup_stdout):
@@ -449,40 +225,332 @@ class SendWorker(Process):
             print "Finished sending thread"
 
 
-def prepare_backup_header(target_directory, passphrase, compressed=False,
-                          encrypted=False,
-                          hmac_algorithm=DEFAULT_HMAC_ALGORITHM,
-                          crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM,
-                          compression_filter=None):
-    header_file_path = os.path.join(target_directory, HEADER_FILENAME)
-    with open(header_file_path, "w") as f:
-        f.write(str("%s=%s\n" % (BackupHeader.version,
-                                 CURRENT_BACKUP_FORMAT_VERSION)))
-        f.write(str("%s=%s\n" % (BackupHeader.hmac_algorithm, hmac_algorithm)))
-        f.write(str("%s=%s\n" % (BackupHeader.crypto_algorithm,
-                                 crypto_algorithm)))
-        f.write(str("%s=%s\n" % (BackupHeader.encrypted, str(encrypted))))
-        f.write(str("%s=%s\n" % (BackupHeader.compressed, str(compressed))))
-        if compressed:
-            f.write(str("%s=%s\n" % (BackupHeader.compression_filter,
-                                     str(compression_filter))))
+class Backup(object):
+    def __init__(self, app, vms_list=None, exclude_list=None, **kwargs):
+        """
+        If vms = None, include all (sensible) VMs;
+        exclude_list is always applied
+        """
+        super(Backup, self).__init__()
 
-    hmac = subprocess.Popen(["openssl", "dgst",
-                             "-" + hmac_algorithm, "-hmac", passphrase],
-                            stdin=open(header_file_path, "r"),
-                            stdout=open(header_file_path + ".hmac", "w"))
-    if hmac.wait() != 0:
-        raise qubes.exc.QubesException("Failed to compute hmac of header file")
-    return HEADER_FILENAME, HEADER_FILENAME + ".hmac"
+        #: progress of the backup - bytes handled of the current VM
+        self.chunk_size = 100 * 1024 * 1024
+        self._current_vm_bytes = 0
+        #: progress of the backup - bytes handled of finished VMs
+        self._done_vms_bytes = 0
+        #: total backup size (set by :py:meth:`get_files_to_backup`)
+        self.total_backup_bytes = 0
+        #: list of PIDs to kill on backup cancel
+        self.processes_to_kill_on_cancel = []
+        self.app = app
+        #: directory for temporary files - set after creating the directory
+        self.tmpdir = None
+
+        # Backup settings - defaults
+        #: should the backup be encrypted?
+        self.encrypted = True
+        #: should the backup be compressed?
+        self.compressed = True
+        #: what passphrase should be used to intergrity protect (and encrypt)
+        #  the backup; required
+        self.passphrase = None
+        #: custom hmac algorithm
+        self.hmac_algorithm = DEFAULT_HMAC_ALGORITHM
+        #: custom encryption algorithm
+        self.crypto_algorithm = DEFAULT_CRYPTO_ALGORITHM
+        #: custom compression filter; a program which process stdin to stdout
+        self.compression_filter = DEFAULT_COMPRESSION_FILTER
+        #: VM to which backup should be sent (if any)
+        self.target_vm = None
+        #: directory to save backup in (either in dom0 or target VM,
+        # depending on :py:attr:`target_vm`
+        self.target_dir = None
+        #: callback for progress reporting. Will be called with one argument
+        # - progress in percents
+        self.progress_callback = None
+
+        for key, value in kwargs.iteritems():
+            if hasattr(self, key):
+                setattr(self, key, value)
+            else:
+                raise AttributeError(key)
+
+        #: whether backup was canceled
+        self.canceled = False
+
+        # FIXME: drop this legacy feature?
+        if isinstance(self.compressed, basestring):
+            self.compression_filter = self.compressed
+            self.compressed = True
+        else:
+            self.compression_filter = DEFAULT_COMPRESSION_FILTER
+
+        if exclude_list is None:
+            exclude_list = []
+
+        if vms_list is None:
+            all_vms = [vm for vm in app.domains]
+            selected_vms = [vm for vm in all_vms if vm.include_in_backups]
+            appvms_to_backup = [vm for vm in selected_vms if
+                                vm.is_appvm() and not vm.internal]
+            template_vms_worth_backingup = [vm for vm in selected_vms if (
+                vm.is_template() and vm.include_in_backups)]
+            dom0 = [app.domains[0]]
+
+            vms_list = appvms_to_backup + \
+                template_vms_worth_backingup + dom0
+
+        self.vms_for_backup = vms_list
+        # Apply exclude list
+        if exclude_list:
+            self.vms_for_backup = [vm for vm in vms_list if vm.name not in
+                                                        exclude_list]
+
+    def __del__(self):
+        if self.tmpdir and os.path.exists(self.tmpdir):
+            shutil.rmtree(self.tmpdir)
+
+    def cancel(self):
+        """Cancel running backup operation. Can be called from another thread.
+        """
+        self.canceled = True
+        for proc in self.processes_to_kill_on_cancel:
+            try:
+                proc.terminate()
+            except:
+                pass
+
+    @staticmethod
+    def _file_to_backup(file_path, subdir=None):
+        sz = get_disk_usage(file_path)
+
+        if subdir is None:
+            abs_file_path = os.path.abspath(file_path)
+            abs_base_dir = os.path.abspath(qubes.config.system_path["qubes_base_dir"]) + '/'
+            abs_file_dir = os.path.dirname(abs_file_path) + '/'
+            (nothing, directory, subdir) = abs_file_dir.partition(abs_base_dir)
+            assert nothing == ""
+            assert directory == abs_base_dir
+        else:
+            if len(subdir) > 0 and not subdir.endswith('/'):
+                subdir += '/'
+        return [{"path": file_path, "size": sz, "subdir": subdir}]
 
 
-def backup_do(app, base_backup_dir, files_to_backup, passphrase,
-              progress_callback=None, encrypted=False, appvm=None,
-              compressed=False, hmac_algorithm=DEFAULT_HMAC_ALGORITHM,
-              crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM):
-    global running_backup_operation
+    def get_files_to_backup(self):
+        files_to_backup = {}
+        total_backup_sz = 0
+        for vm in self.vms_for_backup:
+            if vm.is_template():
+                # handle templates later
+                continue
+            if vm.qid == 0:
+                # handle dom0 later
+                continue
 
-    def queue_put_with_check(proc, vmproc, queue, element):
+            if self.encrypted:
+                subdir = 'vm%d/' % vm.qid
+            else:
+                subdir = None
+
+            vm_files = []
+            if vm.private_img is not None:
+                vm_files += self._file_to_backup(vm.private_img, subdir)
+
+            # TODO: don't backup the icon
+            if vm.is_appvm():
+                vm_files += self._file_to_backup(
+                    vm.icon_path, subdir)
+            if vm.updateable:
+                if os.path.exists(vm.dir_path + "/apps.templates"):
+                    # template
+                    vm_files += self._file_to_backup(
+                        vm.dir_path + "/apps.templates", subdir)
+                else:
+                    # standaloneVM
+                    vm_files += self._file_to_backup(
+                        vm.dir_path + "/apps", subdir)
+
+                if os.path.exists(vm.dir_path + "/kernels"):
+                    vm_files += self._file_to_backup(
+                        vm.dir_path + "/kernels", subdir)
+            # TODO: drop after merging firewall.xml into qubes.xml
+            firewall_conf = os.path.join(vm.dir_path, vm.firewall_conf)
+            if os.path.exists(firewall_conf):
+                vm_files += self._file_to_backup(
+                    firewall_conf, subdir)
+            if 'appmenus_whitelist' in qubes.config.vm_files and \
+                    os.path.exists(os.path.join(vm.dir_path,
+                                                qubes.config.vm_files[
+                                                    'appmenus_whitelist'])):
+                vm_files += self._file_to_backup(
+                    os.path.join(vm.dir_path, qubes.config.vm_files[
+                        'appmenus_whitelist']),
+                    subdir)
+
+            if vm.updateable:
+                vm_files += self._file_to_backup(vm.root_img, subdir)
+            vm_size = reduce(lambda x, y: x + y['size'], vm_files, 0)
+            files_to_backup[vm.qid] = {
+                'vm': vm,
+                'files': vm_files,
+                'subdir': subdir,
+                'size': vm_size,
+            }
+            total_backup_sz += vm_size
+
+        for vm in self.vms_for_backup:
+            if not vm.is_template():
+                # already handled
+                continue
+            if vm.qid == 0:
+                # handle dom0 later
+                continue
+            vm_sz = vm.get_disk_utilization()
+            if self.encrypted:
+                template_subdir = 'vm%d/' % vm.qid
+            else:
+                template_subdir = os.path.relpath(
+                    vm.dir_path,
+                    qubes.config.system_path["qubes_base_dir"]) + '/'
+            template_to_backup = [{"path": vm.dir_path + '/.',
+                                   "size": vm_sz,
+                                   "subdir": template_subdir}]
+            vm_files = template_to_backup
+            vm_size = reduce(lambda x, y: x + y['size'], vm_files, 0)
+            files_to_backup[vm.qid] = {
+                'vm': vm,
+                'files': vm_files,
+                'subdir': template_subdir,
+                'size': vm_size,
+            }
+            total_backup_sz += vm_size
+
+        # Dom0 user home
+        if 0 in [vm.qid for vm in self.vms_for_backup]:
+            local_user = grp.getgrnam('qubes').gr_mem[0]
+            home_dir = pwd.getpwnam(local_user).pw_dir
+            # Home dir should have only user-owned files, so fix it now to prevent
+            # permissions problems - some root-owned files can left after
+            # 'sudo bash' and similar commands
+            subprocess.check_call(['sudo', 'chown', '-R', local_user, home_dir])
+
+            home_sz = get_disk_usage(home_dir)
+            home_to_backup = [
+                {"path": home_dir, "size": home_sz, "subdir": 'dom0-home/'}]
+            vm_files = home_to_backup
+
+            files_to_backup[0] = {
+                'vm': self.app.domains[0],
+                'files': vm_files,
+                'subdir': os.path.join('dom0-home', os.path.basename(home_dir)),
+                'size': home_sz,
+            }
+            total_backup_sz += home_sz
+
+        self.total_backup_bytes = total_backup_sz
+        return files_to_backup
+
+
+    def get_backup_summary(self):
+        summary = ""
+
+        fields_to_display = [
+            {"name": "VM", "width": 16},
+            {"name": "type", "width": 12},
+            {"name": "size", "width": 12}
+        ]
+
+        # Display the header
+        for f in fields_to_display:
+            fmt = "{{0:-^{0}}}-+".format(f["width"] + 1)
+            summary += fmt.format('-')
+        summary += "\n"
+        for f in fields_to_display:
+            fmt = "{{0:>{0}}} |".format(f["width"] + 1)
+            summary += fmt.format(f["name"])
+        summary += "\n"
+        for f in fields_to_display:
+            fmt = "{{0:-^{0}}}-+".format(f["width"] + 1)
+            summary += fmt.format('-')
+        summary += "\n"
+
+        files_to_backup = self.get_files_to_backup()
+
+        for qid, vm_info in files_to_backup.iteritems():
+            s = ""
+            fmt = "{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1)
+            s += fmt.format(vm_info['vm'].name)
+
+            fmt = "{{0:>{0}}} |".format(fields_to_display[1]["width"] + 1)
+            if qid == 0:
+                s += fmt.format("User home")
+            elif vm_info['vm'].is_template():
+                s += fmt.format("Template VM")
+            else:
+                s += fmt.format("VM" + (" + Sys" if vm_info['vm'].updateable else ""))
+
+            vm_size = vm_info['size']
+
+            fmt = "{{0:>{0}}} |".format(fields_to_display[2]["width"] + 1)
+            s += fmt.format(size_to_human(vm_size))
+
+            if qid != 0 and vm_info['vm'].is_running():
+                s += " <-- The VM is running, please shut it down before proceeding " \
+                     "with the backup!"
+
+            summary += s + "\n"
+
+        for f in fields_to_display:
+            fmt = "{{0:-^{0}}}-+".format(f["width"] + 1)
+            summary += fmt.format('-')
+        summary += "\n"
+
+        fmt = "{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1)
+        summary += fmt.format("Total size:")
+        fmt = "{{0:>{0}}} |".format(
+            fields_to_display[1]["width"] + 1 + 2 + fields_to_display[2][
+                "width"] + 1)
+        summary += fmt.format(size_to_human(self.total_backup_bytes))
+        summary += "\n"
+
+        for f in fields_to_display:
+            fmt = "{{0:-^{0}}}-+".format(f["width"] + 1)
+            summary += fmt.format('-')
+        summary += "\n"
+
+        vms_not_for_backup = [vm.name for vm in self.app.domains
+                              if vm not in self.vms_for_backup]
+        summary += "VMs not selected for backup:\n - " + "\n - ".join(
+            vms_not_for_backup)
+
+        return summary
+
+    def prepare_backup_header(self):
+        header_file_path = os.path.join(self.tmpdir, HEADER_FILENAME)
+        backup_header = BackupHeader()
+        backup_header.version = CURRENT_BACKUP_FORMAT_VERSION
+        backup_header.hmac_algorithm = self.hmac_algorithm
+        backup_header.crypto_algorithm = self.crypto_algorithm
+        backup_header.encrypted = self.encrypted
+        backup_header.compressed = self.compressed
+        if self.compressed:
+            backup_header.compression_filter = self.compression_filter
+        backup_header.save(header_file_path)
+
+        hmac = subprocess.Popen(
+            ["openssl", "dgst", "-" + self.hmac_algorithm,
+                "-hmac", self.passphrase],
+            stdin=open(header_file_path, "r"),
+            stdout=open(header_file_path + ".hmac", "w"))
+        if hmac.wait() != 0:
+            raise qubes.exc.QubesException(
+                "Failed to compute hmac of header file")
+        return HEADER_FILENAME, HEADER_FILENAME + ".hmac"
+
+
+    @staticmethod
+    def _queue_put_with_check(proc, vmproc, queue, element):
         if queue.full():
             if not proc.is_alive():
                 if vmproc:
@@ -493,270 +561,286 @@ def backup_do(app, base_backup_dir, files_to_backup, passphrase,
                 raise qubes.exc.QubesException(message)
         queue.put(element)
 
-    total_backup_sz = 0
-    passphrase = passphrase.encode('utf-8')
-    for f in files_to_backup:
-        total_backup_sz += f["size"]
+    def _send_progress_update(self):
+        if callable(self.progress_callback):
+            progress = (
+                100 * (self._done_vms_bytes + self._current_vm_bytes) /
+                self.total_backup_bytes)
+            self.progress_callback(progress)
 
-    if isinstance(compressed, str):
-        compression_filter = compressed
-    else:
-        compression_filter = DEFAULT_COMPRESSION_FILTER
+    def _add_vm_progress(self, bytes_done):
+        self._current_vm_bytes += bytes_done
+        self._send_progress_update()
 
-    running_backup_operation = BackupOperationInfo()
-    vmproc = None
-    tar_sparse = None
-    if appvm is not None:
-        # Prepare the backup target (Qubes service call)
-        backup_target = "QUBESRPC qubes.Backup dom0"
+    def backup_do(self):
+        if self.passphrase is None:
+            raise qubes.exc.QubesException("No passphrase set")
+        qubes_xml = self.app._store
+        self.tmpdir = tempfile.mkdtemp()
+        shutil.copy(qubes_xml, os.path.join(self.tmpdir, 'qubes.xml'))
+        qubes_xml = os.path.join(self.tmpdir, 'qubes.xml')
+        backup_app = qubes.Qubes(qubes_xml)
 
-        # If APPVM, STDOUT is a PIPE
-        vmproc = appvm.run(command=backup_target, passio_popen=True,
-                           passio_stderr=True)
-        vmproc.stdin.write(base_backup_dir.
-                           replace("\r", "").replace("\n", "") + "\n")
-        backup_stdout = vmproc.stdin
-        running_backup_operation.processes_to_kill_on_cancel.append(vmproc)
-    else:
-        # Prepare the backup target (local file)
-        if os.path.isdir(base_backup_dir):
-            backup_target = base_backup_dir + "/qubes-{0}". \
-                format(time.strftime("%Y-%m-%dT%H%M%S"))
+        # FIXME: cache it earlier?
+        files_to_backup = self.get_files_to_backup()
+        # make sure backup_content isn't set initially
+        for vm in backup_app.domains:
+            vm.backup_content = qubes.property.DEFAULT
+
+        for qid, vm_info in files_to_backup.iteritems():
+            if qid != 0 and vm_info['vm'].is_running():
+                raise qubes.exc.QubesVMNotHaltedError(vm_info['vm'])
+            # VM is included in the backup
+            backup_app.domains[qid].backup_content = True
+            backup_app.domains[qid].backup_path = vm_info['subdir']
+            backup_app.domains[qid].backup_size = vm_info['size']
+        backup_app.save()
+
+        passphrase = self.passphrase.encode('utf-8')
+
+        vmproc = None
+        tar_sparse = None
+        if self.target_vm is not None:
+            # Prepare the backup target (Qubes service call)
+            backup_target = "QUBESRPC qubes.Backup dom0"
+
+            # If APPVM, STDOUT is a PIPE
+            vmproc = self.target_vm.run(
+                command=backup_target, passio_popen=True, passio_stderr=True)
+            vmproc.stdin.write(self.target_dir.
+                               replace("\r", "").replace("\n", "") + "\n")
+            backup_stdout = vmproc.stdin
+            self.processes_to_kill_on_cancel.append(vmproc)
         else:
-            backup_target = base_backup_dir
-
-            # Create the target directory
-            if not os.path.exists(os.path.dirname(base_backup_dir)):
-                raise qubes.exc.QubesException(
-                    "ERROR: the backup directory for {0} does not exists".
-                    format(base_backup_dir))
-
-        # If not APPVM, STDOUT is a local file
-        backup_stdout = open(backup_target, 'wb')
-
-    global blocks_backedup
-    blocks_backedup = 0
-    if callable(progress_callback):
-        progress = blocks_backedup * 11 / total_backup_sz
-        progress_callback(progress)
-
-    backup_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/backup_")
-    running_backup_operation.tmpdir_to_remove = backup_tmpdir
-
-    # Tar with tape length does not deals well with stdout (close stdout between
-    # two tapes)
-    # For this reason, we will use named pipes instead
-    if BACKUP_DEBUG:
-        print "Working in", backup_tmpdir
-
-    backup_pipe = os.path.join(backup_tmpdir, "backup_pipe")
-    if BACKUP_DEBUG:
-        print "Creating pipe in:", backup_pipe
-    os.mkfifo(backup_pipe)
-
-    if BACKUP_DEBUG:
-        print "Will backup:", files_to_backup
-
-    header_files = prepare_backup_header(backup_tmpdir, passphrase,
-                                         compressed=bool(compressed),
-                                         encrypted=encrypted,
-                                         hmac_algorithm=hmac_algorithm,
-                                         crypto_algorithm=crypto_algorithm,
-                                         compression_filter=compression_filter)
-
-    # Setup worker to send encrypted data chunks to the backup_target
-    def compute_progress(new_size, total_backup_size):
-        global blocks_backedup
-        blocks_backedup += new_size
-        if callable(progress_callback):
-            this_progress = blocks_backedup / float(total_backup_size)
-            progress_callback(int(round(this_progress * 100, 2)))
-
-    to_send = Queue(10)
-    send_proc = SendWorker(to_send, backup_tmpdir, backup_stdout)
-    send_proc.start()
-
-    for f in header_files:
-        to_send.put(f)
-
-    for filename in files_to_backup:
-        if BACKUP_DEBUG:
-            print "Backing up", filename
-
-        backup_tempfile = os.path.join(backup_tmpdir,
-                                       filename["subdir"],
-                                       os.path.basename(filename["path"]))
-        if BACKUP_DEBUG:
-            print "Using temporary location:", backup_tempfile
-
-        # Ensure the temporary directory exists
-        if not os.path.isdir(os.path.dirname(backup_tempfile)):
-            os.makedirs(os.path.dirname(backup_tempfile))
-
-        # The first tar cmd can use any complex feature as we want. Files will
-        # be verified before untaring this.
-        # Prefix the path in archive with filename["subdir"] to have it
-        # verified during untar
-        tar_cmdline = (["tar", "-Pc", '--sparse',
-                       "-f", backup_pipe,
-                       '-C', os.path.dirname(filename["path"])] +
-                       (['--dereference'] if filename["subdir"] != "dom0-home/"
-                       else []) +
-                       ['--xform', 's:^%s:%s\\0:' % (
-                           os.path.basename(filename["path"]),
-                           filename["subdir"]),
-                       os.path.basename(filename["path"])
-                       ])
-        if compressed:
-            tar_cmdline.insert(-1,
-                               "--use-compress-program=%s" % compression_filter)
-
-        if BACKUP_DEBUG:
-            print " ".join(tar_cmdline)
-
-        # Tips: Popen(bufsize=0)
-        # Pipe: tar-sparse | encryptor [| hmac] | tar | backup_target
-        # Pipe: tar-sparse [| hmac] | tar | backup_target
-        tar_sparse = subprocess.Popen(tar_cmdline, stdin=subprocess.PIPE,
-                                      stderr=(open(os.devnull, 'w')
-                                              if not BACKUP_DEBUG
-                                              else None))
-        running_backup_operation.processes_to_kill_on_cancel.append(tar_sparse)
-
-        # Wait for compressor (tar) process to finish or for any error of other
-        # subprocesses
-        i = 0
-        run_error = "paused"
-        encryptor = None
-        if encrypted:
-            # Start encrypt
-            # If no cipher is provided, the data is forwarded unencrypted !!!
-            encryptor = subprocess.Popen(["openssl", "enc",
-                                          "-e", "-" + crypto_algorithm,
-                                          "-pass", "pass:" + passphrase],
-                                         stdin=open(backup_pipe, 'rb'),
-                                         stdout=subprocess.PIPE)
-            pipe = encryptor.stdout
-        else:
-            pipe = open(backup_pipe, 'rb')
-        while run_error == "paused":
-
-            # Start HMAC
-            hmac = subprocess.Popen(["openssl", "dgst",
-                                     "-" + hmac_algorithm, "-hmac", passphrase],
-                                    stdin=subprocess.PIPE,
-                                    stdout=subprocess.PIPE)
-
-            # Prepare a first chunk
-            chunkfile = backup_tempfile + "." + "%03d" % i
-            i += 1
-            chunkfile_p = open(chunkfile, 'wb')
-
-            common_args = {
-                'backup_target': chunkfile_p,
-                'total_backup_sz': total_backup_sz,
-                'hmac': hmac,
-                'vmproc': vmproc,
-                'addproc': tar_sparse,
-                'progress_callback': compute_progress,
-                'size_limit': 100 * 1024 * 1024,
-            }
-            run_error = wait_backup_feedback(
-                in_stream=pipe, streamproc=encryptor,
-                **common_args)
-            chunkfile_p.close()
-
-            if BACKUP_DEBUG:
-                print "Wait_backup_feedback returned:", run_error
-
-            if running_backup_operation.canceled:
-                try:
-                    tar_sparse.terminate()
-                except:
-                    pass
-                try:
-                    hmac.terminate()
-                except:
-                    pass
-                tar_sparse.wait()
-                hmac.wait()
-                to_send.put("ERROR")
-                send_proc.join()
-                shutil.rmtree(backup_tmpdir)
-                running_backup_operation = None
-                raise BackupCanceledError("Backup canceled")
-            if run_error and run_error != "size_limit":
-                send_proc.terminate()
-                if run_error == "VM" and vmproc:
-                    raise qubes.exc.QubesException(
-                        "Failed to write the backup, VM output:\n" +
-                        vmproc.stderr.read(MAX_STDERR_BYTES))
-                else:
-                    raise qubes.exc.QubesException("Failed to perform backup: error in " +
-                                         run_error)
-
-            # Send the chunk to the backup target
-            queue_put_with_check(
-                send_proc, vmproc, to_send,
-                os.path.relpath(chunkfile, backup_tmpdir))
-
-            # Close HMAC
-            hmac.stdin.close()
-            hmac.wait()
-            if BACKUP_DEBUG:
-                print "HMAC proc return code:", hmac.poll()
-
-            # Write HMAC data next to the chunk file
-            hmac_data = hmac.stdout.read()
-            if BACKUP_DEBUG:
-                print "Writing hmac to", chunkfile + ".hmac"
-            hmac_file = open(chunkfile + ".hmac", 'w')
-            hmac_file.write(hmac_data)
-            hmac_file.flush()
-            hmac_file.close()
-
-            # Send the HMAC to the backup target
-            queue_put_with_check(
-                send_proc, vmproc, to_send,
-                os.path.relpath(chunkfile, backup_tmpdir) + ".hmac")
-
-            if tar_sparse.poll() is None or run_error == "size_limit":
-                run_error = "paused"
+            # Prepare the backup target (local file)
+            if os.path.isdir(self.target_dir):
+                backup_target = self.target_dir + "/qubes-{0}". \
+                    format(time.strftime("%Y-%m-%dT%H%M%S"))
             else:
-                running_backup_operation.processes_to_kill_on_cancel.remove(
-                    tar_sparse)
-                if BACKUP_DEBUG:
-                    print "Finished tar sparse with exit code", tar_sparse \
-                        .poll()
-        pipe.close()
+                backup_target = self.target_dir
 
-    queue_put_with_check(send_proc, vmproc, to_send, "FINISHED")
-    send_proc.join()
-    shutil.rmtree(backup_tmpdir)
+                # Create the target directory
+                if not os.path.exists(os.path.dirname(self.target_dir)):
+                    raise qubes.exc.QubesException(
+                        "ERROR: the backup directory for {0} does not exists".
+                        format(self.target_dir))
 
-    if running_backup_operation.canceled:
-        running_backup_operation = None
-        raise BackupCanceledError("Backup canceled")
+            # If not APPVM, STDOUT is a local file
+            backup_stdout = open(backup_target, 'wb')
 
-    running_backup_operation = None
-
-    if send_proc.exitcode != 0:
-        raise qubes.exc.QubesException(
-            "Failed to send backup: error in the sending process")
-
-    if vmproc:
+        # Tar with tape length does not deals well with stdout (close stdout between
+        # two tapes)
+        # For this reason, we will use named pipes instead
         if BACKUP_DEBUG:
-            print "VMProc1 proc return code:", vmproc.poll()
-            if tar_sparse is not None:
-                print "Sparse1 proc return code:", tar_sparse.poll()
-        vmproc.stdin.close()
+            print "Working in", self.tmpdir
 
-    # Save date of last backup
-    for vm in app.domains:
-        if vm.backup_content:
-            vm.backup_timestamp = datetime.datetime.now()
+        backup_pipe = os.path.join(self.tmpdir, "backup_pipe")
+        if BACKUP_DEBUG:
+            print "Creating pipe in:", backup_pipe
+        os.mkfifo(backup_pipe)
 
-    app.save()
+        if BACKUP_DEBUG:
+            print "Will backup:", files_to_backup
+
+        header_files = self.prepare_backup_header()
+
+        # Setup worker to send encrypted data chunks to the backup_target
+        to_send = Queue(10)
+        send_proc = SendWorker(to_send, self.tmpdir, backup_stdout)
+        send_proc.start()
+
+        for f in header_files:
+            to_send.put(f)
+
+        vm_files_to_backup = self.get_files_to_backup()
+        qubes_xml_info = {
+            'files': self._file_to_backup(qubes_xml, ''),
+            'vm': None,
+            'size': 0,
+            'subdir': '',
+        }
+        for vm_info in itertools.chain([qubes_xml_info],
+                vm_files_to_backup.itervalues()):
+            for file_info in vm_info['files']:
+
+                if BACKUP_DEBUG:
+                    print "Backing up", file_info
+
+                backup_tempfile = os.path.join(self.tmpdir,
+                                               file_info["subdir"],
+                                               os.path.basename(file_info["path"]))
+                if BACKUP_DEBUG:
+                    print "Using temporary location:", backup_tempfile
+
+                # Ensure the temporary directory exists
+                if not os.path.isdir(os.path.dirname(backup_tempfile)):
+                    os.makedirs(os.path.dirname(backup_tempfile))
+
+                # The first tar cmd can use any complex feature as we want. Files will
+                # be verified before untaring this.
+                # Prefix the path in archive with filename["subdir"] to have it
+                # verified during untar
+                tar_cmdline = (["tar", "-Pc", '--sparse',
+                               "-f", backup_pipe,
+                               '-C', os.path.dirname(file_info["path"])] +
+                               (['--dereference'] if file_info["subdir"] != "dom0-home/"
+                               else []) +
+                               ['--xform', 's:^%s:%s\\0:' % (
+                                   os.path.basename(file_info["path"]),
+                                   file_info["subdir"]),
+                               os.path.basename(file_info["path"])
+                               ])
+                if self.compressed:
+                    tar_cmdline.insert(-1,
+                        "--use-compress-program=%s" % self.compression_filter)
+
+                if BACKUP_DEBUG:
+                    print " ".join(tar_cmdline)
+
+                # Tips: Popen(bufsize=0)
+                # Pipe: tar-sparse | encryptor [| hmac] | tar | backup_target
+                # Pipe: tar-sparse [| hmac] | tar | backup_target
+                tar_sparse = subprocess.Popen(tar_cmdline, stdin=subprocess.PIPE,
+                                              stderr=(open(os.devnull, 'w')
+                                                      if not BACKUP_DEBUG
+                                                      else None))
+                self.processes_to_kill_on_cancel.append(tar_sparse)
+
+                # Wait for compressor (tar) process to finish or for any error of other
+                # subprocesses
+                i = 0
+                run_error = "paused"
+                encryptor = None
+                if self.encrypted:
+                    # Start encrypt
+                    # If no cipher is provided, the data is forwarded unencrypted !!!
+                    encryptor = subprocess.Popen([
+                        "openssl", "enc",
+                        "-e", "-" + self.crypto_algorithm,
+                        "-pass", "pass:" + passphrase],
+                        stdin=open(backup_pipe, 'rb'),
+                        stdout=subprocess.PIPE)
+                    pipe = encryptor.stdout
+                else:
+                    pipe = open(backup_pipe, 'rb')
+                while run_error == "paused":
+
+                    # Start HMAC
+                    hmac = subprocess.Popen([
+                        "openssl", "dgst", "-" + self.hmac_algorithm,
+                        "-hmac", passphrase],
+                        stdin=subprocess.PIPE,
+                        stdout=subprocess.PIPE)
+
+                    # Prepare a first chunk
+                    chunkfile = backup_tempfile + "." + "%03d" % i
+                    i += 1
+                    chunkfile_p = open(chunkfile, 'wb')
+
+                    common_args = {
+                        'backup_target': chunkfile_p,
+                        'hmac': hmac,
+                        'vmproc': vmproc,
+                        'addproc': tar_sparse,
+                        'progress_callback': self._add_vm_progress,
+                        'size_limit': self.chunk_size,
+                    }
+                    run_error = wait_backup_feedback(
+                        in_stream=pipe, streamproc=encryptor,
+                        **common_args)
+                    chunkfile_p.close()
+
+                    if BACKUP_DEBUG:
+                        print "Wait_backup_feedback returned:", run_error
+
+                    if self.canceled:
+                        try:
+                            tar_sparse.terminate()
+                        except:
+                            pass
+                        try:
+                            hmac.terminate()
+                        except:
+                            pass
+                        tar_sparse.wait()
+                        hmac.wait()
+                        to_send.put("ERROR")
+                        send_proc.join()
+                        shutil.rmtree(self.tmpdir)
+                        raise BackupCanceledError("Backup canceled")
+                    if run_error and run_error != "size_limit":
+                        send_proc.terminate()
+                        if run_error == "VM" and vmproc:
+                            raise qubes.exc.QubesException(
+                                "Failed to write the backup, VM output:\n" +
+                                vmproc.stderr.read(MAX_STDERR_BYTES))
+                        else:
+                            raise qubes.exc.QubesException("Failed to perform backup: error in " +
+                                                 run_error)
+
+                    # Send the chunk to the backup target
+                    self._queue_put_with_check(
+                        send_proc, vmproc, to_send,
+                        os.path.relpath(chunkfile, self.tmpdir))
+
+                    # Close HMAC
+                    hmac.stdin.close()
+                    hmac.wait()
+                    if BACKUP_DEBUG:
+                        print "HMAC proc return code:", hmac.poll()
+
+                    # Write HMAC data next to the chunk file
+                    hmac_data = hmac.stdout.read()
+                    if BACKUP_DEBUG:
+                        print "Writing hmac to", chunkfile + ".hmac"
+                    hmac_file = open(chunkfile + ".hmac", 'w')
+                    hmac_file.write(hmac_data)
+                    hmac_file.flush()
+                    hmac_file.close()
+
+                    # Send the HMAC to the backup target
+                    self._queue_put_with_check(
+                        send_proc, vmproc, to_send,
+                        os.path.relpath(chunkfile, self.tmpdir) + ".hmac")
+
+                    if tar_sparse.poll() is None or run_error == "size_limit":
+                        run_error = "paused"
+                    else:
+                        self.processes_to_kill_on_cancel.remove(tar_sparse)
+                        if BACKUP_DEBUG:
+                            print "Finished tar sparse with exit code", tar_sparse \
+                                .poll()
+                pipe.close()
+
+            # This VM done, update progress
+            self._done_vms_bytes += vm_info['size']
+            self._current_vm_bytes = 0
+            self._send_progress_update()
+
+        self._queue_put_with_check(send_proc, vmproc, to_send, "FINISHED")
+        send_proc.join()
+        shutil.rmtree(self.tmpdir)
+
+        if self.canceled:
+            raise BackupCanceledError("Backup canceled")
+
+        if send_proc.exitcode != 0:
+            raise qubes.exc.QubesException(
+                "Failed to send backup: error in the sending process")
+
+        if vmproc:
+            if BACKUP_DEBUG:
+                print "VMProc1 proc return code:", vmproc.poll()
+                if tar_sparse is not None:
+                    print "Sparse1 proc return code:", tar_sparse.poll()
+            vmproc.stdin.close()
+
+        # Save date of last backup
+        for vm in self.app.domains:
+            if vm.backup_content:
+                vm.backup_timestamp = datetime.datetime.now()
+
+        self.app.save()
 
 
 '''
@@ -777,7 +861,7 @@ def backup_do(app, base_backup_dir, files_to_backup, passphrase,
 
 
 def wait_backup_feedback(progress_callback, in_stream, streamproc,
-                         backup_target, total_backup_sz, hmac=None, vmproc=None,
+                         backup_target, hmac=None, vmproc=None,
                          addproc=None,
                          size_limit=None):
     buffer_size = 409600
@@ -790,7 +874,8 @@ def wait_backup_feedback(progress_callback, in_stream, streamproc,
         if size_limit and bytes_copied + buffer_size > size_limit:
             return "size_limit"
         buf = in_stream.read(buffer_size)
-        progress_callback(len(buf), total_backup_sz)
+        if callable(progress_callback):
+            progress_callback(len(buf))
         bytes_copied += len(buf)
 
         run_count = 0
@@ -848,46 +933,8 @@ def wait_backup_feedback(progress_callback, in_stream, streamproc,
 
     return run_error
 
-
-def verify_hmac(filename, hmacfile, passphrase, algorithm):
-    if BACKUP_DEBUG:
-        print "Verifying file " + filename
-
-    if hmacfile != filename + ".hmac":
-        raise qubes.exc.QubesException(
-            "ERROR: expected hmac for {}, but got {}".
-            format(filename, hmacfile))
-
-    hmac_proc = subprocess.Popen(["openssl", "dgst", "-" + algorithm,
-                                  "-hmac", passphrase],
-                                 stdin=open(filename, 'rb'),
-                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    hmac_stdout, hmac_stderr = hmac_proc.communicate()
-
-    if len(hmac_stderr) > 0:
-        raise qubes.exc.QubesException(
-            "ERROR: verify file {0}: {1}".format(filename, hmac_stderr))
-    else:
-        if BACKUP_DEBUG:
-            print "Loading hmac for file " + filename
-        hmac = load_hmac(open(hmacfile, 'r').read())
-
-        if len(hmac) > 0 and load_hmac(hmac_stdout) == hmac:
-            os.unlink(hmacfile)
-            if BACKUP_DEBUG:
-                print "File verification OK -> Sending file " + filename
-            return True
-        else:
-            raise qubes.exc.QubesException(
-                "ERROR: invalid hmac for file {0}: {1}. "
-                "Is the passphrase correct?".
-                format(filename, load_hmac(hmac_stdout)))
-    # Not reachable
-    return False
-
-
 class ExtractWorker2(Process):
-    def __init__(self, queue, base_dir, passphrase, encrypted, total_size,
+    def __init__(self, queue, base_dir, passphrase, encrypted,
                  print_callback, error_callback, progress_callback, vmproc=None,
                  compressed=False, crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM,
                  verify_only=False):
@@ -899,7 +946,6 @@ class ExtractWorker2(Process):
         self.compressed = compressed
         self.crypto_algorithm = crypto_algorithm
         self.verify_only = verify_only
-        self.total_size = total_size
         self.blocks_backedup = 0
         self.tar2_process = None
         self.tar2_current_file = None
@@ -919,13 +965,6 @@ class ExtractWorker2(Process):
 
         self.stderr_encoding = sys.stderr.encoding or 'utf-8'
 
-    def compute_progress(self, new_size, _):
-        if self.progress_callback:
-            self.blocks_backedup += new_size
-            progress = self.blocks_backedup / float(self.total_size)
-            progress = int(round(progress * 100, 2))
-            self.progress_callback(progress)
-
     def collect_tar_output(self):
         if not self.tar2_process.stderr:
             return
@@ -1031,7 +1070,6 @@ class ExtractWorker2(Process):
             pipe = open(self.restore_pipe, 'wb')
             common_args = {
                 'backup_target': pipe,
-                'total_backup_sz': self.total_size,
                 'hmac': None,
                 'vmproc': self.vmproc,
                 'addproc': self.tar2_process
@@ -1049,7 +1087,7 @@ class ExtractWorker2(Process):
                     stdout=subprocess.PIPE)
 
                 run_error = wait_backup_feedback(
-                    progress_callback=self.compute_progress,
+                    progress_callback=self.progress_callback,
                     in_stream=self.decryptor_process.stdout,
                     streamproc=self.decryptor_process,
                     **common_args)
@@ -1060,13 +1098,13 @@ class ExtractWorker2(Process):
                     stdout=subprocess.PIPE)
 
                 run_error = wait_backup_feedback(
-                    progress_callback=self.compute_progress,
+                    progress_callback=self.progress_callback,
                     in_stream=self.decompressor_process.stdout,
                     streamproc=self.decompressor_process,
                     **common_args)
             else:
                 run_error = wait_backup_feedback(
-                    progress_callback=self.compute_progress,
+                    progress_callback=self.progress_callback,
                     in_stream=open(filename, "rb"), streamproc=None,
                     **common_args)
 
@@ -1122,12 +1160,12 @@ class ExtractWorker2(Process):
 
 
 class ExtractWorker3(ExtractWorker2):
-    def __init__(self, queue, base_dir, passphrase, encrypted, total_size,
+    def __init__(self, queue, base_dir, passphrase, encrypted,
                  print_callback, error_callback, progress_callback, vmproc=None,
                  compressed=False, crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM,
                  compression_filter=None, verify_only=False):
         super(ExtractWorker3, self).__init__(queue, base_dir, passphrase,
-                                             encrypted, total_size,
+                                             encrypted,
                                              print_callback, error_callback,
                                              progress_callback, vmproc,
                                              compressed, crypto_algorithm,
@@ -1223,14 +1261,13 @@ class ExtractWorker3(ExtractWorker2):
 
             common_args = {
                 'backup_target': input_pipe,
-                'total_backup_sz': self.total_size,
                 'hmac': None,
                 'vmproc': self.vmproc,
                 'addproc': self.tar2_process
             }
 
             run_error = wait_backup_feedback(
-                progress_callback=self.compute_progress,
+                progress_callback=self.progress_callback,
                 in_stream=open(filename, "rb"), streamproc=None,
                 **common_args)
 
@@ -1281,7 +1318,7 @@ class ExtractWorker3(ExtractWorker2):
             self.print_callback("Finished extracting thread")
 
 
-def get_supported_hmac_algo(hmac_algorithm):
+def get_supported_hmac_algo(hmac_algorithm=None):
     # Start with provided default
     if hmac_algorithm:
         yield hmac_algorithm
@@ -1293,141 +1330,258 @@ def get_supported_hmac_algo(hmac_algorithm):
         yield algo.strip()
     proc.wait()
 
-
-def parse_backup_header(filename):
-    header_data = {}
-    with open(filename, 'r') as f:
-        for line in f.readlines():
-            if line.count('=') != 1:
-                raise qubes.exc.QubesException("Invalid backup header (line %s)" % line)
-            (key, value) = line.strip().split('=')
-            if not any([key == getattr(BackupHeader, attr) for attr in dir(
-                    BackupHeader)]):
-                # Ignoring unknown option
-                continue
-            if key in BackupHeader.bool_options:
-                value = value.lower() in ["1", "true", "yes"]
-            elif key in BackupHeader.int_options:
-                value = int(value)
-            header_data[key] = value
-    return header_data
+class BackupRestoreOptions(object):
+    def __init__(self):
+        #: use default NetVM if the one referenced in backup do not exists on
+        #  the host
+        self.use_default_netvm = True
+        #: set NetVM to "none" if the one referenced in backup do not exists
+        # on the host
+        self.use_none_netvm = False
+        #: set template to default if the one referenced in backup do not
+        # exists on the host
+        self.use_default_template = True
+        #: restore dom0 home
+        self.dom0_home = True
+        #: dictionary how what templates should be used instead of those
+        # referenced in backup
+        self.replace_template = {}
+        #: restore dom0 home even if username is different
+        self.ignore_username_mismatch = False
+        #: do not restore data, only verify backup integrity
+        self.verify_only = False
+        #: automatically rename VM during restore, when it would conflict
+        # with existing one
+        self.rename_conflicting = True
+        #: list of VM names to exclude
+        self.exclude = []
 
 
-def restore_vm_dirs(backup_source, restore_tmpdir, passphrase, vms_dirs, vms,
-                    vms_size, print_callback=None, error_callback=None,
-                    progress_callback=None, encrypted=False, appvm=None,
-                    compressed=False, hmac_algorithm=DEFAULT_HMAC_ALGORITHM,
-                    crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM,
-                    verify_only=False,
-                    format_version=CURRENT_BACKUP_FORMAT_VERSION,
-                    compression_filter=None):
-    global running_backup_operation
+class BackupRestore(object):
+    """Usage:
+    >>> restore_op = BackupRestore(...)
+    >>> # adjust restore_op.options here
+    >>> restore_info = restore_op.get_restore_info()
+    >>> # manipulate restore_info to select VMs to restore here
+    >>> restore_op.restore_do(restore_info)
+    """
 
-    if callable(print_callback):
+    def __init__(self, app, backup_location, backup_vm, passphrase):
+        super(BackupRestore, self).__init__()
+
+        #: qubes.Qubes instance
+        self.app = app
+
+        #: options how the backup should be restored
+        self.options = BackupRestoreOptions()
+
+        #: VM from which backup should be retrieved
+        self.backup_vm = backup_vm
+        if backup_vm and backup_vm.qid == 0:
+            self.backup_vm = None
+
+        #: backup path, inside VM pointed by :py:attr:`backup_vm`
+        self.backup_location = backup_location
+
+        #: passphrase protecting backup integrity and optionally decryption
+        self.passphrase = passphrase
+
+        #: temporary directory used to extract the data before moving to the
+        # final location; should be on the same filesystem as /var/lib/qubes
+        self.tmpdir = tempfile.mkdtemp(prefix="restore", dir="/var/tmp")
+
+        #: list of processes (Popen objects) to kill on cancel
+        self.processes_to_kill_on_cancel = []
+
+        #: is the backup operation canceled
+        self.canceled = False
+
+        # TODO: convert to python logging API
+        self.print_callback = print_stdout
+        self.error_callback = print_stderr
+
+        #: report restore progress, called with one argument - percents of
+        # data restored
+        # FIXME: convert to float [0,1]
+        self.progress_callback = None
+
+        #: basic information about the backup
+        self.header_data = self._retrieve_backup_header()
+
+        #: VMs included in the backup
+        self.backup_app = self._process_qubes_xml()
+
+    def cancel(self):
+        """Cancel running backup operation. Can be called from another thread.
+        """
+        self.canceled = True
+        for proc in self.processes_to_kill_on_cancel:
+            try:
+                proc.terminate()
+            except:
+                pass
+
+    def _start_retrieval_process(self, filelist, limit_count, limit_bytes):
+        """Retrieve backup stream and extract it to :py:attr:`tmpdir`
+
+        :param filelist: list of files to extract; listing directory name
+        will extract the whole directory; use empty list to extract the whole
+        archive
+        :param limit_count: maximum number of files to extract
+        :param limit_bytes: maximum size of extracted data
+        :return: a touple of (Popen object of started process, file-like
+        object for reading extracted files list, file-like object for reading
+        errors)
+        """
+
+        vmproc = None
+        if self.backup_vm is not None:
+            # Prepare the backup target (Qubes service call)
+            backup_target = "QUBESRPC qubes.Restore dom0"
+
+            # If APPVM, STDOUT is a PIPE
+            vmproc = self.backup_vm.run(
+                command=backup_target, passio_popen=True, passio_stderr=True)
+            vmproc.stdin.write(
+                self.backup_location.replace("\r", "").replace("\n", "") + "\n")
+
+            # Send to tar2qfile the VMs that should be extracted
+            vmproc.stdin.write(" ".join(filelist) + "\n")
+            self.processes_to_kill_on_cancel.append(vmproc)
+
+            backup_stdin = vmproc.stdout
+            tar1_command = ['/usr/libexec/qubes/qfile-dom0-unpacker',
+                            str(os.getuid()), self.tmpdir, '-v']
+        else:
+            backup_stdin = open(self.backup_location, 'rb')
+
+            tar1_command = ['tar',
+                            '-ixv',
+                            '-C', self.tmpdir] + filelist
+
+        tar1_env = os.environ.copy()
+        tar1_env['UPDATES_MAX_BYTES'] = str(limit_bytes)
+        tar1_env['UPDATES_MAX_FILES'] = str(limit_count)
+        if BACKUP_DEBUG and callable(self.print_callback):
+            self.print_callback("Run command" + unicode(tar1_command))
+        command = subprocess.Popen(
+            tar1_command,
+            stdin=backup_stdin,
+            stdout=vmproc.stdin if vmproc else subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            env=tar1_env)
+        self.processes_to_kill_on_cancel.append(command)
+
+        # qfile-dom0-unpacker output filelist on stderr (and have stdout connected
+        # to the VM), while tar output filelist on stdout
+        if self.backup_vm:
+            filelist_pipe = command.stderr
+            # let qfile-dom0-unpacker hold the only open FD to the write end of
+            # pipe, otherwise qrexec-client will not receive EOF when
+            # qfile-dom0-unpacker terminates
+            vmproc.stdin.close()
+        else:
+            filelist_pipe = command.stdout
+
+        if self.backup_vm:
+            error_pipe = vmproc.stderr
+        else:
+            error_pipe = command.stderr
+        return command, filelist_pipe, error_pipe
+
+    def _verify_hmac(self, filename, hmacfile, algorithm=None):
+        def load_hmac(hmac):
+            hmac = hmac.strip().split("=")
+            if len(hmac) > 1:
+                hmac = hmac[1].strip()
+            else:
+                raise qubes.exc.QubesException("ERROR: invalid hmac file content")
+
+            return hmac
+        if algorithm is None:
+            algorithm = self.header_data.hmac_algorithm
+        passphrase = self.passphrase.encode('utf-8')
         if BACKUP_DEBUG:
-            print_callback("Working in temporary dir:" + restore_tmpdir)
-        print_callback(
-            "Extracting data: " + size_to_human(vms_size) + " to restore")
+            print "Verifying file " + filename
 
-    passphrase = passphrase.encode('utf-8')
-    header_data = None
-    vmproc = None
-    if appvm is not None:
-        # Prepare the backup target (Qubes service call)
-        backup_target = "QUBESRPC qubes.Restore dom0"
+        if hmacfile != filename + ".hmac":
+            raise qubes.exc.QubesException(
+                "ERROR: expected hmac for {}, but got {}".
+                format(filename, hmacfile))
 
-        # If APPVM, STDOUT is a PIPE
-        vmproc = appvm.run(command=backup_target, passio_popen=True,
-                           passio_stderr=True)
-        vmproc.stdin.write(
-            backup_source.replace("\r", "").replace("\n", "") + "\n")
+        hmac_proc = subprocess.Popen(
+            ["openssl", "dgst", "-" + algorithm, "-hmac", passphrase],
+            stdin=open(os.path.join(self.tmpdir, filename), 'rb'),
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        hmac_stdout, hmac_stderr = hmac_proc.communicate()
 
-        # Send to tar2qfile the VMs that should be extracted
-        vmproc.stdin.write(" ".join(vms_dirs) + "\n")
-        if running_backup_operation:
-            running_backup_operation.processes_to_kill_on_cancel.append(vmproc)
+        if len(hmac_stderr) > 0:
+            raise qubes.exc.QubesException(
+                "ERROR: verify file {0}: {1}".format(filename, hmac_stderr))
+        else:
+            if BACKUP_DEBUG:
+                print "Loading hmac for file " + filename
+            hmac = load_hmac(open(os.path.join(self.tmpdir, hmacfile),
+                'r').read())
 
-        backup_stdin = vmproc.stdout
-        tar1_command = ['/usr/libexec/qubes/qfile-dom0-unpacker',
-                        str(os.getuid()), restore_tmpdir, '-v']
-    else:
-        backup_stdin = open(backup_source, 'rb')
+            if len(hmac) > 0 and load_hmac(hmac_stdout) == hmac:
+                os.unlink(os.path.join(self.tmpdir, hmacfile))
+                if BACKUP_DEBUG:
+                    print "File verification OK -> Sending file " + filename
+                return True
+            else:
+                raise qubes.exc.QubesException(
+                    "ERROR: invalid hmac for file {0}: {1}. "
+                    "Is the passphrase correct?".
+                    format(filename, load_hmac(hmac_stdout)))
 
-        tar1_command = ['tar',
-                        '-ixvf', backup_source,
-                        '-C', restore_tmpdir] + vms_dirs
+    def _retrieve_backup_header(self):
+        """Retrieve backup header and qubes.xml. Only backup header is
+        analyzed, qubes.xml is left as-is
+        (not even verified/decrypted/uncompressed)
 
-    tar1_env = os.environ.copy()
-    # TODO: add some safety margin?
-    tar1_env['UPDATES_MAX_BYTES'] = str(vms_size)
-    # Restoring only header
-    if vms_dirs and vms_dirs[0] == HEADER_FILENAME:
-        # backup-header, backup-header.hmac, qubes-xml.000, qubes-xml.000.hmac
-        tar1_env['UPDATES_MAX_FILES'] = '4'
-    else:
-        # Currently each VM consists of at most 7 archives (count
-        # file_to_backup calls in backup_prepare()), but add some safety
-        # margin for further extensions. Each archive is divided into 100MB
-        # chunks. Additionally each file have own hmac file. So assume upper
-        # limit as 2*(10*COUNT_OF_VMS+TOTAL_SIZE/100MB)
-        tar1_env['UPDATES_MAX_FILES'] = str(2 * (10 * len(vms_dirs) +
-                                                 int(vms_size /
-                                                     (100 * 1024 * 1024))))
-    if BACKUP_DEBUG and callable(print_callback):
-        print_callback("Run command" + unicode(tar1_command))
-    command = subprocess.Popen(
-        tar1_command,
-        stdin=backup_stdin,
-        stdout=vmproc.stdin if vmproc else subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        env=tar1_env)
-    if running_backup_operation:
-        running_backup_operation.processes_to_kill_on_cancel.append(command)
+        :return header_data
+        :rtype :py:class:`BackupHeader`
+        """
 
-    # qfile-dom0-unpacker output filelist on stderr (and have stdout connected
-    # to the VM), while tar output filelist on stdout
-    if appvm:
-        filelist_pipe = command.stderr
-        # let qfile-dom0-unpacker hold the only open FD to the write end of
-        # pipe, otherwise qrexec-client will not receive EOF when
-        # qfile-dom0-unpacker terminates
-        vmproc.stdin.close()
-    else:
-        filelist_pipe = command.stdout
+        if not self.backup_vm and os.path.exists(
+                os.path.join(self.backup_location, 'qubes.xml')):
+            # backup format version 1 doesn't have header
+            header_data = BackupHeader()
+            header_data.version = 1
+            return header_data
 
-    expect_tar_error = False
+        (retrieve_proc, filelist_pipe, error_pipe) = \
+            self._start_retrieval_process(
+                ['backup-header', 'backup-header.hmac',
+                'qubes.xml.000', 'qubes.xml.000.hmac'], 4, 1024 * 1024)
 
-    to_extract = Queue()
-    nextfile = None
+        nextfile = None
+        expect_tar_error = False
 
-    # If want to analyze backup header, do it now
-    if vms_dirs and vms_dirs[0] == HEADER_FILENAME:
         filename = filelist_pipe.readline().strip()
         hmacfile = filelist_pipe.readline().strip()
-        if not appvm:
+        # tar output filename before actually extracting it, so wait for the
+        # next one before trying to access it
+        if not self.backup_vm:
             nextfile = filelist_pipe.readline().strip()
 
-        if BACKUP_DEBUG and callable(print_callback):
-            print_callback("Got backup header and hmac: %s, %s" % (filename,
-                                                                   hmacfile))
+        if BACKUP_DEBUG and callable(self.print_callback):
+            self.print_callback("Got backup header and hmac: %s, %s" % (
+                filename, hmacfile))
 
         if not filename or filename == "EOF" or \
                 not hmacfile or hmacfile == "EOF":
-            if appvm:
-                vmproc.wait()
-                proc_error_msg = vmproc.stderr.read(MAX_STDERR_BYTES)
-            else:
-                command.wait()
-                proc_error_msg = command.stderr.read(MAX_STDERR_BYTES)
-            raise qubes.exc.QubesException("Premature end of archive while receiving "
-                                 "backup header. Process output:\n" +
-                                 proc_error_msg)
-        filename = os.path.join(restore_tmpdir, filename)
-        hmacfile = os.path.join(restore_tmpdir, hmacfile)
+            retrieve_proc.wait()
+            proc_error_msg = error_pipe.read(MAX_STDERR_BYTES)
+            raise qubes.exc.QubesException(
+                "Premature end of archive while receiving "
+                "backup header. Process output:\n" + proc_error_msg)
         file_ok = False
+        hmac_algorithm = DEFAULT_HMAC_ALGORITHM
         for hmac_algo in get_supported_hmac_algo(hmac_algorithm):
             try:
-                if verify_hmac(filename, hmacfile, passphrase, hmac_algo):
+                if self._verify_hmac(filename, hmacfile, hmac_algo):
                     file_ok = True
                     hmac_algorithm = hmac_algo
                     break
@@ -1435,896 +1589,762 @@ def restore_vm_dirs(backup_source, restore_tmpdir, passphrase, vms_dirs, vms,
                 # Ignore exception here, try the next algo
                 pass
         if not file_ok:
-            raise qubes.exc.QubesException("Corrupted backup header (hmac verification "
-                                 "failed). Is the password correct?")
+            raise qubes.exc.QubesException(
+                "Corrupted backup header (hmac verification "
+                "failed). Is the password correct?")
         if os.path.basename(filename) == HEADER_FILENAME:
-            header_data = parse_backup_header(filename)
-            if BackupHeader.version in header_data:
-                format_version = header_data[BackupHeader.version]
-            if BackupHeader.crypto_algorithm in header_data:
-                crypto_algorithm = header_data[BackupHeader.crypto_algorithm]
-            if BackupHeader.hmac_algorithm in header_data:
-                hmac_algorithm = header_data[BackupHeader.hmac_algorithm]
-            if BackupHeader.compressed in header_data:
-                compressed = header_data[BackupHeader.compressed]
-            if BackupHeader.encrypted in header_data:
-                encrypted = header_data[BackupHeader.encrypted]
-            if BackupHeader.compression_filter in header_data:
-                compression_filter = header_data[
-                    BackupHeader.compression_filter]
+            filename = os.path.join(self.tmpdir, filename)
+            header_data = BackupHeader(open(filename, 'r').read())
             os.unlink(filename)
         else:
             # if no header found, create one with guessed HMAC algo
-            header_data = {BackupHeader.hmac_algorithm: hmac_algorithm}
-            # If this isn't backup header, pass it to ExtractWorker
-            to_extract.put(filename)
+            header_data = BackupHeader()
+            header_data.version = 2
+            header_data.hmac_algorithm = hmac_algorithm
+            # place explicitly this value, because it is what format_version
+            # 2 have
+            header_data.crypto_algorithm = 'aes-256-cbc'
+            # TODO: set header_data.encrypted to something...
             # when tar do not find expected file in archive, it exit with
             # code 2. This will happen because we've requested backup-header
             # file, but the archive do not contain it. Ignore this particular
             # error.
-            if not appvm:
+            if not self.backup_vm:
                 expect_tar_error = True
 
-    # Setup worker to extract encrypted data chunks to the restore dirs
-    # Create the process here to pass it options extracted from backup header
-    extractor_params = {
-        'queue': to_extract,
-        'base_dir': restore_tmpdir,
-        'passphrase': passphrase,
-        'encrypted': encrypted,
-        'compressed': compressed,
-        'crypto_algorithm': crypto_algorithm,
-        'verify_only': verify_only,
-        'total_size': vms_size,
-        'print_callback': print_callback,
-        'error_callback': error_callback,
-        'progress_callback': progress_callback,
-    }
-    if format_version == 2:
-        extract_proc = ExtractWorker2(**extractor_params)
-    elif format_version == 3:
-        extractor_params['compression_filter'] = compression_filter
-        extract_proc = ExtractWorker3(**extractor_params)
-    else:
-        raise NotImplemented(
-            "Backup format version %d not supported" % format_version)
-    extract_proc.start()
+        if retrieve_proc.wait() != 0 and not expect_tar_error:
+            raise qubes.exc.QubesException(
+                "unable to read the qubes backup file {0} ({1}): {2}".format(
+                    self.backup_location,
+                    retrieve_proc.wait(),
+                    error_pipe.read(MAX_STDERR_BYTES)
+                ))
+        if retrieve_proc in self.processes_to_kill_on_cancel:
+            self.processes_to_kill_on_cancel.remove(retrieve_proc)
+        # wait for other processes (if any)
+        for proc in self.processes_to_kill_on_cancel:
+            if proc.wait() != 0:
+                raise qubes.exc.QubesException(
+                    "Backup header retrieval failed (exit code {})".format(
+                        proc.wait())
+                )
+        return header_data
 
-    try:
-        filename = None
-        while True:
-            if running_backup_operation and running_backup_operation.canceled:
-                break
-            if not extract_proc.is_alive():
-                command.terminate()
-                command.wait()
-                expect_tar_error = True
-                if vmproc:
-                    vmproc.terminate()
-                    vmproc.wait()
-                    vmproc = None
-                break
-            if nextfile is not None:
-                filename = nextfile
-            else:
-                filename = filelist_pipe.readline().strip()
+    def _start_inner_extraction_worker(self, queue):
+        """Start a worker process, extracting inner layer of bacup archive,
+        extract them to :py:attr:`tmpdir`.
+        End the data by pushing "FINISHED" or "ERROR" to the queue.
 
-            if BACKUP_DEBUG and callable(print_callback):
-                print_callback("Getting new file:" + filename)
+        :param queue :py:class:`Queue` object to handle files from
+        """
 
-            if not filename or filename == "EOF":
-                break
+        # Setup worker to extract encrypted data chunks to the restore dirs
+        # Create the process here to pass it options extracted from backup header
+        extractor_params = {
+            'queue': queue,
+            'base_dir': self.tmpdir,
+            'passphrase': self.passphrase,
+            'encrypted': self.header_data.encrypted,
+            'compressed': self.header_data.compressed,
+            'crypto_algorithm': self.header_data.crypto_algorithm,
+            'verify_only': self.options.verify_only,
+            'print_callback': self.print_callback,
+            'error_callback': print_stderr,
+            'progress_callback': self.progress_callback,
+        }
+        format_version = self.header_data.version
+        if format_version == 2:
+            extract_proc = ExtractWorker2(**extractor_params)
+        elif format_version == 3:
+            extractor_params['compression_filter'] = \
+                self.header_data.compression_filter
+            extract_proc = ExtractWorker3(**extractor_params)
+        else:
+            raise NotImplementedError(
+                "Backup format version %d not supported" % format_version)
+        extract_proc.start()
+        return extract_proc
 
-            hmacfile = filelist_pipe.readline().strip()
+    def _process_qubes_xml(self):
+        """Verify, unpack and load qubes.xml. Possibly convert its format if
+        necessary. It expect that :py:attr:`header_data` is already populated,
+        and :py:meth:`retrieve_backup_header` was called.
+        """
+        if self.header_data.version == 1:
+            raise NotImplementedError("TODO: conversion core[12] qubes.xml")
+        else:
+            self._verify_hmac("qubes.xml.000", "qubes.xml.000.hmac")
+            queue = Queue()
+            queue.put("qubes.xml.000")
+            queue.put("FINISHED")
 
-            if running_backup_operation and running_backup_operation.canceled:
-                break
-            # if reading archive directly with tar, wait for next filename -
-            # tar prints filename before processing it, so wait for
-            # the next one to be sure that whole file was extracted
-            if not appvm:
-                nextfile = filelist_pipe.readline().strip()
+        extract_proc = self._start_inner_extraction_worker(queue)
+        extract_proc.join()
+        if extract_proc.exitcode != 0:
+            raise qubes.exc.QubesException(
+                "unable to extract the qubes backup. "
+                "Check extracting process errors.")
 
-            if BACKUP_DEBUG and callable(print_callback):
-                print_callback("Getting hmac:" + hmacfile)
-            if not hmacfile or hmacfile == "EOF":
-                # Premature end of archive, either of tar1_command or
-                # vmproc exited with error
-                break
+        if self.header_data.version in [2]:  # TODO add 3
+            raise NotImplementedError("TODO: conversion core[12] qubes.xml")
+        else:
+            backup_app = qubes.Qubes(os.path.join(self.tmpdir, 'qubes.xml'))
+            # Not needed anymore - all the data stored in backup_app
+            os.unlink(os.path.join(self.tmpdir, 'qubes.xml'))
+            return backup_app
 
-            if not any(map(lambda x: filename.startswith(x), vms_dirs)):
-                if BACKUP_DEBUG and callable(print_callback):
-                    print_callback("Ignoring VM not selected for restore")
-                os.unlink(os.path.join(restore_tmpdir, filename))
-                os.unlink(os.path.join(restore_tmpdir, hmacfile))
+    def _restore_vm_dirs(self, vms_dirs, vms_size):
+        # Currently each VM consists of at most 7 archives (count
+        # file_to_backup calls in backup_prepare()), but add some safety
+        # margin for further extensions. Each archive is divided into 100MB
+        # chunks. Additionally each file have own hmac file. So assume upper
+        # limit as 2*(10*COUNT_OF_VMS+TOTAL_SIZE/100MB)
+        limit_count = str(2 * (10 * len(vms_dirs) +
+                               int(vms_size / (100 * 1024 * 1024))))
+
+        if callable(self.print_callback):
+            if BACKUP_DEBUG:
+                self.print_callback("Working in temporary dir:" +
+                                    self.tmpdir)
+            self.print_callback(
+                "Extracting data: " + size_to_human(vms_size) + " to restore")
+
+        # retrieve backup from the backup stream (either VM, or dom0 file)
+        # TODO: add some safety margin in vms_size?
+        (retrieve_proc, filelist_pipe, error_pipe) = \
+            self._start_retrieval_process(vms_dirs, limit_count, vms_size)
+
+        to_extract = Queue()
+
+        # extract data retrieved by retrieve_proc
+        extract_proc = self._start_inner_extraction_worker(to_extract)
+
+        try:
+            filename = None
+            nextfile = None
+            while True:
+                if self.canceled:
+                    break
+                if not extract_proc.is_alive():
+                    retrieve_proc.terminate()
+                    retrieve_proc.wait()
+                    expect_tar_error = True
+                    if retrieve_proc in self.processes_to_kill_on_cancel:
+                        self.processes_to_kill_on_cancel.remove(retrieve_proc)
+                    # wait for other processes (if any)
+                    for proc in self.processes_to_kill_on_cancel:
+                        proc.wait()
+                    break
+                if nextfile is not None:
+                    filename = nextfile
+                else:
+                    filename = filelist_pipe.readline().strip()
+
+                if BACKUP_DEBUG and callable(self.print_callback):
+                    self.print_callback("Getting new file:" + filename)
+
+                if not filename or filename == "EOF":
+                    break
+
+                hmacfile = filelist_pipe.readline().strip()
+
+                if self.canceled:
+                    break
+                # if reading archive directly with tar, wait for next filename -
+                # tar prints filename before processing it, so wait for
+                # the next one to be sure that whole file was extracted
+                if not self.backup_vm:
+                    nextfile = filelist_pipe.readline().strip()
+
+                if BACKUP_DEBUG and callable(self.print_callback):
+                    self.print_callback("Getting hmac:" + hmacfile)
+                if not hmacfile or hmacfile == "EOF":
+                    # Premature end of archive, either of tar1_command or
+                    # vmproc exited with error
+                    break
+
+                if not any(map(lambda x: filename.startswith(x), vms_dirs)):
+                    if BACKUP_DEBUG and callable(self.print_callback):
+                        self.print_callback("Ignoring VM not selected for "
+                                          "restore")
+                    os.unlink(os.path.join(self.tmpdir, filename))
+                    os.unlink(os.path.join(self.tmpdir, hmacfile))
+                    continue
+
+                if self._verify_hmac(filename, hmacfile):
+                    to_extract.put(os.path.join(self.tmpdir, filename))
+
+            if self.canceled:
+                raise BackupCanceledError("Restore canceled",
+                                          tmpdir=self.tmpdir)
+
+            if retrieve_proc.wait() != 0:
+                raise qubes.exc.QubesException(
+                    "unable to read the qubes backup file {0} ({1}): {2}"
+                    .format(self.backup_location, error_pipe.read(
+                        MAX_STDERR_BYTES)))
+            # wait for other processes (if any)
+            for proc in self.processes_to_kill_on_cancel:
+                # FIXME check 'vmproc' exit code?
+                proc.wait()
+
+            if filename and filename != "EOF":
+                raise qubes.exc.QubesException(
+                    "Premature end of archive, the last file was %s" % filename)
+        except:
+            to_extract.put("ERROR")
+            extract_proc.join()
+            raise
+        else:
+            to_extract.put("FINISHED")
+
+        if BACKUP_DEBUG and callable(self.print_callback):
+            self.print_callback("Waiting for the extraction process to "
+                               "finish...")
+        extract_proc.join()
+        if BACKUP_DEBUG and callable(self.print_callback):
+            self.print_callback("Extraction process finished with code:" +
+                           str(extract_proc.exitcode))
+        if extract_proc.exitcode != 0:
+            raise qubes.exc.QubesException(
+                "unable to extract the qubes backup. "
+                "Check extracting process errors.")
+
+    def generate_new_name_for_conflicting_vm(self, orig_name, restore_info):
+        number = 1
+        if len(orig_name) > 29:
+            orig_name = orig_name[0:29]
+        new_name = orig_name
+        while (new_name in restore_info.keys() or
+               new_name in map(lambda x: x.get('rename_to', None),
+                               restore_info.values()) or
+               new_name in self.app.domains):
+            new_name = str('{}{}'.format(orig_name, number))
+            number += 1
+            if number == 100:
+                # give up
+                return None
+        return new_name
+
+    def restore_info_verify(self, restore_info):
+        for vm in restore_info.keys():
+            if vm in ['dom0']:
                 continue
 
-            if verify_hmac(os.path.join(restore_tmpdir, filename),
-                           os.path.join(restore_tmpdir, hmacfile),
-                           passphrase, hmac_algorithm):
-                to_extract.put(os.path.join(restore_tmpdir, filename))
+            vm_info = restore_info[vm]
 
-        if running_backup_operation and running_backup_operation.canceled:
-            raise BackupCanceledError("Restore canceled",
-                                      tmpdir=restore_tmpdir)
-
-        if command.wait() != 0 and not expect_tar_error:
-            raise qubes.exc.QubesException(
-                "unable to read the qubes backup file {0} ({1}). "
-                "Is it really a backup?".format(backup_source, command.wait()))
-        if vmproc:
-            if vmproc.wait() != 0:
-                raise qubes.exc.QubesException(
-                    "unable to read the qubes backup {0} "
-                    "because of a VM error: {1}".format(
-                        backup_source, vmproc.stderr.read(MAX_STDERR_BYTES)))
-
-        if filename and filename != "EOF":
-            raise qubes.exc.QubesException(
-                "Premature end of archive, the last file was %s" % filename)
-    except:
-        to_extract.put("ERROR")
-        extract_proc.join()
-        raise
-    else:
-        to_extract.put("FINISHED")
-
-    if BACKUP_DEBUG and callable(print_callback):
-        print_callback("Waiting for the extraction process to finish...")
-    extract_proc.join()
-    if BACKUP_DEBUG and callable(print_callback):
-        print_callback("Extraction process finished with code:" +
-                       str(extract_proc.exitcode))
-    if extract_proc.exitcode != 0:
-        raise qubes.exc.QubesException(
-            "unable to extract the qubes backup. "
-            "Check extracting process errors.")
-
-    return header_data
-
-
-def backup_restore_set_defaults(options):
-    if 'use-default-netvm' not in options:
-        options['use-default-netvm'] = False
-    if 'use-none-netvm' not in options:
-        options['use-none-netvm'] = False
-    if 'use-default-template' not in options:
-        options['use-default-template'] = False
-    if 'dom0-home' not in options:
-        options['dom0-home'] = True
-    if 'replace-template' not in options:
-        options['replace-template'] = []
-    if 'ignore-username-mismatch' not in options:
-        options['ignore-username-mismatch'] = False
-    if 'verify-only' not in options:
-        options['verify-only'] = False
-    if 'rename-conflicting' not in options:
-        options['rename-conflicting'] = False
-
-    return options
-
-
-def load_hmac(hmac):
-    hmac = hmac.strip().split("=")
-    if len(hmac) > 1:
-        hmac = hmac[1].strip()
-    else:
-        raise qubes.exc.QubesException("ERROR: invalid hmac file content")
-
-    return hmac
-
-
-def backup_detect_format_version(backup_location):
-    if os.path.exists(os.path.join(backup_location, 'qubes.xml')):
-        return 1
-    else:
-        # this could mean also 3, but not distinguishable until backup header
-        # is read
-        return 2
-
-
-def backup_restore_header(source, passphrase,
-                          print_callback=print_stdout,
-                          error_callback=print_stderr,
-                          encrypted=False, appvm=None, compressed=False,
-                          format_version=None,
-                          hmac_algorithm=DEFAULT_HMAC_ALGORITHM,
-                          crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM):
-    global running_backup_operation
-    running_backup_operation = None
-
-    restore_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/restore_")
-
-    if format_version is None:
-        format_version = backup_detect_format_version(source)
-
-    if format_version == 1:
-        return restore_tmpdir, os.path.join(source, 'qubes.xml'), None
-
-    # tar2qfile matches only beginnings, while tar full path
-    if appvm:
-        extract_filter = [HEADER_FILENAME, 'qubes.xml.000']
-    else:
-        extract_filter = [HEADER_FILENAME, HEADER_FILENAME + '.hmac',
-                          'qubes.xml.000', 'qubes.xml.000.hmac']
-
-    header_data = restore_vm_dirs(source,
-                                  restore_tmpdir,
-                                  passphrase=passphrase,
-                                  vms_dirs=extract_filter,
-                                  vms=None,
-                                  vms_size=HEADER_QUBES_XML_MAX_SIZE,
-                                  format_version=format_version,
-                                  hmac_algorithm=hmac_algorithm,
-                                  crypto_algorithm=crypto_algorithm,
-                                  print_callback=print_callback,
-                                  error_callback=error_callback,
-                                  progress_callback=None,
-                                  encrypted=encrypted,
-                                  compressed=compressed,
-                                  appvm=appvm)
-
-    return (restore_tmpdir, os.path.join(restore_tmpdir, "qubes.xml"),
-            header_data)
-
-
-def generate_new_name_for_conflicting_vm(orig_name, host_collection,
-                                         restore_info):
-    number = 1
-    if len(orig_name) > 29:
-        orig_name = orig_name[0:29]
-    new_name = orig_name
-    while (new_name in restore_info.keys() or
-           new_name in map(lambda x: x.get('rename_to', None),
-                           restore_info.values()) or
-           new_name in host_collection.domains):
-        new_name = str('{}{}'.format(orig_name, number))
-        number += 1
-        if number == 100:
-            # give up
-            return None
-    return new_name
-
-def restore_info_verify(restore_info, host_collection):
-    assert isinstance(host_collection, qubes.Qubes)
-    options = restore_info['$OPTIONS$']
-    for vm in restore_info.keys():
-        if vm in ['$OPTIONS$', 'dom0']:
-            continue
-
-        vm_info = restore_info[vm]
-
-        vm_info.pop('excluded', None)
-        if 'exclude' in options.keys():
-            if vm in options['exclude']:
+            vm_info.pop('excluded', None)
+            if vm in self.options.exclude:
                 vm_info['excluded'] = True
 
-        vm_info.pop('already-exists', None)
-        if not options['verify-only'] and \
-                vm in host_collection.domains:
-            if options['rename-conflicting']:
-                new_name = generate_new_name_for_conflicting_vm(
-                    vm, host_collection, restore_info
-                )
-                if new_name is not None:
-                    vm_info['rename-to'] = new_name
+            vm_info.pop('already-exists', None)
+            if not self.options.verify_only and \
+                    vm in self.app.domains:
+                if self.options.rename_conflicting:
+                    new_name = self.generate_new_name_for_conflicting_vm(
+                        vm, restore_info
+                    )
+                    if new_name is not None:
+                        vm_info['rename-to'] = new_name
+                    else:
+                        vm_info['already-exists'] = True
                 else:
                     vm_info['already-exists'] = True
-            else:
-                vm_info['already-exists'] = True
 
-        # check template
-        vm_info.pop('missing-template', None)
-        if vm_info['template']:
+            # check template
+            vm_info.pop('missing-template', None)
+            if vm_info['template']:
+                template_name = vm_info['template']
+                try:
+                    host_template = self.app.domains[template_name]
+                except KeyError:
+                    host_template = None
+                if not host_template or not host_template.is_template():
+                    # Maybe the (custom) template is in the backup?
+                    if not (template_name in restore_info.keys() and
+                            restore_info[template_name]['vm'].is_template()):
+                        if self.options.use_default_template:
+                            if 'orig-template' not in vm_info.keys():
+                                vm_info['orig-template'] = template_name
+                            vm_info['template'] = self.app.default_template.name
+                        else:
+                            vm_info['missing-template'] = True
+
+            # check netvm
+            vm_info.pop('missing-netvm', None)
+            if vm_info['netvm']:
+                netvm_name = vm_info['netvm']
+
+                try:
+                    netvm_on_host = self.app.domains[netvm_name]
+                except KeyError:
+                    netvm_on_host = None
+                # No netvm on the host?
+                if not ((netvm_on_host is not None) and netvm_on_host.is_netvm()):
+
+                    # Maybe the (custom) netvm is in the backup?
+                    if not (netvm_name in restore_info.keys() and
+                            restore_info[netvm_name]['vm'].is_netvm()):
+                        if self.options.use_default_netvm:
+                            vm_info['netvm'] = self.app.default_netvm.name
+                            vm_info['vm'].uses_default_netvm = True
+                        elif self.options.use_none_netvm:
+                            vm_info['netvm'] = None
+                        else:
+                            vm_info['missing-netvm'] = True
+
+            vm_info['good-to-go'] = not any([(prop in vm_info.keys()) for
+                                             prop in ['missing-netvm',
+                                                      'missing-template',
+                                                      'already-exists',
+                                                      'excluded']])
+
+        # update references to renamed VMs:
+        for vm in restore_info.keys():
+            if vm in ['dom0']:
+                continue
+            vm_info = restore_info[vm]
             template_name = vm_info['template']
-            host_template = host_collection.domains.get(template_name, None)
-            if not host_template or not host_template.is_template():
-                # Maybe the (custom) template is in the backup?
-                if not (template_name in restore_info.keys() and
-                        restore_info[template_name]['vm'].is_template()):
-                    if options['use-default-template']:
-                        if 'orig-template' not in vm_info.keys():
-                            vm_info['orig-template'] = template_name
-                        vm_info['template'] = host_collection \
-                            .default_template.name
-                    else:
-                        vm_info['missing-template'] = True
-
-        # check netvm
-        vm_info.pop('missing-netvm', None)
-        if vm_info['netvm']:
+            if (template_name in restore_info and
+                    restore_info[template_name]['good-to-go'] and
+                    'rename-to' in restore_info[template_name]):
+                vm_info['template'] = restore_info[template_name]['rename-to']
             netvm_name = vm_info['netvm']
+            if (netvm_name in restore_info and
+                    restore_info[netvm_name]['good-to-go'] and
+                    'rename-to' in restore_info[netvm_name]):
+                vm_info['netvm'] = restore_info[netvm_name]['rename-to']
 
-            netvm_on_host = host_collection.domains.get(netvm_name, None)
+        return restore_info
 
-            # No netvm on the host?
-            if not ((netvm_on_host is not None) and netvm_on_host.is_netvm()):
-
-                # Maybe the (custom) netvm is in the backup?
-                if not (netvm_name in restore_info.keys() and
-                        restore_info[netvm_name]['vm'].is_netvm()):
-                    if options['use-default-netvm']:
-                        vm_info['netvm'] = host_collection \
-                            .default_netvm.name
-                        vm_info['vm'].uses_default_netvm = True
-                    elif options['use-none-netvm']:
-                        vm_info['netvm'] = None
-                    else:
-                        vm_info['missing-netvm'] = True
-
-        vm_info['good-to-go'] = not any([(prop in vm_info.keys()) for
-                                         prop in ['missing-netvm',
-                                                  'missing-template',
-                                                  'already-exists',
-                                                  'excluded']])
-
-    # update references to renamed VMs:
-    for vm in restore_info.keys():
-        if vm in ['$OPTIONS$', 'dom0']:
-            continue
-        vm_info = restore_info[vm]
-        template_name = vm_info['template']
-        if (template_name in restore_info and
-                restore_info[template_name]['good-to-go'] and
-                'rename-to' in restore_info[template_name]):
-            vm_info['template'] = restore_info[template_name]['rename-to']
-        netvm_name = vm_info['netvm']
-        if (netvm_name in restore_info and
-                restore_info[netvm_name]['good-to-go'] and
-                'rename-to' in restore_info[netvm_name]):
-            vm_info['netvm'] = restore_info[netvm_name]['rename-to']
-
-    return restore_info
-
-
-def backup_restore_prepare(backup_location, passphrase, options=None,
-                           host_collection=None, encrypted=False, appvm=None,
-                           compressed=False, print_callback=print_stdout,
-                           error_callback=print_stderr,
-                           format_version=None,
-                           hmac_algorithm=DEFAULT_HMAC_ALGORITHM,
-                           crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM):
-    if options is None:
-        options = {}
-    # Defaults
-    backup_restore_set_defaults(options)
-    # Options introduced in backup format 3+, which always have a header,
-    # so no need for fallback in function parameter
-    compression_filter = DEFAULT_COMPRESSION_FILTER
-
-    # Private functions begin
-    def is_vm_included_in_backup_v1(backup_dir, check_vm):
+    def _is_vm_included_in_backup_v1(self, check_vm):
         if check_vm.qid == 0:
-            return os.path.exists(os.path.join(backup_dir, 'dom0-home'))
+            return os.path.exists(
+                os.path.join(self.backup_location, 'dom0-home'))
 
         # DisposableVM
         if check_vm.dir_path is None:
             return False
 
         backup_vm_dir_path = check_vm.dir_path.replace(
-            qubes.config.system_path["qubes_base_dir"], backup_dir)
+            qubes.config.system_path["qubes_base_dir"], self.backup_location)
 
         if os.path.exists(backup_vm_dir_path):
             return True
         else:
             return False
 
-    def is_vm_included_in_backup_v2(_, check_vm):
+    def _is_vm_included_in_backup_v2(self, check_vm):
         if check_vm.backup_content:
             return True
         else:
             return False
 
-    def find_template_name(template, replaces):
-        rx_replace = re.compile("(.*):(.*)")
-        for r in replaces:
-            m = rx_replace.match(r)
-            if m.group(1) == template:
-                return m.group(2)
-
+    def _find_template_name(self, template):
+        if template in self.options.replace_template:
+            return self.options.replace_template[template]
         return template
 
-    # Private functions end
-
-    # Format versions:
-    # 1 - Qubes R1, Qubes R2 beta1, beta2
-    #  2 - Qubes R2 beta3+
-
-    if format_version is None:
-        format_version = backup_detect_format_version(backup_location)
-
-    if format_version == 1:
-        is_vm_included_in_backup = is_vm_included_in_backup_v1
-    elif format_version in [2, 3]:
-        is_vm_included_in_backup = is_vm_included_in_backup_v2
-        if not appvm:
-            if not os.path.isfile(backup_location):
-                raise qubes.exc.QubesException("Invalid backup location (not a file or "
-                                     "directory with qubes.xml)"
-                                     ": %s" % unicode(backup_location))
-    else:
-        raise qubes.exc.QubesException(
-            "Unknown backup format version: %s" % str(format_version))
-
-    (restore_tmpdir, qubes_xml, header_data) = backup_restore_header(
-        backup_location,
-        passphrase,
-        encrypted=encrypted,
-        appvm=appvm,
-        compressed=compressed,
-        hmac_algorithm=hmac_algorithm,
-        crypto_algorithm=crypto_algorithm,
-        print_callback=print_callback,
-        error_callback=error_callback,
-        format_version=format_version)
-
-    if header_data:
-        if BackupHeader.version in header_data:
-            format_version = header_data[BackupHeader.version]
-        if BackupHeader.crypto_algorithm in header_data:
-            crypto_algorithm = header_data[BackupHeader.crypto_algorithm]
-        if BackupHeader.hmac_algorithm in header_data:
-            hmac_algorithm = header_data[BackupHeader.hmac_algorithm]
-        if BackupHeader.compressed in header_data:
-            compressed = header_data[BackupHeader.compressed]
-        if BackupHeader.encrypted in header_data:
-            encrypted = header_data[BackupHeader.encrypted]
-        if BackupHeader.compression_filter in header_data:
-            compression_filter = header_data[BackupHeader.compression_filter]
-
-    if BACKUP_DEBUG:
-        print "Loading file", qubes_xml
-    backup_collection = qubes.Qubes(qubes_xml)
-
-    if host_collection is None:
-        host_collection = qubes.Qubes()
-
-    backup_vms_list = [vm for vm in backup_collection.domains]
-    vms_to_restore = {}
-
-    # ... and the actual data
-    for vm in backup_vms_list:
-        if vm.qid == 0:
-            # Handle dom0 as special case later
-            continue
-        if is_vm_included_in_backup(backup_location, vm):
-            if BACKUP_DEBUG:
-                print vm.name, "is included in backup"
-
-            vms_to_restore[vm.name] = {}
-            vms_to_restore[vm.name]['vm'] = vm
-
-            if vm.template is None:
-                vms_to_restore[vm.name]['template'] = None
-            else:
-                templatevm_name = find_template_name(vm.template.name, options[
-                    'replace-template'])
-                vms_to_restore[vm.name]['template'] = templatevm_name
-
-            if vm.netvm is None:
-                vms_to_restore[vm.name]['netvm'] = None
-            else:
-                netvm_name = vm.netvm.name
-                vms_to_restore[vm.name]['netvm'] = netvm_name
-                # Set to None to not confuse QubesVm object from backup
-                # collection with host collection (further in clone_attrs). Set
-                # directly _netvm to suppress setter action, especially
-                # modifying firewall
-                vm._netvm = None
-
-    # Store restore parameters
-    options['location'] = backup_location
-    options['restore_tmpdir'] = restore_tmpdir
-    options['passphrase'] = passphrase
-    options['encrypted'] = encrypted
-    options['compressed'] = compressed
-    options['compression_filter'] = compression_filter
-    options['hmac_algorithm'] = hmac_algorithm
-    options['crypto_algorithm'] = crypto_algorithm
-    options['appvm'] = appvm
-    options['format_version'] = format_version
-    vms_to_restore['$OPTIONS$'] = options
-
-    vms_to_restore = restore_info_verify(vms_to_restore, host_collection)
-
-    # ...and dom0 home
-    if options['dom0-home'] and \
-            is_vm_included_in_backup(backup_location,
-                backup_collection.domains[0]):
-        vm = backup_collection.domains[0]
-        vms_to_restore['dom0'] = {}
-        if format_version == 1:
-            vms_to_restore['dom0']['subdir'] = \
-                os.listdir(os.path.join(backup_location, 'dom0-home'))[0]
-            vms_to_restore['dom0']['size'] = 0  # unknown
+    def _is_vm_included_in_backup(self, vm):
+        if self.header_data.version == 1:
+            return self._is_vm_included_in_backup_v1(vm)
+        elif self.header_data.version in [2, 3]:
+            return self._is_vm_included_in_backup_v2(vm)
         else:
-            vms_to_restore['dom0']['subdir'] = vm.backup_path
-            vms_to_restore['dom0']['size'] = vm.backup_size
-        local_user = grp.getgrnam('qubes').gr_mem[0]
+            raise qubes.exc.QubesException(
+                "Unknown backup format version: {}".format(
+                    self.header_data.version))
 
-        dom0_home = vms_to_restore['dom0']['subdir']
+    def get_restore_info(self):
+        # Format versions:
+        # 1 - Qubes R1, Qubes R2 beta1, beta2
+        #  2 - Qubes R2 beta3+
 
-        vms_to_restore['dom0']['username'] = os.path.basename(dom0_home)
-        if vms_to_restore['dom0']['username'] != local_user:
-            vms_to_restore['dom0']['username-mismatch'] = True
-            if options['ignore-username-mismatch']:
-                vms_to_restore['dom0']['ignore-username-mismatch'] = True
+        vms_to_restore = {}
+
+        for vm in self.backup_app.domains:
+            if vm.qid == 0:
+                # Handle dom0 as special case later
+                continue
+            if self._is_vm_included_in_backup(vm):
+                if BACKUP_DEBUG:
+                    print vm.name, "is included in backup"
+
+                vms_to_restore[vm.name] = {}
+                vms_to_restore[vm.name]['vm'] = vm
+
+                if not hasattr(vm, 'template'):
+                    vms_to_restore[vm.name]['template'] = None
+                else:
+                    templatevm_name = self._find_template_name(
+                        vm.template.name)
+                    vms_to_restore[vm.name]['template'] = templatevm_name
+
+                if vm.netvm is None:
+                    vms_to_restore[vm.name]['netvm'] = None
+                else:
+                    netvm_name = vm.netvm.name
+                    vms_to_restore[vm.name]['netvm'] = netvm_name
+                    # Set to None to not confuse QubesVm object from backup
+                    # collection with host collection (further in clone_attrs).
+                    vm.netvm = None
+
+        vms_to_restore = self.restore_info_verify(vms_to_restore)
+
+        # ...and dom0 home
+        if self.options.dom0_home and \
+                self._is_vm_included_in_backup(self.backup_app.domains[0]):
+            vm = self.backup_app.domains[0]
+            vms_to_restore['dom0'] = {}
+            if self.header_data.version == 1:
+                vms_to_restore['dom0']['subdir'] = \
+                    os.listdir(os.path.join(
+                        self.backup_location, 'dom0-home'))[0]
+                vms_to_restore['dom0']['size'] = 0  # unknown
             else:
-                vms_to_restore['dom0']['good-to-go'] = False
+                vms_to_restore['dom0']['subdir'] = vm.backup_path
+                vms_to_restore['dom0']['size'] = vm.backup_size
+            local_user = grp.getgrnam('qubes').gr_mem[0]
 
-        if 'good-to-go' not in vms_to_restore['dom0']:
-            vms_to_restore['dom0']['good-to-go'] = True
+            dom0_home = vms_to_restore['dom0']['subdir']
 
-    # Not needed - all the data stored in vms_to_restore
-    if format_version >= 2:
-        os.unlink(qubes_xml)
-    return vms_to_restore
+            vms_to_restore['dom0']['username'] = os.path.basename(dom0_home)
+            if vms_to_restore['dom0']['username'] != local_user:
+                vms_to_restore['dom0']['username-mismatch'] = True
+                if self.options.ignore_username_mismatch:
+                    vms_to_restore['dom0']['ignore-username-mismatch'] = True
+                else:
+                    vms_to_restore['dom0']['good-to-go'] = False
 
+            if 'good-to-go' not in vms_to_restore['dom0']:
+                vms_to_restore['dom0']['good-to-go'] = True
 
-def backup_restore_print_summary(restore_info, print_callback=print_stdout):
-    fields = {
-        "qid": {"func": "vm.qid"},
+        return vms_to_restore
 
-        "name": {"func": "('[' if vm.is_template() else '')\
-                 + ('{' if vm.is_netvm() else '')\
-                 + vm.name \
-                 + (']' if vm.is_template() else '')\
-                 + ('}' if vm.is_netvm() else '')"},
+    def get_restore_summary(self, restore_info):
+        fields = {
+            "qid": {"func": "vm.qid"},
 
-        "type": {"func": "'Tpl' if vm.is_template() else \
-                 'HVM' if vm.type == 'HVM' else \
-                 vm.type.replace('VM','')"},
+            "name": {"func": "('[' if vm.is_template() else '')\
+                     + ('{' if vm.is_netvm() else '')\
+                     + vm.name \
+                     + (']' if vm.is_template() else '')\
+                     + ('}' if vm.is_netvm() else '')"},
 
-        "updbl": {"func": "'Yes' if vm.updateable else ''"},
+            "type": {"func": "'Tpl' if vm.is_template() else \
+                     'App' if isinstance(vm, qubes.vm.appvm.AppVM) else \
+                     vm.__class__.__name__.replace('VM','')"},
 
-        "template": {"func": "'n/a' if vm.is_template() or vm.template is None else\
-                     vm_info['template']"},
+            "updbl": {"func": "'Yes' if vm.updateable else ''"},
 
-        "netvm": {"func": "'n/a' if vm.is_netvm() and not vm.is_proxyvm() else\
-                  ('*' if vm.uses_default_netvm else '') +\
-                    vm_info['netvm'] if vm_info['netvm'] is not None else '-'"},
+            "template": {"func": "'n/a' if vm.is_template() or vm.template is None else\
+                         vm_info['template']"},
 
-        "label": {"func": "vm.label.name"},
-    }
+            "netvm": {"func": "'n/a' if vm.is_netvm() and not vm.is_proxyvm() else\
+                      ('*' if vm.property_is_default('netvm') else '') +\
+                        vm_info['netvm'] if vm_info['netvm'] is not None else '-'"},
 
-    fields_to_display = ["name", "type", "template", "updbl", "netvm", "label"]
+            "label": {"func": "vm.label.name"},
+        }
+
+        fields_to_display = ["name", "type", "template", "updbl", "netvm", "label"]
+
+        # First calculate the maximum width of each field we want to display
+        total_width = 0
+        for f in fields_to_display:
+            fields[f]["max_width"] = len(f)
+            for vm_info in restore_info.values():
+                if 'vm' in vm_info.keys():
+                    # noinspection PyUnusedLocal
+                    vm = vm_info['vm']
+                    l = len(unicode(eval(fields[f]["func"])))
+                    if l > fields[f]["max_width"]:
+                        fields[f]["max_width"] = l
+            total_width += fields[f]["max_width"]
+
+        summary = ""
+        summary += "The following VMs are included in the backup:\n"
+        summary += "\n"
+
+        # Display the header
+        for f in fields_to_display:
+            fmt = "{{0:-^{0}}}-+".format(fields[f]["max_width"] + 1)
+            summary += fmt.format('-')
+        summary += "\n"
+        for f in fields_to_display:
+            fmt = "{{0:>{0}}} |".format(fields[f]["max_width"] + 1)
+            summary += fmt.format(f)
+        summary += "\n"
+        for f in fields_to_display:
+            fmt = "{{0:-^{0}}}-+".format(fields[f]["max_width"] + 1)
+            summary += fmt.format('-')
+        summary += "\n"
 
-    # First calculate the maximum width of each field we want to display
-    total_width = 0
-    for f in fields_to_display:
-        fields[f]["max_width"] = len(f)
         for vm_info in restore_info.values():
-            if 'vm' in vm_info.keys():
-                # noinspection PyUnusedLocal
-                vm = vm_info['vm']
-                l = len(unicode(eval(fields[f]["func"])))
-                if l > fields[f]["max_width"]:
-                    fields[f]["max_width"] = l
-        total_width += fields[f]["max_width"]
+            # Skip non-VM here
+            if 'vm' not in vm_info:
+                continue
+            # noinspection PyUnusedLocal
+            vm = vm_info['vm']
+            s = ""
+            for f in fields_to_display:
+                fmt = "{{0:>{0}}} |".format(fields[f]["max_width"] + 1)
+                s += fmt.format(eval(fields[f]["func"]))
 
-    print_callback("")
-    print_callback("The following VMs are included in the backup:")
-    print_callback("")
-
-    # Display the header
-    s = ""
-    for f in fields_to_display:
-        fmt = "{{0:-^{0}}}-+".format(fields[f]["max_width"] + 1)
-        s += fmt.format('-')
-    print_callback(s)
-    s = ""
-    for f in fields_to_display:
-        fmt = "{{0:>{0}}} |".format(fields[f]["max_width"] + 1)
-        s += fmt.format(f)
-    print_callback(s)
-    s = ""
-    for f in fields_to_display:
-        fmt = "{{0:-^{0}}}-+".format(fields[f]["max_width"] + 1)
-        s += fmt.format('-')
-    print_callback(s)
-
-    for vm_info in restore_info.values():
-        # Skip non-VM here
-        if 'vm' not in vm_info:
-            continue
-        # noinspection PyUnusedLocal
-        vm = vm_info['vm']
-        s = ""
-        for f in fields_to_display:
-            fmt = "{{0:>{0}}} |".format(fields[f]["max_width"] + 1)
-            s += fmt.format(eval(fields[f]["func"]))
-
-        if 'excluded' in vm_info and vm_info['excluded']:
-            s += " <-- Excluded from restore"
-        elif 'already-exists' in vm_info:
-            s += " <-- A VM with the same name already exists on the host!"
-        elif 'missing-template' in vm_info:
-            s += " <-- No matching template on the host or in the backup found!"
-        elif 'missing-netvm' in vm_info:
-            s += " <-- No matching netvm on the host or in the backup found!"
-        else:
-            if 'orig-template' in vm_info:
-                s += " <-- Original template was '%s'" % (vm_info['orig-template'])
-            if 'rename-to' in vm_info:
-                s += " <-- Will be renamed to '%s'" % vm_info['rename-to']
-
-        print_callback(s)
-
-    if 'dom0' in restore_info.keys():
-        s = ""
-        for f in fields_to_display:
-            fmt = "{{0:>{0}}} |".format(fields[f]["max_width"] + 1)
-            if f == "name":
-                s += fmt.format("Dom0")
-            elif f == "type":
-                s += fmt.format("Home")
+            if 'excluded' in vm_info and vm_info['excluded']:
+                s += " <-- Excluded from restore"
+            elif 'already-exists' in vm_info:
+                s += " <-- A VM with the same name already exists on the host!"
+            elif 'missing-template' in vm_info:
+                s += " <-- No matching template on the host or in the backup found!"
+            elif 'missing-netvm' in vm_info:
+                s += " <-- No matching netvm on the host or in the backup found!"
             else:
-                s += fmt.format("")
-        if 'username-mismatch' in restore_info['dom0']:
-            s += " <-- username in backup and dom0 mismatch"
-        if 'ignore-username-mismatch' in restore_info['dom0']:
-            s += " (ignored)"
+                if 'orig-template' in vm_info:
+                    s += " <-- Original template was '%s'" % (vm_info['orig-template'])
+                if 'rename-to' in vm_info:
+                    s += " <-- Will be renamed to '%s'" % vm_info['rename-to']
 
-        print_callback(s)
+            summary += s + "\n"
 
+        if 'dom0' in restore_info.keys():
+            s = ""
+            for f in fields_to_display:
+                fmt = "{{0:>{0}}} |".format(fields[f]["max_width"] + 1)
+                if f == "name":
+                    s += fmt.format("Dom0")
+                elif f == "type":
+                    s += fmt.format("Home")
+                else:
+                    s += fmt.format("")
+            if 'username-mismatch' in restore_info['dom0']:
+                s += " <-- username in backup and dom0 mismatch"
+            if 'ignore-username-mismatch' in restore_info['dom0']:
+                s += " (ignored)"
 
-def backup_restore_do(restore_info,
-                      host_collection=None, print_callback=print_stdout,
-                      error_callback=print_stderr, progress_callback=None,
-                      ):
-    global running_backup_operation
+            summary += s + "\n"
 
-    # Private functions begin
-    def restore_vm_dir_v1(backup_dir, src_dir, dst_dir):
+        return summary
+
+    def _restore_vm_dir_v1(self, src_dir, dst_dir):
 
         backup_src_dir = src_dir.replace(
-            qubes.config.system_path["qubes_base_dir"], backup_dir)
+            qubes.config.system_path["qubes_base_dir"], self.backup_location)
 
         # We prefer to use Linux's cp, because it nicely handles sparse files
-        cp_retcode = subprocess.call(["cp", "-rp", "--reflink=auto", backup_src_dir, dst_dir])
+        cp_retcode = subprocess.call(
+            ["cp", "-rp", "--reflink=auto", backup_src_dir, dst_dir])
         if cp_retcode != 0:
             raise qubes.exc.QubesException(
                 "*** Error while copying file {0} to {1}".format(backup_src_dir,
                                                                  dst_dir))
 
-    # Private functions end
+    def restore_do(self, restore_info):
+        # FIXME handle locking
 
-    options = restore_info['$OPTIONS$']
-    backup_location = options['location']
-    restore_tmpdir = options['restore_tmpdir']
-    passphrase = options['passphrase']
-    encrypted = options['encrypted']
-    compressed = options['compressed']
-    compression_filter = options['compression_filter']
-    hmac_algorithm = options['hmac_algorithm']
-    crypto_algorithm = options['crypto_algorithm']
-    verify_only = options['verify-only']
-    appvm = options['appvm']
-    format_version = options['format_version']
-
-    if format_version is None:
-        format_version = backup_detect_format_version(backup_location)
-
-    lock_obtained = False
-    if host_collection is None:
-        host_collection = qubes.Qubes()
-        # FIXME
-        lock_obtained = True
-
-    # Perform VM restoration in backup order
-    vms_dirs = []
-    vms_size = 0
-    vms = {}
-    for vm_info in restore_info.values():
-        if 'vm' not in vm_info:
-            continue
-        if not vm_info['good-to-go']:
-            continue
-        vm = vm_info['vm']
-        if format_version >= 2:
-            vms_size += vm.backup_size
-            vms_dirs.append(vm.backup_path)
-        vms[vm.name] = vm
-
-    running_backup_operation = BackupOperationInfo()
-
-    if format_version >= 2:
-        if 'dom0' in restore_info.keys() and restore_info['dom0']['good-to-go']:
-            vms_dirs.append(os.path.dirname(restore_info['dom0']['subdir']))
-            vms_size += restore_info['dom0']['size']
-
-        try:
-            restore_vm_dirs(backup_location,
-                            restore_tmpdir,
-                            passphrase=passphrase,
-                            vms_dirs=vms_dirs,
-                            vms=vms,
-                            vms_size=vms_size,
-                            format_version=format_version,
-                            hmac_algorithm=hmac_algorithm,
-                            crypto_algorithm=crypto_algorithm,
-                            verify_only=verify_only,
-                            print_callback=print_callback,
-                            error_callback=error_callback,
-                            progress_callback=progress_callback,
-                            encrypted=encrypted,
-                            compressed=compressed,
-                            compression_filter=compression_filter,
-                            appvm=appvm)
-        except qubes.exc.QubesException:
-            if verify_only:
-                raise
-            else:
-                if callable(print_callback):
-                    print_callback(
-                        "Some errors occurred during data extraction, "
-                        "continuing anyway to restore at least some "
-                        "VMs")
-    else:
-        if verify_only:
-            if callable(print_callback):
-                print_callback("WARNING: Backup verification not supported for "
-                               "this backup format.")
-
-    if verify_only:
-        shutil.rmtree(restore_tmpdir)
-        return
-
-    # First load templates, then other VMs
-    for do_templates in (True, False):
-        if running_backup_operation.canceled:
-            break
-        for vm in vms.values():
-            if running_backup_operation.canceled:
-                # only break the loop to save qubes.xml with already restored
-                # VMs
-                break
-            if vm.is_template != do_templates:
+        # Perform VM restoration in backup order
+        vms_dirs = []
+        vms_size = 0
+        vms = {}
+        for vm_info in restore_info.values():
+            if 'vm' not in vm_info:
                 continue
-            if callable(print_callback):
-                print_callback("-> Restoring {0}...".format(vm.name))
-            retcode = subprocess.call(
-                ["mkdir", "-p", os.path.dirname(vm.dir_path)])
-            if retcode != 0:
-                error_callback("*** Cannot create directory: {0}?!".format(
-                    vm.dir_path))
-                error_callback("Skipping...")
+            if not vm_info['good-to-go']:
                 continue
+            vm = vm_info['vm']
+            if self.header_data.version >= 2:
+                vms_size += vm.backup_size
+                vms_dirs.append(vm.backup_path)
+            vms[vm.name] = vm
 
-            kwargs = {}
-            if hasattr(vm, 'template'):
-                if vm.template is not None:
-                    kwargs['template'] = restore_info[vm.name]['template']
+        if self.header_data.version >= 2:
+            if 'dom0' in restore_info.keys() and \
+                    restore_info['dom0']['good-to-go']:
+                vms_dirs.append(os.path.dirname(restore_info['dom0']['subdir']))
+                vms_size += restore_info['dom0']['size']
+
+            try:
+                self._restore_vm_dirs(vms_dirs=vms_dirs, vms_size=vms_size)
+            except qubes.exc.QubesException:
+                if self.options.verify_only:
+                    raise
                 else:
-                    kwargs['template'] = None
+                    if callable(self.print_callback):
+                        self.print_callback(
+                            "Some errors occurred during data extraction, "
+                            "continuing anyway to restore at least some "
+                            "VMs")
+        else:
+            if self.options.verify_only:
+                if callable(self.print_callback):
+                    self.print_callback("WARNING: Backup verification not "
+                                    "supported for "
+                                   "this backup format.")
 
-            new_vm = None
+        if self.options.verify_only:
+            shutil.rmtree(self.tmpdir)
+            return
+
+        # First load templates, then other VMs
+        for do_templates in (True, False):
+            if self.canceled:
+                break
+            for vm in vms.values():
+                if self.canceled:
+                    # only break the loop to save qubes.xml with already restored
+                    # VMs
+                    break
+                if vm.is_template() != do_templates:
+                    continue
+                if callable(self.print_callback):
+                    self.print_callback("-> Restoring {0}...".format(vm.name))
+                retcode = subprocess.call(
+                    ["mkdir", "-p", os.path.dirname(vm.dir_path)])
+                if retcode != 0:
+                    self.error_callback("*** Cannot create directory: {"
+                                     "0}?!".format(
+                        vm.dir_path))
+                    self.error_callback("Skipping...")
+                    continue
+
+                kwargs = {}
+                if hasattr(vm, 'template'):
+                    if vm.template is not None:
+                        kwargs['template'] = restore_info[vm.name]['template']
+                    else:
+                        kwargs['template'] = None
+
+                new_vm = None
+                vm_name = vm.name
+                if 'rename-to' in restore_info[vm.name]:
+                    vm_name = restore_info[vm.name]['rename-to']
+
+                try:
+                    # first only minimal set, later clone_properties will be called
+                    new_vm = self.app.add_new_vm(
+                        vm.__class__,
+                        name=vm_name,
+                        label=vm.label,
+                        installed_by_rpm=False,
+                        **kwargs)
+                    if os.path.exists(new_vm.dir_path):
+                        move_to_path = tempfile.mkdtemp('', os.path.basename(
+                            new_vm.dir_path), os.path.dirname(new_vm.dir_path))
+                        try:
+                            os.rename(new_vm.dir_path, move_to_path)
+                            self.error_callback(
+                                "*** Directory {} already exists! It has "
+                                "been moved to {}".format(new_vm.dir_path,
+                                                          move_to_path))
+                        except OSError:
+                            self.error_callback(
+                                "*** Directory {} already exists and "
+                                "cannot be moved!".format(new_vm.dir_path))
+                            self.error_callback("Skipping...")
+                            continue
+
+                    if self.header_data.version == 1:
+                        self._restore_vm_dir_v1(vm.dir_path,
+                            os.path.dirname(new_vm.dir_path))
+                    else:
+                        shutil.move(os.path.join(self.tmpdir, vm.backup_path),
+                                    new_vm.dir_path)
+
+                    new_vm.verify_files()
+                except Exception as err:
+                    self.error_callback("ERROR: {0}".format(err))
+                    self.error_callback("*** Skipping VM: {0}".format(vm.name))
+                    if new_vm:
+                        del self.app.domains[new_vm.qid]
+                    continue
+
+                if hasattr(vm, 'kernel'):
+                    # TODO: add a setting for this?
+                    if not vm.property_is_default('kernel') and \
+                            vm.kernel not in \
+                            os.listdir(qubes.config.system_path[
+                                'qubes_kernels_base_dir']):
+                        if callable(self.print_callback):
+                            self.print_callback("WARNING: Kernel %s not "
+                                             "installed, "
+                                           "using default one" % vm.kernel)
+                        vm.kernel = qubes.property.DEFAULT
+                try:
+                    new_vm.clone_properties(vm)
+                except Exception as err:
+                    self.error_callback("ERROR: {0}".format(err))
+                    self.error_callback("*** Some VM property will not be "
+                                     "restored")
+
+                try:
+                    new_vm.fire_event('domain-restore')
+                except Exception as err:
+                    self.error_callback("ERROR during appmenu restore: {"
+                                    "0}".format(err))
+                    self.error_callback(
+                        "*** VM '{0}' will not have appmenus".format(vm.name))
+
+        # Set network dependencies - only non-default netvm setting
+        for vm in vms.values():
             vm_name = vm.name
             if 'rename-to' in restore_info[vm.name]:
                 vm_name = restore_info[vm.name]['rename-to']
-
             try:
-                # first only minimal set, later clone_properties will be called
-                new_vm = host_collection.add_new_vm(
-                    vm.__class__,
-                    name=vm_name,
-                    label=vm.label,
-                    installed_by_rpm=False,
-                    **kwargs)
-                if os.path.exists(new_vm.dir_path):
-                    move_to_path = tempfile.mkdtemp('', os.path.basename(
-                        new_vm.dir_path), os.path.dirname(new_vm.dir_path))
-                    try:
-                        os.rename(new_vm.dir_path, move_to_path)
-                        error_callback(
-                            "*** Directory {} already exists! It has "
-                            "been moved to {}".format(new_vm.dir_path,
-                                                      move_to_path))
-                    except OSError:
-                        error_callback(
-                            "*** Directory {} already exists and "
-                            "cannot be moved!".format(new_vm.dir_path))
-                        error_callback("Skipping...")
-                        continue
-
-                if format_version == 1:
-                    restore_vm_dir_v1(backup_location,
-                                      vm.dir_path,
-                                      os.path.dirname(new_vm.dir_path))
-                elif format_version >= 2:
-                    shutil.move(os.path.join(restore_tmpdir, vm.backup_path),
-                                new_vm.dir_path)
-
-                new_vm.verify_files()
-            except Exception as err:
-                error_callback("ERROR: {0}".format(err))
-                error_callback("*** Skipping VM: {0}".format(vm.name))
-                if new_vm:
-                    del host_collection.domains[new_vm.qid]
+                host_vm = self.app.domains[vm_name]
+            except KeyError:
+                # Failed/skipped VM
                 continue
 
-            if hasattr(vm, 'kernel'):
-                # TODO: add a setting for this?
-                if not vm.is_property_default('kernel') and \
-                        vm.kernel not in \
-                        os.listdir(qubes.config.system_path[
-                            'qubes_kernels_base_dir']):
-                    if callable(print_callback):
-                        print_callback("WARNING: Kernel %s not installed, "
-                                       "using default one" % vm.kernel)
-                    vm.kernel = qubes.property.DEFAULT
-            try:
-                new_vm.clone_attrs(vm)
-            except Exception as err:
-                error_callback("ERROR: {0}".format(err))
-                error_callback("*** Some VM property will not be restored")
+            if not vm.property_is_default('netvm'):
+                if restore_info[vm.name]['netvm'] is not None:
+                    host_vm.netvm = restore_info[vm.name]['netvm']
+                else:
+                    host_vm.netvm = None
 
-            try:
-                new_vm.appmenus_create()
-            except Exception as err:
-                error_callback("ERROR during appmenu restore: {0}".format(err))
-                error_callback(
-                    "*** VM '{0}' will not have appmenus".format(vm.name))
+        self.app.save()
 
-    # Set network dependencies - only non-default netvm setting
-    for vm in vms.values():
-        vm_name = vm.name
-        if 'rename-to' in restore_info[vm.name]:
-            vm_name = restore_info[vm.name]['rename-to']
-        try:
-            host_vm = host_collection.domains[vm_name]
-        except KeyError:
-            # Failed/skipped VM
-            continue
-
-        if not vm.is_property_default('netvm'):
-            if restore_info[vm.name]['netvm'] is not None:
-                host_vm.netvm = restore_info[vm.name]['netvm']
+        if self.canceled:
+            if self.header_data.version >= 2:
+                raise BackupCanceledError("Restore canceled",
+                                          tmpdir=self.tmpdir)
             else:
-                host_vm.netvm = None
+                raise BackupCanceledError("Restore canceled")
 
-    host_collection.save()
+        # ... and dom0 home as last step
+        if 'dom0' in restore_info.keys() and restore_info['dom0']['good-to-go']:
+            backup_path = restore_info['dom0']['subdir']
+            local_user = grp.getgrnam('qubes').gr_mem[0]
+            home_dir = pwd.getpwnam(local_user).pw_dir
+            if self.header_data.version == 1:
+                backup_dom0_home_dir = os.path.join(self.backup_location,
+                    backup_path)
+            else:
+                backup_dom0_home_dir = os.path.join(self.tmpdir, backup_path)
+            restore_home_backupdir = "home-pre-restore-{0}".format(
+                time.strftime("%Y-%m-%d-%H%M%S"))
 
-    if running_backup_operation.canceled:
-        if format_version >= 2:
-            raise BackupCanceledError("Restore canceled",
-                                      tmpdir=restore_tmpdir)
-        else:
-            raise BackupCanceledError("Restore canceled")
+            if callable(self.print_callback):
+                self.print_callback(
+                    "-> Restoring home of user '{0}'...".format(local_user))
+                self.print_callback(
+                    "--> Existing files/dirs backed up in '{0}' dir".format(
+                        restore_home_backupdir))
+            os.mkdir(home_dir + '/' + restore_home_backupdir)
+            for f in os.listdir(backup_dom0_home_dir):
+                home_file = home_dir + '/' + f
+                if os.path.exists(home_file):
+                    os.rename(home_file,
+                              home_dir + '/' + restore_home_backupdir + '/' + f)
+                if self.header_data.version == 1:
+                    subprocess.call(
+                        ["cp", "-nrp", "--reflink=auto", backup_dom0_home_dir + '/' + f, home_file])
+                elif self.header_data.version >= 2:
+                    shutil.move(backup_dom0_home_dir + '/' + f, home_file)
+            retcode = subprocess.call(['sudo', 'chown', '-R', local_user, home_dir])
+            if retcode != 0:
+                self.error_callback("*** Error while setting home directory "
+                                  "owner")
 
-    # ... and dom0 home as last step
-    if 'dom0' in restore_info.keys() and restore_info['dom0']['good-to-go']:
-        backup_path = restore_info['dom0']['subdir']
-        local_user = grp.getgrnam('qubes').gr_mem[0]
-        home_dir = pwd.getpwnam(local_user).pw_dir
-        if format_version == 1:
-            backup_dom0_home_dir = os.path.join(backup_location, backup_path)
-        else:
-            backup_dom0_home_dir = os.path.join(restore_tmpdir, backup_path)
-        restore_home_backupdir = "home-pre-restore-{0}".format(
-            time.strftime("%Y-%m-%d-%H%M%S"))
-
-        if callable(print_callback):
-            print_callback(
-                "-> Restoring home of user '{0}'...".format(local_user))
-            print_callback(
-                "--> Existing files/dirs backed up in '{0}' dir".format(
-                    restore_home_backupdir))
-        os.mkdir(home_dir + '/' + restore_home_backupdir)
-        for f in os.listdir(backup_dom0_home_dir):
-            home_file = home_dir + '/' + f
-            if os.path.exists(home_file):
-                os.rename(home_file,
-                          home_dir + '/' + restore_home_backupdir + '/' + f)
-            if format_version == 1:
-                subprocess.call(
-                    ["cp", "-nrp", "--reflink=auto", backup_dom0_home_dir + '/' + f, home_file])
-            elif format_version >= 2:
-                shutil.move(backup_dom0_home_dir + '/' + f, home_file)
-        retcode = subprocess.call(['sudo', 'chown', '-R', local_user, home_dir])
-        if retcode != 0:
-            error_callback("*** Error while setting home directory owner")
-
-    shutil.rmtree(restore_tmpdir)
+        shutil.rmtree(self.tmpdir)
 
 # vim:sw=4:et:
diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py
index 3c13f701..53b5500e 100644
--- a/qubes/tests/__init__.py
+++ b/qubes/tests/__init__.py
@@ -717,25 +717,22 @@ class BackupTestsMixin(SystemTestsMixin):
 
         return vms
 
-    def make_backup(self, vms, prepare_kwargs=dict(), do_kwargs=dict(),
-                    target=None, expect_failure=False):
+    def make_backup(self, vms, target=None, expect_failure=False, **kwargs):
         if target is None:
             target = self.backupdir
         try:
-            files_to_backup = \
-                qubes.backup.backup_prepare(self.app, vms,
-                                      print_callback=self.print_callback,
-                                      **prepare_kwargs)
+            backup = qubes.backup.Backup(self.app, vms, **kwargs)
         except qubes.exc.QubesException as e:
             if not expect_failure:
                 self.fail("QubesException during backup_prepare: %s" % str(e))
             else:
                 raise
 
+        backup.passphrase = 'qubes'
+        backup.target_dir = target
+
         try:
-            qubes.backup.backup_do(self.app, target, files_to_backup, "qubes",
-                             progress_callback=self.print_progress,
-                             **do_kwargs)
+            backup.backup_do()
         except qubes.exc.QubesException as e:
             if not expect_failure:
                 self.fail("QubesException during backup_do: %s" % str(e))
@@ -754,22 +751,17 @@ class BackupTestsMixin(SystemTestsMixin):
             backupfile = source
 
         with self.assertNotRaises(qubes.exc.QubesException):
-            backup_info = qubes.backup.backup_restore_prepare(
-                backupfile, "qubes",
-                host_collection=self.app,
-                print_callback=self.print_callback,
-                appvm=appvm,
-                options=options or {})
-
+            restore_op = qubes.backup.BackupRestore(
+                self.app, backupfile, appvm, "qubes")
+            if options:
+                for key, value in options.iteritems():
+                    setattr(restore_op.options, key, value)
+            restore_info = restore_op.get_restore_info()
         if self.verbose:
-            qubes.backup.backup_restore_print_summary(backup_info)
+            print restore_op.get_restore_summary(restore_info)
 
         with self.assertNotRaises(qubes.exc.QubesException):
-            qubes.backup.backup_restore_do(
-                backup_info,
-                host_collection=self.app,
-                print_callback=self.print_callback if self.verbose else None,
-                error_callback=self.error_callback)
+            restore_op.restore_do(restore_info)
 
         # maybe someone forgot to call .save()
         self.reload_db()
diff --git a/qubes/tests/int/backup.py b/qubes/tests/int/backup.py
index d47492b4..2acb2754 100644
--- a/qubes/tests/int/backup.py
+++ b/qubes/tests/int/backup.py
@@ -45,7 +45,7 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
 
     def test_001_compressed_backup(self):
         vms = self.create_backup_vms()
-        self.make_backup(vms, do_kwargs={'compressed': True})
+        self.make_backup(vms, compressed=True)
         self.remove_vms(vms)
         self.restore_backup()
         for vm in vms:
@@ -53,7 +53,7 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
 
     def test_002_encrypted_backup(self):
         vms = self.create_backup_vms()
-        self.make_backup(vms, do_kwargs={'encrypted': True})
+        self.make_backup(vms, encrypted=True)
         self.remove_vms(vms)
         self.restore_backup()
         for vm in vms:
@@ -61,10 +61,7 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
 
     def test_003_compressed_encrypted_backup(self):
         vms = self.create_backup_vms()
-        self.make_backup(vms,
-                         do_kwargs={
-                             'compressed': True,
-                             'encrypted': True})
+        self.make_backup(vms, compressed=True, encrypted=True)
         self.remove_vms(vms)
         self.restore_backup()
         for vm in vms:
@@ -96,7 +93,7 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
 
     def test_005_compressed_custom(self):
         vms = self.create_backup_vms()
-        self.make_backup(vms, do_kwargs={'compressed': "bzip2"})
+        self.make_backup(vms, compressed="bzip2")
         self.remove_vms(vms)
         self.restore_backup()
         for vm in vms:
@@ -135,16 +132,16 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
         vms = self.create_backup_vms()
         self.make_backup(vms)
         self.restore_backup(options={
-            'rename-conflicting': True
+            'rename_conflicting': True
         })
         for vm in vms:
-            with self.assertNotRaises(qubes.exc.QubesVMNotFoundError):
+            with self.assertNotRaises(
+                    (qubes.exc.QubesVMNotFoundError, KeyError)):
                 restored_vm = self.app.domains[vm.name + '1']
             if vm.netvm and not vm.property_is_default('netvm'):
                 self.assertEqual(restored_vm.netvm.name, vm.netvm.name + '1')
 
 
-
 class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin):
     def setUp(self):
         super(TC_10_BackupVMMixin, self).setUp()
@@ -160,12 +157,9 @@ class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin):
         vms = self.create_backup_vms()
         self.backupvm.start()
         self.backupvm.run("mkdir '/var/tmp/backup directory'", wait=True)
-        self.make_backup(vms,
-                         do_kwargs={
-                             'appvm': self.backupvm,
-                             'compressed': True,
-                             'encrypted': True},
-                         target='/var/tmp/backup directory')
+        self.make_backup(vms, target_vm=self.backupvm,
+            compressed=True, encrypted=True,
+            target='/var/tmp/backup directory')
         self.remove_vms(vms)
         p = self.backupvm.run("ls /var/tmp/backup*/qubes-backup*",
                               passio_popen=True)
@@ -177,12 +171,9 @@ class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin):
     def test_110_send_to_vm_command(self):
         vms = self.create_backup_vms()
         self.backupvm.start()
-        self.make_backup(vms,
-                         do_kwargs={
-                             'appvm': self.backupvm,
-                             'compressed': True,
-                             'encrypted': True},
-                         target='dd of=/var/tmp/backup-test')
+        self.make_backup(vms, target_vm=self.backupvm,
+            compressed=True, encrypted=True,
+            target='dd of=/var/tmp/backup-test')
         self.remove_vms(vms)
         self.restore_backup(source='dd if=/var/tmp/backup-test',
                             appvm=self.backupvm)
@@ -205,13 +196,10 @@ class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin):
         if retcode != 0:
             raise RuntimeError("Failed to prepare backup directory")
         with self.assertRaises(qubes.exc.QubesException):
-            self.make_backup(vms,
-                             do_kwargs={
-                                 'appvm': self.backupvm,
-                                 'compressed': False,
-                                 'encrypted': True},
-                             target='/home/user/backup',
-                             expect_failure=True)
+            self.make_backup(vms, target_vm=self.backupvm,
+                compressed=False, encrypted=True,
+                target='/home/user/backup',
+                expect_failure=True)
 
 
 def load_tests(loader, tests, pattern):

From 019678bf4d2b17817b33a06bda2d7f98f0f985f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sun, 13 Mar 2016 14:03:53 +0100
Subject: [PATCH 04/32] tests: fix backup test to not break default template...

By default AppVM is template based. This means vm.root_img points at
default template's root image. Change this to StandaloneVM to have
independent root.img.
---
 qubes/tests/__init__.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py
index 53b5500e..5060ced5 100644
--- a/qubes/tests/__init__.py
+++ b/qubes/tests/__init__.py
@@ -42,6 +42,7 @@ import qubes.config
 import qubes.events
 import qubes.backup
 import qubes.exc
+import qubes.vm.standalonevm
 
 XMLPATH = '/var/lib/qubes/qubes-test.xml'
 CLASS_XMLPATH = '/var/lib/qubes/qubes-class-test.xml'
@@ -707,7 +708,8 @@ class BackupTestsMixin(SystemTestsMixin):
         vmname = self.make_vm_name('testhvm1')
         if self.verbose:
             print >>sys.stderr, "-> Creating %s" % vmname
-        testvm2 = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=vmname,
+        testvm2 = self.app.add_new_vm(qubes.vm.standalonevm.StandaloneVM,
+            name=vmname,
             hvm=True, label='red')
         testvm2.create_on_disk()
         self.fill_image(testvm2.root_img, 1024*1024*1024, True)

From 3342f637f4226d8ac1dc6983ba9a311bcc7f1438 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 14 Mar 2016 12:04:07 +0100
Subject: [PATCH 05/32] backup: convert logging to proper python logging API

QubesOS/qubes-issues#1213
QubesOS/qubes-issues#1214
---
 qubes/backup.py         | 293 ++++++++++++++++------------------------
 qubes/tests/__init__.py |  27 ++--
 2 files changed, 132 insertions(+), 188 deletions(-)

diff --git a/qubes/backup.py b/qubes/backup.py
index b201b7fb..6a752ffe 100644
--- a/qubes/backup.py
+++ b/qubes/backup.py
@@ -24,6 +24,7 @@
 #
 from __future__ import unicode_literals
 import itertools
+import logging
 from qubes.utils import size_to_human
 import sys
 import os
@@ -40,8 +41,6 @@ import datetime
 from multiprocessing import Queue, Process
 import qubes
 
-BACKUP_DEBUG = False
-
 HEADER_FILENAME = 'backup-header'
 DEFAULT_CRYPTO_ALGORITHM = 'aes-256-cbc'
 DEFAULT_HMAC_ALGORITHM = 'SHA512'
@@ -55,11 +54,6 @@ HEADER_QUBES_XML_MAX_SIZE = 1024 * 1024
 
 BLKSIZE = 512
 
-def print_stdout(text):
-    print (text)
-
-def print_stderr(text):
-    print >> sys.stderr, (text)
 
 def get_disk_usage_one(st):
     try:
@@ -182,21 +176,19 @@ class SendWorker(Process):
         self.queue = queue
         self.base_dir = base_dir
         self.backup_stdout = backup_stdout
+        self.log = logging.getLogger('qubes.backup')
 
     def run(self):
-        if BACKUP_DEBUG:
-            print "Started sending thread"
+        self.log.debug("Started sending thread")
 
-        if BACKUP_DEBUG:
-            print "Moving to temporary dir", self.base_dir
+        self.log.debug("Moving to temporary dir".format(self.base_dir))
         os.chdir(self.base_dir)
 
         for filename in iter(self.queue.get, None):
             if filename == "FINISHED" or filename == "ERROR":
                 break
 
-            if BACKUP_DEBUG:
-                print "Sending file", filename
+            self.log.debug("Sending file {}".format(filename))
             # This tar used for sending data out need to be as simple, as
             # simple, as featureless as possible. It will not be
             # verified before untaring.
@@ -217,12 +209,10 @@ class SendWorker(Process):
                     "Check console output or ~/.xsession-errors for details.")
 
             # Delete the file as we don't need it anymore
-            if BACKUP_DEBUG:
-                print "Removing file", filename
+            self.log.debug("Removing file {}".format(filename))
             os.remove(filename)
 
-        if BACKUP_DEBUG:
-            print "Finished sending thread"
+        self.log.debug("Finished sending thread")
 
 
 class Backup(object):
@@ -278,6 +268,8 @@ class Backup(object):
         #: whether backup was canceled
         self.canceled = False
 
+        self.log = logging.getLogger('qubes.backup')
+
         # FIXME: drop this legacy feature?
         if isinstance(self.compressed, basestring):
             self.compression_filter = self.compressed
@@ -631,16 +623,13 @@ class Backup(object):
         # Tar with tape length does not deals well with stdout (close stdout between
         # two tapes)
         # For this reason, we will use named pipes instead
-        if BACKUP_DEBUG:
-            print "Working in", self.tmpdir
+        self.log.debug("Working in {}".format(self.tmpdir))
 
         backup_pipe = os.path.join(self.tmpdir, "backup_pipe")
-        if BACKUP_DEBUG:
-            print "Creating pipe in:", backup_pipe
+        self.log.debug("Creating pipe in: {}".format(backup_pipe))
         os.mkfifo(backup_pipe)
 
-        if BACKUP_DEBUG:
-            print "Will backup:", files_to_backup
+        self.log.debug("Will backup: {}".format(files_to_backup))
 
         header_files = self.prepare_backup_header()
 
@@ -663,14 +652,13 @@ class Backup(object):
                 vm_files_to_backup.itervalues()):
             for file_info in vm_info['files']:
 
-                if BACKUP_DEBUG:
-                    print "Backing up", file_info
+                self.log.debug("Backing up {}".format(file_info))
 
                 backup_tempfile = os.path.join(self.tmpdir,
                                                file_info["subdir"],
                                                os.path.basename(file_info["path"]))
-                if BACKUP_DEBUG:
-                    print "Using temporary location:", backup_tempfile
+                self.log.debug("Using temporary location: {}".format(
+                    backup_tempfile))
 
                 # Ensure the temporary directory exists
                 if not os.path.isdir(os.path.dirname(backup_tempfile)):
@@ -694,16 +682,14 @@ class Backup(object):
                     tar_cmdline.insert(-1,
                         "--use-compress-program=%s" % self.compression_filter)
 
-                if BACKUP_DEBUG:
-                    print " ".join(tar_cmdline)
+                self.log.debug(" ".join(tar_cmdline))
 
                 # Tips: Popen(bufsize=0)
                 # Pipe: tar-sparse | encryptor [| hmac] | tar | backup_target
                 # Pipe: tar-sparse [| hmac] | tar | backup_target
-                tar_sparse = subprocess.Popen(tar_cmdline, stdin=subprocess.PIPE,
-                                              stderr=(open(os.devnull, 'w')
-                                                      if not BACKUP_DEBUG
-                                                      else None))
+                # TODO: log handle stderr
+                tar_sparse = subprocess.Popen(
+                    tar_cmdline, stdin=subprocess.PIPE)
                 self.processes_to_kill_on_cancel.append(tar_sparse)
 
                 # Wait for compressor (tar) process to finish or for any error of other
@@ -750,8 +736,8 @@ class Backup(object):
                         **common_args)
                     chunkfile_p.close()
 
-                    if BACKUP_DEBUG:
-                        print "Wait_backup_feedback returned:", run_error
+                    self.log.debug(
+                        "Wait_backup_feedback returned: {}".format(run_error))
 
                     if self.canceled:
                         try:
@@ -786,13 +772,13 @@ class Backup(object):
                     # Close HMAC
                     hmac.stdin.close()
                     hmac.wait()
-                    if BACKUP_DEBUG:
-                        print "HMAC proc return code:", hmac.poll()
+                    self.log.debug("HMAC proc return code: {}".format(
+                        hmac.poll()))
 
                     # Write HMAC data next to the chunk file
                     hmac_data = hmac.stdout.read()
-                    if BACKUP_DEBUG:
-                        print "Writing hmac to", chunkfile + ".hmac"
+                    self.log.debug(
+                        "Writing hmac to {}.hmac".format(chunkfile))
                     hmac_file = open(chunkfile + ".hmac", 'w')
                     hmac_file.write(hmac_data)
                     hmac_file.flush()
@@ -807,9 +793,9 @@ class Backup(object):
                         run_error = "paused"
                     else:
                         self.processes_to_kill_on_cancel.remove(tar_sparse)
-                        if BACKUP_DEBUG:
-                            print "Finished tar sparse with exit code", tar_sparse \
-                                .poll()
+                        self.log.debug(
+                            "Finished tar sparse with exit code {}".format(
+                                tar_sparse.poll()))
                 pipe.close()
 
             # This VM done, update progress
@@ -829,10 +815,10 @@ class Backup(object):
                 "Failed to send backup: error in the sending process")
 
         if vmproc:
-            if BACKUP_DEBUG:
-                print "VMProc1 proc return code:", vmproc.poll()
-                if tar_sparse is not None:
-                    print "Sparse1 proc return code:", tar_sparse.poll()
+            self.log.debug("VMProc1 proc return code: {}".format(vmproc.poll()))
+            if tar_sparse is not None:
+                self.log.debug("Sparse1 proc return code: {}".format(
+                    tar_sparse.poll()))
             vmproc.stdin.close()
 
         # Save date of last backup
@@ -869,6 +855,7 @@ def wait_backup_feedback(progress_callback, in_stream, streamproc,
     run_error = None
     run_count = 1
     bytes_copied = 0
+    log = logging.getLogger('qubes.backup')
     while run_count > 0 and run_error is None:
 
         if size_limit and bytes_copied + buffer_size > size_limit:
@@ -900,8 +887,7 @@ def wait_backup_feedback(progress_callback, in_stream, streamproc,
             if retcode is not None:
                 if retcode != 0:
                     run_error = "VM"
-                    if BACKUP_DEBUG:
-                        print vmproc.stdout.read()
+                    log.debug(vmproc.stdout.read())
             else:
                 # VM should run until the end
                 pass
@@ -935,7 +921,7 @@ def wait_backup_feedback(progress_callback, in_stream, streamproc,
 
 class ExtractWorker2(Process):
     def __init__(self, queue, base_dir, passphrase, encrypted,
-                 print_callback, error_callback, progress_callback, vmproc=None,
+                 progress_callback, vmproc=None,
                  compressed=False, crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM,
                  verify_only=False):
         super(ExtractWorker2, self).__init__()
@@ -952,15 +938,14 @@ class ExtractWorker2(Process):
         self.decompressor_process = None
         self.decryptor_process = None
 
-        self.print_callback = print_callback
-        self.error_callback = error_callback
         self.progress_callback = progress_callback
 
         self.vmproc = vmproc
 
         self.restore_pipe = os.path.join(self.base_dir, "restore_pipe")
-        if BACKUP_DEBUG:
-            print "Creating pipe in:", self.restore_pipe
+
+        self.log = logging.getLogger('qubes.backup.extract')
+        self.log.debug("Creating pipe in: {}".format(self.restore_pipe))
         os.mkfifo(self.restore_pipe)
 
         self.stderr_encoding = sys.stderr.encoding or 'utf-8'
@@ -983,9 +968,10 @@ class ExtractWorker2(Process):
 
         new_lines = map(lambda x: x.decode(self.stderr_encoding), new_lines)
 
-        if not BACKUP_DEBUG:
-            msg_re = re.compile(r".*#[0-9].*restore_pipe")
-            new_lines = filter(lambda x: not msg_re.match(x), new_lines)
+        msg_re = re.compile(r".*#[0-9].*restore_pipe")
+        debug_msg = filter(msg_re.match, new_lines)
+        self.log.debug('tar2_stderr: {}'.format('\n'.join(debug_msg)))
+        new_lines = filter(lambda x: not msg_re.match(x), new_lines)
 
         self.tar2_stderr += new_lines
 
@@ -1005,13 +991,12 @@ class ExtractWorker2(Process):
                     except OSError:
                         pass
                     process.wait()
-            self.error_callback("ERROR: " + unicode(e))
+            self.log.error("ERROR: " + unicode(e))
             raise e, None, exc_traceback
 
     def __run__(self):
-        if BACKUP_DEBUG and callable(self.print_callback):
-            self.print_callback("Started sending thread")
-            self.print_callback("Moving to dir " + self.base_dir)
+        self.log.debug("Started sending thread")
+        self.log.debug("Moving to dir " + self.base_dir)
         os.chdir(self.base_dir)
 
         filename = None
@@ -1020,15 +1005,14 @@ class ExtractWorker2(Process):
             if filename == "FINISHED" or filename == "ERROR":
                 break
 
-            if BACKUP_DEBUG and callable(self.print_callback):
-                self.print_callback("Extracting file " + filename)
+            self.log.debug("Extracting file " + filename)
 
             if filename.endswith('.000'):
                 # next file
                 if self.tar2_process is not None:
                     if self.tar2_process.wait() != 0:
                         self.collect_tar_output()
-                        self.error_callback(
+                        self.log.error(
                             "ERROR: unable to extract files for {0}, tar "
                             "output:\n  {1}".
                             format(self.tar2_current_file,
@@ -1039,13 +1023,10 @@ class ExtractWorker2(Process):
                         self.tar2_current_file = None
 
                 tar2_cmdline = ['tar',
-                                '-%sMk%sf' % ("t" if self.verify_only else "x",
-                                              "v" if BACKUP_DEBUG else ""),
+                                '-%sMkvf' % ("t" if self.verify_only else "x"),
                                 self.restore_pipe,
                                 os.path.relpath(filename.rstrip('.000'))]
-                if BACKUP_DEBUG and callable(self.print_callback):
-                    self.print_callback("Running command " +
-                                        unicode(tar2_cmdline))
+                self.log.debug("Running command " + unicode(tar2_cmdline))
                 self.tar2_process = subprocess.Popen(tar2_cmdline,
                                                      stdin=subprocess.PIPE,
                                                      stderr=subprocess.PIPE)
@@ -1056,13 +1037,12 @@ class ExtractWorker2(Process):
             elif not self.tar2_process:
                 # Extracting of the current archive failed, skip to the next
                 # archive
-                if not BACKUP_DEBUG:
-                    os.remove(filename)
+                # TODO: some debug option to preserve it?
+                os.remove(filename)
                 continue
             else:
                 self.collect_tar_output()
-                if BACKUP_DEBUG and callable(self.print_callback):
-                    self.print_callback("Releasing next chunck")
+                self.log.debug("Releasing next chunck")
                 self.tar2_process.stdin.write("\n")
                 self.tar2_process.stdin.flush()
             self.tar2_current_file = filename
@@ -1112,11 +1092,10 @@ class ExtractWorker2(Process):
                 pipe.close()
             except IOError as e:
                 if e.errno == errno.EPIPE:
-                    if BACKUP_DEBUG:
-                        self.error_callback(
-                            "Got EPIPE while closing pipe to "
-                            "the inner tar process")
-                        # ignore the error
+                    self.log.debug(
+                        "Got EPIPE while closing pipe to "
+                        "the inner tar process")
+                    # ignore the error
                 else:
                     raise
             if len(run_error):
@@ -1128,12 +1107,11 @@ class ExtractWorker2(Process):
                 self.tar2_process.terminate()
                 self.tar2_process.wait()
                 self.tar2_process = None
-                self.error_callback("Error while processing '%s': %s " %
+                self.log.error("Error while processing '%s': %s " %
                                     (self.tar2_current_file, details))
 
             # Delete the file as we don't need it anymore
-            if BACKUP_DEBUG and callable(self.print_callback):
-                self.print_callback("Removing file " + filename)
+            self.log.debug("Removing file " + filename)
             os.remove(filename)
 
         os.unlink(self.restore_pipe)
@@ -1155,18 +1133,16 @@ class ExtractWorker2(Process):
                 # Finished extracting the tar file
                 self.tar2_process = None
 
-        if BACKUP_DEBUG and callable(self.print_callback):
-            self.print_callback("Finished extracting thread")
+        self.log.debug("Finished extracting thread")
 
 
 class ExtractWorker3(ExtractWorker2):
     def __init__(self, queue, base_dir, passphrase, encrypted,
-                 print_callback, error_callback, progress_callback, vmproc=None,
+                 progress_callback, vmproc=None,
                  compressed=False, crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM,
                  compression_filter=None, verify_only=False):
         super(ExtractWorker3, self).__init__(queue, base_dir, passphrase,
                                              encrypted,
-                                             print_callback, error_callback,
                                              progress_callback, vmproc,
                                              compressed, crypto_algorithm,
                                              verify_only)
@@ -1174,9 +1150,8 @@ class ExtractWorker3(ExtractWorker2):
         os.unlink(self.restore_pipe)
 
     def __run__(self):
-        if BACKUP_DEBUG and callable(self.print_callback):
-            self.print_callback("Started sending thread")
-            self.print_callback("Moving to dir " + self.base_dir)
+        self.log.debug("Started sending thread")
+        self.log.debug("Moving to dir " + self.base_dir)
         os.chdir(self.base_dir)
 
         filename = None
@@ -1186,8 +1161,7 @@ class ExtractWorker3(ExtractWorker2):
             if filename == "FINISHED" or filename == "ERROR":
                 break
 
-            if BACKUP_DEBUG and callable(self.print_callback):
-                self.print_callback("Extracting file " + filename)
+            self.log.debug("Extracting file " + filename)
 
             if filename.endswith('.000'):
                 # next file
@@ -1195,7 +1169,7 @@ class ExtractWorker3(ExtractWorker2):
                     input_pipe.close()
                     if self.tar2_process.wait() != 0:
                         self.collect_tar_output()
-                        self.error_callback(
+                        self.log.error(
                             "ERROR: unable to extract files for {0}, tar "
                             "output:\n  {1}".
                             format(self.tar2_current_file,
@@ -1206,8 +1180,7 @@ class ExtractWorker3(ExtractWorker2):
                         self.tar2_current_file = None
 
                 tar2_cmdline = ['tar',
-                                '-%sk%s' % ("t" if self.verify_only else "x",
-                                            "v" if BACKUP_DEBUG else ""),
+                                '-%sk' % ("t" if self.verify_only else "x"),
                                 os.path.relpath(filename.rstrip('.000'))]
                 if self.compressed:
                     if self.compression_filter:
@@ -1218,9 +1191,7 @@ class ExtractWorker3(ExtractWorker2):
                         tar2_cmdline.insert(-1, "--use-compress-program=%s" %
                                             DEFAULT_COMPRESSION_FILTER)
 
-                if BACKUP_DEBUG and callable(self.print_callback):
-                    self.print_callback("Running command " +
-                                        unicode(tar2_cmdline))
+                self.log.debug("Running command " + unicode(tar2_cmdline))
                 if self.encrypted:
                     # Start decrypt
                     self.decryptor_process = subprocess.Popen(
@@ -1251,12 +1222,11 @@ class ExtractWorker3(ExtractWorker2):
             elif not self.tar2_process:
                 # Extracting of the current archive failed, skip to the next
                 # archive
-                if not BACKUP_DEBUG:
-                    os.remove(filename)
+                # TODO: some debug option to preserve it?
+                os.remove(filename)
                 continue
             else:
-                if BACKUP_DEBUG and callable(self.print_callback):
-                    self.print_callback("Releasing next chunck")
+                self.log.debug("Releasing next chunck")
             self.tar2_current_file = filename
 
             common_args = {
@@ -1284,12 +1254,11 @@ class ExtractWorker3(ExtractWorker2):
                 self.tar2_process.terminate()
                 self.tar2_process.wait()
                 self.tar2_process = None
-                self.error_callback("Error while processing '%s': %s " %
+                self.log.error("Error while processing '%s': %s " %
                                     (self.tar2_current_file, details))
 
             # Delete the file as we don't need it anymore
-            if BACKUP_DEBUG and callable(self.print_callback):
-                self.print_callback("Removing file " + filename)
+            self.log.debug("Removing file " + filename)
             os.remove(filename)
 
         if self.tar2_process is not None:
@@ -1314,8 +1283,7 @@ class ExtractWorker3(ExtractWorker2):
                 # Finished extracting the tar file
                 self.tar2_process = None
 
-        if BACKUP_DEBUG and callable(self.print_callback):
-            self.print_callback("Finished extracting thread")
+        self.log.debug("Finished extracting thread")
 
 
 def get_supported_hmac_algo(hmac_algorithm=None):
@@ -1396,15 +1364,13 @@ class BackupRestore(object):
         #: is the backup operation canceled
         self.canceled = False
 
-        # TODO: convert to python logging API
-        self.print_callback = print_stdout
-        self.error_callback = print_stderr
-
         #: report restore progress, called with one argument - percents of
         # data restored
         # FIXME: convert to float [0,1]
         self.progress_callback = None
 
+        self.log = logging.getLogger('qubes.backup')
+
         #: basic information about the backup
         self.header_data = self._retrieve_backup_header()
 
@@ -1462,8 +1428,7 @@ class BackupRestore(object):
         tar1_env = os.environ.copy()
         tar1_env['UPDATES_MAX_BYTES'] = str(limit_bytes)
         tar1_env['UPDATES_MAX_FILES'] = str(limit_count)
-        if BACKUP_DEBUG and callable(self.print_callback):
-            self.print_callback("Run command" + unicode(tar1_command))
+        self.log.debug("Run command" + unicode(tar1_command))
         command = subprocess.Popen(
             tar1_command,
             stdin=backup_stdin,
@@ -1501,8 +1466,7 @@ class BackupRestore(object):
         if algorithm is None:
             algorithm = self.header_data.hmac_algorithm
         passphrase = self.passphrase.encode('utf-8')
-        if BACKUP_DEBUG:
-            print "Verifying file " + filename
+        self.log.debug("Verifying file {}".format(filename))
 
         if hmacfile != filename + ".hmac":
             raise qubes.exc.QubesException(
@@ -1519,15 +1483,14 @@ class BackupRestore(object):
             raise qubes.exc.QubesException(
                 "ERROR: verify file {0}: {1}".format(filename, hmac_stderr))
         else:
-            if BACKUP_DEBUG:
-                print "Loading hmac for file " + filename
+            self.log.debug("Loading hmac for file {}".format(filename))
             hmac = load_hmac(open(os.path.join(self.tmpdir, hmacfile),
                 'r').read())
 
             if len(hmac) > 0 and load_hmac(hmac_stdout) == hmac:
                 os.unlink(os.path.join(self.tmpdir, hmacfile))
-                if BACKUP_DEBUG:
-                    print "File verification OK -> Sending file " + filename
+                self.log.debug(
+                    "File verification OK -> Sending file {}".format(filename))
                 return True
             else:
                 raise qubes.exc.QubesException(
@@ -1566,9 +1529,8 @@ class BackupRestore(object):
         if not self.backup_vm:
             nextfile = filelist_pipe.readline().strip()
 
-        if BACKUP_DEBUG and callable(self.print_callback):
-            self.print_callback("Got backup header and hmac: %s, %s" % (
-                filename, hmacfile))
+        self.log.debug("Got backup header and hmac: {}, {}".format(
+            filename, hmacfile))
 
         if not filename or filename == "EOF" or \
                 not hmacfile or hmacfile == "EOF":
@@ -1648,8 +1610,6 @@ class BackupRestore(object):
             'compressed': self.header_data.compressed,
             'crypto_algorithm': self.header_data.crypto_algorithm,
             'verify_only': self.options.verify_only,
-            'print_callback': self.print_callback,
-            'error_callback': print_stderr,
             'progress_callback': self.progress_callback,
         }
         format_version = self.header_data.version
@@ -1702,12 +1662,9 @@ class BackupRestore(object):
         limit_count = str(2 * (10 * len(vms_dirs) +
                                int(vms_size / (100 * 1024 * 1024))))
 
-        if callable(self.print_callback):
-            if BACKUP_DEBUG:
-                self.print_callback("Working in temporary dir:" +
-                                    self.tmpdir)
-            self.print_callback(
-                "Extracting data: " + size_to_human(vms_size) + " to restore")
+        self.log.debug("Working in temporary dir:" + self.tmpdir)
+        self.log.info(
+            "Extracting data: " + size_to_human(vms_size) + " to restore")
 
         # retrieve backup from the backup stream (either VM, or dom0 file)
         # TODO: add some safety margin in vms_size?
@@ -1740,8 +1697,7 @@ class BackupRestore(object):
                 else:
                     filename = filelist_pipe.readline().strip()
 
-                if BACKUP_DEBUG and callable(self.print_callback):
-                    self.print_callback("Getting new file:" + filename)
+                self.log.debug("Getting new file:" + filename)
 
                 if not filename or filename == "EOF":
                     break
@@ -1756,17 +1712,14 @@ class BackupRestore(object):
                 if not self.backup_vm:
                     nextfile = filelist_pipe.readline().strip()
 
-                if BACKUP_DEBUG and callable(self.print_callback):
-                    self.print_callback("Getting hmac:" + hmacfile)
+                self.log.debug("Getting hmac:" + hmacfile)
                 if not hmacfile or hmacfile == "EOF":
                     # Premature end of archive, either of tar1_command or
                     # vmproc exited with error
                     break
 
                 if not any(map(lambda x: filename.startswith(x), vms_dirs)):
-                    if BACKUP_DEBUG and callable(self.print_callback):
-                        self.print_callback("Ignoring VM not selected for "
-                                          "restore")
+                    self.log.debug("Ignoring VM not selected for restore")
                     os.unlink(os.path.join(self.tmpdir, filename))
                     os.unlink(os.path.join(self.tmpdir, hmacfile))
                     continue
@@ -1798,13 +1751,10 @@ class BackupRestore(object):
         else:
             to_extract.put("FINISHED")
 
-        if BACKUP_DEBUG and callable(self.print_callback):
-            self.print_callback("Waiting for the extraction process to "
-                               "finish...")
+        self.log.debug("Waiting for the extraction process to finish...")
         extract_proc.join()
-        if BACKUP_DEBUG and callable(self.print_callback):
-            self.print_callback("Extraction process finished with code:" +
-                           str(extract_proc.exitcode))
+        self.log.debug("Extraction process finished with code: {}".format(
+            extract_proc.exitcode))
         if extract_proc.exitcode != 0:
             raise qubes.exc.QubesException(
                 "unable to extract the qubes backup. "
@@ -1967,8 +1917,7 @@ class BackupRestore(object):
                 # Handle dom0 as special case later
                 continue
             if self._is_vm_included_in_backup(vm):
-                if BACKUP_DEBUG:
-                    print vm.name, "is included in backup"
+                self.log.debug("{} is included in backup".format(vm.name))
 
                 vms_to_restore[vm.name] = {}
                 vms_to_restore[vm.name]['vm'] = vm
@@ -2169,17 +2118,14 @@ class BackupRestore(object):
                 if self.options.verify_only:
                     raise
                 else:
-                    if callable(self.print_callback):
-                        self.print_callback(
-                            "Some errors occurred during data extraction, "
-                            "continuing anyway to restore at least some "
-                            "VMs")
+                    self.log.warning(
+                        "Some errors occurred during data extraction, "
+                        "continuing anyway to restore at least some "
+                        "VMs")
         else:
             if self.options.verify_only:
-                if callable(self.print_callback):
-                    self.print_callback("WARNING: Backup verification not "
-                                    "supported for "
-                                   "this backup format.")
+                self.log.warning(
+                    "Backup verification not supported for this backup format.")
 
         if self.options.verify_only:
             shutil.rmtree(self.tmpdir)
@@ -2196,15 +2142,13 @@ class BackupRestore(object):
                     break
                 if vm.is_template() != do_templates:
                     continue
-                if callable(self.print_callback):
-                    self.print_callback("-> Restoring {0}...".format(vm.name))
+                self.log.info("-> Restoring {0}...".format(vm.name))
                 retcode = subprocess.call(
                     ["mkdir", "-p", os.path.dirname(vm.dir_path)])
                 if retcode != 0:
-                    self.error_callback("*** Cannot create directory: {"
-                                     "0}?!".format(
+                    self.log.error("*** Cannot create directory: {0}?!".format(
                         vm.dir_path))
-                    self.error_callback("Skipping...")
+                    self.log.warning("Skipping VM {}...".format(vm.name))
                     continue
 
                 kwargs = {}
@@ -2232,15 +2176,16 @@ class BackupRestore(object):
                             new_vm.dir_path), os.path.dirname(new_vm.dir_path))
                         try:
                             os.rename(new_vm.dir_path, move_to_path)
-                            self.error_callback(
+                            self.log.warning(
                                 "*** Directory {} already exists! It has "
                                 "been moved to {}".format(new_vm.dir_path,
                                                           move_to_path))
                         except OSError:
-                            self.error_callback(
+                            self.log.error(
                                 "*** Directory {} already exists and "
                                 "cannot be moved!".format(new_vm.dir_path))
-                            self.error_callback("Skipping...")
+                            self.log.warning("Skipping VM {}...".format(
+                                vm.name))
                             continue
 
                     if self.header_data.version == 1:
@@ -2252,8 +2197,8 @@ class BackupRestore(object):
 
                     new_vm.verify_files()
                 except Exception as err:
-                    self.error_callback("ERROR: {0}".format(err))
-                    self.error_callback("*** Skipping VM: {0}".format(vm.name))
+                    self.log.error("ERROR: {0}".format(err))
+                    self.log.warning("*** Skipping VM: {0}".format(vm.name))
                     if new_vm:
                         del self.app.domains[new_vm.qid]
                     continue
@@ -2264,24 +2209,22 @@ class BackupRestore(object):
                             vm.kernel not in \
                             os.listdir(qubes.config.system_path[
                                 'qubes_kernels_base_dir']):
-                        if callable(self.print_callback):
-                            self.print_callback("WARNING: Kernel %s not "
-                                             "installed, "
+                        self.log.warning("WARNING: Kernel %s not installed, "
                                            "using default one" % vm.kernel)
                         vm.kernel = qubes.property.DEFAULT
                 try:
                     new_vm.clone_properties(vm)
                 except Exception as err:
-                    self.error_callback("ERROR: {0}".format(err))
-                    self.error_callback("*** Some VM property will not be "
+                    self.log.error("ERROR: {0}".format(err))
+                    self.log.warning("*** Some VM property will not be "
                                      "restored")
 
                 try:
                     new_vm.fire_event('domain-restore')
                 except Exception as err:
-                    self.error_callback("ERROR during appmenu restore: {"
+                    self.log.error("ERROR during appmenu restore: {"
                                     "0}".format(err))
-                    self.error_callback(
+                    self.log.warning(
                         "*** VM '{0}' will not have appmenus".format(vm.name))
 
         # Set network dependencies - only non-default netvm setting
@@ -2323,12 +2266,11 @@ class BackupRestore(object):
             restore_home_backupdir = "home-pre-restore-{0}".format(
                 time.strftime("%Y-%m-%d-%H%M%S"))
 
-            if callable(self.print_callback):
-                self.print_callback(
-                    "-> Restoring home of user '{0}'...".format(local_user))
-                self.print_callback(
-                    "--> Existing files/dirs backed up in '{0}' dir".format(
-                        restore_home_backupdir))
+            self.log.info(
+                "Restoring home of user '{0}'...".format(local_user))
+            self.log.info(
+                "Existing files/dirs backed up in '{0}' dir".format(
+                    restore_home_backupdir))
             os.mkdir(home_dir + '/' + restore_home_backupdir)
             for f in os.listdir(backup_dom0_home_dir):
                 home_file = home_dir + '/' + f
@@ -2342,8 +2284,7 @@ class BackupRestore(object):
                     shutil.move(backup_dom0_home_dir + '/' + f, home_file)
             retcode = subprocess.call(['sudo', 'chown', '-R', local_user, home_dir])
             if retcode != 0:
-                self.error_callback("*** Error while setting home directory "
-                                  "owner")
+                self.log.error("*** Error while setting home directory owner")
 
         shutil.rmtree(self.tmpdir)
 
diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py
index 5060ced5..b50993df 100644
--- a/qubes/tests/__init__.py
+++ b/qubes/tests/__init__.py
@@ -631,6 +631,14 @@ class SystemTestsMixin(object):
 
 # noinspection PyAttributeOutsideInit
 class BackupTestsMixin(SystemTestsMixin):
+    class BackupErrorHandler(logging.Handler):
+        def __init__(self, errors_queue, level=logging.NOTSET):
+            super(BackupTestsMixin.BackupErrorHandler, self).__init__(level)
+            self.errors_queue = errors_queue
+
+        def emit(self, record):
+            self.errors_queue.put(record.getMessage())
+
     def setUp(self):
         super(BackupTestsMixin, self).setUp()
         self.init_default_template()
@@ -645,22 +653,17 @@ class BackupTestsMixin(SystemTestsMixin):
             shutil.rmtree(self.backupdir)
         os.mkdir(self.backupdir)
 
+        self.error_handler = self.BackupErrorHandler(self.error_detected,
+            level=logging.WARNING)
+        backup_log = logging.getLogger('qubes.backup')
+        backup_log.addHandler(self.error_handler)
+
     def tearDown(self):
         super(BackupTestsMixin, self).tearDown()
         shutil.rmtree(self.backupdir)
 
-    def print_progress(self, progress):
-        if self.verbose:
-            print >> sys.stderr, "\r-> Backing up files: {0}%...".format(progress)
-
-    def error_callback(self, message):
-        self.error_detected.put(message)
-        if self.verbose:
-            print >> sys.stderr, "ERROR: {0}".format(message)
-
-    def print_callback(self, msg):
-        if self.verbose:
-            print msg
+        backup_log = logging.getLogger('qubes.backup')
+        backup_log.removeHandler(self.error_handler)
 
     def fill_image(self, path, size=None, sparse=False):
         block_size = 4096

From 595dfdc0a9c71f9eed27ce3eaacc7a5a658d5ada Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 14 Mar 2016 12:16:23 +0100
Subject: [PATCH 06/32] backup: add additional verification of backup header

Ensure only alphanumeric characters are used.
---
 qubes/backup.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/qubes/backup.py b/qubes/backup.py
index 6a752ffe..e2773399 100644
--- a/qubes/backup.py
+++ b/qubes/backup.py
@@ -126,9 +126,13 @@ class BackupHeader(object):
             if untrusted_line.count('=') != 1:
                 raise qubes.exc.QubesException("Invalid backup header")
             (key, value) = untrusted_line.strip().split('=')
+            if not re.match(r"^[a-zA-Z0-9-]*$", key):
+                raise qubes.exc.QubesException("Invalid backup header (key)")
             if key not in self.header_keys.keys():
                 # Ignoring unknown option
                 continue
+            if not re.match(r"^[a-zA-Z0-9-]*$", value):
+                raise qubes.exc.QubesException("Invalid backup header (value)")
             if getattr(self, self.header_keys[key]) is not None:
                 raise qubes.exc.QubesException(
                     "Duplicated header line: {}".format(key))

From e5af64f2bf9d9e50e4a0476ce83648e9516f4f7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 14 Mar 2016 12:17:02 +0100
Subject: [PATCH 07/32] backup: fix restore summary table

---
 qubes/backup.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/qubes/backup.py b/qubes/backup.py
index e2773399..8b01aad6 100644
--- a/qubes/backup.py
+++ b/qubes/backup.py
@@ -1990,8 +1990,8 @@ class BackupRestore(object):
 
             "updbl": {"func": "'Yes' if vm.updateable else ''"},
 
-            "template": {"func": "'n/a' if vm.is_template() or vm.template is None else\
-                         vm_info['template']"},
+            "template": {"func": "'n/a' if not hasattr(vm, 'template') is None "
+                                 "else vm_info['template']"},
 
             "netvm": {"func": "'n/a' if vm.is_netvm() and not vm.is_proxyvm() else\
                       ('*' if vm.property_is_default('netvm') else '') +\

From 44eed25511ba718a49f1ff7b4cf0ce804846ac47 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 14 Mar 2016 13:07:58 +0100
Subject: [PATCH 08/32] tests/backup: report failure when error was expected
 but none detected

---
 qubes/tests/__init__.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py
index b50993df..b05348b4 100644
--- a/qubes/tests/__init__.py
+++ b/qubes/tests/__init__.py
@@ -774,6 +774,9 @@ class BackupTestsMixin(SystemTestsMixin):
         errors = []
         if expect_errors is None:
             expect_errors = []
+        else:
+            self.assertFalse(self.error_detected.empty(),
+                "Restore errors expected, but none detected")
         while not self.error_detected.empty():
             current_error = self.error_detected.get()
             if any(map(current_error.startswith, expect_errors)):

From e04ea7512c3d3ee757397a34d3c4639a11170821 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 14 Mar 2016 13:30:55 +0100
Subject: [PATCH 09/32] backup: minor code style

- Don't use catch-all except statement.
- Use str.format instead of "%" operator.
- Use static methods where applicable.
- Remove unused local variables.
- Don't shadow variables from outer scope
---
 qubes/backup.py | 39 ++++++++++++++++++++-------------------
 1 file changed, 20 insertions(+), 19 deletions(-)

diff --git a/qubes/backup.py b/qubes/backup.py
index 8b01aad6..fec8bc38 100644
--- a/qubes/backup.py
+++ b/qubes/backup.py
@@ -313,7 +313,7 @@ class Backup(object):
         for proc in self.processes_to_kill_on_cancel:
             try:
                 proc.terminate()
-            except:
+            except OSError:
                 pass
 
     @staticmethod
@@ -746,11 +746,11 @@ class Backup(object):
                     if self.canceled:
                         try:
                             tar_sparse.terminate()
-                        except:
+                        except OSError:
                             pass
                         try:
                             hmac.terminate()
-                        except:
+                        except OSError:
                             pass
                         tar_sparse.wait()
                         hmac.wait()
@@ -1111,8 +1111,8 @@ class ExtractWorker2(Process):
                 self.tar2_process.terminate()
                 self.tar2_process.wait()
                 self.tar2_process = None
-                self.log.error("Error while processing '%s': %s " %
-                                    (self.tar2_current_file, details))
+                self.log.error("Error while processing '{}': {}".format(
+                    self.tar2_current_file, details))
 
             # Delete the file as we don't need it anymore
             self.log.debug("Removing file " + filename)
@@ -1258,8 +1258,8 @@ class ExtractWorker3(ExtractWorker2):
                 self.tar2_process.terminate()
                 self.tar2_process.wait()
                 self.tar2_process = None
-                self.log.error("Error while processing '%s': %s " %
-                                    (self.tar2_current_file, details))
+                self.log.error("Error while processing '{}': {}".format(
+                    self.tar2_current_file, details))
 
             # Delete the file as we don't need it anymore
             self.log.debug("Removing file " + filename)
@@ -1388,7 +1388,7 @@ class BackupRestore(object):
         for proc in self.processes_to_kill_on_cancel:
             try:
                 proc.terminate()
-            except:
+            except OSError:
                 pass
 
     def _start_retrieval_process(self, filelist, limit_count, limit_bytes):
@@ -1459,14 +1459,15 @@ class BackupRestore(object):
         return command, filelist_pipe, error_pipe
 
     def _verify_hmac(self, filename, hmacfile, algorithm=None):
-        def load_hmac(hmac):
-            hmac = hmac.strip().split("=")
-            if len(hmac) > 1:
-                hmac = hmac[1].strip()
+        def load_hmac(hmac_text):
+            hmac_text = hmac_text.strip().split("=")
+            if len(hmac_text) > 1:
+                hmac_text = hmac_text[1].strip()
             else:
-                raise qubes.exc.QubesException("ERROR: invalid hmac file content")
+                raise qubes.exc.QubesException(
+                    "ERROR: invalid hmac file content")
 
-            return hmac
+            return hmac_text
         if algorithm is None:
             algorithm = self.header_data.hmac_algorithm
         passphrase = self.passphrase.encode('utf-8')
@@ -1523,7 +1524,6 @@ class BackupRestore(object):
                 ['backup-header', 'backup-header.hmac',
                 'qubes.xml.000', 'qubes.xml.000.hmac'], 4, 1024 * 1024)
 
-        nextfile = None
         expect_tar_error = False
 
         filename = filelist_pipe.readline().strip()
@@ -1531,7 +1531,7 @@ class BackupRestore(object):
         # tar output filename before actually extracting it, so wait for the
         # next one before trying to access it
         if not self.backup_vm:
-            nextfile = filelist_pipe.readline().strip()
+            filelist_pipe.readline().strip()
 
         self.log.debug("Got backup header and hmac: {}, {}".format(
             filename, hmacfile))
@@ -1689,7 +1689,6 @@ class BackupRestore(object):
                 if not extract_proc.is_alive():
                     retrieve_proc.terminate()
                     retrieve_proc.wait()
-                    expect_tar_error = True
                     if retrieve_proc in self.processes_to_kill_on_cancel:
                         self.processes_to_kill_on_cancel.remove(retrieve_proc)
                     # wait for other processes (if any)
@@ -1888,7 +1887,8 @@ class BackupRestore(object):
         else:
             return False
 
-    def _is_vm_included_in_backup_v2(self, check_vm):
+    @staticmethod
+    def _is_vm_included_in_backup_v2(check_vm):
         if check_vm.backup_content:
             return True
         else:
@@ -1974,7 +1974,8 @@ class BackupRestore(object):
 
         return vms_to_restore
 
-    def get_restore_summary(self, restore_info):
+    @staticmethod
+    def get_restore_summary(restore_info):
         fields = {
             "qid": {"func": "vm.qid"},
 

From 697a26c8da9ce7a09d51d286c56b28a057c5434d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 14 Mar 2016 13:32:56 +0100
Subject: [PATCH 10/32] backup: update license header

Use link instead of address, as recommended by FSF.
---
 qubes/backup.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/qubes/backup.py b/qubes/backup.py
index fec8bc38..fba9f424 100644
--- a/qubes/backup.py
+++ b/qubes/backup.py
@@ -18,8 +18,7 @@
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+# along with this program. If not, see <http://www.gnu.org/licenses/>
 #
 #
 from __future__ import unicode_literals

From 3fb0754398e707208836261a99df49553dea9357 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 14 Mar 2016 13:34:17 +0100
Subject: [PATCH 11/32] backup: formatting

---
 qubes/backup.py | 93 +++++++++++++++++++++++++++++--------------------
 1 file changed, 55 insertions(+), 38 deletions(-)

diff --git a/qubes/backup.py b/qubes/backup.py
index fba9f424..9bbf4fa3 100644
--- a/qubes/backup.py
+++ b/qubes/backup.py
@@ -60,6 +60,7 @@ def get_disk_usage_one(st):
     except AttributeError:
         return st.st_size
 
+
 def get_disk_usage(path):
     try:
         st = os.lstat(path)
@@ -75,6 +76,7 @@ def get_disk_usage(path):
 
     return ret
 
+
 class BackupCanceledError(qubes.exc.QubesException):
     def __init__(self, msg, tmpdir=None):
         super(BackupCanceledError, self).__init__(msg)
@@ -93,7 +95,7 @@ class BackupHeader(object):
     bool_options = ['encrypted', 'compressed']
     int_options = ['version']
 
-    def __init__(self, header_data = None):
+    def __init__(self, header_data=None):
         # repeat the list to help code completion...
         self.version = None
         self.encrypted = None
@@ -173,6 +175,7 @@ class BackupHeader(object):
                     continue
                 f.write("{!s}={!s}\n".format(key, getattr(self, attr)))
 
+
 class SendWorker(Process):
     def __init__(self, queue, base_dir, backup_stdout):
         super(SendWorker, self).__init__()
@@ -298,8 +301,8 @@ class Backup(object):
         self.vms_for_backup = vms_list
         # Apply exclude list
         if exclude_list:
-            self.vms_for_backup = [vm for vm in vms_list if vm.name not in
-                                                        exclude_list]
+            self.vms_for_backup = [vm for vm in vms_list
+                if vm.name not in exclude_list]
 
     def __del__(self):
         if self.tmpdir and os.path.exists(self.tmpdir):
@@ -321,7 +324,8 @@ class Backup(object):
 
         if subdir is None:
             abs_file_path = os.path.abspath(file_path)
-            abs_base_dir = os.path.abspath(qubes.config.system_path["qubes_base_dir"]) + '/'
+            abs_base_dir = os.path.abspath(
+                qubes.config.system_path["qubes_base_dir"]) + '/'
             abs_file_dir = os.path.dirname(abs_file_path) + '/'
             (nothing, directory, subdir) = abs_file_dir.partition(abs_base_dir)
             assert nothing == ""
@@ -425,9 +429,9 @@ class Backup(object):
         if 0 in [vm.qid for vm in self.vms_for_backup]:
             local_user = grp.getgrnam('qubes').gr_mem[0]
             home_dir = pwd.getpwnam(local_user).pw_dir
-            # Home dir should have only user-owned files, so fix it now to prevent
-            # permissions problems - some root-owned files can left after
-            # 'sudo bash' and similar commands
+            # Home dir should have only user-owned files, so fix it now
+            # to prevent permissions problems - some root-owned files can
+            # left after 'sudo bash' and similar commands
             subprocess.check_call(['sudo', 'chown', '-R', local_user, home_dir])
 
             home_sz = get_disk_usage(home_dir)
@@ -483,7 +487,8 @@ class Backup(object):
             elif vm_info['vm'].is_template():
                 s += fmt.format("Template VM")
             else:
-                s += fmt.format("VM" + (" + Sys" if vm_info['vm'].updateable else ""))
+                s += fmt.format("VM" + (" + Sys" if vm_info['vm'].updateable
+                    else ""))
 
             vm_size = vm_info['size']
 
@@ -623,8 +628,8 @@ class Backup(object):
             # If not APPVM, STDOUT is a local file
             backup_stdout = open(backup_target, 'wb')
 
-        # Tar with tape length does not deals well with stdout (close stdout between
-        # two tapes)
+        # Tar with tape length does not deals well with stdout
+        # (close stdout between two tapes)
         # For this reason, we will use named pipes instead
         self.log.debug("Working in {}".format(self.tmpdir))
 
@@ -657,9 +662,9 @@ class Backup(object):
 
                 self.log.debug("Backing up {}".format(file_info))
 
-                backup_tempfile = os.path.join(self.tmpdir,
-                                               file_info["subdir"],
-                                               os.path.basename(file_info["path"]))
+                backup_tempfile = os.path.join(
+                    self.tmpdir, file_info["subdir"],
+                    os.path.basename(file_info["path"]))
                 self.log.debug("Using temporary location: {}".format(
                     backup_tempfile))
 
@@ -667,20 +672,20 @@ class Backup(object):
                 if not os.path.isdir(os.path.dirname(backup_tempfile)):
                     os.makedirs(os.path.dirname(backup_tempfile))
 
-                # The first tar cmd can use any complex feature as we want. Files will
-                # be verified before untaring this.
+                # The first tar cmd can use any complex feature as we want.
+                # Files will be verified before untaring this.
                 # Prefix the path in archive with filename["subdir"] to have it
                 # verified during untar
                 tar_cmdline = (["tar", "-Pc", '--sparse',
                                "-f", backup_pipe,
                                '-C', os.path.dirname(file_info["path"])] +
-                               (['--dereference'] if file_info["subdir"] != "dom0-home/"
-                               else []) +
+                               (['--dereference'] if
+                                file_info["subdir"] != "dom0-home/" else []) +
                                ['--xform', 's:^%s:%s\\0:' % (
                                    os.path.basename(file_info["path"]),
                                    file_info["subdir"]),
-                               os.path.basename(file_info["path"])
-                               ])
+                                os.path.basename(file_info["path"])
+                                ])
                 if self.compressed:
                     tar_cmdline.insert(-1,
                         "--use-compress-program=%s" % self.compression_filter)
@@ -695,14 +700,15 @@ class Backup(object):
                     tar_cmdline, stdin=subprocess.PIPE)
                 self.processes_to_kill_on_cancel.append(tar_sparse)
 
-                # Wait for compressor (tar) process to finish or for any error of other
-                # subprocesses
+                # Wait for compressor (tar) process to finish or for any
+                # error of other subprocesses
                 i = 0
                 run_error = "paused"
                 encryptor = None
                 if self.encrypted:
                     # Start encrypt
-                    # If no cipher is provided, the data is forwarded unencrypted !!!
+                    # If no cipher is provided,
+                    # the data is forwarded unencrypted !!!
                     encryptor = subprocess.Popen([
                         "openssl", "enc",
                         "-e", "-" + self.crypto_algorithm,
@@ -764,8 +770,9 @@ class Backup(object):
                                 "Failed to write the backup, VM output:\n" +
                                 vmproc.stderr.read(MAX_STDERR_BYTES))
                         else:
-                            raise qubes.exc.QubesException("Failed to perform backup: error in " +
-                                                 run_error)
+                            raise qubes.exc.QubesException(
+                                "Failed to perform backup: error in " +
+                                run_error)
 
                     # Send the chunk to the backup target
                     self._queue_put_with_check(
@@ -922,6 +929,7 @@ def wait_backup_feedback(progress_callback, in_stream, streamproc,
 
     return run_error
 
+
 class ExtractWorker2(Process):
     def __init__(self, queue, base_dir, passphrase, encrypted,
                  progress_callback, vmproc=None,
@@ -1301,6 +1309,7 @@ def get_supported_hmac_algo(hmac_algorithm=None):
         yield algo.strip()
     proc.wait()
 
+
 class BackupRestoreOptions(object):
     def __init__(self):
         #: use default NetVM if the one referenced in backup do not exists on
@@ -1440,8 +1449,9 @@ class BackupRestore(object):
             env=tar1_env)
         self.processes_to_kill_on_cancel.append(command)
 
-        # qfile-dom0-unpacker output filelist on stderr (and have stdout connected
-        # to the VM), while tar output filelist on stdout
+        # qfile-dom0-unpacker output filelist on stderr
+        # and have stdout connected to the VM), while tar output filelist
+        # on stdout
         if self.backup_vm:
             filelist_pipe = command.stderr
             # let qfile-dom0-unpacker hold the only open FD to the write end of
@@ -1604,7 +1614,8 @@ class BackupRestore(object):
         """
 
         # Setup worker to extract encrypted data chunks to the restore dirs
-        # Create the process here to pass it options extracted from backup header
+        # Create the process here to pass it options extracted from
+        # backup header
         extractor_params = {
             'queue': queue,
             'base_dir': self.tmpdir,
@@ -1832,7 +1843,8 @@ class BackupRestore(object):
                 except KeyError:
                     netvm_on_host = None
                 # No netvm on the host?
-                if not ((netvm_on_host is not None) and netvm_on_host.is_netvm()):
+                if not ((netvm_on_host is not None)
+                        and netvm_on_host.is_netvm()):
 
                     # Maybe the (custom) netvm is in the backup?
                     if not (netvm_name in restore_info.keys() and
@@ -1995,12 +2007,14 @@ class BackupRestore(object):
 
             "netvm": {"func": "'n/a' if vm.is_netvm() and not vm.is_proxyvm() else\
                       ('*' if vm.property_is_default('netvm') else '') +\
-                        vm_info['netvm'] if vm_info['netvm'] is not None else '-'"},
+                        vm_info['netvm'] if vm_info['netvm'] is not None "
+                              "else '-'"},
 
             "label": {"func": "vm.label.name"},
         }
 
-        fields_to_display = ["name", "type", "template", "updbl", "netvm", "label"]
+        fields_to_display = ["name", "type", "template", "updbl",
+            "netvm", "label"]
 
         # First calculate the maximum width of each field we want to display
         total_width = 0
@@ -2141,8 +2155,8 @@ class BackupRestore(object):
                 break
             for vm in vms.values():
                 if self.canceled:
-                    # only break the loop to save qubes.xml with already restored
-                    # VMs
+                    # only break the loop to save qubes.xml
+                    # with already restored VMs
                     break
                 if vm.is_template() != do_templates:
                     continue
@@ -2168,7 +2182,8 @@ class BackupRestore(object):
                     vm_name = restore_info[vm.name]['rename-to']
 
                 try:
-                    # first only minimal set, later clone_properties will be called
+                    # first only minimal set, later clone_properties
+                    # will be called
                     new_vm = self.app.add_new_vm(
                         vm.__class__,
                         name=vm_name,
@@ -2214,7 +2229,7 @@ class BackupRestore(object):
                             os.listdir(qubes.config.system_path[
                                 'qubes_kernels_base_dir']):
                         self.log.warning("WARNING: Kernel %s not installed, "
-                                           "using default one" % vm.kernel)
+                        "using default one" % vm.kernel)
                         vm.kernel = qubes.property.DEFAULT
                 try:
                     new_vm.clone_properties(vm)
@@ -2226,8 +2241,8 @@ class BackupRestore(object):
                 try:
                     new_vm.fire_event('domain-restore')
                 except Exception as err:
-                    self.log.error("ERROR during appmenu restore: {"
-                                    "0}".format(err))
+                    self.log.error("ERROR during appmenu restore: "
+                       "{0}".format(err))
                     self.log.warning(
                         "*** VM '{0}' will not have appmenus".format(vm.name))
 
@@ -2283,10 +2298,12 @@ class BackupRestore(object):
                               home_dir + '/' + restore_home_backupdir + '/' + f)
                 if self.header_data.version == 1:
                     subprocess.call(
-                        ["cp", "-nrp", "--reflink=auto", backup_dom0_home_dir + '/' + f, home_file])
+                        ["cp", "-nrp", "--reflink=auto",
+                            backup_dom0_home_dir + '/' + f, home_file])
                 elif self.header_data.version >= 2:
                     shutil.move(backup_dom0_home_dir + '/' + f, home_file)
-            retcode = subprocess.call(['sudo', 'chown', '-R', local_user, home_dir])
+            retcode = subprocess.call(['sudo', 'chown', '-R',
+                local_user, home_dir])
             if retcode != 0:
                 self.log.error("*** Error while setting home directory owner")
 

From d72d8f8445fa80b759e1bc542c81e4a0d2aaff48 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 14 Mar 2016 13:34:40 +0100
Subject: [PATCH 12/32] backup: code inspection annotations

---
 qubes/backup.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/qubes/backup.py b/qubes/backup.py
index 9bbf4fa3..9e182826 100644
--- a/qubes/backup.py
+++ b/qubes/backup.py
@@ -575,6 +575,7 @@ class Backup(object):
     def backup_do(self):
         if self.passphrase is None:
             raise qubes.exc.QubesException("No passphrase set")
+        # noinspection PyProtectedMember
         qubes_xml = self.app._store
         self.tmpdir = tempfile.mkdtemp()
         shutil.copy(qubes_xml, os.path.join(self.tmpdir, 'qubes.xml'))
@@ -2035,14 +2036,17 @@ class BackupRestore(object):
 
         # Display the header
         for f in fields_to_display:
+            # noinspection PyTypeChecker
             fmt = "{{0:-^{0}}}-+".format(fields[f]["max_width"] + 1)
             summary += fmt.format('-')
         summary += "\n"
         for f in fields_to_display:
+            # noinspection PyTypeChecker
             fmt = "{{0:>{0}}} |".format(fields[f]["max_width"] + 1)
             summary += fmt.format(f)
         summary += "\n"
         for f in fields_to_display:
+            # noinspection PyTypeChecker
             fmt = "{{0:-^{0}}}-+".format(fields[f]["max_width"] + 1)
             summary += fmt.format('-')
         summary += "\n"
@@ -2055,6 +2059,7 @@ class BackupRestore(object):
             vm = vm_info['vm']
             s = ""
             for f in fields_to_display:
+                # noinspection PyTypeChecker
                 fmt = "{{0:>{0}}} |".format(fields[f]["max_width"] + 1)
                 s += fmt.format(eval(fields[f]["func"]))
 
@@ -2063,20 +2068,25 @@ class BackupRestore(object):
             elif 'already-exists' in vm_info:
                 s += " <-- A VM with the same name already exists on the host!"
             elif 'missing-template' in vm_info:
-                s += " <-- No matching template on the host or in the backup found!"
+                s += " <-- No matching template on the host " \
+                     "or in the backup found!"
             elif 'missing-netvm' in vm_info:
-                s += " <-- No matching netvm on the host or in the backup found!"
+                s += " <-- No matching netvm on the host " \
+                     "or in the backup found!"
             else:
                 if 'orig-template' in vm_info:
-                    s += " <-- Original template was '%s'" % (vm_info['orig-template'])
+                    s += " <-- Original template was '{}'".format(
+                        vm_info['orig-template'])
                 if 'rename-to' in vm_info:
-                    s += " <-- Will be renamed to '%s'" % vm_info['rename-to']
+                    s += " <-- Will be renamed to '{}'".format(
+                        vm_info['rename-to'])
 
             summary += s + "\n"
 
         if 'dom0' in restore_info.keys():
             s = ""
             for f in fields_to_display:
+                # noinspection PyTypeChecker
                 fmt = "{{0:>{0}}} |".format(fields[f]["max_width"] + 1)
                 if f == "name":
                     s += fmt.format("Dom0")

From 412bcfa88b9b4383ac1b71099f19998639dfd55c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 15 Mar 2016 21:16:24 +0100
Subject: [PATCH 13/32] backup: fix hadling netvm property

---
 qubes/backup.py | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/qubes/backup.py b/qubes/backup.py
index 9e182826..d58cff94 100644
--- a/qubes/backup.py
+++ b/qubes/backup.py
@@ -1851,8 +1851,11 @@ class BackupRestore(object):
                     if not (netvm_name in restore_info.keys() and
                             restore_info[netvm_name]['vm'].is_netvm()):
                         if self.options.use_default_netvm:
-                            vm_info['netvm'] = self.app.default_netvm.name
-                            vm_info['vm'].uses_default_netvm = True
+                            if self.app.default_netvm:
+                                vm_info['netvm'] = self.app.default_netvm.name
+                            else:
+                                vm_info['netvm'] = None
+                            vm_info['vm'].netvm = qubes.property.DEFAULT
                         elif self.options.use_none_netvm:
                             vm_info['netvm'] = None
                         else:
@@ -2236,13 +2239,19 @@ class BackupRestore(object):
                     # TODO: add a setting for this?
                     if not vm.property_is_default('kernel') and \
                             vm.kernel not in \
-                            os.listdir(qubes.config.system_path[
-                                'qubes_kernels_base_dir']):
-                        self.log.warning("WARNING: Kernel %s not installed, "
+                            os.listdir(os.path.join(qubes.config.qubes_base_dir,
+                                qubes.config.system_path[
+                                'qubes_kernels_base_dir'])):
+                        self.log.warning("Kernel %s not installed, "
                         "using default one" % vm.kernel)
                         vm.kernel = qubes.property.DEFAULT
                 try:
-                    new_vm.clone_properties(vm)
+                    # exclude VM references - handled manually according to
+                    # restore options
+                    proplist = [prop for prop in new_vm.property_list()
+                        if prop.clone and prop.__name__ not in
+                              ['template', 'netvm', 'dispvm_netvm']]
+                    new_vm.clone_properties(vm, proplist=proplist)
                 except Exception as err:
                     self.log.error("ERROR: {0}".format(err))
                     self.log.warning("*** Some VM property will not be "

From 304d2b10bc703e8f493ce0354db26db04b65ee19 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 16 Mar 2016 00:57:19 +0100
Subject: [PATCH 14/32] backup: fix handline "None" kernel

---
 qubes/backup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qubes/backup.py b/qubes/backup.py
index d58cff94..a671d2d2 100644
--- a/qubes/backup.py
+++ b/qubes/backup.py
@@ -2237,7 +2237,7 @@ class BackupRestore(object):
 
                 if hasattr(vm, 'kernel'):
                     # TODO: add a setting for this?
-                    if not vm.property_is_default('kernel') and \
+                    if not vm.property_is_default('kernel') and vm.kernel and \
                             vm.kernel not in \
                             os.listdir(os.path.join(qubes.config.qubes_base_dir,
                                 qubes.config.system_path[

From 8c5d42a095fca0e6f22de41e251b792cb594649e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 16 Mar 2016 01:01:13 +0100
Subject: [PATCH 15/32] backup: add support for restoring pre-core3 backups

---
 qubes/backup.py         |  23 +++--
 qubes/core2migration.py | 220 ++++++++++++++++++++++++++++++++++++++++
 rpm_spec/core-dom0.spec |   1 +
 3 files changed, 234 insertions(+), 10 deletions(-)
 create mode 100644 qubes/core2migration.py

diff --git a/qubes/backup.py b/qubes/backup.py
index a671d2d2..fd9b0c59 100644
--- a/qubes/backup.py
+++ b/qubes/backup.py
@@ -39,13 +39,13 @@ import errno
 import datetime
 from multiprocessing import Queue, Process
 import qubes
+import qubes.core2migration
 
 HEADER_FILENAME = 'backup-header'
 DEFAULT_CRYPTO_ALGORITHM = 'aes-256-cbc'
 DEFAULT_HMAC_ALGORITHM = 'SHA512'
 DEFAULT_COMPRESSION_FILTER = 'gzip'
-# TODO: increase version after finishing implementation
-CURRENT_BACKUP_FORMAT_VERSION = '3'
+CURRENT_BACKUP_FORMAT_VERSION = '4'
 # Maximum size of error message get from process stderr (including VM process)
 MAX_STDERR_BYTES = 1024
 # header + qubes.xml max size
@@ -149,7 +149,7 @@ class BackupHeader(object):
         if self.version == 1:
             # header not really present
             pass
-        elif self.version in [2, 3]:
+        elif self.version in [2, 3, 4]:
             expected_attrs = ['version', 'encrypted', 'compressed',
                 'hmac_algorithm']
             if self.encrypted:
@@ -1630,7 +1630,7 @@ class BackupRestore(object):
         format_version = self.header_data.version
         if format_version == 2:
             extract_proc = ExtractWorker2(**extractor_params)
-        elif format_version == 3:
+        elif format_version in [3, 4]:
             extractor_params['compression_filter'] = \
                 self.header_data.compression_filter
             extract_proc = ExtractWorker3(**extractor_params)
@@ -1646,7 +1646,9 @@ class BackupRestore(object):
         and :py:meth:`retrieve_backup_header` was called.
         """
         if self.header_data.version == 1:
-            raise NotImplementedError("TODO: conversion core[12] qubes.xml")
+            backup_app = qubes.core2migration.Core2Qubes(
+                os.path.join(self.backup_location, 'qubes.xml'))
+            return backup_app
         else:
             self._verify_hmac("qubes.xml.000", "qubes.xml.000.hmac")
             queue = Queue()
@@ -1660,13 +1662,14 @@ class BackupRestore(object):
                 "unable to extract the qubes backup. "
                 "Check extracting process errors.")
 
-        if self.header_data.version in [2]:  # TODO add 3
-            raise NotImplementedError("TODO: conversion core[12] qubes.xml")
+        if self.header_data.version in [2, 3]:
+            backup_app = qubes.core2migration.Core2Qubes(
+                os.path.join(self.tmpdir, 'qubes.xml'))
         else:
             backup_app = qubes.Qubes(os.path.join(self.tmpdir, 'qubes.xml'))
             # Not needed anymore - all the data stored in backup_app
-            os.unlink(os.path.join(self.tmpdir, 'qubes.xml'))
-            return backup_app
+        os.unlink(os.path.join(self.tmpdir, 'qubes.xml'))
+        return backup_app
 
     def _restore_vm_dirs(self, vms_dirs, vms_size):
         # Currently each VM consists of at most 7 archives (count
@@ -1917,7 +1920,7 @@ class BackupRestore(object):
     def _is_vm_included_in_backup(self, vm):
         if self.header_data.version == 1:
             return self._is_vm_included_in_backup_v1(vm)
-        elif self.header_data.version in [2, 3]:
+        elif self.header_data.version in [2, 3, 4]:
             return self._is_vm_included_in_backup_v2(vm)
         else:
             raise qubes.exc.QubesException(
diff --git a/qubes/core2migration.py b/qubes/core2migration.py
new file mode 100644
index 00000000..c1518c01
--- /dev/null
+++ b/qubes/core2migration.py
@@ -0,0 +1,220 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# The Qubes OS Project, http://www.qubes-os.org
+#
+# Copyright (C) 2016  Marek Marczykowski-Górecki
+#                                   <marmarek@invisiblethingslab.com>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>
+#
+#
+import os
+import sys
+import qubes
+import qubes.vm.appvm
+import qubes.vm.standalonevm
+import qubes.vm.templatevm
+import qubes.vm.adminvm
+import qubes.ext.r3compatibility
+import lxml.etree
+import xml.parsers.expat
+
+
+class AppVM(qubes.vm.appvm.AppVM):
+    """core2 compatibility AppVM class, with variable dir_path"""
+    dir_path = qubes.property('dir_path',
+        default=(lambda self: self.storage.vmdir),
+        saver=qubes.property.dontsave,
+        doc="VM storage directory",
+    )
+
+    def is_running(self):
+        return False
+
+class StandaloneVM(qubes.vm.standalonevm.StandaloneVM):
+    """core2 compatibility StandaloneVM class, with variable dir_path"""
+    dir_path = qubes.property('dir_path',
+        default=(lambda self: self.storage.vmdir),
+        saver=qubes.property.dontsave,
+        doc="VM storage directory")
+
+    def is_running(self):
+        return False
+
+
+class Core2Qubes(qubes.Qubes):
+
+    def __init__(self, store=None, load=True, **kwargs):
+        if store is None:
+            raise ValueError("store path required")
+        super(Core2Qubes, self).__init__(store, load, **kwargs)
+
+    def load_globals(self, element):
+        default_template = element.get("default_template")
+        self.default_template = int(default_template) \
+            if default_template.lower() != "none" else None
+
+        default_netvm = element.get("default_netvm")
+        if default_netvm is not None:
+            self.default_netvm = int(default_netvm) \
+                if default_netvm != "None" else None
+
+        default_fw_netvm = element.get("default_fw_netvm")
+        if default_fw_netvm is not None:
+            self.default_fw_netvm = int(default_fw_netvm) \
+                if default_fw_netvm != "None" else None
+
+        updatevm = element.get("updatevm")
+        if updatevm is not None:
+            self.updatevm = int(updatevm) \
+                if updatevm != "None" else None
+
+        clockvm = element.get("clockvm")
+        if clockvm is not None:
+            self.clockvm = int(clockvm) \
+                if clockvm != "None" else None
+
+        self.default_kernel = element.get("default_kernel")
+
+    def set_netvm_dependency(self, element):
+        kwargs = {}
+        attr_list = ("qid", "uses_default_netvm", "netvm_qid")
+
+        for attribute in attr_list:
+            kwargs[attribute] = element.get(attribute)
+
+        vm = self.domains[int(kwargs["qid"])]
+
+        if element.get("uses_default_netvm") is None:
+            uses_default_netvm = True
+        else:
+            uses_default_netvm = (
+                True if element.get("uses_default_netvm") == "True" else False)
+        if not uses_default_netvm:
+            netvm_qid = element.get("netvm_qid")
+            if netvm_qid is None or netvm_qid == "none":
+                vm.netvm = None
+            else:
+                vm.netvm = int(netvm_qid)
+
+        # TODO: dispvm_netvm
+
+    def load(self):
+        qubes_store_file = open(self._store, 'r')
+
+        try:
+            qubes_store_file.seek(0)
+            tree = lxml.etree.parse(qubes_store_file)
+        except (EnvironmentError,
+                xml.parsers.expat.ExpatError) as err:
+            self.log.error(err)
+            return False
+
+        self.labels = {
+            1: qubes.Label(1, '0xcc0000', 'red'),
+            2: qubes.Label(2, '0xf57900', 'orange'),
+            3: qubes.Label(3, '0xedd400', 'yellow'),
+            4: qubes.Label(4, '0x73d216', 'green'),
+            5: qubes.Label(5, '0x555753', 'gray'),
+            6: qubes.Label(6, '0x3465a4', 'blue'),
+            7: qubes.Label(7, '0x75507b', 'purple'),
+            8: qubes.Label(8, '0x000000', 'black'),
+        }
+
+        self.domains.add(qubes.vm.adminvm.AdminVM(
+            self, None, qid=0, name='dom0'))
+
+        vm_classes = ["TemplateVm", "TemplateHVm",
+            "AppVm", "HVm", "NetVm", "ProxyVm"]
+        for (vm_class_name) in vm_classes:
+            vms_of_class = tree.findall("Qubes" + vm_class_name)
+            # first non-template based, then template based
+            sorted_vms_of_class = sorted(vms_of_class,
+                key=lambda x: str(x.get('template_qid')).lower() != "none")
+            for element in sorted_vms_of_class:
+                try:
+                    kwargs = {}
+                    if vm_class_name in ["TemplateVm", "TemplateHVm"]:
+                        vm_class = qubes.vm.templatevm.TemplateVM
+                    elif element.get('template_qid').lower() == "none":
+                        kwargs['dir_path'] = element.get('dir_path')
+                        vm_class = StandaloneVM
+                    else:
+                        kwargs['dir_path'] = element.get('dir_path')
+                        kwargs['template'] = int(element.get('template_qid'))
+                        vm_class = AppVM
+                    # simple attributes
+                    for attr in ['installed_by_rpm', 'include_in_backups',
+                            'qrexec_timeout', 'internal', 'label', 'name',
+                            'vcpus', 'memory', 'maxmem', 'default_user',
+                            'debug', 'pci_strictreset', 'mac', 'autostart',
+                            'backup_content', 'backup_path', 'backup_size']:
+                        value = element.get(attr)
+                        if value:
+                            kwargs[attr] = value
+                    # attributes with default value
+                    for attr in ["kernel", "kernelopts"]:
+                        value = element.get(attr)
+                        if value and value.lower() == "none":
+                            value = None
+                        value_is_default = element.get(
+                            "uses_default_{}".format(attr))
+                        if value_is_default and value_is_default.lower() != \
+                                "true":
+                            kwargs[attr] = value
+                    kwargs['hvm'] = "HVm" in vm_class_name
+                    vm = self.add_new_vm(vm_class,
+                        qid=int(element.get('qid')), **kwargs)
+                    services = element.get('services')
+                    if services:
+                        services = eval(services)
+                    else:
+                        services = {}
+                    for service, value in services.iteritems():
+                        feature = service
+                        for repl_feature, repl_service in \
+                                qubes.ext.r3compatibility.\
+                                R3Compatibility.features_to_services.\
+                                iteritems():
+                            if repl_service == service:
+                                feature = repl_feature
+                        vm.features[feature] = value
+                    pcidevs = element.get('pcidevs')
+                    if pcidevs:
+                        pcidevs = eval(pcidevs)
+                    for pcidev in pcidevs:
+                        try:
+                            vm.devices["pci"].attach(pcidev)
+                        except qubes.exc.QubesException as e:
+                            self.log.error("VM {}: {}".format(vm.name, str(e)))
+                except (ValueError, LookupError) as err:
+                    self.log.error("import error ({1}): {2}".format(
+                        vm_class_name, err))
+                    if 'vm' in locals():
+                        del self.domains[vm]
+
+        # After importing all VMs, set netvm references, in the same order
+        for vm_class_name in vm_classes:
+            for element in tree.findall("Qubes" + vm_class_name):
+                try:
+                    self.set_netvm_dependency(element)
+                except (ValueError, LookupError) as err:
+                    self.log.error("VM {}: failed to set netvm dependency: {}".
+                        format(element.get('name'), err))
+
+        self.load_globals(tree.getroot())
+
+    def save(self):
+        raise NotImplementedError("Saving old qubes.xml not supported")
diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec
index 962b10ba..136ffb4f 100644
--- a/rpm_spec/core-dom0.spec
+++ b/rpm_spec/core-dom0.spec
@@ -203,6 +203,7 @@ fi
 %{python_sitelib}/qubes/__init__.py*
 %{python_sitelib}/qubes/backup.py*
 %{python_sitelib}/qubes/config.py*
+%{python_sitelib}/qubes/core2migration.py*
 %{python_sitelib}/qubes/devices.py*
 %{python_sitelib}/qubes/dochelpers.py*
 %{python_sitelib}/qubes/events.py*

From c55ca8004a1ee80ed7b1e5a3cf601d700a169795 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Wed, 16 Mar 2016 01:01:58 +0100
Subject: [PATCH 16/32] tests: port backupcompatibility to core3 API

---
 qubes/tests/__init__.py                       |  2 +-
 .../tests/int}/backupcompatibility.py         | 88 ++++++++++---------
 rpm_spec/core-dom0.spec                       |  1 +
 3 files changed, 49 insertions(+), 42 deletions(-)
 rename {tests => qubes/tests/int}/backupcompatibility.py (93%)

diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py
index b05348b4..09d74a3e 100644
--- a/qubes/tests/__init__.py
+++ b/qubes/tests/__init__.py
@@ -822,7 +822,7 @@ def load_tests(loader, tests, pattern): # pylint: disable=unused-argument
             'qubes.tests.int.network',
 #           'qubes.tests.vm_qrexec_gui',
             'qubes.tests.int.backup',
-#           'qubes.tests.backupcompatibility',
+            'qubes.tests.int.backupcompatibility',
 #           'qubes.tests.regressions',
 
             # tool tests
diff --git a/tests/backupcompatibility.py b/qubes/tests/int/backupcompatibility.py
similarity index 93%
rename from tests/backupcompatibility.py
rename to qubes/tests/int/backupcompatibility.py
index 775db92f..939e6eaf 100644
--- a/tests/backupcompatibility.py
+++ b/qubes/tests/int/backupcompatibility.py
@@ -29,8 +29,6 @@ import subprocess
 import unittest
 import sys
 import re
-from qubes.qubes import QubesVmCollection, QubesException
-from qubes import backup
 
 import qubes.tests
 
@@ -146,6 +144,11 @@ compression-filter=gzip
 '''
 
 class TC_00_BackupCompatibility(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
+
+    def tearDown(self):
+        self.remove_test_vms(prefix="test-")
+        super(TC_00_BackupCompatibility, self).tearDown()
+
     def create_whitelisted_appmenus(self, filename):
         f = open(filename, "w")
         f.write("gnome-terminal.desktop\n")
@@ -401,19 +404,22 @@ class TC_00_BackupCompatibility(qubes.tests.BackupTestsMixin, qubes.tests.QubesT
         f.write(QUBESXML_R1)
         f.close()
 
-        self.restore_backup(self.backupdir, options={
-            'use-default-template': True,
-            'use-default-netvm': True,
-        })
-        self.assertIsNotNone(self.qc.get_vm_by_name("test-template-clone"))
-        self.assertIsNotNone(self.qc.get_vm_by_name("test-testproxy"))
-        self.assertIsNotNone(self.qc.get_vm_by_name("test-work"))
-        self.assertIsNotNone(self.qc.get_vm_by_name("test-standalonevm"))
-        self.assertIsNotNone(self.qc.get_vm_by_name(
-            "test-custom-template-appvm"))
-        self.assertEqual(self.qc.get_vm_by_name("test-custom-template-appvm")
+        self.restore_backup(self.backupdir,
+            options={
+                'use-default-template': True,
+                'use-default-netvm': True,
+            },
+            expect_errors=['Kernel None not installed, using default one']
+        )
+        with self.assertNotRaises(KeyError):
+            vm = self.app.domains["test-template-clone"]
+            vm = self.app.domains["test-testproxy"]
+            vm = self.app.domains["test-work"]
+            vm = self.app.domains["test-standalonevm"]
+            vm = self.app.domains["test-custom-template-appvm"]
+        self.assertEqual(self.app.domains["test-custom-template-appvm"]
                          .template,
-                         self.qc.get_vm_by_name("test-template-clone"))
+                         self.app.domains["test-template-clone"])
 
     def test_200_r2b2(self):
         self.create_v1_files(r2b2=True)
@@ -425,16 +431,16 @@ class TC_00_BackupCompatibility(qubes.tests.BackupTestsMixin, qubes.tests.QubesT
         self.restore_backup(self.backupdir, options={
             'use-default-template': True,
         })
-        self.assertIsNotNone(self.qc.get_vm_by_name("test-template-clone"))
-        self.assertIsNotNone(self.qc.get_vm_by_name("test-testproxy"))
-        self.assertIsNotNone(self.qc.get_vm_by_name("test-work"))
-        self.assertIsNotNone(self.qc.get_vm_by_name("test-testhvm"))
-        self.assertIsNotNone(self.qc.get_vm_by_name("test-standalonevm"))
-        self.assertIsNotNone(self.qc.get_vm_by_name(
-            "test-custom-template-appvm"))
-        self.assertEqual(self.qc.get_vm_by_name("test-custom-template-appvm")
+        with self.assertNotRaises(KeyError):
+            vm = self.app.domains["test-template-clone"]
+            vm = self.app.domains["test-testproxy"]
+            vm = self.app.domains["test-work"]
+            vm = self.app.domains["test-testhvm"]
+            vm = self.app.domains["test-standalonevm"]
+            vm = self.app.domains["test-custom-template-appvm"]
+        self.assertEqual(self.app.domains["test-custom-template-appvm"]
                          .template,
-                         self.qc.get_vm_by_name("test-template-clone"))
+                         self.app.domains["test-template-clone"])
 
     def test_210_r2(self):
         self.create_v3_backup(False)
@@ -443,16 +449,16 @@ class TC_00_BackupCompatibility(qubes.tests.BackupTestsMixin, qubes.tests.QubesT
             'use-default-template': True,
             'use-default-netvm': True,
         })
-        self.assertIsNotNone(self.qc.get_vm_by_name("test-template-clone"))
-        self.assertIsNotNone(self.qc.get_vm_by_name("test-testproxy"))
-        self.assertIsNotNone(self.qc.get_vm_by_name("test-work"))
-        self.assertIsNotNone(self.qc.get_vm_by_name("test-testhvm"))
-        self.assertIsNotNone(self.qc.get_vm_by_name("test-standalonevm"))
-        self.assertIsNotNone(self.qc.get_vm_by_name(
-            "test-custom-template-appvm"))
-        self.assertEqual(self.qc.get_vm_by_name("test-custom-template-appvm")
+        with self.assertNotRaises(KeyError):
+            vm = self.app.domains["test-template-clone"]
+            vm = self.app.domains["test-testproxy"]
+            vm = self.app.domains["test-work"]
+            vm = self.app.domains["test-testhvm"]
+            vm = self.app.domains["test-standalonevm"]
+            vm = self.app.domains["test-custom-template-appvm"]
+        self.assertEqual(self.app.domains["test-custom-template-appvm"]
                          .template,
-                         self.qc.get_vm_by_name("test-template-clone"))
+                         self.app.domains["test-template-clone"])
 
     def test_220_r2_encrypted(self):
         self.create_v3_backup(True)
@@ -461,13 +467,13 @@ class TC_00_BackupCompatibility(qubes.tests.BackupTestsMixin, qubes.tests.QubesT
             'use-default-template': True,
             'use-default-netvm': True,
         })
-        self.assertIsNotNone(self.qc.get_vm_by_name("test-template-clone"))
-        self.assertIsNotNone(self.qc.get_vm_by_name("test-testproxy"))
-        self.assertIsNotNone(self.qc.get_vm_by_name("test-work"))
-        self.assertIsNotNone(self.qc.get_vm_by_name("test-testhvm"))
-        self.assertIsNotNone(self.qc.get_vm_by_name("test-standalonevm"))
-        self.assertIsNotNone(self.qc.get_vm_by_name(
-            "test-custom-template-appvm"))
-        self.assertEqual(self.qc.get_vm_by_name("test-custom-template-appvm")
+        with self.assertNotRaises(KeyError):
+            vm = self.app.domains["test-template-clone"]
+            vm = self.app.domains["test-testproxy"]
+            vm = self.app.domains["test-work"]
+            vm = self.app.domains["test-testhvm"]
+            vm = self.app.domains["test-standalonevm"]
+            vm = self.app.domains["test-custom-template-appvm"]
+        self.assertEqual(self.app.domains["test-custom-template-appvm"]
                          .template,
-                         self.qc.get_vm_by_name("test-template-clone"))
+                         self.app.domains["test-template-clone"])
diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec
index 136ffb4f..488e80eb 100644
--- a/rpm_spec/core-dom0.spec
+++ b/rpm_spec/core-dom0.spec
@@ -275,6 +275,7 @@ fi
 %dir %{python_sitelib}/qubes/tests/int
 %{python_sitelib}/qubes/tests/int/__init__.py*
 %{python_sitelib}/qubes/tests/int/backup.py*
+%{python_sitelib}/qubes/tests/int/backupcompatibility.py*
 %{python_sitelib}/qubes/tests/int/basic.py*
 %{python_sitelib}/qubes/tests/int/dom0_update.py*
 %{python_sitelib}/qubes/tests/int/network.py*

From ad1f1738fa9249fdb38e4be80be4841ad7329f5d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 2 Apr 2016 23:47:41 +0200
Subject: [PATCH 17/32] qubes: make 'store' property public (R/O)

It makes sense to see from what Qubes object was loaded.
---
 qubes/__init__.py | 3 +++
 qubes/backup.py   | 3 +--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/qubes/__init__.py b/qubes/__init__.py
index 1a805cc6..e8758668 100644
--- a/qubes/__init__.py
+++ b/qubes/__init__.py
@@ -1224,6 +1224,9 @@ class Qubes(PropertyHolder):
 
         self.events_enabled = True
 
+    @__builtin__.property
+    def store(self):
+        return self._store
 
     def load(self):
         '''Open qubes.xml
diff --git a/qubes/backup.py b/qubes/backup.py
index fd9b0c59..9961eb2b 100644
--- a/qubes/backup.py
+++ b/qubes/backup.py
@@ -575,8 +575,7 @@ class Backup(object):
     def backup_do(self):
         if self.passphrase is None:
             raise qubes.exc.QubesException("No passphrase set")
-        # noinspection PyProtectedMember
-        qubes_xml = self.app._store
+        qubes_xml = self.app.store
         self.tmpdir = tempfile.mkdtemp()
         shutil.copy(qubes_xml, os.path.join(self.tmpdir, 'qubes.xml'))
         qubes_xml = os.path.join(self.tmpdir, 'qubes.xml')

From c08f5986a99e76fbe81d217a534b2171c6253cc8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 2 Apr 2016 23:51:37 +0200
Subject: [PATCH 18/32] backup: use constants for queue control strings

---
 qubes/backup.py | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/qubes/backup.py b/qubes/backup.py
index 9961eb2b..3fdb8c66 100644
--- a/qubes/backup.py
+++ b/qubes/backup.py
@@ -41,6 +41,10 @@ from multiprocessing import Queue, Process
 import qubes
 import qubes.core2migration
 
+QUEUE_ERROR = "ERROR"
+
+QUEUE_FINISHED = "FINISHED"
+
 HEADER_FILENAME = 'backup-header'
 DEFAULT_CRYPTO_ALGORITHM = 'aes-256-cbc'
 DEFAULT_HMAC_ALGORITHM = 'SHA512'
@@ -191,7 +195,7 @@ class SendWorker(Process):
         os.chdir(self.base_dir)
 
         for filename in iter(self.queue.get, None):
-            if filename == "FINISHED" or filename == "ERROR":
+            if filename in (QUEUE_FINISHED, QUEUE_ERROR):
                 break
 
             self.log.debug("Sending file {}".format(filename))
@@ -759,7 +763,7 @@ class Backup(object):
                             pass
                         tar_sparse.wait()
                         hmac.wait()
-                        to_send.put("ERROR")
+                        to_send.put(QUEUE_ERROR)
                         send_proc.join()
                         shutil.rmtree(self.tmpdir)
                         raise BackupCanceledError("Backup canceled")
@@ -813,7 +817,7 @@ class Backup(object):
             self._current_vm_bytes = 0
             self._send_progress_update()
 
-        self._queue_put_with_check(send_proc, vmproc, to_send, "FINISHED")
+        self._queue_put_with_check(send_proc, vmproc, to_send, QUEUE_FINISHED)
         send_proc.join()
         shutil.rmtree(self.tmpdir)
 
@@ -1013,7 +1017,7 @@ class ExtractWorker2(Process):
         filename = None
 
         for filename in iter(self.queue.get, None):
-            if filename == "FINISHED" or filename == "ERROR":
+            if filename in (QUEUE_FINISHED, QUEUE_ERROR):
                 break
 
             self.log.debug("Extracting file " + filename)
@@ -1128,7 +1132,7 @@ class ExtractWorker2(Process):
         os.unlink(self.restore_pipe)
 
         if self.tar2_process is not None:
-            if filename == "ERROR":
+            if filename == QUEUE_ERROR:
                 self.tar2_process.terminate()
                 self.tar2_process.wait()
             elif self.tar2_process.wait() != 0:
@@ -1169,7 +1173,7 @@ class ExtractWorker3(ExtractWorker2):
 
         input_pipe = None
         for filename in iter(self.queue.get, None):
-            if filename == "FINISHED" or filename == "ERROR":
+            if filename in (QUEUE_FINISHED, QUEUE_ERROR):
                 break
 
             self.log.debug("Extracting file " + filename)
@@ -1274,7 +1278,7 @@ class ExtractWorker3(ExtractWorker2):
 
         if self.tar2_process is not None:
             input_pipe.close()
-            if filename == "ERROR":
+            if filename == QUEUE_ERROR:
                 if self.decryptor_process:
                     self.decryptor_process.terminate()
                     self.decryptor_process.wait()
@@ -1608,7 +1612,7 @@ class BackupRestore(object):
     def _start_inner_extraction_worker(self, queue):
         """Start a worker process, extracting inner layer of bacup archive,
         extract them to :py:attr:`tmpdir`.
-        End the data by pushing "FINISHED" or "ERROR" to the queue.
+        End the data by pushing QUEUE_FINISHED or QUEUE_ERROR to the queue.
 
         :param queue :py:class:`Queue` object to handle files from
         """
@@ -1652,7 +1656,7 @@ class BackupRestore(object):
             self._verify_hmac("qubes.xml.000", "qubes.xml.000.hmac")
             queue = Queue()
             queue.put("qubes.xml.000")
-            queue.put("FINISHED")
+            queue.put(QUEUE_FINISHED)
 
         extract_proc = self._start_inner_extraction_worker(queue)
         extract_proc.join()
@@ -1761,11 +1765,11 @@ class BackupRestore(object):
                 raise qubes.exc.QubesException(
                     "Premature end of archive, the last file was %s" % filename)
         except:
-            to_extract.put("ERROR")
+            to_extract.put(QUEUE_ERROR)
             extract_proc.join()
             raise
         else:
-            to_extract.put("FINISHED")
+            to_extract.put(QUEUE_FINISHED)
 
         self.log.debug("Waiting for the extraction process to finish...")
         extract_proc.join()

From e9b8b6428c3448234011a86d774ea8e8effb3fcd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 2 Apr 2016 23:53:03 +0200
Subject: [PATCH 19/32] backup: remove duplicated get_disk_usage function

---
 qubes/backup.py | 28 +++-------------------------
 1 file changed, 3 insertions(+), 25 deletions(-)

diff --git a/qubes/backup.py b/qubes/backup.py
index 3fdb8c66..9cc938db 100644
--- a/qubes/backup.py
+++ b/qubes/backup.py
@@ -40,6 +40,7 @@ import datetime
 from multiprocessing import Queue, Process
 import qubes
 import qubes.core2migration
+import qubes.storage
 
 QUEUE_ERROR = "ERROR"
 
@@ -58,29 +59,6 @@ HEADER_QUBES_XML_MAX_SIZE = 1024 * 1024
 BLKSIZE = 512
 
 
-def get_disk_usage_one(st):
-    try:
-        return st.st_blocks * BLKSIZE
-    except AttributeError:
-        return st.st_size
-
-
-def get_disk_usage(path):
-    try:
-        st = os.lstat(path)
-    except OSError:
-        return 0
-
-    ret = get_disk_usage_one(st)
-
-    # if path is not a directory, this is skipped
-    for dirpath, dirnames, filenames in os.walk(path):
-        for name in dirnames + filenames:
-            ret += get_disk_usage_one(os.lstat(os.path.join(dirpath, name)))
-
-    return ret
-
-
 class BackupCanceledError(qubes.exc.QubesException):
     def __init__(self, msg, tmpdir=None):
         super(BackupCanceledError, self).__init__(msg)
@@ -324,7 +302,7 @@ class Backup(object):
 
     @staticmethod
     def _file_to_backup(file_path, subdir=None):
-        sz = get_disk_usage(file_path)
+        sz = qubes.storage.get_disk_usage(file_path)
 
         if subdir is None:
             abs_file_path = os.path.abspath(file_path)
@@ -438,7 +416,7 @@ class Backup(object):
             # left after 'sudo bash' and similar commands
             subprocess.check_call(['sudo', 'chown', '-R', local_user, home_dir])
 
-            home_sz = get_disk_usage(home_dir)
+            home_sz = qubes.storage.get_disk_usage(home_dir)
             home_to_backup = [
                 {"path": home_dir, "size": home_sz, "subdir": 'dom0-home/'}]
             vm_files = home_to_backup

From 78dbadab5756d8bda1a1d9b0ed4d4edf5e0762b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 2 Apr 2016 23:53:56 +0200
Subject: [PATCH 20/32] backup: minor: compile regex once

---
 qubes/backup.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/qubes/backup.py b/qubes/backup.py
index 9cc938db..0b6160c4 100644
--- a/qubes/backup.py
+++ b/qubes/backup.py
@@ -58,6 +58,7 @@ HEADER_QUBES_XML_MAX_SIZE = 1024 * 1024
 
 BLKSIZE = 512
 
+_re_alphanum = re.compile(r'^[A-Za-z0-9-]*$')
 
 class BackupCanceledError(qubes.exc.QubesException):
     def __init__(self, msg, tmpdir=None):
@@ -109,12 +110,12 @@ class BackupHeader(object):
             if untrusted_line.count('=') != 1:
                 raise qubes.exc.QubesException("Invalid backup header")
             (key, value) = untrusted_line.strip().split('=')
-            if not re.match(r"^[a-zA-Z0-9-]*$", key):
+            if not _re_alphanum.match(key):
                 raise qubes.exc.QubesException("Invalid backup header (key)")
             if key not in self.header_keys.keys():
                 # Ignoring unknown option
                 continue
-            if not re.match(r"^[a-zA-Z0-9-]*$", value):
+            if not _re_alphanum.match(value):
                 raise qubes.exc.QubesException("Invalid backup header (value)")
             if getattr(self, self.header_keys[key]) is not None:
                 raise qubes.exc.QubesException(

From d9cf64a41da50b7219c13127d69760c4138e5f5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sat, 2 Apr 2016 23:56:00 +0200
Subject: [PATCH 21/32] backup: minor code structure changes

- initialize internal objects object in constructor
- use 'with'
---
 qubes/backup.py | 67 ++++++++++++++++++++++++++-----------------------
 1 file changed, 36 insertions(+), 31 deletions(-)

diff --git a/qubes/backup.py b/qubes/backup.py
index 0b6160c4..b648ed1f 100644
--- a/qubes/backup.py
+++ b/qubes/backup.py
@@ -78,16 +78,23 @@ class BackupHeader(object):
     bool_options = ['encrypted', 'compressed']
     int_options = ['version']
 
-    def __init__(self, header_data=None):
+    def __init__(self,
+            header_data=None,
+            version=None,
+            encrypted=None,
+            compressed=None,
+            compression_filter=None,
+            hmac_algorithm=None,
+            crypto_algorithm=None):
         # repeat the list to help code completion...
-        self.version = None
-        self.encrypted = None
-        self.compressed = None
+        self.version = version
+        self.encrypted = encrypted
+        self.compressed = compressed
         # Options introduced in backup format 3+, which always have a header,
         # so no need for fallback in function parameter
-        self.compression_filter = None
-        self.hmac_algorithm = None
-        self.crypto_algorithm = None
+        self.compression_filter = compression_filter
+        self.hmac_algorithm = hmac_algorithm
+        self.crypto_algorithm = crypto_algorithm
 
         if header_data is not None:
             self.load(header_data)
@@ -109,7 +116,7 @@ class BackupHeader(object):
         for untrusted_line in untrusted_header_text.splitlines():
             if untrusted_line.count('=') != 1:
                 raise qubes.exc.QubesException("Invalid backup header")
-            (key, value) = untrusted_line.strip().split('=')
+            key, value = untrusted_line.strip().split('=', 1)
             if not _re_alphanum.match(key):
                 raise qubes.exc.QubesException("Invalid backup header (key)")
             if key not in self.header_keys.keys():
@@ -283,9 +290,8 @@ class Backup(object):
 
         self.vms_for_backup = vms_list
         # Apply exclude list
-        if exclude_list:
-            self.vms_for_backup = [vm for vm in vms_list
-                if vm.name not in exclude_list]
+        self.vms_for_backup = [vm for vm in vms_list
+            if vm.name not in exclude_list]
 
     def __del__(self):
         if self.tmpdir and os.path.exists(self.tmpdir):
@@ -511,14 +517,14 @@ class Backup(object):
 
     def prepare_backup_header(self):
         header_file_path = os.path.join(self.tmpdir, HEADER_FILENAME)
-        backup_header = BackupHeader()
-        backup_header.version = CURRENT_BACKUP_FORMAT_VERSION
-        backup_header.hmac_algorithm = self.hmac_algorithm
-        backup_header.crypto_algorithm = self.crypto_algorithm
-        backup_header.encrypted = self.encrypted
-        backup_header.compressed = self.compressed
-        if self.compressed:
-            backup_header.compression_filter = self.compression_filter
+        backup_header = BackupHeader(
+            version=CURRENT_BACKUP_FORMAT_VERSION,
+            hmac_algorithm=self.hmac_algorithm,
+            crypto_algorithm=self.crypto_algorithm,
+            encrypted=self.encrypted,
+            compressed=self.compressed,
+            compression_filter=self.compression_filter,
+        )
         backup_header.save(header_file_path)
 
         hmac = subprocess.Popen(
@@ -772,10 +778,8 @@ class Backup(object):
                     hmac_data = hmac.stdout.read()
                     self.log.debug(
                         "Writing hmac to {}.hmac".format(chunkfile))
-                    hmac_file = open(chunkfile + ".hmac", 'w')
-                    hmac_file.write(hmac_data)
-                    hmac_file.flush()
-                    hmac_file.close()
+                    with open(chunkfile + ".hmac", 'w') as hmac_file:
+                        hmac_file.write(hmac_data)
 
                     # Send the HMAC to the backup target
                     self._queue_put_with_check(
@@ -1556,13 +1560,14 @@ class BackupRestore(object):
             os.unlink(filename)
         else:
             # if no header found, create one with guessed HMAC algo
-            header_data = BackupHeader()
-            header_data.version = 2
-            header_data.hmac_algorithm = hmac_algorithm
-            # place explicitly this value, because it is what format_version
-            # 2 have
-            header_data.crypto_algorithm = 'aes-256-cbc'
-            # TODO: set header_data.encrypted to something...
+            header_data = BackupHeader(
+                version=2,
+                hmac_algorithm=hmac_algorithm,
+                # place explicitly this value, because it is what format_version
+                # 2 have
+                crypto_algorithm='aes-256-cbc',
+                # TODO: set encrypted to something...
+            )
             # when tar do not find expected file in archive, it exit with
             # code 2. This will happen because we've requested backup-header
             # file, but the archive do not contain it. Ignore this particular
@@ -1649,7 +1654,7 @@ class BackupRestore(object):
                 os.path.join(self.tmpdir, 'qubes.xml'))
         else:
             backup_app = qubes.Qubes(os.path.join(self.tmpdir, 'qubes.xml'))
-            # Not needed anymore - all the data stored in backup_app
+        # Not needed anymore - all the data stored in backup_app
         os.unlink(os.path.join(self.tmpdir, 'qubes.xml'))
         return backup_app
 

From c01f43a5d3c2c0d4bd779d1021ff2884017f3313 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sun, 3 Apr 2016 00:22:39 +0200
Subject: [PATCH 22/32] backup: unify handling of template and non-template VMs

Handle them the same way - individual files, not the whole directory for
templates.
Also don't backup obsolete 'kernels' subdir - it isn't supported in
core3.
---
 qubes/backup.py | 38 +-------------------------------------
 1 file changed, 1 insertion(+), 37 deletions(-)

diff --git a/qubes/backup.py b/qubes/backup.py
index b648ed1f..d70186d4 100644
--- a/qubes/backup.py
+++ b/qubes/backup.py
@@ -329,9 +329,6 @@ class Backup(object):
         files_to_backup = {}
         total_backup_sz = 0
         for vm in self.vms_for_backup:
-            if vm.is_template():
-                # handle templates later
-                continue
             if vm.qid == 0:
                 # handle dom0 later
                 continue
@@ -345,10 +342,7 @@ class Backup(object):
             if vm.private_img is not None:
                 vm_files += self._file_to_backup(vm.private_img, subdir)
 
-            # TODO: don't backup the icon
-            if vm.is_appvm():
-                vm_files += self._file_to_backup(
-                    vm.icon_path, subdir)
+            vm_files += self._file_to_backup(vm.icon_path, subdir)
             if vm.updateable:
                 if os.path.exists(vm.dir_path + "/apps.templates"):
                     # template
@@ -359,9 +353,6 @@ class Backup(object):
                     vm_files += self._file_to_backup(
                         vm.dir_path + "/apps", subdir)
 
-                if os.path.exists(vm.dir_path + "/kernels"):
-                    vm_files += self._file_to_backup(
-                        vm.dir_path + "/kernels", subdir)
             # TODO: drop after merging firewall.xml into qubes.xml
             firewall_conf = os.path.join(vm.dir_path, vm.firewall_conf)
             if os.path.exists(firewall_conf):
@@ -387,33 +378,6 @@ class Backup(object):
             }
             total_backup_sz += vm_size
 
-        for vm in self.vms_for_backup:
-            if not vm.is_template():
-                # already handled
-                continue
-            if vm.qid == 0:
-                # handle dom0 later
-                continue
-            vm_sz = vm.get_disk_utilization()
-            if self.encrypted:
-                template_subdir = 'vm%d/' % vm.qid
-            else:
-                template_subdir = os.path.relpath(
-                    vm.dir_path,
-                    qubes.config.system_path["qubes_base_dir"]) + '/'
-            template_to_backup = [{"path": vm.dir_path + '/.',
-                                   "size": vm_sz,
-                                   "subdir": template_subdir}]
-            vm_files = template_to_backup
-            vm_size = reduce(lambda x, y: x + y['size'], vm_files, 0)
-            files_to_backup[vm.qid] = {
-                'vm': vm,
-                'files': vm_files,
-                'subdir': template_subdir,
-                'size': vm_size,
-            }
-            total_backup_sz += vm_size
-
         # Dom0 user home
         if 0 in [vm.qid for vm in self.vms_for_backup]:
             local_user = grp.getgrnam('qubes').gr_mem[0]

From b4227f758175562e6382c30498f82fabf465d80d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sun, 3 Apr 2016 03:09:15 +0200
Subject: [PATCH 23/32] backup: use simple classes instead of hashes, use
 events for appmenus

This makes typo errors much easier to find (also using pylint or so).
While at it, also removed explicit appmenus backup, as it should be
provided by appmenus extension.
---
 qubes/backup.py | 140 +++++++++++++++++++++---------------------------
 1 file changed, 62 insertions(+), 78 deletions(-)

diff --git a/qubes/backup.py b/qubes/backup.py
index d70186d4..1adc90b7 100644
--- a/qubes/backup.py
+++ b/qubes/backup.py
@@ -212,6 +212,36 @@ class SendWorker(Process):
 
 
 class Backup(object):
+    class FileToBackup(object):
+        def __init__(self, file_path, subdir=None):
+            sz = qubes.storage.get_disk_usage(file_path)
+
+            if subdir is None:
+                abs_file_path = os.path.abspath(file_path)
+                abs_base_dir = os.path.abspath(
+                    qubes.config.system_path["qubes_base_dir"]) + '/'
+                abs_file_dir = os.path.dirname(abs_file_path) + '/'
+                (nothing, directory, subdir) = abs_file_dir.partition(abs_base_dir)
+                assert nothing == ""
+                assert directory == abs_base_dir
+            else:
+                if len(subdir) > 0 and not subdir.endswith('/'):
+                    subdir += '/'
+
+            self.path = file_path
+            self.size = sz
+            self.subdir = subdir
+
+    class VMToBackup(object):
+        def __init__(self, vm, files, subdir):
+            self.vm = vm
+            self.files = files
+            self.subdir = subdir
+
+        @property
+        def size(self):
+            return reduce(lambda x, y: x + y.size, self.files, 0)
+
     def __init__(self, app, vms_list=None, exclude_list=None, **kwargs):
         """
         If vms = None, include all (sensible) VMs;
@@ -307,27 +337,9 @@ class Backup(object):
             except OSError:
                 pass
 
-    @staticmethod
-    def _file_to_backup(file_path, subdir=None):
-        sz = qubes.storage.get_disk_usage(file_path)
-
-        if subdir is None:
-            abs_file_path = os.path.abspath(file_path)
-            abs_base_dir = os.path.abspath(
-                qubes.config.system_path["qubes_base_dir"]) + '/'
-            abs_file_dir = os.path.dirname(abs_file_path) + '/'
-            (nothing, directory, subdir) = abs_file_dir.partition(abs_base_dir)
-            assert nothing == ""
-            assert directory == abs_base_dir
-        else:
-            if len(subdir) > 0 and not subdir.endswith('/'):
-                subdir += '/'
-        return [{"path": file_path, "size": sz, "subdir": subdir}]
-
 
     def get_files_to_backup(self):
         files_to_backup = {}
-        total_backup_sz = 0
         for vm in self.vms_for_backup:
             if vm.qid == 0:
                 # handle dom0 later
@@ -340,43 +352,20 @@ class Backup(object):
 
             vm_files = []
             if vm.private_img is not None:
-                vm_files += self._file_to_backup(vm.private_img, subdir)
+                vm_files.append(self.FileToBackup(vm.private_img, subdir))
 
-            vm_files += self._file_to_backup(vm.icon_path, subdir)
-            if vm.updateable:
-                if os.path.exists(vm.dir_path + "/apps.templates"):
-                    # template
-                    vm_files += self._file_to_backup(
-                        vm.dir_path + "/apps.templates", subdir)
-                else:
-                    # standaloneVM
-                    vm_files += self._file_to_backup(
-                        vm.dir_path + "/apps", subdir)
+            vm_files.append(self.FileToBackup(vm.icon_path, subdir))
+            vm_files.extend(self.FileToBackup(i, subdir)
+                for i in vm.fire_event('backup-get-files'))
 
             # TODO: drop after merging firewall.xml into qubes.xml
             firewall_conf = os.path.join(vm.dir_path, vm.firewall_conf)
             if os.path.exists(firewall_conf):
-                vm_files += self._file_to_backup(
-                    firewall_conf, subdir)
-            if 'appmenus_whitelist' in qubes.config.vm_files and \
-                    os.path.exists(os.path.join(vm.dir_path,
-                                                qubes.config.vm_files[
-                                                    'appmenus_whitelist'])):
-                vm_files += self._file_to_backup(
-                    os.path.join(vm.dir_path, qubes.config.vm_files[
-                        'appmenus_whitelist']),
-                    subdir)
+                vm_files.append(self.FileToBackup(firewall_conf, subdir))
 
             if vm.updateable:
-                vm_files += self._file_to_backup(vm.root_img, subdir)
-            vm_size = reduce(lambda x, y: x + y['size'], vm_files, 0)
-            files_to_backup[vm.qid] = {
-                'vm': vm,
-                'files': vm_files,
-                'subdir': subdir,
-                'size': vm_size,
-            }
-            total_backup_sz += vm_size
+                vm_files.append(self.FileToBackup(vm.root_img, subdir))
+            files_to_backup[vm.qid] = self.VMToBackup(vm, vm_files, subdir)
 
         # Dom0 user home
         if 0 in [vm.qid for vm in self.vms_for_backup]:
@@ -387,20 +376,16 @@ class Backup(object):
             # left after 'sudo bash' and similar commands
             subprocess.check_call(['sudo', 'chown', '-R', local_user, home_dir])
 
-            home_sz = qubes.storage.get_disk_usage(home_dir)
             home_to_backup = [
-                {"path": home_dir, "size": home_sz, "subdir": 'dom0-home/'}]
+                self.FileToBackup(home_dir, 'dom0-home/')]
             vm_files = home_to_backup
 
-            files_to_backup[0] = {
-                'vm': self.app.domains[0],
-                'files': vm_files,
-                'subdir': os.path.join('dom0-home', os.path.basename(home_dir)),
-                'size': home_sz,
-            }
-            total_backup_sz += home_sz
+            files_to_backup[0] = self.VMToBackup(self.app.domains[0],
+                vm_files,
+                os.path.join('dom0-home', os.path.basename(home_dir)))
 
-        self.total_backup_bytes = total_backup_sz
+        self.total_backup_bytes = reduce(
+            lambda x, y: x + y.size, files_to_backup.values(), 0)
         return files_to_backup
 
 
@@ -541,12 +526,12 @@ class Backup(object):
             vm.backup_content = qubes.property.DEFAULT
 
         for qid, vm_info in files_to_backup.iteritems():
-            if qid != 0 and vm_info['vm'].is_running():
-                raise qubes.exc.QubesVMNotHaltedError(vm_info['vm'])
+            if qid != 0 and vm_info.vm.is_running():
+                raise qubes.exc.QubesVMNotHaltedError(vm_info.vm)
             # VM is included in the backup
-            backup_app.domains[qid].backup_content = True
-            backup_app.domains[qid].backup_path = vm_info['subdir']
-            backup_app.domains[qid].backup_size = vm_info['size']
+            backup_app.domains[qid].backup-content = True
+            backup_app.domains[qid].backup-path = vm_info.subdir
+            backup_app.domains[qid].backup-size = vm_info.size
         backup_app.save()
 
         passphrase = self.passphrase.encode('utf-8')
@@ -603,21 +588,20 @@ class Backup(object):
             to_send.put(f)
 
         vm_files_to_backup = self.get_files_to_backup()
-        qubes_xml_info = {
-            'files': self._file_to_backup(qubes_xml, ''),
-            'vm': None,
-            'size': 0,
-            'subdir': '',
-        }
+        qubes_xml_info = self.VMToBackup(
+            None,
+            [self.FileToBackup(qubes_xml, '')],
+            ''
+        )
         for vm_info in itertools.chain([qubes_xml_info],
                 vm_files_to_backup.itervalues()):
-            for file_info in vm_info['files']:
+            for file_info in vm_info.files:
 
                 self.log.debug("Backing up {}".format(file_info))
 
                 backup_tempfile = os.path.join(
-                    self.tmpdir, file_info["subdir"],
-                    os.path.basename(file_info["path"]))
+                    self.tmpdir, file_info.subdir,
+                    os.path.basename(file_info.path))
                 self.log.debug("Using temporary location: {}".format(
                     backup_tempfile))
 
@@ -631,13 +615,13 @@ class Backup(object):
                 # verified during untar
                 tar_cmdline = (["tar", "-Pc", '--sparse',
                                "-f", backup_pipe,
-                               '-C', os.path.dirname(file_info["path"])] +
+                               '-C', os.path.dirname(file_info.path)] +
                                (['--dereference'] if
-                                file_info["subdir"] != "dom0-home/" else []) +
+                                file_info.subdir != "dom0-home/" else []) +
                                ['--xform', 's:^%s:%s\\0:' % (
-                                   os.path.basename(file_info["path"]),
-                                   file_info["subdir"]),
-                                os.path.basename(file_info["path"])
+                                   os.path.basename(file_info.path),
+                                   file_info.subdir),
+                                os.path.basename(file_info.path)
                                 ])
                 if self.compressed:
                     tar_cmdline.insert(-1,
@@ -760,7 +744,7 @@ class Backup(object):
                 pipe.close()
 
             # This VM done, update progress
-            self._done_vms_bytes += vm_info['size']
+            self._done_vms_bytes += vm_info.size
             self._current_vm_bytes = 0
             self._send_progress_update()
 

From a3051b956b375659e220b6e7d64a0bec5b30a7e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sun, 3 Apr 2016 03:11:57 +0200
Subject: [PATCH 24/32] backup: docstrings, don't expose internal attributes

processes_to_kill_on_cancel shouldn't be initialized by user, so move
below applying user settings.
---
 qubes/backup.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/qubes/backup.py b/qubes/backup.py
index 1adc90b7..3c7d2663 100644
--- a/qubes/backup.py
+++ b/qubes/backup.py
@@ -256,8 +256,7 @@ class Backup(object):
         self._done_vms_bytes = 0
         #: total backup size (set by :py:meth:`get_files_to_backup`)
         self.total_backup_bytes = 0
-        #: list of PIDs to kill on backup cancel
-        self.processes_to_kill_on_cancel = []
+        #: application object
         self.app = app
         #: directory for temporary files - set after creating the directory
         self.tmpdir = None
@@ -268,7 +267,7 @@ class Backup(object):
         #: should the backup be compressed?
         self.compressed = True
         #: what passphrase should be used to intergrity protect (and encrypt)
-        #  the backup; required
+        #: the backup; required
         self.passphrase = None
         #: custom hmac algorithm
         self.hmac_algorithm = DEFAULT_HMAC_ALGORITHM
@@ -279,10 +278,10 @@ class Backup(object):
         #: VM to which backup should be sent (if any)
         self.target_vm = None
         #: directory to save backup in (either in dom0 or target VM,
-        # depending on :py:attr:`target_vm`
+        #: depending on :py:attr:`target_vm`
         self.target_dir = None
         #: callback for progress reporting. Will be called with one argument
-        # - progress in percents
+        #: - progress in percents
         self.progress_callback = None
 
         for key, value in kwargs.iteritems():
@@ -293,6 +292,8 @@ class Backup(object):
 
         #: whether backup was canceled
         self.canceled = False
+        #: list of PIDs to kill on backup cancel
+        self.processes_to_kill_on_cancel = []
 
         self.log = logging.getLogger('qubes.backup')
 

From e8f21929ad223a69da5ca1cb84b9f6de08deb27c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sun, 3 Apr 2016 03:13:27 +0200
Subject: [PATCH 25/32] backup: by default include all the VMs with
 vm.include_in_backups

If there is a need for more robust default, it should be handled at that
property.
---
 qubes/backup.py     | 12 +-----------
 qubes/vm/qubesvm.py |  4 ++--
 2 files changed, 3 insertions(+), 13 deletions(-)

diff --git a/qubes/backup.py b/qubes/backup.py
index 3c7d2663..2cbe6cb4 100644
--- a/qubes/backup.py
+++ b/qubes/backup.py
@@ -308,18 +308,8 @@ class Backup(object):
             exclude_list = []
 
         if vms_list is None:
-            all_vms = [vm for vm in app.domains]
-            selected_vms = [vm for vm in all_vms if vm.include_in_backups]
-            appvms_to_backup = [vm for vm in selected_vms if
-                                vm.is_appvm() and not vm.internal]
-            template_vms_worth_backingup = [vm for vm in selected_vms if (
-                vm.is_template() and vm.include_in_backups)]
-            dom0 = [app.domains[0]]
+            vms_list = [vm for vm in app.domains if vm.include_in_backups]
 
-            vms_list = appvms_to_backup + \
-                template_vms_worth_backingup + dom0
-
-        self.vms_for_backup = vms_list
         # Apply exclude list
         self.vms_for_backup = [vm for vm in vms_list
             if vm.name not in exclude_list]
diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py
index 52e44c43..9fd3769d 100644
--- a/qubes/vm/qubesvm.py
+++ b/qubes/vm/qubesvm.py
@@ -262,8 +262,8 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
         doc='''Setting this to `True` means that VM should be autostarted on
             dom0 boot.''')
 
-    # XXX I don't understand backups
-    include_in_backups = qubes.property('include_in_backups', default=True,
+    include_in_backups = qubes.property('include_in_backups',
+        default=(lambda self: not self.internal),
         type=bool, setter=qubes.property.bool,
         doc='If this domain is to be included in default backup.')
 

From ed88b5a9a381cd1f297456fa6999b5d5d5a526bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sun, 3 Apr 2016 03:19:30 +0200
Subject: [PATCH 26/32] backup: use vm.run_service instead of vm.run

This should be the only way used to call services.
---
 qubes/backup.py | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/qubes/backup.py b/qubes/backup.py
index 2cbe6cb4..235123e3 100644
--- a/qubes/backup.py
+++ b/qubes/backup.py
@@ -531,11 +531,9 @@ class Backup(object):
         tar_sparse = None
         if self.target_vm is not None:
             # Prepare the backup target (Qubes service call)
-            backup_target = "QUBESRPC qubes.Backup dom0"
-
             # If APPVM, STDOUT is a PIPE
-            vmproc = self.target_vm.run(
-                command=backup_target, passio_popen=True, passio_stderr=True)
+            vmproc = self.target_vm.run_service('qubes.Backup',
+                passio_popen=True, passio_stderr=True)
             vmproc.stdin.write(self.target_dir.
                                replace("\r", "").replace("\n", "") + "\n")
             backup_stdout = vmproc.stdin
@@ -1340,12 +1338,9 @@ class BackupRestore(object):
 
         vmproc = None
         if self.backup_vm is not None:
-            # Prepare the backup target (Qubes service call)
-            backup_target = "QUBESRPC qubes.Restore dom0"
-
             # If APPVM, STDOUT is a PIPE
-            vmproc = self.backup_vm.run(
-                command=backup_target, passio_popen=True, passio_stderr=True)
+            vmproc = self.backup_vm.run_service('qubes.Restore',
+                passio_popen=True, passio_stderr=True)
             vmproc.stdin.write(
                 self.backup_location.replace("\r", "").replace("\n", "") + "\n")
 

From 424d3054f3cff3da4f5531fc073214b0e9ff7781 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Sun, 3 Apr 2016 03:33:58 +0200
Subject: [PATCH 27/32] backup: use vm.features to store backup-specific
 metadata

Do not keep them in system qubes.xml.
---
 qubes/backup.py           | 39 ++++++++++++++++++++++-----------------
 qubes/core2migration.py   |  7 +++++--
 qubes/tests/vm/qubesvm.py | 16 ----------------
 qubes/vm/qubesvm.py       | 11 -----------
 4 files changed, 27 insertions(+), 46 deletions(-)

diff --git a/qubes/backup.py b/qubes/backup.py
index 235123e3..9a0521ff 100644
--- a/qubes/backup.py
+++ b/qubes/backup.py
@@ -514,15 +514,15 @@ class Backup(object):
         files_to_backup = self.get_files_to_backup()
         # make sure backup_content isn't set initially
         for vm in backup_app.domains:
-            vm.backup_content = qubes.property.DEFAULT
+            vm.features['backup-content'] = False
 
         for qid, vm_info in files_to_backup.iteritems():
             if qid != 0 and vm_info.vm.is_running():
                 raise qubes.exc.QubesVMNotHaltedError(vm_info.vm)
             # VM is included in the backup
-            backup_app.domains[qid].backup-content = True
-            backup_app.domains[qid].backup-path = vm_info.subdir
-            backup_app.domains[qid].backup-size = vm_info.size
+            backup_app.domains[qid].features['backup-content'] = True
+            backup_app.domains[qid].features['backup-path'] = vm_info.subdir
+            backup_app.domains[qid].features['backup-size'] = vm_info.size
         backup_app.save()
 
         passphrase = self.passphrase.encode('utf-8')
@@ -736,6 +736,9 @@ class Backup(object):
             self._done_vms_bytes += vm_info.size
             self._current_vm_bytes = 0
             self._send_progress_update()
+            # Save date of last backup
+            if vm_info.vm:
+                vm_info.vm.backup_timestamp = datetime.datetime.now()
 
         self._queue_put_with_check(send_proc, vmproc, to_send, QUEUE_FINISHED)
         send_proc.join()
@@ -755,11 +758,6 @@ class Backup(object):
                     tar_sparse.poll()))
             vmproc.stdin.close()
 
-        # Save date of last backup
-        for vm in self.app.domains:
-            if vm.backup_content:
-                vm.backup_timestamp = datetime.datetime.now()
-
         self.app.save()
 
 
@@ -1828,8 +1826,8 @@ class BackupRestore(object):
 
     @staticmethod
     def _is_vm_included_in_backup_v2(check_vm):
-        if check_vm.backup_content:
-            return True
+        if 'backup-content' in check_vm.features:
+            return check_vm.features['backup-content']
         else:
             return False
 
@@ -1894,8 +1892,8 @@ class BackupRestore(object):
                         self.backup_location, 'dom0-home'))[0]
                 vms_to_restore['dom0']['size'] = 0  # unknown
             else:
-                vms_to_restore['dom0']['subdir'] = vm.backup_path
-                vms_to_restore['dom0']['size'] = vm.backup_size
+                vms_to_restore['dom0']['subdir'] = vm.features['backup-path']
+                vms_to_restore['dom0']['size'] = int(vm.features['backup-size'])
             local_user = grp.getgrnam('qubes').gr_mem[0]
 
             dom0_home = vms_to_restore['dom0']['subdir']
@@ -2057,8 +2055,9 @@ class BackupRestore(object):
                 continue
             vm = vm_info['vm']
             if self.header_data.version >= 2:
-                vms_size += vm.backup_size
-                vms_dirs.append(vm.backup_path)
+                if vm.features['backup-size']:
+                    vms_size += int(vm.features['backup-size'])
+                vms_dirs.append(vm.features['backup-path'])
             vms[vm.name] = vm
 
         if self.header_data.version >= 2:
@@ -2148,8 +2147,9 @@ class BackupRestore(object):
                         self._restore_vm_dir_v1(vm.dir_path,
                             os.path.dirname(new_vm.dir_path))
                     else:
-                        shutil.move(os.path.join(self.tmpdir, vm.backup_path),
-                                    new_vm.dir_path)
+                        shutil.move(os.path.join(self.tmpdir,
+                            vm.features['backup-path']),
+                            new_vm.dir_path)
 
                     new_vm.verify_files()
                 except Exception as err:
@@ -2169,6 +2169,11 @@ class BackupRestore(object):
                         self.log.warning("Kernel %s not installed, "
                         "using default one" % vm.kernel)
                         vm.kernel = qubes.property.DEFAULT
+                # remove no longer needed backup metadata
+                if 'backup-content' in vm.features:
+                    del vm.features['backup-content']
+                    del vm.features['backup-size']
+                    del vm.features['backup-path']
                 try:
                     # exclude VM references - handled manually according to
                     # restore options
diff --git a/qubes/core2migration.py b/qubes/core2migration.py
index c1518c01..f19fb3c0 100644
--- a/qubes/core2migration.py
+++ b/qubes/core2migration.py
@@ -159,8 +159,7 @@ class Core2Qubes(qubes.Qubes):
                     for attr in ['installed_by_rpm', 'include_in_backups',
                             'qrexec_timeout', 'internal', 'label', 'name',
                             'vcpus', 'memory', 'maxmem', 'default_user',
-                            'debug', 'pci_strictreset', 'mac', 'autostart',
-                            'backup_content', 'backup_path', 'backup_size']:
+                            'debug', 'pci_strictreset', 'mac', 'autostart']:
                         value = element.get(attr)
                         if value:
                             kwargs[attr] = value
@@ -191,6 +190,10 @@ class Core2Qubes(qubes.Qubes):
                             if repl_service == service:
                                 feature = repl_feature
                         vm.features[feature] = value
+                    for attr in ['backup_content', 'backup_path',
+                        'backup_size']:
+                        value = element.get(attr)
+                        vm.features[attr.replace('_', '-')] = value
                     pcidevs = element.get('pcidevs')
                     if pcidevs:
                         pcidevs = eval(pcidevs)
diff --git a/qubes/tests/vm/qubesvm.py b/qubes/tests/vm/qubesvm.py
index e9041aba..95c0a6f6 100644
--- a/qubes/tests/vm/qubesvm.py
+++ b/qubes/tests/vm/qubesvm.py
@@ -639,22 +639,6 @@ class TC_90_QubesVM(qubes.tests.QubesTestCase):
         vm = self.get_vm()
         self._test_generic_bool_property(vm, 'pci_strictreset')
 
-    def test_380_backup_size(self):
-        vm = self.get_vm()
-        self.assertPropertyDefaultValue(vm, 'backup_size', 0)
-        self.assertPropertyValue(vm, 'backup_size', 0, 0, '0')
-        del vm.backup_size
-        self.assertPropertyDefaultValue(vm, 'backup_size', 0)
-        self.assertPropertyValue(vm, 'backup_size', '0', 0, '0')
-        self.assertPropertyValue(vm, 'backup_size', 300, 300, '300')
-
-    def test_390_backup_path(self):
-        vm = self.get_vm()
-        self.assertPropertyDefaultValue(vm, 'backup_path', '')
-        self.assertPropertyValue(vm, 'backup_path', 'some/dir', 'some/dir')
-        del vm.backup_path
-        self.assertPropertyDefaultValue(vm, 'backup_path', '')
-
     def test_400_backup_timestamp(self):
         vm = self.get_vm()
         timestamp = datetime.datetime(2016, 1, 1, 12, 14, 2)
diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py
index 9fd3769d..0015d6c1 100644
--- a/qubes/vm/qubesvm.py
+++ b/qubes/vm/qubesvm.py
@@ -267,17 +267,6 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
         type=bool, setter=qubes.property.bool,
         doc='If this domain is to be included in default backup.')
 
-    backup_content = qubes.property('backup_content', default=False,
-        type=bool, setter=qubes.property.bool,
-        doc='FIXME')
-
-    backup_size = qubes.property('backup_size', type=int, default=0,
-        doc='FIXME')
-
-    # TODO default=None?
-    backup_path = qubes.property('backup_path', type=str, default='',
-        doc='FIXME')
-
     # format got changed from %s to str(datetime.datetime)
     backup_timestamp = qubes.property('backup_timestamp', default=None,
         setter=(lambda self, prop, value:

From 98c8b7cd22d2be2ec33b58636f503dc7c826a16d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 4 Apr 2016 05:19:59 +0200
Subject: [PATCH 28/32] backup: use simple classes for data storage on restore
 too

---
 qubes/backup.py | 230 ++++++++++++++++++++++++++----------------------
 1 file changed, 127 insertions(+), 103 deletions(-)

diff --git a/qubes/backup.py b/qubes/backup.py
index 9a0521ff..8fc5c5d4 100644
--- a/qubes/backup.py
+++ b/qubes/backup.py
@@ -1268,6 +1268,52 @@ class BackupRestore(object):
     >>> restore_op.restore_do(restore_info)
     """
 
+    class VMToRestore(object):
+        #: VM excluded from restore by user
+        EXCLUDED = object()
+        #: VM with such name already exists on the host
+        ALREADY_EXISTS = object()
+        #: NetVM used by the VM does not exists on the host
+        MISSING_NETVM = object()
+        #: TemplateVM used by the VM does not exists on the host
+        MISSING_TEMPLATE = object()
+
+        def __init__(self, vm):
+            self.vm = vm
+            if 'backup-path' in vm.features:
+                self.subdir = vm.features['backup-path']
+            else:
+                self.subdir = None
+            if 'backup-size' in vm.features and vm.features['backup-size']:
+                self.size = int(vm.features['backup-size'])
+            else:
+                self.size = 0
+            self.problems = set()
+            if hasattr(vm, 'template') and vm.template:
+                self.template = vm.template.name
+            else:
+                self.template = None
+            if vm.netvm:
+                self.netvm = vm.netvm.name
+            else:
+                self.netvm = None
+            self.rename_to = None
+            self.orig_template = None
+
+        @property
+        def good_to_go(self):
+            return len(self.problems) == 0
+
+    class Dom0ToRestore(VMToRestore):
+        #: backup was performed on system with different dom0 username
+        USERNAME_MISMATCH = object()
+
+        def __init__(self, vm, subdir=None):
+            super(BackupRestore.Dom0ToRestore, self).__init__(vm)
+            if subdir:
+                self.subdir = subdir
+            self.username = os.path.basename(subdir)
+
     def __init__(self, app, backup_location, backup_vm, passphrase):
         super(BackupRestore, self).__init__()
 
@@ -1702,7 +1748,7 @@ class BackupRestore(object):
             orig_name = orig_name[0:29]
         new_name = orig_name
         while (new_name in restore_info.keys() or
-               new_name in map(lambda x: x.get('rename_to', None),
+               new_name in map(lambda x: x.rename_to,
                                restore_info.values()) or
                new_name in self.app.domains):
             new_name = str('{}{}'.format(orig_name, number))
@@ -1718,12 +1764,12 @@ class BackupRestore(object):
                 continue
 
             vm_info = restore_info[vm]
+            assert isinstance(vm_info, self.VMToRestore)
 
-            vm_info.pop('excluded', None)
+            vm_info.problems.clear()
             if vm in self.options.exclude:
-                vm_info['excluded'] = True
+                vm_info.problems.add(self.VMToRestore.EXCLUDED)
 
-            vm_info.pop('already-exists', None)
             if not self.options.verify_only and \
                     vm in self.app.domains:
                 if self.options.rename_conflicting:
@@ -1731,16 +1777,15 @@ class BackupRestore(object):
                         vm, restore_info
                     )
                     if new_name is not None:
-                        vm_info['rename-to'] = new_name
+                        vm_info.rename_to = new_name
                     else:
-                        vm_info['already-exists'] = True
+                        vm_info.problems.add(self.VMToRestore.ALREADY_EXISTS)
                 else:
-                    vm_info['already-exists'] = True
+                    vm_info.problems.add(self.VMToRestore.ALREADY_EXISTS)
 
             # check template
-            vm_info.pop('missing-template', None)
-            if vm_info['template']:
-                template_name = vm_info['template']
+            if vm_info.template:
+                template_name = vm_info.template
                 try:
                     host_template = self.app.domains[template_name]
                 except KeyError:
@@ -1748,18 +1793,18 @@ class BackupRestore(object):
                 if not host_template or not host_template.is_template():
                     # Maybe the (custom) template is in the backup?
                     if not (template_name in restore_info.keys() and
-                            restore_info[template_name]['vm'].is_template()):
+                            restore_info[template_name].vm.is_template()):
                         if self.options.use_default_template:
-                            if 'orig-template' not in vm_info.keys():
-                                vm_info['orig-template'] = template_name
-                            vm_info['template'] = self.app.default_template.name
+                            if vm_info.orig_template is None:
+                                vm_info.orig_template = template_name
+                            vm_info.template = self.app.default_template.name
                         else:
-                            vm_info['missing-template'] = True
+                            vm_info.problems.add(
+                                self.VMToRestore.MISSING_TEMPLATE)
 
             # check netvm
-            vm_info.pop('missing-netvm', None)
-            if vm_info['netvm']:
-                netvm_name = vm_info['netvm']
+            if vm_info.netvm:
+                netvm_name = vm_info.netvm
 
                 try:
                     netvm_on_host = self.app.domains[netvm_name]
@@ -1771,39 +1816,34 @@ class BackupRestore(object):
 
                     # Maybe the (custom) netvm is in the backup?
                     if not (netvm_name in restore_info.keys() and
-                            restore_info[netvm_name]['vm'].is_netvm()):
+                            restore_info[netvm_name].vm.is_netvm()):
                         if self.options.use_default_netvm:
                             if self.app.default_netvm:
-                                vm_info['netvm'] = self.app.default_netvm.name
+                                vm_info.netvm = self.app.default_netvm.name
                             else:
-                                vm_info['netvm'] = None
-                            vm_info['vm'].netvm = qubes.property.DEFAULT
+                                vm_info.netvm = None
+                            vm_info.vm.netvm = qubes.property.DEFAULT
                         elif self.options.use_none_netvm:
-                            vm_info['netvm'] = None
+                            vm_info.netvm = None
                         else:
-                            vm_info['missing-netvm'] = True
-
-            vm_info['good-to-go'] = not any([(prop in vm_info.keys()) for
-                                             prop in ['missing-netvm',
-                                                      'missing-template',
-                                                      'already-exists',
-                                                      'excluded']])
+                            vm_info.problems.add(self.VMToRestore.MISSING_NETVM)
 
         # update references to renamed VMs:
         for vm in restore_info.keys():
             if vm in ['dom0']:
                 continue
             vm_info = restore_info[vm]
-            template_name = vm_info['template']
+            assert isinstance(vm_info, self.VMToRestore)
+            template_name = vm_info.template
             if (template_name in restore_info and
-                    restore_info[template_name]['good-to-go'] and
-                    'rename-to' in restore_info[template_name]):
-                vm_info['template'] = restore_info[template_name]['rename-to']
-            netvm_name = vm_info['netvm']
+                    restore_info[template_name].good_to_go and
+                    restore_info[template_name].rename_to):
+                vm_info.template = restore_info[template_name].rename_to
+            netvm_name = vm_info.netvm
             if (netvm_name in restore_info and
-                    restore_info[netvm_name]['good-to-go'] and
-                    'rename-to' in restore_info[netvm_name]):
-                vm_info['netvm'] = restore_info[netvm_name]['rename-to']
+                    restore_info[netvm_name].good_to_go and
+                    restore_info[netvm_name].rename_to):
+                vm_info.netvm = restore_info[netvm_name].rename_to
 
         return restore_info
 
@@ -1860,24 +1900,16 @@ class BackupRestore(object):
             if self._is_vm_included_in_backup(vm):
                 self.log.debug("{} is included in backup".format(vm.name))
 
-                vms_to_restore[vm.name] = {}
-                vms_to_restore[vm.name]['vm'] = vm
+                vms_to_restore[vm.name] = self.VMToRestore(vm)
 
-                if not hasattr(vm, 'template'):
-                    vms_to_restore[vm.name]['template'] = None
-                else:
+                if hasattr(vm, 'template'):
                     templatevm_name = self._find_template_name(
                         vm.template.name)
-                    vms_to_restore[vm.name]['template'] = templatevm_name
+                    vms_to_restore[vm.name].template = templatevm_name
 
-                if vm.netvm is None:
-                    vms_to_restore[vm.name]['netvm'] = None
-                else:
-                    netvm_name = vm.netvm.name
-                    vms_to_restore[vm.name]['netvm'] = netvm_name
-                    # Set to None to not confuse QubesVm object from backup
-                    # collection with host collection (further in clone_attrs).
-                    vm.netvm = None
+                # Set to None to not confuse QubesVm object from backup
+                # collection with host collection (further in clone_attrs).
+                vm.netvm = None
 
         vms_to_restore = self.restore_info_verify(vms_to_restore)
 
@@ -1885,29 +1917,18 @@ class BackupRestore(object):
         if self.options.dom0_home and \
                 self._is_vm_included_in_backup(self.backup_app.domains[0]):
             vm = self.backup_app.domains[0]
-            vms_to_restore['dom0'] = {}
             if self.header_data.version == 1:
-                vms_to_restore['dom0']['subdir'] = \
-                    os.listdir(os.path.join(
-                        self.backup_location, 'dom0-home'))[0]
-                vms_to_restore['dom0']['size'] = 0  # unknown
+                subdir = os.listdir(os.path.join(self.backup_location,
+                    'dom0-home'))[0]
             else:
-                vms_to_restore['dom0']['subdir'] = vm.features['backup-path']
-                vms_to_restore['dom0']['size'] = int(vm.features['backup-size'])
+                subdir = None
+            vms_to_restore['dom0'] = self.Dom0ToRestore(vm, subdir)
             local_user = grp.getgrnam('qubes').gr_mem[0]
 
-            dom0_home = vms_to_restore['dom0']['subdir']
-
-            vms_to_restore['dom0']['username'] = os.path.basename(dom0_home)
-            if vms_to_restore['dom0']['username'] != local_user:
-                vms_to_restore['dom0']['username-mismatch'] = True
-                if self.options.ignore_username_mismatch:
-                    vms_to_restore['dom0']['ignore-username-mismatch'] = True
-                else:
-                    vms_to_restore['dom0']['good-to-go'] = False
-
-            if 'good-to-go' not in vms_to_restore['dom0']:
-                vms_to_restore['dom0']['good-to-go'] = True
+            if vms_to_restore['dom0'].username != local_user:
+                if not self.options.ignore_username_mismatch:
+                    vms_to_restore['dom0'].problems.add(
+                        self.Dom0ToRestore.USERNAME_MISMATCH)
 
         return vms_to_restore
 
@@ -1929,11 +1950,11 @@ class BackupRestore(object):
             "updbl": {"func": "'Yes' if vm.updateable else ''"},
 
             "template": {"func": "'n/a' if not hasattr(vm, 'template') is None "
-                                 "else vm_info['template']"},
+                                 "else vm_info.template"},
 
             "netvm": {"func": "'n/a' if vm.is_netvm() and not vm.is_proxyvm() else\
                       ('*' if vm.property_is_default('netvm') else '') +\
-                        vm_info['netvm'] if vm_info['netvm'] is not None "
+                        vm_info.netvm if vm_info.netvm is not None "
                               "else '-'"},
 
             "label": {"func": "vm.label.name"},
@@ -1947,9 +1968,9 @@ class BackupRestore(object):
         for f in fields_to_display:
             fields[f]["max_width"] = len(f)
             for vm_info in restore_info.values():
-                if 'vm' in vm_info.keys():
+                if vm_info.vm:
                     # noinspection PyUnusedLocal
-                    vm = vm_info['vm']
+                    vm = vm_info.vm
                     l = len(unicode(eval(fields[f]["func"])))
                     if l > fields[f]["max_width"]:
                         fields[f]["max_width"] = l
@@ -1977,34 +1998,37 @@ class BackupRestore(object):
         summary += "\n"
 
         for vm_info in restore_info.values():
+            assert isinstance(vm_info, BackupRestore.VMToRestore)
             # Skip non-VM here
-            if 'vm' not in vm_info:
+            if not vm_info.vm:
                 continue
             # noinspection PyUnusedLocal
-            vm = vm_info['vm']
+            vm = vm_info.vm
             s = ""
             for f in fields_to_display:
                 # noinspection PyTypeChecker
                 fmt = "{{0:>{0}}} |".format(fields[f]["max_width"] + 1)
                 s += fmt.format(eval(fields[f]["func"]))
 
-            if 'excluded' in vm_info and vm_info['excluded']:
+            if BackupRestore.VMToRestore.EXCLUDED in vm_info.problems:
                 s += " <-- Excluded from restore"
-            elif 'already-exists' in vm_info:
+            elif BackupRestore.VMToRestore.ALREADY_EXISTS in vm_info.problems:
                 s += " <-- A VM with the same name already exists on the host!"
-            elif 'missing-template' in vm_info:
+            elif BackupRestore.VMToRestore.MISSING_TEMPLATE in \
+                    vm_info.problems:
                 s += " <-- No matching template on the host " \
                      "or in the backup found!"
-            elif 'missing-netvm' in vm_info:
+            elif BackupRestore.VMToRestore.MISSING_NETVM in \
+                    vm_info.problems:
                 s += " <-- No matching netvm on the host " \
                      "or in the backup found!"
             else:
-                if 'orig-template' in vm_info:
+                if vm_info.orig_template:
                     s += " <-- Original template was '{}'".format(
-                        vm_info['orig-template'])
-                if 'rename-to' in vm_info:
+                        vm_info.orig_template)
+                if vm_info.rename_to:
                     s += " <-- Will be renamed to '{}'".format(
-                        vm_info['rename-to'])
+                        vm_info.rename_to)
 
             summary += s + "\n"
 
@@ -2019,10 +2043,9 @@ class BackupRestore(object):
                     s += fmt.format("Home")
                 else:
                     s += fmt.format("")
-            if 'username-mismatch' in restore_info['dom0']:
+            if BackupRestore.Dom0ToRestore.USERNAME_MISMATCH in \
+                    restore_info['dom0'].problems:
                 s += " <-- username in backup and dom0 mismatch"
-            if 'ignore-username-mismatch' in restore_info['dom0']:
-                s += " (ignored)"
 
             summary += s + "\n"
 
@@ -2049,11 +2072,12 @@ class BackupRestore(object):
         vms_size = 0
         vms = {}
         for vm_info in restore_info.values():
-            if 'vm' not in vm_info:
+            assert isinstance(vm_info, self.VMToRestore)
+            if not vm_info.vm:
                 continue
-            if not vm_info['good-to-go']:
+            if not vm_info.good_to_go:
                 continue
-            vm = vm_info['vm']
+            vm = vm_info.vm
             if self.header_data.version >= 2:
                 if vm.features['backup-size']:
                     vms_size += int(vm.features['backup-size'])
@@ -2062,9 +2086,9 @@ class BackupRestore(object):
 
         if self.header_data.version >= 2:
             if 'dom0' in restore_info.keys() and \
-                    restore_info['dom0']['good-to-go']:
-                vms_dirs.append(os.path.dirname(restore_info['dom0']['subdir']))
-                vms_size += restore_info['dom0']['size']
+                    restore_info['dom0'].good_to_go:
+                vms_dirs.append(os.path.dirname(restore_info['dom0'].subdir))
+                vms_size += restore_info['dom0'].size
 
             try:
                 self._restore_vm_dirs(vms_dirs=vms_dirs, vms_size=vms_size)
@@ -2108,14 +2132,14 @@ class BackupRestore(object):
                 kwargs = {}
                 if hasattr(vm, 'template'):
                     if vm.template is not None:
-                        kwargs['template'] = restore_info[vm.name]['template']
+                        kwargs['template'] = restore_info[vm.name].template
                     else:
                         kwargs['template'] = None
 
                 new_vm = None
                 vm_name = vm.name
-                if 'rename-to' in restore_info[vm.name]:
-                    vm_name = restore_info[vm.name]['rename-to']
+                if restore_info[vm.name].rename_to:
+                    vm_name = restore_info[vm.name].rename_to
 
                 try:
                     # first only minimal set, later clone_properties
@@ -2197,8 +2221,8 @@ class BackupRestore(object):
         # Set network dependencies - only non-default netvm setting
         for vm in vms.values():
             vm_name = vm.name
-            if 'rename-to' in restore_info[vm.name]:
-                vm_name = restore_info[vm.name]['rename-to']
+            if restore_info[vm.name].rename_to:
+                vm_name = restore_info[vm.name].rename_to
             try:
                 host_vm = self.app.domains[vm_name]
             except KeyError:
@@ -2206,8 +2230,8 @@ class BackupRestore(object):
                 continue
 
             if not vm.property_is_default('netvm'):
-                if restore_info[vm.name]['netvm'] is not None:
-                    host_vm.netvm = restore_info[vm.name]['netvm']
+                if restore_info[vm.name].netvm is not None:
+                    host_vm.netvm = restore_info[vm.name].netvm
                 else:
                     host_vm.netvm = None
 
@@ -2221,8 +2245,8 @@ class BackupRestore(object):
                 raise BackupCanceledError("Restore canceled")
 
         # ... and dom0 home as last step
-        if 'dom0' in restore_info.keys() and restore_info['dom0']['good-to-go']:
-            backup_path = restore_info['dom0']['subdir']
+        if 'dom0' in restore_info.keys() and restore_info['dom0'].good_to_go:
+            backup_path = restore_info['dom0'].subdir
             local_user = grp.getgrnam('qubes').gr_mem[0]
             home_dir = pwd.getpwnam(local_user).pw_dir
             if self.header_data.version == 1:

From 20d53fbf69e02c4e64b1e09c48d87c3f2bb866c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 4 Apr 2016 14:11:19 +0200
Subject: [PATCH 29/32] tests: more VMs combinations in backup

VM with custom template, non-default NetVM

This requires to VMs being removed in reversed order (to first remove
VM, then its template).
---
 qubes/tests/__init__.py   | 17 +++++++++++++++++
 qubes/tests/int/backup.py | 18 +++++++++---------
 2 files changed, 26 insertions(+), 9 deletions(-)

diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py
index 09d74a3e..e95ae3d4 100644
--- a/qubes/tests/__init__.py
+++ b/qubes/tests/__init__.py
@@ -718,6 +718,23 @@ class BackupTestsMixin(SystemTestsMixin):
         self.fill_image(testvm2.root_img, 1024*1024*1024, True)
         vms.append(testvm2)
 
+        vmname = self.make_vm_name('template')
+        if self.verbose:
+            print >>sys.stderr, "-> Creating %s" % vmname
+        testvm3 = self.app.add_new_vm(qubes.vm.templatevm.TemplateVM,
+            name=vmname, label='red')
+        testvm3.create_on_disk()
+        self.fill_image(testvm3.root_img, 100*1024*1024, True)
+        vms.append(testvm3)
+
+        vmname = self.make_vm_name('custom')
+        if self.verbose:
+            print >>sys.stderr, "-> Creating %s" % vmname
+        testvm4 = self.app.add_new_vm(qubes.vm.appvm.AppVM,
+            name=vmname, template=testvm3, label='red')
+        testvm4.create_on_disk()
+        vms.append(testvm4)
+
         self.app.save()
 
         return vms
diff --git a/qubes/tests/int/backup.py b/qubes/tests/int/backup.py
index 2acb2754..c11b0948 100644
--- a/qubes/tests/int/backup.py
+++ b/qubes/tests/int/backup.py
@@ -38,7 +38,7 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
     def test_000_basic_backup(self):
         vms = self.create_backup_vms()
         self.make_backup(vms)
-        self.remove_vms(vms)
+        self.remove_vms(reversed(vms))
         self.restore_backup()
         for vm in vms:
             self.assertIn(vm.name, self.app.domains)
@@ -46,7 +46,7 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
     def test_001_compressed_backup(self):
         vms = self.create_backup_vms()
         self.make_backup(vms, compressed=True)
-        self.remove_vms(vms)
+        self.remove_vms(reversed(vms))
         self.restore_backup()
         for vm in vms:
             self.assertIn(vm.name, self.app.domains)
@@ -54,7 +54,7 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
     def test_002_encrypted_backup(self):
         vms = self.create_backup_vms()
         self.make_backup(vms, encrypted=True)
-        self.remove_vms(vms)
+        self.remove_vms(reversed(vms))
         self.restore_backup()
         for vm in vms:
             self.assertIn(vm.name, self.app.domains)
@@ -62,7 +62,7 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
     def test_003_compressed_encrypted_backup(self):
         vms = self.create_backup_vms()
         self.make_backup(vms, compressed=True, encrypted=True)
-        self.remove_vms(vms)
+        self.remove_vms(reversed(vms))
         self.restore_backup()
         for vm in vms:
             self.assertIn(vm.name, self.app.domains)
@@ -85,7 +85,7 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
         self.app.save()
 
         self.make_backup(vms)
-        self.remove_vms(vms)
+        self.remove_vms(reversed(vms))
         self.restore_backup()
         for vm in vms:
             self.assertIn(vm.name, self.app.domains)
@@ -94,7 +94,7 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
     def test_005_compressed_custom(self):
         vms = self.create_backup_vms()
         self.make_backup(vms, compressed="bzip2")
-        self.remove_vms(vms)
+        self.remove_vms(reversed(vms))
         self.restore_backup()
         for vm in vms:
             self.assertIn(vm.name, self.app.domains)
@@ -113,7 +113,7 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
         """
         vms = self.create_backup_vms()
         self.make_backup(vms)
-        self.remove_vms(vms)
+        self.remove_vms(reversed(vms))
         test_dir = vms[0].dir_path
         os.mkdir(test_dir)
         with open(os.path.join(test_dir, 'some-file.txt'), 'w') as f:
@@ -160,7 +160,7 @@ class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin):
         self.make_backup(vms, target_vm=self.backupvm,
             compressed=True, encrypted=True,
             target='/var/tmp/backup directory')
-        self.remove_vms(vms)
+        self.remove_vms(reversed(vms))
         p = self.backupvm.run("ls /var/tmp/backup*/qubes-backup*",
                               passio_popen=True)
         (backup_path, _) = p.communicate()
@@ -174,7 +174,7 @@ class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin):
         self.make_backup(vms, target_vm=self.backupvm,
             compressed=True, encrypted=True,
             target='dd of=/var/tmp/backup-test')
-        self.remove_vms(vms)
+        self.remove_vms(reversed(vms))
         self.restore_backup(source='dd if=/var/tmp/backup-test',
                             appvm=self.backupvm)
 

From 71a7730168511ee16bf5801394d8629010160ab2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Mon, 4 Apr 2016 22:06:21 +0200
Subject: [PATCH 30/32] backup: simplify ordering of restoring VMs

No other change than reducing two nested loops into one.
---
 qubes/backup.py | 192 ++++++++++++++++++++++++------------------------
 1 file changed, 94 insertions(+), 98 deletions(-)

diff --git a/qubes/backup.py b/qubes/backup.py
index 8fc5c5d4..394e925c 100644
--- a/qubes/backup.py
+++ b/qubes/backup.py
@@ -2110,113 +2110,109 @@ class BackupRestore(object):
             return
 
         # First load templates, then other VMs
-        for do_templates in (True, False):
+        for vm in sorted(vms.values(), key=lambda x: x.is_template(),
+                reverse=True):
             if self.canceled:
+                # only break the loop to save qubes.xml
+                # with already restored VMs
                 break
-            for vm in vms.values():
-                if self.canceled:
-                    # only break the loop to save qubes.xml
-                    # with already restored VMs
-                    break
-                if vm.is_template() != do_templates:
-                    continue
-                self.log.info("-> Restoring {0}...".format(vm.name))
-                retcode = subprocess.call(
-                    ["mkdir", "-p", os.path.dirname(vm.dir_path)])
-                if retcode != 0:
-                    self.log.error("*** Cannot create directory: {0}?!".format(
-                        vm.dir_path))
-                    self.log.warning("Skipping VM {}...".format(vm.name))
-                    continue
+            self.log.info("-> Restoring {0}...".format(vm.name))
+            retcode = subprocess.call(
+                ["mkdir", "-p", os.path.dirname(vm.dir_path)])
+            if retcode != 0:
+                self.log.error("*** Cannot create directory: {0}?!".format(
+                    vm.dir_path))
+                self.log.warning("Skipping VM {}...".format(vm.name))
+                continue
 
-                kwargs = {}
-                if hasattr(vm, 'template'):
-                    if vm.template is not None:
-                        kwargs['template'] = restore_info[vm.name].template
-                    else:
-                        kwargs['template'] = None
+            kwargs = {}
+            if hasattr(vm, 'template'):
+                if vm.template is not None:
+                    kwargs['template'] = restore_info[vm.name].template
+                else:
+                    kwargs['template'] = None
 
-                new_vm = None
-                vm_name = vm.name
-                if restore_info[vm.name].rename_to:
-                    vm_name = restore_info[vm.name].rename_to
+            new_vm = None
+            vm_name = vm.name
+            if restore_info[vm.name].rename_to:
+                vm_name = restore_info[vm.name].rename_to
 
-                try:
-                    # first only minimal set, later clone_properties
-                    # will be called
-                    new_vm = self.app.add_new_vm(
-                        vm.__class__,
-                        name=vm_name,
-                        label=vm.label,
-                        installed_by_rpm=False,
-                        **kwargs)
-                    if os.path.exists(new_vm.dir_path):
-                        move_to_path = tempfile.mkdtemp('', os.path.basename(
-                            new_vm.dir_path), os.path.dirname(new_vm.dir_path))
-                        try:
-                            os.rename(new_vm.dir_path, move_to_path)
-                            self.log.warning(
-                                "*** Directory {} already exists! It has "
-                                "been moved to {}".format(new_vm.dir_path,
-                                                          move_to_path))
-                        except OSError:
-                            self.log.error(
-                                "*** Directory {} already exists and "
-                                "cannot be moved!".format(new_vm.dir_path))
-                            self.log.warning("Skipping VM {}...".format(
-                                vm.name))
-                            continue
+            try:
+                # first only minimal set, later clone_properties
+                # will be called
+                new_vm = self.app.add_new_vm(
+                    vm.__class__,
+                    name=vm_name,
+                    label=vm.label,
+                    installed_by_rpm=False,
+                    **kwargs)
+                if os.path.exists(new_vm.dir_path):
+                    move_to_path = tempfile.mkdtemp('', os.path.basename(
+                        new_vm.dir_path), os.path.dirname(new_vm.dir_path))
+                    try:
+                        os.rename(new_vm.dir_path, move_to_path)
+                        self.log.warning(
+                            "*** Directory {} already exists! It has "
+                            "been moved to {}".format(new_vm.dir_path,
+                                                      move_to_path))
+                    except OSError:
+                        self.log.error(
+                            "*** Directory {} already exists and "
+                            "cannot be moved!".format(new_vm.dir_path))
+                        self.log.warning("Skipping VM {}...".format(
+                            vm.name))
+                        continue
 
-                    if self.header_data.version == 1:
-                        self._restore_vm_dir_v1(vm.dir_path,
-                            os.path.dirname(new_vm.dir_path))
-                    else:
-                        shutil.move(os.path.join(self.tmpdir,
-                            vm.features['backup-path']),
-                            new_vm.dir_path)
+                if self.header_data.version == 1:
+                    self._restore_vm_dir_v1(vm.dir_path,
+                        os.path.dirname(new_vm.dir_path))
+                else:
+                    shutil.move(os.path.join(self.tmpdir,
+                        vm.features['backup-path']),
+                        new_vm.dir_path)
 
-                    new_vm.verify_files()
-                except Exception as err:
-                    self.log.error("ERROR: {0}".format(err))
-                    self.log.warning("*** Skipping VM: {0}".format(vm.name))
-                    if new_vm:
-                        del self.app.domains[new_vm.qid]
-                    continue
+                new_vm.verify_files()
+            except Exception as err:
+                self.log.error("ERROR: {0}".format(err))
+                self.log.warning("*** Skipping VM: {0}".format(vm.name))
+                if new_vm:
+                    del self.app.domains[new_vm.qid]
+                continue
 
-                if hasattr(vm, 'kernel'):
-                    # TODO: add a setting for this?
-                    if not vm.property_is_default('kernel') and vm.kernel and \
-                            vm.kernel not in \
-                            os.listdir(os.path.join(qubes.config.qubes_base_dir,
-                                qubes.config.system_path[
-                                'qubes_kernels_base_dir'])):
-                        self.log.warning("Kernel %s not installed, "
-                        "using default one" % vm.kernel)
-                        vm.kernel = qubes.property.DEFAULT
-                # remove no longer needed backup metadata
-                if 'backup-content' in vm.features:
-                    del vm.features['backup-content']
-                    del vm.features['backup-size']
-                    del vm.features['backup-path']
-                try:
-                    # exclude VM references - handled manually according to
-                    # restore options
-                    proplist = [prop for prop in new_vm.property_list()
-                        if prop.clone and prop.__name__ not in
-                              ['template', 'netvm', 'dispvm_netvm']]
-                    new_vm.clone_properties(vm, proplist=proplist)
-                except Exception as err:
-                    self.log.error("ERROR: {0}".format(err))
-                    self.log.warning("*** Some VM property will not be "
-                                     "restored")
+            if hasattr(vm, 'kernel'):
+                # TODO: add a setting for this?
+                if not vm.property_is_default('kernel') and vm.kernel and \
+                        vm.kernel not in \
+                        os.listdir(os.path.join(qubes.config.qubes_base_dir,
+                            qubes.config.system_path[
+                            'qubes_kernels_base_dir'])):
+                    self.log.warning("Kernel %s not installed, "
+                    "using default one" % vm.kernel)
+                    vm.kernel = qubes.property.DEFAULT
+            # remove no longer needed backup metadata
+            if 'backup-content' in vm.features:
+                del vm.features['backup-content']
+                del vm.features['backup-size']
+                del vm.features['backup-path']
+            try:
+                # exclude VM references - handled manually according to
+                # restore options
+                proplist = [prop for prop in new_vm.property_list()
+                    if prop.clone and prop.__name__ not in
+                          ['template', 'netvm', 'dispvm_netvm']]
+                new_vm.clone_properties(vm, proplist=proplist)
+            except Exception as err:
+                self.log.error("ERROR: {0}".format(err))
+                self.log.warning("*** Some VM property will not be "
+                                 "restored")
 
-                try:
-                    new_vm.fire_event('domain-restore')
-                except Exception as err:
-                    self.log.error("ERROR during appmenu restore: "
-                       "{0}".format(err))
-                    self.log.warning(
-                        "*** VM '{0}' will not have appmenus".format(vm.name))
+            try:
+                new_vm.fire_event('domain-restore')
+            except Exception as err:
+                self.log.error("ERROR during appmenu restore: "
+                   "{0}".format(err))
+                self.log.warning(
+                    "*** VM '{0}' will not have appmenus".format(vm.name))
 
         # Set network dependencies - only non-default netvm setting
         for vm in vms.values():

From 2261e6a4c776f31b7c5f56da5d39c2778b9d8823 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 5 Apr 2016 00:34:19 +0200
Subject: [PATCH 31/32] backup: simplify auto-renaming handling

Don't try to track names in restore_info_verify, instead resolve them at
actual restore time.
---
 qubes/backup.py | 60 +++++++++++++++++--------------------------------
 1 file changed, 20 insertions(+), 40 deletions(-)

diff --git a/qubes/backup.py b/qubes/backup.py
index 394e925c..fa3be91b 100644
--- a/qubes/backup.py
+++ b/qubes/backup.py
@@ -1297,7 +1297,7 @@ class BackupRestore(object):
                 self.netvm = vm.netvm.name
             else:
                 self.netvm = None
-            self.rename_to = None
+            self.name = vm.name
             self.orig_template = None
 
         @property
@@ -1748,7 +1748,7 @@ class BackupRestore(object):
             orig_name = orig_name[0:29]
         new_name = orig_name
         while (new_name in restore_info.keys() or
-               new_name in map(lambda x: x.rename_to,
+               new_name in map(lambda x: x.name,
                                restore_info.values()) or
                new_name in self.app.domains):
             new_name = str('{}{}'.format(orig_name, number))
@@ -1777,7 +1777,7 @@ class BackupRestore(object):
                         vm, restore_info
                     )
                     if new_name is not None:
-                        vm_info.rename_to = new_name
+                        vm_info.name = new_name
                     else:
                         vm_info.problems.add(self.VMToRestore.ALREADY_EXISTS)
                 else:
@@ -1793,6 +1793,7 @@ class BackupRestore(object):
                 if not host_template or not host_template.is_template():
                     # Maybe the (custom) template is in the backup?
                     if not (template_name in restore_info.keys() and
+                            restore_info[template_name].good_to_go and
                             restore_info[template_name].vm.is_template()):
                         if self.options.use_default_template:
                             if vm_info.orig_template is None:
@@ -1803,7 +1804,7 @@ class BackupRestore(object):
                                 self.VMToRestore.MISSING_TEMPLATE)
 
             # check netvm
-            if vm_info.netvm:
+            if not vm_info.vm.property_is_default('netvm') and vm_info.netvm:
                 netvm_name = vm_info.netvm
 
                 try:
@@ -1816,35 +1817,15 @@ class BackupRestore(object):
 
                     # Maybe the (custom) netvm is in the backup?
                     if not (netvm_name in restore_info.keys() and
+                            restore_info[netvm_name].good_to_go and
                             restore_info[netvm_name].vm.is_netvm()):
                         if self.options.use_default_netvm:
-                            if self.app.default_netvm:
-                                vm_info.netvm = self.app.default_netvm.name
-                            else:
-                                vm_info.netvm = None
                             vm_info.vm.netvm = qubes.property.DEFAULT
                         elif self.options.use_none_netvm:
                             vm_info.netvm = None
                         else:
                             vm_info.problems.add(self.VMToRestore.MISSING_NETVM)
 
-        # update references to renamed VMs:
-        for vm in restore_info.keys():
-            if vm in ['dom0']:
-                continue
-            vm_info = restore_info[vm]
-            assert isinstance(vm_info, self.VMToRestore)
-            template_name = vm_info.template
-            if (template_name in restore_info and
-                    restore_info[template_name].good_to_go and
-                    restore_info[template_name].rename_to):
-                vm_info.template = restore_info[template_name].rename_to
-            netvm_name = vm_info.netvm
-            if (netvm_name in restore_info and
-                    restore_info[netvm_name].good_to_go and
-                    restore_info[netvm_name].rename_to):
-                vm_info.netvm = restore_info[netvm_name].rename_to
-
         return restore_info
 
     def _is_vm_included_in_backup_v1(self, check_vm):
@@ -2026,9 +2007,9 @@ class BackupRestore(object):
                 if vm_info.orig_template:
                     s += " <-- Original template was '{}'".format(
                         vm_info.orig_template)
-                if vm_info.rename_to:
+                if vm_info.name != vm_info.vm.name:
                     s += " <-- Will be renamed to '{}'".format(
-                        vm_info.rename_to)
+                        vm_info.name)
 
             summary += s + "\n"
 
@@ -2127,15 +2108,15 @@ class BackupRestore(object):
 
             kwargs = {}
             if hasattr(vm, 'template'):
-                if vm.template is not None:
-                    kwargs['template'] = restore_info[vm.name].template
-                else:
-                    kwargs['template'] = None
+                template = restore_info[vm.name].template
+                # handle potentially renamed template
+                if template in restore_info \
+                        and restore_info[template].good_to_go:
+                    template = restore_info[template].name
+                kwargs['template'] = template
 
             new_vm = None
-            vm_name = vm.name
-            if restore_info[vm.name].rename_to:
-                vm_name = restore_info[vm.name].rename_to
+            vm_name = restore_info[vm.name].name
 
             try:
                 # first only minimal set, later clone_properties
@@ -2216,9 +2197,8 @@ class BackupRestore(object):
 
         # Set network dependencies - only non-default netvm setting
         for vm in vms.values():
-            vm_name = vm.name
-            if restore_info[vm.name].rename_to:
-                vm_name = restore_info[vm.name].rename_to
+            vm_info = restore_info[vm.name]
+            vm_name = vm_info.name
             try:
                 host_vm = self.app.domains[vm_name]
             except KeyError:
@@ -2226,10 +2206,10 @@ class BackupRestore(object):
                 continue
 
             if not vm.property_is_default('netvm'):
-                if restore_info[vm.name].netvm is not None:
-                    host_vm.netvm = restore_info[vm.name].netvm
+                if vm_info.netvm in restore_info:
+                    host_vm.netvm = restore_info[vm_info.netvm].name
                 else:
-                    host_vm.netvm = None
+                    host_vm.netvm = vm_info.netvm
 
         self.app.save()
 

From c28f50f6fe54cd40eec72d4c130f7248e96c6ba5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
 <marmarek@invisiblethingslab.com>
Date: Tue, 5 Apr 2016 00:35:57 +0200
Subject: [PATCH 32/32] backup: minor fixes

---
 qubes/backup.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/qubes/backup.py b/qubes/backup.py
index fa3be91b..506ed63e 100644
--- a/qubes/backup.py
+++ b/qubes/backup.py
@@ -1795,7 +1795,8 @@ class BackupRestore(object):
                     if not (template_name in restore_info.keys() and
                             restore_info[template_name].good_to_go and
                             restore_info[template_name].vm.is_template()):
-                        if self.options.use_default_template:
+                        if self.options.use_default_template and \
+                                self.app.default_template:
                             if vm_info.orig_template is None:
                                 vm_info.orig_template = template_name
                             vm_info.template = self.app.default_template.name
@@ -1813,12 +1814,12 @@ class BackupRestore(object):
                     netvm_on_host = None
                 # No netvm on the host?
                 if not ((netvm_on_host is not None)
-                        and netvm_on_host.is_netvm()):
+                        and netvm_on_host.provides_network):
 
                     # Maybe the (custom) netvm is in the backup?
                     if not (netvm_name in restore_info.keys() and
                             restore_info[netvm_name].good_to_go and
-                            restore_info[netvm_name].vm.is_netvm()):
+                            restore_info[netvm_name].vm.provides_network):
                         if self.options.use_default_netvm:
                             vm_info.vm.netvm = qubes.property.DEFAULT
                         elif self.options.use_none_netvm: