backup: reimplemented restore function through an AppVM

The VM size is now stored inside the backup specification file in order to compute progress.

dom0/qvm-core/qubesutils.py

@@ -906,11 +906,13 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca
             s +=  " <-- The VM is running, please shut it down before proceeding with the backup!"
             there_are_running_vms = True
 
+        print_callback(s)
+
     # Build Backup VMs reference file
     # Format: vm_name:tarball_path\n
     backup_reference_file = open(os.path.join(qubes_base_dir,"backup_targets"),'w')
     for vm in vms_for_backup:
-        backup_reference_file.write(vm.name+":"+vm.dir_path.split(qubes_base_dir)[1]+"\n")
+        backup_reference_file.write(vm.name+":"+vm.dir_path.split(qubes_base_dir)[1]+":"+str(vm.get_disk_utilization())+"\n")
     backup_reference_file.flush()
     backup_reference_file.close()
     files_to_backup = file_to_backup(backup_reference_file.name,os.stat(backup_reference_file.name).st_size) + files_to_backup
@@ -1011,7 +1013,6 @@ def backup_do(base_backup_dir, files_to_backup, progress_callback = None):
         progress_callback(progress)
 
 def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, encrypt=False, appvm=None):
-    print appvm,base_backup_dir
     total_backup_sz = 0
     for file in files_to_backup:
         total_backup_sz += file["size"]
@@ -1069,6 +1070,26 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e
         compressor = subprocess.Popen (tar_cmdline,stdout=backup_stdout,stderr=feedback_file)
         encryptor  = None
 
+    run_error = wait_backup_feedback(progress_callback, feedback_file, total_backup_sz, compressor, encryptor, vmproc)
+
+    feedback_file.close()
+    backup_stdout.close()
+
+    # Check returns code of compressor and encryptor and qubes vm retcode
+    if run_error != None:
+        try:
+            if compressor != None:
+                compressor.terminate()
+            if encryptor != None:
+                encryptor.terminate()
+            if vmproc != None:
+                vmproc.terminate()
+        except OSError:
+            pass
+        raise QubesException("Failed to perform backup: error with "+run_error)
+
+
+def wait_backup_feedback(progress_callback, feedback_file, total_backup_sz, compressor, encryptor, vmproc):
     # Get tar backup feedback
     feedback_file_r = open(feedback_file.name,'r')
     run_error = None
@@ -1104,7 +1125,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e
             retcode = vmproc.poll()
             if retcode != None:
                 if retcode != 0:
-                    run_error = "VM "+appvm
+                    run_error = "VM"
                     print vmproc.stdout.read()
             else:
                 # VM should run until the end
@@ -1112,8 +1133,56 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e
 
     # Cleanup
     feedback_file_r.close()
+
+    return run_error
+
+def restore_vm_dir (backup_dir, src_dir, dst_dir, vm_spec, print_callback=None, error_callback=None, encrypted=False, appvm=None):
+
+    #backup_src_dir = src_dir.replace (qubes_base_dir, backup_dir)
+
+    vmproc = None
+    if appvm != None:
+        # Prepare the backup target (Qubes service call)
+        backup_target = "QUBESRPC qubes.Restore none"
+
+        # does the vm exist?
+        qvm_collection = QubesVmCollection()
+        qvm_collection.lock_db_for_reading()
+        qvm_collection.load()
+    
+        vm = qvm_collection.get_vm_by_name(appvm)
+        if vm is None or vm.qid not in qvm_collection:
+            raise QubesException("VM {0} does not exist".format(appvm))
+
+        qvm_collection.unlock_db()
+
+        # If APPVM, STDOUT is a PIPE
+        vmproc = vm.run(command = backup_target, passio_popen = True)
+        vmproc.stdin.write(backup_dir.replace("\r","").replace("\n","")+"\n")
+        backup_stdin = vmproc.stdout
+    else:
+        backup_stdin = open(backup_dir,'rb')
+
+    tar_cmdline = ["tar", "-xzv",'--sparse','-C',qubes_base_dir,'--checkpoint=10000']
+
+    tar_cmdline.append(src_dir.split(os.path.normpath(qubes_base_dir)+"/")[1])
+
+    #print ("Will backup using command",tar_cmdline)
+
+    import tempfile
+    feedback_file = tempfile.NamedTemporaryFile()
+    if encrypted:
+        encryptor  = subprocess.Popen (["gpg2", "--decrypt"], stdin=backup_stdin, stdout=subprocess.PIPE,stderr=subprocess.PIPE)
+        compressor = subprocess.Popen (tar_cmdline,stdin=encryptor.stdout,stderr=feedback_file,stdout=subprocess.PIPE)
+    else:
+        compressor = subprocess.Popen (tar_cmdline,stdin=backup_stdin,stderr=feedback_file,stdout=subprocess.PIPE)
+        encryptor  = None
+
+    run_error = wait_backup_feedback(print_callback, feedback_file, vm_spec["size"], compressor, encryptor, vmproc)
+
+    # Cleanup
     feedback_file.close()
-    backup_stdout.close()
+    backup_stdin.close()
 
     # Check returns code of compressor and encryptor and qubes vm retcode
     if run_error != None:
@@ -1142,14 +1211,14 @@ def backup_restore_set_defaults(options):
 
     return options
 
-def backup_restore_header(restore_target, progress_callback = None, encrypt=False, appvm=None):
+def backup_restore_header(restore_target, encrypt=False, appvm=None):
     # Simulate dd if=backup_file count=10 | file -
     # Simulate dd if=backup_file count=10 | gpg2 -d | tar xzv -O
     # analysis  = subprocess.Popen()
     vmproc = None
     if appvm != None:
         # Prepare the backup target (Qubes service call)
-        backup_target = "QUBESRPC qubes.Backup none"
+        restore_command = "QUBESRPC qubes.Restore none"
 
         # does the vm exist?
         qvm_collection = QubesVmCollection()
@@ -1163,14 +1232,14 @@ def backup_restore_header(restore_target, progress_callback = None, encrypt=Fals
         qvm_collection.unlock_db()
 
         # If APPVM, STDOUT is a PIPE
-        vmproc = vm.run(command = restore_target, passio_popen = True)
+        vmproc = vm.run(command = restore_command, passio_popen = True)
         vmproc.stdin.write(restore_target.replace("\r","").replace("\n","")+"\n")
-        headers = vmproc.stdout.read(4096)
+
+        headers = vmproc.stdout.read(4096*4)
         vmproc.terminate()
 
         if len(headers) <= 0:
-            print vmproc.stderr.read()
+            raise QubesException("ERROR: unable to read the backup target {0}".format(restore_target))
-            raise QubesException("ERROR: the backup directory {0} does not exists".format(restore_target))
 
     else:
         # Create the target directory
@@ -1178,7 +1247,7 @@ def backup_restore_header(restore_target, progress_callback = None, encrypt=Fals
             raise QubesException("ERROR: the backup directory {0} does not exists".format(restore_target))
 
         fp = open(restore_target,'rb')
-        headers = fp.read(4096)
+        headers = fp.read(4096*4)
 
     is_encrypted = False
 
@@ -1194,25 +1263,31 @@ def backup_restore_header(restore_target, progress_callback = None, encrypt=Fals
             print stderr
             raise QubesException("ERROR: unable to decrypt the backup {0}. Is it really encrypted?".format(restore_target))
 
-    command = subprocess.Popen(['tar', 'xzv', '-O', 'backup_targets'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
+    command = subprocess.Popen(['tar', 'xzv', '-O', 'backup_targets','qubes.xml'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
-    vm_list,stderr = command.communicate(headers)
+    headers,stderr = command.communicate(headers)
-    if len(vm_list) <= 0:
+    if len(headers) <= 0:
         print stderr
         raise QubesException("ERROR: unable to read the qubes backup file {0}. Is it really a backup?".format(restore_target))
 
-    vms_in_backup = []
+    vms_in_backup = {}
-    for vm in vm_list.split("\n"):
+    for vm in headers.split("\n"):
-        vms_in_backup.append(vm.strip("\r\t\n ").split(":"))
+        match = re.match("^(?P<name>[^:]+):(?P<path>[^:]+):(?P<size>[0-9]+)$",vm,re.MULTILINE)
+        if match:
+            item = match.groupdict()
+            item["size"] = int(item["size"])
+            vms_in_backup[item["name"]] = item
+            headers = headers.replace(vm,"")
+        else:
+            break
 
-    print vms_in_backup
+    return is_encrypted, vms_in_backup, headers
-    return is_encrypted, vms_in_backup
 
-def backup_restore_prepare(backup_dir, options = {}, host_collection = None):
+def backup_restore_prepare(backup_dir, backup_content, qubes_xml, options = {}, host_collection = None, encrypt=False, appvm=None):
-    backup_restore_header(backup_dir)
     # Defaults
     backup_restore_set_defaults(options)
 
     #### Private functions begin
+    '''
     def is_vm_included_in_backup (backup_dir, vm):
         if vm.qid == 0:
             # Dom0 is not included, obviously
@@ -1224,7 +1299,12 @@ def backup_restore_prepare(backup_dir, options = {}, host_collection = None):
             return True
         else:
             return False
-
+    '''
+    def is_vm_included_in_backup (backup_dir, vm):
+        for item in backup_content.keys():
+            if vm.name == item:
+                return True
+        return False
     def find_template_name(template, replaces):
         rx_replace = re.compile("(.*):(.*)")
         for r in replaces:
@@ -1235,14 +1315,19 @@ def backup_restore_prepare(backup_dir, options = {}, host_collection = None):
         return template
 
     #### Private functions end
-
+    '''
     if not os.path.exists (backup_dir):
         raise QubesException("The backup directory doesn't exist!")
+    '''
+    backup_collection = QubesVmCollection()
+    import StringIO
+    backup_collection.qubes_store_file=StringIO.StringIO(qubes_xml)
+
+
+    #backup_collection.lock_db_for_reading()
 
-    backup_collection = QubesVmCollection(store_filename = backup_dir + "/qubes.xml")
-    backup_collection.lock_db_for_reading()
     backup_collection.load()
-
+    
     if host_collection is None:
         host_collection = QubesVmCollection()
         host_collection.lock_db_for_reading()
@@ -1250,9 +1335,11 @@ def backup_restore_prepare(backup_dir, options = {}, host_collection = None):
         host_collection.unlock_db()
 
     backup_vms_list = [vm for vm in backup_collection.values()]
+
     host_vms_list = [vm for vm in host_collection.values()]
     vms_to_restore = {}
 
+
     there_are_conflicting_vms = False
     there_are_missing_templates = False
     there_are_missing_netvms = False
@@ -1325,6 +1412,7 @@ def backup_restore_prepare(backup_dir, options = {}, host_collection = None):
                 vms_to_restore[vm.name]['good-to-go'] = True
 
     # ...and dom0 home
+    # TODO, replace this part of code to handle the new backup format using tar
     if options['dom0-home'] and os.path.exists(backup_dir + '/dom0-home'):
         vms_to_restore['dom0'] = {}
         local_user = grp.getgrnam('qubes').gr_mem[0]
@@ -1443,10 +1531,11 @@ def backup_restore_print_summary(restore_info, print_callback = print_stdout):
 
         print_callback(s)
 
-def backup_restore_do(backup_dir, restore_info, host_collection = None, print_callback = print_stdout, error_callback = print_stderr):
+def backup_restore_do(backup_dir, restore_info, restore_vms, host_collection = None, print_callback = print_stdout, error_callback = print_stderr, encrypted=False, appvm = None ):
 
     ### Private functions begin
-    def restore_vm_dir (backup_dir, src_dir, dst_dir):
+    '''
+    def restore_vm_dir (backup_dir, src_dir, dst_dir, print_callback, error_callback, encrypted, appvm):
 
         backup_src_dir = src_dir.replace (qubes_base_dir, backup_dir)
 
@@ -1454,6 +1543,7 @@ def backup_restore_do(backup_dir, restore_info, host_collection = None, print_ca
         retcode = subprocess.call (["cp", "-rp", backup_src_dir, dst_dir])
         if retcode != 0:
             raise QubesException("*** Error while copying file {0} to {1}".format(backup_src_dir, dest_dir))
+    '''
     ### Private functions end
 
     lock_obtained = False
@@ -1495,7 +1585,8 @@ def backup_restore_do(backup_dir, restore_info, host_collection = None, print_ca
                                                    dir_path=vm.dir_path,
                                                    template=template,
                                                    installed_by_rpm=False)
-                restore_vm_dir (backup_dir, vm.dir_path, os.path.dirname(new_vm.dir_path));
+
+                restore_vm_dir (backup_dir, vm.dir_path, os.path.dirname(new_vm.dir_path), restore_vms[vm.name], print_callback, error_callback, encrypted, appvm)
 
                 new_vm.verify_files()
             except Exception as err:

dom0/qvm-tools/qvm-backup-restore

@@ -22,7 +22,8 @@
 
 from qubes.qubes import QubesVmCollection
 from qubes.qubes import QubesException
-from qubes.qubesutils import backup_restore_prepare
+from qubes.qubesutils import backup_restore_header
+from qubes.qubesutils import backup_restore_prepare 
 from qubes.qubesutils import backup_restore_print_summary
 from qubes.qubesutils import backup_restore_do
 from optparse import OptionParser
@@ -58,6 +59,12 @@ def main():
     parser.add_option ("--ignore-username-mismatch", action="store_true", dest="ignore_username_mismatch", default=False,
                        help="Ignore dom0 username mismatch while restoring homedir")
 
+    parser.add_option ("-d", "--dest-vm", action="store", dest="appvm",
+                       help="The AppVM to send backups to")
+
+    parser.add_option ("-e", "--encrypted", action="store_true", dest="decrypt", default=False,
+                       help="The backup is encrypted")
+
     (options, args) = parser.parse_args ()
 
     if (len (args) != 1):
@@ -66,9 +73,9 @@ def main():
 
     backup_dir = args[0]
 
-    if not os.path.exists (backup_dir):
+    #if not os.path.exists (backup_dir):
-        print >> sys.stderr, "The backup directory doesn't exist!"
+    #    print >> sys.stderr, "The backup directory doesn't exist!"
-        exit(1)
+    #    exit(1)
 
     host_collection = QubesVmCollection()
     host_collection.lock_db_for_writing()
@@ -87,9 +94,13 @@ def main():
     if options.exclude:
         restore_options['exclude'] = options.exclude
 
+
+    print >> sys.stderr, "Checking backup content..."
+    encrypted, vms, qubes_xml = backup_restore_header(backup_dir, options.decrypt, appvm=options.appvm)
+
     restore_info = None
     try:
-        restore_info = backup_restore_prepare(backup_dir, options=restore_options, host_collection=host_collection)
+        restore_info = backup_restore_prepare(backup_dir, vms, qubes_xml, options=restore_options, host_collection=host_collection, encrypt=encrypted, appvm=options.appvm)
     except QubesException as e:
         print >> sys.stderr, "ERROR: %s" % str(e)
         exit(1)
@@ -113,8 +124,6 @@ def main():
         if 'username-mismatch' in vm_info.keys():
             dom0_username_mismatch = True
 
-    print
-
     if os.geteuid() == 0:
         print >> sys.stderr, "*** Running this tool as root is strongly discouraged, this will lead you in permissions problems."
         if options.force_root:
@@ -179,7 +188,8 @@ def main():
     if not (prompt == "y" or prompt == "Y"):
         exit (0)
 
-    backup_restore_do(backup_dir, restore_info, host_collection=host_collection)
+
+    backup_restore_do(backup_dir, restore_info, vms, host_collection=host_collection, encrypted=encrypted, appvm=options.appvm)
 
     host_collection.unlock_db()