backup: major revamp of the backup code to include backup to dom0, backup to vm, better cleanup code

dom0/qvm-core/qubesutils.py

@@ -1003,7 +1003,16 @@ def backup_do(base_backup_dir, files_to_backup, progress_callback = None):
         progress = bytes_backedup * 100 / total_backup_sz
         progress_callback(progress)
 
-def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = None, encrypt=False):
+def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, encrypt=False, appvm=None):
+    print appvm,base_backup_dir
+    total_backup_sz = 0
+    for file in files_to_backup:
+        total_backup_sz += file["size"]
+
+    vmproc = None
+    if appvm != None:
+        # Prepare the backup target (Qubes service call)
+        backup_target = "QUBESRPC qubes.Backup none"
 
         # does the vm exist?
         qvm_collection = QubesVmCollection()
@@ -1016,97 +1025,101 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback =
 
         qvm_collection.unlock_db()
 
-    total_backup_sz = 0
+        # If APPVM, STDOUT is a PIPE
-    for file in files_to_backup:
+        vmproc = vm.run(command = backup_target, passio_popen = True)
-        total_backup_sz += file["size"]
+        vmproc.stdin.write(base_backup_dir.replace("\r","").replace("\n","")+"\n")
+        backup_stdout = vmproc.stdin
 
-    backup_dir = base_backup_dir + "/qubes-{0}".format (time.strftime("%Y-%m-%d-%H%M%S"))
+    else:
-    '''
+        # Prepare the backup target (local file)
-    if os.path.exists (backup_dir):
+        backup_target = base_backup_dir + "/qubes-{0}".format (time.strftime("%Y-%m-%d-%H%M%S"))
-        raise QubesException("ERROR: the path {0} already exists?!".format(backup_dir))
+
+        # Create the target directory
+        if not os.path.exists (base_backup_dir):
+            raise QubesException("ERROR: the backup directory {0} does not exists".format(base_backup_dir))
+
+        # If not APPVM, STDOUT is a local file
+        backup_stdout = open(backup_target,'wb')
 
-    os.mkdir (backup_dir)
 
-    if not os.path.exists (backup_dir):
-        raise QubesException("Strange: couldn't create backup dir: {0}?!".format(backup_dir))
-    '''
     blocks_backedup = 0
-
     progress = blocks_backedup * 11 / total_backup_sz
     progress_callback(progress)
-    dest_dir = backup_dir + '/' + file["subdir"]
-    if file["subdir"] != "":
-        retcode = vm.run("mkdir -p " + dest_dir, wait = True)
-        if retcode != 0:
-            raise QubesException("Cannot create directory: {0}?!".format(dest_dir))
-
-    file["basename"] = os.path.basename(file["path"])
-    vm.run("mkdir -p {0}".format(dest_dir))
 
     tar_cmdline = ["tar", "-PczO",'--sparse','-C','/var/lib/qubes','--checkpoint=10000']
 
     for filename in files_to_backup:
         tar_cmdline.append(filename["path"].split("/var/lib/qubes/")[1])
-    print ("Will backup using command",tar_cmdline)
+    #print ("Will backup using command",tar_cmdline)
-
-    retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"]), passio_popen = True)
 
     import tempfile
     feedback_file = tempfile.NamedTemporaryFile()
-    print feedback_file
+    #print feedback_file
     if encrypt:
         compressor = subprocess.Popen (tar_cmdline,stdout=subprocess.PIPE,stderr=feedback_file)
-        encryptor  = subprocess.Popen (["gpg2", "-c", "--force-mdc", "-o-"], stdin=compressor.stdout, stdout=retcode.stdin)
+        encryptor  = subprocess.Popen (["gpg2", "-c", "--force-mdc", "-o-"], stdin=compressor.stdout, stdout=backup_stdout)
-        '''
-        print "Wait for encryptor"
-        encryptor.wait()
-        print "End waiting"
-
-        if encryptor.returncode != 0:
-            raise QubesException("Failed to backup file {0} with error {1}".format(file["basename"]))
-        '''
     else:
-        compressor = subprocess.Popen (tar_cmdline,stdout=retcode.stdin)
+        compressor = subprocess.Popen (tar_cmdline,stdout=backup_stdout,stderr=feedback_file)
+        encryptor  = None
 
-        '''
+    # Get tar backup feedback
-        for checkpoint in compressor.stderr:
-            print "Checkpoints:",len(checkpoints)
-
-            match = re.search('tar:.*(\d+)',checkpoints)
-            if match:
-                print bytes_backedup,total_backup_sz
-                progress = int(match.group(1)) * 100 / total_backup_sz
-                progress_callback(progress)
-        '''
-    '''
-    print "Wait for compressor"
-    compressor.wait()
-    print "End waiting"
-    '''
     feedback_file_r = open(feedback_file.name,'r')
-    while compressor.poll()==None or (encryptor!=None and encryptor.poll()==None):
+    run_error = None
+    run_count = 1
+    while run_count > 0 and run_error == None:
         time.sleep(1)
-        #print "Polling:", compressor.poll(),encryptor.poll()
-        #print feedback_file_r.tell(),feedback_file_r.closed,feedback_file_r.name,feedback_file_r.readline()
 
         match = re.search("tar: [^0-9]+([0-9]+)",feedback_file_r.readline())
-        print match
         if match:
             blocks_backedup = int(match.group(1))
             progress = blocks_backedup * 11.024 * 1024 / total_backup_sz
             #print blocks_backedup,total_backup_sz,progress
             progress_callback(round(progress*100,2))
 
+        run_count = 0
+        if compressor:
+            retcode=compressor.poll()
+            if retcode != None:
+                if retcode != 0:
+                    run_error = "compressor"
+            else:
+                run_count += 1
+
+        if encryptor:
+            retcode=encryptor.poll()
+            if retcode != None:
+                if retcode != 0:
+                    run_error = "encryptor"
+            else:
+                run_count += 1
+
+        if vmproc:
+            retcode = vmproc.poll()
+            if retcode != None:
+                if retcode != 0:
+                    run_error = "VM "+appvm
+                    print vmproc.stdout.read()
+            else:
+                # VM should run until the end
+                pass
+
+    # Cleanup
     feedback_file_r.close()
     feedback_file.close()
+    backup_stdout.close()
 
-    retcode.terminate()
+    # Check returns code of compressor and encryptor and qubes vm retcode
-    '''
+    if run_error != None:
-    if compressor.returncode != 0:
+        try:
-        raise QubesException("Failed to backup file {0} with error {1}".format(file["basename"]))
+            if compressor != None:
-    '''    
+                compressor.terminate()
-
+            if encryptor != None:
-
+                encryptor.terminate()
+            if vmproc != None:
+                vmproc.terminate()
+        except OSError:
+            pass
+        raise QubesException("Failed to perform backup: error with "+run_error)
 
 def backup_restore_set_defaults(options):
     if 'use-default-netvm' not in options:

dom0/qvm-tools/qvm-backup

@@ -76,7 +76,7 @@ def main():
         exit (0)
 
     try:
-        backup_do_copy(options.appvm, base_backup_dir, files_to_backup, progress_callback=print_progress, encrypt=options.encrypt)
+        backup_do_copy(base_backup_dir, files_to_backup, progress_callback=print_progress, encrypt=options.encrypt,appvm=options.appvm)
     except QubesException as e:
         print >>sys.stderr, "ERROR: %s" % str(e)
         exit(1)

qubes_rpc/qubes.Backup

@@ -0,0 +1,23 @@
+echo Starting Backupcopy
+read args
+echo Arguments: $args 
+if [ -d "$args" ] ; then
+  echo "Performing backup to directory $args"
+  TARGET="$args/qubes-backup-`date +'%Y-%d-%d-%H%M%S'`"
+  echo "Copying STDIN data to $TARGET"
+  cat > $TARGET
+else
+  echo "Checking if arguments is matching a command"
+  COMMAND=`echo $args | cut -d ' ' -f 1`
+  TYPE=`type -t $COMMAND`
+  if [ "$TYPE" == "file" ] ; then
+    echo "Redirecting STDIN to $args"
+    # Parsing args to handle quotes correctly
+    # Dangerous method if args are uncontrolled
+    eval "set -- $args"
+    $@
+  else
+    echo "Invalid command $COMMAND"
+    exit 1
+  fi
+fi