backup: implemented backup mecanism using tar_sparse+encryption+hmac generation

dom0/qvm-core/qubesutils.py

@@ -1054,73 +1054,139 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e
     progress = blocks_backedup * 11 / total_backup_sz
     progress_callback(progress)
 
-    tar_cmdline = ["tar", "-PczO",'--sparse','-C',qubes_base_dir,'--checkpoint=10000']
-
-    for filename in files_to_backup:
-        tar_cmdline.append(filename["path"].split(os.path.normpath(qubes_base_dir)+"/")[1])
-    #print ("Will backup using command",tar_cmdline)
-
     import tempfile
     feedback_file = tempfile.NamedTemporaryFile()
-    #print feedback_file
-    if encrypt:
-        compressor = subprocess.Popen (tar_cmdline,stdout=subprocess.PIPE,stderr=feedback_file)
-        encryptor  = subprocess.Popen (["gpg2", "-c", "--force-mdc", "-o-"], stdin=compressor.stdout, stdout=backup_stdout)
-    else:
-        compressor = subprocess.Popen (tar_cmdline,stdout=backup_stdout,stderr=feedback_file)
-        encryptor  = None
 
-    run_error = wait_backup_feedback(progress_callback, feedback_file, total_backup_sz, compressor, encryptor, vmproc)
+    for filename in files_to_backup:
+        print "Backing up",filename
+        tar_cmdline = ["tar", "-PcO",'--sparse','-C',qubes_base_dir,
+            filename["path"].split(os.path.normpath(qubes_base_dir)+"/")[1]
+            ]
 
-    feedback_file.close()
+        # Prepare all subprocesses with the right stdin, stdout
+        if encrypt:
+            # Tips: Popen(bufsize=0)
+            compressor = subprocess.Popen (tar_cmdline,stdout=subprocess.PIPE)
+
+            encryptor  = subprocess.Popen (["openssl", "enc", "-e", "-k", "azerty"], stdin=compressor.stdout, stdout=subprocess.PIPE)
+            hmac       = subprocess.Popen (["openssl", "dgst", "-hmac", "azerty"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+
+            streamproc = encryptor
+            addproc    = compressor
+        else:
+            compressor = subprocess.Popen (tar_cmdline,stdout=subprocess.PIPE)
+
+            encryptor  = None
+            hmac       = subprocess.Popen (["openssl", "dgst", "-hmac", "azerty"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+
+            streamproc = compressor
+            addproc    = None
+
+        # Wait for compressor (tar) process to finish or for any error of other subprocesses
+        run_error = wait_backup_feedback(progress_callback, streamproc, backup_stdout, total_backup_sz, hmac=hmac, vmproc=vmproc, addproc=addproc)
+        if len(run_error) > 0:
+            raise QubesException("Failed to perform backup: error with "+run_error)
+
+        # Wait for all remaining subprocess to finish
+        if addproc:
+            addproc.wait()
+            print "Addproc:",addproc.poll()
+
+        streamproc.wait()
+        print "Streamproc:",streamproc.poll()
+
+        hmac.stdin.close()
+        hmac.wait()
+        print "HMAC:",hmac.poll()
+
+        # Write HMAC data next to the original file
+        hmac_data = hmac.stdout.read()
+        print "Writing hmac to",filename['path']+".hmac"
+        hmac_file = open(filename['path']+".hmac",'w')
+        hmac_file.write(hmac_data)
+        hmac_file.flush()
+        hmac_file.close()
+
+        # Send the hmac file to the backup target
+        tar_cmdline[-1] += ".hmac"
+        print tar_cmdline
+        streamproc = subprocess.Popen(tar_cmdline,stdout=subprocess.PIPE)
+        run_error = wait_backup_feedback(progress_callback, streamproc, backup_stdout, total_backup_sz, vmproc=vmproc)
+        if len(run_error) > 0:
+            raise QubesException("Failed to perform backup: error with "+run_error)
+
+        streamproc.wait()
+        print "HMAC sent:",streamproc.poll()
+        
+    # Close the backup target and wait for it to finish
     backup_stdout.close()
-
+    if vmproc:
-    # Check returns code of compressor and encryptor and qubes vm retcode
+        print "VMProc1:",vmproc.poll()
-    if run_error != None:
+        vmproc.wait()
-        try:
+        print "VMProc2:",vmproc.poll()
-            if compressor != None:
-                compressor.terminate()
-            if encryptor != None:
-                encryptor.terminate()
-            if vmproc != None:
-                vmproc.terminate()
-        except OSError:
-            pass
-        raise QubesException("Failed to perform backup: error with "+run_error)
 
 
-def wait_backup_feedback(progress_callback, feedback_file, total_backup_sz, compressor, encryptor, vmproc):
+def wait_backup_feedback(progress_callback, streamproc, backup_target, total_backup_sz, hmac=None, vmproc=None, addproc=None, remove_trailing_bytes=0):
-    # Get tar backup feedback
+
-    feedback_file_r = open(feedback_file.name,'r')
+    buffer_size = 4096
+
     run_error = None
     run_count = 1
+    blocks_backedup = 0
     while run_count > 0 and run_error == None:
-        time.sleep(1)
 
-        match = re.search("tar: [^0-9]+([0-9]+)",feedback_file_r.readline())
+        buffer = streamproc.stdout.read(buffer_size)
-        if match:
+        #print "Read",len(buffer)
-            blocks_backedup = int(match.group(1))
+
-            progress = blocks_backedup * 11.024 * 1024 / total_backup_sz
+        blocks_backedup += len(buffer)
-            #print blocks_backedup,total_backup_sz,progress
+
-            progress_callback(round(progress*100,2))
+        progress = blocks_backedup / float(total_backup_sz)
+        progress_callback(round(progress*100,2))
 
         run_count = 0
-        if compressor:
+        if hmac:
-            retcode=compressor.poll()
+            retcode=hmac.poll()
             if retcode != None:
                 if retcode != 0:
-                    run_error = "compressor"
+                    run_error = "hmac"
             else:
                 run_count += 1
 
-        if encryptor:
+        if addproc:
-            retcode=encryptor.poll()
+            retcode=addproc.poll()
             if retcode != None:
                 if retcode != 0:
-                    run_error = "encryptor"
+                    run_error = "addproc"
             else:
                 run_count += 1
 
+        retcode=streamproc.poll()
+        if retcode != None:
+            if retcode != 0:
+                run_error = "streamproc"
+            elif retcode == 0 and len(buffer) <= 0:
+                return ""
+            else:
+                if remove_trailing_bytes > 0:
+                    print buffer.encode("hex")
+                    buffer = buffer[:-remove_trailing_bytes]
+                    print buffer.encode("hex")
+
+                backup_target.write(buffer)
+
+                if hmac:
+                    hmac.stdin.write(buffer)
+
+                run_count += 1
+        else:
+            # Process still running
+            backup_target.write(buffer)
+
+            if hmac:
+                hmac.stdin.write(buffer)
+
+            run_count += 1
+
         if vmproc:
             retcode = vmproc.poll()
             if retcode != None:
@@ -1131,9 +1197,6 @@ def wait_backup_feedback(progress_callback, feedback_file, total_backup_sz, comp
                 # VM should run until the end
                 pass
 
-    # Cleanup
-    feedback_file_r.close()
-
     return run_error
 
 def restore_vm_dir (backup_dir, src_dir, dst_dir, vm_spec, print_callback=None, error_callback=None, encrypted=False, appvm=None):