From bf2dd7f0c74527d033f51f011ce15c02b33f7c1d Mon Sep 17 00:00:00 2001 From: Andrew Sorensen Date: Tue, 26 Mar 2013 18:36:29 -0700 Subject: [PATCH 01/34] dom0: allow backup to AppVM instead of just local block device --- dom0/qvm-core/qubesutils.py | 44 +++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 2fb63aa9..f6b89c8c 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -780,10 +780,10 @@ def file_to_backup (file_path, sz = None): def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_callback = print_stdout): """If vms = None, include all (sensible) VMs; exclude_list is always applied""" - + ''' if not os.path.exists (base_backup_dir): raise QubesException("The target directory doesn't exist!") - + ''' files_to_backup = file_to_backup (qubes_store_filename) if exclude_list is None: @@ -955,7 +955,7 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca fmt="{{0:-^{0}}}-+".format(f["width"] + 1) s += fmt.format('-') print_callback(s) - + ''' stat = os.statvfs(base_backup_dir) backup_fs_free_sz = stat.f_bsize * stat.f_bavail print_callback("") @@ -966,7 +966,7 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca raise QubesException("Please shutdown all VMs before proceeding.") print_callback("-> Available space: {0}".format(size_to_human(backup_fs_free_sz))) - + ''' return files_to_backup def backup_do(base_backup_dir, files_to_backup, progress_callback = None): @@ -1003,6 +1003,42 @@ def backup_do(base_backup_dir, files_to_backup, progress_callback = None): progress = bytes_backedup * 100 / total_backup_sz progress_callback(progress) +def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = None): + + total_backup_sz = 0 + for file in files_to_backup: + total_backup_sz += file["size"] + + backup_dir = base_backup_dir + "/qubes-{0}".format (time.strftime("%Y-%m-%d-%H%M%S")) + ''' + if os.path.exists (backup_dir): + raise QubesException("ERROR: the path {0} already exists?!".format(backup_dir)) + + os.mkdir (backup_dir) + + if not os.path.exists (backup_dir): + raise QubesException("Strange: couldn't create backup dir: {0}?!".format(backup_dir)) + ''' + bytes_backedup = 0 + for file in files_to_backup: + # We prefer to use Linux's cp, because it nicely handles sparse files + progress = bytes_backedup * 100 / total_backup_sz + progress_callback(progress) + dest_dir = backup_dir + '/' + file["subdir"] + if file["subdir"] != "": + retcode = subprocess.call (["qvm-run", "-p", appvm, "mkdir -p " + dest_dir]) + if retcode != 0: + raise QubesException("Cannot create directory: {0}?!".format(dest_dir)) + + file["basename"] = os.path.basename(file["path"]) + compressor = subprocess.Popen (["gzip", "-c", file["path"]], stdout=subprocess.PIPE) + subprocess.Popen (["qvm-run", "--pass-io", "-p", appvm, "cat > " + backup_dir + "/" + file["basename"]], stdin=compressor.stdout) + + bytes_backedup += file["size"] + progress = bytes_backedup * 100 / total_backup_sz + progress_callback(progress) + + def backup_restore_set_defaults(options): if 'use-default-netvm' not in options: options['use-default-netvm'] = False From 65822f60337b93d087c2fcb225764b50926b526b Mon Sep 17 00:00:00 2001 From: Andrew Sorensen Date: Wed, 27 Mar 2013 11:04:39 -0700 Subject: [PATCH 02/34] dom0: switch backup compression to .tar.gz, properly handle folders. --- dom0/qvm-core/qubesutils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index f6b89c8c..27371429 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1031,8 +1031,8 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = raise QubesException("Cannot create directory: {0}?!".format(dest_dir)) file["basename"] = os.path.basename(file["path"]) - compressor = subprocess.Popen (["gzip", "-c", file["path"]], stdout=subprocess.PIPE) - subprocess.Popen (["qvm-run", "--pass-io", "-p", appvm, "cat > " + backup_dir + "/" + file["basename"]], stdin=compressor.stdout) + compressor = subprocess.Popen (["tar", "-PcOz", file["path"]], stdout=subprocess.PIPE) + subprocess.Popen (["qvm-run", "--pass-io", "-p", appvm, "cat > " + dest_dir + file["basename"] + ".tar.gz"], stdin=compressor.stdout) bytes_backedup += file["size"] progress = bytes_backedup * 100 / total_backup_sz From 7cacc3db482f033cee6ece401f5bd9ae7bc1b5a3 Mon Sep 17 00:00:00 2001 From: Andrew Sorensen Date: Sat, 22 Jun 2013 18:02:40 -0700 Subject: [PATCH 03/34] dom0: edit qvm-backup to use backup_do_copy, throw error when appvm is not found --- dom0/qvm-core/qubesutils.py | 11 ++++++++++- dom0/qvm-tools/qvm-backup | 4 ++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 27371429..4263345e 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1005,6 +1005,15 @@ def backup_do(base_backup_dir, files_to_backup, progress_callback = None): def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = None): + # does the vm exist? + qvm_collection = QubesVmCollection() + qvm_collection.lock_db_for_reading() + qvm_collection.load() + + vm = qvm_collection.get_vm_by_name(appvm) + if vm is None or vm.qid not in qvm_collection: + raise QubesException("VM {0} does not exist".format(appvm)) + total_backup_sz = 0 for file in files_to_backup: total_backup_sz += file["size"] @@ -1026,7 +1035,7 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = progress_callback(progress) dest_dir = backup_dir + '/' + file["subdir"] if file["subdir"] != "": - retcode = subprocess.call (["qvm-run", "-p", appvm, "mkdir -p " + dest_dir]) + retcode = vm.run(["qvm-run", "-p", appvm, "mkdir -p " + dest_dir]) if retcode != 0: raise QubesException("Cannot create directory: {0}?!".format(dest_dir)) diff --git a/dom0/qvm-tools/qvm-backup b/dom0/qvm-tools/qvm-backup index cee29e3b..fafb9eb2 100755 --- a/dom0/qvm-tools/qvm-backup +++ b/dom0/qvm-tools/qvm-backup @@ -22,7 +22,7 @@ from qubes.qubes import QubesVmCollection from qubes.qubes import QubesException -from qubes.qubesutils import backup_prepare, backup_do +from qubes.qubesutils import backup_prepare, backup_do_copy from optparse import OptionParser import os import sys @@ -72,7 +72,7 @@ def main(): exit (0) try: - backup_do(base_backup_dir, files_to_backup, progress_callback=print_progress) + backup_do_copy("storage", base_backup_dir, files_to_backup, progress_callback=print_progress) except QubesException as e: print >>sys.stderr, "ERROR: %s" % str(e) exit(1) From 246e8c383dac369c947653262bdeee17977acad9 Mon Sep 17 00:00:00 2001 From: Andrew Sorensen Date: Sat, 22 Jun 2013 18:22:11 -0700 Subject: [PATCH 04/34] dom0: fix directory creation --- dom0/qvm-core/qubesutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 4263345e..1a155050 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1035,7 +1035,7 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = progress_callback(progress) dest_dir = backup_dir + '/' + file["subdir"] if file["subdir"] != "": - retcode = vm.run(["qvm-run", "-p", appvm, "mkdir -p " + dest_dir]) + retcode = vm.run("mkdir -p " + dest_dir) if retcode != 0: raise QubesException("Cannot create directory: {0}?!".format(dest_dir)) From 93162df677406cee8a1769156f66b7908d98121c Mon Sep 17 00:00:00 2001 From: Andrew Sorensen Date: Sat, 22 Jun 2013 19:59:16 -0700 Subject: [PATCH 05/34] dom0: use vm.run() instead of subprocess.Popen() directly --- dom0/qvm-core/qubesutils.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 1a155050..a4b9e7c5 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1030,7 +1030,6 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = ''' bytes_backedup = 0 for file in files_to_backup: - # We prefer to use Linux's cp, because it nicely handles sparse files progress = bytes_backedup * 100 / total_backup_sz progress_callback(progress) dest_dir = backup_dir + '/' + file["subdir"] @@ -1040,9 +1039,13 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = raise QubesException("Cannot create directory: {0}?!".format(dest_dir)) file["basename"] = os.path.basename(file["path"]) - compressor = subprocess.Popen (["tar", "-PcOz", file["path"]], stdout=subprocess.PIPE) - subprocess.Popen (["qvm-run", "--pass-io", "-p", appvm, "cat > " + dest_dir + file["basename"] + ".tar.gz"], stdin=compressor.stdout) - + vm.run("mkdir -p {0}".format(dest_dir)) + retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"] + ".tar.gz"), passio_popen = True) + compressor = subprocess.Popen (["tar", "-PcOz", file["path"]], stdout=retcode.stdin) + compressor.wait() + if compressor.retcode != 0: + raise QubesException("Failed to backup file {0} with error {1}".format(file["basename"])) + bytes_backedup += file["size"] progress = bytes_backedup * 100 / total_backup_sz progress_callback(progress) From c2f157c2d2f1eea93519925091a4d398ccba203a Mon Sep 17 00:00:00 2001 From: Andrew Sorensen Date: Sat, 22 Jun 2013 20:30:32 -0700 Subject: [PATCH 06/34] dom0: close Popen when the transfer is complete --- dom0/qvm-core/qubesutils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index a4b9e7c5..92b6449c 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1043,7 +1043,8 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"] + ".tar.gz"), passio_popen = True) compressor = subprocess.Popen (["tar", "-PcOz", file["path"]], stdout=retcode.stdin) compressor.wait() - if compressor.retcode != 0: + retcode.terminate() + if compressor.returncode != 0: raise QubesException("Failed to backup file {0} with error {1}".format(file["basename"])) bytes_backedup += file["size"] From 3d7af2f7f51e91f74d8a6be3efbd70e440cb39e2 Mon Sep 17 00:00:00 2001 From: Andrew Sorensen Date: Sat, 22 Jun 2013 20:59:07 -0700 Subject: [PATCH 07/34] dom0: allow the user to set the AppVM --- dom0/qvm-tools/qvm-backup | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dom0/qvm-tools/qvm-backup b/dom0/qvm-tools/qvm-backup index fafb9eb2..a8f1d50d 100755 --- a/dom0/qvm-tools/qvm-backup +++ b/dom0/qvm-tools/qvm-backup @@ -38,6 +38,8 @@ def main(): help="Exclude the specified VM from backup (might be repeated)") parser.add_option ("--force-root", action="store_true", dest="force_root", default=False, help="Force to run, even with root privileges") + parser.add_option ("-d", "--dest-vm", action="store", dest="appvm", + help="The AppVM to send backups to") (options, args) = parser.parse_args () @@ -72,7 +74,7 @@ def main(): exit (0) try: - backup_do_copy("storage", base_backup_dir, files_to_backup, progress_callback=print_progress) + backup_do_copy(options.appvm, base_backup_dir, files_to_backup, progress_callback=print_progress) except QubesException as e: print >>sys.stderr, "ERROR: %s" % str(e) exit(1) From 4ed00f123d707c4614e164be66a5ae542fbf4383 Mon Sep 17 00:00:00 2001 From: Andrew Sorensen Date: Sat, 22 Jun 2013 21:19:59 -0700 Subject: [PATCH 08/34] dom0: allow user to decide if encryption should be used, close qvm database --- dom0/qvm-core/qubesutils.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 92b6449c..f30f02f6 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1003,7 +1003,7 @@ def backup_do(base_backup_dir, files_to_backup, progress_callback = None): progress = bytes_backedup * 100 / total_backup_sz progress_callback(progress) -def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = None): +def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = None, encrypt=False): # does the vm exist? qvm_collection = QubesVmCollection() @@ -1014,6 +1014,8 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = if vm is None or vm.qid not in qvm_collection: raise QubesException("VM {0} does not exist".format(appvm)) + qvm_collection.unlock_db() + total_backup_sz = 0 for file in files_to_backup: total_backup_sz += file["size"] @@ -1040,8 +1042,13 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = file["basename"] = os.path.basename(file["path"]) vm.run("mkdir -p {0}".format(dest_dir)) - retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"] + ".tar.gz"), passio_popen = True) - compressor = subprocess.Popen (["tar", "-PcOz", file["path"]], stdout=retcode.stdin) + if encrypt: + retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"] + ".gpg"), passio_popen = True) + compressor = subprocess.Popen (["gpg", "-ac", "--force-mdc", "-o-", file["path"]], stdout=retcode.stdin) + else: + retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"] + ".tar.gz"), passio_popen = True) + compressor = subprocess.Popen (["tar", "-PcOz", file["path"]], stdout=retcode.stdin) + compressor.wait() retcode.terminate() if compressor.returncode != 0: From 1d2990e9380ba81af091c0a460d981a439299dc4 Mon Sep 17 00:00:00 2001 From: Andrew Sorensen Date: Sat, 22 Jun 2013 21:27:58 -0700 Subject: [PATCH 09/34] dom0: add option for encryption to qvm-backup --- dom0/qvm-tools/qvm-backup | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dom0/qvm-tools/qvm-backup b/dom0/qvm-tools/qvm-backup index a8f1d50d..4fef7314 100755 --- a/dom0/qvm-tools/qvm-backup +++ b/dom0/qvm-tools/qvm-backup @@ -40,6 +40,8 @@ def main(): help="Force to run, even with root privileges") parser.add_option ("-d", "--dest-vm", action="store", dest="appvm", help="The AppVM to send backups to") + parser.add_option ("-e", "--encrypt", action="store_true", dest="encrypt", default=False, + help="Encrypts the backup") (options, args) = parser.parse_args () @@ -74,7 +76,7 @@ def main(): exit (0) try: - backup_do_copy(options.appvm, base_backup_dir, files_to_backup, progress_callback=print_progress) + backup_do_copy(options.appvm, base_backup_dir, files_to_backup, progress_callback=print_progress, encrypt=options.encrypt) except QubesException as e: print >>sys.stderr, "ERROR: %s" % str(e) exit(1) From ead479804e3f9679f1abc8da7002492758105b9a Mon Sep 17 00:00:00 2001 From: Andrew Sorensen Date: Sun, 7 Jul 2013 00:00:07 -0700 Subject: [PATCH 10/34] dom0: wait for folder to be created before adding file --- dom0/qvm-core/qubesutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index f30f02f6..d84b7a8b 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1036,7 +1036,7 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = progress_callback(progress) dest_dir = backup_dir + '/' + file["subdir"] if file["subdir"] != "": - retcode = vm.run("mkdir -p " + dest_dir) + retcode = vm.run("mkdir -p " + dest_dir, wait = True) if retcode != 0: raise QubesException("Cannot create directory: {0}?!".format(dest_dir)) From fb8748f3e9d292ad9bf790fcf3169faef9ad8925 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Wed, 14 Aug 2013 10:18:05 +0200 Subject: [PATCH 11/34] backup: implemented use of tar+gpg2 instead of only encrypting files --- dom0/qvm-core/qubesutils.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index d84b7a8b..8aba4943 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1042,15 +1042,32 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = file["basename"] = os.path.basename(file["path"]) vm.run("mkdir -p {0}".format(dest_dir)) - if encrypt: - retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"] + ".gpg"), passio_popen = True) - compressor = subprocess.Popen (["gpg", "-ac", "--force-mdc", "-o-", file["path"]], stdout=retcode.stdin) - else: - retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"] + ".tar.gz"), passio_popen = True) - compressor = subprocess.Popen (["tar", "-PcOz", file["path"]], stdout=retcode.stdin) + retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"]), passio_popen = True) + + if encrypt: + compressor = subprocess.Popen (["tar", "-PcO",'--checkpoint=10000', file["path"]],stdout=subprocess.PIPE) + encryptor = subprocess.Popen (["gpg2", "-ac", "--force-mdc", "-o-"], stdin=compressor.stdout, stdout=retcode.stdin) + encryptor.wait() + + if encryptor.returncode != 0: + raise QubesException("Failed to backup file {0} with error {1}".format(file["basename"])) + else: + compressor = subprocess.Popen (["tar", "-PcOz",'--checkpoint=10000', file["path"]],stdout=retcode.stdin) + + ''' + for checkpoint in compressor.stderr: + print "Checkpoints:",len(checkpoints) + + match = re.search('tar:.*(\d+)',checkpoints) + if match: + print bytes_backedup,total_backup_sz + progress = int(match.group(1)) * 100 / total_backup_sz + progress_callback(progress) + ''' compressor.wait() retcode.terminate() + if compressor.returncode != 0: raise QubesException("Failed to backup file {0} with error {1}".format(file["basename"])) From 6c09189b774af32c97b84e93b8c5710851c5851f Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Wed, 14 Aug 2013 10:19:15 +0200 Subject: [PATCH 12/34] backup: improved performance by optimizing tar and gpg options --- dom0/qvm-core/qubesutils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 8aba4943..642414cf 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1046,8 +1046,8 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"]), passio_popen = True) if encrypt: - compressor = subprocess.Popen (["tar", "-PcO",'--checkpoint=10000', file["path"]],stdout=subprocess.PIPE) - encryptor = subprocess.Popen (["gpg2", "-ac", "--force-mdc", "-o-"], stdin=compressor.stdout, stdout=retcode.stdin) + compressor = subprocess.Popen (["tar", "-PcO",'--sparse','--checkpoint=10000', file["path"]],stdout=subprocess.PIPE) + encryptor = subprocess.Popen (["gpg2", "-c", "--force-mdc", "-o-"], stdin=compressor.stdout, stdout=retcode.stdin) encryptor.wait() if encryptor.returncode != 0: From aea789d0a17beee9993ca6a684581d74a7563b7e Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Wed, 14 Aug 2013 10:21:41 +0200 Subject: [PATCH 13/34] backup: implemented use of a single tar file instead of creation of multiple file during backup --- dom0/qvm-core/qubesutils.py | 58 ++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 642414cf..bc22a331 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1031,29 +1031,35 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = raise QubesException("Strange: couldn't create backup dir: {0}?!".format(backup_dir)) ''' bytes_backedup = 0 - for file in files_to_backup: - progress = bytes_backedup * 100 / total_backup_sz - progress_callback(progress) - dest_dir = backup_dir + '/' + file["subdir"] - if file["subdir"] != "": - retcode = vm.run("mkdir -p " + dest_dir, wait = True) - if retcode != 0: - raise QubesException("Cannot create directory: {0}?!".format(dest_dir)) - file["basename"] = os.path.basename(file["path"]) - vm.run("mkdir -p {0}".format(dest_dir)) + progress = bytes_backedup * 100 / total_backup_sz + progress_callback(progress) + dest_dir = backup_dir + '/' + file["subdir"] + if file["subdir"] != "": + retcode = vm.run("mkdir -p " + dest_dir, wait = True) + if retcode != 0: + raise QubesException("Cannot create directory: {0}?!".format(dest_dir)) - retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"]), passio_popen = True) + file["basename"] = os.path.basename(file["path"]) + vm.run("mkdir -p {0}".format(dest_dir)) - if encrypt: - compressor = subprocess.Popen (["tar", "-PcO",'--sparse','--checkpoint=10000', file["path"]],stdout=subprocess.PIPE) - encryptor = subprocess.Popen (["gpg2", "-c", "--force-mdc", "-o-"], stdin=compressor.stdout, stdout=retcode.stdin) - encryptor.wait() + tar_cmdline = ["tar", "-PcO",'--sparse','-C','/var/lib/qubes','--checkpoint=10000'] - if encryptor.returncode != 0: - raise QubesException("Failed to backup file {0} with error {1}".format(file["basename"])) - else: - compressor = subprocess.Popen (["tar", "-PcOz",'--checkpoint=10000', file["path"]],stdout=retcode.stdin) + for filename in files_to_backup: + tar_cmdline.append(filename["path"].split("/var/lib/qubes/")[1]) + print ("Will backup using command",tar_cmdline) + + retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"]), passio_popen = True) + + if encrypt: + compressor = subprocess.Popen (tar_cmdline,stdout=subprocess.PIPE) + encryptor = subprocess.Popen (["gpg2", "-c", "--force-mdc", "-o-"], stdin=compressor.stdout, stdout=retcode.stdin) + encryptor.wait() + + if encryptor.returncode != 0: + raise QubesException("Failed to backup file {0} with error {1}".format(file["basename"])) + else: + compressor = subprocess.Popen (tar_cmdline,stdout=retcode.stdin) ''' for checkpoint in compressor.stderr: @@ -1065,15 +1071,15 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = progress = int(match.group(1)) * 100 / total_backup_sz progress_callback(progress) ''' - compressor.wait() - retcode.terminate() + compressor.wait() + retcode.terminate() - if compressor.returncode != 0: - raise QubesException("Failed to backup file {0} with error {1}".format(file["basename"])) + if compressor.returncode != 0: + raise QubesException("Failed to backup file {0} with error {1}".format(file["basename"])) - bytes_backedup += file["size"] - progress = bytes_backedup * 100 / total_backup_sz - progress_callback(progress) + bytes_backedup += file["size"] + progress = bytes_backedup * 100 / total_backup_sz + progress_callback(progress) def backup_restore_set_defaults(options): From fbb26d89b445bea460eb5e2c900994ea0d5f475b Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Wed, 14 Aug 2013 10:23:04 +0200 Subject: [PATCH 14/34] backup: implemented progress feedback using tar checkpoint and a temporary file for tar output --- dom0/qvm-core/qubesutils.py | 44 +++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index bc22a331..01a1c14a 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1030,9 +1030,9 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = if not os.path.exists (backup_dir): raise QubesException("Strange: couldn't create backup dir: {0}?!".format(backup_dir)) ''' - bytes_backedup = 0 + blocks_backedup = 0 - progress = bytes_backedup * 100 / total_backup_sz + progress = blocks_backedup * 11 / total_backup_sz progress_callback(progress) dest_dir = backup_dir + '/' + file["subdir"] if file["subdir"] != "": @@ -1043,7 +1043,7 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = file["basename"] = os.path.basename(file["path"]) vm.run("mkdir -p {0}".format(dest_dir)) - tar_cmdline = ["tar", "-PcO",'--sparse','-C','/var/lib/qubes','--checkpoint=10000'] + tar_cmdline = ["tar", "-PczO",'--sparse','-C','/var/lib/qubes','--checkpoint=10000'] for filename in files_to_backup: tar_cmdline.append(filename["path"].split("/var/lib/qubes/")[1]) @@ -1051,13 +1051,20 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"]), passio_popen = True) + import tempfile + feedback_file = tempfile.NamedTemporaryFile() + print feedback_file if encrypt: - compressor = subprocess.Popen (tar_cmdline,stdout=subprocess.PIPE) + compressor = subprocess.Popen (tar_cmdline,stdout=subprocess.PIPE,stderr=feedback_file) encryptor = subprocess.Popen (["gpg2", "-c", "--force-mdc", "-o-"], stdin=compressor.stdout, stdout=retcode.stdin) + ''' + print "Wait for encryptor" encryptor.wait() + print "End waiting" if encryptor.returncode != 0: raise QubesException("Failed to backup file {0} with error {1}".format(file["basename"])) + ''' else: compressor = subprocess.Popen (tar_cmdline,stdout=retcode.stdin) @@ -1071,15 +1078,34 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = progress = int(match.group(1)) * 100 / total_backup_sz progress_callback(progress) ''' + ''' + print "Wait for compressor" compressor.wait() - retcode.terminate() + print "End waiting" + ''' + feedback_file_r = open(feedback_file.name,'r') + while compressor.poll()==None or (encryptor!=None and encryptor.poll()==None): + time.sleep(1) + #print "Polling:", compressor.poll(),encryptor.poll() + #print feedback_file_r.tell(),feedback_file_r.closed,feedback_file_r.name,feedback_file_r.readline() + match = re.search("tar: [^0-9]+([0-9]+)",feedback_file_r.readline()) + print match + if match: + blocks_backedup = int(match.group(1)) + progress = blocks_backedup * 11.024 * 1024 / total_backup_sz + #print blocks_backedup,total_backup_sz,progress + progress_callback(round(progress*100,2)) + + feedback_file_r.close() + feedback_file.close() + + retcode.terminate() + ''' if compressor.returncode != 0: raise QubesException("Failed to backup file {0} with error {1}".format(file["basename"])) - - bytes_backedup += file["size"] - progress = bytes_backedup * 100 / total_backup_sz - progress_callback(progress) + ''' + def backup_restore_set_defaults(options): From 5fa8d732ae70b47828625ad20806315e42479911 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Wed, 14 Aug 2013 10:26:58 +0200 Subject: [PATCH 15/34] backup: major revamp of the backup code to include backup to dom0, backup to vm, better cleanup code --- dom0/qvm-core/qubesutils.py | 153 +++++++++++++++++++----------------- dom0/qvm-tools/qvm-backup | 2 +- qubes_rpc/qubes.Backup | 23 ++++++ 3 files changed, 107 insertions(+), 71 deletions(-) create mode 100644 qubes_rpc/qubes.Backup diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 01a1c14a..de5212ee 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1003,110 +1003,123 @@ def backup_do(base_backup_dir, files_to_backup, progress_callback = None): progress = bytes_backedup * 100 / total_backup_sz progress_callback(progress) -def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = None, encrypt=False): - - # does the vm exist? - qvm_collection = QubesVmCollection() - qvm_collection.lock_db_for_reading() - qvm_collection.load() - - vm = qvm_collection.get_vm_by_name(appvm) - if vm is None or vm.qid not in qvm_collection: - raise QubesException("VM {0} does not exist".format(appvm)) - - qvm_collection.unlock_db() - +def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, encrypt=False, appvm=None): + print appvm,base_backup_dir total_backup_sz = 0 for file in files_to_backup: total_backup_sz += file["size"] + + vmproc = None + if appvm != None: + # Prepare the backup target (Qubes service call) + backup_target = "QUBESRPC qubes.Backup none" + + # does the vm exist? + qvm_collection = QubesVmCollection() + qvm_collection.lock_db_for_reading() + qvm_collection.load() - backup_dir = base_backup_dir + "/qubes-{0}".format (time.strftime("%Y-%m-%d-%H%M%S")) - ''' - if os.path.exists (backup_dir): - raise QubesException("ERROR: the path {0} already exists?!".format(backup_dir)) + vm = qvm_collection.get_vm_by_name(appvm) + if vm is None or vm.qid not in qvm_collection: + raise QubesException("VM {0} does not exist".format(appvm)) + + qvm_collection.unlock_db() + + # If APPVM, STDOUT is a PIPE + vmproc = vm.run(command = backup_target, passio_popen = True) + vmproc.stdin.write(base_backup_dir.replace("\r","").replace("\n","")+"\n") + backup_stdout = vmproc.stdin + + else: + # Prepare the backup target (local file) + backup_target = base_backup_dir + "/qubes-{0}".format (time.strftime("%Y-%m-%d-%H%M%S")) + + # Create the target directory + if not os.path.exists (base_backup_dir): + raise QubesException("ERROR: the backup directory {0} does not exists".format(base_backup_dir)) + + # If not APPVM, STDOUT is a local file + backup_stdout = open(backup_target,'wb') - os.mkdir (backup_dir) - if not os.path.exists (backup_dir): - raise QubesException("Strange: couldn't create backup dir: {0}?!".format(backup_dir)) - ''' blocks_backedup = 0 - progress = blocks_backedup * 11 / total_backup_sz progress_callback(progress) - dest_dir = backup_dir + '/' + file["subdir"] - if file["subdir"] != "": - retcode = vm.run("mkdir -p " + dest_dir, wait = True) - if retcode != 0: - raise QubesException("Cannot create directory: {0}?!".format(dest_dir)) - - file["basename"] = os.path.basename(file["path"]) - vm.run("mkdir -p {0}".format(dest_dir)) tar_cmdline = ["tar", "-PczO",'--sparse','-C','/var/lib/qubes','--checkpoint=10000'] for filename in files_to_backup: tar_cmdline.append(filename["path"].split("/var/lib/qubes/")[1]) - print ("Will backup using command",tar_cmdline) - - retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"]), passio_popen = True) + #print ("Will backup using command",tar_cmdline) import tempfile feedback_file = tempfile.NamedTemporaryFile() - print feedback_file + #print feedback_file if encrypt: compressor = subprocess.Popen (tar_cmdline,stdout=subprocess.PIPE,stderr=feedback_file) - encryptor = subprocess.Popen (["gpg2", "-c", "--force-mdc", "-o-"], stdin=compressor.stdout, stdout=retcode.stdin) - ''' - print "Wait for encryptor" - encryptor.wait() - print "End waiting" - - if encryptor.returncode != 0: - raise QubesException("Failed to backup file {0} with error {1}".format(file["basename"])) - ''' + encryptor = subprocess.Popen (["gpg2", "-c", "--force-mdc", "-o-"], stdin=compressor.stdout, stdout=backup_stdout) else: - compressor = subprocess.Popen (tar_cmdline,stdout=retcode.stdin) + compressor = subprocess.Popen (tar_cmdline,stdout=backup_stdout,stderr=feedback_file) + encryptor = None - ''' - for checkpoint in compressor.stderr: - print "Checkpoints:",len(checkpoints) - - match = re.search('tar:.*(\d+)',checkpoints) - if match: - print bytes_backedup,total_backup_sz - progress = int(match.group(1)) * 100 / total_backup_sz - progress_callback(progress) - ''' - ''' - print "Wait for compressor" - compressor.wait() - print "End waiting" - ''' + # Get tar backup feedback feedback_file_r = open(feedback_file.name,'r') - while compressor.poll()==None or (encryptor!=None and encryptor.poll()==None): + run_error = None + run_count = 1 + while run_count > 0 and run_error == None: time.sleep(1) - #print "Polling:", compressor.poll(),encryptor.poll() - #print feedback_file_r.tell(),feedback_file_r.closed,feedback_file_r.name,feedback_file_r.readline() match = re.search("tar: [^0-9]+([0-9]+)",feedback_file_r.readline()) - print match if match: blocks_backedup = int(match.group(1)) progress = blocks_backedup * 11.024 * 1024 / total_backup_sz #print blocks_backedup,total_backup_sz,progress progress_callback(round(progress*100,2)) + run_count = 0 + if compressor: + retcode=compressor.poll() + if retcode != None: + if retcode != 0: + run_error = "compressor" + else: + run_count += 1 + + if encryptor: + retcode=encryptor.poll() + if retcode != None: + if retcode != 0: + run_error = "encryptor" + else: + run_count += 1 + + if vmproc: + retcode = vmproc.poll() + if retcode != None: + if retcode != 0: + run_error = "VM "+appvm + print vmproc.stdout.read() + else: + # VM should run until the end + pass + + # Cleanup feedback_file_r.close() feedback_file.close() + backup_stdout.close() - retcode.terminate() - ''' - if compressor.returncode != 0: - raise QubesException("Failed to backup file {0} with error {1}".format(file["basename"])) - ''' - - + # Check returns code of compressor and encryptor and qubes vm retcode + if run_error != None: + try: + if compressor != None: + compressor.terminate() + if encryptor != None: + encryptor.terminate() + if vmproc != None: + vmproc.terminate() + except OSError: + pass + raise QubesException("Failed to perform backup: error with "+run_error) def backup_restore_set_defaults(options): if 'use-default-netvm' not in options: diff --git a/dom0/qvm-tools/qvm-backup b/dom0/qvm-tools/qvm-backup index 4fef7314..ea3a43dd 100755 --- a/dom0/qvm-tools/qvm-backup +++ b/dom0/qvm-tools/qvm-backup @@ -76,7 +76,7 @@ def main(): exit (0) try: - backup_do_copy(options.appvm, base_backup_dir, files_to_backup, progress_callback=print_progress, encrypt=options.encrypt) + backup_do_copy(base_backup_dir, files_to_backup, progress_callback=print_progress, encrypt=options.encrypt,appvm=options.appvm) except QubesException as e: print >>sys.stderr, "ERROR: %s" % str(e) exit(1) diff --git a/qubes_rpc/qubes.Backup b/qubes_rpc/qubes.Backup new file mode 100644 index 00000000..6e3f1d48 --- /dev/null +++ b/qubes_rpc/qubes.Backup @@ -0,0 +1,23 @@ +echo Starting Backupcopy +read args +echo Arguments: $args +if [ -d "$args" ] ; then + echo "Performing backup to directory $args" + TARGET="$args/qubes-backup-`date +'%Y-%d-%d-%H%M%S'`" + echo "Copying STDIN data to $TARGET" + cat > $TARGET +else + echo "Checking if arguments is matching a command" + COMMAND=`echo $args | cut -d ' ' -f 1` + TYPE=`type -t $COMMAND` + if [ "$TYPE" == "file" ] ; then + echo "Redirecting STDIN to $args" + # Parsing args to handle quotes correctly + # Dangerous method if args are uncontrolled + eval "set -- $args" + $@ + else + echo "Invalid command $COMMAND" + exit 1 + fi +fi From 9784ca87f688ff0d32e6bf444940c8cd0ff720b6 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Fri, 16 Aug 2013 09:12:06 +0200 Subject: [PATCH 16/34] backup: implemented mecanism to read only the backup headers --- dom0/qvm-core/qubesutils.py | 78 +++++++++++++++++++++++++++++++++++-- 1 file changed, 75 insertions(+), 3 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index de5212ee..248a69d6 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -906,7 +906,14 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca s += " <-- The VM is running, please shut it down before proceeding with the backup!" there_are_running_vms = True - print_callback(s) + # Build Backup VMs reference file + # Format: vm_name:tarball_path\n + backup_reference_file = open(os.path.join(qubes_base_dir,"backup_targets"),'w') + for vm in vms_for_backup: + backup_reference_file.write(vm.name+":"+vm.dir_path.split(qubes_base_dir)[1]+"\n") + backup_reference_file.flush() + backup_reference_file.close() + files_to_backup = file_to_backup(backup_reference_file.name,os.stat(backup_reference_file.name).st_size) + files_to_backup # Dom0 user home if not 'dom0' in exclude_list: @@ -1046,10 +1053,10 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e progress = blocks_backedup * 11 / total_backup_sz progress_callback(progress) - tar_cmdline = ["tar", "-PczO",'--sparse','-C','/var/lib/qubes','--checkpoint=10000'] + tar_cmdline = ["tar", "-PczO",'--sparse','-C',qubes_base_dir,'--checkpoint=10000'] for filename in files_to_backup: - tar_cmdline.append(filename["path"].split("/var/lib/qubes/")[1]) + tar_cmdline.append(filename["path"].split(os.path.normpath(qubes_base_dir)+"/")[1]) #print ("Will backup using command",tar_cmdline) import tempfile @@ -1135,8 +1142,73 @@ def backup_restore_set_defaults(options): return options +def backup_restore_header(restore_target, progress_callback = None, encrypt=False, appvm=None): + # Simulate dd if=backup_file count=10 | file - + # Simulate dd if=backup_file count=10 | gpg2 -d | tar xzv -O + # analysis = subprocess.Popen() + vmproc = None + if appvm != None: + # Prepare the backup target (Qubes service call) + backup_target = "QUBESRPC qubes.Backup none" + + # does the vm exist? + qvm_collection = QubesVmCollection() + qvm_collection.lock_db_for_reading() + qvm_collection.load() + + vm = qvm_collection.get_vm_by_name(appvm) + if vm is None or vm.qid not in qvm_collection: + raise QubesException("VM {0} does not exist".format(appvm)) + + qvm_collection.unlock_db() + + # If APPVM, STDOUT is a PIPE + vmproc = vm.run(command = restore_target, passio_popen = True) + vmproc.stdin.write(restore_target.replace("\r","").replace("\n","")+"\n") + headers = vmproc.stdout.read(4096) + vmproc.terminate() + + if len(headers) <= 0: + print vmproc.stderr.read() + raise QubesException("ERROR: the backup directory {0} does not exists".format(restore_target)) + + else: + # Create the target directory + if not os.path.exists (restore_target): + raise QubesException("ERROR: the backup directory {0} does not exists".format(restore_target)) + + fp = open(restore_target,'rb') + headers = fp.read(4096) + + is_encrypted = False + + command = subprocess.Popen(['file','-'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + stdout,stderr = command.communicate(headers) + if not stdout.find('gzip compressed data') >= 0: + command = subprocess.Popen(['gpg2','--decrypt'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + stdout,stderr = command.communicate(headers) + if len(stdout) > 0: + headers = stdout + is_encrypted = True + else: + print stderr + raise QubesException("ERROR: unable to decrypt the backup {0}. Is it really encrypted?".format(restore_target)) + + command = subprocess.Popen(['tar', 'xzv', '-O', 'backup_targets'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + vm_list,stderr = command.communicate(headers) + if len(vm_list) <= 0: + print stderr + raise QubesException("ERROR: unable to read the qubes backup file {0}. Is it really a backup?".format(restore_target)) + + vms_in_backup = [] + for vm in vm_list.split("\n"): + vms_in_backup.append(vm.strip("\r\t\n ").split(":")) + + print vms_in_backup + return is_encrypted, vms_in_backup def backup_restore_prepare(backup_dir, options = {}, host_collection = None): + backup_restore_header(backup_dir) # Defaults backup_restore_set_defaults(options) From 836c604473823b374175c6e4b41a4674556e1ce5 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Mon, 19 Aug 2013 16:48:29 +0200 Subject: [PATCH 17/34] backup: reimplemented restore function through an AppVM The VM size is now stored inside the backup specification file in order to compute progress. --- dom0/qvm-core/qubesutils.py | 149 ++++++++++++++++++++++++------ dom0/qvm-tools/qvm-backup-restore | 26 ++++-- 2 files changed, 138 insertions(+), 37 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 248a69d6..e91a3a55 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -906,11 +906,13 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca s += " <-- The VM is running, please shut it down before proceeding with the backup!" there_are_running_vms = True + print_callback(s) + # Build Backup VMs reference file # Format: vm_name:tarball_path\n backup_reference_file = open(os.path.join(qubes_base_dir,"backup_targets"),'w') for vm in vms_for_backup: - backup_reference_file.write(vm.name+":"+vm.dir_path.split(qubes_base_dir)[1]+"\n") + backup_reference_file.write(vm.name+":"+vm.dir_path.split(qubes_base_dir)[1]+":"+str(vm.get_disk_utilization())+"\n") backup_reference_file.flush() backup_reference_file.close() files_to_backup = file_to_backup(backup_reference_file.name,os.stat(backup_reference_file.name).st_size) + files_to_backup @@ -1011,7 +1013,6 @@ def backup_do(base_backup_dir, files_to_backup, progress_callback = None): progress_callback(progress) def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, encrypt=False, appvm=None): - print appvm,base_backup_dir total_backup_sz = 0 for file in files_to_backup: total_backup_sz += file["size"] @@ -1069,6 +1070,26 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e compressor = subprocess.Popen (tar_cmdline,stdout=backup_stdout,stderr=feedback_file) encryptor = None + run_error = wait_backup_feedback(progress_callback, feedback_file, total_backup_sz, compressor, encryptor, vmproc) + + feedback_file.close() + backup_stdout.close() + + # Check returns code of compressor and encryptor and qubes vm retcode + if run_error != None: + try: + if compressor != None: + compressor.terminate() + if encryptor != None: + encryptor.terminate() + if vmproc != None: + vmproc.terminate() + except OSError: + pass + raise QubesException("Failed to perform backup: error with "+run_error) + + +def wait_backup_feedback(progress_callback, feedback_file, total_backup_sz, compressor, encryptor, vmproc): # Get tar backup feedback feedback_file_r = open(feedback_file.name,'r') run_error = None @@ -1104,7 +1125,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e retcode = vmproc.poll() if retcode != None: if retcode != 0: - run_error = "VM "+appvm + run_error = "VM" print vmproc.stdout.read() else: # VM should run until the end @@ -1112,8 +1133,56 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e # Cleanup feedback_file_r.close() + + return run_error + +def restore_vm_dir (backup_dir, src_dir, dst_dir, vm_spec, print_callback=None, error_callback=None, encrypted=False, appvm=None): + + #backup_src_dir = src_dir.replace (qubes_base_dir, backup_dir) + + vmproc = None + if appvm != None: + # Prepare the backup target (Qubes service call) + backup_target = "QUBESRPC qubes.Restore none" + + # does the vm exist? + qvm_collection = QubesVmCollection() + qvm_collection.lock_db_for_reading() + qvm_collection.load() + + vm = qvm_collection.get_vm_by_name(appvm) + if vm is None or vm.qid not in qvm_collection: + raise QubesException("VM {0} does not exist".format(appvm)) + + qvm_collection.unlock_db() + + # If APPVM, STDOUT is a PIPE + vmproc = vm.run(command = backup_target, passio_popen = True) + vmproc.stdin.write(backup_dir.replace("\r","").replace("\n","")+"\n") + backup_stdin = vmproc.stdout + else: + backup_stdin = open(backup_dir,'rb') + + tar_cmdline = ["tar", "-xzv",'--sparse','-C',qubes_base_dir,'--checkpoint=10000'] + + tar_cmdline.append(src_dir.split(os.path.normpath(qubes_base_dir)+"/")[1]) + + #print ("Will backup using command",tar_cmdline) + + import tempfile + feedback_file = tempfile.NamedTemporaryFile() + if encrypted: + encryptor = subprocess.Popen (["gpg2", "--decrypt"], stdin=backup_stdin, stdout=subprocess.PIPE,stderr=subprocess.PIPE) + compressor = subprocess.Popen (tar_cmdline,stdin=encryptor.stdout,stderr=feedback_file,stdout=subprocess.PIPE) + else: + compressor = subprocess.Popen (tar_cmdline,stdin=backup_stdin,stderr=feedback_file,stdout=subprocess.PIPE) + encryptor = None + + run_error = wait_backup_feedback(print_callback, feedback_file, vm_spec["size"], compressor, encryptor, vmproc) + + # Cleanup feedback_file.close() - backup_stdout.close() + backup_stdin.close() # Check returns code of compressor and encryptor and qubes vm retcode if run_error != None: @@ -1142,14 +1211,14 @@ def backup_restore_set_defaults(options): return options -def backup_restore_header(restore_target, progress_callback = None, encrypt=False, appvm=None): +def backup_restore_header(restore_target, encrypt=False, appvm=None): # Simulate dd if=backup_file count=10 | file - # Simulate dd if=backup_file count=10 | gpg2 -d | tar xzv -O # analysis = subprocess.Popen() vmproc = None if appvm != None: # Prepare the backup target (Qubes service call) - backup_target = "QUBESRPC qubes.Backup none" + restore_command = "QUBESRPC qubes.Restore none" # does the vm exist? qvm_collection = QubesVmCollection() @@ -1163,14 +1232,14 @@ def backup_restore_header(restore_target, progress_callback = None, encrypt=Fals qvm_collection.unlock_db() # If APPVM, STDOUT is a PIPE - vmproc = vm.run(command = restore_target, passio_popen = True) + vmproc = vm.run(command = restore_command, passio_popen = True) vmproc.stdin.write(restore_target.replace("\r","").replace("\n","")+"\n") - headers = vmproc.stdout.read(4096) + + headers = vmproc.stdout.read(4096*4) vmproc.terminate() if len(headers) <= 0: - print vmproc.stderr.read() - raise QubesException("ERROR: the backup directory {0} does not exists".format(restore_target)) + raise QubesException("ERROR: unable to read the backup target {0}".format(restore_target)) else: # Create the target directory @@ -1178,7 +1247,7 @@ def backup_restore_header(restore_target, progress_callback = None, encrypt=Fals raise QubesException("ERROR: the backup directory {0} does not exists".format(restore_target)) fp = open(restore_target,'rb') - headers = fp.read(4096) + headers = fp.read(4096*4) is_encrypted = False @@ -1194,25 +1263,31 @@ def backup_restore_header(restore_target, progress_callback = None, encrypt=Fals print stderr raise QubesException("ERROR: unable to decrypt the backup {0}. Is it really encrypted?".format(restore_target)) - command = subprocess.Popen(['tar', 'xzv', '-O', 'backup_targets'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) - vm_list,stderr = command.communicate(headers) - if len(vm_list) <= 0: + command = subprocess.Popen(['tar', 'xzv', '-O', 'backup_targets','qubes.xml'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + headers,stderr = command.communicate(headers) + if len(headers) <= 0: print stderr raise QubesException("ERROR: unable to read the qubes backup file {0}. Is it really a backup?".format(restore_target)) - vms_in_backup = [] - for vm in vm_list.split("\n"): - vms_in_backup.append(vm.strip("\r\t\n ").split(":")) + vms_in_backup = {} + for vm in headers.split("\n"): + match = re.match("^(?P[^:]+):(?P[^:]+):(?P[0-9]+)$",vm,re.MULTILINE) + if match: + item = match.groupdict() + item["size"] = int(item["size"]) + vms_in_backup[item["name"]] = item + headers = headers.replace(vm,"") + else: + break - print vms_in_backup - return is_encrypted, vms_in_backup + return is_encrypted, vms_in_backup, headers -def backup_restore_prepare(backup_dir, options = {}, host_collection = None): - backup_restore_header(backup_dir) +def backup_restore_prepare(backup_dir, backup_content, qubes_xml, options = {}, host_collection = None, encrypt=False, appvm=None): # Defaults backup_restore_set_defaults(options) #### Private functions begin + ''' def is_vm_included_in_backup (backup_dir, vm): if vm.qid == 0: # Dom0 is not included, obviously @@ -1224,7 +1299,12 @@ def backup_restore_prepare(backup_dir, options = {}, host_collection = None): return True else: return False - + ''' + def is_vm_included_in_backup (backup_dir, vm): + for item in backup_content.keys(): + if vm.name == item: + return True + return False def find_template_name(template, replaces): rx_replace = re.compile("(.*):(.*)") for r in replaces: @@ -1235,14 +1315,19 @@ def backup_restore_prepare(backup_dir, options = {}, host_collection = None): return template #### Private functions end - + ''' if not os.path.exists (backup_dir): raise QubesException("The backup directory doesn't exist!") + ''' + backup_collection = QubesVmCollection() + import StringIO + backup_collection.qubes_store_file=StringIO.StringIO(qubes_xml) + + + #backup_collection.lock_db_for_reading() - backup_collection = QubesVmCollection(store_filename = backup_dir + "/qubes.xml") - backup_collection.lock_db_for_reading() backup_collection.load() - + if host_collection is None: host_collection = QubesVmCollection() host_collection.lock_db_for_reading() @@ -1250,9 +1335,11 @@ def backup_restore_prepare(backup_dir, options = {}, host_collection = None): host_collection.unlock_db() backup_vms_list = [vm for vm in backup_collection.values()] + host_vms_list = [vm for vm in host_collection.values()] vms_to_restore = {} + there_are_conflicting_vms = False there_are_missing_templates = False there_are_missing_netvms = False @@ -1325,6 +1412,7 @@ def backup_restore_prepare(backup_dir, options = {}, host_collection = None): vms_to_restore[vm.name]['good-to-go'] = True # ...and dom0 home + # TODO, replace this part of code to handle the new backup format using tar if options['dom0-home'] and os.path.exists(backup_dir + '/dom0-home'): vms_to_restore['dom0'] = {} local_user = grp.getgrnam('qubes').gr_mem[0] @@ -1443,10 +1531,11 @@ def backup_restore_print_summary(restore_info, print_callback = print_stdout): print_callback(s) -def backup_restore_do(backup_dir, restore_info, host_collection = None, print_callback = print_stdout, error_callback = print_stderr): +def backup_restore_do(backup_dir, restore_info, restore_vms, host_collection = None, print_callback = print_stdout, error_callback = print_stderr, encrypted=False, appvm = None ): ### Private functions begin - def restore_vm_dir (backup_dir, src_dir, dst_dir): + ''' + def restore_vm_dir (backup_dir, src_dir, dst_dir, print_callback, error_callback, encrypted, appvm): backup_src_dir = src_dir.replace (qubes_base_dir, backup_dir) @@ -1454,6 +1543,7 @@ def backup_restore_do(backup_dir, restore_info, host_collection = None, print_ca retcode = subprocess.call (["cp", "-rp", backup_src_dir, dst_dir]) if retcode != 0: raise QubesException("*** Error while copying file {0} to {1}".format(backup_src_dir, dest_dir)) + ''' ### Private functions end lock_obtained = False @@ -1495,7 +1585,8 @@ def backup_restore_do(backup_dir, restore_info, host_collection = None, print_ca dir_path=vm.dir_path, template=template, installed_by_rpm=False) - restore_vm_dir (backup_dir, vm.dir_path, os.path.dirname(new_vm.dir_path)); + + restore_vm_dir (backup_dir, vm.dir_path, os.path.dirname(new_vm.dir_path), restore_vms[vm.name], print_callback, error_callback, encrypted, appvm) new_vm.verify_files() except Exception as err: diff --git a/dom0/qvm-tools/qvm-backup-restore b/dom0/qvm-tools/qvm-backup-restore index 6b1a590d..05c56374 100755 --- a/dom0/qvm-tools/qvm-backup-restore +++ b/dom0/qvm-tools/qvm-backup-restore @@ -22,7 +22,8 @@ from qubes.qubes import QubesVmCollection from qubes.qubes import QubesException -from qubes.qubesutils import backup_restore_prepare +from qubes.qubesutils import backup_restore_header +from qubes.qubesutils import backup_restore_prepare from qubes.qubesutils import backup_restore_print_summary from qubes.qubesutils import backup_restore_do from optparse import OptionParser @@ -58,6 +59,12 @@ def main(): parser.add_option ("--ignore-username-mismatch", action="store_true", dest="ignore_username_mismatch", default=False, help="Ignore dom0 username mismatch while restoring homedir") + parser.add_option ("-d", "--dest-vm", action="store", dest="appvm", + help="The AppVM to send backups to") + + parser.add_option ("-e", "--encrypted", action="store_true", dest="decrypt", default=False, + help="The backup is encrypted") + (options, args) = parser.parse_args () if (len (args) != 1): @@ -66,9 +73,9 @@ def main(): backup_dir = args[0] - if not os.path.exists (backup_dir): - print >> sys.stderr, "The backup directory doesn't exist!" - exit(1) + #if not os.path.exists (backup_dir): + # print >> sys.stderr, "The backup directory doesn't exist!" + # exit(1) host_collection = QubesVmCollection() host_collection.lock_db_for_writing() @@ -87,9 +94,13 @@ def main(): if options.exclude: restore_options['exclude'] = options.exclude + + print >> sys.stderr, "Checking backup content..." + encrypted, vms, qubes_xml = backup_restore_header(backup_dir, options.decrypt, appvm=options.appvm) + restore_info = None try: - restore_info = backup_restore_prepare(backup_dir, options=restore_options, host_collection=host_collection) + restore_info = backup_restore_prepare(backup_dir, vms, qubes_xml, options=restore_options, host_collection=host_collection, encrypt=encrypted, appvm=options.appvm) except QubesException as e: print >> sys.stderr, "ERROR: %s" % str(e) exit(1) @@ -113,8 +124,6 @@ def main(): if 'username-mismatch' in vm_info.keys(): dom0_username_mismatch = True - print - if os.geteuid() == 0: print >> sys.stderr, "*** Running this tool as root is strongly discouraged, this will lead you in permissions problems." if options.force_root: @@ -179,7 +188,8 @@ def main(): if not (prompt == "y" or prompt == "Y"): exit (0) - backup_restore_do(backup_dir, restore_info, host_collection=host_collection) + + backup_restore_do(backup_dir, restore_info, vms, host_collection=host_collection, encrypted=encrypted, appvm=options.appvm) host_collection.unlock_db() From 89fde55cd954232a58394a34bc33f00c57272407 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Mon, 19 Aug 2013 16:50:46 +0200 Subject: [PATCH 18/34] backup: Added rpc restoration file --- qubes_rpc/qubes.Restore | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 qubes_rpc/qubes.Restore diff --git a/qubes_rpc/qubes.Restore b/qubes_rpc/qubes.Restore new file mode 100644 index 00000000..dfea1448 --- /dev/null +++ b/qubes_rpc/qubes.Restore @@ -0,0 +1,23 @@ +echo Starting Restorecopy >2 +read args +echo Arguments: $args >2 +if [ -f "$args" ] ; then + echo "Performing restore from backup file $args" >2 + TARGET="$args" + echo "Copying $TARGET to STDOUT" >2 + cat $TARGET +else + echo "Checking if arguments is matching a command" >2 + COMMAND=`echo $args | cut -d ' ' -f 1` + TYPE=`type -t $COMMAND` + if [ "$TYPE" == "file" ] ; then + echo "Redirecting $args to STDOUT" >2 + # Parsing args to handle quotes correctly + # Dangerous method if args are uncontrolled + eval "set -- $args" + $@ + else + echo "Invalid command $COMMAND" >2 + exit 1 + fi +fi From 5edca4ac90b260fc72dd95bc02570208c672ba23 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Tue, 10 Sep 2013 09:20:49 +0200 Subject: [PATCH 19/34] backup: code cleanup --- dom0/qvm-core/qubesutils.py | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index e91a3a55..8f831743 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1249,16 +1249,11 @@ def backup_restore_header(restore_target, encrypt=False, appvm=None): fp = open(restore_target,'rb') headers = fp.read(4096*4) - is_encrypted = False - - command = subprocess.Popen(['file','-'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) - stdout,stderr = command.communicate(headers) - if not stdout.find('gzip compressed data') >= 0: + if encrypt: command = subprocess.Popen(['gpg2','--decrypt'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) stdout,stderr = command.communicate(headers) if len(stdout) > 0: headers = stdout - is_encrypted = True else: print stderr raise QubesException("ERROR: unable to decrypt the backup {0}. Is it really encrypted?".format(restore_target)) @@ -1280,26 +1275,13 @@ def backup_restore_header(restore_target, encrypt=False, appvm=None): else: break - return is_encrypted, vms_in_backup, headers + return vms_in_backup, headers def backup_restore_prepare(backup_dir, backup_content, qubes_xml, options = {}, host_collection = None, encrypt=False, appvm=None): # Defaults backup_restore_set_defaults(options) #### Private functions begin - ''' - def is_vm_included_in_backup (backup_dir, vm): - if vm.qid == 0: - # Dom0 is not included, obviously - return False - - backup_vm_dir_path = vm.dir_path.replace (qubes_base_dir, backup_dir) - - if os.path.exists (backup_vm_dir_path): - return True - else: - return False - ''' def is_vm_included_in_backup (backup_dir, vm): for item in backup_content.keys(): if vm.name == item: @@ -1313,12 +1295,8 @@ def backup_restore_prepare(backup_dir, backup_content, qubes_xml, options = {}, return m.group(2) return template - #### Private functions end - ''' - if not os.path.exists (backup_dir): - raise QubesException("The backup directory doesn't exist!") - ''' + backup_collection = QubesVmCollection() import StringIO backup_collection.qubes_store_file=StringIO.StringIO(qubes_xml) From 4ae4bdc45258f681c71b3cbbb19cc1954927d80f Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Tue, 10 Sep 2013 09:22:35 +0200 Subject: [PATCH 20/34] backup: implemented backup mecanism using tar_sparse+encryption+hmac generation --- dom0/qvm-core/qubesutils.py | 157 +++++++++++++++++++++++++----------- 1 file changed, 110 insertions(+), 47 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 8f831743..0d308e5a 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1054,73 +1054,139 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e progress = blocks_backedup * 11 / total_backup_sz progress_callback(progress) - tar_cmdline = ["tar", "-PczO",'--sparse','-C',qubes_base_dir,'--checkpoint=10000'] - - for filename in files_to_backup: - tar_cmdline.append(filename["path"].split(os.path.normpath(qubes_base_dir)+"/")[1]) - #print ("Will backup using command",tar_cmdline) - import tempfile feedback_file = tempfile.NamedTemporaryFile() - #print feedback_file - if encrypt: - compressor = subprocess.Popen (tar_cmdline,stdout=subprocess.PIPE,stderr=feedback_file) - encryptor = subprocess.Popen (["gpg2", "-c", "--force-mdc", "-o-"], stdin=compressor.stdout, stdout=backup_stdout) - else: - compressor = subprocess.Popen (tar_cmdline,stdout=backup_stdout,stderr=feedback_file) - encryptor = None - run_error = wait_backup_feedback(progress_callback, feedback_file, total_backup_sz, compressor, encryptor, vmproc) + for filename in files_to_backup: + print "Backing up",filename + tar_cmdline = ["tar", "-PcO",'--sparse','-C',qubes_base_dir, + filename["path"].split(os.path.normpath(qubes_base_dir)+"/")[1] + ] - feedback_file.close() + # Prepare all subprocesses with the right stdin, stdout + if encrypt: + # Tips: Popen(bufsize=0) + compressor = subprocess.Popen (tar_cmdline,stdout=subprocess.PIPE) + + encryptor = subprocess.Popen (["openssl", "enc", "-e", "-k", "azerty"], stdin=compressor.stdout, stdout=subprocess.PIPE) + hmac = subprocess.Popen (["openssl", "dgst", "-hmac", "azerty"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) + + streamproc = encryptor + addproc = compressor + else: + compressor = subprocess.Popen (tar_cmdline,stdout=subprocess.PIPE) + + encryptor = None + hmac = subprocess.Popen (["openssl", "dgst", "-hmac", "azerty"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) + + streamproc = compressor + addproc = None + + # Wait for compressor (tar) process to finish or for any error of other subprocesses + run_error = wait_backup_feedback(progress_callback, streamproc, backup_stdout, total_backup_sz, hmac=hmac, vmproc=vmproc, addproc=addproc) + if len(run_error) > 0: + raise QubesException("Failed to perform backup: error with "+run_error) + + # Wait for all remaining subprocess to finish + if addproc: + addproc.wait() + print "Addproc:",addproc.poll() + + streamproc.wait() + print "Streamproc:",streamproc.poll() + + hmac.stdin.close() + hmac.wait() + print "HMAC:",hmac.poll() + + # Write HMAC data next to the original file + hmac_data = hmac.stdout.read() + print "Writing hmac to",filename['path']+".hmac" + hmac_file = open(filename['path']+".hmac",'w') + hmac_file.write(hmac_data) + hmac_file.flush() + hmac_file.close() + + # Send the hmac file to the backup target + tar_cmdline[-1] += ".hmac" + print tar_cmdline + streamproc = subprocess.Popen(tar_cmdline,stdout=subprocess.PIPE) + run_error = wait_backup_feedback(progress_callback, streamproc, backup_stdout, total_backup_sz, vmproc=vmproc) + if len(run_error) > 0: + raise QubesException("Failed to perform backup: error with "+run_error) + + streamproc.wait() + print "HMAC sent:",streamproc.poll() + + # Close the backup target and wait for it to finish backup_stdout.close() - - # Check returns code of compressor and encryptor and qubes vm retcode - if run_error != None: - try: - if compressor != None: - compressor.terminate() - if encryptor != None: - encryptor.terminate() - if vmproc != None: - vmproc.terminate() - except OSError: - pass - raise QubesException("Failed to perform backup: error with "+run_error) + if vmproc: + print "VMProc1:",vmproc.poll() + vmproc.wait() + print "VMProc2:",vmproc.poll() -def wait_backup_feedback(progress_callback, feedback_file, total_backup_sz, compressor, encryptor, vmproc): - # Get tar backup feedback - feedback_file_r = open(feedback_file.name,'r') +def wait_backup_feedback(progress_callback, streamproc, backup_target, total_backup_sz, hmac=None, vmproc=None, addproc=None, remove_trailing_bytes=0): + + buffer_size = 4096 + run_error = None run_count = 1 + blocks_backedup = 0 while run_count > 0 and run_error == None: - time.sleep(1) - match = re.search("tar: [^0-9]+([0-9]+)",feedback_file_r.readline()) - if match: - blocks_backedup = int(match.group(1)) - progress = blocks_backedup * 11.024 * 1024 / total_backup_sz - #print blocks_backedup,total_backup_sz,progress - progress_callback(round(progress*100,2)) + buffer = streamproc.stdout.read(buffer_size) + #print "Read",len(buffer) + + blocks_backedup += len(buffer) + + progress = blocks_backedup / float(total_backup_sz) + progress_callback(round(progress*100,2)) run_count = 0 - if compressor: - retcode=compressor.poll() + if hmac: + retcode=hmac.poll() if retcode != None: if retcode != 0: - run_error = "compressor" + run_error = "hmac" else: run_count += 1 - if encryptor: - retcode=encryptor.poll() + if addproc: + retcode=addproc.poll() if retcode != None: if retcode != 0: - run_error = "encryptor" + run_error = "addproc" else: run_count += 1 + retcode=streamproc.poll() + if retcode != None: + if retcode != 0: + run_error = "streamproc" + elif retcode == 0 and len(buffer) <= 0: + return "" + else: + if remove_trailing_bytes > 0: + print buffer.encode("hex") + buffer = buffer[:-remove_trailing_bytes] + print buffer.encode("hex") + + backup_target.write(buffer) + + if hmac: + hmac.stdin.write(buffer) + + run_count += 1 + else: + # Process still running + backup_target.write(buffer) + + if hmac: + hmac.stdin.write(buffer) + + run_count += 1 + if vmproc: retcode = vmproc.poll() if retcode != None: @@ -1131,9 +1197,6 @@ def wait_backup_feedback(progress_callback, feedback_file, total_backup_sz, comp # VM should run until the end pass - # Cleanup - feedback_file_r.close() - return run_error def restore_vm_dir (backup_dir, src_dir, dst_dir, vm_spec, print_callback=None, error_callback=None, encrypted=False, appvm=None): From a85f3a7d8e93122298b262ac241722dd61b69cd5 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Tue, 10 Sep 2013 09:24:25 +0200 Subject: [PATCH 21/34] backup: introduced a second tar pass to send encrypted data to an AppVM The backup process is now tar_sparse | encrypt | hmac | tar | appvm --- dom0/qvm-core/qubesutils.py | 171 +++++++++++++++++++++++------------- 1 file changed, 112 insertions(+), 59 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 0d308e5a..dcf510d6 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -915,7 +915,7 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca backup_reference_file.write(vm.name+":"+vm.dir_path.split(qubes_base_dir)[1]+":"+str(vm.get_disk_utilization())+"\n") backup_reference_file.flush() backup_reference_file.close() - files_to_backup = file_to_backup(backup_reference_file.name,os.stat(backup_reference_file.name).st_size) + files_to_backup + #files_to_backup = file_to_backup(backup_reference_file.name,os.stat(backup_reference_file.name).st_size) + files_to_backup # Dom0 user home if not 'dom0' in exclude_list: @@ -1056,76 +1056,125 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e import tempfile feedback_file = tempfile.NamedTemporaryFile() + backup_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/backup_") + + # Tar with tapelength does not deals well with stdout (close stdout between two tapes) + # For this reason, we will use named pipes instead + print "Working in",backup_tmpdir + + backup_pipe = os.path.join(backup_tmpdir,"backup_pipe") + print "Creating pipe in:",backup_pipe + print os.mkfifo(backup_pipe) + + print "Will backup:",files_to_backup for filename in files_to_backup: print "Backing up",filename - tar_cmdline = ["tar", "-PcO",'--sparse','-C',qubes_base_dir, + + backup_tempfile = os.path.join(backup_tmpdir,filename["path"].split(os.path.normpath(qubes_base_dir)+"/")[1]) + print "Using temporary location:",backup_tempfile + + # Ensure the temporary directory exists + + if not os.path.isdir(os.path.dirname(backup_tempfile)): + os.makedirs(os.path.dirname(backup_tempfile)) + + tar_cmdline = ["tar", "-Pc", "-f", backup_pipe,'--sparse','--tape-length',str(1000000),'-C',qubes_base_dir, filename["path"].split(os.path.normpath(qubes_base_dir)+"/")[1] ] - # Prepare all subprocesses with the right stdin, stdout - if encrypt: - # Tips: Popen(bufsize=0) - compressor = subprocess.Popen (tar_cmdline,stdout=subprocess.PIPE) + print " ".join(tar_cmdline) - encryptor = subprocess.Popen (["openssl", "enc", "-e", "-k", "azerty"], stdin=compressor.stdout, stdout=subprocess.PIPE) - hmac = subprocess.Popen (["openssl", "dgst", "-hmac", "azerty"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) - - streamproc = encryptor - addproc = compressor - else: - compressor = subprocess.Popen (tar_cmdline,stdout=subprocess.PIPE) - - encryptor = None - hmac = subprocess.Popen (["openssl", "dgst", "-hmac", "azerty"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) - - streamproc = compressor - addproc = None + # Tips: Popen(bufsize=0) + # Pipe: tar-sparse | encryptor [| hmac] | tar | backup_target + # Pipe: tar-sparse [| hmac] | tar | backup_target + tar_sparse = subprocess.Popen (tar_cmdline,stdin=subprocess.PIPE) # Wait for compressor (tar) process to finish or for any error of other subprocesses - run_error = wait_backup_feedback(progress_callback, streamproc, backup_stdout, total_backup_sz, hmac=hmac, vmproc=vmproc, addproc=addproc) - if len(run_error) > 0: - raise QubesException("Failed to perform backup: error with "+run_error) + i=0 + run_error = "paused" + running = [] + while run_error == "paused": + # Start encrypt + pipe = open(backup_pipe,'rb') + # If no cipher is provided, the data is forwarded unencrypted !!! + encryptor = subprocess.Popen (["openssl", "enc", "-e", "-aes-256-cbc", "-pass", "pass:azerty"], stdin=pipe, stdout=subprocess.PIPE) + + # Start HMAC + hmac = subprocess.Popen (["openssl", "dgst", "-hmac", "azerty"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) + + # Prepare a first chunk + chunkfile = backup_tempfile + "." + "%03d" % i + i += 1 + chunkfile_p = open(chunkfile,'wb') + run_error = wait_backup_feedback(progress_callback, encryptor, chunkfile_p, total_backup_sz, hmac=hmac, vmproc=vmproc, addproc=tar_sparse) + chunkfile_p.close() + + print "Wait_backup_feedback returned:",run_error + + if len(run_error) > 0: + raise QubesException("Failed to perform backup: error with "+run_error) + + # Send the chunk to the backup target + tar_final_cmd = ["tar", "-cO", "-C", backup_tmpdir, chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]] + final_proc = subprocess.Popen (tar_final_cmd, stdin=subprocess.PIPE, stdout=backup_stdout) + final_proc.wait() + + # Close HMAC + hmac.stdin.close() + hmac.wait() + print "HMAC:",hmac.poll() + + # Write HMAC data next to the chunk file + hmac_data = hmac.stdout.read() + print "Writing hmac to",chunkfile+".hmac" + hmac_file = open(chunkfile+".hmac",'w') + hmac_file.write(hmac_data) + hmac_file.flush() + hmac_file.close() + + # Send the HMAC to the backup target + tar_final_cmd = ["tar", "-cO", "-C", backup_tmpdir, chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]+".hmac"] + final_proc = subprocess.Popen (tar_final_cmd, stdin=subprocess.PIPE, stdout=backup_stdout) + final_proc.wait() + + if tar_sparse.poll() == None: + # Release the next chunk + print "Release next chunk for process:",tar_sparse.poll() + #tar_sparse.stdout = subprocess.PIPE + tar_sparse.stdin.write("\n") + run_error="paused" + else: + print "Finished tar sparse with error",tar_sparse.poll() + # Wait for all remaining subprocess to finish - if addproc: - addproc.wait() - print "Addproc:",addproc.poll() + #if addproc: + # addproc.wait() + # print "Addproc:",addproc.poll() - streamproc.wait() - print "Streamproc:",streamproc.poll() + #streamproc.wait() + #print "Streamproc:",streamproc.poll() - hmac.stdin.close() - hmac.wait() - print "HMAC:",hmac.poll() - - # Write HMAC data next to the original file - hmac_data = hmac.stdout.read() - print "Writing hmac to",filename['path']+".hmac" - hmac_file = open(filename['path']+".hmac",'w') - hmac_file.write(hmac_data) - hmac_file.flush() - hmac_file.close() - - # Send the hmac file to the backup target - tar_cmdline[-1] += ".hmac" - print tar_cmdline - streamproc = subprocess.Popen(tar_cmdline,stdout=subprocess.PIPE) - run_error = wait_backup_feedback(progress_callback, streamproc, backup_stdout, total_backup_sz, vmproc=vmproc) - if len(run_error) > 0: - raise QubesException("Failed to perform backup: error with "+run_error) - - streamproc.wait() - print "HMAC sent:",streamproc.poll() + #streamproc.wait() # Close the backup target and wait for it to finish - backup_stdout.close() + #backup_stdout.close() if vmproc: print "VMProc1:",vmproc.poll() - vmproc.wait() - print "VMProc2:",vmproc.poll() - + print "Sparse1:",tar_sparse.poll() + vmproc.stdin.close() +''' +' Wait for backup chunk to finish +' - Monitor all the processes (streamproc, hmac, vmproc, addproc) for errors +' - Copy stdout of streamproc to backup_target and hmac stdin if available +' - Compute progress based on total_backup_sz and send progress to progress_callback function +' - Returns if +' - one of the monitored processes error out (streamproc, hmac, vmproc, addproc), along with the processe that failed +' - all of the monitored processes except vmproc finished successfully (vmproc termination is controlled by the python script) +' - streamproc does not delivers any data anymore (return with the error "paused") +''' def wait_backup_feedback(progress_callback, streamproc, backup_target, total_backup_sz, hmac=None, vmproc=None, addproc=None, remove_trailing_bytes=0): buffer_size = 4096 @@ -1136,12 +1185,11 @@ def wait_backup_feedback(progress_callback, streamproc, backup_target, total_bac while run_count > 0 and run_error == None: buffer = streamproc.stdout.read(buffer_size) - #print "Read",len(buffer) blocks_backedup += len(buffer) progress = blocks_backedup / float(total_backup_sz) - progress_callback(round(progress*100,2)) + #progress_callback(round(progress*100,2)) run_count = 0 if hmac: @@ -1154,6 +1202,7 @@ def wait_backup_feedback(progress_callback, streamproc, backup_target, total_bac if addproc: retcode=addproc.poll() + print "Tar proc status:",retcode if retcode != None: if retcode != 0: run_error = "addproc" @@ -1164,13 +1213,16 @@ def wait_backup_feedback(progress_callback, streamproc, backup_target, total_bac if retcode != None: if retcode != 0: run_error = "streamproc" + print "INFO: run error" elif retcode == 0 and len(buffer) <= 0: + print "INFO: no data" return "" else: - if remove_trailing_bytes > 0: - print buffer.encode("hex") - buffer = buffer[:-remove_trailing_bytes] - print buffer.encode("hex") + print "INFO: last packet" + #if remove_trailing_bytes > 0: + # print buffer.encode("hex") + # buffer = buffer[:-remove_trailing_bytes] + # print buffer.encode("hex") backup_target.write(buffer) @@ -1179,6 +1231,7 @@ def wait_backup_feedback(progress_callback, streamproc, backup_target, total_bac run_count += 1 else: + print "Process running:",len(buffer) # Process still running backup_target.write(buffer) From 23065f6fa04206009e15760ababf5461c5540f00 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Tue, 10 Sep 2013 09:25:44 +0200 Subject: [PATCH 22/34] backup: use a thread to send data to AppVM in parallel to tar main operations. Additionnally, temporary files are removed once data has been sent --- dom0/qvm-core/qubesutils.py | 67 +++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index dcf510d6..afcec82d 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1068,6 +1068,40 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e print "Will backup:",files_to_backup + # Setup worker to send encrypted data chunks to the backup_target + from multiprocessing import Queue,Process + class Send_Worker(Process): + def __init__(self,queue,base_dir,backup_stdout): + super(Send_Worker, self).__init__() + self.queue = queue + self.base_dir = base_dir + self.backup_stdout = backup_stdout + + def run(self): + print "Started sending thread" + + print "Moving to temporary dir",self.base_dir + os.chdir(self.base_dir) + + for filename in iter(self.queue.get,None): + if filename == "FINISHED": + break + + print "Sending file",filename + tar_final_cmd = ["tar", "-cO", "-C", self.base_dir, filename] + final_proc = subprocess.Popen (tar_final_cmd, stdin=subprocess.PIPE, stdout=self.backup_stdout) + final_proc.wait() + + # Delete the file as we don't need it anymore + print "Removing file",filename + os.remove(filename) + + print "Finished sending thread" + + to_send = Queue() + send_proc = Send_Worker(to_send, backup_tmpdir, backup_stdout) + send_proc.start() + for filename in files_to_backup: print "Backing up",filename @@ -1117,9 +1151,10 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e raise QubesException("Failed to perform backup: error with "+run_error) # Send the chunk to the backup target - tar_final_cmd = ["tar", "-cO", "-C", backup_tmpdir, chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]] - final_proc = subprocess.Popen (tar_final_cmd, stdin=subprocess.PIPE, stdout=backup_stdout) - final_proc.wait() + to_send.put(chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]) + #tar_final_cmd = ["tar", "-cO", "-C", backup_tmpdir, chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]] + #final_proc = subprocess.Popen (tar_final_cmd, stdin=subprocess.PIPE, stdout=backup_stdout) + #final_proc.wait() # Close HMAC hmac.stdin.close() @@ -1135,9 +1170,10 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e hmac_file.close() # Send the HMAC to the backup target - tar_final_cmd = ["tar", "-cO", "-C", backup_tmpdir, chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]+".hmac"] - final_proc = subprocess.Popen (tar_final_cmd, stdin=subprocess.PIPE, stdout=backup_stdout) - final_proc.wait() + to_send.put(chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]+".hmac") + #tar_final_cmd = ["tar", "-cO", "-C", backup_tmpdir, chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]+".hmac"] + #final_proc = subprocess.Popen (tar_final_cmd, stdin=subprocess.PIPE, stdout=backup_stdout) + #final_proc.wait() if tar_sparse.poll() == None: # Release the next chunk @@ -1148,18 +1184,13 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e else: print "Finished tar sparse with error",tar_sparse.poll() - # Wait for all remaining subprocess to finish - #if addproc: - # addproc.wait() - # print "Addproc:",addproc.poll() - - #streamproc.wait() - #print "Streamproc:",streamproc.poll() - - #streamproc.wait() # Close the backup target and wait for it to finish #backup_stdout.close() + + to_send.put("FINISHED") + send_proc.join() + if vmproc: print "VMProc1:",vmproc.poll() print "Sparse1:",tar_sparse.poll() @@ -1189,7 +1220,7 @@ def wait_backup_feedback(progress_callback, streamproc, backup_target, total_bac blocks_backedup += len(buffer) progress = blocks_backedup / float(total_backup_sz) - #progress_callback(round(progress*100,2)) + progress_callback(round(progress*100,2)) run_count = 0 if hmac: @@ -1202,7 +1233,7 @@ def wait_backup_feedback(progress_callback, streamproc, backup_target, total_bac if addproc: retcode=addproc.poll() - print "Tar proc status:",retcode + #print "Tar proc status:",retcode if retcode != None: if retcode != 0: run_error = "addproc" @@ -1231,7 +1262,7 @@ def wait_backup_feedback(progress_callback, streamproc, backup_target, total_bac run_count += 1 else: - print "Process running:",len(buffer) + #print "Process running:",len(buffer) # Process still running backup_target.write(buffer) From 361741b8aaf6244e3a4d01f6b100d577aaa2a424 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Tue, 10 Sep 2013 09:27:51 +0200 Subject: [PATCH 23/34] backup: multiple fixes for the backup process, including non-encrypted backups - Ensure backup without encryption is working - Implemented progress feedback through a global variable - Ask user for a passphrase used for encryption or for verification --- dom0/qvm-core/qubesutils.py | 122 +++++++++++++++++++++--------------- 1 file changed, 71 insertions(+), 51 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index afcec82d..454b2719 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1049,6 +1049,8 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e # If not APPVM, STDOUT is a local file backup_stdout = open(backup_target,'wb') + passphrase = raw_input("Please enter the pass phrase that will be used to encrypt/verify the backup:\n") + passphrase = passphrase.replace("\r","").replace("\n","") blocks_backedup = 0 progress = blocks_backedup * 11 / total_backup_sz @@ -1088,7 +1090,8 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e break print "Sending file",filename - tar_final_cmd = ["tar", "-cO", "-C", self.base_dir, filename] + # This tar used for sending data out need to be as simple, as simple, as featureless as possible. It will not be verified before untaring. + tar_final_cmd = ["tar", "-cO", "--posix", "-C", self.base_dir, filename] final_proc = subprocess.Popen (tar_final_cmd, stdin=subprocess.PIPE, stdout=self.backup_stdout) final_proc.wait() @@ -1098,6 +1101,14 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e print "Finished sending thread" + global blocks_backedup + blocks_backedup = 0 + def compute_progress(new_size, total_backup_sz): + global blocks_backedup + blocks_backedup += new_size + progress = blocks_backedup / float(total_backup_sz) + progress_callback(round(progress*100,2)) + to_send = Queue() send_proc = Send_Worker(to_send, backup_tmpdir, backup_stdout) send_proc.start() @@ -1113,6 +1124,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e if not os.path.isdir(os.path.dirname(backup_tempfile)): os.makedirs(os.path.dirname(backup_tempfile)) + # The first tar cmd can use any complex feature as we want. Files will be verified before untaring this. tar_cmdline = ["tar", "-Pc", "-f", backup_pipe,'--sparse','--tape-length',str(1000000),'-C',qubes_base_dir, filename["path"].split(os.path.normpath(qubes_base_dir)+"/")[1] ] @@ -1130,19 +1142,25 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e running = [] while run_error == "paused": - # Start encrypt pipe = open(backup_pipe,'rb') - # If no cipher is provided, the data is forwarded unencrypted !!! - encryptor = subprocess.Popen (["openssl", "enc", "-e", "-aes-256-cbc", "-pass", "pass:azerty"], stdin=pipe, stdout=subprocess.PIPE) # Start HMAC - hmac = subprocess.Popen (["openssl", "dgst", "-hmac", "azerty"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) + hmac = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], stdin=subprocess.PIPE, stdout=subprocess.PIPE) # Prepare a first chunk chunkfile = backup_tempfile + "." + "%03d" % i i += 1 chunkfile_p = open(chunkfile,'wb') - run_error = wait_backup_feedback(progress_callback, encryptor, chunkfile_p, total_backup_sz, hmac=hmac, vmproc=vmproc, addproc=tar_sparse) + + if encrypt: + # Start encrypt + # If no cipher is provided, the data is forwarded unencrypted !!! + # Also note that the + encryptor = subprocess.Popen (["openssl", "enc", "-e", "-aes-256-cbc", "-pass", "pass:"+passphrase], stdin=pipe, stdout=subprocess.PIPE) + run_error = wait_backup_feedback(compute_progress, encryptor.stdout, encryptor, chunkfile_p, total_backup_sz, hmac=hmac, vmproc=vmproc, addproc=tar_sparse) + else: + run_error = wait_backup_feedback(compute_progress, pipe, None, chunkfile_p, total_backup_sz, hmac=hmac, vmproc=vmproc, addproc=tar_sparse) + chunkfile_p.close() print "Wait_backup_feedback returned:",run_error @@ -1152,9 +1170,6 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e # Send the chunk to the backup target to_send.put(chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]) - #tar_final_cmd = ["tar", "-cO", "-C", backup_tmpdir, chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]] - #final_proc = subprocess.Popen (tar_final_cmd, stdin=subprocess.PIPE, stdout=backup_stdout) - #final_proc.wait() # Close HMAC hmac.stdin.close() @@ -1171,10 +1186,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e # Send the HMAC to the backup target to_send.put(chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]+".hmac") - #tar_final_cmd = ["tar", "-cO", "-C", backup_tmpdir, chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]+".hmac"] - #final_proc = subprocess.Popen (tar_final_cmd, stdin=subprocess.PIPE, stdout=backup_stdout) - #final_proc.wait() - + if tar_sparse.poll() == None: # Release the next chunk print "Release next chunk for process:",tar_sparse.poll() @@ -1184,7 +1196,8 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e else: print "Finished tar sparse with error",tar_sparse.poll() - + pipe.close() + # Close the backup target and wait for it to finish #backup_stdout.close() @@ -1206,7 +1219,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e ' - all of the monitored processes except vmproc finished successfully (vmproc termination is controlled by the python script) ' - streamproc does not delivers any data anymore (return with the error "paused") ''' -def wait_backup_feedback(progress_callback, streamproc, backup_target, total_backup_sz, hmac=None, vmproc=None, addproc=None, remove_trailing_bytes=0): +def wait_backup_feedback(progress_callback, in_stream, streamproc, backup_target, total_backup_sz, hmac=None, vmproc=None, addproc=None, remove_trailing_bytes=0): buffer_size = 4096 @@ -1215,12 +1228,9 @@ def wait_backup_feedback(progress_callback, streamproc, backup_target, total_bac blocks_backedup = 0 while run_count > 0 and run_error == None: - buffer = streamproc.stdout.read(buffer_size) + buffer = in_stream.read(buffer_size) - blocks_backedup += len(buffer) - - progress = blocks_backedup / float(total_backup_sz) - progress_callback(round(progress*100,2)) + progress_callback(len(buffer),total_backup_sz) run_count = 0 if hmac: @@ -1240,37 +1250,6 @@ def wait_backup_feedback(progress_callback, streamproc, backup_target, total_bac else: run_count += 1 - retcode=streamproc.poll() - if retcode != None: - if retcode != 0: - run_error = "streamproc" - print "INFO: run error" - elif retcode == 0 and len(buffer) <= 0: - print "INFO: no data" - return "" - else: - print "INFO: last packet" - #if remove_trailing_bytes > 0: - # print buffer.encode("hex") - # buffer = buffer[:-remove_trailing_bytes] - # print buffer.encode("hex") - - backup_target.write(buffer) - - if hmac: - hmac.stdin.write(buffer) - - run_count += 1 - else: - #print "Process running:",len(buffer) - # Process still running - backup_target.write(buffer) - - if hmac: - hmac.stdin.write(buffer) - - run_count += 1 - if vmproc: retcode = vmproc.poll() if retcode != None: @@ -1281,6 +1260,47 @@ def wait_backup_feedback(progress_callback, streamproc, backup_target, total_bac # VM should run until the end pass + if streamproc: + retcode=streamproc.poll() + if retcode != None: + if retcode != 0: + run_error = "streamproc" + elif retcode == 0 and len(buffer) <= 0: + return "" + else: + #print "INFO: last packet" + #if remove_trailing_bytes > 0: + # print buffer.encode("hex") + # buffer = buffer[:-remove_trailing_bytes] + # print buffer.encode("hex") + + backup_target.write(buffer) + + if hmac: + hmac.stdin.write(buffer) + + run_count += 1 + else: + #print "Process running:",len(buffer) + # Process still running + backup_target.write(buffer) + + if hmac: + hmac.stdin.write(buffer) + + run_count += 1 + + else: + if len(buffer) <= 0: + return "" + else: + backup_target.write(buffer) + + if hmac: + hmac.stdin.write(buffer) + + + return run_error def restore_vm_dir (backup_dir, src_dir, dst_dir, vm_spec, print_callback=None, error_callback=None, encrypted=False, appvm=None): From c805ff6aeb5afe517e5694ea06418e2e44bb148e Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Wed, 25 Sep 2013 09:51:17 +0200 Subject: [PATCH 24/34] backup: implement header restoration for the new backup format --- dom0/qvm-core/qubes.py | 6 +- dom0/qvm-core/qubesutils.py | 141 ++++++++++++++++-------------- dom0/qvm-tools/qvm-backup-restore | 9 +- 3 files changed, 87 insertions(+), 69 deletions(-) diff --git a/dom0/qvm-core/qubes.py b/dom0/qvm-core/qubes.py index 6edaebfd..8a404529 100755 --- a/dom0/qvm-core/qubes.py +++ b/dom0/qvm-core/qubes.py @@ -285,6 +285,9 @@ class QubesVm(object): # for backward compatibility (or another rare case): kernel=None -> kernel in VM dir 'self.dir_path + "/" + default_kernels_subdir' }, "_start_guid_first": { 'eval': 'False' }, + "backup_content" : { 'default': False }, + "backup_size" : { 'default': 0, "eval": "int(value)" }, + "backup_path" : { 'default': "" }, } ### Mark attrs for XML inclusion @@ -293,7 +296,8 @@ class QubesVm(object): 'uses_default_kernel', 'kernel', 'uses_default_kernelopts',\ 'kernelopts', 'services', 'installed_by_rpm',\ 'uses_default_netvm', 'include_in_backups', 'debug',\ - 'default_user', 'qrexec_timeout' ]: + 'default_user', 'qrexec_timeout', + 'backup_content', 'backup_size', 'backup_path' ]: attrs[prop]['save'] = 'str(self.%s)' % prop # Simple paths for prop in ['conf_file', 'root_img', 'volatile_img', 'private_img']: diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 454b2719..ec02e369 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -789,12 +789,11 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca if exclude_list is None: exclude_list = [] + qvm_collection = None if vms_list is None: qvm_collection = QubesVmCollection() - qvm_collection.lock_db_for_reading() + qvm_collection.lock_db_for_writing() qvm_collection.load() - # FIXME: should be after backup completed - qvm_collection.unlock_db() all_vms = [vm for vm in qvm_collection.values()] appvms_to_backup = [vm for vm in all_vms if vm.is_appvm() and not vm.internal] @@ -908,14 +907,13 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca print_callback(s) - # Build Backup VMs reference file - # Format: vm_name:tarball_path\n - backup_reference_file = open(os.path.join(qubes_base_dir,"backup_targets"),'w') - for vm in vms_for_backup: - backup_reference_file.write(vm.name+":"+vm.dir_path.split(qubes_base_dir)[1]+":"+str(vm.get_disk_utilization())+"\n") - backup_reference_file.flush() - backup_reference_file.close() - #files_to_backup = file_to_backup(backup_reference_file.name,os.stat(backup_reference_file.name).st_size) + files_to_backup + vm.backup_content = True + vm.backup_size = vm.get_disk_utilization() + vm.backup_path = vm.dir_path.split(qubes_base_dir)[1] + + qvm_collection.save() + # FIXME: should be after backup completed + qvm_collection.unlock_db() # Dom0 user home if not 'dom0' in exclude_list: @@ -1306,6 +1304,8 @@ def wait_backup_feedback(progress_callback, in_stream, streamproc, backup_target def restore_vm_dir (backup_dir, src_dir, dst_dir, vm_spec, print_callback=None, error_callback=None, encrypted=False, appvm=None): #backup_src_dir = src_dir.replace (qubes_base_dir, backup_dir) + print "Restore vm dir:",backup_dir, src_dir, dst_dir, vm_spec + vmproc = None if appvm != None: @@ -1378,11 +1378,29 @@ def backup_restore_set_defaults(options): return options -def backup_restore_header(restore_target, encrypt=False, appvm=None): +def load_hmac(hmac): + hmac = hmac.strip(" \t\r\n").split("=") + if len(hmac) > 1: + hmac = hmac[1].strip() + else: + raise QubesException("ERROR: invalid hmac file content") + + return hmac + +def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None): # Simulate dd if=backup_file count=10 | file - # Simulate dd if=backup_file count=10 | gpg2 -d | tar xzv -O # analysis = subprocess.Popen() vmproc = None + + import tempfile + feedback_file = tempfile.NamedTemporaryFile() + backup_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/restore_") + + # Tar with tapelength does not deals well with stdout (close stdout between two tapes) + # For this reason, we will use named pipes instead + print "Working in",backup_tmpdir + if appvm != None: # Prepare the backup target (Qubes service call) restore_command = "QUBESRPC qubes.Restore none" @@ -1391,7 +1409,7 @@ def backup_restore_header(restore_target, encrypt=False, appvm=None): qvm_collection = QubesVmCollection() qvm_collection.lock_db_for_reading() qvm_collection.load() - + vm = qvm_collection.get_vm_by_name(appvm) if vm is None or vm.qid not in qvm_collection: raise QubesException("VM {0} does not exist".format(appvm)) @@ -1402,7 +1420,7 @@ def backup_restore_header(restore_target, encrypt=False, appvm=None): vmproc = vm.run(command = restore_command, passio_popen = True) vmproc.stdin.write(restore_target.replace("\r","").replace("\n","")+"\n") - headers = vmproc.stdout.read(4096*4) + headers = vmproc.stdout.read(4096*64) vmproc.terminate() if len(headers) <= 0: @@ -1414,46 +1432,58 @@ def backup_restore_header(restore_target, encrypt=False, appvm=None): raise QubesException("ERROR: the backup directory {0} does not exists".format(restore_target)) fp = open(restore_target,'rb') - headers = fp.read(4096*4) + headers = fp.read(4096*16) - if encrypt: - command = subprocess.Popen(['gpg2','--decrypt'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) - stdout,stderr = command.communicate(headers) - if len(stdout) > 0: - headers = stdout - else: - print stderr - raise QubesException("ERROR: unable to decrypt the backup {0}. Is it really encrypted?".format(restore_target)) - - command = subprocess.Popen(['tar', 'xzv', '-O', 'backup_targets','qubes.xml'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + # FIXME: Use a safer program such as cpio, modified uncompress.c, or try to extract it from the APPVM directly + command = subprocess.Popen(['tar', '-i', '-xv', '-C', backup_tmpdir, 'qubes.xml.*'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) headers,stderr = command.communicate(headers) if len(headers) <= 0: print stderr raise QubesException("ERROR: unable to read the qubes backup file {0}. Is it really a backup?".format(restore_target)) - vms_in_backup = {} - for vm in headers.split("\n"): - match = re.match("^(?P[^:]+):(?P[^:]+):(?P[0-9]+)$",vm,re.MULTILINE) - if match: - item = match.groupdict() - item["size"] = int(item["size"]) - vms_in_backup[item["name"]] = item - headers = headers.replace(vm,"") + print "Retrieved headers",headers + + for filename in headers.splitlines(): + print "Loading hmac for file",filename + hmac = load_hmac(open(os.path.join(backup_tmpdir,filename+".hmac"),'r').read()) + + print "Verifying file",filename + hmac_proc = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], stdin=open(os.path.join(backup_tmpdir,filename),'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout,stderr = hmac_proc.communicate() + if len(stderr) > 0: + raise QubesException("ERROR: verify file {0}: {1}".format((filename,stderr))) else: - break + if len(hmac) > 0 and load_hmac(stdout) == hmac: + print "File verification OK -> Extracting archive",filename + # FIXME: implement function for encrypted file + command = subprocess.Popen(['tar', '-xvf', os.path.join(backup_tmpdir,filename), '-C', backup_tmpdir],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + headers,stderr = command.communicate(headers) + if len(headers) <= 0: + print stderr + raise QubesException("ERROR: unable to read the qubes backup file {0}. Is it really a backup?".format(restore_target)) + else: + return os.path.join(backup_tmpdir,headers.strip(" \r\n\t")) - return vms_in_backup, headers + else: + raise QubesException("ERROR: invalid hmac for file {0}: {1}. Is the passphrase correct?".format(filename,load_hmac(stdout))) -def backup_restore_prepare(backup_dir, backup_content, qubes_xml, options = {}, host_collection = None, encrypt=False, appvm=None): + return None + +def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, host_collection = None, encrypt=False, appvm=None): # Defaults backup_restore_set_defaults(options) #### Private functions begin def is_vm_included_in_backup (backup_dir, vm): - for item in backup_content.keys(): - if vm.name == item: - return True - return False + if vm.qid == 0: + # Dom0 is not included, obviously + return False + + if vm.backup_content: + return True + else: + return False + def find_template_name(template, replaces): rx_replace = re.compile("(.*):(.*)") for r in replaces: @@ -1463,14 +1493,9 @@ def backup_restore_prepare(backup_dir, backup_content, qubes_xml, options = {}, return template #### Private functions end - - backup_collection = QubesVmCollection() - import StringIO - backup_collection.qubes_store_file=StringIO.StringIO(qubes_xml) - - - #backup_collection.lock_db_for_reading() - + print "Loading file",qubes_xml + backup_collection = QubesVmCollection(store_filename = qubes_xml) + backup_collection.lock_db_for_reading() backup_collection.load() if host_collection is None: @@ -1480,11 +1505,9 @@ def backup_restore_prepare(backup_dir, backup_content, qubes_xml, options = {}, host_collection.unlock_db() backup_vms_list = [vm for vm in backup_collection.values()] - host_vms_list = [vm for vm in host_collection.values()] vms_to_restore = {} - there_are_conflicting_vms = False there_are_missing_templates = False there_are_missing_netvms = False @@ -1493,6 +1516,7 @@ def backup_restore_prepare(backup_dir, backup_content, qubes_xml, options = {}, # ... and the actual data for vm in backup_vms_list: if is_vm_included_in_backup (backup_dir, vm): + print vm.name,"is included in backup" vms_to_restore[vm.name] = {} vms_to_restore[vm.name]['vm'] = vm; @@ -1676,20 +1700,7 @@ def backup_restore_print_summary(restore_info, print_callback = print_stdout): print_callback(s) -def backup_restore_do(backup_dir, restore_info, restore_vms, host_collection = None, print_callback = print_stdout, error_callback = print_stderr, encrypted=False, appvm = None ): - - ### Private functions begin - ''' - def restore_vm_dir (backup_dir, src_dir, dst_dir, print_callback, error_callback, encrypted, appvm): - - backup_src_dir = src_dir.replace (qubes_base_dir, backup_dir) - - # We prefer to use Linux's cp, because it nicely handles sparse files - retcode = subprocess.call (["cp", "-rp", backup_src_dir, dst_dir]) - if retcode != 0: - raise QubesException("*** Error while copying file {0} to {1}".format(backup_src_dir, dest_dir)) - ''' - ### Private functions end +def backup_restore_do(backup_dir, restore_info, host_collection = None, print_callback = print_stdout, error_callback = print_stderr, encrypted=False, appvm = None ): lock_obtained = False if host_collection is None: @@ -1731,7 +1742,7 @@ def backup_restore_do(backup_dir, restore_info, restore_vms, host_collection = N template=template, installed_by_rpm=False) - restore_vm_dir (backup_dir, vm.dir_path, os.path.dirname(new_vm.dir_path), restore_vms[vm.name], print_callback, error_callback, encrypted, appvm) + restore_vm_dir (backup_dir, vm.dir_path, os.path.dirname(new_vm.dir_path), vm, print_callback, error_callback, encrypted, appvm) new_vm.verify_files() except Exception as err: diff --git a/dom0/qvm-tools/qvm-backup-restore b/dom0/qvm-tools/qvm-backup-restore index 05c56374..3755b788 100755 --- a/dom0/qvm-tools/qvm-backup-restore +++ b/dom0/qvm-tools/qvm-backup-restore @@ -95,12 +95,15 @@ def main(): restore_options['exclude'] = options.exclude + passphrase = raw_input("Please enter the pass phrase that will be used to decrypt/verify the backup:\n") + passphrase = passphrase.replace("\r","").replace("\n","") + print >> sys.stderr, "Checking backup content..." - encrypted, vms, qubes_xml = backup_restore_header(backup_dir, options.decrypt, appvm=options.appvm) + qubes_xml = backup_restore_header(backup_dir, passphrase, options.decrypt, appvm=options.appvm) restore_info = None try: - restore_info = backup_restore_prepare(backup_dir, vms, qubes_xml, options=restore_options, host_collection=host_collection, encrypt=encrypted, appvm=options.appvm) + restore_info = backup_restore_prepare(backup_dir, qubes_xml, passphrase, options=restore_options, host_collection=host_collection, encrypt=options.decrypt, appvm=options.appvm) except QubesException as e: print >> sys.stderr, "ERROR: %s" % str(e) exit(1) @@ -189,7 +192,7 @@ def main(): exit (0) - backup_restore_do(backup_dir, restore_info, vms, host_collection=host_collection, encrypted=encrypted, appvm=options.appvm) + backup_restore_do(backup_dir, restore_info, host_collection=host_collection, encrypted=options.decrypt, appvm=options.appvm) host_collection.unlock_db() From af230b33a9e9ae28ce6aef6b5589c7b37a869624 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Thu, 26 Sep 2013 10:30:57 +0200 Subject: [PATCH 25/34] backup: implemented validated file extraction for non encrypted backups --- dom0/qvm-core/qubesutils.py | 178 ++++++++++++++++++++++-------- dom0/qvm-tools/qvm-backup-restore | 6 +- 2 files changed, 137 insertions(+), 47 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index ec02e369..85f32167 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -909,7 +909,7 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca vm.backup_content = True vm.backup_size = vm.get_disk_utilization() - vm.backup_path = vm.dir_path.split(qubes_base_dir)[1] + vm.backup_path = vm.dir_path.split(os.path.normpath(qubes_base_dir)+"/")[1] qvm_collection.save() # FIXME: should be after backup completed @@ -1164,6 +1164,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e print "Wait_backup_feedback returned:",run_error if len(run_error) > 0: + send_proc.terminate() raise QubesException("Failed to perform backup: error with "+run_error) # Send the chunk to the backup target @@ -1172,7 +1173,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e # Close HMAC hmac.stdin.close() hmac.wait() - print "HMAC:",hmac.poll() + print "HMAC proc return code:",hmac.poll() # Write HMAC data next to the chunk file hmac_data = hmac.stdout.read() @@ -1203,8 +1204,8 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e send_proc.join() if vmproc: - print "VMProc1:",vmproc.poll() - print "Sparse1:",tar_sparse.poll() + print "VMProc1 proc return code:",vmproc.poll() + print "Sparse1 proc return code:",tar_sparse.poll() vmproc.stdin.close() ''' @@ -1227,7 +1228,6 @@ def wait_backup_feedback(progress_callback, in_stream, streamproc, backup_target while run_count > 0 and run_error == None: buffer = in_stream.read(buffer_size) - progress_callback(len(buffer),total_backup_sz) run_count = 0 @@ -1301,11 +1301,84 @@ def wait_backup_feedback(progress_callback, in_stream, streamproc, backup_target return run_error -def restore_vm_dir (backup_dir, src_dir, dst_dir, vm_spec, print_callback=None, error_callback=None, encrypted=False, appvm=None): +def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_size, print_callback=None, error_callback=None, encrypted=False, appvm=None): + + # Setup worker to extract encrypted data chunks to the restore dirs + from multiprocessing import Queue,Process + class Extract_Worker(Process): + def __init__(self,queue,base_dir,passphrase,encrypted,total_size): + super(Extract_Worker, self).__init__() + self.queue = queue + self.base_dir = base_dir + self.passphrase = passphrase + self.encrypted = encrypted + self.total_size = total_size + self.blocks_backedup = 0 + self.tar2_command = None + + self.restore_pipe = os.path.join(self.base_dir,"restore_pipe") + print "Creating pipe in:",self.restore_pipe + print os.mkfifo(self.restore_pipe) + + def compute_progress(self, new_size, total_size): + self.blocks_backedup += new_size + progress = self.blocks_backedup / float(self.total_size) + print_callback(round(progress*100,2)) + + def run(self): + print "Started extracting thread" + + print "Moving to temporary dir",self.base_dir + os.chdir(self.base_dir) + + for filename in iter(self.queue.get,None): + if filename == "FINISHED": + break + + print "Extracting file",filename,"to",self.base_dir + + if self.tar2_command == None: + self.tar2_command = ['tar', '--tape-length','1000000', '-xvf', self.restore_pipe] + self.tar2_command = subprocess.Popen(self.tar2_command,stdin=subprocess.PIPE) + + pipe = open(self.restore_pipe,'r+b') + if self.encrypted: + # Start decrypt + encryptor = subprocess.Popen (["openssl", "enc", "-d", "-aes-256-cbc", "-pass", "pass:"+passphrase], stdin=open(filename,'rb'), stdout=pipe) + + # progress_callback, in_stream, streamproc, backup_target, total_backup_sz, hmac=None, vmproc=None, addproc=None, remove_trailing_bytes=0): + run_error = wait_backup_feedback(self.compute_progress, pipe, encryptor, self.tar2_command.stdin, self.total_size, hmac=None, vmproc=None, addproc=self.tar2_command) + else: + run_error = wait_backup_feedback(self.compute_progress, open(filename,"rb"), None, pipe, self.total_size, hmac=None, vmproc=None, addproc=self.tar2_command) + + print "Run error:",run_error + print self.tar2_command.poll(), + + # Close named pipe so that tar knowns the file has been entirely read + pipe.close() + + if self.tar2_command.poll() != None: + if self.tar2_command.poll() != 0: + raise QubesException("ERROR: unable to extract files for {0}.".format(filename)) + else: + # Finished extracting the tar file + self.tar2_command = None + else: + print "Releasing next chunck" + self.tar2_command.stdin.write("\n") + + # Delete the file as we don't need it anymore + print "Removing file",filename + os.remove(filename) + + print "Finished extracting thread" + + to_extract = Queue() + extract_proc = Extract_Worker(to_extract, backup_tmpdir, passphrase, encrypted, vms_size) + extract_proc.start() #backup_src_dir = src_dir.replace (qubes_base_dir, backup_dir) - print "Restore vm dir:",backup_dir, src_dir, dst_dir, vm_spec - + print "Restore vm dirs:",backup_dir, vms_dirs, vms, vms_size vmproc = None if appvm != None: @@ -1330,39 +1403,42 @@ def restore_vm_dir (backup_dir, src_dir, dst_dir, vm_spec, print_callback=None, else: backup_stdin = open(backup_dir,'rb') - tar_cmdline = ["tar", "-xzv",'--sparse','-C',qubes_base_dir,'--checkpoint=10000'] + # FIXME: Use a safer program such as cpio, modified uncompress.c, or try to extract it from the APPVM directly based on Joana recommendation. + tar1_command = ['tar', '-i', '-xv', '-C', backup_tmpdir] + tar1_command.extend(vms_dirs) + print "Running 'safe' command",tar1_command + command = subprocess.Popen(tar1_command, stdin=backup_stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - tar_cmdline.append(src_dir.split(os.path.normpath(qubes_base_dir)+"/")[1]) + while command.poll() == None and vmproc.poll() == None: + filename = command.stdout.readline().strip(" \t\r\n") + print "Getting new file:",filename + hmacfile = command.stdout.readline().strip(" \t\r\n") + print "Getting hmac:",hmacfile + + print "Verifying file",filename + hmac_proc = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], stdin=open(os.path.join(backup_tmpdir,filename),'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout,stderr = hmac_proc.communicate() + if len(stderr) > 0: + raise QubesException("ERROR: verify file {0}: {1}".format((filename,stderr))) + else: + print "Loading hmac for file",filename + hmac = load_hmac(open(os.path.join(backup_tmpdir,filename+".hmac"),'r').read()) - #print ("Will backup using command",tar_cmdline) + if len(hmac) > 0 and load_hmac(stdout) == hmac: + print "File verification OK. Sending",filename,"to extraction thread" + # Send the chunk to the backup target + to_extract.put(os.path.join(backup_tmpdir,filename)) - import tempfile - feedback_file = tempfile.NamedTemporaryFile() - if encrypted: - encryptor = subprocess.Popen (["gpg2", "--decrypt"], stdin=backup_stdin, stdout=subprocess.PIPE,stderr=subprocess.PIPE) - compressor = subprocess.Popen (tar_cmdline,stdin=encryptor.stdout,stderr=feedback_file,stdout=subprocess.PIPE) - else: - compressor = subprocess.Popen (tar_cmdline,stdin=backup_stdin,stderr=feedback_file,stdout=subprocess.PIPE) - encryptor = None + else: + raise QubesException("ERROR: invalid hmac for file {0}: {1}. Is the passphrase correct?".format(filename,load_hmac(stdout))) - run_error = wait_backup_feedback(print_callback, feedback_file, vm_spec["size"], compressor, encryptor, vmproc) - - # Cleanup - feedback_file.close() - backup_stdin.close() - - # Check returns code of compressor and encryptor and qubes vm retcode - if run_error != None: - try: - if compressor != None: - compressor.terminate() - if encryptor != None: - encryptor.terminate() - if vmproc != None: - vmproc.terminate() - except OSError: - pass - raise QubesException("Failed to perform backup: error with "+run_error) + if command.poll() != 0: + raise QubesException("ERROR: unable to read the qubes backup file {0}. Is it really a backup?".format(restore_target)) + if vmproc.poll() != 0: + raise QubesException("ERROR: unable to read the qubes backup {0} because of a VM error: {1}".format(restore_target,vmproc.stderr.read())) + + to_extract.put("FINISHED") + to_extract.wait() def backup_restore_set_defaults(options): if 'use-default-netvm' not in options: @@ -1434,7 +1510,6 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) fp = open(restore_target,'rb') headers = fp.read(4096*16) - # FIXME: Use a safer program such as cpio, modified uncompress.c, or try to extract it from the APPVM directly command = subprocess.Popen(['tar', '-i', '-xv', '-C', backup_tmpdir, 'qubes.xml.*'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) headers,stderr = command.communicate(headers) if len(headers) <= 0: @@ -1455,14 +1530,14 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) else: if len(hmac) > 0 and load_hmac(stdout) == hmac: print "File verification OK -> Extracting archive",filename - # FIXME: implement function for encrypted file + # FIXME: handle encrypted file command = subprocess.Popen(['tar', '-xvf', os.path.join(backup_tmpdir,filename), '-C', backup_tmpdir],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) headers,stderr = command.communicate(headers) if len(headers) <= 0: print stderr raise QubesException("ERROR: unable to read the qubes backup file {0}. Is it really a backup?".format(restore_target)) else: - return os.path.join(backup_tmpdir,headers.strip(" \r\n\t")) + return (backup_tmpdir,headers.strip(" \r\n\t")) else: raise QubesException("ERROR: invalid hmac for file {0}: {1}. Is the passphrase correct?".format(filename,load_hmac(stdout))) @@ -1581,7 +1656,7 @@ def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, host vms_to_restore[vm.name]['good-to-go'] = True # ...and dom0 home - # TODO, replace this part of code to handle the new backup format using tar + # FIXME, replace this part of code to handle the new backup format using tar if options['dom0-home'] and os.path.exists(backup_dir + '/dom0-home'): vms_to_restore['dom0'] = {} local_user = grp.getgrnam('qubes').gr_mem[0] @@ -1700,7 +1775,7 @@ def backup_restore_print_summary(restore_info, print_callback = print_stdout): print_callback(s) -def backup_restore_do(backup_dir, restore_info, host_collection = None, print_callback = print_stdout, error_callback = print_stderr, encrypted=False, appvm = None ): +def backup_restore_do(backup_dir, restore_tmpdir, passphrase, restore_info, host_collection = None, print_callback = print_stdout, error_callback = print_stderr, encrypted=False, appvm = None ): lock_obtained = False if host_collection is None: @@ -1709,6 +1784,23 @@ def backup_restore_do(backup_dir, restore_info, host_collection = None, print_ca host_collection.load() lock_obtained = True + # Perform VM restoration in backup order + vms_dirs = [] + vms_size = 0 + vms = {} + for vm_info in restore_info.values(): + if not vm_info['good-to-go']: + continue + if 'vm' not in vm_info: + continue + vm = vm_info['vm'] + vms_size += vm.backup_size + vms_dirs.append(vm.backup_path+"*") + vms[vm.name] = vm + + restore_vm_dirs (backup_dir, restore_tmpdir, passphrase, vms_dirs, vms, vms_size, print_callback, error_callback, encrypted, appvm) + + # Add VM in right order for (vm_class_name, vm_class) in sorted(QubesVmClasses.items(), key=lambda _x: _x[1].load_order): @@ -1742,8 +1834,6 @@ def backup_restore_do(backup_dir, restore_info, host_collection = None, print_ca template=template, installed_by_rpm=False) - restore_vm_dir (backup_dir, vm.dir_path, os.path.dirname(new_vm.dir_path), vm, print_callback, error_callback, encrypted, appvm) - new_vm.verify_files() except Exception as err: error_callback("ERROR: {0}".format(err)) diff --git a/dom0/qvm-tools/qvm-backup-restore b/dom0/qvm-tools/qvm-backup-restore index 3755b788..bf1c1253 100755 --- a/dom0/qvm-tools/qvm-backup-restore +++ b/dom0/qvm-tools/qvm-backup-restore @@ -99,11 +99,11 @@ def main(): passphrase = passphrase.replace("\r","").replace("\n","") print >> sys.stderr, "Checking backup content..." - qubes_xml = backup_restore_header(backup_dir, passphrase, options.decrypt, appvm=options.appvm) + restore_tmpdir,qubes_xml = backup_restore_header(backup_dir, passphrase, options.decrypt, appvm=options.appvm) restore_info = None try: - restore_info = backup_restore_prepare(backup_dir, qubes_xml, passphrase, options=restore_options, host_collection=host_collection, encrypt=options.decrypt, appvm=options.appvm) + restore_info = backup_restore_prepare(backup_dir,os.path.join(restore_tmpdir, qubes_xml), passphrase, options=restore_options, host_collection=host_collection, encrypt=options.decrypt, appvm=options.appvm) except QubesException as e: print >> sys.stderr, "ERROR: %s" % str(e) exit(1) @@ -192,7 +192,7 @@ def main(): exit (0) - backup_restore_do(backup_dir, restore_info, host_collection=host_collection, encrypted=options.decrypt, appvm=options.appvm) + backup_restore_do(backup_dir,restore_tmpdir, passphrase, restore_info, host_collection=host_collection, encrypted=options.decrypt, appvm=options.appvm) host_collection.unlock_db() From 26fb5b3d25a28e305b3609f561b53b250c67ca35 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Fri, 27 Sep 2013 09:41:40 +0200 Subject: [PATCH 26/34] backup: fixes to use the backup GUI --- dom0/qvm-core/qubesutils.py | 68 +++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 85f32167..c705f476 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1105,7 +1105,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e global blocks_backedup blocks_backedup += new_size progress = blocks_backedup / float(total_backup_sz) - progress_callback(round(progress*100,2)) + progress_callback(int(round(progress*100,2))) to_send = Queue() send_proc = Send_Worker(to_send, backup_tmpdir, backup_stdout) @@ -1298,15 +1298,14 @@ def wait_backup_feedback(progress_callback, in_stream, streamproc, backup_target hmac.stdin.write(buffer) - return run_error -def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_size, print_callback=None, error_callback=None, encrypted=False, appvm=None): +def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_size, print_callback=None, error_callback=None, progress_callback=None, encrypted=False, appvm=None): # Setup worker to extract encrypted data chunks to the restore dirs from multiprocessing import Queue,Process class Extract_Worker(Process): - def __init__(self,queue,base_dir,passphrase,encrypted,total_size): + def __init__(self,queue,base_dir,passphrase,encrypted,total_size,print_callback,error_callback,progress_callback): super(Extract_Worker, self).__init__() self.queue = queue self.base_dir = base_dir @@ -1316,6 +1315,10 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s self.blocks_backedup = 0 self.tar2_command = None + self.print_callback = print_callback + self.error_callback = error_callback + self.progress_callback = progress_callback + self.restore_pipe = os.path.join(self.base_dir,"restore_pipe") print "Creating pipe in:",self.restore_pipe print os.mkfifo(self.restore_pipe) @@ -1323,22 +1326,24 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s def compute_progress(self, new_size, total_size): self.blocks_backedup += new_size progress = self.blocks_backedup / float(self.total_size) - print_callback(round(progress*100,2)) + progress = int(round(progress*100,2)) + self.progress_callback(progress) def run(self): - print "Started extracting thread" + self.print_callback("Started sending thread") - print "Moving to temporary dir",self.base_dir + self.print_callback("Moving to temporary dir "+self.base_dir) os.chdir(self.base_dir) for filename in iter(self.queue.get,None): if filename == "FINISHED": break - print "Extracting file",filename,"to",self.base_dir + self.print_callback("Extracting file "+filename+" to "+self.base_dir) if self.tar2_command == None: self.tar2_command = ['tar', '--tape-length','1000000', '-xvf', self.restore_pipe] + self.print_callback("Running command "+str(self.tar2_command)) self.tar2_command = subprocess.Popen(self.tar2_command,stdin=subprocess.PIPE) pipe = open(self.restore_pipe,'r+b') @@ -1351,11 +1356,8 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s else: run_error = wait_backup_feedback(self.compute_progress, open(filename,"rb"), None, pipe, self.total_size, hmac=None, vmproc=None, addproc=self.tar2_command) - print "Run error:",run_error - print self.tar2_command.poll(), - - # Close named pipe so that tar knowns the file has been entirely read - pipe.close() + self.print_callback("Run error:"+run_error) + self.print_callback(str(self.tar2_command.poll())) if self.tar2_command.poll() != None: if self.tar2_command.poll() != 0: @@ -1363,22 +1365,24 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s else: # Finished extracting the tar file self.tar2_command = None + else: - print "Releasing next chunck" + self.print_callback("Releasing next chunck") + pipe.close() self.tar2_command.stdin.write("\n") # Delete the file as we don't need it anymore - print "Removing file",filename + self.print_callback("Removing file "+filename) os.remove(filename) - print "Finished extracting thread" + self.print_callback("Finished extracting thread") to_extract = Queue() - extract_proc = Extract_Worker(to_extract, backup_tmpdir, passphrase, encrypted, vms_size) + extract_proc = Extract_Worker(to_extract, backup_tmpdir, passphrase, encrypted, vms_size, print_callback, error_callback, progress_callback) extract_proc.start() - #backup_src_dir = src_dir.replace (qubes_base_dir, backup_dir) - print "Restore vm dirs:",backup_dir, vms_dirs, vms, vms_size + print_callback("Working in temporary dir:"+backup_tmpdir) + print_callback(str(vms_size)+" bytes to restore") vmproc = None if appvm != None: @@ -1403,29 +1407,35 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s else: backup_stdin = open(backup_dir,'rb') - # FIXME: Use a safer program such as cpio, modified uncompress.c, or try to extract it from the APPVM directly based on Joana recommendation. + # FIXME: Use a safer program such as cpio, modified uncompress.c, or try to extract it from tar1_command = ['tar', '-i', '-xv', '-C', backup_tmpdir] tar1_command.extend(vms_dirs) - print "Running 'safe' command",tar1_command + print_callback("Run command"+str(tar1_command)) command = subprocess.Popen(tar1_command, stdin=backup_stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE) while command.poll() == None and vmproc.poll() == None: + filename = command.stdout.readline().strip(" \t\r\n") - print "Getting new file:",filename + + print_callback("Getting new file:"+filename) + hmacfile = command.stdout.readline().strip(" \t\r\n") - print "Getting hmac:",hmacfile + + print_callback("Getting hmac:"+hmacfile) - print "Verifying file",filename + print_callback("Verifying file"+filename) + hmac_proc = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], stdin=open(os.path.join(backup_tmpdir,filename),'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout,stderr = hmac_proc.communicate() + if len(stderr) > 0: raise QubesException("ERROR: verify file {0}: {1}".format((filename,stderr))) else: - print "Loading hmac for file",filename + print_callback("Loading hmac for file"+filename) hmac = load_hmac(open(os.path.join(backup_tmpdir,filename+".hmac"),'r').read()) if len(hmac) > 0 and load_hmac(stdout) == hmac: - print "File verification OK. Sending",filename,"to extraction thread" + print_callback("File verification OK -> Sending file "+filename+" for extraction") # Send the chunk to the backup target to_extract.put(os.path.join(backup_tmpdir,filename)) @@ -1438,7 +1448,7 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s raise QubesException("ERROR: unable to read the qubes backup {0} because of a VM error: {1}".format(restore_target,vmproc.stderr.read())) to_extract.put("FINISHED") - to_extract.wait() + extract_proc.join() def backup_restore_set_defaults(options): if 'use-default-netvm' not in options: @@ -1775,7 +1785,7 @@ def backup_restore_print_summary(restore_info, print_callback = print_stdout): print_callback(s) -def backup_restore_do(backup_dir, restore_tmpdir, passphrase, restore_info, host_collection = None, print_callback = print_stdout, error_callback = print_stderr, encrypted=False, appvm = None ): +def backup_restore_do(backup_dir, restore_tmpdir, passphrase, restore_info, host_collection = None, print_callback = print_stdout, error_callback = print_stderr, progress_callback = None, encrypted=False, appvm=None): lock_obtained = False if host_collection is None: @@ -1798,7 +1808,7 @@ def backup_restore_do(backup_dir, restore_tmpdir, passphrase, restore_info, host vms_dirs.append(vm.backup_path+"*") vms[vm.name] = vm - restore_vm_dirs (backup_dir, restore_tmpdir, passphrase, vms_dirs, vms, vms_size, print_callback, error_callback, encrypted, appvm) + restore_vm_dirs (backup_dir, restore_tmpdir, passphrase, vms_dirs, vms, vms_size, print_callback, error_callback, progress_callback, encrypted, appvm) # Add VM in right order From a594af02dea10865eb048b9db1080e9af3455a0d Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Fri, 27 Sep 2013 09:47:33 +0200 Subject: [PATCH 27/34] backup: fix of a bug introduced in the qvm-backup-restore command by adding a progress_callback --- dom0/qvm-core/qubesutils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index c705f476..b1d9f437 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1377,6 +1377,10 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s self.print_callback("Finished extracting thread") + if progress_callback == None: + def progress_callback(data): + pass + to_extract = Queue() extract_proc = Extract_Worker(to_extract, backup_tmpdir, passphrase, encrypted, vms_size, print_callback, error_callback, progress_callback) extract_proc.start() From c2645bcb924f99e143600f9c706e1f7dcfa7902c Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Sat, 28 Sep 2013 12:24:33 +0200 Subject: [PATCH 28/34] backup: fix a misplaced passphrase request breaking the backup GUI --- dom0/qvm-core/qubesutils.py | 5 +---- dom0/qvm-tools/qvm-backup | 5 ++++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index b1d9f437..87c01f61 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1010,7 +1010,7 @@ def backup_do(base_backup_dir, files_to_backup, progress_callback = None): progress = bytes_backedup * 100 / total_backup_sz progress_callback(progress) -def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, encrypt=False, appvm=None): +def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callback = None, encrypt=False, appvm=None): total_backup_sz = 0 for file in files_to_backup: total_backup_sz += file["size"] @@ -1047,9 +1047,6 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e # If not APPVM, STDOUT is a local file backup_stdout = open(backup_target,'wb') - passphrase = raw_input("Please enter the pass phrase that will be used to encrypt/verify the backup:\n") - passphrase = passphrase.replace("\r","").replace("\n","") - blocks_backedup = 0 progress = blocks_backedup * 11 / total_backup_sz progress_callback(progress) diff --git a/dom0/qvm-tools/qvm-backup b/dom0/qvm-tools/qvm-backup index ea3a43dd..e7bc0f0a 100755 --- a/dom0/qvm-tools/qvm-backup +++ b/dom0/qvm-tools/qvm-backup @@ -75,8 +75,11 @@ def main(): if not (prompt == "y" or prompt == "Y"): exit (0) + passphrase = raw_input("Please enter the pass phrase that will be used to encrypt/verify the backup:\n") + passphrase = passphrase.replace("\r","").replace("\n","") + try: - backup_do_copy(base_backup_dir, files_to_backup, progress_callback=print_progress, encrypt=options.encrypt,appvm=options.appvm) + backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callback=print_progress, encrypt=options.encrypt,appvm=options.appvm) except QubesException as e: print >>sys.stderr, "ERROR: %s" % str(e) exit(1) From 1a80893ef0454b725dfaf17d6b607d1a3a6fcc75 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Mon, 30 Sep 2013 10:26:04 +0200 Subject: [PATCH 29/34] backup: fixed decryption during backup restoration process --- dom0/qvm-core/qubesutils.py | 99 ++++++++++++++++++++++--------------- 1 file changed, 58 insertions(+), 41 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 87c01f61..120ab4cf 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1302,7 +1302,7 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s # Setup worker to extract encrypted data chunks to the restore dirs from multiprocessing import Queue,Process class Extract_Worker(Process): - def __init__(self,queue,base_dir,passphrase,encrypted,total_size,print_callback,error_callback,progress_callback): + def __init__(self,queue,base_dir,passphrase,encrypted,total_size,print_callback,error_callback,progress_callback,vmproc=None): super(Extract_Worker, self).__init__() self.queue = queue self.base_dir = base_dir @@ -1316,6 +1316,8 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s self.error_callback = error_callback self.progress_callback = progress_callback + self.vmproc = vmproc + self.restore_pipe = os.path.join(self.base_dir,"restore_pipe") print "Creating pipe in:",self.restore_pipe print os.mkfifo(self.restore_pipe) @@ -1344,14 +1346,17 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s self.tar2_command = subprocess.Popen(self.tar2_command,stdin=subprocess.PIPE) pipe = open(self.restore_pipe,'r+b') - if self.encrypted: - # Start decrypt - encryptor = subprocess.Popen (["openssl", "enc", "-d", "-aes-256-cbc", "-pass", "pass:"+passphrase], stdin=open(filename,'rb'), stdout=pipe) + if self.encrypted: + # Start decrypt + encryptor = subprocess.Popen (["openssl", "enc", "-d", "-aes-256-cbc", "-pass", "pass:"+passphrase], stdin=open(filename,'rb'), stdout=subprocess.PIPE) # progress_callback, in_stream, streamproc, backup_target, total_backup_sz, hmac=None, vmproc=None, addproc=None, remove_trailing_bytes=0): - run_error = wait_backup_feedback(self.compute_progress, pipe, encryptor, self.tar2_command.stdin, self.total_size, hmac=None, vmproc=None, addproc=self.tar2_command) - else: - run_error = wait_backup_feedback(self.compute_progress, open(filename,"rb"), None, pipe, self.total_size, hmac=None, vmproc=None, addproc=self.tar2_command) + run_error = wait_backup_feedback(self.compute_progress, encryptor.stdout, encryptor, pipe, self.total_size, hmac=None, vmproc=self.vmproc, addproc=self.tar2_command) + #print "End wait_backup_feedback",run_error,self.tar2_command.poll(),encryptor.poll() + else: + run_error = wait_backup_feedback(self.compute_progress, open(filename,"rb"), None, pipe, self.total_size, hmac=None, vmproc=self.vmproc, addproc=self.tar2_command) + + pipe.close() self.print_callback("Run error:"+run_error) self.print_callback(str(self.tar2_command.poll())) @@ -1365,7 +1370,6 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s else: self.print_callback("Releasing next chunck") - pipe.close() self.tar2_command.stdin.write("\n") # Delete the file as we don't need it anymore @@ -1484,6 +1488,8 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) feedback_file = tempfile.NamedTemporaryFile() backup_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/restore_") + os.chdir(backup_tmpdir) + # Tar with tapelength does not deals well with stdout (close stdout between two tapes) # For this reason, we will use named pipes instead print "Working in",backup_tmpdir @@ -1507,12 +1513,6 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) vmproc = vm.run(command = restore_command, passio_popen = True) vmproc.stdin.write(restore_target.replace("\r","").replace("\n","")+"\n") - headers = vmproc.stdout.read(4096*64) - vmproc.terminate() - - if len(headers) <= 0: - raise QubesException("ERROR: unable to read the backup target {0}".format(restore_target)) - else: # Create the target directory if not os.path.exists (restore_target): @@ -1521,37 +1521,53 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) fp = open(restore_target,'rb') headers = fp.read(4096*16) - command = subprocess.Popen(['tar', '-i', '-xv', '-C', backup_tmpdir, 'qubes.xml.*'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) - headers,stderr = command.communicate(headers) - if len(headers) <= 0: - print stderr - raise QubesException("ERROR: unable to read the qubes backup file {0}. Is it really a backup?".format(restore_target)) - print "Retrieved headers",headers + command = subprocess.Popen(['tar', '-i', '-xv', '-C', backup_tmpdir, 'qubes.xml*'],stdin=vmproc.stdout,stdout=subprocess.PIPE,stderr=subprocess.PIPE) - for filename in headers.splitlines(): - print "Loading hmac for file",filename - hmac = load_hmac(open(os.path.join(backup_tmpdir,filename+".hmac"),'r').read()) + filename = command.stdout.readline().strip(" \t\r\n") + print "Getting file",filename - print "Verifying file",filename - hmac_proc = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], stdin=open(os.path.join(backup_tmpdir,filename),'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout,stderr = hmac_proc.communicate() - if len(stderr) > 0: - raise QubesException("ERROR: verify file {0}: {1}".format((filename,stderr))) - else: - if len(hmac) > 0 and load_hmac(stdout) == hmac: - print "File verification OK -> Extracting archive",filename - # FIXME: handle encrypted file - command = subprocess.Popen(['tar', '-xvf', os.path.join(backup_tmpdir,filename), '-C', backup_tmpdir],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) - headers,stderr = command.communicate(headers) - if len(headers) <= 0: - print stderr - raise QubesException("ERROR: unable to read the qubes backup file {0}. Is it really a backup?".format(restore_target)) - else: - return (backup_tmpdir,headers.strip(" \r\n\t")) + hmacfile = command.stdout.readline().strip(" \t\r\n") + print "Getting hmac",hmacfile + while not os.path.exists(os.path.join(backup_tmpdir,hmacfile)): + time.sleep(1000) + + command.terminate() + command.wait() + vmproc.terminate() + vmproc.wait() + + print "Loading hmac for file",filename + hmac = load_hmac(open(os.path.join(backup_tmpdir,filename+".hmac"),'r').read()) + + print "Successfully retrieved headers" + + print "Verifying file",filename + hmac_proc = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], stdin=open(os.path.join(backup_tmpdir,filename),'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout,stderr = hmac_proc.communicate() + if len(stderr) > 0: + raise QubesException("ERROR: verify file {0}: {1}".format((filename,stderr))) + else: + if len(hmac) > 0 and load_hmac(stdout) == hmac: + print "File verification OK -> Extracting archive",filename + if encrypt: + encryptor = subprocess.Popen (["openssl", "enc", "-d", "-aes-256-cbc", "-pass", "pass:"+passphrase], stdin=open(os.path.join(backup_tmpdir,filename),'rb'), stdout=subprocess.PIPE) + tarhead_command = subprocess.Popen(['tar', '--tape-length','1000000', '-xv'],stdin=encryptor.stdout) else: - raise QubesException("ERROR: invalid hmac for file {0}: {1}. Is the passphrase correct?".format(filename,load_hmac(stdout))) + encryptor = None + tarhead_command = subprocess.Popen(['tar', '--tape-length','1000000', '-xvf', os.path.join(backup_tmpdir,filename)]) + + tarhead_command.wait() + if encryptor: + if encryptor.poll() != 0: + raise QubesException("ERROR: unable to decrypt file {0}".format(filename)) + if tarhead_command.poll() != 0: + raise QubesException("ERROR: unable to extract the qubes.xml file. Is archive encrypted?") + + return (backup_tmpdir,"qubes.xml") + else: + raise QubesException("ERROR: unable to verify the qubes.xml file. Is the passphrase correct?") return None @@ -1653,7 +1669,8 @@ def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, host # Maybe the (custom) netvm is in the backup? netvm_on_backup = backup_collection.get_vm_by_name (netvm_name) - if not ((netvm_on_backup is not None) and netvm_on_backup.is_netvm() and is_vm_included_in_backup(backup_dir, netvm_on_backup)): + if not ((netvm_on_backup is not None) and netvm_on_backup.is_netvm() and is_vm_ +included_in_backup(backup_dir, netvm_on_backup)): if options['use-default-netvm']: vms_to_restore[vm.name]['netvm'] = host_collection.get_default_netvm().name vm.uses_default_netvm = True From 04d38055a63f474f418cc1d969eaa4f5c036236f Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Mon, 30 Sep 2013 14:54:14 +0200 Subject: [PATCH 30/34] backup: fixed vm extraction path and untracked errors --- dom0/qvm-core/qubesutils.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 120ab4cf..e10ad4d6 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1199,6 +1199,9 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba to_send.put("FINISHED") send_proc.join() + + if send_proc.exitcode != 0: + raise QubesException("Failed to send backup: error in the sending process") if vmproc: print "VMProc1 proc return code:",vmproc.poll() @@ -1331,17 +1334,20 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s def run(self): self.print_callback("Started sending thread") - self.print_callback("Moving to temporary dir "+self.base_dir) + self.print_callback("Moving to dir "+self.base_dir) os.chdir(self.base_dir) for filename in iter(self.queue.get,None): if filename == "FINISHED": break - self.print_callback("Extracting file "+filename+" to "+self.base_dir) + self.print_callback("Extracting file "+filename+" to "+qubes_base_dir) if self.tar2_command == None: - self.tar2_command = ['tar', '--tape-length','1000000', '-xvf', self.restore_pipe] + # FIXME: Make the extraction safer by avoiding to erase other vms: + # - extracting directly to the target directory (based on the vm name and by using the --strip=2). + # - ensuring that the leading slashs are ignored when extracting (can also be obtained by running with --strip ?) + self.tar2_command = ['tar', '--tape-length','1000000', '-C', qubes_base_dir, '-xvf', self.restore_pipe] self.print_callback("Running command "+str(self.tar2_command)) self.tar2_command = subprocess.Popen(self.tar2_command,stdin=subprocess.PIPE) @@ -1452,8 +1458,14 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s if vmproc.poll() != 0: raise QubesException("ERROR: unable to read the qubes backup {0} because of a VM error: {1}".format(restore_target,vmproc.stderr.read())) + print "Extraction process status:",extract_proc.exitcode + to_extract.put("FINISHED") + print_callback("Waiting for the extraction process to finish...") extract_proc.join() + print_callback("Extraction process finished with code:"+str(extract_proc.exitcode)) + if extract_proc.exitcode != 0: + raise QubesException("ERROR: unable to extract the qubes backup. Check extracting process errors.") def backup_restore_set_defaults(options): if 'use-default-netvm' not in options: @@ -1552,9 +1564,11 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) if len(hmac) > 0 and load_hmac(stdout) == hmac: print "File verification OK -> Extracting archive",filename if encrypt: + print "Starting decryption process" encryptor = subprocess.Popen (["openssl", "enc", "-d", "-aes-256-cbc", "-pass", "pass:"+passphrase], stdin=open(os.path.join(backup_tmpdir,filename),'rb'), stdout=subprocess.PIPE) tarhead_command = subprocess.Popen(['tar', '--tape-length','1000000', '-xv'],stdin=encryptor.stdout) else: + print "No decryption process required" encryptor = None tarhead_command = subprocess.Popen(['tar', '--tape-length','1000000', '-xvf', os.path.join(backup_tmpdir,filename)]) @@ -1669,8 +1683,7 @@ def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, host # Maybe the (custom) netvm is in the backup? netvm_on_backup = backup_collection.get_vm_by_name (netvm_name) - if not ((netvm_on_backup is not None) and netvm_on_backup.is_netvm() and is_vm_ -included_in_backup(backup_dir, netvm_on_backup)): + if not ((netvm_on_backup is not None) and netvm_on_backup.is_netvm() and is_vm_included_in_backup(backup_dir, netvm_on_backup)): if options['use-default-netvm']: vms_to_restore[vm.name]['netvm'] = host_collection.get_default_netvm().name vm.uses_default_netvm = True From 3e4637415abc11860fb1f1b537b448749088a38c Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Tue, 1 Oct 2013 12:09:13 +0200 Subject: [PATCH 31/34] backup: improved error handling during restore process --- dom0/qvm-core/qubesutils.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index e10ad4d6..4ab0c13f 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1522,7 +1522,7 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) qvm_collection.unlock_db() # If APPVM, STDOUT is a PIPE - vmproc = vm.run(command = restore_command, passio_popen = True) + vmproc = vm.run(command = restore_command, passio_popen = True, passio_stderr = True) vmproc.stdin.write(restore_target.replace("\r","").replace("\n","")+"\n") else: @@ -1544,9 +1544,19 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) while not os.path.exists(os.path.join(backup_tmpdir,hmacfile)): time.sleep(1000) - + command.terminate() command.wait() + + if vmproc.poll() != None and vmproc.poll() != 0: + error = vmproc.stderr.read() + print error,vmproc.poll(),command.poll() + raise QubesException("ERROR: retrieving backup headers:{0}".format(error)) + elif command.poll() != None and command.poll() not in [0,-15]: + error = command.stderr.read() + print error,vmproc.poll(),command.poll() + raise QubesException("ERROR: retrieving backup headers:{0}".format(error)) + vmproc.terminate() vmproc.wait() From 6c7322d324121be42f3bc1321cc9f54121935f47 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Mon, 21 Oct 2013 15:39:16 +0200 Subject: [PATCH 32/34] backup: fixed reinitialisation of all backup flags in qubes.xml --- dom0/qvm-core/qubesutils.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 4ab0c13f..ed4b114a 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -907,9 +907,14 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca print_callback(s) - vm.backup_content = True - vm.backup_size = vm.get_disk_utilization() - vm.backup_path = vm.dir_path.split(os.path.normpath(qubes_base_dir)+"/")[1] + # Initialize backup flag on all VMs + for vm in qvm_collection.values(): + vm.backup_content = False + + if vm in vms_for_backup: + vm.backup_content = True + vm.backup_size = vm.get_disk_utilization() + vm.backup_path = vm.dir_path.split(os.path.normpath(qubes_base_dir)+"/")[1] qvm_collection.save() # FIXME: should be after backup completed From b96ec61e17be2e2e6482a62118c93f0908ca24b7 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Mon, 21 Oct 2013 15:50:54 +0200 Subject: [PATCH 33/34] backup: implemented header restoration using tar2qfile --- dom0/qvm-core/qubesutils.py | 51 +++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index ed4b114a..3a6024fc 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1495,6 +1495,20 @@ def load_hmac(hmac): return hmac +import struct +def get_qfile_error(buffer): + error = struct.unpack("I",buffer[0:4])[0] + error_msg = { 0: "COPY_FILE_OK", + 1: "COPY_FILE_READ_EOF", + 2: "COPY_FILE_READ_ERROR", + 3: "COPY_FILE_WRITE_ERROR", + } + + if error in error_msg.keys(): + return error_msg[error] + else: + return "UNKNOWN_ERROR_"+str(error) + def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None): # Simulate dd if=backup_file count=10 | file - # Simulate dd if=backup_file count=10 | gpg2 -d | tar xzv -O @@ -1529,7 +1543,6 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) # If APPVM, STDOUT is a PIPE vmproc = vm.run(command = restore_command, passio_popen = True, passio_stderr = True) vmproc.stdin.write(restore_target.replace("\r","").replace("\n","")+"\n") - else: # Create the target directory if not os.path.exists (restore_target): @@ -1538,32 +1551,44 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) fp = open(restore_target,'rb') headers = fp.read(4096*16) + tar1_command = ['/usr/lib/qubes/qfile-dom0-unpacker', str(os.getuid()), backup_tmpdir] + command = subprocess.Popen(tar1_command,stdin=vmproc.stdout,stdout=subprocess.PIPE,stderr=subprocess.PIPE) - command = subprocess.Popen(['tar', '-i', '-xv', '-C', backup_tmpdir, 'qubes.xml*'],stdin=vmproc.stdout,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + result_header = command.stdout.read() - filename = command.stdout.readline().strip(" \t\r\n") - print "Getting file",filename + if vmproc.poll() != None: + error = vmproc.stderr.read() + print error + print vmproc.poll(),command.poll() + raise QubesException("ERROR: Immediate VM error while retrieving backup headers:{0}".format(error)) - hmacfile = command.stdout.readline().strip(" \t\r\n") - print "Getting hmac",hmacfile + filename = "qubes.xml.000" - while not os.path.exists(os.path.join(backup_tmpdir,hmacfile)): - time.sleep(1000) + print result_header.encode("hex") + error_msg = get_qfile_error(result_header) + if error_msg != "COPY_FILE_OK": + print vmproc.stdout.read() + raise QubesException("ERROR: unpacking backup headers: {0}".format(error_msg)) + if not os.path.exists(os.path.join(backup_tmpdir,filename+".hmac")): + raise QubesException("ERROR: header not extracted correctly: {0}".format(os.path.join(backup_tmpdir,filename+".hmac"))) command.terminate() command.wait() if vmproc.poll() != None and vmproc.poll() != 0: error = vmproc.stderr.read() - print error,vmproc.poll(),command.poll() - raise QubesException("ERROR: retrieving backup headers:{0}".format(error)) + print error + print vmproc.poll(),command.poll() + raise QubesException("ERROR: VM error retrieving backup headers") elif command.poll() != None and command.poll() not in [0,-15]: error = command.stderr.read() - print error,vmproc.poll(),command.poll() + print error + print vmproc.poll(),command.poll() raise QubesException("ERROR: retrieving backup headers:{0}".format(error)) - vmproc.terminate() - vmproc.wait() + if vmproc.poll() == None: + vmproc.terminate() + vmproc.wait() print "Loading hmac for file",filename hmac = load_hmac(open(os.path.join(backup_tmpdir,filename+".hmac"),'r').read()) From 1a7d1702566dd6415ac65ec7f82d62677e6b989c Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Mon, 21 Oct 2013 16:04:15 +0200 Subject: [PATCH 34/34] backup: use tar2qfile in qubes.Restore RPC file --- qubes_rpc/qubes.Restore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes_rpc/qubes.Restore b/qubes_rpc/qubes.Restore index dfea1448..96a76e53 100644 --- a/qubes_rpc/qubes.Restore +++ b/qubes_rpc/qubes.Restore @@ -5,7 +5,7 @@ if [ -f "$args" ] ; then echo "Performing restore from backup file $args" >2 TARGET="$args" echo "Copying $TARGET to STDOUT" >2 - cat $TARGET + cat $TARGET | /usr/lib/qubes/tar2qfile else echo "Checking if arguments is matching a command" >2 COMMAND=`echo $args | cut -d ' ' -f 1`