From bf2dd7f0c74527d033f51f011ce15c02b33f7c1d Mon Sep 17 00:00:00 2001 From: Andrew Sorensen Date: Tue, 26 Mar 2013 18:36:29 -0700 Subject: [PATCH 01/82] dom0: allow backup to AppVM instead of just local block device --- dom0/qvm-core/qubesutils.py | 44 +++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 2fb63aa9..f6b89c8c 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -780,10 +780,10 @@ def file_to_backup (file_path, sz = None): def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_callback = print_stdout): """If vms = None, include all (sensible) VMs; exclude_list is always applied""" - + ''' if not os.path.exists (base_backup_dir): raise QubesException("The target directory doesn't exist!") - + ''' files_to_backup = file_to_backup (qubes_store_filename) if exclude_list is None: @@ -955,7 +955,7 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca fmt="{{0:-^{0}}}-+".format(f["width"] + 1) s += fmt.format('-') print_callback(s) - + ''' stat = os.statvfs(base_backup_dir) backup_fs_free_sz = stat.f_bsize * stat.f_bavail print_callback("") @@ -966,7 +966,7 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca raise QubesException("Please shutdown all VMs before proceeding.") print_callback("-> Available space: {0}".format(size_to_human(backup_fs_free_sz))) - + ''' return files_to_backup def backup_do(base_backup_dir, files_to_backup, progress_callback = None): @@ -1003,6 +1003,42 @@ def backup_do(base_backup_dir, files_to_backup, progress_callback = None): progress = bytes_backedup * 100 / total_backup_sz progress_callback(progress) +def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = None): + + total_backup_sz = 0 + for file in files_to_backup: + total_backup_sz += file["size"] + + backup_dir = base_backup_dir + "/qubes-{0}".format (time.strftime("%Y-%m-%d-%H%M%S")) + ''' + if os.path.exists (backup_dir): + raise QubesException("ERROR: the path {0} already exists?!".format(backup_dir)) + + os.mkdir (backup_dir) + + if not os.path.exists (backup_dir): + raise QubesException("Strange: couldn't create backup dir: {0}?!".format(backup_dir)) + ''' + bytes_backedup = 0 + for file in files_to_backup: + # We prefer to use Linux's cp, because it nicely handles sparse files + progress = bytes_backedup * 100 / total_backup_sz + progress_callback(progress) + dest_dir = backup_dir + '/' + file["subdir"] + if file["subdir"] != "": + retcode = subprocess.call (["qvm-run", "-p", appvm, "mkdir -p " + dest_dir]) + if retcode != 0: + raise QubesException("Cannot create directory: {0}?!".format(dest_dir)) + + file["basename"] = os.path.basename(file["path"]) + compressor = subprocess.Popen (["gzip", "-c", file["path"]], stdout=subprocess.PIPE) + subprocess.Popen (["qvm-run", "--pass-io", "-p", appvm, "cat > " + backup_dir + "/" + file["basename"]], stdin=compressor.stdout) + + bytes_backedup += file["size"] + progress = bytes_backedup * 100 / total_backup_sz + progress_callback(progress) + + def backup_restore_set_defaults(options): if 'use-default-netvm' not in options: options['use-default-netvm'] = False From 65822f60337b93d087c2fcb225764b50926b526b Mon Sep 17 00:00:00 2001 From: Andrew Sorensen Date: Wed, 27 Mar 2013 11:04:39 -0700 Subject: [PATCH 02/82] dom0: switch backup compression to .tar.gz, properly handle folders. --- dom0/qvm-core/qubesutils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index f6b89c8c..27371429 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1031,8 +1031,8 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = raise QubesException("Cannot create directory: {0}?!".format(dest_dir)) file["basename"] = os.path.basename(file["path"]) - compressor = subprocess.Popen (["gzip", "-c", file["path"]], stdout=subprocess.PIPE) - subprocess.Popen (["qvm-run", "--pass-io", "-p", appvm, "cat > " + backup_dir + "/" + file["basename"]], stdin=compressor.stdout) + compressor = subprocess.Popen (["tar", "-PcOz", file["path"]], stdout=subprocess.PIPE) + subprocess.Popen (["qvm-run", "--pass-io", "-p", appvm, "cat > " + dest_dir + file["basename"] + ".tar.gz"], stdin=compressor.stdout) bytes_backedup += file["size"] progress = bytes_backedup * 100 / total_backup_sz From 7cacc3db482f033cee6ece401f5bd9ae7bc1b5a3 Mon Sep 17 00:00:00 2001 From: Andrew Sorensen Date: Sat, 22 Jun 2013 18:02:40 -0700 Subject: [PATCH 03/82] dom0: edit qvm-backup to use backup_do_copy, throw error when appvm is not found --- dom0/qvm-core/qubesutils.py | 11 ++++++++++- dom0/qvm-tools/qvm-backup | 4 ++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 27371429..4263345e 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1005,6 +1005,15 @@ def backup_do(base_backup_dir, files_to_backup, progress_callback = None): def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = None): + # does the vm exist? + qvm_collection = QubesVmCollection() + qvm_collection.lock_db_for_reading() + qvm_collection.load() + + vm = qvm_collection.get_vm_by_name(appvm) + if vm is None or vm.qid not in qvm_collection: + raise QubesException("VM {0} does not exist".format(appvm)) + total_backup_sz = 0 for file in files_to_backup: total_backup_sz += file["size"] @@ -1026,7 +1035,7 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = progress_callback(progress) dest_dir = backup_dir + '/' + file["subdir"] if file["subdir"] != "": - retcode = subprocess.call (["qvm-run", "-p", appvm, "mkdir -p " + dest_dir]) + retcode = vm.run(["qvm-run", "-p", appvm, "mkdir -p " + dest_dir]) if retcode != 0: raise QubesException("Cannot create directory: {0}?!".format(dest_dir)) diff --git a/dom0/qvm-tools/qvm-backup b/dom0/qvm-tools/qvm-backup index cee29e3b..fafb9eb2 100755 --- a/dom0/qvm-tools/qvm-backup +++ b/dom0/qvm-tools/qvm-backup @@ -22,7 +22,7 @@ from qubes.qubes import QubesVmCollection from qubes.qubes import QubesException -from qubes.qubesutils import backup_prepare, backup_do +from qubes.qubesutils import backup_prepare, backup_do_copy from optparse import OptionParser import os import sys @@ -72,7 +72,7 @@ def main(): exit (0) try: - backup_do(base_backup_dir, files_to_backup, progress_callback=print_progress) + backup_do_copy("storage", base_backup_dir, files_to_backup, progress_callback=print_progress) except QubesException as e: print >>sys.stderr, "ERROR: %s" % str(e) exit(1) From 246e8c383dac369c947653262bdeee17977acad9 Mon Sep 17 00:00:00 2001 From: Andrew Sorensen Date: Sat, 22 Jun 2013 18:22:11 -0700 Subject: [PATCH 04/82] dom0: fix directory creation --- dom0/qvm-core/qubesutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 4263345e..1a155050 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1035,7 +1035,7 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = progress_callback(progress) dest_dir = backup_dir + '/' + file["subdir"] if file["subdir"] != "": - retcode = vm.run(["qvm-run", "-p", appvm, "mkdir -p " + dest_dir]) + retcode = vm.run("mkdir -p " + dest_dir) if retcode != 0: raise QubesException("Cannot create directory: {0}?!".format(dest_dir)) From 93162df677406cee8a1769156f66b7908d98121c Mon Sep 17 00:00:00 2001 From: Andrew Sorensen Date: Sat, 22 Jun 2013 19:59:16 -0700 Subject: [PATCH 05/82] dom0: use vm.run() instead of subprocess.Popen() directly --- dom0/qvm-core/qubesutils.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 1a155050..a4b9e7c5 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1030,7 +1030,6 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = ''' bytes_backedup = 0 for file in files_to_backup: - # We prefer to use Linux's cp, because it nicely handles sparse files progress = bytes_backedup * 100 / total_backup_sz progress_callback(progress) dest_dir = backup_dir + '/' + file["subdir"] @@ -1040,9 +1039,13 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = raise QubesException("Cannot create directory: {0}?!".format(dest_dir)) file["basename"] = os.path.basename(file["path"]) - compressor = subprocess.Popen (["tar", "-PcOz", file["path"]], stdout=subprocess.PIPE) - subprocess.Popen (["qvm-run", "--pass-io", "-p", appvm, "cat > " + dest_dir + file["basename"] + ".tar.gz"], stdin=compressor.stdout) - + vm.run("mkdir -p {0}".format(dest_dir)) + retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"] + ".tar.gz"), passio_popen = True) + compressor = subprocess.Popen (["tar", "-PcOz", file["path"]], stdout=retcode.stdin) + compressor.wait() + if compressor.retcode != 0: + raise QubesException("Failed to backup file {0} with error {1}".format(file["basename"])) + bytes_backedup += file["size"] progress = bytes_backedup * 100 / total_backup_sz progress_callback(progress) From c2f157c2d2f1eea93519925091a4d398ccba203a Mon Sep 17 00:00:00 2001 From: Andrew Sorensen Date: Sat, 22 Jun 2013 20:30:32 -0700 Subject: [PATCH 06/82] dom0: close Popen when the transfer is complete --- dom0/qvm-core/qubesutils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index a4b9e7c5..92b6449c 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1043,7 +1043,8 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"] + ".tar.gz"), passio_popen = True) compressor = subprocess.Popen (["tar", "-PcOz", file["path"]], stdout=retcode.stdin) compressor.wait() - if compressor.retcode != 0: + retcode.terminate() + if compressor.returncode != 0: raise QubesException("Failed to backup file {0} with error {1}".format(file["basename"])) bytes_backedup += file["size"] From 3d7af2f7f51e91f74d8a6be3efbd70e440cb39e2 Mon Sep 17 00:00:00 2001 From: Andrew Sorensen Date: Sat, 22 Jun 2013 20:59:07 -0700 Subject: [PATCH 07/82] dom0: allow the user to set the AppVM --- dom0/qvm-tools/qvm-backup | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dom0/qvm-tools/qvm-backup b/dom0/qvm-tools/qvm-backup index fafb9eb2..a8f1d50d 100755 --- a/dom0/qvm-tools/qvm-backup +++ b/dom0/qvm-tools/qvm-backup @@ -38,6 +38,8 @@ def main(): help="Exclude the specified VM from backup (might be repeated)") parser.add_option ("--force-root", action="store_true", dest="force_root", default=False, help="Force to run, even with root privileges") + parser.add_option ("-d", "--dest-vm", action="store", dest="appvm", + help="The AppVM to send backups to") (options, args) = parser.parse_args () @@ -72,7 +74,7 @@ def main(): exit (0) try: - backup_do_copy("storage", base_backup_dir, files_to_backup, progress_callback=print_progress) + backup_do_copy(options.appvm, base_backup_dir, files_to_backup, progress_callback=print_progress) except QubesException as e: print >>sys.stderr, "ERROR: %s" % str(e) exit(1) From 4ed00f123d707c4614e164be66a5ae542fbf4383 Mon Sep 17 00:00:00 2001 From: Andrew Sorensen Date: Sat, 22 Jun 2013 21:19:59 -0700 Subject: [PATCH 08/82] dom0: allow user to decide if encryption should be used, close qvm database --- dom0/qvm-core/qubesutils.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 92b6449c..f30f02f6 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1003,7 +1003,7 @@ def backup_do(base_backup_dir, files_to_backup, progress_callback = None): progress = bytes_backedup * 100 / total_backup_sz progress_callback(progress) -def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = None): +def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = None, encrypt=False): # does the vm exist? qvm_collection = QubesVmCollection() @@ -1014,6 +1014,8 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = if vm is None or vm.qid not in qvm_collection: raise QubesException("VM {0} does not exist".format(appvm)) + qvm_collection.unlock_db() + total_backup_sz = 0 for file in files_to_backup: total_backup_sz += file["size"] @@ -1040,8 +1042,13 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = file["basename"] = os.path.basename(file["path"]) vm.run("mkdir -p {0}".format(dest_dir)) - retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"] + ".tar.gz"), passio_popen = True) - compressor = subprocess.Popen (["tar", "-PcOz", file["path"]], stdout=retcode.stdin) + if encrypt: + retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"] + ".gpg"), passio_popen = True) + compressor = subprocess.Popen (["gpg", "-ac", "--force-mdc", "-o-", file["path"]], stdout=retcode.stdin) + else: + retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"] + ".tar.gz"), passio_popen = True) + compressor = subprocess.Popen (["tar", "-PcOz", file["path"]], stdout=retcode.stdin) + compressor.wait() retcode.terminate() if compressor.returncode != 0: From 1d2990e9380ba81af091c0a460d981a439299dc4 Mon Sep 17 00:00:00 2001 From: Andrew Sorensen Date: Sat, 22 Jun 2013 21:27:58 -0700 Subject: [PATCH 09/82] dom0: add option for encryption to qvm-backup --- dom0/qvm-tools/qvm-backup | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dom0/qvm-tools/qvm-backup b/dom0/qvm-tools/qvm-backup index a8f1d50d..4fef7314 100755 --- a/dom0/qvm-tools/qvm-backup +++ b/dom0/qvm-tools/qvm-backup @@ -40,6 +40,8 @@ def main(): help="Force to run, even with root privileges") parser.add_option ("-d", "--dest-vm", action="store", dest="appvm", help="The AppVM to send backups to") + parser.add_option ("-e", "--encrypt", action="store_true", dest="encrypt", default=False, + help="Encrypts the backup") (options, args) = parser.parse_args () @@ -74,7 +76,7 @@ def main(): exit (0) try: - backup_do_copy(options.appvm, base_backup_dir, files_to_backup, progress_callback=print_progress) + backup_do_copy(options.appvm, base_backup_dir, files_to_backup, progress_callback=print_progress, encrypt=options.encrypt) except QubesException as e: print >>sys.stderr, "ERROR: %s" % str(e) exit(1) From ead479804e3f9679f1abc8da7002492758105b9a Mon Sep 17 00:00:00 2001 From: Andrew Sorensen Date: Sun, 7 Jul 2013 00:00:07 -0700 Subject: [PATCH 10/82] dom0: wait for folder to be created before adding file --- dom0/qvm-core/qubesutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index f30f02f6..d84b7a8b 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1036,7 +1036,7 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = progress_callback(progress) dest_dir = backup_dir + '/' + file["subdir"] if file["subdir"] != "": - retcode = vm.run("mkdir -p " + dest_dir) + retcode = vm.run("mkdir -p " + dest_dir, wait = True) if retcode != 0: raise QubesException("Cannot create directory: {0}?!".format(dest_dir)) From fb8748f3e9d292ad9bf790fcf3169faef9ad8925 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Wed, 14 Aug 2013 10:18:05 +0200 Subject: [PATCH 11/82] backup: implemented use of tar+gpg2 instead of only encrypting files --- dom0/qvm-core/qubesutils.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index d84b7a8b..8aba4943 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1042,15 +1042,32 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = file["basename"] = os.path.basename(file["path"]) vm.run("mkdir -p {0}".format(dest_dir)) - if encrypt: - retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"] + ".gpg"), passio_popen = True) - compressor = subprocess.Popen (["gpg", "-ac", "--force-mdc", "-o-", file["path"]], stdout=retcode.stdin) - else: - retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"] + ".tar.gz"), passio_popen = True) - compressor = subprocess.Popen (["tar", "-PcOz", file["path"]], stdout=retcode.stdin) + retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"]), passio_popen = True) + + if encrypt: + compressor = subprocess.Popen (["tar", "-PcO",'--checkpoint=10000', file["path"]],stdout=subprocess.PIPE) + encryptor = subprocess.Popen (["gpg2", "-ac", "--force-mdc", "-o-"], stdin=compressor.stdout, stdout=retcode.stdin) + encryptor.wait() + + if encryptor.returncode != 0: + raise QubesException("Failed to backup file {0} with error {1}".format(file["basename"])) + else: + compressor = subprocess.Popen (["tar", "-PcOz",'--checkpoint=10000', file["path"]],stdout=retcode.stdin) + + ''' + for checkpoint in compressor.stderr: + print "Checkpoints:",len(checkpoints) + + match = re.search('tar:.*(\d+)',checkpoints) + if match: + print bytes_backedup,total_backup_sz + progress = int(match.group(1)) * 100 / total_backup_sz + progress_callback(progress) + ''' compressor.wait() retcode.terminate() + if compressor.returncode != 0: raise QubesException("Failed to backup file {0} with error {1}".format(file["basename"])) From 6c09189b774af32c97b84e93b8c5710851c5851f Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Wed, 14 Aug 2013 10:19:15 +0200 Subject: [PATCH 12/82] backup: improved performance by optimizing tar and gpg options --- dom0/qvm-core/qubesutils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 8aba4943..642414cf 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1046,8 +1046,8 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"]), passio_popen = True) if encrypt: - compressor = subprocess.Popen (["tar", "-PcO",'--checkpoint=10000', file["path"]],stdout=subprocess.PIPE) - encryptor = subprocess.Popen (["gpg2", "-ac", "--force-mdc", "-o-"], stdin=compressor.stdout, stdout=retcode.stdin) + compressor = subprocess.Popen (["tar", "-PcO",'--sparse','--checkpoint=10000', file["path"]],stdout=subprocess.PIPE) + encryptor = subprocess.Popen (["gpg2", "-c", "--force-mdc", "-o-"], stdin=compressor.stdout, stdout=retcode.stdin) encryptor.wait() if encryptor.returncode != 0: From aea789d0a17beee9993ca6a684581d74a7563b7e Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Wed, 14 Aug 2013 10:21:41 +0200 Subject: [PATCH 13/82] backup: implemented use of a single tar file instead of creation of multiple file during backup --- dom0/qvm-core/qubesutils.py | 58 ++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 642414cf..bc22a331 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1031,29 +1031,35 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = raise QubesException("Strange: couldn't create backup dir: {0}?!".format(backup_dir)) ''' bytes_backedup = 0 - for file in files_to_backup: - progress = bytes_backedup * 100 / total_backup_sz - progress_callback(progress) - dest_dir = backup_dir + '/' + file["subdir"] - if file["subdir"] != "": - retcode = vm.run("mkdir -p " + dest_dir, wait = True) - if retcode != 0: - raise QubesException("Cannot create directory: {0}?!".format(dest_dir)) - file["basename"] = os.path.basename(file["path"]) - vm.run("mkdir -p {0}".format(dest_dir)) + progress = bytes_backedup * 100 / total_backup_sz + progress_callback(progress) + dest_dir = backup_dir + '/' + file["subdir"] + if file["subdir"] != "": + retcode = vm.run("mkdir -p " + dest_dir, wait = True) + if retcode != 0: + raise QubesException("Cannot create directory: {0}?!".format(dest_dir)) - retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"]), passio_popen = True) + file["basename"] = os.path.basename(file["path"]) + vm.run("mkdir -p {0}".format(dest_dir)) - if encrypt: - compressor = subprocess.Popen (["tar", "-PcO",'--sparse','--checkpoint=10000', file["path"]],stdout=subprocess.PIPE) - encryptor = subprocess.Popen (["gpg2", "-c", "--force-mdc", "-o-"], stdin=compressor.stdout, stdout=retcode.stdin) - encryptor.wait() + tar_cmdline = ["tar", "-PcO",'--sparse','-C','/var/lib/qubes','--checkpoint=10000'] - if encryptor.returncode != 0: - raise QubesException("Failed to backup file {0} with error {1}".format(file["basename"])) - else: - compressor = subprocess.Popen (["tar", "-PcOz",'--checkpoint=10000', file["path"]],stdout=retcode.stdin) + for filename in files_to_backup: + tar_cmdline.append(filename["path"].split("/var/lib/qubes/")[1]) + print ("Will backup using command",tar_cmdline) + + retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"]), passio_popen = True) + + if encrypt: + compressor = subprocess.Popen (tar_cmdline,stdout=subprocess.PIPE) + encryptor = subprocess.Popen (["gpg2", "-c", "--force-mdc", "-o-"], stdin=compressor.stdout, stdout=retcode.stdin) + encryptor.wait() + + if encryptor.returncode != 0: + raise QubesException("Failed to backup file {0} with error {1}".format(file["basename"])) + else: + compressor = subprocess.Popen (tar_cmdline,stdout=retcode.stdin) ''' for checkpoint in compressor.stderr: @@ -1065,15 +1071,15 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = progress = int(match.group(1)) * 100 / total_backup_sz progress_callback(progress) ''' - compressor.wait() - retcode.terminate() + compressor.wait() + retcode.terminate() - if compressor.returncode != 0: - raise QubesException("Failed to backup file {0} with error {1}".format(file["basename"])) + if compressor.returncode != 0: + raise QubesException("Failed to backup file {0} with error {1}".format(file["basename"])) - bytes_backedup += file["size"] - progress = bytes_backedup * 100 / total_backup_sz - progress_callback(progress) + bytes_backedup += file["size"] + progress = bytes_backedup * 100 / total_backup_sz + progress_callback(progress) def backup_restore_set_defaults(options): From fbb26d89b445bea460eb5e2c900994ea0d5f475b Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Wed, 14 Aug 2013 10:23:04 +0200 Subject: [PATCH 14/82] backup: implemented progress feedback using tar checkpoint and a temporary file for tar output --- dom0/qvm-core/qubesutils.py | 44 +++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index bc22a331..01a1c14a 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1030,9 +1030,9 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = if not os.path.exists (backup_dir): raise QubesException("Strange: couldn't create backup dir: {0}?!".format(backup_dir)) ''' - bytes_backedup = 0 + blocks_backedup = 0 - progress = bytes_backedup * 100 / total_backup_sz + progress = blocks_backedup * 11 / total_backup_sz progress_callback(progress) dest_dir = backup_dir + '/' + file["subdir"] if file["subdir"] != "": @@ -1043,7 +1043,7 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = file["basename"] = os.path.basename(file["path"]) vm.run("mkdir -p {0}".format(dest_dir)) - tar_cmdline = ["tar", "-PcO",'--sparse','-C','/var/lib/qubes','--checkpoint=10000'] + tar_cmdline = ["tar", "-PczO",'--sparse','-C','/var/lib/qubes','--checkpoint=10000'] for filename in files_to_backup: tar_cmdline.append(filename["path"].split("/var/lib/qubes/")[1]) @@ -1051,13 +1051,20 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"]), passio_popen = True) + import tempfile + feedback_file = tempfile.NamedTemporaryFile() + print feedback_file if encrypt: - compressor = subprocess.Popen (tar_cmdline,stdout=subprocess.PIPE) + compressor = subprocess.Popen (tar_cmdline,stdout=subprocess.PIPE,stderr=feedback_file) encryptor = subprocess.Popen (["gpg2", "-c", "--force-mdc", "-o-"], stdin=compressor.stdout, stdout=retcode.stdin) + ''' + print "Wait for encryptor" encryptor.wait() + print "End waiting" if encryptor.returncode != 0: raise QubesException("Failed to backup file {0} with error {1}".format(file["basename"])) + ''' else: compressor = subprocess.Popen (tar_cmdline,stdout=retcode.stdin) @@ -1071,15 +1078,34 @@ def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = progress = int(match.group(1)) * 100 / total_backup_sz progress_callback(progress) ''' + ''' + print "Wait for compressor" compressor.wait() - retcode.terminate() + print "End waiting" + ''' + feedback_file_r = open(feedback_file.name,'r') + while compressor.poll()==None or (encryptor!=None and encryptor.poll()==None): + time.sleep(1) + #print "Polling:", compressor.poll(),encryptor.poll() + #print feedback_file_r.tell(),feedback_file_r.closed,feedback_file_r.name,feedback_file_r.readline() + match = re.search("tar: [^0-9]+([0-9]+)",feedback_file_r.readline()) + print match + if match: + blocks_backedup = int(match.group(1)) + progress = blocks_backedup * 11.024 * 1024 / total_backup_sz + #print blocks_backedup,total_backup_sz,progress + progress_callback(round(progress*100,2)) + + feedback_file_r.close() + feedback_file.close() + + retcode.terminate() + ''' if compressor.returncode != 0: raise QubesException("Failed to backup file {0} with error {1}".format(file["basename"])) - - bytes_backedup += file["size"] - progress = bytes_backedup * 100 / total_backup_sz - progress_callback(progress) + ''' + def backup_restore_set_defaults(options): From 5fa8d732ae70b47828625ad20806315e42479911 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Wed, 14 Aug 2013 10:26:58 +0200 Subject: [PATCH 15/82] backup: major revamp of the backup code to include backup to dom0, backup to vm, better cleanup code --- dom0/qvm-core/qubesutils.py | 153 +++++++++++++++++++----------------- dom0/qvm-tools/qvm-backup | 2 +- qubes_rpc/qubes.Backup | 23 ++++++ 3 files changed, 107 insertions(+), 71 deletions(-) create mode 100644 qubes_rpc/qubes.Backup diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 01a1c14a..de5212ee 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1003,110 +1003,123 @@ def backup_do(base_backup_dir, files_to_backup, progress_callback = None): progress = bytes_backedup * 100 / total_backup_sz progress_callback(progress) -def backup_do_copy(appvm, base_backup_dir, files_to_backup, progress_callback = None, encrypt=False): - - # does the vm exist? - qvm_collection = QubesVmCollection() - qvm_collection.lock_db_for_reading() - qvm_collection.load() - - vm = qvm_collection.get_vm_by_name(appvm) - if vm is None or vm.qid not in qvm_collection: - raise QubesException("VM {0} does not exist".format(appvm)) - - qvm_collection.unlock_db() - +def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, encrypt=False, appvm=None): + print appvm,base_backup_dir total_backup_sz = 0 for file in files_to_backup: total_backup_sz += file["size"] + + vmproc = None + if appvm != None: + # Prepare the backup target (Qubes service call) + backup_target = "QUBESRPC qubes.Backup none" + + # does the vm exist? + qvm_collection = QubesVmCollection() + qvm_collection.lock_db_for_reading() + qvm_collection.load() - backup_dir = base_backup_dir + "/qubes-{0}".format (time.strftime("%Y-%m-%d-%H%M%S")) - ''' - if os.path.exists (backup_dir): - raise QubesException("ERROR: the path {0} already exists?!".format(backup_dir)) + vm = qvm_collection.get_vm_by_name(appvm) + if vm is None or vm.qid not in qvm_collection: + raise QubesException("VM {0} does not exist".format(appvm)) + + qvm_collection.unlock_db() + + # If APPVM, STDOUT is a PIPE + vmproc = vm.run(command = backup_target, passio_popen = True) + vmproc.stdin.write(base_backup_dir.replace("\r","").replace("\n","")+"\n") + backup_stdout = vmproc.stdin + + else: + # Prepare the backup target (local file) + backup_target = base_backup_dir + "/qubes-{0}".format (time.strftime("%Y-%m-%d-%H%M%S")) + + # Create the target directory + if not os.path.exists (base_backup_dir): + raise QubesException("ERROR: the backup directory {0} does not exists".format(base_backup_dir)) + + # If not APPVM, STDOUT is a local file + backup_stdout = open(backup_target,'wb') - os.mkdir (backup_dir) - if not os.path.exists (backup_dir): - raise QubesException("Strange: couldn't create backup dir: {0}?!".format(backup_dir)) - ''' blocks_backedup = 0 - progress = blocks_backedup * 11 / total_backup_sz progress_callback(progress) - dest_dir = backup_dir + '/' + file["subdir"] - if file["subdir"] != "": - retcode = vm.run("mkdir -p " + dest_dir, wait = True) - if retcode != 0: - raise QubesException("Cannot create directory: {0}?!".format(dest_dir)) - - file["basename"] = os.path.basename(file["path"]) - vm.run("mkdir -p {0}".format(dest_dir)) tar_cmdline = ["tar", "-PczO",'--sparse','-C','/var/lib/qubes','--checkpoint=10000'] for filename in files_to_backup: tar_cmdline.append(filename["path"].split("/var/lib/qubes/")[1]) - print ("Will backup using command",tar_cmdline) - - retcode = vm.run(command = "cat > {0}".format(dest_dir + file["basename"]), passio_popen = True) + #print ("Will backup using command",tar_cmdline) import tempfile feedback_file = tempfile.NamedTemporaryFile() - print feedback_file + #print feedback_file if encrypt: compressor = subprocess.Popen (tar_cmdline,stdout=subprocess.PIPE,stderr=feedback_file) - encryptor = subprocess.Popen (["gpg2", "-c", "--force-mdc", "-o-"], stdin=compressor.stdout, stdout=retcode.stdin) - ''' - print "Wait for encryptor" - encryptor.wait() - print "End waiting" - - if encryptor.returncode != 0: - raise QubesException("Failed to backup file {0} with error {1}".format(file["basename"])) - ''' + encryptor = subprocess.Popen (["gpg2", "-c", "--force-mdc", "-o-"], stdin=compressor.stdout, stdout=backup_stdout) else: - compressor = subprocess.Popen (tar_cmdline,stdout=retcode.stdin) + compressor = subprocess.Popen (tar_cmdline,stdout=backup_stdout,stderr=feedback_file) + encryptor = None - ''' - for checkpoint in compressor.stderr: - print "Checkpoints:",len(checkpoints) - - match = re.search('tar:.*(\d+)',checkpoints) - if match: - print bytes_backedup,total_backup_sz - progress = int(match.group(1)) * 100 / total_backup_sz - progress_callback(progress) - ''' - ''' - print "Wait for compressor" - compressor.wait() - print "End waiting" - ''' + # Get tar backup feedback feedback_file_r = open(feedback_file.name,'r') - while compressor.poll()==None or (encryptor!=None and encryptor.poll()==None): + run_error = None + run_count = 1 + while run_count > 0 and run_error == None: time.sleep(1) - #print "Polling:", compressor.poll(),encryptor.poll() - #print feedback_file_r.tell(),feedback_file_r.closed,feedback_file_r.name,feedback_file_r.readline() match = re.search("tar: [^0-9]+([0-9]+)",feedback_file_r.readline()) - print match if match: blocks_backedup = int(match.group(1)) progress = blocks_backedup * 11.024 * 1024 / total_backup_sz #print blocks_backedup,total_backup_sz,progress progress_callback(round(progress*100,2)) + run_count = 0 + if compressor: + retcode=compressor.poll() + if retcode != None: + if retcode != 0: + run_error = "compressor" + else: + run_count += 1 + + if encryptor: + retcode=encryptor.poll() + if retcode != None: + if retcode != 0: + run_error = "encryptor" + else: + run_count += 1 + + if vmproc: + retcode = vmproc.poll() + if retcode != None: + if retcode != 0: + run_error = "VM "+appvm + print vmproc.stdout.read() + else: + # VM should run until the end + pass + + # Cleanup feedback_file_r.close() feedback_file.close() + backup_stdout.close() - retcode.terminate() - ''' - if compressor.returncode != 0: - raise QubesException("Failed to backup file {0} with error {1}".format(file["basename"])) - ''' - - + # Check returns code of compressor and encryptor and qubes vm retcode + if run_error != None: + try: + if compressor != None: + compressor.terminate() + if encryptor != None: + encryptor.terminate() + if vmproc != None: + vmproc.terminate() + except OSError: + pass + raise QubesException("Failed to perform backup: error with "+run_error) def backup_restore_set_defaults(options): if 'use-default-netvm' not in options: diff --git a/dom0/qvm-tools/qvm-backup b/dom0/qvm-tools/qvm-backup index 4fef7314..ea3a43dd 100755 --- a/dom0/qvm-tools/qvm-backup +++ b/dom0/qvm-tools/qvm-backup @@ -76,7 +76,7 @@ def main(): exit (0) try: - backup_do_copy(options.appvm, base_backup_dir, files_to_backup, progress_callback=print_progress, encrypt=options.encrypt) + backup_do_copy(base_backup_dir, files_to_backup, progress_callback=print_progress, encrypt=options.encrypt,appvm=options.appvm) except QubesException as e: print >>sys.stderr, "ERROR: %s" % str(e) exit(1) diff --git a/qubes_rpc/qubes.Backup b/qubes_rpc/qubes.Backup new file mode 100644 index 00000000..6e3f1d48 --- /dev/null +++ b/qubes_rpc/qubes.Backup @@ -0,0 +1,23 @@ +echo Starting Backupcopy +read args +echo Arguments: $args +if [ -d "$args" ] ; then + echo "Performing backup to directory $args" + TARGET="$args/qubes-backup-`date +'%Y-%d-%d-%H%M%S'`" + echo "Copying STDIN data to $TARGET" + cat > $TARGET +else + echo "Checking if arguments is matching a command" + COMMAND=`echo $args | cut -d ' ' -f 1` + TYPE=`type -t $COMMAND` + if [ "$TYPE" == "file" ] ; then + echo "Redirecting STDIN to $args" + # Parsing args to handle quotes correctly + # Dangerous method if args are uncontrolled + eval "set -- $args" + $@ + else + echo "Invalid command $COMMAND" + exit 1 + fi +fi From 9784ca87f688ff0d32e6bf444940c8cd0ff720b6 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Fri, 16 Aug 2013 09:12:06 +0200 Subject: [PATCH 16/82] backup: implemented mecanism to read only the backup headers --- dom0/qvm-core/qubesutils.py | 78 +++++++++++++++++++++++++++++++++++-- 1 file changed, 75 insertions(+), 3 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index de5212ee..248a69d6 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -906,7 +906,14 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca s += " <-- The VM is running, please shut it down before proceeding with the backup!" there_are_running_vms = True - print_callback(s) + # Build Backup VMs reference file + # Format: vm_name:tarball_path\n + backup_reference_file = open(os.path.join(qubes_base_dir,"backup_targets"),'w') + for vm in vms_for_backup: + backup_reference_file.write(vm.name+":"+vm.dir_path.split(qubes_base_dir)[1]+"\n") + backup_reference_file.flush() + backup_reference_file.close() + files_to_backup = file_to_backup(backup_reference_file.name,os.stat(backup_reference_file.name).st_size) + files_to_backup # Dom0 user home if not 'dom0' in exclude_list: @@ -1046,10 +1053,10 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e progress = blocks_backedup * 11 / total_backup_sz progress_callback(progress) - tar_cmdline = ["tar", "-PczO",'--sparse','-C','/var/lib/qubes','--checkpoint=10000'] + tar_cmdline = ["tar", "-PczO",'--sparse','-C',qubes_base_dir,'--checkpoint=10000'] for filename in files_to_backup: - tar_cmdline.append(filename["path"].split("/var/lib/qubes/")[1]) + tar_cmdline.append(filename["path"].split(os.path.normpath(qubes_base_dir)+"/")[1]) #print ("Will backup using command",tar_cmdline) import tempfile @@ -1135,8 +1142,73 @@ def backup_restore_set_defaults(options): return options +def backup_restore_header(restore_target, progress_callback = None, encrypt=False, appvm=None): + # Simulate dd if=backup_file count=10 | file - + # Simulate dd if=backup_file count=10 | gpg2 -d | tar xzv -O + # analysis = subprocess.Popen() + vmproc = None + if appvm != None: + # Prepare the backup target (Qubes service call) + backup_target = "QUBESRPC qubes.Backup none" + + # does the vm exist? + qvm_collection = QubesVmCollection() + qvm_collection.lock_db_for_reading() + qvm_collection.load() + + vm = qvm_collection.get_vm_by_name(appvm) + if vm is None or vm.qid not in qvm_collection: + raise QubesException("VM {0} does not exist".format(appvm)) + + qvm_collection.unlock_db() + + # If APPVM, STDOUT is a PIPE + vmproc = vm.run(command = restore_target, passio_popen = True) + vmproc.stdin.write(restore_target.replace("\r","").replace("\n","")+"\n") + headers = vmproc.stdout.read(4096) + vmproc.terminate() + + if len(headers) <= 0: + print vmproc.stderr.read() + raise QubesException("ERROR: the backup directory {0} does not exists".format(restore_target)) + + else: + # Create the target directory + if not os.path.exists (restore_target): + raise QubesException("ERROR: the backup directory {0} does not exists".format(restore_target)) + + fp = open(restore_target,'rb') + headers = fp.read(4096) + + is_encrypted = False + + command = subprocess.Popen(['file','-'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + stdout,stderr = command.communicate(headers) + if not stdout.find('gzip compressed data') >= 0: + command = subprocess.Popen(['gpg2','--decrypt'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + stdout,stderr = command.communicate(headers) + if len(stdout) > 0: + headers = stdout + is_encrypted = True + else: + print stderr + raise QubesException("ERROR: unable to decrypt the backup {0}. Is it really encrypted?".format(restore_target)) + + command = subprocess.Popen(['tar', 'xzv', '-O', 'backup_targets'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + vm_list,stderr = command.communicate(headers) + if len(vm_list) <= 0: + print stderr + raise QubesException("ERROR: unable to read the qubes backup file {0}. Is it really a backup?".format(restore_target)) + + vms_in_backup = [] + for vm in vm_list.split("\n"): + vms_in_backup.append(vm.strip("\r\t\n ").split(":")) + + print vms_in_backup + return is_encrypted, vms_in_backup def backup_restore_prepare(backup_dir, options = {}, host_collection = None): + backup_restore_header(backup_dir) # Defaults backup_restore_set_defaults(options) From 836c604473823b374175c6e4b41a4674556e1ce5 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Mon, 19 Aug 2013 16:48:29 +0200 Subject: [PATCH 17/82] backup: reimplemented restore function through an AppVM The VM size is now stored inside the backup specification file in order to compute progress. --- dom0/qvm-core/qubesutils.py | 149 ++++++++++++++++++++++++------ dom0/qvm-tools/qvm-backup-restore | 26 ++++-- 2 files changed, 138 insertions(+), 37 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 248a69d6..e91a3a55 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -906,11 +906,13 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca s += " <-- The VM is running, please shut it down before proceeding with the backup!" there_are_running_vms = True + print_callback(s) + # Build Backup VMs reference file # Format: vm_name:tarball_path\n backup_reference_file = open(os.path.join(qubes_base_dir,"backup_targets"),'w') for vm in vms_for_backup: - backup_reference_file.write(vm.name+":"+vm.dir_path.split(qubes_base_dir)[1]+"\n") + backup_reference_file.write(vm.name+":"+vm.dir_path.split(qubes_base_dir)[1]+":"+str(vm.get_disk_utilization())+"\n") backup_reference_file.flush() backup_reference_file.close() files_to_backup = file_to_backup(backup_reference_file.name,os.stat(backup_reference_file.name).st_size) + files_to_backup @@ -1011,7 +1013,6 @@ def backup_do(base_backup_dir, files_to_backup, progress_callback = None): progress_callback(progress) def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, encrypt=False, appvm=None): - print appvm,base_backup_dir total_backup_sz = 0 for file in files_to_backup: total_backup_sz += file["size"] @@ -1069,6 +1070,26 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e compressor = subprocess.Popen (tar_cmdline,stdout=backup_stdout,stderr=feedback_file) encryptor = None + run_error = wait_backup_feedback(progress_callback, feedback_file, total_backup_sz, compressor, encryptor, vmproc) + + feedback_file.close() + backup_stdout.close() + + # Check returns code of compressor and encryptor and qubes vm retcode + if run_error != None: + try: + if compressor != None: + compressor.terminate() + if encryptor != None: + encryptor.terminate() + if vmproc != None: + vmproc.terminate() + except OSError: + pass + raise QubesException("Failed to perform backup: error with "+run_error) + + +def wait_backup_feedback(progress_callback, feedback_file, total_backup_sz, compressor, encryptor, vmproc): # Get tar backup feedback feedback_file_r = open(feedback_file.name,'r') run_error = None @@ -1104,7 +1125,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e retcode = vmproc.poll() if retcode != None: if retcode != 0: - run_error = "VM "+appvm + run_error = "VM" print vmproc.stdout.read() else: # VM should run until the end @@ -1112,8 +1133,56 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e # Cleanup feedback_file_r.close() + + return run_error + +def restore_vm_dir (backup_dir, src_dir, dst_dir, vm_spec, print_callback=None, error_callback=None, encrypted=False, appvm=None): + + #backup_src_dir = src_dir.replace (qubes_base_dir, backup_dir) + + vmproc = None + if appvm != None: + # Prepare the backup target (Qubes service call) + backup_target = "QUBESRPC qubes.Restore none" + + # does the vm exist? + qvm_collection = QubesVmCollection() + qvm_collection.lock_db_for_reading() + qvm_collection.load() + + vm = qvm_collection.get_vm_by_name(appvm) + if vm is None or vm.qid not in qvm_collection: + raise QubesException("VM {0} does not exist".format(appvm)) + + qvm_collection.unlock_db() + + # If APPVM, STDOUT is a PIPE + vmproc = vm.run(command = backup_target, passio_popen = True) + vmproc.stdin.write(backup_dir.replace("\r","").replace("\n","")+"\n") + backup_stdin = vmproc.stdout + else: + backup_stdin = open(backup_dir,'rb') + + tar_cmdline = ["tar", "-xzv",'--sparse','-C',qubes_base_dir,'--checkpoint=10000'] + + tar_cmdline.append(src_dir.split(os.path.normpath(qubes_base_dir)+"/")[1]) + + #print ("Will backup using command",tar_cmdline) + + import tempfile + feedback_file = tempfile.NamedTemporaryFile() + if encrypted: + encryptor = subprocess.Popen (["gpg2", "--decrypt"], stdin=backup_stdin, stdout=subprocess.PIPE,stderr=subprocess.PIPE) + compressor = subprocess.Popen (tar_cmdline,stdin=encryptor.stdout,stderr=feedback_file,stdout=subprocess.PIPE) + else: + compressor = subprocess.Popen (tar_cmdline,stdin=backup_stdin,stderr=feedback_file,stdout=subprocess.PIPE) + encryptor = None + + run_error = wait_backup_feedback(print_callback, feedback_file, vm_spec["size"], compressor, encryptor, vmproc) + + # Cleanup feedback_file.close() - backup_stdout.close() + backup_stdin.close() # Check returns code of compressor and encryptor and qubes vm retcode if run_error != None: @@ -1142,14 +1211,14 @@ def backup_restore_set_defaults(options): return options -def backup_restore_header(restore_target, progress_callback = None, encrypt=False, appvm=None): +def backup_restore_header(restore_target, encrypt=False, appvm=None): # Simulate dd if=backup_file count=10 | file - # Simulate dd if=backup_file count=10 | gpg2 -d | tar xzv -O # analysis = subprocess.Popen() vmproc = None if appvm != None: # Prepare the backup target (Qubes service call) - backup_target = "QUBESRPC qubes.Backup none" + restore_command = "QUBESRPC qubes.Restore none" # does the vm exist? qvm_collection = QubesVmCollection() @@ -1163,14 +1232,14 @@ def backup_restore_header(restore_target, progress_callback = None, encrypt=Fals qvm_collection.unlock_db() # If APPVM, STDOUT is a PIPE - vmproc = vm.run(command = restore_target, passio_popen = True) + vmproc = vm.run(command = restore_command, passio_popen = True) vmproc.stdin.write(restore_target.replace("\r","").replace("\n","")+"\n") - headers = vmproc.stdout.read(4096) + + headers = vmproc.stdout.read(4096*4) vmproc.terminate() if len(headers) <= 0: - print vmproc.stderr.read() - raise QubesException("ERROR: the backup directory {0} does not exists".format(restore_target)) + raise QubesException("ERROR: unable to read the backup target {0}".format(restore_target)) else: # Create the target directory @@ -1178,7 +1247,7 @@ def backup_restore_header(restore_target, progress_callback = None, encrypt=Fals raise QubesException("ERROR: the backup directory {0} does not exists".format(restore_target)) fp = open(restore_target,'rb') - headers = fp.read(4096) + headers = fp.read(4096*4) is_encrypted = False @@ -1194,25 +1263,31 @@ def backup_restore_header(restore_target, progress_callback = None, encrypt=Fals print stderr raise QubesException("ERROR: unable to decrypt the backup {0}. Is it really encrypted?".format(restore_target)) - command = subprocess.Popen(['tar', 'xzv', '-O', 'backup_targets'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) - vm_list,stderr = command.communicate(headers) - if len(vm_list) <= 0: + command = subprocess.Popen(['tar', 'xzv', '-O', 'backup_targets','qubes.xml'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + headers,stderr = command.communicate(headers) + if len(headers) <= 0: print stderr raise QubesException("ERROR: unable to read the qubes backup file {0}. Is it really a backup?".format(restore_target)) - vms_in_backup = [] - for vm in vm_list.split("\n"): - vms_in_backup.append(vm.strip("\r\t\n ").split(":")) + vms_in_backup = {} + for vm in headers.split("\n"): + match = re.match("^(?P[^:]+):(?P[^:]+):(?P[0-9]+)$",vm,re.MULTILINE) + if match: + item = match.groupdict() + item["size"] = int(item["size"]) + vms_in_backup[item["name"]] = item + headers = headers.replace(vm,"") + else: + break - print vms_in_backup - return is_encrypted, vms_in_backup + return is_encrypted, vms_in_backup, headers -def backup_restore_prepare(backup_dir, options = {}, host_collection = None): - backup_restore_header(backup_dir) +def backup_restore_prepare(backup_dir, backup_content, qubes_xml, options = {}, host_collection = None, encrypt=False, appvm=None): # Defaults backup_restore_set_defaults(options) #### Private functions begin + ''' def is_vm_included_in_backup (backup_dir, vm): if vm.qid == 0: # Dom0 is not included, obviously @@ -1224,7 +1299,12 @@ def backup_restore_prepare(backup_dir, options = {}, host_collection = None): return True else: return False - + ''' + def is_vm_included_in_backup (backup_dir, vm): + for item in backup_content.keys(): + if vm.name == item: + return True + return False def find_template_name(template, replaces): rx_replace = re.compile("(.*):(.*)") for r in replaces: @@ -1235,14 +1315,19 @@ def backup_restore_prepare(backup_dir, options = {}, host_collection = None): return template #### Private functions end - + ''' if not os.path.exists (backup_dir): raise QubesException("The backup directory doesn't exist!") + ''' + backup_collection = QubesVmCollection() + import StringIO + backup_collection.qubes_store_file=StringIO.StringIO(qubes_xml) + + + #backup_collection.lock_db_for_reading() - backup_collection = QubesVmCollection(store_filename = backup_dir + "/qubes.xml") - backup_collection.lock_db_for_reading() backup_collection.load() - + if host_collection is None: host_collection = QubesVmCollection() host_collection.lock_db_for_reading() @@ -1250,9 +1335,11 @@ def backup_restore_prepare(backup_dir, options = {}, host_collection = None): host_collection.unlock_db() backup_vms_list = [vm for vm in backup_collection.values()] + host_vms_list = [vm for vm in host_collection.values()] vms_to_restore = {} + there_are_conflicting_vms = False there_are_missing_templates = False there_are_missing_netvms = False @@ -1325,6 +1412,7 @@ def backup_restore_prepare(backup_dir, options = {}, host_collection = None): vms_to_restore[vm.name]['good-to-go'] = True # ...and dom0 home + # TODO, replace this part of code to handle the new backup format using tar if options['dom0-home'] and os.path.exists(backup_dir + '/dom0-home'): vms_to_restore['dom0'] = {} local_user = grp.getgrnam('qubes').gr_mem[0] @@ -1443,10 +1531,11 @@ def backup_restore_print_summary(restore_info, print_callback = print_stdout): print_callback(s) -def backup_restore_do(backup_dir, restore_info, host_collection = None, print_callback = print_stdout, error_callback = print_stderr): +def backup_restore_do(backup_dir, restore_info, restore_vms, host_collection = None, print_callback = print_stdout, error_callback = print_stderr, encrypted=False, appvm = None ): ### Private functions begin - def restore_vm_dir (backup_dir, src_dir, dst_dir): + ''' + def restore_vm_dir (backup_dir, src_dir, dst_dir, print_callback, error_callback, encrypted, appvm): backup_src_dir = src_dir.replace (qubes_base_dir, backup_dir) @@ -1454,6 +1543,7 @@ def backup_restore_do(backup_dir, restore_info, host_collection = None, print_ca retcode = subprocess.call (["cp", "-rp", backup_src_dir, dst_dir]) if retcode != 0: raise QubesException("*** Error while copying file {0} to {1}".format(backup_src_dir, dest_dir)) + ''' ### Private functions end lock_obtained = False @@ -1495,7 +1585,8 @@ def backup_restore_do(backup_dir, restore_info, host_collection = None, print_ca dir_path=vm.dir_path, template=template, installed_by_rpm=False) - restore_vm_dir (backup_dir, vm.dir_path, os.path.dirname(new_vm.dir_path)); + + restore_vm_dir (backup_dir, vm.dir_path, os.path.dirname(new_vm.dir_path), restore_vms[vm.name], print_callback, error_callback, encrypted, appvm) new_vm.verify_files() except Exception as err: diff --git a/dom0/qvm-tools/qvm-backup-restore b/dom0/qvm-tools/qvm-backup-restore index 6b1a590d..05c56374 100755 --- a/dom0/qvm-tools/qvm-backup-restore +++ b/dom0/qvm-tools/qvm-backup-restore @@ -22,7 +22,8 @@ from qubes.qubes import QubesVmCollection from qubes.qubes import QubesException -from qubes.qubesutils import backup_restore_prepare +from qubes.qubesutils import backup_restore_header +from qubes.qubesutils import backup_restore_prepare from qubes.qubesutils import backup_restore_print_summary from qubes.qubesutils import backup_restore_do from optparse import OptionParser @@ -58,6 +59,12 @@ def main(): parser.add_option ("--ignore-username-mismatch", action="store_true", dest="ignore_username_mismatch", default=False, help="Ignore dom0 username mismatch while restoring homedir") + parser.add_option ("-d", "--dest-vm", action="store", dest="appvm", + help="The AppVM to send backups to") + + parser.add_option ("-e", "--encrypted", action="store_true", dest="decrypt", default=False, + help="The backup is encrypted") + (options, args) = parser.parse_args () if (len (args) != 1): @@ -66,9 +73,9 @@ def main(): backup_dir = args[0] - if not os.path.exists (backup_dir): - print >> sys.stderr, "The backup directory doesn't exist!" - exit(1) + #if not os.path.exists (backup_dir): + # print >> sys.stderr, "The backup directory doesn't exist!" + # exit(1) host_collection = QubesVmCollection() host_collection.lock_db_for_writing() @@ -87,9 +94,13 @@ def main(): if options.exclude: restore_options['exclude'] = options.exclude + + print >> sys.stderr, "Checking backup content..." + encrypted, vms, qubes_xml = backup_restore_header(backup_dir, options.decrypt, appvm=options.appvm) + restore_info = None try: - restore_info = backup_restore_prepare(backup_dir, options=restore_options, host_collection=host_collection) + restore_info = backup_restore_prepare(backup_dir, vms, qubes_xml, options=restore_options, host_collection=host_collection, encrypt=encrypted, appvm=options.appvm) except QubesException as e: print >> sys.stderr, "ERROR: %s" % str(e) exit(1) @@ -113,8 +124,6 @@ def main(): if 'username-mismatch' in vm_info.keys(): dom0_username_mismatch = True - print - if os.geteuid() == 0: print >> sys.stderr, "*** Running this tool as root is strongly discouraged, this will lead you in permissions problems." if options.force_root: @@ -179,7 +188,8 @@ def main(): if not (prompt == "y" or prompt == "Y"): exit (0) - backup_restore_do(backup_dir, restore_info, host_collection=host_collection) + + backup_restore_do(backup_dir, restore_info, vms, host_collection=host_collection, encrypted=encrypted, appvm=options.appvm) host_collection.unlock_db() From 89fde55cd954232a58394a34bc33f00c57272407 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Mon, 19 Aug 2013 16:50:46 +0200 Subject: [PATCH 18/82] backup: Added rpc restoration file --- qubes_rpc/qubes.Restore | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 qubes_rpc/qubes.Restore diff --git a/qubes_rpc/qubes.Restore b/qubes_rpc/qubes.Restore new file mode 100644 index 00000000..dfea1448 --- /dev/null +++ b/qubes_rpc/qubes.Restore @@ -0,0 +1,23 @@ +echo Starting Restorecopy >2 +read args +echo Arguments: $args >2 +if [ -f "$args" ] ; then + echo "Performing restore from backup file $args" >2 + TARGET="$args" + echo "Copying $TARGET to STDOUT" >2 + cat $TARGET +else + echo "Checking if arguments is matching a command" >2 + COMMAND=`echo $args | cut -d ' ' -f 1` + TYPE=`type -t $COMMAND` + if [ "$TYPE" == "file" ] ; then + echo "Redirecting $args to STDOUT" >2 + # Parsing args to handle quotes correctly + # Dangerous method if args are uncontrolled + eval "set -- $args" + $@ + else + echo "Invalid command $COMMAND" >2 + exit 1 + fi +fi From 5edca4ac90b260fc72dd95bc02570208c672ba23 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Tue, 10 Sep 2013 09:20:49 +0200 Subject: [PATCH 19/82] backup: code cleanup --- dom0/qvm-core/qubesutils.py | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index e91a3a55..8f831743 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1249,16 +1249,11 @@ def backup_restore_header(restore_target, encrypt=False, appvm=None): fp = open(restore_target,'rb') headers = fp.read(4096*4) - is_encrypted = False - - command = subprocess.Popen(['file','-'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) - stdout,stderr = command.communicate(headers) - if not stdout.find('gzip compressed data') >= 0: + if encrypt: command = subprocess.Popen(['gpg2','--decrypt'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) stdout,stderr = command.communicate(headers) if len(stdout) > 0: headers = stdout - is_encrypted = True else: print stderr raise QubesException("ERROR: unable to decrypt the backup {0}. Is it really encrypted?".format(restore_target)) @@ -1280,26 +1275,13 @@ def backup_restore_header(restore_target, encrypt=False, appvm=None): else: break - return is_encrypted, vms_in_backup, headers + return vms_in_backup, headers def backup_restore_prepare(backup_dir, backup_content, qubes_xml, options = {}, host_collection = None, encrypt=False, appvm=None): # Defaults backup_restore_set_defaults(options) #### Private functions begin - ''' - def is_vm_included_in_backup (backup_dir, vm): - if vm.qid == 0: - # Dom0 is not included, obviously - return False - - backup_vm_dir_path = vm.dir_path.replace (qubes_base_dir, backup_dir) - - if os.path.exists (backup_vm_dir_path): - return True - else: - return False - ''' def is_vm_included_in_backup (backup_dir, vm): for item in backup_content.keys(): if vm.name == item: @@ -1313,12 +1295,8 @@ def backup_restore_prepare(backup_dir, backup_content, qubes_xml, options = {}, return m.group(2) return template - #### Private functions end - ''' - if not os.path.exists (backup_dir): - raise QubesException("The backup directory doesn't exist!") - ''' + backup_collection = QubesVmCollection() import StringIO backup_collection.qubes_store_file=StringIO.StringIO(qubes_xml) From 4ae4bdc45258f681c71b3cbbb19cc1954927d80f Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Tue, 10 Sep 2013 09:22:35 +0200 Subject: [PATCH 20/82] backup: implemented backup mecanism using tar_sparse+encryption+hmac generation --- dom0/qvm-core/qubesutils.py | 157 +++++++++++++++++++++++++----------- 1 file changed, 110 insertions(+), 47 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 8f831743..0d308e5a 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1054,73 +1054,139 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e progress = blocks_backedup * 11 / total_backup_sz progress_callback(progress) - tar_cmdline = ["tar", "-PczO",'--sparse','-C',qubes_base_dir,'--checkpoint=10000'] - - for filename in files_to_backup: - tar_cmdline.append(filename["path"].split(os.path.normpath(qubes_base_dir)+"/")[1]) - #print ("Will backup using command",tar_cmdline) - import tempfile feedback_file = tempfile.NamedTemporaryFile() - #print feedback_file - if encrypt: - compressor = subprocess.Popen (tar_cmdline,stdout=subprocess.PIPE,stderr=feedback_file) - encryptor = subprocess.Popen (["gpg2", "-c", "--force-mdc", "-o-"], stdin=compressor.stdout, stdout=backup_stdout) - else: - compressor = subprocess.Popen (tar_cmdline,stdout=backup_stdout,stderr=feedback_file) - encryptor = None - run_error = wait_backup_feedback(progress_callback, feedback_file, total_backup_sz, compressor, encryptor, vmproc) + for filename in files_to_backup: + print "Backing up",filename + tar_cmdline = ["tar", "-PcO",'--sparse','-C',qubes_base_dir, + filename["path"].split(os.path.normpath(qubes_base_dir)+"/")[1] + ] - feedback_file.close() + # Prepare all subprocesses with the right stdin, stdout + if encrypt: + # Tips: Popen(bufsize=0) + compressor = subprocess.Popen (tar_cmdline,stdout=subprocess.PIPE) + + encryptor = subprocess.Popen (["openssl", "enc", "-e", "-k", "azerty"], stdin=compressor.stdout, stdout=subprocess.PIPE) + hmac = subprocess.Popen (["openssl", "dgst", "-hmac", "azerty"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) + + streamproc = encryptor + addproc = compressor + else: + compressor = subprocess.Popen (tar_cmdline,stdout=subprocess.PIPE) + + encryptor = None + hmac = subprocess.Popen (["openssl", "dgst", "-hmac", "azerty"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) + + streamproc = compressor + addproc = None + + # Wait for compressor (tar) process to finish or for any error of other subprocesses + run_error = wait_backup_feedback(progress_callback, streamproc, backup_stdout, total_backup_sz, hmac=hmac, vmproc=vmproc, addproc=addproc) + if len(run_error) > 0: + raise QubesException("Failed to perform backup: error with "+run_error) + + # Wait for all remaining subprocess to finish + if addproc: + addproc.wait() + print "Addproc:",addproc.poll() + + streamproc.wait() + print "Streamproc:",streamproc.poll() + + hmac.stdin.close() + hmac.wait() + print "HMAC:",hmac.poll() + + # Write HMAC data next to the original file + hmac_data = hmac.stdout.read() + print "Writing hmac to",filename['path']+".hmac" + hmac_file = open(filename['path']+".hmac",'w') + hmac_file.write(hmac_data) + hmac_file.flush() + hmac_file.close() + + # Send the hmac file to the backup target + tar_cmdline[-1] += ".hmac" + print tar_cmdline + streamproc = subprocess.Popen(tar_cmdline,stdout=subprocess.PIPE) + run_error = wait_backup_feedback(progress_callback, streamproc, backup_stdout, total_backup_sz, vmproc=vmproc) + if len(run_error) > 0: + raise QubesException("Failed to perform backup: error with "+run_error) + + streamproc.wait() + print "HMAC sent:",streamproc.poll() + + # Close the backup target and wait for it to finish backup_stdout.close() - - # Check returns code of compressor and encryptor and qubes vm retcode - if run_error != None: - try: - if compressor != None: - compressor.terminate() - if encryptor != None: - encryptor.terminate() - if vmproc != None: - vmproc.terminate() - except OSError: - pass - raise QubesException("Failed to perform backup: error with "+run_error) + if vmproc: + print "VMProc1:",vmproc.poll() + vmproc.wait() + print "VMProc2:",vmproc.poll() -def wait_backup_feedback(progress_callback, feedback_file, total_backup_sz, compressor, encryptor, vmproc): - # Get tar backup feedback - feedback_file_r = open(feedback_file.name,'r') +def wait_backup_feedback(progress_callback, streamproc, backup_target, total_backup_sz, hmac=None, vmproc=None, addproc=None, remove_trailing_bytes=0): + + buffer_size = 4096 + run_error = None run_count = 1 + blocks_backedup = 0 while run_count > 0 and run_error == None: - time.sleep(1) - match = re.search("tar: [^0-9]+([0-9]+)",feedback_file_r.readline()) - if match: - blocks_backedup = int(match.group(1)) - progress = blocks_backedup * 11.024 * 1024 / total_backup_sz - #print blocks_backedup,total_backup_sz,progress - progress_callback(round(progress*100,2)) + buffer = streamproc.stdout.read(buffer_size) + #print "Read",len(buffer) + + blocks_backedup += len(buffer) + + progress = blocks_backedup / float(total_backup_sz) + progress_callback(round(progress*100,2)) run_count = 0 - if compressor: - retcode=compressor.poll() + if hmac: + retcode=hmac.poll() if retcode != None: if retcode != 0: - run_error = "compressor" + run_error = "hmac" else: run_count += 1 - if encryptor: - retcode=encryptor.poll() + if addproc: + retcode=addproc.poll() if retcode != None: if retcode != 0: - run_error = "encryptor" + run_error = "addproc" else: run_count += 1 + retcode=streamproc.poll() + if retcode != None: + if retcode != 0: + run_error = "streamproc" + elif retcode == 0 and len(buffer) <= 0: + return "" + else: + if remove_trailing_bytes > 0: + print buffer.encode("hex") + buffer = buffer[:-remove_trailing_bytes] + print buffer.encode("hex") + + backup_target.write(buffer) + + if hmac: + hmac.stdin.write(buffer) + + run_count += 1 + else: + # Process still running + backup_target.write(buffer) + + if hmac: + hmac.stdin.write(buffer) + + run_count += 1 + if vmproc: retcode = vmproc.poll() if retcode != None: @@ -1131,9 +1197,6 @@ def wait_backup_feedback(progress_callback, feedback_file, total_backup_sz, comp # VM should run until the end pass - # Cleanup - feedback_file_r.close() - return run_error def restore_vm_dir (backup_dir, src_dir, dst_dir, vm_spec, print_callback=None, error_callback=None, encrypted=False, appvm=None): From a85f3a7d8e93122298b262ac241722dd61b69cd5 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Tue, 10 Sep 2013 09:24:25 +0200 Subject: [PATCH 21/82] backup: introduced a second tar pass to send encrypted data to an AppVM The backup process is now tar_sparse | encrypt | hmac | tar | appvm --- dom0/qvm-core/qubesutils.py | 171 +++++++++++++++++++++++------------- 1 file changed, 112 insertions(+), 59 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 0d308e5a..dcf510d6 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -915,7 +915,7 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca backup_reference_file.write(vm.name+":"+vm.dir_path.split(qubes_base_dir)[1]+":"+str(vm.get_disk_utilization())+"\n") backup_reference_file.flush() backup_reference_file.close() - files_to_backup = file_to_backup(backup_reference_file.name,os.stat(backup_reference_file.name).st_size) + files_to_backup + #files_to_backup = file_to_backup(backup_reference_file.name,os.stat(backup_reference_file.name).st_size) + files_to_backup # Dom0 user home if not 'dom0' in exclude_list: @@ -1056,76 +1056,125 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e import tempfile feedback_file = tempfile.NamedTemporaryFile() + backup_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/backup_") + + # Tar with tapelength does not deals well with stdout (close stdout between two tapes) + # For this reason, we will use named pipes instead + print "Working in",backup_tmpdir + + backup_pipe = os.path.join(backup_tmpdir,"backup_pipe") + print "Creating pipe in:",backup_pipe + print os.mkfifo(backup_pipe) + + print "Will backup:",files_to_backup for filename in files_to_backup: print "Backing up",filename - tar_cmdline = ["tar", "-PcO",'--sparse','-C',qubes_base_dir, + + backup_tempfile = os.path.join(backup_tmpdir,filename["path"].split(os.path.normpath(qubes_base_dir)+"/")[1]) + print "Using temporary location:",backup_tempfile + + # Ensure the temporary directory exists + + if not os.path.isdir(os.path.dirname(backup_tempfile)): + os.makedirs(os.path.dirname(backup_tempfile)) + + tar_cmdline = ["tar", "-Pc", "-f", backup_pipe,'--sparse','--tape-length',str(1000000),'-C',qubes_base_dir, filename["path"].split(os.path.normpath(qubes_base_dir)+"/")[1] ] - # Prepare all subprocesses with the right stdin, stdout - if encrypt: - # Tips: Popen(bufsize=0) - compressor = subprocess.Popen (tar_cmdline,stdout=subprocess.PIPE) + print " ".join(tar_cmdline) - encryptor = subprocess.Popen (["openssl", "enc", "-e", "-k", "azerty"], stdin=compressor.stdout, stdout=subprocess.PIPE) - hmac = subprocess.Popen (["openssl", "dgst", "-hmac", "azerty"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) - - streamproc = encryptor - addproc = compressor - else: - compressor = subprocess.Popen (tar_cmdline,stdout=subprocess.PIPE) - - encryptor = None - hmac = subprocess.Popen (["openssl", "dgst", "-hmac", "azerty"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) - - streamproc = compressor - addproc = None + # Tips: Popen(bufsize=0) + # Pipe: tar-sparse | encryptor [| hmac] | tar | backup_target + # Pipe: tar-sparse [| hmac] | tar | backup_target + tar_sparse = subprocess.Popen (tar_cmdline,stdin=subprocess.PIPE) # Wait for compressor (tar) process to finish or for any error of other subprocesses - run_error = wait_backup_feedback(progress_callback, streamproc, backup_stdout, total_backup_sz, hmac=hmac, vmproc=vmproc, addproc=addproc) - if len(run_error) > 0: - raise QubesException("Failed to perform backup: error with "+run_error) + i=0 + run_error = "paused" + running = [] + while run_error == "paused": + # Start encrypt + pipe = open(backup_pipe,'rb') + # If no cipher is provided, the data is forwarded unencrypted !!! + encryptor = subprocess.Popen (["openssl", "enc", "-e", "-aes-256-cbc", "-pass", "pass:azerty"], stdin=pipe, stdout=subprocess.PIPE) + + # Start HMAC + hmac = subprocess.Popen (["openssl", "dgst", "-hmac", "azerty"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) + + # Prepare a first chunk + chunkfile = backup_tempfile + "." + "%03d" % i + i += 1 + chunkfile_p = open(chunkfile,'wb') + run_error = wait_backup_feedback(progress_callback, encryptor, chunkfile_p, total_backup_sz, hmac=hmac, vmproc=vmproc, addproc=tar_sparse) + chunkfile_p.close() + + print "Wait_backup_feedback returned:",run_error + + if len(run_error) > 0: + raise QubesException("Failed to perform backup: error with "+run_error) + + # Send the chunk to the backup target + tar_final_cmd = ["tar", "-cO", "-C", backup_tmpdir, chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]] + final_proc = subprocess.Popen (tar_final_cmd, stdin=subprocess.PIPE, stdout=backup_stdout) + final_proc.wait() + + # Close HMAC + hmac.stdin.close() + hmac.wait() + print "HMAC:",hmac.poll() + + # Write HMAC data next to the chunk file + hmac_data = hmac.stdout.read() + print "Writing hmac to",chunkfile+".hmac" + hmac_file = open(chunkfile+".hmac",'w') + hmac_file.write(hmac_data) + hmac_file.flush() + hmac_file.close() + + # Send the HMAC to the backup target + tar_final_cmd = ["tar", "-cO", "-C", backup_tmpdir, chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]+".hmac"] + final_proc = subprocess.Popen (tar_final_cmd, stdin=subprocess.PIPE, stdout=backup_stdout) + final_proc.wait() + + if tar_sparse.poll() == None: + # Release the next chunk + print "Release next chunk for process:",tar_sparse.poll() + #tar_sparse.stdout = subprocess.PIPE + tar_sparse.stdin.write("\n") + run_error="paused" + else: + print "Finished tar sparse with error",tar_sparse.poll() + # Wait for all remaining subprocess to finish - if addproc: - addproc.wait() - print "Addproc:",addproc.poll() + #if addproc: + # addproc.wait() + # print "Addproc:",addproc.poll() - streamproc.wait() - print "Streamproc:",streamproc.poll() + #streamproc.wait() + #print "Streamproc:",streamproc.poll() - hmac.stdin.close() - hmac.wait() - print "HMAC:",hmac.poll() - - # Write HMAC data next to the original file - hmac_data = hmac.stdout.read() - print "Writing hmac to",filename['path']+".hmac" - hmac_file = open(filename['path']+".hmac",'w') - hmac_file.write(hmac_data) - hmac_file.flush() - hmac_file.close() - - # Send the hmac file to the backup target - tar_cmdline[-1] += ".hmac" - print tar_cmdline - streamproc = subprocess.Popen(tar_cmdline,stdout=subprocess.PIPE) - run_error = wait_backup_feedback(progress_callback, streamproc, backup_stdout, total_backup_sz, vmproc=vmproc) - if len(run_error) > 0: - raise QubesException("Failed to perform backup: error with "+run_error) - - streamproc.wait() - print "HMAC sent:",streamproc.poll() + #streamproc.wait() # Close the backup target and wait for it to finish - backup_stdout.close() + #backup_stdout.close() if vmproc: print "VMProc1:",vmproc.poll() - vmproc.wait() - print "VMProc2:",vmproc.poll() - + print "Sparse1:",tar_sparse.poll() + vmproc.stdin.close() +''' +' Wait for backup chunk to finish +' - Monitor all the processes (streamproc, hmac, vmproc, addproc) for errors +' - Copy stdout of streamproc to backup_target and hmac stdin if available +' - Compute progress based on total_backup_sz and send progress to progress_callback function +' - Returns if +' - one of the monitored processes error out (streamproc, hmac, vmproc, addproc), along with the processe that failed +' - all of the monitored processes except vmproc finished successfully (vmproc termination is controlled by the python script) +' - streamproc does not delivers any data anymore (return with the error "paused") +''' def wait_backup_feedback(progress_callback, streamproc, backup_target, total_backup_sz, hmac=None, vmproc=None, addproc=None, remove_trailing_bytes=0): buffer_size = 4096 @@ -1136,12 +1185,11 @@ def wait_backup_feedback(progress_callback, streamproc, backup_target, total_bac while run_count > 0 and run_error == None: buffer = streamproc.stdout.read(buffer_size) - #print "Read",len(buffer) blocks_backedup += len(buffer) progress = blocks_backedup / float(total_backup_sz) - progress_callback(round(progress*100,2)) + #progress_callback(round(progress*100,2)) run_count = 0 if hmac: @@ -1154,6 +1202,7 @@ def wait_backup_feedback(progress_callback, streamproc, backup_target, total_bac if addproc: retcode=addproc.poll() + print "Tar proc status:",retcode if retcode != None: if retcode != 0: run_error = "addproc" @@ -1164,13 +1213,16 @@ def wait_backup_feedback(progress_callback, streamproc, backup_target, total_bac if retcode != None: if retcode != 0: run_error = "streamproc" + print "INFO: run error" elif retcode == 0 and len(buffer) <= 0: + print "INFO: no data" return "" else: - if remove_trailing_bytes > 0: - print buffer.encode("hex") - buffer = buffer[:-remove_trailing_bytes] - print buffer.encode("hex") + print "INFO: last packet" + #if remove_trailing_bytes > 0: + # print buffer.encode("hex") + # buffer = buffer[:-remove_trailing_bytes] + # print buffer.encode("hex") backup_target.write(buffer) @@ -1179,6 +1231,7 @@ def wait_backup_feedback(progress_callback, streamproc, backup_target, total_bac run_count += 1 else: + print "Process running:",len(buffer) # Process still running backup_target.write(buffer) From 23065f6fa04206009e15760ababf5461c5540f00 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Tue, 10 Sep 2013 09:25:44 +0200 Subject: [PATCH 22/82] backup: use a thread to send data to AppVM in parallel to tar main operations. Additionnally, temporary files are removed once data has been sent --- dom0/qvm-core/qubesutils.py | 67 +++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index dcf510d6..afcec82d 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1068,6 +1068,40 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e print "Will backup:",files_to_backup + # Setup worker to send encrypted data chunks to the backup_target + from multiprocessing import Queue,Process + class Send_Worker(Process): + def __init__(self,queue,base_dir,backup_stdout): + super(Send_Worker, self).__init__() + self.queue = queue + self.base_dir = base_dir + self.backup_stdout = backup_stdout + + def run(self): + print "Started sending thread" + + print "Moving to temporary dir",self.base_dir + os.chdir(self.base_dir) + + for filename in iter(self.queue.get,None): + if filename == "FINISHED": + break + + print "Sending file",filename + tar_final_cmd = ["tar", "-cO", "-C", self.base_dir, filename] + final_proc = subprocess.Popen (tar_final_cmd, stdin=subprocess.PIPE, stdout=self.backup_stdout) + final_proc.wait() + + # Delete the file as we don't need it anymore + print "Removing file",filename + os.remove(filename) + + print "Finished sending thread" + + to_send = Queue() + send_proc = Send_Worker(to_send, backup_tmpdir, backup_stdout) + send_proc.start() + for filename in files_to_backup: print "Backing up",filename @@ -1117,9 +1151,10 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e raise QubesException("Failed to perform backup: error with "+run_error) # Send the chunk to the backup target - tar_final_cmd = ["tar", "-cO", "-C", backup_tmpdir, chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]] - final_proc = subprocess.Popen (tar_final_cmd, stdin=subprocess.PIPE, stdout=backup_stdout) - final_proc.wait() + to_send.put(chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]) + #tar_final_cmd = ["tar", "-cO", "-C", backup_tmpdir, chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]] + #final_proc = subprocess.Popen (tar_final_cmd, stdin=subprocess.PIPE, stdout=backup_stdout) + #final_proc.wait() # Close HMAC hmac.stdin.close() @@ -1135,9 +1170,10 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e hmac_file.close() # Send the HMAC to the backup target - tar_final_cmd = ["tar", "-cO", "-C", backup_tmpdir, chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]+".hmac"] - final_proc = subprocess.Popen (tar_final_cmd, stdin=subprocess.PIPE, stdout=backup_stdout) - final_proc.wait() + to_send.put(chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]+".hmac") + #tar_final_cmd = ["tar", "-cO", "-C", backup_tmpdir, chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]+".hmac"] + #final_proc = subprocess.Popen (tar_final_cmd, stdin=subprocess.PIPE, stdout=backup_stdout) + #final_proc.wait() if tar_sparse.poll() == None: # Release the next chunk @@ -1148,18 +1184,13 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e else: print "Finished tar sparse with error",tar_sparse.poll() - # Wait for all remaining subprocess to finish - #if addproc: - # addproc.wait() - # print "Addproc:",addproc.poll() - - #streamproc.wait() - #print "Streamproc:",streamproc.poll() - - #streamproc.wait() # Close the backup target and wait for it to finish #backup_stdout.close() + + to_send.put("FINISHED") + send_proc.join() + if vmproc: print "VMProc1:",vmproc.poll() print "Sparse1:",tar_sparse.poll() @@ -1189,7 +1220,7 @@ def wait_backup_feedback(progress_callback, streamproc, backup_target, total_bac blocks_backedup += len(buffer) progress = blocks_backedup / float(total_backup_sz) - #progress_callback(round(progress*100,2)) + progress_callback(round(progress*100,2)) run_count = 0 if hmac: @@ -1202,7 +1233,7 @@ def wait_backup_feedback(progress_callback, streamproc, backup_target, total_bac if addproc: retcode=addproc.poll() - print "Tar proc status:",retcode + #print "Tar proc status:",retcode if retcode != None: if retcode != 0: run_error = "addproc" @@ -1231,7 +1262,7 @@ def wait_backup_feedback(progress_callback, streamproc, backup_target, total_bac run_count += 1 else: - print "Process running:",len(buffer) + #print "Process running:",len(buffer) # Process still running backup_target.write(buffer) From 361741b8aaf6244e3a4d01f6b100d577aaa2a424 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Tue, 10 Sep 2013 09:27:51 +0200 Subject: [PATCH 23/82] backup: multiple fixes for the backup process, including non-encrypted backups - Ensure backup without encryption is working - Implemented progress feedback through a global variable - Ask user for a passphrase used for encryption or for verification --- dom0/qvm-core/qubesutils.py | 122 +++++++++++++++++++++--------------- 1 file changed, 71 insertions(+), 51 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index afcec82d..454b2719 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1049,6 +1049,8 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e # If not APPVM, STDOUT is a local file backup_stdout = open(backup_target,'wb') + passphrase = raw_input("Please enter the pass phrase that will be used to encrypt/verify the backup:\n") + passphrase = passphrase.replace("\r","").replace("\n","") blocks_backedup = 0 progress = blocks_backedup * 11 / total_backup_sz @@ -1088,7 +1090,8 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e break print "Sending file",filename - tar_final_cmd = ["tar", "-cO", "-C", self.base_dir, filename] + # This tar used for sending data out need to be as simple, as simple, as featureless as possible. It will not be verified before untaring. + tar_final_cmd = ["tar", "-cO", "--posix", "-C", self.base_dir, filename] final_proc = subprocess.Popen (tar_final_cmd, stdin=subprocess.PIPE, stdout=self.backup_stdout) final_proc.wait() @@ -1098,6 +1101,14 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e print "Finished sending thread" + global blocks_backedup + blocks_backedup = 0 + def compute_progress(new_size, total_backup_sz): + global blocks_backedup + blocks_backedup += new_size + progress = blocks_backedup / float(total_backup_sz) + progress_callback(round(progress*100,2)) + to_send = Queue() send_proc = Send_Worker(to_send, backup_tmpdir, backup_stdout) send_proc.start() @@ -1113,6 +1124,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e if not os.path.isdir(os.path.dirname(backup_tempfile)): os.makedirs(os.path.dirname(backup_tempfile)) + # The first tar cmd can use any complex feature as we want. Files will be verified before untaring this. tar_cmdline = ["tar", "-Pc", "-f", backup_pipe,'--sparse','--tape-length',str(1000000),'-C',qubes_base_dir, filename["path"].split(os.path.normpath(qubes_base_dir)+"/")[1] ] @@ -1130,19 +1142,25 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e running = [] while run_error == "paused": - # Start encrypt pipe = open(backup_pipe,'rb') - # If no cipher is provided, the data is forwarded unencrypted !!! - encryptor = subprocess.Popen (["openssl", "enc", "-e", "-aes-256-cbc", "-pass", "pass:azerty"], stdin=pipe, stdout=subprocess.PIPE) # Start HMAC - hmac = subprocess.Popen (["openssl", "dgst", "-hmac", "azerty"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) + hmac = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], stdin=subprocess.PIPE, stdout=subprocess.PIPE) # Prepare a first chunk chunkfile = backup_tempfile + "." + "%03d" % i i += 1 chunkfile_p = open(chunkfile,'wb') - run_error = wait_backup_feedback(progress_callback, encryptor, chunkfile_p, total_backup_sz, hmac=hmac, vmproc=vmproc, addproc=tar_sparse) + + if encrypt: + # Start encrypt + # If no cipher is provided, the data is forwarded unencrypted !!! + # Also note that the + encryptor = subprocess.Popen (["openssl", "enc", "-e", "-aes-256-cbc", "-pass", "pass:"+passphrase], stdin=pipe, stdout=subprocess.PIPE) + run_error = wait_backup_feedback(compute_progress, encryptor.stdout, encryptor, chunkfile_p, total_backup_sz, hmac=hmac, vmproc=vmproc, addproc=tar_sparse) + else: + run_error = wait_backup_feedback(compute_progress, pipe, None, chunkfile_p, total_backup_sz, hmac=hmac, vmproc=vmproc, addproc=tar_sparse) + chunkfile_p.close() print "Wait_backup_feedback returned:",run_error @@ -1152,9 +1170,6 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e # Send the chunk to the backup target to_send.put(chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]) - #tar_final_cmd = ["tar", "-cO", "-C", backup_tmpdir, chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]] - #final_proc = subprocess.Popen (tar_final_cmd, stdin=subprocess.PIPE, stdout=backup_stdout) - #final_proc.wait() # Close HMAC hmac.stdin.close() @@ -1171,10 +1186,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e # Send the HMAC to the backup target to_send.put(chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]+".hmac") - #tar_final_cmd = ["tar", "-cO", "-C", backup_tmpdir, chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]+".hmac"] - #final_proc = subprocess.Popen (tar_final_cmd, stdin=subprocess.PIPE, stdout=backup_stdout) - #final_proc.wait() - + if tar_sparse.poll() == None: # Release the next chunk print "Release next chunk for process:",tar_sparse.poll() @@ -1184,7 +1196,8 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e else: print "Finished tar sparse with error",tar_sparse.poll() - + pipe.close() + # Close the backup target and wait for it to finish #backup_stdout.close() @@ -1206,7 +1219,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e ' - all of the monitored processes except vmproc finished successfully (vmproc termination is controlled by the python script) ' - streamproc does not delivers any data anymore (return with the error "paused") ''' -def wait_backup_feedback(progress_callback, streamproc, backup_target, total_backup_sz, hmac=None, vmproc=None, addproc=None, remove_trailing_bytes=0): +def wait_backup_feedback(progress_callback, in_stream, streamproc, backup_target, total_backup_sz, hmac=None, vmproc=None, addproc=None, remove_trailing_bytes=0): buffer_size = 4096 @@ -1215,12 +1228,9 @@ def wait_backup_feedback(progress_callback, streamproc, backup_target, total_bac blocks_backedup = 0 while run_count > 0 and run_error == None: - buffer = streamproc.stdout.read(buffer_size) + buffer = in_stream.read(buffer_size) - blocks_backedup += len(buffer) - - progress = blocks_backedup / float(total_backup_sz) - progress_callback(round(progress*100,2)) + progress_callback(len(buffer),total_backup_sz) run_count = 0 if hmac: @@ -1240,37 +1250,6 @@ def wait_backup_feedback(progress_callback, streamproc, backup_target, total_bac else: run_count += 1 - retcode=streamproc.poll() - if retcode != None: - if retcode != 0: - run_error = "streamproc" - print "INFO: run error" - elif retcode == 0 and len(buffer) <= 0: - print "INFO: no data" - return "" - else: - print "INFO: last packet" - #if remove_trailing_bytes > 0: - # print buffer.encode("hex") - # buffer = buffer[:-remove_trailing_bytes] - # print buffer.encode("hex") - - backup_target.write(buffer) - - if hmac: - hmac.stdin.write(buffer) - - run_count += 1 - else: - #print "Process running:",len(buffer) - # Process still running - backup_target.write(buffer) - - if hmac: - hmac.stdin.write(buffer) - - run_count += 1 - if vmproc: retcode = vmproc.poll() if retcode != None: @@ -1281,6 +1260,47 @@ def wait_backup_feedback(progress_callback, streamproc, backup_target, total_bac # VM should run until the end pass + if streamproc: + retcode=streamproc.poll() + if retcode != None: + if retcode != 0: + run_error = "streamproc" + elif retcode == 0 and len(buffer) <= 0: + return "" + else: + #print "INFO: last packet" + #if remove_trailing_bytes > 0: + # print buffer.encode("hex") + # buffer = buffer[:-remove_trailing_bytes] + # print buffer.encode("hex") + + backup_target.write(buffer) + + if hmac: + hmac.stdin.write(buffer) + + run_count += 1 + else: + #print "Process running:",len(buffer) + # Process still running + backup_target.write(buffer) + + if hmac: + hmac.stdin.write(buffer) + + run_count += 1 + + else: + if len(buffer) <= 0: + return "" + else: + backup_target.write(buffer) + + if hmac: + hmac.stdin.write(buffer) + + + return run_error def restore_vm_dir (backup_dir, src_dir, dst_dir, vm_spec, print_callback=None, error_callback=None, encrypted=False, appvm=None): From c805ff6aeb5afe517e5694ea06418e2e44bb148e Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Wed, 25 Sep 2013 09:51:17 +0200 Subject: [PATCH 24/82] backup: implement header restoration for the new backup format --- dom0/qvm-core/qubes.py | 6 +- dom0/qvm-core/qubesutils.py | 141 ++++++++++++++++-------------- dom0/qvm-tools/qvm-backup-restore | 9 +- 3 files changed, 87 insertions(+), 69 deletions(-) diff --git a/dom0/qvm-core/qubes.py b/dom0/qvm-core/qubes.py index 6edaebfd..8a404529 100755 --- a/dom0/qvm-core/qubes.py +++ b/dom0/qvm-core/qubes.py @@ -285,6 +285,9 @@ class QubesVm(object): # for backward compatibility (or another rare case): kernel=None -> kernel in VM dir 'self.dir_path + "/" + default_kernels_subdir' }, "_start_guid_first": { 'eval': 'False' }, + "backup_content" : { 'default': False }, + "backup_size" : { 'default': 0, "eval": "int(value)" }, + "backup_path" : { 'default': "" }, } ### Mark attrs for XML inclusion @@ -293,7 +296,8 @@ class QubesVm(object): 'uses_default_kernel', 'kernel', 'uses_default_kernelopts',\ 'kernelopts', 'services', 'installed_by_rpm',\ 'uses_default_netvm', 'include_in_backups', 'debug',\ - 'default_user', 'qrexec_timeout' ]: + 'default_user', 'qrexec_timeout', + 'backup_content', 'backup_size', 'backup_path' ]: attrs[prop]['save'] = 'str(self.%s)' % prop # Simple paths for prop in ['conf_file', 'root_img', 'volatile_img', 'private_img']: diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 454b2719..ec02e369 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -789,12 +789,11 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca if exclude_list is None: exclude_list = [] + qvm_collection = None if vms_list is None: qvm_collection = QubesVmCollection() - qvm_collection.lock_db_for_reading() + qvm_collection.lock_db_for_writing() qvm_collection.load() - # FIXME: should be after backup completed - qvm_collection.unlock_db() all_vms = [vm for vm in qvm_collection.values()] appvms_to_backup = [vm for vm in all_vms if vm.is_appvm() and not vm.internal] @@ -908,14 +907,13 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca print_callback(s) - # Build Backup VMs reference file - # Format: vm_name:tarball_path\n - backup_reference_file = open(os.path.join(qubes_base_dir,"backup_targets"),'w') - for vm in vms_for_backup: - backup_reference_file.write(vm.name+":"+vm.dir_path.split(qubes_base_dir)[1]+":"+str(vm.get_disk_utilization())+"\n") - backup_reference_file.flush() - backup_reference_file.close() - #files_to_backup = file_to_backup(backup_reference_file.name,os.stat(backup_reference_file.name).st_size) + files_to_backup + vm.backup_content = True + vm.backup_size = vm.get_disk_utilization() + vm.backup_path = vm.dir_path.split(qubes_base_dir)[1] + + qvm_collection.save() + # FIXME: should be after backup completed + qvm_collection.unlock_db() # Dom0 user home if not 'dom0' in exclude_list: @@ -1306,6 +1304,8 @@ def wait_backup_feedback(progress_callback, in_stream, streamproc, backup_target def restore_vm_dir (backup_dir, src_dir, dst_dir, vm_spec, print_callback=None, error_callback=None, encrypted=False, appvm=None): #backup_src_dir = src_dir.replace (qubes_base_dir, backup_dir) + print "Restore vm dir:",backup_dir, src_dir, dst_dir, vm_spec + vmproc = None if appvm != None: @@ -1378,11 +1378,29 @@ def backup_restore_set_defaults(options): return options -def backup_restore_header(restore_target, encrypt=False, appvm=None): +def load_hmac(hmac): + hmac = hmac.strip(" \t\r\n").split("=") + if len(hmac) > 1: + hmac = hmac[1].strip() + else: + raise QubesException("ERROR: invalid hmac file content") + + return hmac + +def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None): # Simulate dd if=backup_file count=10 | file - # Simulate dd if=backup_file count=10 | gpg2 -d | tar xzv -O # analysis = subprocess.Popen() vmproc = None + + import tempfile + feedback_file = tempfile.NamedTemporaryFile() + backup_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/restore_") + + # Tar with tapelength does not deals well with stdout (close stdout between two tapes) + # For this reason, we will use named pipes instead + print "Working in",backup_tmpdir + if appvm != None: # Prepare the backup target (Qubes service call) restore_command = "QUBESRPC qubes.Restore none" @@ -1391,7 +1409,7 @@ def backup_restore_header(restore_target, encrypt=False, appvm=None): qvm_collection = QubesVmCollection() qvm_collection.lock_db_for_reading() qvm_collection.load() - + vm = qvm_collection.get_vm_by_name(appvm) if vm is None or vm.qid not in qvm_collection: raise QubesException("VM {0} does not exist".format(appvm)) @@ -1402,7 +1420,7 @@ def backup_restore_header(restore_target, encrypt=False, appvm=None): vmproc = vm.run(command = restore_command, passio_popen = True) vmproc.stdin.write(restore_target.replace("\r","").replace("\n","")+"\n") - headers = vmproc.stdout.read(4096*4) + headers = vmproc.stdout.read(4096*64) vmproc.terminate() if len(headers) <= 0: @@ -1414,46 +1432,58 @@ def backup_restore_header(restore_target, encrypt=False, appvm=None): raise QubesException("ERROR: the backup directory {0} does not exists".format(restore_target)) fp = open(restore_target,'rb') - headers = fp.read(4096*4) + headers = fp.read(4096*16) - if encrypt: - command = subprocess.Popen(['gpg2','--decrypt'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) - stdout,stderr = command.communicate(headers) - if len(stdout) > 0: - headers = stdout - else: - print stderr - raise QubesException("ERROR: unable to decrypt the backup {0}. Is it really encrypted?".format(restore_target)) - - command = subprocess.Popen(['tar', 'xzv', '-O', 'backup_targets','qubes.xml'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + # FIXME: Use a safer program such as cpio, modified uncompress.c, or try to extract it from the APPVM directly + command = subprocess.Popen(['tar', '-i', '-xv', '-C', backup_tmpdir, 'qubes.xml.*'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) headers,stderr = command.communicate(headers) if len(headers) <= 0: print stderr raise QubesException("ERROR: unable to read the qubes backup file {0}. Is it really a backup?".format(restore_target)) - vms_in_backup = {} - for vm in headers.split("\n"): - match = re.match("^(?P[^:]+):(?P[^:]+):(?P[0-9]+)$",vm,re.MULTILINE) - if match: - item = match.groupdict() - item["size"] = int(item["size"]) - vms_in_backup[item["name"]] = item - headers = headers.replace(vm,"") + print "Retrieved headers",headers + + for filename in headers.splitlines(): + print "Loading hmac for file",filename + hmac = load_hmac(open(os.path.join(backup_tmpdir,filename+".hmac"),'r').read()) + + print "Verifying file",filename + hmac_proc = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], stdin=open(os.path.join(backup_tmpdir,filename),'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout,stderr = hmac_proc.communicate() + if len(stderr) > 0: + raise QubesException("ERROR: verify file {0}: {1}".format((filename,stderr))) else: - break + if len(hmac) > 0 and load_hmac(stdout) == hmac: + print "File verification OK -> Extracting archive",filename + # FIXME: implement function for encrypted file + command = subprocess.Popen(['tar', '-xvf', os.path.join(backup_tmpdir,filename), '-C', backup_tmpdir],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + headers,stderr = command.communicate(headers) + if len(headers) <= 0: + print stderr + raise QubesException("ERROR: unable to read the qubes backup file {0}. Is it really a backup?".format(restore_target)) + else: + return os.path.join(backup_tmpdir,headers.strip(" \r\n\t")) - return vms_in_backup, headers + else: + raise QubesException("ERROR: invalid hmac for file {0}: {1}. Is the passphrase correct?".format(filename,load_hmac(stdout))) -def backup_restore_prepare(backup_dir, backup_content, qubes_xml, options = {}, host_collection = None, encrypt=False, appvm=None): + return None + +def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, host_collection = None, encrypt=False, appvm=None): # Defaults backup_restore_set_defaults(options) #### Private functions begin def is_vm_included_in_backup (backup_dir, vm): - for item in backup_content.keys(): - if vm.name == item: - return True - return False + if vm.qid == 0: + # Dom0 is not included, obviously + return False + + if vm.backup_content: + return True + else: + return False + def find_template_name(template, replaces): rx_replace = re.compile("(.*):(.*)") for r in replaces: @@ -1463,14 +1493,9 @@ def backup_restore_prepare(backup_dir, backup_content, qubes_xml, options = {}, return template #### Private functions end - - backup_collection = QubesVmCollection() - import StringIO - backup_collection.qubes_store_file=StringIO.StringIO(qubes_xml) - - - #backup_collection.lock_db_for_reading() - + print "Loading file",qubes_xml + backup_collection = QubesVmCollection(store_filename = qubes_xml) + backup_collection.lock_db_for_reading() backup_collection.load() if host_collection is None: @@ -1480,11 +1505,9 @@ def backup_restore_prepare(backup_dir, backup_content, qubes_xml, options = {}, host_collection.unlock_db() backup_vms_list = [vm for vm in backup_collection.values()] - host_vms_list = [vm for vm in host_collection.values()] vms_to_restore = {} - there_are_conflicting_vms = False there_are_missing_templates = False there_are_missing_netvms = False @@ -1493,6 +1516,7 @@ def backup_restore_prepare(backup_dir, backup_content, qubes_xml, options = {}, # ... and the actual data for vm in backup_vms_list: if is_vm_included_in_backup (backup_dir, vm): + print vm.name,"is included in backup" vms_to_restore[vm.name] = {} vms_to_restore[vm.name]['vm'] = vm; @@ -1676,20 +1700,7 @@ def backup_restore_print_summary(restore_info, print_callback = print_stdout): print_callback(s) -def backup_restore_do(backup_dir, restore_info, restore_vms, host_collection = None, print_callback = print_stdout, error_callback = print_stderr, encrypted=False, appvm = None ): - - ### Private functions begin - ''' - def restore_vm_dir (backup_dir, src_dir, dst_dir, print_callback, error_callback, encrypted, appvm): - - backup_src_dir = src_dir.replace (qubes_base_dir, backup_dir) - - # We prefer to use Linux's cp, because it nicely handles sparse files - retcode = subprocess.call (["cp", "-rp", backup_src_dir, dst_dir]) - if retcode != 0: - raise QubesException("*** Error while copying file {0} to {1}".format(backup_src_dir, dest_dir)) - ''' - ### Private functions end +def backup_restore_do(backup_dir, restore_info, host_collection = None, print_callback = print_stdout, error_callback = print_stderr, encrypted=False, appvm = None ): lock_obtained = False if host_collection is None: @@ -1731,7 +1742,7 @@ def backup_restore_do(backup_dir, restore_info, restore_vms, host_collection = N template=template, installed_by_rpm=False) - restore_vm_dir (backup_dir, vm.dir_path, os.path.dirname(new_vm.dir_path), restore_vms[vm.name], print_callback, error_callback, encrypted, appvm) + restore_vm_dir (backup_dir, vm.dir_path, os.path.dirname(new_vm.dir_path), vm, print_callback, error_callback, encrypted, appvm) new_vm.verify_files() except Exception as err: diff --git a/dom0/qvm-tools/qvm-backup-restore b/dom0/qvm-tools/qvm-backup-restore index 05c56374..3755b788 100755 --- a/dom0/qvm-tools/qvm-backup-restore +++ b/dom0/qvm-tools/qvm-backup-restore @@ -95,12 +95,15 @@ def main(): restore_options['exclude'] = options.exclude + passphrase = raw_input("Please enter the pass phrase that will be used to decrypt/verify the backup:\n") + passphrase = passphrase.replace("\r","").replace("\n","") + print >> sys.stderr, "Checking backup content..." - encrypted, vms, qubes_xml = backup_restore_header(backup_dir, options.decrypt, appvm=options.appvm) + qubes_xml = backup_restore_header(backup_dir, passphrase, options.decrypt, appvm=options.appvm) restore_info = None try: - restore_info = backup_restore_prepare(backup_dir, vms, qubes_xml, options=restore_options, host_collection=host_collection, encrypt=encrypted, appvm=options.appvm) + restore_info = backup_restore_prepare(backup_dir, qubes_xml, passphrase, options=restore_options, host_collection=host_collection, encrypt=options.decrypt, appvm=options.appvm) except QubesException as e: print >> sys.stderr, "ERROR: %s" % str(e) exit(1) @@ -189,7 +192,7 @@ def main(): exit (0) - backup_restore_do(backup_dir, restore_info, vms, host_collection=host_collection, encrypted=encrypted, appvm=options.appvm) + backup_restore_do(backup_dir, restore_info, host_collection=host_collection, encrypted=options.decrypt, appvm=options.appvm) host_collection.unlock_db() From af230b33a9e9ae28ce6aef6b5589c7b37a869624 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Thu, 26 Sep 2013 10:30:57 +0200 Subject: [PATCH 25/82] backup: implemented validated file extraction for non encrypted backups --- dom0/qvm-core/qubesutils.py | 178 ++++++++++++++++++++++-------- dom0/qvm-tools/qvm-backup-restore | 6 +- 2 files changed, 137 insertions(+), 47 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index ec02e369..85f32167 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -909,7 +909,7 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca vm.backup_content = True vm.backup_size = vm.get_disk_utilization() - vm.backup_path = vm.dir_path.split(qubes_base_dir)[1] + vm.backup_path = vm.dir_path.split(os.path.normpath(qubes_base_dir)+"/")[1] qvm_collection.save() # FIXME: should be after backup completed @@ -1164,6 +1164,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e print "Wait_backup_feedback returned:",run_error if len(run_error) > 0: + send_proc.terminate() raise QubesException("Failed to perform backup: error with "+run_error) # Send the chunk to the backup target @@ -1172,7 +1173,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e # Close HMAC hmac.stdin.close() hmac.wait() - print "HMAC:",hmac.poll() + print "HMAC proc return code:",hmac.poll() # Write HMAC data next to the chunk file hmac_data = hmac.stdout.read() @@ -1203,8 +1204,8 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e send_proc.join() if vmproc: - print "VMProc1:",vmproc.poll() - print "Sparse1:",tar_sparse.poll() + print "VMProc1 proc return code:",vmproc.poll() + print "Sparse1 proc return code:",tar_sparse.poll() vmproc.stdin.close() ''' @@ -1227,7 +1228,6 @@ def wait_backup_feedback(progress_callback, in_stream, streamproc, backup_target while run_count > 0 and run_error == None: buffer = in_stream.read(buffer_size) - progress_callback(len(buffer),total_backup_sz) run_count = 0 @@ -1301,11 +1301,84 @@ def wait_backup_feedback(progress_callback, in_stream, streamproc, backup_target return run_error -def restore_vm_dir (backup_dir, src_dir, dst_dir, vm_spec, print_callback=None, error_callback=None, encrypted=False, appvm=None): +def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_size, print_callback=None, error_callback=None, encrypted=False, appvm=None): + + # Setup worker to extract encrypted data chunks to the restore dirs + from multiprocessing import Queue,Process + class Extract_Worker(Process): + def __init__(self,queue,base_dir,passphrase,encrypted,total_size): + super(Extract_Worker, self).__init__() + self.queue = queue + self.base_dir = base_dir + self.passphrase = passphrase + self.encrypted = encrypted + self.total_size = total_size + self.blocks_backedup = 0 + self.tar2_command = None + + self.restore_pipe = os.path.join(self.base_dir,"restore_pipe") + print "Creating pipe in:",self.restore_pipe + print os.mkfifo(self.restore_pipe) + + def compute_progress(self, new_size, total_size): + self.blocks_backedup += new_size + progress = self.blocks_backedup / float(self.total_size) + print_callback(round(progress*100,2)) + + def run(self): + print "Started extracting thread" + + print "Moving to temporary dir",self.base_dir + os.chdir(self.base_dir) + + for filename in iter(self.queue.get,None): + if filename == "FINISHED": + break + + print "Extracting file",filename,"to",self.base_dir + + if self.tar2_command == None: + self.tar2_command = ['tar', '--tape-length','1000000', '-xvf', self.restore_pipe] + self.tar2_command = subprocess.Popen(self.tar2_command,stdin=subprocess.PIPE) + + pipe = open(self.restore_pipe,'r+b') + if self.encrypted: + # Start decrypt + encryptor = subprocess.Popen (["openssl", "enc", "-d", "-aes-256-cbc", "-pass", "pass:"+passphrase], stdin=open(filename,'rb'), stdout=pipe) + + # progress_callback, in_stream, streamproc, backup_target, total_backup_sz, hmac=None, vmproc=None, addproc=None, remove_trailing_bytes=0): + run_error = wait_backup_feedback(self.compute_progress, pipe, encryptor, self.tar2_command.stdin, self.total_size, hmac=None, vmproc=None, addproc=self.tar2_command) + else: + run_error = wait_backup_feedback(self.compute_progress, open(filename,"rb"), None, pipe, self.total_size, hmac=None, vmproc=None, addproc=self.tar2_command) + + print "Run error:",run_error + print self.tar2_command.poll(), + + # Close named pipe so that tar knowns the file has been entirely read + pipe.close() + + if self.tar2_command.poll() != None: + if self.tar2_command.poll() != 0: + raise QubesException("ERROR: unable to extract files for {0}.".format(filename)) + else: + # Finished extracting the tar file + self.tar2_command = None + else: + print "Releasing next chunck" + self.tar2_command.stdin.write("\n") + + # Delete the file as we don't need it anymore + print "Removing file",filename + os.remove(filename) + + print "Finished extracting thread" + + to_extract = Queue() + extract_proc = Extract_Worker(to_extract, backup_tmpdir, passphrase, encrypted, vms_size) + extract_proc.start() #backup_src_dir = src_dir.replace (qubes_base_dir, backup_dir) - print "Restore vm dir:",backup_dir, src_dir, dst_dir, vm_spec - + print "Restore vm dirs:",backup_dir, vms_dirs, vms, vms_size vmproc = None if appvm != None: @@ -1330,39 +1403,42 @@ def restore_vm_dir (backup_dir, src_dir, dst_dir, vm_spec, print_callback=None, else: backup_stdin = open(backup_dir,'rb') - tar_cmdline = ["tar", "-xzv",'--sparse','-C',qubes_base_dir,'--checkpoint=10000'] + # FIXME: Use a safer program such as cpio, modified uncompress.c, or try to extract it from the APPVM directly based on Joana recommendation. + tar1_command = ['tar', '-i', '-xv', '-C', backup_tmpdir] + tar1_command.extend(vms_dirs) + print "Running 'safe' command",tar1_command + command = subprocess.Popen(tar1_command, stdin=backup_stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - tar_cmdline.append(src_dir.split(os.path.normpath(qubes_base_dir)+"/")[1]) + while command.poll() == None and vmproc.poll() == None: + filename = command.stdout.readline().strip(" \t\r\n") + print "Getting new file:",filename + hmacfile = command.stdout.readline().strip(" \t\r\n") + print "Getting hmac:",hmacfile + + print "Verifying file",filename + hmac_proc = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], stdin=open(os.path.join(backup_tmpdir,filename),'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout,stderr = hmac_proc.communicate() + if len(stderr) > 0: + raise QubesException("ERROR: verify file {0}: {1}".format((filename,stderr))) + else: + print "Loading hmac for file",filename + hmac = load_hmac(open(os.path.join(backup_tmpdir,filename+".hmac"),'r').read()) - #print ("Will backup using command",tar_cmdline) + if len(hmac) > 0 and load_hmac(stdout) == hmac: + print "File verification OK. Sending",filename,"to extraction thread" + # Send the chunk to the backup target + to_extract.put(os.path.join(backup_tmpdir,filename)) - import tempfile - feedback_file = tempfile.NamedTemporaryFile() - if encrypted: - encryptor = subprocess.Popen (["gpg2", "--decrypt"], stdin=backup_stdin, stdout=subprocess.PIPE,stderr=subprocess.PIPE) - compressor = subprocess.Popen (tar_cmdline,stdin=encryptor.stdout,stderr=feedback_file,stdout=subprocess.PIPE) - else: - compressor = subprocess.Popen (tar_cmdline,stdin=backup_stdin,stderr=feedback_file,stdout=subprocess.PIPE) - encryptor = None + else: + raise QubesException("ERROR: invalid hmac for file {0}: {1}. Is the passphrase correct?".format(filename,load_hmac(stdout))) - run_error = wait_backup_feedback(print_callback, feedback_file, vm_spec["size"], compressor, encryptor, vmproc) - - # Cleanup - feedback_file.close() - backup_stdin.close() - - # Check returns code of compressor and encryptor and qubes vm retcode - if run_error != None: - try: - if compressor != None: - compressor.terminate() - if encryptor != None: - encryptor.terminate() - if vmproc != None: - vmproc.terminate() - except OSError: - pass - raise QubesException("Failed to perform backup: error with "+run_error) + if command.poll() != 0: + raise QubesException("ERROR: unable to read the qubes backup file {0}. Is it really a backup?".format(restore_target)) + if vmproc.poll() != 0: + raise QubesException("ERROR: unable to read the qubes backup {0} because of a VM error: {1}".format(restore_target,vmproc.stderr.read())) + + to_extract.put("FINISHED") + to_extract.wait() def backup_restore_set_defaults(options): if 'use-default-netvm' not in options: @@ -1434,7 +1510,6 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) fp = open(restore_target,'rb') headers = fp.read(4096*16) - # FIXME: Use a safer program such as cpio, modified uncompress.c, or try to extract it from the APPVM directly command = subprocess.Popen(['tar', '-i', '-xv', '-C', backup_tmpdir, 'qubes.xml.*'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) headers,stderr = command.communicate(headers) if len(headers) <= 0: @@ -1455,14 +1530,14 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) else: if len(hmac) > 0 and load_hmac(stdout) == hmac: print "File verification OK -> Extracting archive",filename - # FIXME: implement function for encrypted file + # FIXME: handle encrypted file command = subprocess.Popen(['tar', '-xvf', os.path.join(backup_tmpdir,filename), '-C', backup_tmpdir],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) headers,stderr = command.communicate(headers) if len(headers) <= 0: print stderr raise QubesException("ERROR: unable to read the qubes backup file {0}. Is it really a backup?".format(restore_target)) else: - return os.path.join(backup_tmpdir,headers.strip(" \r\n\t")) + return (backup_tmpdir,headers.strip(" \r\n\t")) else: raise QubesException("ERROR: invalid hmac for file {0}: {1}. Is the passphrase correct?".format(filename,load_hmac(stdout))) @@ -1581,7 +1656,7 @@ def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, host vms_to_restore[vm.name]['good-to-go'] = True # ...and dom0 home - # TODO, replace this part of code to handle the new backup format using tar + # FIXME, replace this part of code to handle the new backup format using tar if options['dom0-home'] and os.path.exists(backup_dir + '/dom0-home'): vms_to_restore['dom0'] = {} local_user = grp.getgrnam('qubes').gr_mem[0] @@ -1700,7 +1775,7 @@ def backup_restore_print_summary(restore_info, print_callback = print_stdout): print_callback(s) -def backup_restore_do(backup_dir, restore_info, host_collection = None, print_callback = print_stdout, error_callback = print_stderr, encrypted=False, appvm = None ): +def backup_restore_do(backup_dir, restore_tmpdir, passphrase, restore_info, host_collection = None, print_callback = print_stdout, error_callback = print_stderr, encrypted=False, appvm = None ): lock_obtained = False if host_collection is None: @@ -1709,6 +1784,23 @@ def backup_restore_do(backup_dir, restore_info, host_collection = None, print_ca host_collection.load() lock_obtained = True + # Perform VM restoration in backup order + vms_dirs = [] + vms_size = 0 + vms = {} + for vm_info in restore_info.values(): + if not vm_info['good-to-go']: + continue + if 'vm' not in vm_info: + continue + vm = vm_info['vm'] + vms_size += vm.backup_size + vms_dirs.append(vm.backup_path+"*") + vms[vm.name] = vm + + restore_vm_dirs (backup_dir, restore_tmpdir, passphrase, vms_dirs, vms, vms_size, print_callback, error_callback, encrypted, appvm) + + # Add VM in right order for (vm_class_name, vm_class) in sorted(QubesVmClasses.items(), key=lambda _x: _x[1].load_order): @@ -1742,8 +1834,6 @@ def backup_restore_do(backup_dir, restore_info, host_collection = None, print_ca template=template, installed_by_rpm=False) - restore_vm_dir (backup_dir, vm.dir_path, os.path.dirname(new_vm.dir_path), vm, print_callback, error_callback, encrypted, appvm) - new_vm.verify_files() except Exception as err: error_callback("ERROR: {0}".format(err)) diff --git a/dom0/qvm-tools/qvm-backup-restore b/dom0/qvm-tools/qvm-backup-restore index 3755b788..bf1c1253 100755 --- a/dom0/qvm-tools/qvm-backup-restore +++ b/dom0/qvm-tools/qvm-backup-restore @@ -99,11 +99,11 @@ def main(): passphrase = passphrase.replace("\r","").replace("\n","") print >> sys.stderr, "Checking backup content..." - qubes_xml = backup_restore_header(backup_dir, passphrase, options.decrypt, appvm=options.appvm) + restore_tmpdir,qubes_xml = backup_restore_header(backup_dir, passphrase, options.decrypt, appvm=options.appvm) restore_info = None try: - restore_info = backup_restore_prepare(backup_dir, qubes_xml, passphrase, options=restore_options, host_collection=host_collection, encrypt=options.decrypt, appvm=options.appvm) + restore_info = backup_restore_prepare(backup_dir,os.path.join(restore_tmpdir, qubes_xml), passphrase, options=restore_options, host_collection=host_collection, encrypt=options.decrypt, appvm=options.appvm) except QubesException as e: print >> sys.stderr, "ERROR: %s" % str(e) exit(1) @@ -192,7 +192,7 @@ def main(): exit (0) - backup_restore_do(backup_dir, restore_info, host_collection=host_collection, encrypted=options.decrypt, appvm=options.appvm) + backup_restore_do(backup_dir,restore_tmpdir, passphrase, restore_info, host_collection=host_collection, encrypted=options.decrypt, appvm=options.appvm) host_collection.unlock_db() From 26fb5b3d25a28e305b3609f561b53b250c67ca35 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Fri, 27 Sep 2013 09:41:40 +0200 Subject: [PATCH 26/82] backup: fixes to use the backup GUI --- dom0/qvm-core/qubesutils.py | 68 +++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 85f32167..c705f476 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1105,7 +1105,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e global blocks_backedup blocks_backedup += new_size progress = blocks_backedup / float(total_backup_sz) - progress_callback(round(progress*100,2)) + progress_callback(int(round(progress*100,2))) to_send = Queue() send_proc = Send_Worker(to_send, backup_tmpdir, backup_stdout) @@ -1298,15 +1298,14 @@ def wait_backup_feedback(progress_callback, in_stream, streamproc, backup_target hmac.stdin.write(buffer) - return run_error -def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_size, print_callback=None, error_callback=None, encrypted=False, appvm=None): +def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_size, print_callback=None, error_callback=None, progress_callback=None, encrypted=False, appvm=None): # Setup worker to extract encrypted data chunks to the restore dirs from multiprocessing import Queue,Process class Extract_Worker(Process): - def __init__(self,queue,base_dir,passphrase,encrypted,total_size): + def __init__(self,queue,base_dir,passphrase,encrypted,total_size,print_callback,error_callback,progress_callback): super(Extract_Worker, self).__init__() self.queue = queue self.base_dir = base_dir @@ -1316,6 +1315,10 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s self.blocks_backedup = 0 self.tar2_command = None + self.print_callback = print_callback + self.error_callback = error_callback + self.progress_callback = progress_callback + self.restore_pipe = os.path.join(self.base_dir,"restore_pipe") print "Creating pipe in:",self.restore_pipe print os.mkfifo(self.restore_pipe) @@ -1323,22 +1326,24 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s def compute_progress(self, new_size, total_size): self.blocks_backedup += new_size progress = self.blocks_backedup / float(self.total_size) - print_callback(round(progress*100,2)) + progress = int(round(progress*100,2)) + self.progress_callback(progress) def run(self): - print "Started extracting thread" + self.print_callback("Started sending thread") - print "Moving to temporary dir",self.base_dir + self.print_callback("Moving to temporary dir "+self.base_dir) os.chdir(self.base_dir) for filename in iter(self.queue.get,None): if filename == "FINISHED": break - print "Extracting file",filename,"to",self.base_dir + self.print_callback("Extracting file "+filename+" to "+self.base_dir) if self.tar2_command == None: self.tar2_command = ['tar', '--tape-length','1000000', '-xvf', self.restore_pipe] + self.print_callback("Running command "+str(self.tar2_command)) self.tar2_command = subprocess.Popen(self.tar2_command,stdin=subprocess.PIPE) pipe = open(self.restore_pipe,'r+b') @@ -1351,11 +1356,8 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s else: run_error = wait_backup_feedback(self.compute_progress, open(filename,"rb"), None, pipe, self.total_size, hmac=None, vmproc=None, addproc=self.tar2_command) - print "Run error:",run_error - print self.tar2_command.poll(), - - # Close named pipe so that tar knowns the file has been entirely read - pipe.close() + self.print_callback("Run error:"+run_error) + self.print_callback(str(self.tar2_command.poll())) if self.tar2_command.poll() != None: if self.tar2_command.poll() != 0: @@ -1363,22 +1365,24 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s else: # Finished extracting the tar file self.tar2_command = None + else: - print "Releasing next chunck" + self.print_callback("Releasing next chunck") + pipe.close() self.tar2_command.stdin.write("\n") # Delete the file as we don't need it anymore - print "Removing file",filename + self.print_callback("Removing file "+filename) os.remove(filename) - print "Finished extracting thread" + self.print_callback("Finished extracting thread") to_extract = Queue() - extract_proc = Extract_Worker(to_extract, backup_tmpdir, passphrase, encrypted, vms_size) + extract_proc = Extract_Worker(to_extract, backup_tmpdir, passphrase, encrypted, vms_size, print_callback, error_callback, progress_callback) extract_proc.start() - #backup_src_dir = src_dir.replace (qubes_base_dir, backup_dir) - print "Restore vm dirs:",backup_dir, vms_dirs, vms, vms_size + print_callback("Working in temporary dir:"+backup_tmpdir) + print_callback(str(vms_size)+" bytes to restore") vmproc = None if appvm != None: @@ -1403,29 +1407,35 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s else: backup_stdin = open(backup_dir,'rb') - # FIXME: Use a safer program such as cpio, modified uncompress.c, or try to extract it from the APPVM directly based on Joana recommendation. + # FIXME: Use a safer program such as cpio, modified uncompress.c, or try to extract it from tar1_command = ['tar', '-i', '-xv', '-C', backup_tmpdir] tar1_command.extend(vms_dirs) - print "Running 'safe' command",tar1_command + print_callback("Run command"+str(tar1_command)) command = subprocess.Popen(tar1_command, stdin=backup_stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE) while command.poll() == None and vmproc.poll() == None: + filename = command.stdout.readline().strip(" \t\r\n") - print "Getting new file:",filename + + print_callback("Getting new file:"+filename) + hmacfile = command.stdout.readline().strip(" \t\r\n") - print "Getting hmac:",hmacfile + + print_callback("Getting hmac:"+hmacfile) - print "Verifying file",filename + print_callback("Verifying file"+filename) + hmac_proc = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], stdin=open(os.path.join(backup_tmpdir,filename),'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout,stderr = hmac_proc.communicate() + if len(stderr) > 0: raise QubesException("ERROR: verify file {0}: {1}".format((filename,stderr))) else: - print "Loading hmac for file",filename + print_callback("Loading hmac for file"+filename) hmac = load_hmac(open(os.path.join(backup_tmpdir,filename+".hmac"),'r').read()) if len(hmac) > 0 and load_hmac(stdout) == hmac: - print "File verification OK. Sending",filename,"to extraction thread" + print_callback("File verification OK -> Sending file "+filename+" for extraction") # Send the chunk to the backup target to_extract.put(os.path.join(backup_tmpdir,filename)) @@ -1438,7 +1448,7 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s raise QubesException("ERROR: unable to read the qubes backup {0} because of a VM error: {1}".format(restore_target,vmproc.stderr.read())) to_extract.put("FINISHED") - to_extract.wait() + extract_proc.join() def backup_restore_set_defaults(options): if 'use-default-netvm' not in options: @@ -1775,7 +1785,7 @@ def backup_restore_print_summary(restore_info, print_callback = print_stdout): print_callback(s) -def backup_restore_do(backup_dir, restore_tmpdir, passphrase, restore_info, host_collection = None, print_callback = print_stdout, error_callback = print_stderr, encrypted=False, appvm = None ): +def backup_restore_do(backup_dir, restore_tmpdir, passphrase, restore_info, host_collection = None, print_callback = print_stdout, error_callback = print_stderr, progress_callback = None, encrypted=False, appvm=None): lock_obtained = False if host_collection is None: @@ -1798,7 +1808,7 @@ def backup_restore_do(backup_dir, restore_tmpdir, passphrase, restore_info, host vms_dirs.append(vm.backup_path+"*") vms[vm.name] = vm - restore_vm_dirs (backup_dir, restore_tmpdir, passphrase, vms_dirs, vms, vms_size, print_callback, error_callback, encrypted, appvm) + restore_vm_dirs (backup_dir, restore_tmpdir, passphrase, vms_dirs, vms, vms_size, print_callback, error_callback, progress_callback, encrypted, appvm) # Add VM in right order From a594af02dea10865eb048b9db1080e9af3455a0d Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Fri, 27 Sep 2013 09:47:33 +0200 Subject: [PATCH 27/82] backup: fix of a bug introduced in the qvm-backup-restore command by adding a progress_callback --- dom0/qvm-core/qubesutils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index c705f476..b1d9f437 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1377,6 +1377,10 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s self.print_callback("Finished extracting thread") + if progress_callback == None: + def progress_callback(data): + pass + to_extract = Queue() extract_proc = Extract_Worker(to_extract, backup_tmpdir, passphrase, encrypted, vms_size, print_callback, error_callback, progress_callback) extract_proc.start() From c2645bcb924f99e143600f9c706e1f7dcfa7902c Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Sat, 28 Sep 2013 12:24:33 +0200 Subject: [PATCH 28/82] backup: fix a misplaced passphrase request breaking the backup GUI --- dom0/qvm-core/qubesutils.py | 5 +---- dom0/qvm-tools/qvm-backup | 5 ++++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index b1d9f437..87c01f61 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1010,7 +1010,7 @@ def backup_do(base_backup_dir, files_to_backup, progress_callback = None): progress = bytes_backedup * 100 / total_backup_sz progress_callback(progress) -def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, encrypt=False, appvm=None): +def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callback = None, encrypt=False, appvm=None): total_backup_sz = 0 for file in files_to_backup: total_backup_sz += file["size"] @@ -1047,9 +1047,6 @@ def backup_do_copy(base_backup_dir, files_to_backup, progress_callback = None, e # If not APPVM, STDOUT is a local file backup_stdout = open(backup_target,'wb') - passphrase = raw_input("Please enter the pass phrase that will be used to encrypt/verify the backup:\n") - passphrase = passphrase.replace("\r","").replace("\n","") - blocks_backedup = 0 progress = blocks_backedup * 11 / total_backup_sz progress_callback(progress) diff --git a/dom0/qvm-tools/qvm-backup b/dom0/qvm-tools/qvm-backup index ea3a43dd..e7bc0f0a 100755 --- a/dom0/qvm-tools/qvm-backup +++ b/dom0/qvm-tools/qvm-backup @@ -75,8 +75,11 @@ def main(): if not (prompt == "y" or prompt == "Y"): exit (0) + passphrase = raw_input("Please enter the pass phrase that will be used to encrypt/verify the backup:\n") + passphrase = passphrase.replace("\r","").replace("\n","") + try: - backup_do_copy(base_backup_dir, files_to_backup, progress_callback=print_progress, encrypt=options.encrypt,appvm=options.appvm) + backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callback=print_progress, encrypt=options.encrypt,appvm=options.appvm) except QubesException as e: print >>sys.stderr, "ERROR: %s" % str(e) exit(1) From 1a80893ef0454b725dfaf17d6b607d1a3a6fcc75 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Mon, 30 Sep 2013 10:26:04 +0200 Subject: [PATCH 29/82] backup: fixed decryption during backup restoration process --- dom0/qvm-core/qubesutils.py | 99 ++++++++++++++++++++++--------------- 1 file changed, 58 insertions(+), 41 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 87c01f61..120ab4cf 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1302,7 +1302,7 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s # Setup worker to extract encrypted data chunks to the restore dirs from multiprocessing import Queue,Process class Extract_Worker(Process): - def __init__(self,queue,base_dir,passphrase,encrypted,total_size,print_callback,error_callback,progress_callback): + def __init__(self,queue,base_dir,passphrase,encrypted,total_size,print_callback,error_callback,progress_callback,vmproc=None): super(Extract_Worker, self).__init__() self.queue = queue self.base_dir = base_dir @@ -1316,6 +1316,8 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s self.error_callback = error_callback self.progress_callback = progress_callback + self.vmproc = vmproc + self.restore_pipe = os.path.join(self.base_dir,"restore_pipe") print "Creating pipe in:",self.restore_pipe print os.mkfifo(self.restore_pipe) @@ -1344,14 +1346,17 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s self.tar2_command = subprocess.Popen(self.tar2_command,stdin=subprocess.PIPE) pipe = open(self.restore_pipe,'r+b') - if self.encrypted: - # Start decrypt - encryptor = subprocess.Popen (["openssl", "enc", "-d", "-aes-256-cbc", "-pass", "pass:"+passphrase], stdin=open(filename,'rb'), stdout=pipe) + if self.encrypted: + # Start decrypt + encryptor = subprocess.Popen (["openssl", "enc", "-d", "-aes-256-cbc", "-pass", "pass:"+passphrase], stdin=open(filename,'rb'), stdout=subprocess.PIPE) # progress_callback, in_stream, streamproc, backup_target, total_backup_sz, hmac=None, vmproc=None, addproc=None, remove_trailing_bytes=0): - run_error = wait_backup_feedback(self.compute_progress, pipe, encryptor, self.tar2_command.stdin, self.total_size, hmac=None, vmproc=None, addproc=self.tar2_command) - else: - run_error = wait_backup_feedback(self.compute_progress, open(filename,"rb"), None, pipe, self.total_size, hmac=None, vmproc=None, addproc=self.tar2_command) + run_error = wait_backup_feedback(self.compute_progress, encryptor.stdout, encryptor, pipe, self.total_size, hmac=None, vmproc=self.vmproc, addproc=self.tar2_command) + #print "End wait_backup_feedback",run_error,self.tar2_command.poll(),encryptor.poll() + else: + run_error = wait_backup_feedback(self.compute_progress, open(filename,"rb"), None, pipe, self.total_size, hmac=None, vmproc=self.vmproc, addproc=self.tar2_command) + + pipe.close() self.print_callback("Run error:"+run_error) self.print_callback(str(self.tar2_command.poll())) @@ -1365,7 +1370,6 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s else: self.print_callback("Releasing next chunck") - pipe.close() self.tar2_command.stdin.write("\n") # Delete the file as we don't need it anymore @@ -1484,6 +1488,8 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) feedback_file = tempfile.NamedTemporaryFile() backup_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/restore_") + os.chdir(backup_tmpdir) + # Tar with tapelength does not deals well with stdout (close stdout between two tapes) # For this reason, we will use named pipes instead print "Working in",backup_tmpdir @@ -1507,12 +1513,6 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) vmproc = vm.run(command = restore_command, passio_popen = True) vmproc.stdin.write(restore_target.replace("\r","").replace("\n","")+"\n") - headers = vmproc.stdout.read(4096*64) - vmproc.terminate() - - if len(headers) <= 0: - raise QubesException("ERROR: unable to read the backup target {0}".format(restore_target)) - else: # Create the target directory if not os.path.exists (restore_target): @@ -1521,37 +1521,53 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) fp = open(restore_target,'rb') headers = fp.read(4096*16) - command = subprocess.Popen(['tar', '-i', '-xv', '-C', backup_tmpdir, 'qubes.xml.*'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) - headers,stderr = command.communicate(headers) - if len(headers) <= 0: - print stderr - raise QubesException("ERROR: unable to read the qubes backup file {0}. Is it really a backup?".format(restore_target)) - print "Retrieved headers",headers + command = subprocess.Popen(['tar', '-i', '-xv', '-C', backup_tmpdir, 'qubes.xml*'],stdin=vmproc.stdout,stdout=subprocess.PIPE,stderr=subprocess.PIPE) - for filename in headers.splitlines(): - print "Loading hmac for file",filename - hmac = load_hmac(open(os.path.join(backup_tmpdir,filename+".hmac"),'r').read()) + filename = command.stdout.readline().strip(" \t\r\n") + print "Getting file",filename - print "Verifying file",filename - hmac_proc = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], stdin=open(os.path.join(backup_tmpdir,filename),'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout,stderr = hmac_proc.communicate() - if len(stderr) > 0: - raise QubesException("ERROR: verify file {0}: {1}".format((filename,stderr))) - else: - if len(hmac) > 0 and load_hmac(stdout) == hmac: - print "File verification OK -> Extracting archive",filename - # FIXME: handle encrypted file - command = subprocess.Popen(['tar', '-xvf', os.path.join(backup_tmpdir,filename), '-C', backup_tmpdir],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) - headers,stderr = command.communicate(headers) - if len(headers) <= 0: - print stderr - raise QubesException("ERROR: unable to read the qubes backup file {0}. Is it really a backup?".format(restore_target)) - else: - return (backup_tmpdir,headers.strip(" \r\n\t")) + hmacfile = command.stdout.readline().strip(" \t\r\n") + print "Getting hmac",hmacfile + while not os.path.exists(os.path.join(backup_tmpdir,hmacfile)): + time.sleep(1000) + + command.terminate() + command.wait() + vmproc.terminate() + vmproc.wait() + + print "Loading hmac for file",filename + hmac = load_hmac(open(os.path.join(backup_tmpdir,filename+".hmac"),'r').read()) + + print "Successfully retrieved headers" + + print "Verifying file",filename + hmac_proc = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], stdin=open(os.path.join(backup_tmpdir,filename),'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout,stderr = hmac_proc.communicate() + if len(stderr) > 0: + raise QubesException("ERROR: verify file {0}: {1}".format((filename,stderr))) + else: + if len(hmac) > 0 and load_hmac(stdout) == hmac: + print "File verification OK -> Extracting archive",filename + if encrypt: + encryptor = subprocess.Popen (["openssl", "enc", "-d", "-aes-256-cbc", "-pass", "pass:"+passphrase], stdin=open(os.path.join(backup_tmpdir,filename),'rb'), stdout=subprocess.PIPE) + tarhead_command = subprocess.Popen(['tar', '--tape-length','1000000', '-xv'],stdin=encryptor.stdout) else: - raise QubesException("ERROR: invalid hmac for file {0}: {1}. Is the passphrase correct?".format(filename,load_hmac(stdout))) + encryptor = None + tarhead_command = subprocess.Popen(['tar', '--tape-length','1000000', '-xvf', os.path.join(backup_tmpdir,filename)]) + + tarhead_command.wait() + if encryptor: + if encryptor.poll() != 0: + raise QubesException("ERROR: unable to decrypt file {0}".format(filename)) + if tarhead_command.poll() != 0: + raise QubesException("ERROR: unable to extract the qubes.xml file. Is archive encrypted?") + + return (backup_tmpdir,"qubes.xml") + else: + raise QubesException("ERROR: unable to verify the qubes.xml file. Is the passphrase correct?") return None @@ -1653,7 +1669,8 @@ def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, host # Maybe the (custom) netvm is in the backup? netvm_on_backup = backup_collection.get_vm_by_name (netvm_name) - if not ((netvm_on_backup is not None) and netvm_on_backup.is_netvm() and is_vm_included_in_backup(backup_dir, netvm_on_backup)): + if not ((netvm_on_backup is not None) and netvm_on_backup.is_netvm() and is_vm_ +included_in_backup(backup_dir, netvm_on_backup)): if options['use-default-netvm']: vms_to_restore[vm.name]['netvm'] = host_collection.get_default_netvm().name vm.uses_default_netvm = True From 04d38055a63f474f418cc1d969eaa4f5c036236f Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Mon, 30 Sep 2013 14:54:14 +0200 Subject: [PATCH 30/82] backup: fixed vm extraction path and untracked errors --- dom0/qvm-core/qubesutils.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 120ab4cf..e10ad4d6 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1199,6 +1199,9 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba to_send.put("FINISHED") send_proc.join() + + if send_proc.exitcode != 0: + raise QubesException("Failed to send backup: error in the sending process") if vmproc: print "VMProc1 proc return code:",vmproc.poll() @@ -1331,17 +1334,20 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s def run(self): self.print_callback("Started sending thread") - self.print_callback("Moving to temporary dir "+self.base_dir) + self.print_callback("Moving to dir "+self.base_dir) os.chdir(self.base_dir) for filename in iter(self.queue.get,None): if filename == "FINISHED": break - self.print_callback("Extracting file "+filename+" to "+self.base_dir) + self.print_callback("Extracting file "+filename+" to "+qubes_base_dir) if self.tar2_command == None: - self.tar2_command = ['tar', '--tape-length','1000000', '-xvf', self.restore_pipe] + # FIXME: Make the extraction safer by avoiding to erase other vms: + # - extracting directly to the target directory (based on the vm name and by using the --strip=2). + # - ensuring that the leading slashs are ignored when extracting (can also be obtained by running with --strip ?) + self.tar2_command = ['tar', '--tape-length','1000000', '-C', qubes_base_dir, '-xvf', self.restore_pipe] self.print_callback("Running command "+str(self.tar2_command)) self.tar2_command = subprocess.Popen(self.tar2_command,stdin=subprocess.PIPE) @@ -1452,8 +1458,14 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s if vmproc.poll() != 0: raise QubesException("ERROR: unable to read the qubes backup {0} because of a VM error: {1}".format(restore_target,vmproc.stderr.read())) + print "Extraction process status:",extract_proc.exitcode + to_extract.put("FINISHED") + print_callback("Waiting for the extraction process to finish...") extract_proc.join() + print_callback("Extraction process finished with code:"+str(extract_proc.exitcode)) + if extract_proc.exitcode != 0: + raise QubesException("ERROR: unable to extract the qubes backup. Check extracting process errors.") def backup_restore_set_defaults(options): if 'use-default-netvm' not in options: @@ -1552,9 +1564,11 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) if len(hmac) > 0 and load_hmac(stdout) == hmac: print "File verification OK -> Extracting archive",filename if encrypt: + print "Starting decryption process" encryptor = subprocess.Popen (["openssl", "enc", "-d", "-aes-256-cbc", "-pass", "pass:"+passphrase], stdin=open(os.path.join(backup_tmpdir,filename),'rb'), stdout=subprocess.PIPE) tarhead_command = subprocess.Popen(['tar', '--tape-length','1000000', '-xv'],stdin=encryptor.stdout) else: + print "No decryption process required" encryptor = None tarhead_command = subprocess.Popen(['tar', '--tape-length','1000000', '-xvf', os.path.join(backup_tmpdir,filename)]) @@ -1669,8 +1683,7 @@ def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, host # Maybe the (custom) netvm is in the backup? netvm_on_backup = backup_collection.get_vm_by_name (netvm_name) - if not ((netvm_on_backup is not None) and netvm_on_backup.is_netvm() and is_vm_ -included_in_backup(backup_dir, netvm_on_backup)): + if not ((netvm_on_backup is not None) and netvm_on_backup.is_netvm() and is_vm_included_in_backup(backup_dir, netvm_on_backup)): if options['use-default-netvm']: vms_to_restore[vm.name]['netvm'] = host_collection.get_default_netvm().name vm.uses_default_netvm = True From 3e4637415abc11860fb1f1b537b448749088a38c Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Tue, 1 Oct 2013 12:09:13 +0200 Subject: [PATCH 31/82] backup: improved error handling during restore process --- dom0/qvm-core/qubesutils.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index e10ad4d6..4ab0c13f 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1522,7 +1522,7 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) qvm_collection.unlock_db() # If APPVM, STDOUT is a PIPE - vmproc = vm.run(command = restore_command, passio_popen = True) + vmproc = vm.run(command = restore_command, passio_popen = True, passio_stderr = True) vmproc.stdin.write(restore_target.replace("\r","").replace("\n","")+"\n") else: @@ -1544,9 +1544,19 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) while not os.path.exists(os.path.join(backup_tmpdir,hmacfile)): time.sleep(1000) - + command.terminate() command.wait() + + if vmproc.poll() != None and vmproc.poll() != 0: + error = vmproc.stderr.read() + print error,vmproc.poll(),command.poll() + raise QubesException("ERROR: retrieving backup headers:{0}".format(error)) + elif command.poll() != None and command.poll() not in [0,-15]: + error = command.stderr.read() + print error,vmproc.poll(),command.poll() + raise QubesException("ERROR: retrieving backup headers:{0}".format(error)) + vmproc.terminate() vmproc.wait() From 6c7322d324121be42f3bc1321cc9f54121935f47 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Mon, 21 Oct 2013 15:39:16 +0200 Subject: [PATCH 32/82] backup: fixed reinitialisation of all backup flags in qubes.xml --- dom0/qvm-core/qubesutils.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index 4ab0c13f..ed4b114a 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -907,9 +907,14 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca print_callback(s) - vm.backup_content = True - vm.backup_size = vm.get_disk_utilization() - vm.backup_path = vm.dir_path.split(os.path.normpath(qubes_base_dir)+"/")[1] + # Initialize backup flag on all VMs + for vm in qvm_collection.values(): + vm.backup_content = False + + if vm in vms_for_backup: + vm.backup_content = True + vm.backup_size = vm.get_disk_utilization() + vm.backup_path = vm.dir_path.split(os.path.normpath(qubes_base_dir)+"/")[1] qvm_collection.save() # FIXME: should be after backup completed From b96ec61e17be2e2e6482a62118c93f0908ca24b7 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Mon, 21 Oct 2013 15:50:54 +0200 Subject: [PATCH 33/82] backup: implemented header restoration using tar2qfile --- dom0/qvm-core/qubesutils.py | 51 +++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/dom0/qvm-core/qubesutils.py b/dom0/qvm-core/qubesutils.py index ed4b114a..3a6024fc 100644 --- a/dom0/qvm-core/qubesutils.py +++ b/dom0/qvm-core/qubesutils.py @@ -1495,6 +1495,20 @@ def load_hmac(hmac): return hmac +import struct +def get_qfile_error(buffer): + error = struct.unpack("I",buffer[0:4])[0] + error_msg = { 0: "COPY_FILE_OK", + 1: "COPY_FILE_READ_EOF", + 2: "COPY_FILE_READ_ERROR", + 3: "COPY_FILE_WRITE_ERROR", + } + + if error in error_msg.keys(): + return error_msg[error] + else: + return "UNKNOWN_ERROR_"+str(error) + def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None): # Simulate dd if=backup_file count=10 | file - # Simulate dd if=backup_file count=10 | gpg2 -d | tar xzv -O @@ -1529,7 +1543,6 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) # If APPVM, STDOUT is a PIPE vmproc = vm.run(command = restore_command, passio_popen = True, passio_stderr = True) vmproc.stdin.write(restore_target.replace("\r","").replace("\n","")+"\n") - else: # Create the target directory if not os.path.exists (restore_target): @@ -1538,32 +1551,44 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) fp = open(restore_target,'rb') headers = fp.read(4096*16) + tar1_command = ['/usr/lib/qubes/qfile-dom0-unpacker', str(os.getuid()), backup_tmpdir] + command = subprocess.Popen(tar1_command,stdin=vmproc.stdout,stdout=subprocess.PIPE,stderr=subprocess.PIPE) - command = subprocess.Popen(['tar', '-i', '-xv', '-C', backup_tmpdir, 'qubes.xml*'],stdin=vmproc.stdout,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + result_header = command.stdout.read() - filename = command.stdout.readline().strip(" \t\r\n") - print "Getting file",filename + if vmproc.poll() != None: + error = vmproc.stderr.read() + print error + print vmproc.poll(),command.poll() + raise QubesException("ERROR: Immediate VM error while retrieving backup headers:{0}".format(error)) - hmacfile = command.stdout.readline().strip(" \t\r\n") - print "Getting hmac",hmacfile + filename = "qubes.xml.000" - while not os.path.exists(os.path.join(backup_tmpdir,hmacfile)): - time.sleep(1000) + print result_header.encode("hex") + error_msg = get_qfile_error(result_header) + if error_msg != "COPY_FILE_OK": + print vmproc.stdout.read() + raise QubesException("ERROR: unpacking backup headers: {0}".format(error_msg)) + if not os.path.exists(os.path.join(backup_tmpdir,filename+".hmac")): + raise QubesException("ERROR: header not extracted correctly: {0}".format(os.path.join(backup_tmpdir,filename+".hmac"))) command.terminate() command.wait() if vmproc.poll() != None and vmproc.poll() != 0: error = vmproc.stderr.read() - print error,vmproc.poll(),command.poll() - raise QubesException("ERROR: retrieving backup headers:{0}".format(error)) + print error + print vmproc.poll(),command.poll() + raise QubesException("ERROR: VM error retrieving backup headers") elif command.poll() != None and command.poll() not in [0,-15]: error = command.stderr.read() - print error,vmproc.poll(),command.poll() + print error + print vmproc.poll(),command.poll() raise QubesException("ERROR: retrieving backup headers:{0}".format(error)) - vmproc.terminate() - vmproc.wait() + if vmproc.poll() == None: + vmproc.terminate() + vmproc.wait() print "Loading hmac for file",filename hmac = load_hmac(open(os.path.join(backup_tmpdir,filename+".hmac"),'r').read()) From 1a7d1702566dd6415ac65ec7f82d62677e6b989c Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Mon, 21 Oct 2013 16:04:15 +0200 Subject: [PATCH 34/82] backup: use tar2qfile in qubes.Restore RPC file --- qubes_rpc/qubes.Restore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qubes_rpc/qubes.Restore b/qubes_rpc/qubes.Restore index dfea1448..96a76e53 100644 --- a/qubes_rpc/qubes.Restore +++ b/qubes_rpc/qubes.Restore @@ -5,7 +5,7 @@ if [ -f "$args" ] ; then echo "Performing restore from backup file $args" >2 TARGET="$args" echo "Copying $TARGET to STDOUT" >2 - cat $TARGET + cat $TARGET | /usr/lib/qubes/tar2qfile else echo "Checking if arguments is matching a command" >2 COMMAND=`echo $args | cut -d ' ' -f 1` From a56ceb92c55f28240b96af79178785d32755d611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 9 Nov 2013 15:55:17 +0100 Subject: [PATCH 35/82] backup: update for new API - use system_path hash Instead of a bunch of global variables --- core/qubesutils.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index e96ce13d..66ccb789 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -916,7 +916,7 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca if vm in vms_for_backup: vm.backup_content = True vm.backup_size = vm.get_disk_utilization() - vm.backup_path = vm.dir_path.split(os.path.normpath(qubes_base_dir)+"/")[1] + vm.backup_path = vm.dir_path.split(os.path.normpath(system_path["qubes_base_dir"])+"/")[1] qvm_collection.save() # FIXME: should be after backup completed @@ -1118,7 +1118,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba for filename in files_to_backup: print "Backing up",filename - backup_tempfile = os.path.join(backup_tmpdir,filename["path"].split(os.path.normpath(qubes_base_dir)+"/")[1]) + backup_tempfile = os.path.join(backup_tmpdir,filename["path"].split(os.path.normpath(system_path["qubes_base_dir"])+"/")[1]) print "Using temporary location:",backup_tempfile # Ensure the temporary directory exists @@ -1127,8 +1127,8 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba os.makedirs(os.path.dirname(backup_tempfile)) # The first tar cmd can use any complex feature as we want. Files will be verified before untaring this. - tar_cmdline = ["tar", "-Pc", "-f", backup_pipe,'--sparse','--tape-length',str(1000000),'-C',qubes_base_dir, - filename["path"].split(os.path.normpath(qubes_base_dir)+"/")[1] + tar_cmdline = ["tar", "-Pc", "-f", backup_pipe,'--sparse','--tape-length',str(1000000),'-C',system_path["qubes_base_dir"], + filename["path"].split(os.path.normpath(system_path["qubes_base_dir"])+"/")[1] ] print " ".join(tar_cmdline) @@ -1348,13 +1348,14 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s if filename == "FINISHED": break - self.print_callback("Extracting file "+filename+" to "+qubes_base_dir) + self.print_callback("Extracting file "+filename+" to "+system_path["qubes_base_dir"]) if self.tar2_command == None: # FIXME: Make the extraction safer by avoiding to erase other vms: # - extracting directly to the target directory (based on the vm name and by using the --strip=2). # - ensuring that the leading slashs are ignored when extracting (can also be obtained by running with --strip ?) - self.tar2_command = ['tar', '--tape-length','1000000', '-C', qubes_base_dir, '-xvf', self.restore_pipe] + # marmarek: use other (local) variable for command line + self.tar2_command = ['tar', '--tape-length','1000000', '-C', system_path["qubes_base_dir"], '-xvf', self.restore_pipe] self.print_callback("Running command "+str(self.tar2_command)) self.tar2_command = subprocess.Popen(self.tar2_command,stdin=subprocess.PIPE) From 34b03fe2b3bed3e945c3818dd3b15dbc47c7912a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 9 Nov 2013 16:46:06 +0100 Subject: [PATCH 36/82] backup: fix setting backup qubes.xml attributes Cannot compare QubesVm objects, because we have different instances of QubesVmCollection. So compare QID instead. --- core/qubesutils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index 66ccb789..3c7ddf8d 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -789,12 +789,11 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca if exclude_list is None: exclude_list = [] - qvm_collection = None - if vms_list is None: - qvm_collection = QubesVmCollection() - qvm_collection.lock_db_for_writing() - qvm_collection.load() + qvm_collection = QubesVmCollection() + qvm_collection.lock_db_for_writing() + qvm_collection.load() + if vms_list is None: all_vms = [vm for vm in qvm_collection.values()] selected_vms = [vm for vm in all_vms if vm.include_in_backups] appvms_to_backup = [vm for vm in selected_vms if vm.is_appvm() and not vm.internal] @@ -910,10 +909,11 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca print_callback(s) # Initialize backup flag on all VMs + vms_for_backup_qid = [vm.qid for vm in vms_for_backup] for vm in qvm_collection.values(): vm.backup_content = False - if vm in vms_for_backup: + if vm.qid in vms_for_backup_qid: vm.backup_content = True vm.backup_size = vm.get_disk_utilization() vm.backup_path = vm.dir_path.split(os.path.normpath(system_path["qubes_base_dir"])+"/")[1] From 5cebff34bda0ef31f6dcd634056323588bad1cec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 9 Nov 2013 16:51:25 +0100 Subject: [PATCH 37/82] backup: include qubes.xml It was commented out by mistake. --- core/qubesutils.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index 3c7ddf8d..695fbbcb 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -779,12 +779,7 @@ def file_to_backup (file_path, sz = None): def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_callback = print_stdout): """If vms = None, include all (sensible) VMs; exclude_list is always applied""" - ''' - if not os.path.exists (base_backup_dir): - raise QubesException("The target directory doesn't exist!") - files_to_backup = file_to_backup (system_path["qubes_store_filename"]) - ''' if exclude_list is None: exclude_list = [] @@ -969,6 +964,7 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca fmt="{{0:-^{0}}}-+".format(f["width"] + 1) s += fmt.format('-') print_callback(s) + # TODO: check at least if backing up to local drive ''' stat = os.statvfs(base_backup_dir) backup_fs_free_sz = stat.f_bsize * stat.f_bavail From b84ba998a303ded57dbaebd7b7ea602ffb64d0bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 9 Nov 2013 16:56:59 +0100 Subject: [PATCH 38/82] backup: fix misused variables - most likely copy&paste error --- core/qubesutils.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index 695fbbcb..1bd42180 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -1050,6 +1050,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba # If not APPVM, STDOUT is a local file backup_stdout = open(backup_target,'wb') + global blocks_backedup blocks_backedup = 0 progress = blocks_backedup * 11 / total_backup_sz progress_callback(progress) @@ -1098,9 +1099,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba os.remove(filename) print "Finished sending thread" - - global blocks_backedup - blocks_backedup = 0 + def compute_progress(new_size, total_backup_sz): global blocks_backedup blocks_backedup += new_size @@ -1458,9 +1457,9 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s raise QubesException("ERROR: invalid hmac for file {0}: {1}. Is the passphrase correct?".format(filename,load_hmac(stdout))) if command.poll() != 0: - raise QubesException("ERROR: unable to read the qubes backup file {0}. Is it really a backup?".format(restore_target)) + raise QubesException("ERROR: unable to read the qubes backup file {0}. Is it really a backup?".format(backup_dir)) if vmproc.poll() != 0: - raise QubesException("ERROR: unable to read the qubes backup {0} because of a VM error: {1}".format(restore_target,vmproc.stderr.read())) + raise QubesException("ERROR: unable to read the qubes backup {0} because of a VM error: {1}".format(backup_dir,vmproc.stderr.read())) print "Extraction process status:",extract_proc.exitcode From 89b6069bda3799d8cd07a297f8a3e253b51dc178 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 9 Nov 2013 17:06:29 +0100 Subject: [PATCH 39/82] backup: whitespace fixes --- core/qubesutils.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index 1bd42180..d9b751d4 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -1027,7 +1027,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba qvm_collection = QubesVmCollection() qvm_collection.lock_db_for_reading() qvm_collection.load() - + vm = qvm_collection.get_vm_by_name(appvm) if vm is None or vm.qid not in qvm_collection: raise QubesException("VM {0} does not exist".format(appvm)) @@ -1093,7 +1093,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba tar_final_cmd = ["tar", "-cO", "--posix", "-C", self.base_dir, filename] final_proc = subprocess.Popen (tar_final_cmd, stdin=subprocess.PIPE, stdout=self.backup_stdout) final_proc.wait() - + # Delete the file as we don't need it anymore print "Removing file",filename os.remove(filename) @@ -1148,7 +1148,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba chunkfile = backup_tempfile + "." + "%03d" % i i += 1 chunkfile_p = open(chunkfile,'wb') - + if encrypt: # Start encrypt # If no cipher is provided, the data is forwarded unencrypted !!! @@ -1193,15 +1193,12 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba run_error="paused" else: print "Finished tar sparse with error",tar_sparse.poll() - - pipe.close() - # Close the backup target and wait for it to finish - #backup_stdout.close() + pipe.close() to_send.put("FINISHED") send_proc.join() - + if send_proc.exitcode != 0: raise QubesException("Failed to send backup: error in the sending process") @@ -1380,7 +1377,7 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s else: self.print_callback("Releasing next chunck") self.tar2_command.stdin.write("\n") - + # Delete the file as we don't need it anymore self.print_callback("Removing file "+filename) os.remove(filename) @@ -1407,7 +1404,7 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s qvm_collection = QubesVmCollection() qvm_collection.lock_db_for_reading() qvm_collection.load() - + vm = qvm_collection.get_vm_by_name(appvm) if vm is None or vm.qid not in qvm_collection: raise QubesException("VM {0} does not exist".format(appvm)) @@ -1557,7 +1554,7 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) if vmproc.poll() != None: error = vmproc.stderr.read() print error - print vmproc.poll(),command.poll() + print vmproc.poll(),command.poll() raise QubesException("ERROR: Immediate VM error while retrieving backup headers:{0}".format(error)) filename = "qubes.xml.000" @@ -1651,7 +1648,7 @@ def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, host backup_collection = QubesVmCollection(store_filename = qubes_xml) backup_collection.lock_db_for_reading() backup_collection.load() - + if host_collection is None: host_collection = QubesVmCollection() host_collection.lock_db_for_reading() From 54f08e00b9dad3dab1eff38d5c5dd04852fdba13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 9 Nov 2013 17:09:44 +0100 Subject: [PATCH 40/82] backup: fix race condition with inner tar process Do not assume that tar will finish quickly - explicitly wait for either process termination or request for the next archive part. --- core/qubesutils.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index d9b751d4..4fd3ed1a 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -1342,6 +1342,7 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s self.print_callback("Extracting file "+filename+" to "+system_path["qubes_base_dir"]) + pipe = open(self.restore_pipe,'r+b') if self.tar2_command == None: # FIXME: Make the extraction safer by avoiding to erase other vms: # - extracting directly to the target directory (based on the vm name and by using the --strip=2). @@ -1349,9 +1350,8 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s # marmarek: use other (local) variable for command line self.tar2_command = ['tar', '--tape-length','1000000', '-C', system_path["qubes_base_dir"], '-xvf', self.restore_pipe] self.print_callback("Running command "+str(self.tar2_command)) - self.tar2_command = subprocess.Popen(self.tar2_command,stdin=subprocess.PIPE) + self.tar2_command = subprocess.Popen(self.tar2_command, stdin=subprocess.PIPE, stderr=subprocess.PIPE) - pipe = open(self.restore_pipe,'r+b') if self.encrypted: # Start decrypt encryptor = subprocess.Popen (["openssl", "enc", "-d", "-aes-256-cbc", "-pass", "pass:"+passphrase], stdin=open(filename,'rb'), stdout=subprocess.PIPE) @@ -1364,11 +1364,13 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s pipe.close() - self.print_callback("Run error:"+run_error) - self.print_callback(str(self.tar2_command.poll())) - - if self.tar2_command.poll() != None: - if self.tar2_command.poll() != 0: + # tar2 input closed, wait for either it finishes, or prompt for the next + # file part; in both cases we can use read() on stderr - in the former case + # it will return "" (EOF) + tar2_stderr=self.tar2_command.stderr.readline() + if tar2_stderr == "": + # EOF, so collect process exit status + if self.tar2_command.wait() != 0: raise QubesException("ERROR: unable to extract files for {0}.".format(filename)) else: # Finished extracting the tar file From 661a1ba4afb5fee40987d98997a4d0129e2ae39f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 9 Nov 2013 17:12:14 +0100 Subject: [PATCH 41/82] backup: comment update --- core/qubesutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index 4fd3ed1a..42992f29 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -1215,7 +1215,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba ' - Returns if ' - one of the monitored processes error out (streamproc, hmac, vmproc, addproc), along with the processe that failed ' - all of the monitored processes except vmproc finished successfully (vmproc termination is controlled by the python script) -' - streamproc does not delivers any data anymore (return with the error "paused") +' - streamproc does not delivers any data anymore (return with the error "") ''' def wait_backup_feedback(progress_callback, in_stream, streamproc, backup_target, total_backup_sz, hmac=None, vmproc=None, addproc=None, remove_trailing_bytes=0): From c61a4570e867b780dcef592324a5b94fb1e1da0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 9 Nov 2013 17:12:41 +0100 Subject: [PATCH 42/82] backup: use 'dom0' as source domain for RPC calls --- core/qubesutils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index 42992f29..6e28cec5 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -1400,7 +1400,7 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s vmproc = None if appvm != None: # Prepare the backup target (Qubes service call) - backup_target = "QUBESRPC qubes.Restore none" + backup_target = "QUBESRPC qubes.Restore dom0" # does the vm exist? qvm_collection = QubesVmCollection() @@ -1524,7 +1524,7 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) if appvm != None: # Prepare the backup target (Qubes service call) - restore_command = "QUBESRPC qubes.Restore none" + restore_command = "QUBESRPC qubes.Restore dom0" # does the vm exist? qvm_collection = QubesVmCollection() From 3c993c619c9e9d4769406b327cd1200d2a9e084f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 9 Nov 2013 17:20:18 +0100 Subject: [PATCH 43/82] backup: restore: handle VM data with qfile format instead of simple tar This was already partially implemented, but only for backup header (qubes.xml). Fix handling of vmproc object (available only when backup in another VM). Also fix some race conditions - wait for process termination, not only check its exit code (which would be None if process still running). --- core/qubesutils.py | 84 +++++++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 39 deletions(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index 6e28cec5..a19dff1f 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -1417,22 +1417,41 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s vmproc = vm.run(command = backup_target, passio_popen = True) vmproc.stdin.write(backup_dir.replace("\r","").replace("\n","")+"\n") backup_stdin = vmproc.stdout + tar1_command = ['/usr/libexec/qubes/qfile-dom0-unpacker', str(os.getuid()), backup_tmpdir, '-v'] else: backup_stdin = open(backup_dir,'rb') - # FIXME: Use a safer program such as cpio, modified uncompress.c, or try to extract it from - tar1_command = ['tar', '-i', '-xv', '-C', backup_tmpdir] - tar1_command.extend(vms_dirs) + tar1_command = ['tar', '-i', '-xvf', backup_dir, '-C', backup_tmpdir] + + # Remove already processed qubes.xml.000, because qfile-dom0-unpacker will + # refuse to override files + os.unlink(os.path.join(backup_tmpdir,'qubes.xml.000')) + os.unlink(os.path.join(backup_tmpdir,'qubes.xml.000.hmac')) + tar1_env = os.environ.copy() + # TODO: add some safety margin? + tar1_env['UPDATES_MAX_BYTES'] = str(vms_size) + tar1_env['UPDATES_MAX_FILES'] = '0' print_callback("Run command"+str(tar1_command)) - command = subprocess.Popen(tar1_command, stdin=backup_stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + command = subprocess.Popen(tar1_command, + stdin=backup_stdin, + stdout=vmproc.stdin if vmproc else subprocess.PIPE, + stderr=subprocess.PIPE, + env=tar1_env) + + # qfile-dom0-unpacker output filelist on stderr (and have stdout connected + # to the VM), while tar output filelist on stdout + if appvm: + filelist_pipe = command.stderr + else: + filelist_pipe = command.stdout while command.poll() == None and vmproc.poll() == None: - filename = command.stdout.readline().strip(" \t\r\n") + filename = filelist_pipe.readline().strip(" \t\r\n") print_callback("Getting new file:"+filename) - hmacfile = command.stdout.readline().strip(" \t\r\n") + hmacfile = filelist_pipe.readline().strip(" \t\r\n") print_callback("Getting hmac:"+hmacfile) @@ -1492,20 +1511,6 @@ def load_hmac(hmac): return hmac -import struct -def get_qfile_error(buffer): - error = struct.unpack("I",buffer[0:4])[0] - error_msg = { 0: "COPY_FILE_OK", - 1: "COPY_FILE_READ_EOF", - 2: "COPY_FILE_READ_ERROR", - 3: "COPY_FILE_WRITE_ERROR", - } - - if error in error_msg.keys(): - return error_msg[error] - else: - return "UNKNOWN_ERROR_"+str(error) - def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None): # Simulate dd if=backup_file count=10 | file - # Simulate dd if=backup_file count=10 | gpg2 -d | tar xzv -O @@ -1522,6 +1527,8 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) # For this reason, we will use named pipes instead print "Working in",backup_tmpdir + + tar1_env = os.environ.copy() if appvm != None: # Prepare the backup target (Qubes service call) restore_command = "QUBESRPC qubes.Restore dom0" @@ -1540,20 +1547,25 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) # If APPVM, STDOUT is a PIPE vmproc = vm.run(command = restore_command, passio_popen = True, passio_stderr = True) vmproc.stdin.write(restore_target.replace("\r","").replace("\n","")+"\n") + tar1_command = ['/usr/libexec/qubes/qfile-dom0-unpacker', str(os.getuid()), backup_tmpdir, '-v'] + # extract only qubes.xml.000 and qubes.xml.000.hmac + tar1_env['UPDATES_MAX_FILES'] = '2' else: - # Create the target directory + # Check source file if not os.path.exists (restore_target): raise QubesException("ERROR: the backup directory {0} does not exists".format(restore_target)) - fp = open(restore_target,'rb') - headers = fp.read(4096*16) + # TODO: perhaps pass only first 40kB here? Tar uses seek to skip files, + # so not a big problem, but still it might save some time + tar1_command = ['tar', '-i', '-xvf', restore_target, '-C', backup_tmpdir, 'qubes.xml.000', 'qubes.xml.000.hmac'] - tar1_command = ['/usr/lib/qubes/qfile-dom0-unpacker', str(os.getuid()), backup_tmpdir] - command = subprocess.Popen(tar1_command,stdin=vmproc.stdout,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + command = subprocess.Popen(tar1_command, + stdin=vmproc.stdout if vmproc else None, + stdout=vmproc.stdin if vmproc else subprocess.PIPE, + stderr=subprocess.PIPE, + env=tar1_env) - result_header = command.stdout.read() - - if vmproc.poll() != None: + if vmproc and vmproc.poll() != None: error = vmproc.stderr.read() print error print vmproc.poll(),command.poll() @@ -1561,29 +1573,23 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) filename = "qubes.xml.000" - print result_header.encode("hex") - error_msg = get_qfile_error(result_header) - if error_msg != "COPY_FILE_OK": - print vmproc.stdout.read() - raise QubesException("ERROR: unpacking backup headers: {0}".format(error_msg)) + command.wait() + if not os.path.exists(os.path.join(backup_tmpdir,filename+".hmac")): raise QubesException("ERROR: header not extracted correctly: {0}".format(os.path.join(backup_tmpdir,filename+".hmac"))) - command.terminate() - command.wait() - - if vmproc.poll() != None and vmproc.poll() != 0: + if vmproc and vmproc.poll() != None and vmproc.poll() != 0: error = vmproc.stderr.read() print error print vmproc.poll(),command.poll() raise QubesException("ERROR: VM error retrieving backup headers") - elif command.poll() != None and command.poll() not in [0,-15]: + elif command.returncode not in [0,-15,122]: error = command.stderr.read() print error print vmproc.poll(),command.poll() raise QubesException("ERROR: retrieving backup headers:{0}".format(error)) - if vmproc.poll() == None: + if vmproc and vmproc.poll() == None: vmproc.terminate() vmproc.wait() From 1880f61c2d668e3ec0e476fad7846487ce7131eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 9 Nov 2013 18:53:57 +0100 Subject: [PATCH 44/82] backup: restore: process files until EOF received Not only until unpacker process is running. This is another race condition, which would cause some data left in the pipe buffer not processed. --- core/qubesutils.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index a19dff1f..4ad305cd 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -1445,18 +1445,24 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s else: filelist_pipe = command.stdout - while command.poll() == None and vmproc.poll() == None: + while True: filename = filelist_pipe.readline().strip(" \t\r\n") print_callback("Getting new file:"+filename) + if not filename or filename=="EOF": + break + hmacfile = filelist_pipe.readline().strip(" \t\r\n") - print_callback("Getting hmac:"+hmacfile) - - print_callback("Verifying file"+filename) + if hmacfile != filename + ".hmac": + raise QubesException("ERROR: expected hmac for {}, but got {}".format(filename, hmacfile)) + + print_callback("Verifying file "+filename) + + print os.path.join(backup_tmpdir,filename) hmac_proc = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], stdin=open(os.path.join(backup_tmpdir,filename),'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout,stderr = hmac_proc.communicate() @@ -1474,11 +1480,11 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s else: raise QubesException("ERROR: invalid hmac for file {0}: {1}. Is the passphrase correct?".format(filename,load_hmac(stdout))) - if command.poll() != 0: - raise QubesException("ERROR: unable to read the qubes backup file {0}. Is it really a backup?".format(backup_dir)) - if vmproc.poll() != 0: - raise QubesException("ERROR: unable to read the qubes backup {0} because of a VM error: {1}".format(backup_dir,vmproc.stderr.read())) - + if command.wait() != 0: + raise QubesException("ERROR: unable to read the qubes backup file {0} ({1}). Is it really a backup?".format(backup_dir, command.wait())) + if vmproc: + if vmproc.wait() != 0: + raise QubesException("ERROR: unable to read the qubes backup {0} because of a VM error: {1}".format(backup_dir,vmproc.stderr.read())) print "Extraction process status:",extract_proc.exitcode to_extract.put("FINISHED") From 319158d5b1c60d9a9b3bea6c3da0d5befe202ea4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 9 Nov 2013 18:54:52 +0100 Subject: [PATCH 45/82] backup: restore: Ignore qubes.xml in the second restore pass Already processed in backup prepare phase). This is only because qfile-dom0-unpacker doesn't support selective unpack (like tar do). This should be extended to skip also VMs not selected for restore. --- core/qubesutils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/qubesutils.py b/core/qubesutils.py index 4ad305cd..2f1167b7 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -1460,6 +1460,13 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s if hmacfile != filename + ".hmac": raise QubesException("ERROR: expected hmac for {}, but got {}".format(filename, hmacfile)) + # skip qubes.xml after receiving its hmac to skip both of them + if filename == 'qubes.xml.000': + print_callback("Ignoring already processed qubes.xml") + continue + + # FIXME: skip VMs not selected for restore + print_callback("Verifying file "+filename) print os.path.join(backup_tmpdir,filename) From 3666d6ced9daf58a8143b840245b8524c9f7394b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 9 Nov 2013 18:56:08 +0100 Subject: [PATCH 46/82] backup: wait for process termination in restore header phase One more race condition, which could cause qvm-backup-restore hang. --- core/qubesutils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index 2f1167b7..f24cc7d7 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -1630,9 +1630,9 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) tarhead_command.wait() if encryptor: - if encryptor.poll() != 0: + if encryptor.wait() != 0: raise QubesException("ERROR: unable to decrypt file {0}".format(filename)) - if tarhead_command.poll() != 0: + if tarhead_command.wait() != 0: raise QubesException("ERROR: unable to extract the qubes.xml file. Is archive encrypted?") return (backup_tmpdir,"qubes.xml") From 8cffdea41f017d0d318eee8a060e067dd0028318 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sat, 9 Nov 2013 18:57:26 +0100 Subject: [PATCH 47/82] backup: move qubes.{Backup,Restore} services to core-agent-linux repo --- qubes-rpc/qubes.Backup | 23 ----------------------- qubes-rpc/qubes.Restore | 23 ----------------------- 2 files changed, 46 deletions(-) delete mode 100644 qubes-rpc/qubes.Backup delete mode 100644 qubes-rpc/qubes.Restore diff --git a/qubes-rpc/qubes.Backup b/qubes-rpc/qubes.Backup deleted file mode 100644 index 6e3f1d48..00000000 --- a/qubes-rpc/qubes.Backup +++ /dev/null @@ -1,23 +0,0 @@ -echo Starting Backupcopy -read args -echo Arguments: $args -if [ -d "$args" ] ; then - echo "Performing backup to directory $args" - TARGET="$args/qubes-backup-`date +'%Y-%d-%d-%H%M%S'`" - echo "Copying STDIN data to $TARGET" - cat > $TARGET -else - echo "Checking if arguments is matching a command" - COMMAND=`echo $args | cut -d ' ' -f 1` - TYPE=`type -t $COMMAND` - if [ "$TYPE" == "file" ] ; then - echo "Redirecting STDIN to $args" - # Parsing args to handle quotes correctly - # Dangerous method if args are uncontrolled - eval "set -- $args" - $@ - else - echo "Invalid command $COMMAND" - exit 1 - fi -fi diff --git a/qubes-rpc/qubes.Restore b/qubes-rpc/qubes.Restore deleted file mode 100644 index 96a76e53..00000000 --- a/qubes-rpc/qubes.Restore +++ /dev/null @@ -1,23 +0,0 @@ -echo Starting Restorecopy >2 -read args -echo Arguments: $args >2 -if [ -f "$args" ] ; then - echo "Performing restore from backup file $args" >2 - TARGET="$args" - echo "Copying $TARGET to STDOUT" >2 - cat $TARGET | /usr/lib/qubes/tar2qfile -else - echo "Checking if arguments is matching a command" >2 - COMMAND=`echo $args | cut -d ' ' -f 1` - TYPE=`type -t $COMMAND` - if [ "$TYPE" == "file" ] ; then - echo "Redirecting $args to STDOUT" >2 - # Parsing args to handle quotes correctly - # Dangerous method if args are uncontrolled - eval "set -- $args" - $@ - else - echo "Invalid command $COMMAND" >2 - exit 1 - fi -fi From e875ae9d06a581faf88c6503bd0e81ed4c559248 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Thu, 14 Nov 2013 20:53:04 +0100 Subject: [PATCH 48/82] backups: use tar2qfile filtering to enable partial backup restore --- core/qubesutils.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index f24cc7d7..95c53729 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -1416,6 +1416,13 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s # If APPVM, STDOUT is a PIPE vmproc = vm.run(command = backup_target, passio_popen = True) vmproc.stdin.write(backup_dir.replace("\r","").replace("\n","")+"\n") + + # Send to tar2qfile the VMs that should be extracted + vmpaths = [] + for vmobj in vms.values(): + vmpaths.append(vmobj.backup_path) + vmproc.stdin.write(" ".join(vmpaths)+"\n") + backup_stdin = vmproc.stdout tar1_command = ['/usr/libexec/qubes/qfile-dom0-unpacker', str(os.getuid()), backup_tmpdir, '-v'] else: @@ -1560,8 +1567,12 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) # If APPVM, STDOUT is a PIPE vmproc = vm.run(command = restore_command, passio_popen = True, passio_stderr = True) vmproc.stdin.write(restore_target.replace("\r","").replace("\n","")+"\n") + + # Ask to tar2qfile to only extract qubes.xml.* + vmproc.stdin.write("qubes.xml\n") + tar1_command = ['/usr/libexec/qubes/qfile-dom0-unpacker', str(os.getuid()), backup_tmpdir, '-v'] - # extract only qubes.xml.000 and qubes.xml.000.hmac + # Ask qfile-dom0-unpacker to extract only qubes.xml.000 and qubes.xml.000.hmac tar1_env['UPDATES_MAX_FILES'] = '2' else: # Check source file From 51f119326b4c02f473f4572ec89b7800b2faa2b5 Mon Sep 17 00:00:00 2001 From: Olivier MEDOC Date: Tue, 19 Nov 2013 13:41:38 +0100 Subject: [PATCH 49/82] backup: improved error handling during restore --- core/qubesutils.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index 95c53729..f4622970 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -1599,20 +1599,23 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) command.wait() - if not os.path.exists(os.path.join(backup_tmpdir,filename+".hmac")): - raise QubesException("ERROR: header not extracted correctly: {0}".format(os.path.join(backup_tmpdir,filename+".hmac"))) + # Let the time to vmproc process to crash + time.sleep(2) if vmproc and vmproc.poll() != None and vmproc.poll() != 0: error = vmproc.stderr.read() print error print vmproc.poll(),command.poll() - raise QubesException("ERROR: VM error retrieving backup headers") + raise QubesException("ERROR: AppVM error retrieving backup headers: {0}".format(error)) elif command.returncode not in [0,-15,122]: error = command.stderr.read() print error print vmproc.poll(),command.poll() raise QubesException("ERROR: retrieving backup headers:{0}".format(error)) + if not os.path.exists(os.path.join(backup_tmpdir,filename+".hmac")): + raise QubesException("ERROR: header not extracted correctly: {0}".format(os.path.join(backup_tmpdir,filename+".hmac"))) + if vmproc and vmproc.poll() == None: vmproc.terminate() vmproc.wait() @@ -1639,10 +1642,9 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) encryptor = None tarhead_command = subprocess.Popen(['tar', '--tape-length','1000000', '-xvf', os.path.join(backup_tmpdir,filename)]) - tarhead_command.wait() if encryptor: if encryptor.wait() != 0: - raise QubesException("ERROR: unable to decrypt file {0}".format(filename)) + raise QubesException("ERROR: unable to decrypt file {0}. Bad password or unencrypted archive?".format(filename)) if tarhead_command.wait() != 0: raise QubesException("ERROR: unable to extract the qubes.xml file. Is archive encrypted?") From e7701d9c5d5469dd00846314a7da8cdd34b284c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 24 Nov 2013 03:15:44 +0100 Subject: [PATCH 50/82] backup: check for disk space if target is local directory --- core/qubesutils.py | 9 --------- qvm-tools/qvm-backup | 13 +++++++++++++ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index f4622970..89ddc740 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -964,19 +964,10 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca fmt="{{0:-^{0}}}-+".format(f["width"] + 1) s += fmt.format('-') print_callback(s) - # TODO: check at least if backing up to local drive - ''' - stat = os.statvfs(base_backup_dir) - backup_fs_free_sz = stat.f_bsize * stat.f_bavail - print_callback("") - if (total_backup_sz > backup_fs_free_sz): - raise QubesException("Not enough space available on the backup filesystem!") if (there_are_running_vms): raise QubesException("Please shutdown all VMs before proceeding.") - print_callback("-> Available space: {0}".format(size_to_human(backup_fs_free_sz))) - ''' return files_to_backup def backup_do(base_backup_dir, files_to_backup, progress_callback = None): diff --git a/qvm-tools/qvm-backup b/qvm-tools/qvm-backup index 46d48096..b8ddac61 100755 --- a/qvm-tools/qvm-backup +++ b/qvm-tools/qvm-backup @@ -75,6 +75,19 @@ def main(): print >>sys.stderr, "ERROR: %s" % str(e) exit(1) + total_backup_sz = reduce(lambda size, file: size+file["size"], + files_to_backup, 0) + + if not options.appvm: + stat = os.statvfs(base_backup_dir) + backup_fs_free_sz = stat.f_bsize * stat.f_bavail + print + if (total_backup_sz > backup_fs_free_sz): + print >>sys.stderr, "ERROR: Not enough space available on the backup filesystem!" + exit(1) + + print "-> Available space: {0}".format(size_to_human(backup_fs_free_sz)) + prompt = raw_input ("Do you want to proceed? [y/N] ") if not (prompt == "y" or prompt == "Y"): exit (0) From 61b3a81e822bdb69f81b853595610e848d8c4f37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 24 Nov 2013 03:17:15 +0100 Subject: [PATCH 51/82] backup: remove unused argument from backup_prepare --- core/qubesutils.py | 2 +- qvm-tools/qvm-backup | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index 89ddc740..a5c8f71d 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -777,7 +777,7 @@ def file_to_backup (file_path, sz = None): assert dir == abs_base_dir return [ { "path" : file_path, "size": sz, "subdir": subdir} ] -def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_callback = print_stdout): +def backup_prepare(vms_list = None, exclude_list = [], print_callback = print_stdout): """If vms = None, include all (sensible) VMs; exclude_list is always applied""" files_to_backup = file_to_backup (system_path["qubes_store_filename"]) diff --git a/qvm-tools/qvm-backup b/qvm-tools/qvm-backup index b8ddac61..ef8994f6 100755 --- a/qvm-tools/qvm-backup +++ b/qvm-tools/qvm-backup @@ -70,7 +70,7 @@ def main(): files_to_backup = None try: - files_to_backup = backup_prepare(base_backup_dir, vms_list=vms, exclude_list=options.exclude_list) + files_to_backup = backup_prepare(vms_list=vms, exclude_list=options.exclude_list) except QubesException as e: print >>sys.stderr, "ERROR: %s" % str(e) exit(1) From 6c61e79ebf48ca6f55eb1aafad9a491046af3db4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 24 Nov 2013 03:19:11 +0100 Subject: [PATCH 52/82] backups: don't echo entered passwords --- qvm-tools/qvm-backup | 4 ++-- qvm-tools/qvm-backup-restore | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/qvm-tools/qvm-backup b/qvm-tools/qvm-backup index ef8994f6..cc76471e 100755 --- a/qvm-tools/qvm-backup +++ b/qvm-tools/qvm-backup @@ -26,6 +26,7 @@ from qubes.qubesutils import backup_prepare, backup_do_copy from optparse import OptionParser import os import sys +import getpass def print_progress(progress): print >> sys.stderr, "\r-> Backing up files: {0}%...".format (progress), @@ -92,8 +93,7 @@ def main(): if not (prompt == "y" or prompt == "Y"): exit (0) - passphrase = raw_input("Please enter the pass phrase that will be used to encrypt/verify the backup:\n") - passphrase = passphrase.replace("\r","").replace("\n","") + passphrase = getpass.getpass("Please enter the pass phrase that will be used to encrypt/verify the backup: ") try: backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callback=print_progress, encrypt=options.encrypt,appvm=options.appvm) diff --git a/qvm-tools/qvm-backup-restore b/qvm-tools/qvm-backup-restore index bf1c1253..6ac22088 100755 --- a/qvm-tools/qvm-backup-restore +++ b/qvm-tools/qvm-backup-restore @@ -30,6 +30,7 @@ from optparse import OptionParser import os import sys +import getpass def main(): usage = "usage: %prog [options] " @@ -95,8 +96,7 @@ def main(): restore_options['exclude'] = options.exclude - passphrase = raw_input("Please enter the pass phrase that will be used to decrypt/verify the backup:\n") - passphrase = passphrase.replace("\r","").replace("\n","") + passphrase = getpass.getpass("Please enter the pass phrase that will be used to decrypt/verify the backup: ") print >> sys.stderr, "Checking backup content..." restore_tmpdir,qubes_xml = backup_restore_header(backup_dir, passphrase, options.decrypt, appvm=options.appvm) From 3d1b40f25c73fab3bfe95e33ea1d56be1a92beb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 24 Nov 2013 03:23:27 +0100 Subject: [PATCH 53/82] backups: keep file without path in inner tar archive It is senseless to have full file path in multiple locations: - external archive - qubes.xml - internal archive Also it is more logical to have only "private.img" file in archive placed in "appvms/untrusted/private.img.000". Although this is rather cosmetic change for VMs data, it is required to backup arbitrary directory, like dom0 user home. Also use os.path.* instead of manual string operations (split, partition). It is more foolproof. --- core/qubesutils.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index a5c8f71d..2ffe5543 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -1104,7 +1104,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba for filename in files_to_backup: print "Backing up",filename - backup_tempfile = os.path.join(backup_tmpdir,filename["path"].split(os.path.normpath(system_path["qubes_base_dir"])+"/")[1]) + backup_tempfile = os.path.join(backup_tmpdir, filename["subdir"], os.path.basename(filename["path"])) print "Using temporary location:",backup_tempfile # Ensure the temporary directory exists @@ -1113,8 +1113,8 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba os.makedirs(os.path.dirname(backup_tempfile)) # The first tar cmd can use any complex feature as we want. Files will be verified before untaring this. - tar_cmdline = ["tar", "-Pc", "-f", backup_pipe,'--sparse','--tape-length',str(1000000),'-C',system_path["qubes_base_dir"], - filename["path"].split(os.path.normpath(system_path["qubes_base_dir"])+"/")[1] + tar_cmdline = ["tar", "-Pc", "-f", backup_pipe,'--sparse','--tape-length',str(1000000),'-C',os.path.dirname(filename["path"]), + os.path.basename(filename["path"]) ] print " ".join(tar_cmdline) @@ -1158,7 +1158,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba raise QubesException("Failed to perform backup: error with "+run_error) # Send the chunk to the backup target - to_send.put(chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]) + to_send.put(os.path.relpath(chunkfile, backup_tmpdir)) # Close HMAC hmac.stdin.close() @@ -1174,7 +1174,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba hmac_file.close() # Send the HMAC to the backup target - to_send.put(chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]+".hmac") + to_send.put(os.path.relpath(chunkfile, backup_tmpdir)+".hmac") if tar_sparse.poll() == None: # Release the next chunk @@ -1332,16 +1332,19 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s break self.print_callback("Extracting file "+filename+" to "+system_path["qubes_base_dir"]) + dirname = os.path.join(system_path["qubes_base_dir"], + os.path.dirname(os.path.relpath(filename))) + if not os.path.exists(dirname): + os.makedirs(dirname) pipe = open(self.restore_pipe,'r+b') if self.tar2_command == None: # FIXME: Make the extraction safer by avoiding to erase other vms: # - extracting directly to the target directory (based on the vm name and by using the --strip=2). # - ensuring that the leading slashs are ignored when extracting (can also be obtained by running with --strip ?) - # marmarek: use other (local) variable for command line - self.tar2_command = ['tar', '--tape-length','1000000', '-C', system_path["qubes_base_dir"], '-xvf', self.restore_pipe] - self.print_callback("Running command "+str(self.tar2_command)) - self.tar2_command = subprocess.Popen(self.tar2_command, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + tar2_cmdline = ['tar', '--tape-length','1000000', '-C', dirname, '-xvf', self.restore_pipe] + self.print_callback("Running command "+str(tar2_cmdline)) + self.tar2_command = subprocess.Popen(tar2_cmdline, stdin=subprocess.PIPE, stderr=subprocess.PIPE) if self.encrypted: # Start decrypt From 005db6a5ab2fdfc37464feaf8e714a8d95f6e844 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 24 Nov 2013 03:28:21 +0100 Subject: [PATCH 54/82] backups: fix race condition in "tape" change event during backup Ensure that outer tar/encryptor gets all the data *and EOF* before signalling inner tar to continue. Previously it could happen that inner tar begins to write next data chunk, while qvm-backup still holds previous data chunk open. --- core/qubesutils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index 2ffe5543..46d5fa60 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -1173,6 +1173,8 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba hmac_file.flush() hmac_file.close() + pipe.close() + # Send the HMAC to the backup target to_send.put(os.path.relpath(chunkfile, backup_tmpdir)+".hmac") @@ -1185,7 +1187,6 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba else: print "Finished tar sparse with error",tar_sparse.poll() - pipe.close() to_send.put("FINISHED") send_proc.join() From c64b6c04cef9c96cda260332b10062ac9c74828d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 24 Nov 2013 03:32:57 +0100 Subject: [PATCH 55/82] backups: make all the debug easy to disable --- core/qubesutils.py | 184 ++++++++++++++++++++++++++++++--------------- 1 file changed, 122 insertions(+), 62 deletions(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index 46d5fa60..a0d6b8cd 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -37,6 +37,8 @@ import xen.lowlevel.xc import xen.lowlevel.xl import xen.lowlevel.xs +BACKUP_DEBUG = True + def mbytes_to_kmg(size): if size > 1024: return "%d GiB" % (size/1024) @@ -1052,13 +1054,16 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba # Tar with tapelength does not deals well with stdout (close stdout between two tapes) # For this reason, we will use named pipes instead - print "Working in",backup_tmpdir + if BACKUP_DEBUG: + print "Working in",backup_tmpdir backup_pipe = os.path.join(backup_tmpdir,"backup_pipe") - print "Creating pipe in:",backup_pipe - print os.mkfifo(backup_pipe) + if BACKUP_DEBUG: + print "Creating pipe in:",backup_pipe + os.mkfifo(backup_pipe) - print "Will backup:",files_to_backup + if BACKUP_DEBUG: + print "Will backup:",files_to_backup # Setup worker to send encrypted data chunks to the backup_target from multiprocessing import Queue,Process @@ -1070,26 +1075,31 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba self.backup_stdout = backup_stdout def run(self): - print "Started sending thread" + if BACKUP_DEBUG: + print "Started sending thread" - print "Moving to temporary dir",self.base_dir + if BACKUP_DEBUG: + print "Moving to temporary dir",self.base_dir os.chdir(self.base_dir) for filename in iter(self.queue.get,None): if filename == "FINISHED": break - print "Sending file",filename + if BACKUP_DEBUG: + print "Sending file",filename # This tar used for sending data out need to be as simple, as simple, as featureless as possible. It will not be verified before untaring. tar_final_cmd = ["tar", "-cO", "--posix", "-C", self.base_dir, filename] final_proc = subprocess.Popen (tar_final_cmd, stdin=subprocess.PIPE, stdout=self.backup_stdout) final_proc.wait() # Delete the file as we don't need it anymore - print "Removing file",filename + if BACKUP_DEBUG: + print "Removing file",filename os.remove(filename) - print "Finished sending thread" + if BACKUP_DEBUG: + print "Finished sending thread" def compute_progress(new_size, total_backup_sz): global blocks_backedup @@ -1102,10 +1112,12 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba send_proc.start() for filename in files_to_backup: - print "Backing up",filename + if BACKUP_DEBUG: + print "Backing up",filename backup_tempfile = os.path.join(backup_tmpdir, filename["subdir"], os.path.basename(filename["path"])) - print "Using temporary location:",backup_tempfile + if BACKUP_DEBUG: + print "Using temporary location:",backup_tempfile # Ensure the temporary directory exists @@ -1117,7 +1129,8 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba os.path.basename(filename["path"]) ] - print " ".join(tar_cmdline) + if BACKUP_DEBUG: + print " ".join(tar_cmdline) # Tips: Popen(bufsize=0) # Pipe: tar-sparse | encryptor [| hmac] | tar | backup_target @@ -1151,7 +1164,8 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba chunkfile_p.close() - print "Wait_backup_feedback returned:",run_error + if BACKUP_DEBUG: + print "Wait_backup_feedback returned:",run_error if len(run_error) > 0: send_proc.terminate() @@ -1163,11 +1177,13 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba # Close HMAC hmac.stdin.close() hmac.wait() - print "HMAC proc return code:",hmac.poll() + if BACKUP_DEBUG: + print "HMAC proc return code:",hmac.poll() # Write HMAC data next to the chunk file hmac_data = hmac.stdout.read() - print "Writing hmac to",chunkfile+".hmac" + if BACKUP_DEBUG: + print "Writing hmac to",chunkfile+".hmac" hmac_file = open(chunkfile+".hmac",'w') hmac_file.write(hmac_data) hmac_file.flush() @@ -1180,13 +1196,14 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba if tar_sparse.poll() == None: # Release the next chunk - print "Release next chunk for process:",tar_sparse.poll() + if BACKUP_DEBUG: + print "Release next chunk for process:",tar_sparse.poll() #tar_sparse.stdout = subprocess.PIPE tar_sparse.stdin.write("\n") run_error="paused" else: - print "Finished tar sparse with error",tar_sparse.poll() - + if BACKUP_DEBUG: + print "Finished tar sparse with error",tar_sparse.poll() to_send.put("FINISHED") send_proc.join() @@ -1195,8 +1212,9 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba raise QubesException("Failed to send backup: error in the sending process") if vmproc: - print "VMProc1 proc return code:",vmproc.poll() - print "Sparse1 proc return code:",tar_sparse.poll() + if BACKUP_DEBUG: + print "VMProc1 proc return code:",vmproc.poll() + print "Sparse1 proc return code:",tar_sparse.poll() vmproc.stdin.close() ''' @@ -1244,6 +1262,7 @@ def wait_backup_feedback(progress_callback, in_stream, streamproc, backup_target if retcode != None: if retcode != 0: run_error = "VM" + #FIXME: ?? if BACKUP_DEBUG: print vmproc.stdout.read() else: # VM should run until the end @@ -1313,8 +1332,9 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s self.vmproc = vmproc self.restore_pipe = os.path.join(self.base_dir,"restore_pipe") - print "Creating pipe in:",self.restore_pipe - print os.mkfifo(self.restore_pipe) + if BACKUP_DEBUG: + print "Creating pipe in:",self.restore_pipe + os.mkfifo(self.restore_pipe) def compute_progress(self, new_size, total_size): self.blocks_backedup += new_size @@ -1323,18 +1343,19 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s self.progress_callback(progress) def run(self): - self.print_callback("Started sending thread") - - self.print_callback("Moving to dir "+self.base_dir) + if BACKUP_DEBUG: + self.print_callback("Started sending thread") + self.print_callback("Moving to dir "+self.base_dir) os.chdir(self.base_dir) for filename in iter(self.queue.get,None): if filename == "FINISHED": break - self.print_callback("Extracting file "+filename+" to "+system_path["qubes_base_dir"]) dirname = os.path.join(system_path["qubes_base_dir"], os.path.dirname(os.path.relpath(filename))) + if BACKUP_DEBUG: + self.print_callback("Extracting file "+filename+" to "+dirname) if not os.path.exists(dirname): os.makedirs(dirname) @@ -1343,8 +1364,12 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s # FIXME: Make the extraction safer by avoiding to erase other vms: # - extracting directly to the target directory (based on the vm name and by using the --strip=2). # - ensuring that the leading slashs are ignored when extracting (can also be obtained by running with --strip ?) - tar2_cmdline = ['tar', '--tape-length','1000000', '-C', dirname, '-xvf', self.restore_pipe] - self.print_callback("Running command "+str(tar2_cmdline)) + tar2_cmdline = ['tar', + '--tape-length','1000000', + '-C', dirname, + '-x%sf' % ("v" if BACKUP_DEBUG else ""), self.restore_pipe] + if BACKUP_DEBUG: + self.print_callback("Running command "+str(tar2_cmdline)) self.tar2_command = subprocess.Popen(tar2_cmdline, stdin=subprocess.PIPE, stderr=subprocess.PIPE) if self.encrypted: @@ -1372,14 +1397,17 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s self.tar2_command = None else: - self.print_callback("Releasing next chunck") + if BACKUP_DEBUG: + self.print_callback("Releasing next chunck") self.tar2_command.stdin.write("\n") # Delete the file as we don't need it anymore - self.print_callback("Removing file "+filename) + if BACKUP_DEBUG: + self.print_callback("Removing file "+filename) os.remove(filename) - self.print_callback("Finished extracting thread") + if BACKUP_DEBUG: + self.print_callback("Finished extracting thread") if progress_callback == None: def progress_callback(data): @@ -1389,7 +1417,8 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s extract_proc = Extract_Worker(to_extract, backup_tmpdir, passphrase, encrypted, vms_size, print_callback, error_callback, progress_callback) extract_proc.start() - print_callback("Working in temporary dir:"+backup_tmpdir) + if BACKUP_DEBUG: + print_callback("Working in temporary dir:"+backup_tmpdir) print_callback(str(vms_size)+" bytes to restore") vmproc = None @@ -1423,7 +1452,9 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s else: backup_stdin = open(backup_dir,'rb') - tar1_command = ['tar', '-i', '-xvf', backup_dir, '-C', backup_tmpdir] + tar1_command = ['tar', + '-ix%sf' % ("v" if BACKUP_DEBUG else ""), backup_dir, + '-C', backup_tmpdir] # Remove already processed qubes.xml.000, because qfile-dom0-unpacker will # refuse to override files @@ -1433,7 +1464,8 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s # TODO: add some safety margin? tar1_env['UPDATES_MAX_BYTES'] = str(vms_size) tar1_env['UPDATES_MAX_FILES'] = '0' - print_callback("Run command"+str(tar1_command)) + if BACKUP_DEBUG: + print_callback("Run command"+str(tar1_command)) command = subprocess.Popen(tar1_command, stdin=backup_stdin, stdout=vmproc.stdin if vmproc else subprocess.PIPE, @@ -1451,38 +1483,45 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s filename = filelist_pipe.readline().strip(" \t\r\n") - print_callback("Getting new file:"+filename) + if BACKUP_DEBUG: + print_callback("Getting new file:"+filename) if not filename or filename=="EOF": break hmacfile = filelist_pipe.readline().strip(" \t\r\n") - print_callback("Getting hmac:"+hmacfile) + if BACKUP_DEBUG: + print_callback("Getting hmac:"+hmacfile) if hmacfile != filename + ".hmac": raise QubesException("ERROR: expected hmac for {}, but got {}".format(filename, hmacfile)) # skip qubes.xml after receiving its hmac to skip both of them if filename == 'qubes.xml.000': - print_callback("Ignoring already processed qubes.xml") + if BACKUP_DEBUG: + print_callback("Ignoring already processed qubes.xml") continue # FIXME: skip VMs not selected for restore - print_callback("Verifying file "+filename) + if BACKUP_DEBUG: + print_callback("Verifying file "+filename) - print os.path.join(backup_tmpdir,filename) + if BACKUP_DEBUG: + print os.path.join(backup_tmpdir,filename) hmac_proc = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], stdin=open(os.path.join(backup_tmpdir,filename),'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout,stderr = hmac_proc.communicate() if len(stderr) > 0: raise QubesException("ERROR: verify file {0}: {1}".format((filename,stderr))) else: - print_callback("Loading hmac for file"+filename) + if BACKUP_DEBUG: + print_callback("Loading hmac for file"+filename) hmac = load_hmac(open(os.path.join(backup_tmpdir,filename+".hmac"),'r').read()) if len(hmac) > 0 and load_hmac(stdout) == hmac: - print_callback("File verification OK -> Sending file "+filename+" for extraction") + if BACKUP_DEBUG: + print_callback("File verification OK -> Sending file "+filename+" for extraction") # Send the chunk to the backup target to_extract.put(os.path.join(backup_tmpdir,filename)) @@ -1494,12 +1533,15 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s if vmproc: if vmproc.wait() != 0: raise QubesException("ERROR: unable to read the qubes backup {0} because of a VM error: {1}".format(backup_dir,vmproc.stderr.read())) - print "Extraction process status:",extract_proc.exitcode + if BACKUP_DEBUG: + print "Extraction process status:",extract_proc.exitcode to_extract.put("FINISHED") - print_callback("Waiting for the extraction process to finish...") + if BACKUP_DEBUG: + print_callback("Waiting for the extraction process to finish...") extract_proc.join() - print_callback("Extraction process finished with code:"+str(extract_proc.exitcode)) + if BACKUP_DEBUG: + print_callback("Extraction process finished with code:"+str(extract_proc.exitcode)) if extract_proc.exitcode != 0: raise QubesException("ERROR: unable to extract the qubes backup. Check extracting process errors.") @@ -1540,7 +1582,8 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) # Tar with tapelength does not deals well with stdout (close stdout between two tapes) # For this reason, we will use named pipes instead - print "Working in",backup_tmpdir + if BACKUP_DEBUG: + print "Working in",backup_tmpdir tar1_env = os.environ.copy() @@ -1576,7 +1619,9 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) # TODO: perhaps pass only first 40kB here? Tar uses seek to skip files, # so not a big problem, but still it might save some time - tar1_command = ['tar', '-i', '-xvf', restore_target, '-C', backup_tmpdir, 'qubes.xml.000', 'qubes.xml.000.hmac'] + tar1_command = ['tar', + '-xi%sf' % ("v" if BACKUP_DEBUG else ""), restore_target, + '-C', backup_tmpdir, 'qubes.xml.000', 'qubes.xml.000.hmac'] command = subprocess.Popen(tar1_command, stdin=vmproc.stdout if vmproc else None, @@ -1586,8 +1631,9 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) if vmproc and vmproc.poll() != None: error = vmproc.stderr.read() - print error - print vmproc.poll(),command.poll() + if BACKUP_DEBUG: + print error + print vmproc.poll(),command.poll() raise QubesException("ERROR: Immediate VM error while retrieving backup headers:{0}".format(error)) filename = "qubes.xml.000" @@ -1599,13 +1645,15 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) if vmproc and vmproc.poll() != None and vmproc.poll() != 0: error = vmproc.stderr.read() - print error - print vmproc.poll(),command.poll() + if BACKUP_DEBUG: + print error + print vmproc.poll(),command.poll() raise QubesException("ERROR: AppVM error retrieving backup headers: {0}".format(error)) elif command.returncode not in [0,-15,122]: error = command.stderr.read() - print error - print vmproc.poll(),command.poll() + if BACKUP_DEBUG: + print error + print vmproc.poll(),command.poll() raise QubesException("ERROR: retrieving backup headers:{0}".format(error)) if not os.path.exists(os.path.join(backup_tmpdir,filename+".hmac")): @@ -1615,27 +1663,37 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) vmproc.terminate() vmproc.wait() - print "Loading hmac for file",filename + if BACKUP_DEBUG: + print "Loading hmac for file",filename hmac = load_hmac(open(os.path.join(backup_tmpdir,filename+".hmac"),'r').read()) - print "Successfully retrieved headers" + if BACKUP_DEBUG: + print "Successfully retrieved headers" - print "Verifying file",filename + if BACKUP_DEBUG: + print "Verifying file",filename hmac_proc = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], stdin=open(os.path.join(backup_tmpdir,filename),'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout,stderr = hmac_proc.communicate() if len(stderr) > 0: raise QubesException("ERROR: verify file {0}: {1}".format((filename,stderr))) else: if len(hmac) > 0 and load_hmac(stdout) == hmac: - print "File verification OK -> Extracting archive",filename + if BACKUP_DEBUG: + print "File verification OK -> Extracting archive",filename if encrypt: - print "Starting decryption process" + if BACKUP_DEBUG: + print "Starting decryption process" encryptor = subprocess.Popen (["openssl", "enc", "-d", "-aes-256-cbc", "-pass", "pass:"+passphrase], stdin=open(os.path.join(backup_tmpdir,filename),'rb'), stdout=subprocess.PIPE) - tarhead_command = subprocess.Popen(['tar', '--tape-length','1000000', '-xv'],stdin=encryptor.stdout) + tarhead_command = subprocess.Popen(['tar', + '--tape-length','1000000', + '-x%s' % ("v" if BACKUP_DEBUG else "")],stdin=encryptor.stdout) else: - print "No decryption process required" + if BACKUP_DEBUG: + print "No decryption process required" encryptor = None - tarhead_command = subprocess.Popen(['tar', '--tape-length','1000000', '-xvf', os.path.join(backup_tmpdir,filename)]) + tarhead_command = subprocess.Popen(['tar', + '--tape-length', '1000000', + '-x%sf' % ("v" if BACKUP_DEBUG else ""), os.path.join(backup_tmpdir,filename)]) if encryptor: if encryptor.wait() != 0: @@ -1673,7 +1731,8 @@ def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, host return template #### Private functions end - print "Loading file",qubes_xml + if BACKUP_DEBUG: + print "Loading file",qubes_xml backup_collection = QubesVmCollection(store_filename = qubes_xml) backup_collection.lock_db_for_reading() backup_collection.load() @@ -1696,7 +1755,8 @@ def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, host # ... and the actual data for vm in backup_vms_list: if is_vm_included_in_backup (backup_dir, vm): - print vm.name,"is included in backup" + if BACKUP_DEBUG: + print vm.name,"is included in backup" vms_to_restore[vm.name] = {} vms_to_restore[vm.name]['vm'] = vm; From 5477aea877cfeb42a8149833045337a82866da49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 24 Nov 2013 03:44:30 +0100 Subject: [PATCH 56/82] backups: increase buffer size for better performance After this change the bottleneck is qrexec throughput. --- core/qubesutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index a0d6b8cd..eafac982 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -1229,7 +1229,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba ''' def wait_backup_feedback(progress_callback, in_stream, streamproc, backup_target, total_backup_sz, hmac=None, vmproc=None, addproc=None, remove_trailing_bytes=0): - buffer_size = 4096 + buffer_size = 409600 run_error = None run_count = 1 From c306b9c00af82f65206365401d26f066c86ce48a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 24 Nov 2013 03:45:40 +0100 Subject: [PATCH 57/82] backups: increase readability of long function calls --- core/qubesutils.py | 13 +++++++++++-- qvm-tools/qvm-backup | 5 ++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index eafac982..af6efac7 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -1963,8 +1963,17 @@ def backup_restore_do(backup_dir, restore_tmpdir, passphrase, restore_info, host vms_dirs.append(vm.backup_path+"*") vms[vm.name] = vm - restore_vm_dirs (backup_dir, restore_tmpdir, passphrase, vms_dirs, vms, vms_size, print_callback, error_callback, progress_callback, encrypted, appvm) - + restore_vm_dirs (backup_dir, + restore_tmpdir, + passphrase=passphrase, + vms_dirs=vms_dirs, + vms=vms, + vms_size=vms_size, + print_callback=print_callback, + error_callback=error_callback, + progress_callback=progress_callback, + encrypted=encrypted, + appvm=appvm) # Add VM in right order for (vm_class_name, vm_class) in sorted(QubesVmClasses.items(), diff --git a/qvm-tools/qvm-backup b/qvm-tools/qvm-backup index cc76471e..6f96a97d 100755 --- a/qvm-tools/qvm-backup +++ b/qvm-tools/qvm-backup @@ -96,7 +96,10 @@ def main(): passphrase = getpass.getpass("Please enter the pass phrase that will be used to encrypt/verify the backup: ") try: - backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callback=print_progress, encrypt=options.encrypt,appvm=options.appvm) + backup_do_copy(base_backup_dir, files_to_backup, passphrase, + progress_callback=print_progress, + encrypt=options.encrypt, + appvm=options.appvm) except QubesException as e: print >>sys.stderr, "ERROR: %s" % str(e) exit(1) From a64f7c12adc69d5c39be27e58e51824bff0f6e0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 24 Nov 2013 04:16:29 +0100 Subject: [PATCH 58/82] backups: desperate try to improve readability Especially kill long lines. --- core/qubesutils.py | 346 +++++++++++++++++++++++++++++++-------------- 1 file changed, 239 insertions(+), 107 deletions(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index af6efac7..a4ab4ee7 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -1006,7 +1006,8 @@ def backup_do(base_backup_dir, files_to_backup, progress_callback = None): progress = bytes_backedup * 100 / total_backup_sz progress_callback(progress) -def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callback = None, encrypt=False, appvm=None): +def backup_do_copy(base_backup_dir, files_to_backup, passphrase,\ + progress_callback = None, encrypt=False, appvm=None): total_backup_sz = 0 for file in files_to_backup: total_backup_sz += file["size"] @@ -1029,16 +1030,20 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba # If APPVM, STDOUT is a PIPE vmproc = vm.run(command = backup_target, passio_popen = True) - vmproc.stdin.write(base_backup_dir.replace("\r","").replace("\n","")+"\n") + vmproc.stdin.write(base_backup_dir.\ + replace("\r","").replace("\n","")+"\n") backup_stdout = vmproc.stdin else: # Prepare the backup target (local file) - backup_target = base_backup_dir + "/qubes-{0}".format (time.strftime("%Y-%m-%d-%H%M%S")) + backup_target = base_backup_dir + "/qubes-{0}".\ + format (time.strftime("%Y-%m-%d-%H%M%S")) # Create the target directory if not os.path.exists (base_backup_dir): - raise QubesException("ERROR: the backup directory {0} does not exists".format(base_backup_dir)) + raise QubesException( + "ERROR: the backup directory {0} does not exists".\ + format(base_backup_dir)) # If not APPVM, STDOUT is a local file backup_stdout = open(backup_target,'wb') @@ -1052,23 +1057,24 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba feedback_file = tempfile.NamedTemporaryFile() backup_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/backup_") - # Tar with tapelength does not deals well with stdout (close stdout between two tapes) + # Tar with tapelength does not deals well with stdout (close stdout between + # two tapes) # For this reason, we will use named pipes instead if BACKUP_DEBUG: - print "Working in",backup_tmpdir + print "Working in", backup_tmpdir backup_pipe = os.path.join(backup_tmpdir,"backup_pipe") if BACKUP_DEBUG: - print "Creating pipe in:",backup_pipe + print "Creating pipe in:", backup_pipe os.mkfifo(backup_pipe) if BACKUP_DEBUG: - print "Will backup:",files_to_backup + print "Will backup:", files_to_backup # Setup worker to send encrypted data chunks to the backup_target from multiprocessing import Queue,Process class Send_Worker(Process): - def __init__(self,queue,base_dir,backup_stdout): + def __init__(self, queue, base_dir, backup_stdout): super(Send_Worker, self).__init__() self.queue = queue self.base_dir = base_dir @@ -1079,7 +1085,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba print "Started sending thread" if BACKUP_DEBUG: - print "Moving to temporary dir",self.base_dir + print "Moving to temporary dir", self.base_dir os.chdir(self.base_dir) for filename in iter(self.queue.get,None): @@ -1087,15 +1093,19 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba break if BACKUP_DEBUG: - print "Sending file",filename - # This tar used for sending data out need to be as simple, as simple, as featureless as possible. It will not be verified before untaring. - tar_final_cmd = ["tar", "-cO", "--posix", "-C", self.base_dir, filename] - final_proc = subprocess.Popen (tar_final_cmd, stdin=subprocess.PIPE, stdout=self.backup_stdout) + print "Sending file", filename + # This tar used for sending data out need to be as simple, as + # simple, as featureless as possible. It will not be + # verified before untaring. + tar_final_cmd = ["tar", "-cO", "--posix", + "-C", self.base_dir, filename] + final_proc = subprocess.Popen (tar_final_cmd, + stdin=subprocess.PIPE, stdout=self.backup_stdout) final_proc.wait() # Delete the file as we don't need it anymore if BACKUP_DEBUG: - print "Removing file",filename + print "Removing file", filename os.remove(filename) if BACKUP_DEBUG: @@ -1113,19 +1123,25 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba for filename in files_to_backup: if BACKUP_DEBUG: - print "Backing up",filename + print "Backing up", filename - backup_tempfile = os.path.join(backup_tmpdir, filename["subdir"], os.path.basename(filename["path"])) + backup_tempfile = os.path.join(backup_tmpdir, + filename["subdir"], + os.path.basename(filename["path"])) if BACKUP_DEBUG: - print "Using temporary location:",backup_tempfile + print "Using temporary location:", backup_tempfile # Ensure the temporary directory exists if not os.path.isdir(os.path.dirname(backup_tempfile)): os.makedirs(os.path.dirname(backup_tempfile)) - # The first tar cmd can use any complex feature as we want. Files will be verified before untaring this. - tar_cmdline = ["tar", "-Pc", "-f", backup_pipe,'--sparse','--tape-length',str(1000000),'-C',os.path.dirname(filename["path"]), + # The first tar cmd can use any complex feature as we want. Files will + # be verified before untaring this. + tar_cmdline = ["tar", "-Pc", '--sparse' + "-f", backup_pipe, + '--tape-length', str(1000000), + '-C', os.path.dirname(filename["path"]), os.path.basename(filename["path"]) ] @@ -1135,10 +1151,11 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba # Tips: Popen(bufsize=0) # Pipe: tar-sparse | encryptor [| hmac] | tar | backup_target # Pipe: tar-sparse [| hmac] | tar | backup_target - tar_sparse = subprocess.Popen (tar_cmdline,stdin=subprocess.PIPE) + tar_sparse = subprocess.Popen (tar_cmdline, stdin=subprocess.PIPE) - # Wait for compressor (tar) process to finish or for any error of other subprocesses - i=0 + # Wait for compressor (tar) process to finish or for any error of other + # subprocesses + i = 0 run_error = "paused" running = [] while run_error == "paused": @@ -1146,7 +1163,8 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba pipe = open(backup_pipe,'rb') # Start HMAC - hmac = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], stdin=subprocess.PIPE, stdout=subprocess.PIPE) + hmac = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], + stdin=subprocess.PIPE, stdout=subprocess.PIPE) # Prepare a first chunk chunkfile = backup_tempfile + "." + "%03d" % i @@ -1157,19 +1175,39 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba # Start encrypt # If no cipher is provided, the data is forwarded unencrypted !!! # Also note that the - encryptor = subprocess.Popen (["openssl", "enc", "-e", "-aes-256-cbc", "-pass", "pass:"+passphrase], stdin=pipe, stdout=subprocess.PIPE) - run_error = wait_backup_feedback(compute_progress, encryptor.stdout, encryptor, chunkfile_p, total_backup_sz, hmac=hmac, vmproc=vmproc, addproc=tar_sparse) + encryptor = subprocess.Popen (["openssl", "enc", + "-e", "-aes-256-cbc", + "-pass", "pass:"+passphrase], + stdin=pipe, stdout=subprocess.PIPE) + run_error = wait_backup_feedback( + progress_callback=compute_progress, + in_stream=encryptor.stdout, + streamproc=encryptor, + backup_target=chunkfile_p, + total_backup_sz=total_backup_sz, + hmac=hmac, + vmproc=vmproc, + addproc=tar_sparse) else: - run_error = wait_backup_feedback(compute_progress, pipe, None, chunkfile_p, total_backup_sz, hmac=hmac, vmproc=vmproc, addproc=tar_sparse) + run_error = wait_backup_feedback( + progress_callback=compute_progress, + in_stream=pipe, + streamproc=None, + backup_target=chunkfile_p, + total_backup_sz=total_backup_sz, + hmac=hmac, + vmproc=vmproc, + addproc=tar_sparse) chunkfile_p.close() if BACKUP_DEBUG: - print "Wait_backup_feedback returned:",run_error + print "Wait_backup_feedback returned:", run_error if len(run_error) > 0: send_proc.terminate() - raise QubesException("Failed to perform backup: error with "+run_error) + raise QubesException("Failed to perform backup: error with "+ \ + run_error) # Send the chunk to the backup target to_send.put(os.path.relpath(chunkfile, backup_tmpdir)) @@ -1178,12 +1216,12 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba hmac.stdin.close() hmac.wait() if BACKUP_DEBUG: - print "HMAC proc return code:",hmac.poll() + print "HMAC proc return code:", hmac.poll() # Write HMAC data next to the chunk file hmac_data = hmac.stdout.read() if BACKUP_DEBUG: - print "Writing hmac to",chunkfile+".hmac" + print "Writing hmac to", chunkfile+".hmac" hmac_file = open(chunkfile+".hmac",'w') hmac_file.write(hmac_data) hmac_file.flush() @@ -1197,13 +1235,13 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba if tar_sparse.poll() == None: # Release the next chunk if BACKUP_DEBUG: - print "Release next chunk for process:",tar_sparse.poll() + print "Release next chunk for process:", tar_sparse.poll() #tar_sparse.stdout = subprocess.PIPE tar_sparse.stdin.write("\n") run_error="paused" else: if BACKUP_DEBUG: - print "Finished tar sparse with error",tar_sparse.poll() + print "Finished tar sparse with error", tar_sparse.poll() to_send.put("FINISHED") send_proc.join() @@ -1213,21 +1251,27 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callba if vmproc: if BACKUP_DEBUG: - print "VMProc1 proc return code:",vmproc.poll() - print "Sparse1 proc return code:",tar_sparse.poll() + print "VMProc1 proc return code:", vmproc.poll() + print "Sparse1 proc return code:", tar_sparse.poll() vmproc.stdin.close() ''' ' Wait for backup chunk to finish ' - Monitor all the processes (streamproc, hmac, vmproc, addproc) for errors ' - Copy stdout of streamproc to backup_target and hmac stdin if available -' - Compute progress based on total_backup_sz and send progress to progress_callback function +' - Compute progress based on total_backup_sz and send progress to +' progress_callback function ' - Returns if -' - one of the monitored processes error out (streamproc, hmac, vmproc, addproc), along with the processe that failed -' - all of the monitored processes except vmproc finished successfully (vmproc termination is controlled by the python script) -' - streamproc does not delivers any data anymore (return with the error "") +' - one of the monitored processes error out (streamproc, hmac, vmproc, +' addproc), along with the processe that failed +' - all of the monitored processes except vmproc finished successfully +' (vmproc termination is controlled by the python script) +' - streamproc does not delivers any data anymore (return with the error +' "") ''' -def wait_backup_feedback(progress_callback, in_stream, streamproc, backup_target, total_backup_sz, hmac=None, vmproc=None, addproc=None, remove_trailing_bytes=0): +def wait_backup_feedback(progress_callback, in_stream, streamproc, + backup_target, total_backup_sz, hmac=None, vmproc=None, addproc=None, + remove_trailing_bytes=0): buffer_size = 409600 @@ -1237,7 +1281,7 @@ def wait_backup_feedback(progress_callback, in_stream, streamproc, backup_target while run_count > 0 and run_error == None: buffer = in_stream.read(buffer_size) - progress_callback(len(buffer),total_backup_sz) + progress_callback(len(buffer), total_backup_sz) run_count = 0 if hmac: @@ -1250,7 +1294,7 @@ def wait_backup_feedback(progress_callback, in_stream, streamproc, backup_target if addproc: retcode=addproc.poll() - #print "Tar proc status:",retcode + #print "Tar proc status:", retcode if retcode != None: if retcode != 0: run_error = "addproc" @@ -1289,7 +1333,7 @@ def wait_backup_feedback(progress_callback, in_stream, streamproc, backup_target run_count += 1 else: - #print "Process running:",len(buffer) + #print "Process running:", len(buffer) # Process still running backup_target.write(buffer) @@ -1310,12 +1354,15 @@ def wait_backup_feedback(progress_callback, in_stream, streamproc, backup_target return run_error -def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_size, print_callback=None, error_callback=None, progress_callback=None, encrypted=False, appvm=None): +def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, + vms_size, print_callback=None, error_callback=None, + progress_callback=None, encrypted=False, appvm=None): # Setup worker to extract encrypted data chunks to the restore dirs from multiprocessing import Queue,Process class Extract_Worker(Process): - def __init__(self,queue,base_dir,passphrase,encrypted,total_size,print_callback,error_callback,progress_callback,vmproc=None): + def __init__(self, queue, base_dir, passphrase, encrypted, total_size, + print_callback, error_callback, progress_callback, vmproc=None): super(Extract_Worker, self).__init__() self.queue = queue self.base_dir = base_dir @@ -1333,7 +1380,7 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s self.restore_pipe = os.path.join(self.base_dir,"restore_pipe") if BACKUP_DEBUG: - print "Creating pipe in:",self.restore_pipe + print "Creating pipe in:", self.restore_pipe os.mkfifo(self.restore_pipe) def compute_progress(self, new_size, total_size): @@ -1361,37 +1408,62 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s pipe = open(self.restore_pipe,'r+b') if self.tar2_command == None: - # FIXME: Make the extraction safer by avoiding to erase other vms: - # - extracting directly to the target directory (based on the vm name and by using the --strip=2). - # - ensuring that the leading slashs are ignored when extracting (can also be obtained by running with --strip ?) + # FIXME: Make the extraction safer by avoiding to erase + # other vms: + # - extracting directly to the target directory (based on + # the vm name and by using the --strip=2). + # - ensuring that the leading slashs are ignored when + # extracting (can also be obtained by running with --strip ?) tar2_cmdline = ['tar', '--tape-length','1000000', '-C', dirname, '-x%sf' % ("v" if BACKUP_DEBUG else ""), self.restore_pipe] if BACKUP_DEBUG: self.print_callback("Running command "+str(tar2_cmdline)) - self.tar2_command = subprocess.Popen(tar2_cmdline, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + self.tar2_command = subprocess.Popen(tar2_cmdline, + stdin=subprocess.PIPE, stderr=subprocess.PIPE) if self.encrypted: # Start decrypt - encryptor = subprocess.Popen (["openssl", "enc", "-d", "-aes-256-cbc", "-pass", "pass:"+passphrase], stdin=open(filename,'rb'), stdout=subprocess.PIPE) + encryptor = subprocess.Popen (["openssl", "enc", + "-d", "-aes-256-cbc", + "-pass", "pass:"+passphrase], + stdin=open(filename,'rb'), + stdout=subprocess.PIPE) - # progress_callback, in_stream, streamproc, backup_target, total_backup_sz, hmac=None, vmproc=None, addproc=None, remove_trailing_bytes=0): - run_error = wait_backup_feedback(self.compute_progress, encryptor.stdout, encryptor, pipe, self.total_size, hmac=None, vmproc=self.vmproc, addproc=self.tar2_command) - #print "End wait_backup_feedback",run_error,self.tar2_command.poll(),encryptor.poll() + run_error = wait_backup_feedback( + progress_callback=self.compute_progress, + in_stream=encryptor.stdout, + streamproc=encryptor, + backup_target=pipe, + total_backup_sz=self.total_size, + hmac=None, + vmproc=self.vmproc, + addproc=self.tar2_command) else: - run_error = wait_backup_feedback(self.compute_progress, open(filename,"rb"), None, pipe, self.total_size, hmac=None, vmproc=self.vmproc, addproc=self.tar2_command) + run_error = wait_backup_feedback( + progress_callback=self.compute_progress, + in_stream=open(filename,"rb"), + streamproc=None, + backup_target=pipe, + total_backup_sz=self.total_size, + hmac=None, + vmproc=self.vmproc, + addproc=self.tar2_command) + pipe.close() - # tar2 input closed, wait for either it finishes, or prompt for the next - # file part; in both cases we can use read() on stderr - in the former case - # it will return "" (EOF) + # tar2 input closed, wait for either it finishes, or prompt for + # the next file part; in both cases we can use read() on stderr + # - in the former case it will return "" (EOF) tar2_stderr=self.tar2_command.stderr.readline() if tar2_stderr == "": # EOF, so collect process exit status if self.tar2_command.wait() != 0: - raise QubesException("ERROR: unable to extract files for {0}.".format(filename)) + raise QubesException( + "ERROR: unable to extract files for {0}.".\ + format(filename)) else: # Finished extracting the tar file self.tar2_command = None @@ -1414,7 +1486,14 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s pass to_extract = Queue() - extract_proc = Extract_Worker(to_extract, backup_tmpdir, passphrase, encrypted, vms_size, print_callback, error_callback, progress_callback) + extract_proc = Extract_Worker(queue=to_extract, + base_dir=backup_tmpdir, + passphrase=passphrase, + encrypted=encrypted, + total_size=vms_size, + print_callback=print_callback, + error_callback=error_callback, + progress_callback=progress_callback) extract_proc.start() if BACKUP_DEBUG: @@ -1448,7 +1527,8 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s vmproc.stdin.write(" ".join(vmpaths)+"\n") backup_stdin = vmproc.stdout - tar1_command = ['/usr/libexec/qubes/qfile-dom0-unpacker', str(os.getuid()), backup_tmpdir, '-v'] + tar1_command = ['/usr/libexec/qubes/qfile-dom0-unpacker', + str(os.getuid()), backup_tmpdir, '-v'] else: backup_stdin = open(backup_dir,'rb') @@ -1494,7 +1574,9 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s print_callback("Getting hmac:"+hmacfile) if hmacfile != filename + ".hmac": - raise QubesException("ERROR: expected hmac for {}, but got {}".format(filename, hmacfile)) + raise QubesException( + "ERROR: expected hmac for {}, but got {}".\ + format(filename, hmacfile)) # skip qubes.xml after receiving its hmac to skip both of them if filename == 'qubes.xml.000': @@ -1508,42 +1590,58 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_s print_callback("Verifying file "+filename) if BACKUP_DEBUG: - print os.path.join(backup_tmpdir,filename) - hmac_proc = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], stdin=open(os.path.join(backup_tmpdir,filename),'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout,stderr = hmac_proc.communicate() + print os.path.join(backup_tmpdir, filename) + hmac_proc = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], + stdin=open(os.path.join(backup_tmpdir, filename),'rb'), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = hmac_proc.communicate() if len(stderr) > 0: - raise QubesException("ERROR: verify file {0}: {1}".format((filename,stderr))) + raise QubesException("ERROR: verify file {0}: {1}".format((filename, stderr))) else: if BACKUP_DEBUG: print_callback("Loading hmac for file"+filename) - hmac = load_hmac(open(os.path.join(backup_tmpdir,filename+".hmac"),'r').read()) + hmac = load_hmac( + open(os.path.join(backup_tmpdir, + filename+".hmac"),'r').read()) if len(hmac) > 0 and load_hmac(stdout) == hmac: if BACKUP_DEBUG: - print_callback("File verification OK -> Sending file "+filename+" for extraction") + print_callback("File verification OK -> Sending file " + \ + filename+" for extraction") # Send the chunk to the backup target - to_extract.put(os.path.join(backup_tmpdir,filename)) + to_extract.put(os.path.join(backup_tmpdir, filename)) else: - raise QubesException("ERROR: invalid hmac for file {0}: {1}. Is the passphrase correct?".format(filename,load_hmac(stdout))) + raise QubesException( + "ERROR: invalid hmac for file {0}: {1}. " \ + "Is the passphrase correct?".\ + format(filename, load_hmac(stdout))) if command.wait() != 0: - raise QubesException("ERROR: unable to read the qubes backup file {0} ({1}). Is it really a backup?".format(backup_dir, command.wait())) + raise QubesException( + "ERROR: unable to read the qubes backup file {0} ({1}). " \ + "Is it really a backup?".format(backup_dir, command.wait())) if vmproc: if vmproc.wait() != 0: - raise QubesException("ERROR: unable to read the qubes backup {0} because of a VM error: {1}".format(backup_dir,vmproc.stderr.read())) + raise QubesException( + "ERROR: unable to read the qubes backup {0} " \ + "because of a VM error: {1}".format( + backup_dir, vmproc.stderr.read())) if BACKUP_DEBUG: - print "Extraction process status:",extract_proc.exitcode + print "Extraction process status:", extract_proc.exitcode to_extract.put("FINISHED") if BACKUP_DEBUG: print_callback("Waiting for the extraction process to finish...") extract_proc.join() if BACKUP_DEBUG: - print_callback("Extraction process finished with code:"+str(extract_proc.exitcode)) + print_callback("Extraction process finished with code:" + \ + str(extract_proc.exitcode)) if extract_proc.exitcode != 0: - raise QubesException("ERROR: unable to extract the qubes backup. Check extracting process errors.") + raise QubesException( + "ERROR: unable to extract the qubes backup. " \ + "Check extracting process errors.") def backup_restore_set_defaults(options): if 'use-default-netvm' not in options: @@ -1568,7 +1666,8 @@ def load_hmac(hmac): return hmac -def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None): +def backup_restore_header(restore_target, passphrase, + encrypt=False, appvm=None): # Simulate dd if=backup_file count=10 | file - # Simulate dd if=backup_file count=10 | gpg2 -d | tar xzv -O # analysis = subprocess.Popen() @@ -1580,10 +1679,11 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) os.chdir(backup_tmpdir) - # Tar with tapelength does not deals well with stdout (close stdout between two tapes) + # Tar with tapelength does not deals well with stdout (close stdout between + # two tapes) # For this reason, we will use named pipes instead if BACKUP_DEBUG: - print "Working in",backup_tmpdir + print "Working in", backup_tmpdir tar1_env = os.environ.copy() @@ -1603,19 +1703,23 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) qvm_collection.unlock_db() # If APPVM, STDOUT is a PIPE - vmproc = vm.run(command = restore_command, passio_popen = True, passio_stderr = True) + vmproc = vm.run(command = restore_command, passio_popen = True, + passio_stderr = True) vmproc.stdin.write(restore_target.replace("\r","").replace("\n","")+"\n") # Ask to tar2qfile to only extract qubes.xml.* vmproc.stdin.write("qubes.xml\n") - tar1_command = ['/usr/libexec/qubes/qfile-dom0-unpacker', str(os.getuid()), backup_tmpdir, '-v'] + tar1_command = ['/usr/libexec/qubes/qfile-dom0-unpacker', + str(os.getuid()), backup_tmpdir, '-v'] # Ask qfile-dom0-unpacker to extract only qubes.xml.000 and qubes.xml.000.hmac tar1_env['UPDATES_MAX_FILES'] = '2' else: # Check source file if not os.path.exists (restore_target): - raise QubesException("ERROR: the backup directory {0} does not exists".format(restore_target)) + raise QubesException( + "ERROR: the backup directory {0} does not exists".\ + format(restore_target)) # TODO: perhaps pass only first 40kB here? Tar uses seek to skip files, # so not a big problem, but still it might save some time @@ -1633,8 +1737,10 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) error = vmproc.stderr.read() if BACKUP_DEBUG: print error - print vmproc.poll(),command.poll() - raise QubesException("ERROR: Immediate VM error while retrieving backup headers:{0}".format(error)) + print vmproc.poll(), command.poll() + raise QubesException( + "ERROR: Immediate VM error while retrieving backup headers:{0}".\ + format(error)) filename = "qubes.xml.000" @@ -1647,67 +1753,88 @@ def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None) error = vmproc.stderr.read() if BACKUP_DEBUG: print error - print vmproc.poll(),command.poll() - raise QubesException("ERROR: AppVM error retrieving backup headers: {0}".format(error)) + print vmproc.poll(), command.poll() + raise QubesException( + "ERROR: AppVM error retrieving backup headers: {0}".\ + format(error)) elif command.returncode not in [0,-15,122]: error = command.stderr.read() if BACKUP_DEBUG: print error - print vmproc.poll(),command.poll() - raise QubesException("ERROR: retrieving backup headers:{0}".format(error)) + print vmproc.poll(), command.poll() + raise QubesException("ERROR: retrieving backup headers: {0}".\ + format(error)) - if not os.path.exists(os.path.join(backup_tmpdir,filename+".hmac")): - raise QubesException("ERROR: header not extracted correctly: {0}".format(os.path.join(backup_tmpdir,filename+".hmac"))) + if not os.path.exists(os.path.join(backup_tmpdir, filename+".hmac")): + raise QubesException("ERROR: header not extracted correctly: {0}".\ + format(os.path.join(backup_tmpdir, filename+".hmac"))) if vmproc and vmproc.poll() == None: vmproc.terminate() vmproc.wait() if BACKUP_DEBUG: - print "Loading hmac for file",filename - hmac = load_hmac(open(os.path.join(backup_tmpdir,filename+".hmac"),'r').read()) + print "Loading hmac for file", filename + hmac = load_hmac( + open(os.path.join(backup_tmpdir, filename+".hmac"),'r').read()) if BACKUP_DEBUG: print "Successfully retrieved headers" if BACKUP_DEBUG: - print "Verifying file",filename - hmac_proc = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], stdin=open(os.path.join(backup_tmpdir,filename),'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout,stderr = hmac_proc.communicate() + print "Verifying file", filename + hmac_proc = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], + stdin=open(os.path.join(backup_tmpdir, filename),'rb'), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = hmac_proc.communicate() if len(stderr) > 0: - raise QubesException("ERROR: verify file {0}: {1}".format((filename,stderr))) + raise QubesException("ERROR: verify file {0}: {1}".format((filename, stderr))) else: if len(hmac) > 0 and load_hmac(stdout) == hmac: if BACKUP_DEBUG: - print "File verification OK -> Extracting archive",filename + print "File verification OK -> Extracting archive", filename if encrypt: if BACKUP_DEBUG: print "Starting decryption process" - encryptor = subprocess.Popen (["openssl", "enc", "-d", "-aes-256-cbc", "-pass", "pass:"+passphrase], stdin=open(os.path.join(backup_tmpdir,filename),'rb'), stdout=subprocess.PIPE) + encryptor = subprocess.Popen (["openssl", "enc", + "-d", "-aes-256-cbc", + "-pass", "pass:"+passphrase], + stdin=open(os.path.join(backup_tmpdir, filename),'rb'), + stdout=subprocess.PIPE) tarhead_command = subprocess.Popen(['tar', '--tape-length','1000000', - '-x%s' % ("v" if BACKUP_DEBUG else "")],stdin=encryptor.stdout) + '-x%s' % ("v" if BACKUP_DEBUG else "")], + stdin=encryptor.stdout) else: if BACKUP_DEBUG: print "No decryption process required" encryptor = None tarhead_command = subprocess.Popen(['tar', '--tape-length', '1000000', - '-x%sf' % ("v" if BACKUP_DEBUG else ""), os.path.join(backup_tmpdir,filename)]) + '-x%sf' % ("v" if BACKUP_DEBUG else ""), + os.path.join(backup_tmpdir, filename)]) if encryptor: if encryptor.wait() != 0: - raise QubesException("ERROR: unable to decrypt file {0}. Bad password or unencrypted archive?".format(filename)) + raise QubesException( + "ERROR: unable to decrypt file {0}. " \ + "Bad password or unencrypted archive?".\ + format(filename)) if tarhead_command.wait() != 0: - raise QubesException("ERROR: unable to extract the qubes.xml file. Is archive encrypted?") + raise QubesException( + "ERROR: unable to extract the qubes.xml file. " \ + "Is archive encrypted?") - return (backup_tmpdir,"qubes.xml") + return (backup_tmpdir, "qubes.xml") else: - raise QubesException("ERROR: unable to verify the qubes.xml file. Is the passphrase correct?") + raise QubesException( + "ERROR: unable to verify the qubes.xml file. " \ + "Is the passphrase correct?") return None -def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, host_collection = None, encrypt=False, appvm=None): +def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, + host_collection = None, encrypt=False, appvm=None): # Defaults backup_restore_set_defaults(options) @@ -1732,7 +1859,7 @@ def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, host return template #### Private functions end if BACKUP_DEBUG: - print "Loading file",qubes_xml + print "Loading file", qubes_xml backup_collection = QubesVmCollection(store_filename = qubes_xml) backup_collection.lock_db_for_reading() backup_collection.load() @@ -1807,7 +1934,9 @@ def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, host # Maybe the (custom) netvm is in the backup? netvm_on_backup = backup_collection.get_vm_by_name (netvm_name) - if not ((netvm_on_backup is not None) and netvm_on_backup.is_netvm() and is_vm_included_in_backup(backup_dir, netvm_on_backup)): + if not ((netvm_on_backup is not None) and \ + netvm_on_backup.is_netvm() and \ + is_vm_included_in_backup(backup_dir, netvm_on_backup)): if options['use-default-netvm']: vms_to_restore[vm.name]['netvm'] = host_collection.get_default_netvm().name vm.uses_default_netvm = True @@ -1940,7 +2069,10 @@ def backup_restore_print_summary(restore_info, print_callback = print_stdout): print_callback(s) -def backup_restore_do(backup_dir, restore_tmpdir, passphrase, restore_info, host_collection = None, print_callback = print_stdout, error_callback = print_stderr, progress_callback = None, encrypted=False, appvm=None): +def backup_restore_do(backup_dir, restore_tmpdir, passphrase, restore_info, + host_collection = None, print_callback = print_stdout, + error_callback = print_stderr, progress_callback = None, + encrypted=False, appvm=None): lock_obtained = False if host_collection is None: From dc6fd3c8f380c4d1a612ba67bc400b2566162915 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 24 Nov 2013 23:50:39 +0100 Subject: [PATCH 59/82] core: store dom0 info in qubes.xml At least to have there info about its backup. --- core-modules/006QubesDom0NetVm.py | 11 ++++++++--- core/qubes.py | 10 ++++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/core-modules/006QubesDom0NetVm.py b/core-modules/006QubesDom0NetVm.py index 3c1de9fd..43251e48 100644 --- a/core-modules/006QubesDom0NetVm.py +++ b/core-modules/006QubesDom0NetVm.py @@ -26,6 +26,10 @@ from qubes.qubes import defaults from qubes.qubes import QubesException,dry_run class QubesDom0NetVm(QubesNetVm): + + # In which order load this VM type from qubes.xml + load_order = 10 + def __init__(self, **kwargs): super(QubesDom0NetVm, self).__init__(qid=0, name="dom0", netid=0, dir_path=None, @@ -35,6 +39,10 @@ class QubesDom0NetVm(QubesNetVm): **kwargs) self.xid = 0 + @property + def type(self): + return "Dom0NetVM" + def is_running(self): return True @@ -83,9 +91,6 @@ class QubesDom0NetVm(QubesNetVm): domains = xc.domain_getinfo(0, 1) return domains[0] - def create_xml_element(self): - return None - def verify_files(self): return True diff --git a/core/qubes.py b/core/qubes.py index 2fcbb8d7..0532b1de 100755 --- a/core/qubes.py +++ b/core/qubes.py @@ -609,10 +609,6 @@ class QubesVmCollection(dict): def load(self): self.clear() - dom0vm = QubesDom0NetVm (collection=self) - self[dom0vm.qid] = dom0vm - self.default_netvm_qid = 0 - try: tree = lxml.etree.parse(self.qubes_store_file) except (EnvironmentError, @@ -661,6 +657,12 @@ class QubesVmCollection(dict): # using 123/udp port) if self.clockvm_qid is not None: self[self.clockvm_qid].services['ntpd'] = False + + # Add dom0 if wasn't present in qubes.xml + if not 0 in self.keys(): + dom0vm = QubesDom0NetVm (collection=self) + self[dom0vm.qid] = dom0vm + return True def pop(self, qid): From bc59d7e054f05d84f691e0d62b38a4fcc1ed0e84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 25 Nov 2013 00:36:40 +0100 Subject: [PATCH 60/82] backups: include file path in internal archive, implement dom0 home restore This is mostly revert of "3d1b40f backups: keep file without path in inner tar archive" in terms of archive format, but the code is more robust than old one. Especially reuse already computed dir paths. Also restore only requested files (based on selected VMs and its qubes.xml data). Change the restore workflow to restore files first to temporary directory, then move to final dirs. This approach: - will be compatible with hashed vm name in the archive path - is required to handle dom0 home backup (directory outside of /var/lib/qubes) - it should be also more defensive - make any changes in /var/lib/qubes only after successful extraction of files and creating Qubes*Vm object Second change in this commit is implement of dom0 home backup/restore. As qubes.xml now contains data about dom0, we have information whether it is included in the backup (before getting actual files). --- core/qubesutils.py | 80 +++++++++++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 33 deletions(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index a4ab4ee7..de7742ad 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -28,6 +28,7 @@ import sys import os import subprocess import re +import shutil import time import grp,pwd from datetime import datetime @@ -913,11 +914,7 @@ def backup_prepare(vms_list = None, exclude_list = [], print_callback = print_st if vm.qid in vms_for_backup_qid: vm.backup_content = True vm.backup_size = vm.get_disk_utilization() - vm.backup_path = vm.dir_path.split(os.path.normpath(system_path["qubes_base_dir"])+"/")[1] - - qvm_collection.save() - # FIXME: should be after backup completed - qvm_collection.unlock_db() + vm.backup_path = os.path.relpath(vm.dir_path, system_path["qubes_base_dir"]) # Dom0 user home if not 'dom0' in exclude_list: @@ -929,9 +926,14 @@ def backup_prepare(vms_list = None, exclude_list = [], print_callback = print_st subprocess.check_call(['sudo', 'chown', '-R', local_user, home_dir]) home_sz = get_disk_usage(home_dir) - home_to_backup = [ { "path" : home_dir, "size": home_sz, "subdir": 'dom0-home'} ] + home_to_backup = [ { "path" : home_dir, "size": home_sz, "subdir": 'dom0-home/'} ] files_to_backup += home_to_backup + vm = qvm_collection[0] + vm.backup_content = True + vm.backup_size = home_sz + vm.backup_path = os.path.join('dom0-home', os.path.basename(home_dir)) + s = "" fmt="{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1) s += fmt.format('Dom0') @@ -944,6 +946,10 @@ def backup_prepare(vms_list = None, exclude_list = [], print_callback = print_st print_callback(s) + qvm_collection.save() + # FIXME: should be after backup completed + qvm_collection.unlock_db() + total_backup_sz = 0 for file in files_to_backup: total_backup_sz += file["size"] @@ -1138,10 +1144,12 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase,\ # The first tar cmd can use any complex feature as we want. Files will # be verified before untaring this. - tar_cmdline = ["tar", "-Pc", '--sparse' + # Prefix the path in archive with filename["subdir"] to have it verified during untar + tar_cmdline = ["tar", "-Pc", '--sparse', "-f", backup_pipe, '--tape-length', str(1000000), '-C', os.path.dirname(filename["path"]), + '--xform', 's:^[a-z]:%s\\0:' % filename["subdir"], os.path.basename(filename["path"]) ] @@ -1399,8 +1407,6 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, if filename == "FINISHED": break - dirname = os.path.join(system_path["qubes_base_dir"], - os.path.dirname(os.path.relpath(filename))) if BACKUP_DEBUG: self.print_callback("Extracting file "+filename+" to "+dirname) if not os.path.exists(dirname): @@ -1416,8 +1422,8 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, # extracting (can also be obtained by running with --strip ?) tar2_cmdline = ['tar', '--tape-length','1000000', - '-C', dirname, - '-x%sf' % ("v" if BACKUP_DEBUG else ""), self.restore_pipe] + '-xk%sf' % ("v" if BACKUP_DEBUG else ""), self.restore_pipe, + os.path.relpath(filename.rstrip('.000'))] if BACKUP_DEBUG: self.print_callback("Running command "+str(tar2_cmdline)) self.tar2_command = subprocess.Popen(tar2_cmdline, @@ -1521,10 +1527,7 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vmproc.stdin.write(backup_dir.replace("\r","").replace("\n","")+"\n") # Send to tar2qfile the VMs that should be extracted - vmpaths = [] - for vmobj in vms.values(): - vmpaths.append(vmobj.backup_path) - vmproc.stdin.write(" ".join(vmpaths)+"\n") + vmproc.stdin.write(" ".join(vms_dirs)+"\n") backup_stdin = vmproc.stdout tar1_command = ['/usr/libexec/qubes/qfile-dom0-unpacker', @@ -1534,7 +1537,7 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, tar1_command = ['tar', '-ix%sf' % ("v" if BACKUP_DEBUG else ""), backup_dir, - '-C', backup_tmpdir] + '-C', backup_tmpdir] + vms_dirs # Remove already processed qubes.xml.000, because qfile-dom0-unpacker will # refuse to override files @@ -1584,7 +1587,12 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, print_callback("Ignoring already processed qubes.xml") continue - # FIXME: skip VMs not selected for restore + if not any(map(lambda x: filename.startswith(x), vms_dirs)): + if BACKUP_DEBUG: + print_callback("Ignoring VM not selected for restore") + os.unlink(filename) + os.unlink(hmacfile) + continue if BACKUP_DEBUG: print_callback("Verifying file "+filename) @@ -1840,10 +1848,6 @@ def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, #### Private functions begin def is_vm_included_in_backup (backup_dir, vm): - if vm.qid == 0: - # Dom0 is not included, obviously - return False - if vm.backup_content: return True else: @@ -1881,6 +1885,9 @@ def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, restore_home = False # ... and the actual data for vm in backup_vms_list: + if vm.qid == 0: + # Handle dom0 as special case later + continue if is_vm_included_in_backup (backup_dir, vm): if BACKUP_DEBUG: print vm.name,"is included in backup" @@ -1950,17 +1957,18 @@ def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, vms_to_restore[vm.name]['good-to-go'] = True # ...and dom0 home - # FIXME, replace this part of code to handle the new backup format using tar - if options['dom0-home'] and os.path.exists(backup_dir + '/dom0-home'): + if options['dom0-home'] and \ + is_vm_included_in_backup(backup_dir, backup_collection[0]): + vm = backup_collection[0] vms_to_restore['dom0'] = {} + vms_to_restore['dom0']['subdir'] = vm.backup_path + vms_to_restore['dom0']['size'] = vm.backup_size local_user = grp.getgrnam('qubes').gr_mem[0] - dom0_homes = os.listdir(backup_dir + '/dom0-home') - if len(dom0_homes) > 1: - raise QubesException("More than one dom0 homedir in backup") + dom0_home = vm.backup_path - vms_to_restore['dom0']['username'] = dom0_homes[0] - if dom0_homes[0] != local_user: + vms_to_restore['dom0']['username'] = os.path.basename(dom0_home) + if vms_to_restore['dom0']['username'] != local_user: vms_to_restore['dom0']['username-mismatch'] = True if not options['ignore-dom0-username-mismatch']: vms_to_restore['dom0']['good-to-go'] = False @@ -2092,9 +2100,13 @@ def backup_restore_do(backup_dir, restore_tmpdir, passphrase, restore_info, continue vm = vm_info['vm'] vms_size += vm.backup_size - vms_dirs.append(vm.backup_path+"*") + vms_dirs.append(vm.backup_path) vms[vm.name] = vm + if 'dom0' in restore_info.keys() and restore_info['dom0']['good-to-go']: + vms_dirs.append('dom0-home') + vms_size += restore_info['dom0']['size'] + restore_vm_dirs (backup_dir, restore_tmpdir, passphrase=passphrase, @@ -2119,13 +2131,12 @@ def backup_restore_do(backup_dir, restore_tmpdir, passphrase, restore_info, if not vm.__class__ == vm_class: continue print_callback("-> Restoring {type} {0}...".format(vm.name, type=vm_class_name)) - retcode = subprocess.call (["mkdir", "-p", vm.dir_path]) + retcode = subprocess.call (["mkdir", "-p", os.path.dirname(vm.dir_path)]) if retcode != 0: error_callback("*** Cannot create directory: {0}?!".format(dest_dir)) error_callback("Skipping...") continue - template = None if vm.template is not None: template_name = vm_info['template'] @@ -2140,6 +2151,9 @@ def backup_restore_do(backup_dir, restore_tmpdir, passphrase, restore_info, template=template, installed_by_rpm=False) + shutil.move(os.path.join(restore_tmpdir, vm.backup_path), + new_vm.dir_path) + new_vm.verify_files() except Exception as err: error_callback("ERROR: {0}".format(err)) @@ -2181,10 +2195,10 @@ def backup_restore_do(backup_dir, restore_tmpdir, passphrase, restore_info, # ... and dom0 home as last step if 'dom0' in restore_info.keys() and restore_info['dom0']['good-to-go']: - backup_info = restore_info['dom0'] + backup_path = restore_info['dom0']['subdir'] local_user = grp.getgrnam('qubes').gr_mem[0] home_dir = pwd.getpwnam(local_user).pw_dir - backup_dom0_home_dir = backup_dir + '/dom0-home/' + backup_info['username'] + backup_dom0_home_dir = os.path.join(restore_tmpdir, backup_path) restore_home_backupdir = "home-pre-restore-{0}".format (time.strftime("%Y-%m-%d-%H%M%S")) print_callback("-> Restoring home of user '{0}'...".format(local_user)) From 7229b78bbf5b9be8353370757d5c81468c85fb3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 25 Nov 2013 00:48:00 +0100 Subject: [PATCH 61/82] backups: minor reduce code duplication --- core/qubesutils.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index de7742ad..ba30a74b 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -1429,6 +1429,13 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, self.tar2_command = subprocess.Popen(tar2_cmdline, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + common_args = { + 'backup_target': pipe, + 'total_backup_sz': self.total_size, + 'hmac': None, + 'vmproc': self.vmproc, + 'addproc': self.tar2_command + } if self.encrypted: # Start decrypt encryptor = subprocess.Popen (["openssl", "enc", @@ -1439,24 +1446,13 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, run_error = wait_backup_feedback( progress_callback=self.compute_progress, - in_stream=encryptor.stdout, - streamproc=encryptor, - backup_target=pipe, - total_backup_sz=self.total_size, - hmac=None, - vmproc=self.vmproc, - addproc=self.tar2_command) + in_stream=encryptor.stdout, streamproc=encryptor, + **common_args) else: run_error = wait_backup_feedback( progress_callback=self.compute_progress, - in_stream=open(filename,"rb"), - streamproc=None, - backup_target=pipe, - total_backup_sz=self.total_size, - hmac=None, - vmproc=self.vmproc, - addproc=self.tar2_command) - + in_stream=open(filename,"rb"), streamproc=None, + **common_args) pipe.close() From 10a01010bb9b756689268986db5550fb9266eff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 25 Nov 2013 00:48:54 +0100 Subject: [PATCH 62/82] backups: fix handling multi-volume archive during restore We can't wait for tar next volume prompt using stderr.readline(), because tar don't output EOL marker after this prompt. The other way would be switching file descriptor to non-blocking mode and using lower level os.read(), but this looks like more error-prone way (races...). So change idea of handling such archives: after switching to next archive volume, simply send '\n' to tar (which will receive when needed). When getting "*.000" file, assume that previous archive was over and wait for previous tar process. Then start the new one. Also don't give explicit tape length, only turn multi-volume mode on. So will correctly handle all multi-volume archives, regardless of its size. --- core/qubesutils.py | 77 ++++++++++++++++++++++------------------------ 1 file changed, 36 insertions(+), 41 deletions(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index ba30a74b..666c1564 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -1379,6 +1379,7 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, self.total_size = total_size self.blocks_backedup = 0 self.tar2_command = None + self.tar2_current_file = None self.print_callback = print_callback self.error_callback = error_callback @@ -1408,27 +1409,36 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, break if BACKUP_DEBUG: - self.print_callback("Extracting file "+filename+" to "+dirname) - if not os.path.exists(dirname): - os.makedirs(dirname) + self.print_callback("Extracting file "+filename) + + if filename.endswith('.000'): + # next file + if self.tar2_command != None: + if self.tar2_command.wait() != 0: + raise QubesException( + "ERROR: unable to extract files for {0}.".\ + format(self.tar2_current_file)) + else: + # Finished extracting the tar file + self.tar2_command = None + self.tar2_current_file = None - pipe = open(self.restore_pipe,'r+b') - if self.tar2_command == None: - # FIXME: Make the extraction safer by avoiding to erase - # other vms: - # - extracting directly to the target directory (based on - # the vm name and by using the --strip=2). - # - ensuring that the leading slashs are ignored when - # extracting (can also be obtained by running with --strip ?) tar2_cmdline = ['tar', - '--tape-length','1000000', - '-xk%sf' % ("v" if BACKUP_DEBUG else ""), self.restore_pipe, + '-xMk%sf' % ("v" if BACKUP_DEBUG else ""), self.restore_pipe, os.path.relpath(filename.rstrip('.000'))] if BACKUP_DEBUG: self.print_callback("Running command "+str(tar2_cmdline)) self.tar2_command = subprocess.Popen(tar2_cmdline, - stdin=subprocess.PIPE, stderr=subprocess.PIPE) + stdin=subprocess.PIPE, + stderr=(None if BACKUP_DEBUG else open('/dev/null', 'w'))) + else: + if BACKUP_DEBUG: + self.print_callback("Releasing next chunck") + self.tar2_command.stdin.write("\n") + self.tar2_command.stdin.flush() + self.tar2_current_file = filename + pipe = open(self.restore_pipe,'wb') common_args = { 'backup_target': pipe, 'total_backup_sz': self.total_size, @@ -1456,30 +1466,20 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, pipe.close() - # tar2 input closed, wait for either it finishes, or prompt for - # the next file part; in both cases we can use read() on stderr - # - in the former case it will return "" (EOF) - tar2_stderr=self.tar2_command.stderr.readline() - if tar2_stderr == "": - # EOF, so collect process exit status - if self.tar2_command.wait() != 0: - raise QubesException( - "ERROR: unable to extract files for {0}.".\ - format(filename)) - else: - # Finished extracting the tar file - self.tar2_command = None - - else: - if BACKUP_DEBUG: - self.print_callback("Releasing next chunck") - self.tar2_command.stdin.write("\n") - # Delete the file as we don't need it anymore if BACKUP_DEBUG: self.print_callback("Removing file "+filename) os.remove(filename) + if self.tar2_command != None: + if self.tar2_command.wait() != 0: + raise QubesException( + "ERROR: unable to extract files for {0}.".\ + format(self.tar2_current_file)) + else: + # Finished extracting the tar file + self.tar2_command = None + if BACKUP_DEBUG: self.print_callback("Finished extracting thread") @@ -1750,10 +1750,7 @@ def backup_restore_header(restore_target, passphrase, command.wait() - # Let the time to vmproc process to crash - time.sleep(2) - - if vmproc and vmproc.poll() != None and vmproc.poll() != 0: + if vmproc and vmproc.wait() != 0: error = vmproc.stderr.read() if BACKUP_DEBUG: print error @@ -1806,16 +1803,14 @@ def backup_restore_header(restore_target, passphrase, stdin=open(os.path.join(backup_tmpdir, filename),'rb'), stdout=subprocess.PIPE) tarhead_command = subprocess.Popen(['tar', - '--tape-length','1000000', - '-x%s' % ("v" if BACKUP_DEBUG else "")], + '-xM%s' % ("v" if BACKUP_DEBUG else "")], stdin=encryptor.stdout) else: if BACKUP_DEBUG: print "No decryption process required" encryptor = None tarhead_command = subprocess.Popen(['tar', - '--tape-length', '1000000', - '-x%sf' % ("v" if BACKUP_DEBUG else ""), + '-xM%sf' % ("v" if BACKUP_DEBUG else ""), os.path.join(backup_tmpdir, filename)]) if encryptor: From 07ae02915f9cf62344345315afd136b90ceaadfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 25 Nov 2013 00:55:10 +0100 Subject: [PATCH 63/82] backups: add missing import in qvm-backup --- qvm-tools/qvm-backup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qvm-tools/qvm-backup b/qvm-tools/qvm-backup index 6f96a97d..601bd009 100755 --- a/qvm-tools/qvm-backup +++ b/qvm-tools/qvm-backup @@ -22,7 +22,7 @@ from qubes.qubes import QubesVmCollection from qubes.qubes import QubesException -from qubes.qubesutils import backup_prepare, backup_do_copy +from qubes.qubesutils import backup_prepare, backup_do_copy, size_to_human from optparse import OptionParser import os import sys From e31c3ae8e7a3343607839e2edbcfc90a484f6611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 25 Nov 2013 00:55:59 +0100 Subject: [PATCH 64/82] backup: reduce volume size to 100M and limit queue length This way backup process won't need more than 1GB for temporary files and also will give more precise progress information. For now it looks like the slowest element is qrexec, so without such limit, all the data would be prepared (basically making second copy of it in dom0) while only first few files would be transfered to the VM. Also backup progress is calculated based on preparation thread, so when it finishes there is some other time needed to flush all the data to the VM. Limiting this amount makes progress somehow more accurate (but still off by 1GB...). --- core/qubesutils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index 666c1564..8fe9d272 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -1123,7 +1123,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase,\ progress = blocks_backedup / float(total_backup_sz) progress_callback(int(round(progress*100,2))) - to_send = Queue() + to_send = Queue(10) send_proc = Send_Worker(to_send, backup_tmpdir, backup_stdout) send_proc.start() @@ -1147,7 +1147,7 @@ def backup_do_copy(base_backup_dir, files_to_backup, passphrase,\ # Prefix the path in archive with filename["subdir"] to have it verified during untar tar_cmdline = ["tar", "-Pc", '--sparse', "-f", backup_pipe, - '--tape-length', str(1000000), + '--tape-length', str(100000), '-C', os.path.dirname(filename["path"]), '--xform', 's:^[a-z]:%s\\0:' % filename["subdir"], os.path.basename(filename["path"]) @@ -1487,7 +1487,7 @@ def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, def progress_callback(data): pass - to_extract = Queue() + to_extract = Queue() extract_proc = Extract_Worker(queue=to_extract, base_dir=backup_tmpdir, passphrase=passphrase, From 657beaf65560d1d85823ffcb2eda1f990e2fdb35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 25 Nov 2013 01:11:29 +0100 Subject: [PATCH 65/82] backups: move extracted dom0 home from /var/tmp instead of copy --- core/qubesutils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/qubesutils.py b/core/qubesutils.py index 8fe9d272..304e5a24 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -2199,9 +2199,7 @@ def backup_restore_do(backup_dir, restore_tmpdir, passphrase, restore_info, home_file = home_dir + '/' + f if os.path.exists(home_file): os.rename(home_file, home_dir + '/' + restore_home_backupdir + '/' + f) - retcode = subprocess.call (["cp", "-nrp", backup_dom0_home_dir + '/' + f, home_file]) - if retcode != 0: - error_callback("*** Error while copying file {0} to {1}".format(backup_dom0_home_dir + '/' + f, home_file)) + shutil.move(backup_dom0_home_dir + '/' + f, home_file) retcode = subprocess.call(['sudo', 'chown', '-R', local_user, home_dir]) if retcode != 0: error_callback("*** Error while setting home directory owner") From c781a522d86d9fbfe7e93e26340bb7aa4bb8ae3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 25 Nov 2013 05:41:13 +0100 Subject: [PATCH 66/82] backups: move backup code to separate file Also some major cleanups: Reduce some more code duplication (verify_hmac, simplify backup_restore_prepare). Rename backup_dir/backup_tmpdir variables to better match its purpose. Rename backup_do_copy back to backup_do. Require QubesVm object (instead of VM name) as appvm param. --- core/backup.py | 1279 ++++++++++++++++++++++++++++++ core/qubesutils.py | 1452 ---------------------------------- qvm-tools/qvm-backup | 14 +- qvm-tools/qvm-backup-restore | 35 +- rpm_spec/core-dom0.spec | 5 + 5 files changed, 1322 insertions(+), 1463 deletions(-) create mode 100644 core/backup.py diff --git a/core/backup.py b/core/backup.py new file mode 100644 index 00000000..6ea883e7 --- /dev/null +++ b/core/backup.py @@ -0,0 +1,1279 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# The Qubes OS Project, http://www.qubes-os.org +# +# Copyright (C) 2013 Marek Marczykowski-Górecki +# Copyright (C) 2013 Olivier Médoc +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# + +from qubes import QubesException,QubesVmCollection +from qubes import QubesVmClasses +from qubes import system_path,vm_files +from qubesutils import size_to_human, print_stdout, print_stderr +import sys +import os +import subprocess +import re +import shutil +import time +import grp,pwd +from multiprocessing import Queue,Process + +BACKUP_DEBUG = True + +def get_disk_usage(file_or_dir): + if not os.path.exists(file_or_dir): + return 0 + + p = subprocess.Popen (["du", "-s", "--block-size=1", file_or_dir], + stdout=subprocess.PIPE) + result = p.communicate() + m = re.match(r"^(\d+)\s.*", result[0]) + sz = int(m.group(1)) if m is not None else 0 + return sz + + +def file_to_backup (file_path, sz = None): + if sz is None: + sz = os.path.getsize (system_path["qubes_store_filename"]) + + abs_file_path = os.path.abspath (file_path) + abs_base_dir = os.path.abspath (system_path["qubes_base_dir"]) + '/' + abs_file_dir = os.path.dirname (abs_file_path) + '/' + (nothing, dir, subdir) = abs_file_dir.partition (abs_base_dir) + assert nothing == "" + assert dir == abs_base_dir + return [ { "path" : file_path, "size": sz, "subdir": subdir} ] + +def backup_prepare(vms_list = None, exclude_list = [], print_callback = print_stdout): + """If vms = None, include all (sensible) VMs; exclude_list is always applied""" + files_to_backup = file_to_backup (system_path["qubes_store_filename"]) + + if exclude_list is None: + exclude_list = [] + + qvm_collection = QubesVmCollection() + qvm_collection.lock_db_for_writing() + qvm_collection.load() + + if vms_list is None: + all_vms = [vm for vm in qvm_collection.values()] + selected_vms = [vm for vm in all_vms if vm.include_in_backups] + appvms_to_backup = [vm for vm in selected_vms if vm.is_appvm() and not vm.internal] + netvms_to_backup = [vm for vm in selected_vms if vm.is_netvm() and not vm.qid == 0] + template_vms_worth_backingup = [vm for vm in selected_vms if (vm.is_template() and not vm.installed_by_rpm)] + + vms_list = appvms_to_backup + netvms_to_backup + template_vms_worth_backingup + + vms_for_backup = vms_list + # Apply exclude list + if exclude_list: + vms_for_backup = [vm for vm in vms_list if vm.name not in exclude_list] + + no_vms = len (vms_for_backup) + + there_are_running_vms = False + + fields_to_display = [ + { "name": "VM", "width": 16}, + { "name": "type","width": 12 }, + { "name": "size", "width": 12} + ] + + # Display the header + s = "" + for f in fields_to_display: + fmt="{{0:-^{0}}}-+".format(f["width"] + 1) + s += fmt.format('-') + print_callback(s) + s = "" + for f in fields_to_display: + fmt="{{0:>{0}}} |".format(f["width"] + 1) + s += fmt.format(f["name"]) + print_callback(s) + s = "" + for f in fields_to_display: + fmt="{{0:-^{0}}}-+".format(f["width"] + 1) + s += fmt.format('-') + print_callback(s) + + for vm in vms_for_backup: + if vm.is_template(): + # handle templates later + continue + + if vm.private_img is not None: + vm_sz = vm.get_disk_usage (vm.private_img) + files_to_backup += file_to_backup(vm.private_img, vm_sz ) + + if vm.is_appvm(): + files_to_backup += file_to_backup(vm.icon_path) + if vm.updateable: + if os.path.exists(vm.dir_path + "/apps.templates"): + # template + files_to_backup += file_to_backup(vm.dir_path + "/apps.templates") + else: + # standaloneVM + files_to_backup += file_to_backup(vm.dir_path + "/apps") + + if os.path.exists(vm.dir_path + "/kernels"): + files_to_backup += file_to_backup(vm.dir_path + "/kernels") + if os.path.exists (vm.firewall_conf): + files_to_backup += file_to_backup(vm.firewall_conf) + if 'appmenus_whitelist' in vm_files and \ + os.path.exists(vm.dir_path + vm_files['appmenus_whitelist']): + files_to_backup += file_to_backup(vm.dir_path + vm_files['appmenus_whitelist']) + + if vm.updateable: + sz = vm.get_disk_usage(vm.root_img) + files_to_backup += file_to_backup(vm.root_img, sz) + vm_sz += sz + + s = "" + fmt="{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1) + s += fmt.format(vm.name) + + fmt="{{0:>{0}}} |".format(fields_to_display[1]["width"] + 1) + if vm.is_netvm(): + s += fmt.format("NetVM" + (" + Sys" if vm.updateable else "")) + else: + s += fmt.format("AppVM" + (" + Sys" if vm.updateable else "")) + + fmt="{{0:>{0}}} |".format(fields_to_display[2]["width"] + 1) + s += fmt.format(size_to_human(vm_sz)) + + if vm.is_running(): + s += " <-- The VM is running, please shut it down before proceeding with the backup!" + there_are_running_vms = True + + print_callback(s) + + for vm in vms_for_backup: + if not vm.is_template(): + # already handled + continue + vm_sz = vm.get_disk_utilization() + files_to_backup += file_to_backup (vm.dir_path, vm_sz) + + s = "" + fmt="{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1) + s += fmt.format(vm.name) + + fmt="{{0:>{0}}} |".format(fields_to_display[1]["width"] + 1) + s += fmt.format("Template VM") + + fmt="{{0:>{0}}} |".format(fields_to_display[2]["width"] + 1) + s += fmt.format(size_to_human(vm_sz)) + + if vm.is_running(): + s += " <-- The VM is running, please shut it down before proceeding with the backup!" + there_are_running_vms = True + + print_callback(s) + + # Initialize backup flag on all VMs + vms_for_backup_qid = [vm.qid for vm in vms_for_backup] + for vm in qvm_collection.values(): + vm.backup_content = False + + if vm.qid in vms_for_backup_qid: + vm.backup_content = True + vm.backup_size = vm.get_disk_utilization() + vm.backup_path = os.path.relpath(vm.dir_path, system_path["qubes_base_dir"]) + + # Dom0 user home + if not 'dom0' in exclude_list: + local_user = grp.getgrnam('qubes').gr_mem[0] + home_dir = pwd.getpwnam(local_user).pw_dir + # Home dir should have only user-owned files, so fix it now to prevent + # permissions problems - some root-owned files can left after + # 'sudo bash' and similar commands + subprocess.check_call(['sudo', 'chown', '-R', local_user, home_dir]) + + home_sz = get_disk_usage(home_dir) + home_to_backup = [ { "path" : home_dir, "size": home_sz, "subdir": 'dom0-home/'} ] + files_to_backup += home_to_backup + + vm = qvm_collection[0] + vm.backup_content = True + vm.backup_size = home_sz + vm.backup_path = os.path.join('dom0-home', os.path.basename(home_dir)) + + s = "" + fmt="{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1) + s += fmt.format('Dom0') + + fmt="{{0:>{0}}} |".format(fields_to_display[1]["width"] + 1) + s += fmt.format("User home") + + fmt="{{0:>{0}}} |".format(fields_to_display[2]["width"] + 1) + s += fmt.format(size_to_human(home_sz)) + + print_callback(s) + + qvm_collection.save() + # FIXME: should be after backup completed + qvm_collection.unlock_db() + + total_backup_sz = 0 + for file in files_to_backup: + total_backup_sz += file["size"] + + s = "" + for f in fields_to_display: + fmt="{{0:-^{0}}}-+".format(f["width"] + 1) + s += fmt.format('-') + print_callback(s) + + s = "" + fmt="{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1) + s += fmt.format("Total size:") + fmt="{{0:>{0}}} |".format(fields_to_display[1]["width"] + 1 + 2 + fields_to_display[2]["width"] + 1) + s += fmt.format(size_to_human(total_backup_sz)) + print_callback(s) + + s = "" + for f in fields_to_display: + fmt="{{0:-^{0}}}-+".format(f["width"] + 1) + s += fmt.format('-') + print_callback(s) + + if (there_are_running_vms): + raise QubesException("Please shutdown all VMs before proceeding.") + + return files_to_backup + +class Send_Worker(Process): + def __init__(self, queue, base_dir, backup_stdout): + super(Send_Worker, self).__init__() + self.queue = queue + self.base_dir = base_dir + self.backup_stdout = backup_stdout + + def run(self): + if BACKUP_DEBUG: + print "Started sending thread" + + if BACKUP_DEBUG: + print "Moving to temporary dir", self.base_dir + os.chdir(self.base_dir) + + for filename in iter(self.queue.get,None): + if filename == "FINISHED": + break + + if BACKUP_DEBUG: + print "Sending file", filename + # This tar used for sending data out need to be as simple, as + # simple, as featureless as possible. It will not be + # verified before untaring. + tar_final_cmd = ["tar", "-cO", "--posix", + "-C", self.base_dir, filename] + final_proc = subprocess.Popen (tar_final_cmd, + stdin=subprocess.PIPE, stdout=self.backup_stdout) + final_proc.wait() + + # Delete the file as we don't need it anymore + if BACKUP_DEBUG: + print "Removing file", filename + os.remove(filename) + + if BACKUP_DEBUG: + print "Finished sending thread" + +def backup_do(base_backup_dir, files_to_backup, passphrase,\ + progress_callback = None, encrypt=False, appvm=None): + total_backup_sz = 0 + for file in files_to_backup: + total_backup_sz += file["size"] + + vmproc = None + if appvm != None: + # Prepare the backup target (Qubes service call) + backup_target = "QUBESRPC qubes.Backup none" + + # If APPVM, STDOUT is a PIPE + vmproc = appvm.run(command = backup_target, passio_popen = True) + vmproc.stdin.write(base_backup_dir.\ + replace("\r","").replace("\n","")+"\n") + backup_stdout = vmproc.stdin + else: + # Prepare the backup target (local file) + backup_target = base_backup_dir + "/qubes-{0}".\ + format (time.strftime("%Y-%m-%d-%H%M%S")) + + # Create the target directory + if not os.path.exists (base_backup_dir): + raise QubesException( + "ERROR: the backup directory {0} does not exists".\ + format(base_backup_dir)) + + # If not APPVM, STDOUT is a local file + backup_stdout = open(backup_target,'wb') + + global blocks_backedup + blocks_backedup = 0 + progress = blocks_backedup * 11 / total_backup_sz + progress_callback(progress) + + import tempfile + feedback_file = tempfile.NamedTemporaryFile() + backup_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/backup_") + + # Tar with tapelength does not deals well with stdout (close stdout between + # two tapes) + # For this reason, we will use named pipes instead + if BACKUP_DEBUG: + print "Working in", backup_tmpdir + + backup_pipe = os.path.join(backup_tmpdir,"backup_pipe") + if BACKUP_DEBUG: + print "Creating pipe in:", backup_pipe + os.mkfifo(backup_pipe) + + if BACKUP_DEBUG: + print "Will backup:", files_to_backup + + # Setup worker to send encrypted data chunks to the backup_target + + def compute_progress(new_size, total_backup_sz): + global blocks_backedup + blocks_backedup += new_size + progress = blocks_backedup / float(total_backup_sz) + progress_callback(int(round(progress*100,2))) + + to_send = Queue(10) + send_proc = Send_Worker(to_send, backup_tmpdir, backup_stdout) + send_proc.start() + + for filename in files_to_backup: + if BACKUP_DEBUG: + print "Backing up", filename + + backup_tempfile = os.path.join(backup_tmpdir, + filename["subdir"], + os.path.basename(filename["path"])) + if BACKUP_DEBUG: + print "Using temporary location:", backup_tempfile + + # Ensure the temporary directory exists + if not os.path.isdir(os.path.dirname(backup_tempfile)): + os.makedirs(os.path.dirname(backup_tempfile)) + + # The first tar cmd can use any complex feature as we want. Files will + # be verified before untaring this. + # Prefix the path in archive with filename["subdir"] to have it verified during untar + tar_cmdline = ["tar", "-Pc", '--sparse', + "-f", backup_pipe, + '--tape-length', str(100000), + '-C', os.path.dirname(filename["path"]), + '--xform', 's:^[a-z]:%s\\0:' % filename["subdir"], + os.path.basename(filename["path"]) + ] + + if BACKUP_DEBUG: + print " ".join(tar_cmdline) + + # Tips: Popen(bufsize=0) + # Pipe: tar-sparse | encryptor [| hmac] | tar | backup_target + # Pipe: tar-sparse [| hmac] | tar | backup_target + tar_sparse = subprocess.Popen (tar_cmdline, stdin=subprocess.PIPE) + + # Wait for compressor (tar) process to finish or for any error of other + # subprocesses + i = 0 + run_error = "paused" + running = [] + while run_error == "paused": + + pipe = open(backup_pipe,'rb') + + # Start HMAC + hmac = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], + stdin=subprocess.PIPE, stdout=subprocess.PIPE) + + # Prepare a first chunk + chunkfile = backup_tempfile + "." + "%03d" % i + i += 1 + chunkfile_p = open(chunkfile,'wb') + + common_args = { + 'backup_target': chunkfile_p, + 'total_backup_sz': total_backup_sz, + 'hmac': hmac, + 'vmproc': vmproc, + 'addproc': tar_sparse + } + if encrypt: + # Start encrypt + # If no cipher is provided, the data is forwarded unencrypted !!! + encryptor = subprocess.Popen (["openssl", "enc", + "-e", "-aes-256-cbc", + "-pass", "pass:"+passphrase], + stdin=pipe, stdout=subprocess.PIPE) + run_error = wait_backup_feedback( + progress_callback=compute_progress, + in_stream=encryptor.stdout, streamproc=encryptor, + **common_args) + else: + run_error = wait_backup_feedback( + progress_callback=compute_progress, + in_stream=pipe, streamproc=None, + **common_args) + + chunkfile_p.close() + + if BACKUP_DEBUG: + print "Wait_backup_feedback returned:", run_error + + if len(run_error) > 0: + send_proc.terminate() + raise QubesException("Failed to perform backup: error with "+ \ + run_error) + + # Send the chunk to the backup target + to_send.put(os.path.relpath(chunkfile, backup_tmpdir)) + + # Close HMAC + hmac.stdin.close() + hmac.wait() + if BACKUP_DEBUG: + print "HMAC proc return code:", hmac.poll() + + # Write HMAC data next to the chunk file + hmac_data = hmac.stdout.read() + if BACKUP_DEBUG: + print "Writing hmac to", chunkfile+".hmac" + hmac_file = open(chunkfile+".hmac",'w') + hmac_file.write(hmac_data) + hmac_file.flush() + hmac_file.close() + + pipe.close() + + # Send the HMAC to the backup target + to_send.put(os.path.relpath(chunkfile, backup_tmpdir)+".hmac") + + if tar_sparse.poll() == None: + # Release the next chunk + if BACKUP_DEBUG: + print "Release next chunk for process:", tar_sparse.poll() + #tar_sparse.stdout = subprocess.PIPE + tar_sparse.stdin.write("\n") + tar_sparse.stdin.flush() + run_error="paused" + else: + if BACKUP_DEBUG: + print "Finished tar sparse with error", tar_sparse.poll() + + to_send.put("FINISHED") + send_proc.join() + + if send_proc.exitcode != 0: + raise QubesException("Failed to send backup: error in the sending process") + + if vmproc: + if BACKUP_DEBUG: + print "VMProc1 proc return code:", vmproc.poll() + print "Sparse1 proc return code:", tar_sparse.poll() + vmproc.stdin.close() + + shutil.rmtree(restore_tmpdir) + +''' +' Wait for backup chunk to finish +' - Monitor all the processes (streamproc, hmac, vmproc, addproc) for errors +' - Copy stdout of streamproc to backup_target and hmac stdin if available +' - Compute progress based on total_backup_sz and send progress to +' progress_callback function +' - Returns if +' - one of the monitored processes error out (streamproc, hmac, vmproc, +' addproc), along with the processe that failed +' - all of the monitored processes except vmproc finished successfully +' (vmproc termination is controlled by the python script) +' - streamproc does not delivers any data anymore (return with the error +' "") +''' +def wait_backup_feedback(progress_callback, in_stream, streamproc, + backup_target, total_backup_sz, hmac=None, vmproc=None, addproc=None, + remove_trailing_bytes=0): + + buffer_size = 409600 + + run_error = None + run_count = 1 + blocks_backedup = 0 + while run_count > 0 and run_error == None: + + buffer = in_stream.read(buffer_size) + progress_callback(len(buffer), total_backup_sz) + + run_count = 0 + if hmac: + retcode=hmac.poll() + if retcode != None: + if retcode != 0: + run_error = "hmac" + else: + run_count += 1 + + if addproc: + retcode=addproc.poll() + if retcode != None: + if retcode != 0: + run_error = "addproc" + else: + run_count += 1 + + if vmproc: + retcode = vmproc.poll() + if retcode != None: + if retcode != 0: + run_error = "VM" + if BACKUP_DEBUG: + print vmproc.stdout.read() + else: + # VM should run until the end + pass + + if streamproc: + retcode=streamproc.poll() + if retcode != None: + if retcode != 0: + run_error = "streamproc" + break + elif retcode == 0 and len(buffer) <= 0: + return "" + run_count += 1 + + else: + if len(buffer) <= 0: + return "" + + backup_target.write(buffer) + + if hmac: + hmac.stdin.write(buffer) + + return run_error + +def verify_hmac(filename, hmacfile, passphrase): + if BACKUP_DEBUG: + print "Verifying file "+filename + + if hmacfile != filename + ".hmac": + raise QubesException( + "ERROR: expected hmac for {}, but got {}".\ + format(filename, hmacfile)) + + hmac_proc = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], + stdin=open(filename,'rb'), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + hmac_stdout, hmac_stderr = hmac_proc.communicate() + + if len(hmac_stderr) > 0: + raise QubesException("ERROR: verify file {0}: {1}".format((filename, hmac_stderr))) + else: + if BACKUP_DEBUG: + print "Loading hmac for file " + filename + hmac = load_hmac(open(hmacfile,'r').read()) + + if len(hmac) > 0 and load_hmac(hmac_stdout) == hmac: + os.unlink(hmacfile) + if BACKUP_DEBUG: + print "File verification OK -> Sending file " + filename + return True + else: + raise QubesException( + "ERROR: invalid hmac for file {0}: {1}. " \ + "Is the passphrase correct?".\ + format(filename, load_hmac(hmac_stdout))) + # Not reachable + return False + +class Extract_Worker(Process): + def __init__(self, queue, base_dir, passphrase, encrypted, total_size, + print_callback, error_callback, progress_callback, vmproc=None): + super(Extract_Worker, self).__init__() + self.queue = queue + self.base_dir = base_dir + self.passphrase = passphrase + self.encrypted = encrypted + self.total_size = total_size + self.blocks_backedup = 0 + self.tar2_command = None + self.tar2_current_file = None + + self.print_callback = print_callback + self.error_callback = error_callback + self.progress_callback = progress_callback + + self.vmproc = vmproc + + self.restore_pipe = os.path.join(self.base_dir,"restore_pipe") + if BACKUP_DEBUG: + print "Creating pipe in:", self.restore_pipe + os.mkfifo(self.restore_pipe) + + def compute_progress(self, new_size, total_size): + self.blocks_backedup += new_size + progress = self.blocks_backedup / float(self.total_size) + progress = int(round(progress*100,2)) + self.progress_callback(progress) + + def run(self): + if BACKUP_DEBUG: + self.print_callback("Started sending thread") + self.print_callback("Moving to dir "+self.base_dir) + os.chdir(self.base_dir) + + for filename in iter(self.queue.get,None): + if filename == "FINISHED": + break + + if BACKUP_DEBUG: + self.print_callback("Extracting file "+filename) + + if filename.endswith('.000'): + # next file + if self.tar2_command != None: + if self.tar2_command.wait() != 0: + raise QubesException( + "ERROR: unable to extract files for {0}.".\ + format(self.tar2_current_file)) + else: + # Finished extracting the tar file + self.tar2_command = None + self.tar2_current_file = None + + tar2_cmdline = ['tar', + '-xMk%sf' % ("v" if BACKUP_DEBUG else ""), self.restore_pipe, + os.path.relpath(filename.rstrip('.000'))] + if BACKUP_DEBUG: + self.print_callback("Running command "+str(tar2_cmdline)) + self.tar2_command = subprocess.Popen(tar2_cmdline, + stdin=subprocess.PIPE, + stderr=(None if BACKUP_DEBUG else open('/dev/null', 'w'))) + else: + if BACKUP_DEBUG: + self.print_callback("Releasing next chunck") + self.tar2_command.stdin.write("\n") + self.tar2_command.stdin.flush() + self.tar2_current_file = filename + + pipe = open(self.restore_pipe,'wb') + common_args = { + 'backup_target': pipe, + 'total_backup_sz': self.total_size, + 'hmac': None, + 'vmproc': self.vmproc, + 'addproc': self.tar2_command + } + if self.encrypted: + # Start decrypt + encryptor = subprocess.Popen (["openssl", "enc", + "-d", "-aes-256-cbc", + "-pass", "pass:"+passphrase], + stdin=open(filename,'rb'), + stdout=subprocess.PIPE) + + run_error = wait_backup_feedback( + progress_callback=self.compute_progress, + in_stream=encryptor.stdout, streamproc=encryptor, + **common_args) + else: + run_error = wait_backup_feedback( + progress_callback=self.compute_progress, + in_stream=open(filename,"rb"), streamproc=None, + **common_args) + + pipe.close() + + # Delete the file as we don't need it anymore + if BACKUP_DEBUG: + self.print_callback("Removing file "+filename) + os.remove(filename) + + if self.tar2_command != None: + if self.tar2_command.wait() != 0: + raise QubesException( + "ERROR: unable to extract files for {0}.".\ + format(self.tar2_current_file)) + else: + # Finished extracting the tar file + self.tar2_command = None + + os.unlink(self.restore_pipe) + if BACKUP_DEBUG: + self.print_callback("Finished extracting thread") + +def restore_vm_dirs (backup_source, restore_tmpdir, passphrase, vms_dirs, vms, + vms_size, print_callback=None, error_callback=None, + progress_callback=None, encrypted=False, appvm=None): + + # Setup worker to extract encrypted data chunks to the restore dirs + if progress_callback == None: + def progress_callback(data): + pass + + to_extract = Queue() + extract_proc = Extract_Worker(queue=to_extract, + base_dir=restore_tmpdir, + passphrase=passphrase, + encrypted=encrypted, + total_size=vms_size, + print_callback=print_callback, + error_callback=error_callback, + progress_callback=progress_callback) + extract_proc.start() + + if BACKUP_DEBUG: + print_callback("Working in temporary dir:"+restore_tmpdir) + print_callback(str(vms_size)+" bytes to restore") + + vmproc = None + if appvm != None: + # Prepare the backup target (Qubes service call) + backup_target = "QUBESRPC qubes.Restore dom0" + + # If APPVM, STDOUT is a PIPE + vmproc = appvm.run(command = backup_target, passio_popen = True) + vmproc.stdin.write(backup_source.replace("\r","").replace("\n","")+"\n") + + # Send to tar2qfile the VMs that should be extracted + vmproc.stdin.write(" ".join(vms_dirs)+"\n") + + backup_stdin = vmproc.stdout + tar1_command = ['/usr/libexec/qubes/qfile-dom0-unpacker', + str(os.getuid()), restore_tmpdir, '-v'] + else: + backup_stdin = open(backup_source,'rb') + + tar1_command = ['tar', + '-ix%sf' % ("v" if BACKUP_DEBUG else ""), backup_source, + '-C', restore_tmpdir] + vms_dirs + + # Remove already processed qubes.xml.000, because qfile-dom0-unpacker will + # refuse to override files + tar1_env = os.environ.copy() + # TODO: add some safety margin? + tar1_env['UPDATES_MAX_BYTES'] = str(vms_size) + # Restoring only header + if len(vms_dirs) == 2 and vms_dirs[0] == 'qubes.xml.000': + tar1_env['UPDATES_MAX_FILES'] = '2' + else: + tar1_env['UPDATES_MAX_FILES'] = '0' + if BACKUP_DEBUG: + print_callback("Run command"+str(tar1_command)) + command = subprocess.Popen(tar1_command, + stdin=backup_stdin, + stdout=vmproc.stdin if vmproc else subprocess.PIPE, + stderr=subprocess.PIPE, + env=tar1_env) + + # qfile-dom0-unpacker output filelist on stderr (and have stdout connected + # to the VM), while tar output filelist on stdout + if appvm: + filelist_pipe = command.stderr + else: + filelist_pipe = command.stdout + + while True: + + filename = filelist_pipe.readline().strip(" \t\r\n") + + if BACKUP_DEBUG: + print_callback("Getting new file:"+filename) + + if not filename or filename=="EOF": + break + + hmacfile = filelist_pipe.readline().strip(" \t\r\n") + if BACKUP_DEBUG: + print_callback("Getting hmac:"+hmacfile) + + if not any(map(lambda x: filename.startswith(x), vms_dirs)): + if BACKUP_DEBUG: + print_callback("Ignoring VM not selected for restore") + os.unlink(os.path.join(restore_tmpdir, filename)) + os.unlink(os.path.join(restore_tmpdir, hmacfile)) + continue + + if verify_hmac(os.path.join(restore_tmpdir,filename), + os.path.join(restore_tmpdir,hmacfile), + passphrase): + to_extract.put(os.path.join(restore_tmpdir, filename)) + + if command.wait() != 0: + raise QubesException( + "ERROR: unable to read the qubes backup file {0} ({1}). " \ + "Is it really a backup?".format(backup_source, command.wait())) + if vmproc: + if vmproc.wait() != 0: + raise QubesException( + "ERROR: unable to read the qubes backup {0} " \ + "because of a VM error: {1}".format( + backup_source, vmproc.stderr.read())) + if BACKUP_DEBUG: + print "Extraction process status:", extract_proc.exitcode + + to_extract.put("FINISHED") + if BACKUP_DEBUG: + print_callback("Waiting for the extraction process to finish...") + extract_proc.join() + if BACKUP_DEBUG: + print_callback("Extraction process finished with code:" + \ + str(extract_proc.exitcode)) + if extract_proc.exitcode != 0: + raise QubesException( + "ERROR: unable to extract the qubes backup. " \ + "Check extracting process errors.") + +def backup_restore_set_defaults(options): + if 'use-default-netvm' not in options: + options['use-default-netvm'] = False + if 'use-none-netvm' not in options: + options['use-none-netvm'] = False + if 'use-default-template' not in options: + options['use-default-template'] = False + if 'dom0-home' not in options: + options['dom0-home'] = True + if 'replace-template' not in options: + options['replace-template'] = [] + + return options + +def load_hmac(hmac): + hmac = hmac.strip(" \t\r\n").split("=") + if len(hmac) > 1: + hmac = hmac[1].strip() + else: + raise QubesException("ERROR: invalid hmac file content") + + return hmac + +def backup_restore_header(source, passphrase, + print_callback = print_stdout, error_callback = print_stderr, + encrypted=False, appvm=None): + + # Simulate dd if=backup_file count=10 | file - + # Simulate dd if=backup_file count=10 | gpg2 -d | tar xzv -O + # analysis = subprocess.Popen() + vmproc = None + + import tempfile + feedback_file = tempfile.NamedTemporaryFile() + restore_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/restore_") + + os.chdir(restore_tmpdir) + + if BACKUP_DEBUG: + print "Working in", restore_tmpdir + + restore_vm_dirs (source, + restore_tmpdir, + passphrase=passphrase, + vms_dirs=['qubes.xml.000', 'qubes.xml.000.hmac'], + vms=None, + vms_size=40000, + print_callback=print_callback, + error_callback=error_callback, + progress_callback=None, + encrypted=encrypted, + appvm=appvm) + + return (restore_tmpdir, "qubes.xml") + +def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, + host_collection = None, encrypt=False, appvm=None): + # Defaults + backup_restore_set_defaults(options) + + #### Private functions begin + def is_vm_included_in_backup (backup_dir, vm): + if vm.backup_content: + return True + else: + return False + + def find_template_name(template, replaces): + rx_replace = re.compile("(.*):(.*)") + for r in replaces: + m = rx_replace.match(r) + if m.group(1) == template: + return m.group(2) + + return template + #### Private functions end + if BACKUP_DEBUG: + print "Loading file", qubes_xml + backup_collection = QubesVmCollection(store_filename = qubes_xml) + backup_collection.lock_db_for_reading() + backup_collection.load() + + if host_collection is None: + host_collection = QubesVmCollection() + host_collection.lock_db_for_reading() + host_collection.load() + host_collection.unlock_db() + + backup_vms_list = [vm for vm in backup_collection.values()] + host_vms_list = [vm for vm in host_collection.values()] + vms_to_restore = {} + + there_are_conflicting_vms = False + there_are_missing_templates = False + there_are_missing_netvms = False + dom0_username_mismatch = False + restore_home = False + # ... and the actual data + for vm in backup_vms_list: + if vm.qid == 0: + # Handle dom0 as special case later + continue + if is_vm_included_in_backup (backup_dir, vm): + if BACKUP_DEBUG: + print vm.name,"is included in backup" + + vms_to_restore[vm.name] = {} + vms_to_restore[vm.name]['vm'] = vm; + if 'exclude' in options.keys(): + vms_to_restore[vm.name]['excluded'] = vm.name in options['exclude'] + vms_to_restore[vm.name]['good-to-go'] = False + + if host_collection.get_vm_by_name (vm.name) is not None: + vms_to_restore[vm.name]['already-exists'] = True + vms_to_restore[vm.name]['good-to-go'] = False + + if vm.template is None: + vms_to_restore[vm.name]['template'] = None + else: + templatevm_name = find_template_name(vm.template.name, options['replace-template']) + vms_to_restore[vm.name]['template'] = templatevm_name + template_vm_on_host = host_collection.get_vm_by_name (templatevm_name) + + # No template on the host? + if not ((template_vm_on_host is not None) and template_vm_on_host.is_template()): + # Maybe the (custom) template is in the backup? + template_vm_on_backup = backup_collection.get_vm_by_name (templatevm_name) + if template_vm_on_backup is None or not \ + (is_vm_included_in_backup(backup_dir, template_vm_on_backup) and \ + template_vm_on_backup.is_template()): + if options['use-default-template']: + vms_to_restore[vm.name]['orig-template'] = templatevm_name + vms_to_restore[vm.name]['template'] = host_collection.get_default_template().name + else: + vms_to_restore[vm.name]['missing-template'] = True + vms_to_restore[vm.name]['good-to-go'] = False + + if vm.netvm is None: + vms_to_restore[vm.name]['netvm'] = None + else: + netvm_name = vm.netvm.name + vms_to_restore[vm.name]['netvm'] = netvm_name + # Set to None to not confuse QubesVm object from backup + # collection with host collection (further in clone_attrs). Set + # directly _netvm to suppress setter action, especially + # modifying firewall + vm._netvm = None + + netvm_on_host = host_collection.get_vm_by_name (netvm_name) + + # No netvm on the host? + if not ((netvm_on_host is not None) and netvm_on_host.is_netvm()): + + # Maybe the (custom) netvm is in the backup? + netvm_on_backup = backup_collection.get_vm_by_name (netvm_name) + if not ((netvm_on_backup is not None) and \ + netvm_on_backup.is_netvm() and \ + is_vm_included_in_backup(backup_dir, netvm_on_backup)): + if options['use-default-netvm']: + vms_to_restore[vm.name]['netvm'] = host_collection.get_default_netvm().name + vm.uses_default_netvm = True + elif options['use-none-netvm']: + vms_to_restore[vm.name]['netvm'] = None + else: + vms_to_restore[vm.name]['missing-netvm'] = True + vms_to_restore[vm.name]['good-to-go'] = False + + if 'good-to-go' not in vms_to_restore[vm.name].keys(): + vms_to_restore[vm.name]['good-to-go'] = True + + # ...and dom0 home + if options['dom0-home'] and \ + is_vm_included_in_backup(backup_dir, backup_collection[0]): + vm = backup_collection[0] + vms_to_restore['dom0'] = {} + vms_to_restore['dom0']['subdir'] = vm.backup_path + vms_to_restore['dom0']['size'] = vm.backup_size + local_user = grp.getgrnam('qubes').gr_mem[0] + + dom0_home = vm.backup_path + + vms_to_restore['dom0']['username'] = os.path.basename(dom0_home) + if vms_to_restore['dom0']['username'] != local_user: + vms_to_restore['dom0']['username-mismatch'] = True + if not options['ignore-dom0-username-mismatch']: + vms_to_restore['dom0']['good-to-go'] = False + + if 'good-to-go' not in vms_to_restore['dom0']: + vms_to_restore['dom0']['good-to-go'] = True + + # Not needed - all the data stored in vms_to_restore + os.unlink(qubes_xml) + return vms_to_restore + +def backup_restore_print_summary(restore_info, print_callback = print_stdout): + fields = { + "qid": {"func": "vm.qid"}, + + "name": {"func": "('[' if vm.is_template() else '')\ + + ('{' if vm.is_netvm() else '')\ + + vm.name \ + + (']' if vm.is_template() else '')\ + + ('}' if vm.is_netvm() else '')"}, + + "type": {"func": "'Tpl' if vm.is_template() else \ + 'HVM' if vm.type == 'HVM' else \ + vm.type.replace('VM','')"}, + + "updbl" : {"func": "'Yes' if vm.updateable else ''"}, + + "template": {"func": "'n/a' if vm.is_template() or vm.template is None else\ + vm_info['template']"}, + + "netvm": {"func": "'n/a' if vm.is_netvm() and not vm.is_proxyvm() else\ + ('*' if vm.uses_default_netvm else '') +\ + vm_info['netvm'] if vm_info['netvm'] is not None else '-'"}, + + "label" : {"func" : "vm.label.name"}, + } + + fields_to_display = ["name", "type", "template", "updbl", "netvm", "label" ] + + # First calculate the maximum width of each field we want to display + total_width = 0; + for f in fields_to_display: + fields[f]["max_width"] = len(f) + for vm_info in restore_info.values(): + if 'vm' in vm_info.keys(): + vm = vm_info['vm'] + l = len(str(eval(fields[f]["func"]))) + if l > fields[f]["max_width"]: + fields[f]["max_width"] = l + total_width += fields[f]["max_width"] + + print_callback("") + print_callback("The following VMs are included in the backup:") + print_callback("") + + # Display the header + s = "" + for f in fields_to_display: + fmt="{{0:-^{0}}}-+".format(fields[f]["max_width"] + 1) + s += fmt.format('-') + print_callback(s) + s = "" + for f in fields_to_display: + fmt="{{0:>{0}}} |".format(fields[f]["max_width"] + 1) + s += fmt.format(f) + print_callback(s) + s = "" + for f in fields_to_display: + fmt="{{0:-^{0}}}-+".format(fields[f]["max_width"] + 1) + s += fmt.format('-') + print_callback(s) + + for vm_info in restore_info.values(): + # Skip non-VM here + if not 'vm' in vm_info: + continue + vm = vm_info['vm'] + s = "" + for f in fields_to_display: + fmt="{{0:>{0}}} |".format(fields[f]["max_width"] + 1) + s += fmt.format(eval(fields[f]["func"])) + + if 'excluded' in vm_info and vm_info['excluded']: + s += " <-- Excluded from restore" + elif 'already-exists' in vm_info: + s += " <-- A VM with the same name already exists on the host!" + elif 'missing-template' in vm_info: + s += " <-- No matching template on the host or in the backup found!" + elif 'missing-netvm' in vm_info: + s += " <-- No matching netvm on the host or in the backup found!" + elif 'orig-template' in vm_info: + s += " <-- Original template was '%s'" % (vm_info['orig-template']) + + print_callback(s) + + if 'dom0' in restore_info.keys(): + s = "" + for f in fields_to_display: + fmt="{{0:>{0}}} |".format(fields[f]["max_width"] + 1) + if f == "name": + s += fmt.format("Dom0") + elif f == "type": + s += fmt.format("Home") + else: + s += fmt.format("") + if 'username-mismatch' in restore_info['dom0']: + s += " <-- username in backup and dom0 mismatch" + + print_callback(s) + +def backup_restore_do(backup_dir, restore_tmpdir, passphrase, restore_info, + host_collection = None, print_callback = print_stdout, + error_callback = print_stderr, progress_callback = None, + encrypted=False, appvm=None): + + lock_obtained = False + if host_collection is None: + host_collection = QubesVmCollection() + host_collection.lock_db_for_writing() + host_collection.load() + lock_obtained = True + + # Perform VM restoration in backup order + vms_dirs = [] + vms_size = 0 + vms = {} + for vm_info in restore_info.values(): + if not vm_info['good-to-go']: + continue + if 'vm' not in vm_info: + continue + vm = vm_info['vm'] + vms_size += vm.backup_size + vms_dirs.append(vm.backup_path) + vms[vm.name] = vm + + if 'dom0' in restore_info.keys() and restore_info['dom0']['good-to-go']: + vms_dirs.append('dom0-home') + vms_size += restore_info['dom0']['size'] + + restore_vm_dirs (backup_dir, + restore_tmpdir, + passphrase=passphrase, + vms_dirs=vms_dirs, + vms=vms, + vms_size=vms_size, + print_callback=print_callback, + error_callback=error_callback, + progress_callback=progress_callback, + encrypted=encrypted, + appvm=appvm) + + # Add VM in right order + for (vm_class_name, vm_class) in sorted(QubesVmClasses.items(), + key=lambda _x: _x[1].load_order): + for vm_info in restore_info.values(): + if not vm_info['good-to-go']: + continue + if 'vm' not in vm_info: + continue + vm = vm_info['vm'] + if not vm.__class__ == vm_class: + continue + print_callback("-> Restoring {type} {0}...".format(vm.name, type=vm_class_name)) + retcode = subprocess.call (["mkdir", "-p", os.path.dirname(vm.dir_path)]) + if retcode != 0: + error_callback("*** Cannot create directory: {0}?!".format(dest_dir)) + error_callback("Skipping...") + continue + + template = None + if vm.template is not None: + template_name = vm_info['template'] + template = host_collection.get_vm_by_name(template_name) + + new_vm = None + + try: + new_vm = host_collection.add_new_vm(vm_class_name, name=vm.name, + conf_file=vm.conf_file, + dir_path=vm.dir_path, + template=template, + installed_by_rpm=False) + + shutil.move(os.path.join(restore_tmpdir, vm.backup_path), + new_vm.dir_path) + + new_vm.verify_files() + except Exception as err: + error_callback("ERROR: {0}".format(err)) + error_callback("*** Skipping VM: {0}".format(vm.name)) + if new_vm: + host_collection.pop(new_vm.qid) + continue + + try: + new_vm.clone_attrs(vm) + except Exception as err: + error_callback("ERROR: {0}".format(err)) + error_callback("*** Some VM property will not be restored") + + try: + new_vm.appmenus_create(verbose=True) + except Exception as err: + error_callback("ERROR during appmenu restore: {0}".format(err)) + error_callback("*** VM '{0}' will not have appmenus".format(vm.name)) + + # Set network dependencies - only non-default netvm setting + for vm_info in restore_info.values(): + if not vm_info['good-to-go']: + continue + if 'vm' not in vm_info: + continue + vm = vm_info['vm'] + host_vm = host_collection.get_vm_by_name(vm.name) + if host_vm is None: + # Failed/skipped VM + continue + + if not vm.uses_default_netvm: + host_vm.netvm = host_collection.get_vm_by_name (vm_info['netvm']) if vm_info['netvm'] is not None else None + + host_collection.save() + if lock_obtained: + host_collection.unlock_db() + + # ... and dom0 home as last step + if 'dom0' in restore_info.keys() and restore_info['dom0']['good-to-go']: + backup_path = restore_info['dom0']['subdir'] + local_user = grp.getgrnam('qubes').gr_mem[0] + home_dir = pwd.getpwnam(local_user).pw_dir + backup_dom0_home_dir = os.path.join(restore_tmpdir, backup_path) + restore_home_backupdir = "home-pre-restore-{0}".format (time.strftime("%Y-%m-%d-%H%M%S")) + + print_callback("-> Restoring home of user '{0}'...".format(local_user)) + print_callback("--> Existing files/dirs backed up in '{0}' dir".format(restore_home_backupdir)) + os.mkdir(home_dir + '/' + restore_home_backupdir) + for f in os.listdir(backup_dom0_home_dir): + home_file = home_dir + '/' + f + if os.path.exists(home_file): + os.rename(home_file, home_dir + '/' + restore_home_backupdir + '/' + f) + shutil.move(backup_dom0_home_dir + '/' + f, home_file) + retcode = subprocess.call(['sudo', 'chown', '-R', local_user, home_dir]) + if retcode != 0: + error_callback("*** Error while setting home directory owner") + + shutil.rmtree(restore_tmpdir) + +# vim:sw=4:et: diff --git a/core/qubesutils.py b/core/qubesutils.py index 304e5a24..9034540e 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -38,8 +38,6 @@ import xen.lowlevel.xc import xen.lowlevel.xl import xen.lowlevel.xs -BACKUP_DEBUG = True - def mbytes_to_kmg(size): if size > 1024: return "%d GiB" % (size/1024) @@ -754,1454 +752,4 @@ class QubesWatch(object): while True: self.watch_single() -######## Backups ######### - -def get_disk_usage(file_or_dir): - if not os.path.exists(file_or_dir): - return 0 - - p = subprocess.Popen (["du", "-s", "--block-size=1", file_or_dir], - stdout=subprocess.PIPE) - result = p.communicate() - m = re.match(r"^(\d+)\s.*", result[0]) - sz = int(m.group(1)) if m is not None else 0 - return sz - - -def file_to_backup (file_path, sz = None): - if sz is None: - sz = os.path.getsize (system_path["qubes_store_filename"]) - - abs_file_path = os.path.abspath (file_path) - abs_base_dir = os.path.abspath (system_path["qubes_base_dir"]) + '/' - abs_file_dir = os.path.dirname (abs_file_path) + '/' - (nothing, dir, subdir) = abs_file_dir.partition (abs_base_dir) - assert nothing == "" - assert dir == abs_base_dir - return [ { "path" : file_path, "size": sz, "subdir": subdir} ] - -def backup_prepare(vms_list = None, exclude_list = [], print_callback = print_stdout): - """If vms = None, include all (sensible) VMs; exclude_list is always applied""" - files_to_backup = file_to_backup (system_path["qubes_store_filename"]) - - if exclude_list is None: - exclude_list = [] - - qvm_collection = QubesVmCollection() - qvm_collection.lock_db_for_writing() - qvm_collection.load() - - if vms_list is None: - all_vms = [vm for vm in qvm_collection.values()] - selected_vms = [vm for vm in all_vms if vm.include_in_backups] - appvms_to_backup = [vm for vm in selected_vms if vm.is_appvm() and not vm.internal] - netvms_to_backup = [vm for vm in selected_vms if vm.is_netvm() and not vm.qid == 0] - template_vms_worth_backingup = [vm for vm in selected_vms if (vm.is_template() and not vm.installed_by_rpm)] - - vms_list = appvms_to_backup + netvms_to_backup + template_vms_worth_backingup - - vms_for_backup = vms_list - # Apply exclude list - if exclude_list: - vms_for_backup = [vm for vm in vms_list if vm.name not in exclude_list] - - no_vms = len (vms_for_backup) - - there_are_running_vms = False - - fields_to_display = [ - { "name": "VM", "width": 16}, - { "name": "type","width": 12 }, - { "name": "size", "width": 12} - ] - - # Display the header - s = "" - for f in fields_to_display: - fmt="{{0:-^{0}}}-+".format(f["width"] + 1) - s += fmt.format('-') - print_callback(s) - s = "" - for f in fields_to_display: - fmt="{{0:>{0}}} |".format(f["width"] + 1) - s += fmt.format(f["name"]) - print_callback(s) - s = "" - for f in fields_to_display: - fmt="{{0:-^{0}}}-+".format(f["width"] + 1) - s += fmt.format('-') - print_callback(s) - - for vm in vms_for_backup: - if vm.is_template(): - # handle templates later - continue - - if vm.private_img is not None: - vm_sz = vm.get_disk_usage (vm.private_img) - files_to_backup += file_to_backup(vm.private_img, vm_sz ) - - if vm.is_appvm(): - files_to_backup += file_to_backup(vm.icon_path) - if vm.updateable: - if os.path.exists(vm.dir_path + "/apps.templates"): - # template - files_to_backup += file_to_backup(vm.dir_path + "/apps.templates") - else: - # standaloneVM - files_to_backup += file_to_backup(vm.dir_path + "/apps") - - if os.path.exists(vm.dir_path + "/kernels"): - files_to_backup += file_to_backup(vm.dir_path + "/kernels") - if os.path.exists (vm.firewall_conf): - files_to_backup += file_to_backup(vm.firewall_conf) - if 'appmenus_whitelist' in vm_files and \ - os.path.exists(vm.dir_path + vm_files['appmenus_whitelist']): - files_to_backup += file_to_backup(vm.dir_path + vm_files['appmenus_whitelist']) - - if vm.updateable: - sz = vm.get_disk_usage(vm.root_img) - files_to_backup += file_to_backup(vm.root_img, sz) - vm_sz += sz - - s = "" - fmt="{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1) - s += fmt.format(vm.name) - - fmt="{{0:>{0}}} |".format(fields_to_display[1]["width"] + 1) - if vm.is_netvm(): - s += fmt.format("NetVM" + (" + Sys" if vm.updateable else "")) - else: - s += fmt.format("AppVM" + (" + Sys" if vm.updateable else "")) - - fmt="{{0:>{0}}} |".format(fields_to_display[2]["width"] + 1) - s += fmt.format(size_to_human(vm_sz)) - - if vm.is_running(): - s += " <-- The VM is running, please shut it down before proceeding with the backup!" - there_are_running_vms = True - - print_callback(s) - - for vm in vms_for_backup: - if not vm.is_template(): - # already handled - continue - vm_sz = vm.get_disk_utilization() - files_to_backup += file_to_backup (vm.dir_path, vm_sz) - - s = "" - fmt="{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1) - s += fmt.format(vm.name) - - fmt="{{0:>{0}}} |".format(fields_to_display[1]["width"] + 1) - s += fmt.format("Template VM") - - fmt="{{0:>{0}}} |".format(fields_to_display[2]["width"] + 1) - s += fmt.format(size_to_human(vm_sz)) - - if vm.is_running(): - s += " <-- The VM is running, please shut it down before proceeding with the backup!" - there_are_running_vms = True - - print_callback(s) - - # Initialize backup flag on all VMs - vms_for_backup_qid = [vm.qid for vm in vms_for_backup] - for vm in qvm_collection.values(): - vm.backup_content = False - - if vm.qid in vms_for_backup_qid: - vm.backup_content = True - vm.backup_size = vm.get_disk_utilization() - vm.backup_path = os.path.relpath(vm.dir_path, system_path["qubes_base_dir"]) - - # Dom0 user home - if not 'dom0' in exclude_list: - local_user = grp.getgrnam('qubes').gr_mem[0] - home_dir = pwd.getpwnam(local_user).pw_dir - # Home dir should have only user-owned files, so fix it now to prevent - # permissions problems - some root-owned files can left after - # 'sudo bash' and similar commands - subprocess.check_call(['sudo', 'chown', '-R', local_user, home_dir]) - - home_sz = get_disk_usage(home_dir) - home_to_backup = [ { "path" : home_dir, "size": home_sz, "subdir": 'dom0-home/'} ] - files_to_backup += home_to_backup - - vm = qvm_collection[0] - vm.backup_content = True - vm.backup_size = home_sz - vm.backup_path = os.path.join('dom0-home', os.path.basename(home_dir)) - - s = "" - fmt="{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1) - s += fmt.format('Dom0') - - fmt="{{0:>{0}}} |".format(fields_to_display[1]["width"] + 1) - s += fmt.format("User home") - - fmt="{{0:>{0}}} |".format(fields_to_display[2]["width"] + 1) - s += fmt.format(size_to_human(home_sz)) - - print_callback(s) - - qvm_collection.save() - # FIXME: should be after backup completed - qvm_collection.unlock_db() - - total_backup_sz = 0 - for file in files_to_backup: - total_backup_sz += file["size"] - - s = "" - for f in fields_to_display: - fmt="{{0:-^{0}}}-+".format(f["width"] + 1) - s += fmt.format('-') - print_callback(s) - - s = "" - fmt="{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1) - s += fmt.format("Total size:") - fmt="{{0:>{0}}} |".format(fields_to_display[1]["width"] + 1 + 2 + fields_to_display[2]["width"] + 1) - s += fmt.format(size_to_human(total_backup_sz)) - print_callback(s) - - s = "" - for f in fields_to_display: - fmt="{{0:-^{0}}}-+".format(f["width"] + 1) - s += fmt.format('-') - print_callback(s) - - if (there_are_running_vms): - raise QubesException("Please shutdown all VMs before proceeding.") - - return files_to_backup - -def backup_do(base_backup_dir, files_to_backup, progress_callback = None): - - total_backup_sz = 0 - for file in files_to_backup: - total_backup_sz += file["size"] - - backup_dir = base_backup_dir + "/qubes-{0}".format (time.strftime("%Y-%m-%d-%H%M%S")) - if os.path.exists (backup_dir): - raise QubesException("ERROR: the path {0} already exists?!".format(backup_dir)) - - os.mkdir (backup_dir) - - if not os.path.exists (backup_dir): - raise QubesException("Strange: couldn't create backup dir: {0}?!".format(backup_dir)) - - bytes_backedup = 0 - for file in files_to_backup: - # We prefer to use Linux's cp, because it nicely handles sparse files - progress = bytes_backedup * 100 / total_backup_sz - progress_callback(progress) - dest_dir = backup_dir + '/' + file["subdir"] - if file["subdir"] != "": - retcode = subprocess.call (["mkdir", "-p", dest_dir]) - if retcode != 0: - raise QubesException("Cannot create directory: {0}?!".format(dest_dir)) - - retcode = subprocess.call (["cp", "-rp", file["path"], dest_dir]) - if retcode != 0: - raise QubesException("Error while copying file {0} to {1}".format(file["path"], dest_dir)) - - bytes_backedup += file["size"] - progress = bytes_backedup * 100 / total_backup_sz - progress_callback(progress) - -def backup_do_copy(base_backup_dir, files_to_backup, passphrase,\ - progress_callback = None, encrypt=False, appvm=None): - total_backup_sz = 0 - for file in files_to_backup: - total_backup_sz += file["size"] - - vmproc = None - if appvm != None: - # Prepare the backup target (Qubes service call) - backup_target = "QUBESRPC qubes.Backup none" - - # does the vm exist? - qvm_collection = QubesVmCollection() - qvm_collection.lock_db_for_reading() - qvm_collection.load() - - vm = qvm_collection.get_vm_by_name(appvm) - if vm is None or vm.qid not in qvm_collection: - raise QubesException("VM {0} does not exist".format(appvm)) - - qvm_collection.unlock_db() - - # If APPVM, STDOUT is a PIPE - vmproc = vm.run(command = backup_target, passio_popen = True) - vmproc.stdin.write(base_backup_dir.\ - replace("\r","").replace("\n","")+"\n") - backup_stdout = vmproc.stdin - - else: - # Prepare the backup target (local file) - backup_target = base_backup_dir + "/qubes-{0}".\ - format (time.strftime("%Y-%m-%d-%H%M%S")) - - # Create the target directory - if not os.path.exists (base_backup_dir): - raise QubesException( - "ERROR: the backup directory {0} does not exists".\ - format(base_backup_dir)) - - # If not APPVM, STDOUT is a local file - backup_stdout = open(backup_target,'wb') - - global blocks_backedup - blocks_backedup = 0 - progress = blocks_backedup * 11 / total_backup_sz - progress_callback(progress) - - import tempfile - feedback_file = tempfile.NamedTemporaryFile() - backup_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/backup_") - - # Tar with tapelength does not deals well with stdout (close stdout between - # two tapes) - # For this reason, we will use named pipes instead - if BACKUP_DEBUG: - print "Working in", backup_tmpdir - - backup_pipe = os.path.join(backup_tmpdir,"backup_pipe") - if BACKUP_DEBUG: - print "Creating pipe in:", backup_pipe - os.mkfifo(backup_pipe) - - if BACKUP_DEBUG: - print "Will backup:", files_to_backup - - # Setup worker to send encrypted data chunks to the backup_target - from multiprocessing import Queue,Process - class Send_Worker(Process): - def __init__(self, queue, base_dir, backup_stdout): - super(Send_Worker, self).__init__() - self.queue = queue - self.base_dir = base_dir - self.backup_stdout = backup_stdout - - def run(self): - if BACKUP_DEBUG: - print "Started sending thread" - - if BACKUP_DEBUG: - print "Moving to temporary dir", self.base_dir - os.chdir(self.base_dir) - - for filename in iter(self.queue.get,None): - if filename == "FINISHED": - break - - if BACKUP_DEBUG: - print "Sending file", filename - # This tar used for sending data out need to be as simple, as - # simple, as featureless as possible. It will not be - # verified before untaring. - tar_final_cmd = ["tar", "-cO", "--posix", - "-C", self.base_dir, filename] - final_proc = subprocess.Popen (tar_final_cmd, - stdin=subprocess.PIPE, stdout=self.backup_stdout) - final_proc.wait() - - # Delete the file as we don't need it anymore - if BACKUP_DEBUG: - print "Removing file", filename - os.remove(filename) - - if BACKUP_DEBUG: - print "Finished sending thread" - - def compute_progress(new_size, total_backup_sz): - global blocks_backedup - blocks_backedup += new_size - progress = blocks_backedup / float(total_backup_sz) - progress_callback(int(round(progress*100,2))) - - to_send = Queue(10) - send_proc = Send_Worker(to_send, backup_tmpdir, backup_stdout) - send_proc.start() - - for filename in files_to_backup: - if BACKUP_DEBUG: - print "Backing up", filename - - backup_tempfile = os.path.join(backup_tmpdir, - filename["subdir"], - os.path.basename(filename["path"])) - if BACKUP_DEBUG: - print "Using temporary location:", backup_tempfile - - # Ensure the temporary directory exists - - if not os.path.isdir(os.path.dirname(backup_tempfile)): - os.makedirs(os.path.dirname(backup_tempfile)) - - # The first tar cmd can use any complex feature as we want. Files will - # be verified before untaring this. - # Prefix the path in archive with filename["subdir"] to have it verified during untar - tar_cmdline = ["tar", "-Pc", '--sparse', - "-f", backup_pipe, - '--tape-length', str(100000), - '-C', os.path.dirname(filename["path"]), - '--xform', 's:^[a-z]:%s\\0:' % filename["subdir"], - os.path.basename(filename["path"]) - ] - - if BACKUP_DEBUG: - print " ".join(tar_cmdline) - - # Tips: Popen(bufsize=0) - # Pipe: tar-sparse | encryptor [| hmac] | tar | backup_target - # Pipe: tar-sparse [| hmac] | tar | backup_target - tar_sparse = subprocess.Popen (tar_cmdline, stdin=subprocess.PIPE) - - # Wait for compressor (tar) process to finish or for any error of other - # subprocesses - i = 0 - run_error = "paused" - running = [] - while run_error == "paused": - - pipe = open(backup_pipe,'rb') - - # Start HMAC - hmac = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], - stdin=subprocess.PIPE, stdout=subprocess.PIPE) - - # Prepare a first chunk - chunkfile = backup_tempfile + "." + "%03d" % i - i += 1 - chunkfile_p = open(chunkfile,'wb') - - if encrypt: - # Start encrypt - # If no cipher is provided, the data is forwarded unencrypted !!! - # Also note that the - encryptor = subprocess.Popen (["openssl", "enc", - "-e", "-aes-256-cbc", - "-pass", "pass:"+passphrase], - stdin=pipe, stdout=subprocess.PIPE) - run_error = wait_backup_feedback( - progress_callback=compute_progress, - in_stream=encryptor.stdout, - streamproc=encryptor, - backup_target=chunkfile_p, - total_backup_sz=total_backup_sz, - hmac=hmac, - vmproc=vmproc, - addproc=tar_sparse) - else: - run_error = wait_backup_feedback( - progress_callback=compute_progress, - in_stream=pipe, - streamproc=None, - backup_target=chunkfile_p, - total_backup_sz=total_backup_sz, - hmac=hmac, - vmproc=vmproc, - addproc=tar_sparse) - - chunkfile_p.close() - - if BACKUP_DEBUG: - print "Wait_backup_feedback returned:", run_error - - if len(run_error) > 0: - send_proc.terminate() - raise QubesException("Failed to perform backup: error with "+ \ - run_error) - - # Send the chunk to the backup target - to_send.put(os.path.relpath(chunkfile, backup_tmpdir)) - - # Close HMAC - hmac.stdin.close() - hmac.wait() - if BACKUP_DEBUG: - print "HMAC proc return code:", hmac.poll() - - # Write HMAC data next to the chunk file - hmac_data = hmac.stdout.read() - if BACKUP_DEBUG: - print "Writing hmac to", chunkfile+".hmac" - hmac_file = open(chunkfile+".hmac",'w') - hmac_file.write(hmac_data) - hmac_file.flush() - hmac_file.close() - - pipe.close() - - # Send the HMAC to the backup target - to_send.put(os.path.relpath(chunkfile, backup_tmpdir)+".hmac") - - if tar_sparse.poll() == None: - # Release the next chunk - if BACKUP_DEBUG: - print "Release next chunk for process:", tar_sparse.poll() - #tar_sparse.stdout = subprocess.PIPE - tar_sparse.stdin.write("\n") - run_error="paused" - else: - if BACKUP_DEBUG: - print "Finished tar sparse with error", tar_sparse.poll() - - to_send.put("FINISHED") - send_proc.join() - - if send_proc.exitcode != 0: - raise QubesException("Failed to send backup: error in the sending process") - - if vmproc: - if BACKUP_DEBUG: - print "VMProc1 proc return code:", vmproc.poll() - print "Sparse1 proc return code:", tar_sparse.poll() - vmproc.stdin.close() - -''' -' Wait for backup chunk to finish -' - Monitor all the processes (streamproc, hmac, vmproc, addproc) for errors -' - Copy stdout of streamproc to backup_target and hmac stdin if available -' - Compute progress based on total_backup_sz and send progress to -' progress_callback function -' - Returns if -' - one of the monitored processes error out (streamproc, hmac, vmproc, -' addproc), along with the processe that failed -' - all of the monitored processes except vmproc finished successfully -' (vmproc termination is controlled by the python script) -' - streamproc does not delivers any data anymore (return with the error -' "") -''' -def wait_backup_feedback(progress_callback, in_stream, streamproc, - backup_target, total_backup_sz, hmac=None, vmproc=None, addproc=None, - remove_trailing_bytes=0): - - buffer_size = 409600 - - run_error = None - run_count = 1 - blocks_backedup = 0 - while run_count > 0 and run_error == None: - - buffer = in_stream.read(buffer_size) - progress_callback(len(buffer), total_backup_sz) - - run_count = 0 - if hmac: - retcode=hmac.poll() - if retcode != None: - if retcode != 0: - run_error = "hmac" - else: - run_count += 1 - - if addproc: - retcode=addproc.poll() - #print "Tar proc status:", retcode - if retcode != None: - if retcode != 0: - run_error = "addproc" - else: - run_count += 1 - - if vmproc: - retcode = vmproc.poll() - if retcode != None: - if retcode != 0: - run_error = "VM" - #FIXME: ?? if BACKUP_DEBUG: - print vmproc.stdout.read() - else: - # VM should run until the end - pass - - if streamproc: - retcode=streamproc.poll() - if retcode != None: - if retcode != 0: - run_error = "streamproc" - elif retcode == 0 and len(buffer) <= 0: - return "" - else: - #print "INFO: last packet" - #if remove_trailing_bytes > 0: - # print buffer.encode("hex") - # buffer = buffer[:-remove_trailing_bytes] - # print buffer.encode("hex") - - backup_target.write(buffer) - - if hmac: - hmac.stdin.write(buffer) - - run_count += 1 - else: - #print "Process running:", len(buffer) - # Process still running - backup_target.write(buffer) - - if hmac: - hmac.stdin.write(buffer) - - run_count += 1 - - else: - if len(buffer) <= 0: - return "" - else: - backup_target.write(buffer) - - if hmac: - hmac.stdin.write(buffer) - - - return run_error - -def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, - vms_size, print_callback=None, error_callback=None, - progress_callback=None, encrypted=False, appvm=None): - - # Setup worker to extract encrypted data chunks to the restore dirs - from multiprocessing import Queue,Process - class Extract_Worker(Process): - def __init__(self, queue, base_dir, passphrase, encrypted, total_size, - print_callback, error_callback, progress_callback, vmproc=None): - super(Extract_Worker, self).__init__() - self.queue = queue - self.base_dir = base_dir - self.passphrase = passphrase - self.encrypted = encrypted - self.total_size = total_size - self.blocks_backedup = 0 - self.tar2_command = None - self.tar2_current_file = None - - self.print_callback = print_callback - self.error_callback = error_callback - self.progress_callback = progress_callback - - self.vmproc = vmproc - - self.restore_pipe = os.path.join(self.base_dir,"restore_pipe") - if BACKUP_DEBUG: - print "Creating pipe in:", self.restore_pipe - os.mkfifo(self.restore_pipe) - - def compute_progress(self, new_size, total_size): - self.blocks_backedup += new_size - progress = self.blocks_backedup / float(self.total_size) - progress = int(round(progress*100,2)) - self.progress_callback(progress) - - def run(self): - if BACKUP_DEBUG: - self.print_callback("Started sending thread") - self.print_callback("Moving to dir "+self.base_dir) - os.chdir(self.base_dir) - - for filename in iter(self.queue.get,None): - if filename == "FINISHED": - break - - if BACKUP_DEBUG: - self.print_callback("Extracting file "+filename) - - if filename.endswith('.000'): - # next file - if self.tar2_command != None: - if self.tar2_command.wait() != 0: - raise QubesException( - "ERROR: unable to extract files for {0}.".\ - format(self.tar2_current_file)) - else: - # Finished extracting the tar file - self.tar2_command = None - self.tar2_current_file = None - - tar2_cmdline = ['tar', - '-xMk%sf' % ("v" if BACKUP_DEBUG else ""), self.restore_pipe, - os.path.relpath(filename.rstrip('.000'))] - if BACKUP_DEBUG: - self.print_callback("Running command "+str(tar2_cmdline)) - self.tar2_command = subprocess.Popen(tar2_cmdline, - stdin=subprocess.PIPE, - stderr=(None if BACKUP_DEBUG else open('/dev/null', 'w'))) - else: - if BACKUP_DEBUG: - self.print_callback("Releasing next chunck") - self.tar2_command.stdin.write("\n") - self.tar2_command.stdin.flush() - self.tar2_current_file = filename - - pipe = open(self.restore_pipe,'wb') - common_args = { - 'backup_target': pipe, - 'total_backup_sz': self.total_size, - 'hmac': None, - 'vmproc': self.vmproc, - 'addproc': self.tar2_command - } - if self.encrypted: - # Start decrypt - encryptor = subprocess.Popen (["openssl", "enc", - "-d", "-aes-256-cbc", - "-pass", "pass:"+passphrase], - stdin=open(filename,'rb'), - stdout=subprocess.PIPE) - - run_error = wait_backup_feedback( - progress_callback=self.compute_progress, - in_stream=encryptor.stdout, streamproc=encryptor, - **common_args) - else: - run_error = wait_backup_feedback( - progress_callback=self.compute_progress, - in_stream=open(filename,"rb"), streamproc=None, - **common_args) - - pipe.close() - - # Delete the file as we don't need it anymore - if BACKUP_DEBUG: - self.print_callback("Removing file "+filename) - os.remove(filename) - - if self.tar2_command != None: - if self.tar2_command.wait() != 0: - raise QubesException( - "ERROR: unable to extract files for {0}.".\ - format(self.tar2_current_file)) - else: - # Finished extracting the tar file - self.tar2_command = None - - if BACKUP_DEBUG: - self.print_callback("Finished extracting thread") - - if progress_callback == None: - def progress_callback(data): - pass - - to_extract = Queue() - extract_proc = Extract_Worker(queue=to_extract, - base_dir=backup_tmpdir, - passphrase=passphrase, - encrypted=encrypted, - total_size=vms_size, - print_callback=print_callback, - error_callback=error_callback, - progress_callback=progress_callback) - extract_proc.start() - - if BACKUP_DEBUG: - print_callback("Working in temporary dir:"+backup_tmpdir) - print_callback(str(vms_size)+" bytes to restore") - - vmproc = None - if appvm != None: - # Prepare the backup target (Qubes service call) - backup_target = "QUBESRPC qubes.Restore dom0" - - # does the vm exist? - qvm_collection = QubesVmCollection() - qvm_collection.lock_db_for_reading() - qvm_collection.load() - - vm = qvm_collection.get_vm_by_name(appvm) - if vm is None or vm.qid not in qvm_collection: - raise QubesException("VM {0} does not exist".format(appvm)) - - qvm_collection.unlock_db() - - # If APPVM, STDOUT is a PIPE - vmproc = vm.run(command = backup_target, passio_popen = True) - vmproc.stdin.write(backup_dir.replace("\r","").replace("\n","")+"\n") - - # Send to tar2qfile the VMs that should be extracted - vmproc.stdin.write(" ".join(vms_dirs)+"\n") - - backup_stdin = vmproc.stdout - tar1_command = ['/usr/libexec/qubes/qfile-dom0-unpacker', - str(os.getuid()), backup_tmpdir, '-v'] - else: - backup_stdin = open(backup_dir,'rb') - - tar1_command = ['tar', - '-ix%sf' % ("v" if BACKUP_DEBUG else ""), backup_dir, - '-C', backup_tmpdir] + vms_dirs - - # Remove already processed qubes.xml.000, because qfile-dom0-unpacker will - # refuse to override files - os.unlink(os.path.join(backup_tmpdir,'qubes.xml.000')) - os.unlink(os.path.join(backup_tmpdir,'qubes.xml.000.hmac')) - tar1_env = os.environ.copy() - # TODO: add some safety margin? - tar1_env['UPDATES_MAX_BYTES'] = str(vms_size) - tar1_env['UPDATES_MAX_FILES'] = '0' - if BACKUP_DEBUG: - print_callback("Run command"+str(tar1_command)) - command = subprocess.Popen(tar1_command, - stdin=backup_stdin, - stdout=vmproc.stdin if vmproc else subprocess.PIPE, - stderr=subprocess.PIPE, - env=tar1_env) - - # qfile-dom0-unpacker output filelist on stderr (and have stdout connected - # to the VM), while tar output filelist on stdout - if appvm: - filelist_pipe = command.stderr - else: - filelist_pipe = command.stdout - - while True: - - filename = filelist_pipe.readline().strip(" \t\r\n") - - if BACKUP_DEBUG: - print_callback("Getting new file:"+filename) - - if not filename or filename=="EOF": - break - - hmacfile = filelist_pipe.readline().strip(" \t\r\n") - if BACKUP_DEBUG: - print_callback("Getting hmac:"+hmacfile) - - if hmacfile != filename + ".hmac": - raise QubesException( - "ERROR: expected hmac for {}, but got {}".\ - format(filename, hmacfile)) - - # skip qubes.xml after receiving its hmac to skip both of them - if filename == 'qubes.xml.000': - if BACKUP_DEBUG: - print_callback("Ignoring already processed qubes.xml") - continue - - if not any(map(lambda x: filename.startswith(x), vms_dirs)): - if BACKUP_DEBUG: - print_callback("Ignoring VM not selected for restore") - os.unlink(filename) - os.unlink(hmacfile) - continue - - if BACKUP_DEBUG: - print_callback("Verifying file "+filename) - - if BACKUP_DEBUG: - print os.path.join(backup_tmpdir, filename) - hmac_proc = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], - stdin=open(os.path.join(backup_tmpdir, filename),'rb'), - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = hmac_proc.communicate() - - if len(stderr) > 0: - raise QubesException("ERROR: verify file {0}: {1}".format((filename, stderr))) - else: - if BACKUP_DEBUG: - print_callback("Loading hmac for file"+filename) - hmac = load_hmac( - open(os.path.join(backup_tmpdir, - filename+".hmac"),'r').read()) - - if len(hmac) > 0 and load_hmac(stdout) == hmac: - if BACKUP_DEBUG: - print_callback("File verification OK -> Sending file " + \ - filename+" for extraction") - # Send the chunk to the backup target - to_extract.put(os.path.join(backup_tmpdir, filename)) - - else: - raise QubesException( - "ERROR: invalid hmac for file {0}: {1}. " \ - "Is the passphrase correct?".\ - format(filename, load_hmac(stdout))) - - if command.wait() != 0: - raise QubesException( - "ERROR: unable to read the qubes backup file {0} ({1}). " \ - "Is it really a backup?".format(backup_dir, command.wait())) - if vmproc: - if vmproc.wait() != 0: - raise QubesException( - "ERROR: unable to read the qubes backup {0} " \ - "because of a VM error: {1}".format( - backup_dir, vmproc.stderr.read())) - if BACKUP_DEBUG: - print "Extraction process status:", extract_proc.exitcode - - to_extract.put("FINISHED") - if BACKUP_DEBUG: - print_callback("Waiting for the extraction process to finish...") - extract_proc.join() - if BACKUP_DEBUG: - print_callback("Extraction process finished with code:" + \ - str(extract_proc.exitcode)) - if extract_proc.exitcode != 0: - raise QubesException( - "ERROR: unable to extract the qubes backup. " \ - "Check extracting process errors.") - -def backup_restore_set_defaults(options): - if 'use-default-netvm' not in options: - options['use-default-netvm'] = False - if 'use-none-netvm' not in options: - options['use-none-netvm'] = False - if 'use-default-template' not in options: - options['use-default-template'] = False - if 'dom0-home' not in options: - options['dom0-home'] = True - if 'replace-template' not in options: - options['replace-template'] = [] - - return options - -def load_hmac(hmac): - hmac = hmac.strip(" \t\r\n").split("=") - if len(hmac) > 1: - hmac = hmac[1].strip() - else: - raise QubesException("ERROR: invalid hmac file content") - - return hmac - -def backup_restore_header(restore_target, passphrase, - encrypt=False, appvm=None): - # Simulate dd if=backup_file count=10 | file - - # Simulate dd if=backup_file count=10 | gpg2 -d | tar xzv -O - # analysis = subprocess.Popen() - vmproc = None - - import tempfile - feedback_file = tempfile.NamedTemporaryFile() - backup_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/restore_") - - os.chdir(backup_tmpdir) - - # Tar with tapelength does not deals well with stdout (close stdout between - # two tapes) - # For this reason, we will use named pipes instead - if BACKUP_DEBUG: - print "Working in", backup_tmpdir - - - tar1_env = os.environ.copy() - if appvm != None: - # Prepare the backup target (Qubes service call) - restore_command = "QUBESRPC qubes.Restore dom0" - - # does the vm exist? - qvm_collection = QubesVmCollection() - qvm_collection.lock_db_for_reading() - qvm_collection.load() - - vm = qvm_collection.get_vm_by_name(appvm) - if vm is None or vm.qid not in qvm_collection: - raise QubesException("VM {0} does not exist".format(appvm)) - - qvm_collection.unlock_db() - - # If APPVM, STDOUT is a PIPE - vmproc = vm.run(command = restore_command, passio_popen = True, - passio_stderr = True) - vmproc.stdin.write(restore_target.replace("\r","").replace("\n","")+"\n") - - # Ask to tar2qfile to only extract qubes.xml.* - vmproc.stdin.write("qubes.xml\n") - - tar1_command = ['/usr/libexec/qubes/qfile-dom0-unpacker', - str(os.getuid()), backup_tmpdir, '-v'] - # Ask qfile-dom0-unpacker to extract only qubes.xml.000 and qubes.xml.000.hmac - tar1_env['UPDATES_MAX_FILES'] = '2' - else: - # Check source file - if not os.path.exists (restore_target): - raise QubesException( - "ERROR: the backup directory {0} does not exists".\ - format(restore_target)) - - # TODO: perhaps pass only first 40kB here? Tar uses seek to skip files, - # so not a big problem, but still it might save some time - tar1_command = ['tar', - '-xi%sf' % ("v" if BACKUP_DEBUG else ""), restore_target, - '-C', backup_tmpdir, 'qubes.xml.000', 'qubes.xml.000.hmac'] - - command = subprocess.Popen(tar1_command, - stdin=vmproc.stdout if vmproc else None, - stdout=vmproc.stdin if vmproc else subprocess.PIPE, - stderr=subprocess.PIPE, - env=tar1_env) - - if vmproc and vmproc.poll() != None: - error = vmproc.stderr.read() - if BACKUP_DEBUG: - print error - print vmproc.poll(), command.poll() - raise QubesException( - "ERROR: Immediate VM error while retrieving backup headers:{0}".\ - format(error)) - - filename = "qubes.xml.000" - - command.wait() - - if vmproc and vmproc.wait() != 0: - error = vmproc.stderr.read() - if BACKUP_DEBUG: - print error - print vmproc.poll(), command.poll() - raise QubesException( - "ERROR: AppVM error retrieving backup headers: {0}".\ - format(error)) - elif command.returncode not in [0,-15,122]: - error = command.stderr.read() - if BACKUP_DEBUG: - print error - print vmproc.poll(), command.poll() - raise QubesException("ERROR: retrieving backup headers: {0}".\ - format(error)) - - if not os.path.exists(os.path.join(backup_tmpdir, filename+".hmac")): - raise QubesException("ERROR: header not extracted correctly: {0}".\ - format(os.path.join(backup_tmpdir, filename+".hmac"))) - - if vmproc and vmproc.poll() == None: - vmproc.terminate() - vmproc.wait() - - if BACKUP_DEBUG: - print "Loading hmac for file", filename - hmac = load_hmac( - open(os.path.join(backup_tmpdir, filename+".hmac"),'r').read()) - - if BACKUP_DEBUG: - print "Successfully retrieved headers" - - if BACKUP_DEBUG: - print "Verifying file", filename - hmac_proc = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], - stdin=open(os.path.join(backup_tmpdir, filename),'rb'), - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = hmac_proc.communicate() - if len(stderr) > 0: - raise QubesException("ERROR: verify file {0}: {1}".format((filename, stderr))) - else: - if len(hmac) > 0 and load_hmac(stdout) == hmac: - if BACKUP_DEBUG: - print "File verification OK -> Extracting archive", filename - if encrypt: - if BACKUP_DEBUG: - print "Starting decryption process" - encryptor = subprocess.Popen (["openssl", "enc", - "-d", "-aes-256-cbc", - "-pass", "pass:"+passphrase], - stdin=open(os.path.join(backup_tmpdir, filename),'rb'), - stdout=subprocess.PIPE) - tarhead_command = subprocess.Popen(['tar', - '-xM%s' % ("v" if BACKUP_DEBUG else "")], - stdin=encryptor.stdout) - else: - if BACKUP_DEBUG: - print "No decryption process required" - encryptor = None - tarhead_command = subprocess.Popen(['tar', - '-xM%sf' % ("v" if BACKUP_DEBUG else ""), - os.path.join(backup_tmpdir, filename)]) - - if encryptor: - if encryptor.wait() != 0: - raise QubesException( - "ERROR: unable to decrypt file {0}. " \ - "Bad password or unencrypted archive?".\ - format(filename)) - if tarhead_command.wait() != 0: - raise QubesException( - "ERROR: unable to extract the qubes.xml file. " \ - "Is archive encrypted?") - - return (backup_tmpdir, "qubes.xml") - else: - raise QubesException( - "ERROR: unable to verify the qubes.xml file. " \ - "Is the passphrase correct?") - - return None - -def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, - host_collection = None, encrypt=False, appvm=None): - # Defaults - backup_restore_set_defaults(options) - - #### Private functions begin - def is_vm_included_in_backup (backup_dir, vm): - if vm.backup_content: - return True - else: - return False - - def find_template_name(template, replaces): - rx_replace = re.compile("(.*):(.*)") - for r in replaces: - m = rx_replace.match(r) - if m.group(1) == template: - return m.group(2) - - return template - #### Private functions end - if BACKUP_DEBUG: - print "Loading file", qubes_xml - backup_collection = QubesVmCollection(store_filename = qubes_xml) - backup_collection.lock_db_for_reading() - backup_collection.load() - - if host_collection is None: - host_collection = QubesVmCollection() - host_collection.lock_db_for_reading() - host_collection.load() - host_collection.unlock_db() - - backup_vms_list = [vm for vm in backup_collection.values()] - host_vms_list = [vm for vm in host_collection.values()] - vms_to_restore = {} - - there_are_conflicting_vms = False - there_are_missing_templates = False - there_are_missing_netvms = False - dom0_username_mismatch = False - restore_home = False - # ... and the actual data - for vm in backup_vms_list: - if vm.qid == 0: - # Handle dom0 as special case later - continue - if is_vm_included_in_backup (backup_dir, vm): - if BACKUP_DEBUG: - print vm.name,"is included in backup" - - vms_to_restore[vm.name] = {} - vms_to_restore[vm.name]['vm'] = vm; - if 'exclude' in options.keys(): - vms_to_restore[vm.name]['excluded'] = vm.name in options['exclude'] - vms_to_restore[vm.name]['good-to-go'] = False - - if host_collection.get_vm_by_name (vm.name) is not None: - vms_to_restore[vm.name]['already-exists'] = True - vms_to_restore[vm.name]['good-to-go'] = False - - if vm.template is None: - vms_to_restore[vm.name]['template'] = None - else: - templatevm_name = find_template_name(vm.template.name, options['replace-template']) - vms_to_restore[vm.name]['template'] = templatevm_name - template_vm_on_host = host_collection.get_vm_by_name (templatevm_name) - - # No template on the host? - if not ((template_vm_on_host is not None) and template_vm_on_host.is_template()): - # Maybe the (custom) template is in the backup? - template_vm_on_backup = backup_collection.get_vm_by_name (templatevm_name) - if template_vm_on_backup is None or not \ - (is_vm_included_in_backup(backup_dir, template_vm_on_backup) and \ - template_vm_on_backup.is_template()): - if options['use-default-template']: - vms_to_restore[vm.name]['orig-template'] = templatevm_name - vms_to_restore[vm.name]['template'] = host_collection.get_default_template().name - else: - vms_to_restore[vm.name]['missing-template'] = True - vms_to_restore[vm.name]['good-to-go'] = False - - if vm.netvm is None: - vms_to_restore[vm.name]['netvm'] = None - else: - netvm_name = vm.netvm.name - vms_to_restore[vm.name]['netvm'] = netvm_name - # Set to None to not confuse QubesVm object from backup - # collection with host collection (further in clone_attrs). Set - # directly _netvm to suppress setter action, especially - # modifying firewall - vm._netvm = None - - netvm_on_host = host_collection.get_vm_by_name (netvm_name) - - # No netvm on the host? - if not ((netvm_on_host is not None) and netvm_on_host.is_netvm()): - - # Maybe the (custom) netvm is in the backup? - netvm_on_backup = backup_collection.get_vm_by_name (netvm_name) - if not ((netvm_on_backup is not None) and \ - netvm_on_backup.is_netvm() and \ - is_vm_included_in_backup(backup_dir, netvm_on_backup)): - if options['use-default-netvm']: - vms_to_restore[vm.name]['netvm'] = host_collection.get_default_netvm().name - vm.uses_default_netvm = True - elif options['use-none-netvm']: - vms_to_restore[vm.name]['netvm'] = None - else: - vms_to_restore[vm.name]['missing-netvm'] = True - vms_to_restore[vm.name]['good-to-go'] = False - - if 'good-to-go' not in vms_to_restore[vm.name].keys(): - vms_to_restore[vm.name]['good-to-go'] = True - - # ...and dom0 home - if options['dom0-home'] and \ - is_vm_included_in_backup(backup_dir, backup_collection[0]): - vm = backup_collection[0] - vms_to_restore['dom0'] = {} - vms_to_restore['dom0']['subdir'] = vm.backup_path - vms_to_restore['dom0']['size'] = vm.backup_size - local_user = grp.getgrnam('qubes').gr_mem[0] - - dom0_home = vm.backup_path - - vms_to_restore['dom0']['username'] = os.path.basename(dom0_home) - if vms_to_restore['dom0']['username'] != local_user: - vms_to_restore['dom0']['username-mismatch'] = True - if not options['ignore-dom0-username-mismatch']: - vms_to_restore['dom0']['good-to-go'] = False - - if 'good-to-go' not in vms_to_restore['dom0']: - vms_to_restore['dom0']['good-to-go'] = True - - return vms_to_restore - -def backup_restore_print_summary(restore_info, print_callback = print_stdout): - fields = { - "qid": {"func": "vm.qid"}, - - "name": {"func": "('[' if vm.is_template() else '')\ - + ('{' if vm.is_netvm() else '')\ - + vm.name \ - + (']' if vm.is_template() else '')\ - + ('}' if vm.is_netvm() else '')"}, - - "type": {"func": "'Tpl' if vm.is_template() else \ - 'HVM' if vm.type == 'HVM' else \ - vm.type.replace('VM','')"}, - - "updbl" : {"func": "'Yes' if vm.updateable else ''"}, - - "template": {"func": "'n/a' if vm.is_template() or vm.template is None else\ - vm_info['template']"}, - - "netvm": {"func": "'n/a' if vm.is_netvm() and not vm.is_proxyvm() else\ - ('*' if vm.uses_default_netvm else '') +\ - vm_info['netvm'] if vm_info['netvm'] is not None else '-'"}, - - "label" : {"func" : "vm.label.name"}, - } - - fields_to_display = ["name", "type", "template", "updbl", "netvm", "label" ] - - # First calculate the maximum width of each field we want to display - total_width = 0; - for f in fields_to_display: - fields[f]["max_width"] = len(f) - for vm_info in restore_info.values(): - if 'vm' in vm_info.keys(): - vm = vm_info['vm'] - l = len(str(eval(fields[f]["func"]))) - if l > fields[f]["max_width"]: - fields[f]["max_width"] = l - total_width += fields[f]["max_width"] - - print_callback("") - print_callback("The following VMs are included in the backup:") - print_callback("") - - # Display the header - s = "" - for f in fields_to_display: - fmt="{{0:-^{0}}}-+".format(fields[f]["max_width"] + 1) - s += fmt.format('-') - print_callback(s) - s = "" - for f in fields_to_display: - fmt="{{0:>{0}}} |".format(fields[f]["max_width"] + 1) - s += fmt.format(f) - print_callback(s) - s = "" - for f in fields_to_display: - fmt="{{0:-^{0}}}-+".format(fields[f]["max_width"] + 1) - s += fmt.format('-') - print_callback(s) - - for vm_info in restore_info.values(): - # Skip non-VM here - if not 'vm' in vm_info: - continue - vm = vm_info['vm'] - s = "" - for f in fields_to_display: - fmt="{{0:>{0}}} |".format(fields[f]["max_width"] + 1) - s += fmt.format(eval(fields[f]["func"])) - - if 'excluded' in vm_info and vm_info['excluded']: - s += " <-- Excluded from restore" - elif 'already-exists' in vm_info: - s += " <-- A VM with the same name already exists on the host!" - elif 'missing-template' in vm_info: - s += " <-- No matching template on the host or in the backup found!" - elif 'missing-netvm' in vm_info: - s += " <-- No matching netvm on the host or in the backup found!" - elif 'orig-template' in vm_info: - s += " <-- Original template was '%s'" % (vm_info['orig-template']) - - print_callback(s) - - if 'dom0' in restore_info.keys(): - s = "" - for f in fields_to_display: - fmt="{{0:>{0}}} |".format(fields[f]["max_width"] + 1) - if f == "name": - s += fmt.format("Dom0") - elif f == "type": - s += fmt.format("Home") - else: - s += fmt.format("") - if 'username-mismatch' in restore_info['dom0']: - s += " <-- username in backup and dom0 mismatch" - - print_callback(s) - -def backup_restore_do(backup_dir, restore_tmpdir, passphrase, restore_info, - host_collection = None, print_callback = print_stdout, - error_callback = print_stderr, progress_callback = None, - encrypted=False, appvm=None): - - lock_obtained = False - if host_collection is None: - host_collection = QubesVmCollection() - host_collection.lock_db_for_writing() - host_collection.load() - lock_obtained = True - - # Perform VM restoration in backup order - vms_dirs = [] - vms_size = 0 - vms = {} - for vm_info in restore_info.values(): - if not vm_info['good-to-go']: - continue - if 'vm' not in vm_info: - continue - vm = vm_info['vm'] - vms_size += vm.backup_size - vms_dirs.append(vm.backup_path) - vms[vm.name] = vm - - if 'dom0' in restore_info.keys() and restore_info['dom0']['good-to-go']: - vms_dirs.append('dom0-home') - vms_size += restore_info['dom0']['size'] - - restore_vm_dirs (backup_dir, - restore_tmpdir, - passphrase=passphrase, - vms_dirs=vms_dirs, - vms=vms, - vms_size=vms_size, - print_callback=print_callback, - error_callback=error_callback, - progress_callback=progress_callback, - encrypted=encrypted, - appvm=appvm) - - # Add VM in right order - for (vm_class_name, vm_class) in sorted(QubesVmClasses.items(), - key=lambda _x: _x[1].load_order): - for vm_info in restore_info.values(): - if not vm_info['good-to-go']: - continue - if 'vm' not in vm_info: - continue - vm = vm_info['vm'] - if not vm.__class__ == vm_class: - continue - print_callback("-> Restoring {type} {0}...".format(vm.name, type=vm_class_name)) - retcode = subprocess.call (["mkdir", "-p", os.path.dirname(vm.dir_path)]) - if retcode != 0: - error_callback("*** Cannot create directory: {0}?!".format(dest_dir)) - error_callback("Skipping...") - continue - - template = None - if vm.template is not None: - template_name = vm_info['template'] - template = host_collection.get_vm_by_name(template_name) - - new_vm = None - - try: - new_vm = host_collection.add_new_vm(vm_class_name, name=vm.name, - conf_file=vm.conf_file, - dir_path=vm.dir_path, - template=template, - installed_by_rpm=False) - - shutil.move(os.path.join(restore_tmpdir, vm.backup_path), - new_vm.dir_path) - - new_vm.verify_files() - except Exception as err: - error_callback("ERROR: {0}".format(err)) - error_callback("*** Skipping VM: {0}".format(vm.name)) - if new_vm: - host_collection.pop(new_vm.qid) - continue - - try: - new_vm.clone_attrs(vm) - except Exception as err: - error_callback("ERROR: {0}".format(err)) - error_callback("*** Some VM property will not be restored") - - try: - new_vm.appmenus_create(verbose=True) - except Exception as err: - error_callback("ERROR during appmenu restore: {0}".format(err)) - error_callback("*** VM '{0}' will not have appmenus".format(vm.name)) - - # Set network dependencies - only non-default netvm setting - for vm_info in restore_info.values(): - if not vm_info['good-to-go']: - continue - if 'vm' not in vm_info: - continue - vm = vm_info['vm'] - host_vm = host_collection.get_vm_by_name(vm.name) - if host_vm is None: - # Failed/skipped VM - continue - - if not vm.uses_default_netvm: - host_vm.netvm = host_collection.get_vm_by_name (vm_info['netvm']) if vm_info['netvm'] is not None else None - - host_collection.save() - if lock_obtained: - host_collection.unlock_db() - - # ... and dom0 home as last step - if 'dom0' in restore_info.keys() and restore_info['dom0']['good-to-go']: - backup_path = restore_info['dom0']['subdir'] - local_user = grp.getgrnam('qubes').gr_mem[0] - home_dir = pwd.getpwnam(local_user).pw_dir - backup_dom0_home_dir = os.path.join(restore_tmpdir, backup_path) - restore_home_backupdir = "home-pre-restore-{0}".format (time.strftime("%Y-%m-%d-%H%M%S")) - - print_callback("-> Restoring home of user '{0}'...".format(local_user)) - print_callback("--> Existing files/dirs backed up in '{0}' dir".format(restore_home_backupdir)) - os.mkdir(home_dir + '/' + restore_home_backupdir) - for f in os.listdir(backup_dom0_home_dir): - home_file = home_dir + '/' + f - if os.path.exists(home_file): - os.rename(home_file, home_dir + '/' + restore_home_backupdir + '/' + f) - shutil.move(backup_dom0_home_dir + '/' + f, home_file) - retcode = subprocess.call(['sudo', 'chown', '-R', local_user, home_dir]) - if retcode != 0: - error_callback("*** Error while setting home directory owner") - # vim:sw=4:et: diff --git a/qvm-tools/qvm-backup b/qvm-tools/qvm-backup index 601bd009..108875eb 100755 --- a/qvm-tools/qvm-backup +++ b/qvm-tools/qvm-backup @@ -22,7 +22,8 @@ from qubes.qubes import QubesVmCollection from qubes.qubes import QubesException -from qubes.qubesutils import backup_prepare, backup_do_copy, size_to_human +from qubes.backup import backup_prepare, backup_do +from qubes.qubesutils import size_to_human from optparse import OptionParser import os import sys @@ -80,6 +81,8 @@ def main(): files_to_backup, 0) if not options.appvm: + appvm = None + stat = os.statvfs(base_backup_dir) backup_fs_free_sz = stat.f_bsize * stat.f_bavail print @@ -88,6 +91,11 @@ def main(): exit(1) print "-> Available space: {0}".format(size_to_human(backup_fs_free_sz)) + else: + appvm = qvm_collection.get_vm_by_name(options.appvm) + if appvm is None: + print >>sys.stderr, "ERROR: VM {0} does not exist".format(options.appvm) + exit(1) prompt = raw_input ("Do you want to proceed? [y/N] ") if not (prompt == "y" or prompt == "Y"): @@ -96,10 +104,10 @@ def main(): passphrase = getpass.getpass("Please enter the pass phrase that will be used to encrypt/verify the backup: ") try: - backup_do_copy(base_backup_dir, files_to_backup, passphrase, + backup_do(base_backup_dir, files_to_backup, passphrase, progress_callback=print_progress, encrypt=options.encrypt, - appvm=options.appvm) + appvm=appvm) except QubesException as e: print >>sys.stderr, "ERROR: %s" % str(e) exit(1) diff --git a/qvm-tools/qvm-backup-restore b/qvm-tools/qvm-backup-restore index 6ac22088..506646f6 100755 --- a/qvm-tools/qvm-backup-restore +++ b/qvm-tools/qvm-backup-restore @@ -22,10 +22,10 @@ from qubes.qubes import QubesVmCollection from qubes.qubes import QubesException -from qubes.qubesutils import backup_restore_header -from qubes.qubesutils import backup_restore_prepare -from qubes.qubesutils import backup_restore_print_summary -from qubes.qubesutils import backup_restore_do +from qubes.backup import backup_restore_header +from qubes.backup import backup_restore_prepare +from qubes.backup import backup_restore_print_summary +from qubes.backup import backup_restore_do from optparse import OptionParser import os @@ -95,15 +95,28 @@ def main(): if options.exclude: restore_options['exclude'] = options.exclude + appvm = None + if options.appvm is not None: + appvm = qvm_collection.get_vm_by_name(options.appvm) + if appvm is None: + print >>sys.stderr, "ERROR: VM {0} does not exist".format(options.appvm) + exit(1) passphrase = getpass.getpass("Please enter the pass phrase that will be used to decrypt/verify the backup: ") print >> sys.stderr, "Checking backup content..." - restore_tmpdir,qubes_xml = backup_restore_header(backup_dir, passphrase, options.decrypt, appvm=options.appvm) + restore_tmpdir,qubes_xml = backup_restore_header(backup_dir, passphrase, encrypted=options.decrypt, appvm=options.appvm) restore_info = None try: - restore_info = backup_restore_prepare(backup_dir,os.path.join(restore_tmpdir, qubes_xml), passphrase, options=restore_options, host_collection=host_collection, encrypt=options.decrypt, appvm=options.appvm) + restore_info = backup_restore_prepare( + backup_dir, + os.path.join(restore_tmpdir, qubes_xml), + passphrase, + options=restore_options, + host_collection=host_collection, + encrypt=options.decrypt, + appvm=appvm) except QubesException as e: print >> sys.stderr, "ERROR: %s" % str(e) exit(1) @@ -147,7 +160,7 @@ def main(): print "The above VMs will be copied and added to your system." print "Exisiting VMs will not be removed." - + if there_are_missing_templates: print >> sys.stderr, "*** One or more template VM is missing on the host! ***" if not (options.skip_broken or options.ignore_missing): @@ -192,7 +205,13 @@ def main(): exit (0) - backup_restore_do(backup_dir,restore_tmpdir, passphrase, restore_info, host_collection=host_collection, encrypted=options.decrypt, appvm=options.appvm) + backup_restore_do(backup_dir, + restore_tmpdir, + passphrase, + restore_info, + host_collection=host_collection, + encrypted=options.decrypt, + appvm=appvm) host_collection.unlock_db() diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec index a31aaac1..b5d56b84 100644 --- a/rpm_spec/core-dom0.spec +++ b/rpm_spec/core-dom0.spec @@ -109,6 +109,8 @@ cp core/qubesutils.py $RPM_BUILD_ROOT%{python_sitearch}/qubes cp core/qubesutils.py[co] $RPM_BUILD_ROOT%{python_sitearch}/qubes cp core/guihelpers.py $RPM_BUILD_ROOT%{python_sitearch}/qubes cp core/guihelpers.py[co] $RPM_BUILD_ROOT%{python_sitearch}/qubes +cp core/backup.py $RPM_BUILD_ROOT%{python_sitearch}/qubes +cp core/backup.py[co] $RPM_BUILD_ROOT%{python_sitearch}/qubes cp core/__init__.py $RPM_BUILD_ROOT%{python_sitearch}/qubes cp core/__init__.py[co] $RPM_BUILD_ROOT%{python_sitearch}/qubes cp qmemman/qmemman*py $RPM_BUILD_ROOT%{python_sitearch}/qubes @@ -272,6 +274,9 @@ fi %{python_sitearch}/qubes/guihelpers.py %{python_sitearch}/qubes/guihelpers.pyc %{python_sitearch}/qubes/guihelpers.pyo +%{python_sitearch}/qubes/backup.py +%{python_sitearch}/qubes/backup.pyc +%{python_sitearch}/qubes/backup.pyo %{python_sitearch}/qubes/__init__.py %{python_sitearch}/qubes/__init__.pyc %{python_sitearch}/qubes/__init__.pyo From 3a898db663103dac6ce30ef9143310f6daa8b6bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 25 Nov 2013 05:42:47 +0100 Subject: [PATCH 67/82] backups: Prompt for password twice for verification --- qvm-tools/qvm-backup | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/qvm-tools/qvm-backup b/qvm-tools/qvm-backup index 108875eb..334f70fb 100755 --- a/qvm-tools/qvm-backup +++ b/qvm-tools/qvm-backup @@ -102,6 +102,10 @@ def main(): exit (0) passphrase = getpass.getpass("Please enter the pass phrase that will be used to encrypt/verify the backup: ") + passphrase2 = getpass.getpass("Enter again for verification: ") + if passphrase != passphrase2: + print >>sys.stderr, "ERROR: Password mismatch" + exit(1) try: backup_do(base_backup_dir, files_to_backup, passphrase, From d7e3f3cb0aa7cf6cfd623197acc420238de20e2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 25 Nov 2013 05:43:15 +0100 Subject: [PATCH 68/82] backups: qvm-backup: check if /var/tmp have enough space --- qvm-tools/qvm-backup | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/qvm-tools/qvm-backup b/qvm-tools/qvm-backup index 334f70fb..ae264a69 100755 --- a/qvm-tools/qvm-backup +++ b/qvm-tools/qvm-backup @@ -97,6 +97,14 @@ def main(): print >>sys.stderr, "ERROR: VM {0} does not exist".format(options.appvm) exit(1) + stat = os.statvfs('/var/tmp') + backup_fs_free_sz = stat.f_bsize * stat.f_bavail + print + if (backup_fs_free_sz < 1000000000): + print >>sys.stderr, "ERROR: Not enough space available " \ + "on the local filesystem (needs 1GB for temporary files)!" + exit(1) + prompt = raw_input ("Do you want to proceed? [y/N] ") if not (prompt == "y" or prompt == "Y"): exit (0) From cc379270808871bcf7835fa8151b7ac088863fde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 25 Nov 2013 05:46:57 +0100 Subject: [PATCH 69/82] backups: fix backup cleanup --- core/backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/backup.py b/core/backup.py index 6ea883e7..afd937c4 100644 --- a/core/backup.py +++ b/core/backup.py @@ -494,7 +494,7 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,\ print "Sparse1 proc return code:", tar_sparse.poll() vmproc.stdin.close() - shutil.rmtree(restore_tmpdir) + shutil.rmtree(backup_tmpdir) ''' ' Wait for backup chunk to finish From bf6bf8ed8ff16c66fcd69cc7904a08df259b46e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 25 Nov 2013 06:31:38 +0100 Subject: [PATCH 70/82] backups: fix backup header extraction Pass only 'qubes.xml.000' to tar2qfile - this way it will stop reading the source after requested file(s). --- core/backup.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/core/backup.py b/core/backup.py index afd937c4..3bcf46a5 100644 --- a/core/backup.py +++ b/core/backup.py @@ -769,13 +769,11 @@ def restore_vm_dirs (backup_source, restore_tmpdir, passphrase, vms_dirs, vms, '-ix%sf' % ("v" if BACKUP_DEBUG else ""), backup_source, '-C', restore_tmpdir] + vms_dirs - # Remove already processed qubes.xml.000, because qfile-dom0-unpacker will - # refuse to override files tar1_env = os.environ.copy() # TODO: add some safety margin? tar1_env['UPDATES_MAX_BYTES'] = str(vms_size) # Restoring only header - if len(vms_dirs) == 2 and vms_dirs[0] == 'qubes.xml.000': + if vms_dirs and vms_dirs[0] == 'qubes.xml.000': tar1_env['UPDATES_MAX_FILES'] = '2' else: tar1_env['UPDATES_MAX_FILES'] = '0' @@ -886,10 +884,16 @@ def backup_restore_header(source, passphrase, if BACKUP_DEBUG: print "Working in", restore_tmpdir + # tar2qfile matches only beginnings, while tar full path + if appvm: + extract_filter = ['qubes.xml.000'] + else: + extract_filter = ['qubes.xml.000', 'qubes.xml.000.hmac'] + restore_vm_dirs (source, restore_tmpdir, passphrase=passphrase, - vms_dirs=['qubes.xml.000', 'qubes.xml.000.hmac'], + vms_dirs=extract_filter, vms=None, vms_size=40000, print_callback=print_callback, From 0743531244a86ce3644ee6fc008218c58a8cbdc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 25 Nov 2013 06:33:06 +0100 Subject: [PATCH 71/82] backups: fix VM exclude logic (restore) --- core/backup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/backup.py b/core/backup.py index 3bcf46a5..1403031c 100644 --- a/core/backup.py +++ b/core/backup.py @@ -959,7 +959,8 @@ def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, vms_to_restore[vm.name]['vm'] = vm; if 'exclude' in options.keys(): vms_to_restore[vm.name]['excluded'] = vm.name in options['exclude'] - vms_to_restore[vm.name]['good-to-go'] = False + if vms_to_restore[vm.name]['excluded']: + vms_to_restore[vm.name]['good-to-go'] = False if host_collection.get_vm_by_name (vm.name) is not None: vms_to_restore[vm.name]['already-exists'] = True From 0fcceb324dfd9c077f54be0c2b17a4e090915d96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 25 Nov 2013 06:33:31 +0100 Subject: [PATCH 72/82] backups: fix appvm handling in qvm-backup-restore --- qvm-tools/qvm-backup-restore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qvm-tools/qvm-backup-restore b/qvm-tools/qvm-backup-restore index 506646f6..31eb5313 100755 --- a/qvm-tools/qvm-backup-restore +++ b/qvm-tools/qvm-backup-restore @@ -97,7 +97,7 @@ def main(): appvm = None if options.appvm is not None: - appvm = qvm_collection.get_vm_by_name(options.appvm) + appvm = host_collection.get_vm_by_name(options.appvm) if appvm is None: print >>sys.stderr, "ERROR: VM {0} does not exist".format(options.appvm) exit(1) @@ -105,7 +105,7 @@ def main(): passphrase = getpass.getpass("Please enter the pass phrase that will be used to decrypt/verify the backup: ") print >> sys.stderr, "Checking backup content..." - restore_tmpdir,qubes_xml = backup_restore_header(backup_dir, passphrase, encrypted=options.decrypt, appvm=options.appvm) + restore_tmpdir,qubes_xml = backup_restore_header(backup_dir, passphrase, encrypted=options.decrypt, appvm=appvm) restore_info = None try: From 0b0d50edf35c5d93b6f225e54d772fc3dd79d8eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Tue, 26 Nov 2013 16:45:51 +0100 Subject: [PATCH 73/82] backups: move import at the beginning of .py file --- core/backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/backup.py b/core/backup.py index 1403031c..1d472c80 100644 --- a/core/backup.py +++ b/core/backup.py @@ -31,6 +31,7 @@ import os import subprocess import re import shutil +import tempfile import time import grp,pwd from multiprocessing import Queue,Process @@ -332,7 +333,6 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,\ progress = blocks_backedup * 11 / total_backup_sz progress_callback(progress) - import tempfile feedback_file = tempfile.NamedTemporaryFile() backup_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/backup_") From 50662bf090860b826eeaef53bf4c2d9da9f3a0c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Tue, 26 Nov 2013 16:46:09 +0100 Subject: [PATCH 74/82] backups: correctly calculate size of file to backup --- core/backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/backup.py b/core/backup.py index 1d472c80..fbf2c7ff 100644 --- a/core/backup.py +++ b/core/backup.py @@ -52,7 +52,7 @@ def get_disk_usage(file_or_dir): def file_to_backup (file_path, sz = None): if sz is None: - sz = os.path.getsize (system_path["qubes_store_filename"]) + sz = get_disk_usage (file_path) abs_file_path = os.path.abspath (file_path) abs_base_dir = os.path.abspath (system_path["qubes_base_dir"]) + '/' From 2d68b79bff6a2421cd393ffc97e2e5131995cd89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Tue, 26 Nov 2013 16:46:34 +0100 Subject: [PATCH 75/82] backups: fix backup of templates Template is saved as single archive of the whole VM directory. Preserve backup directory structure regardless of its content - in this case it means we need "." archive (with template directory content) placed in "vm-tempates//" backup directory. This allows restore process to select right files to restore regardless of VM type. --- core/backup.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/core/backup.py b/core/backup.py index fbf2c7ff..456d84d4 100644 --- a/core/backup.py +++ b/core/backup.py @@ -170,7 +170,14 @@ def backup_prepare(vms_list = None, exclude_list = [], print_callback = print_st # already handled continue vm_sz = vm.get_disk_utilization() - files_to_backup += file_to_backup (vm.dir_path, vm_sz) + template_subdir = os.path.relpath( + vm.dir_path, + system_path["qubes_base_dir"]) + '/' + template_to_backup = [ { + "path": vm.dir_path + '/.', + "size": vm_sz, + "subdir": template_subdir } ] + files_to_backup += template_to_backup s = "" fmt="{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1) @@ -258,6 +265,10 @@ def backup_prepare(vms_list = None, exclude_list = [], print_callback = print_st if (there_are_running_vms): raise QubesException("Please shutdown all VMs before proceeding.") + for fileinfo in files_to_backup: + assert len(fileinfo["subdir"]) == 0 or fileinfo["subdir"][-1] == '/', \ + "'subdir' must ends with a '/': %s" % str(fileinfo) + return files_to_backup class Send_Worker(Process): @@ -383,7 +394,7 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,\ "-f", backup_pipe, '--tape-length', str(100000), '-C', os.path.dirname(filename["path"]), - '--xform', 's:^[a-z]:%s\\0:' % filename["subdir"], + '--xform', 's:^[^/]:%s\\0:' % filename["subdir"], os.path.basename(filename["path"]) ] From 8bdea5b0ab80aa4ace005191575a378f1e1b5f8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 27 Nov 2013 03:18:14 +0100 Subject: [PATCH 76/82] backups: fix backup of selected appmenus for VM This wasn't working for a long time... --- core/backup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/backup.py b/core/backup.py index 456d84d4..823e9d42 100644 --- a/core/backup.py +++ b/core/backup.py @@ -138,8 +138,9 @@ def backup_prepare(vms_list = None, exclude_list = [], print_callback = print_st if os.path.exists (vm.firewall_conf): files_to_backup += file_to_backup(vm.firewall_conf) if 'appmenus_whitelist' in vm_files and \ - os.path.exists(vm.dir_path + vm_files['appmenus_whitelist']): - files_to_backup += file_to_backup(vm.dir_path + vm_files['appmenus_whitelist']) + os.path.exists(os.path.join(vm.dir_path, vm_files['appmenus_whitelist'])): + files_to_backup += file_to_backup( + os.path.join(vm.dir_path, vm_files['appmenus_whitelist'])) if vm.updateable: sz = vm.get_disk_usage(vm.root_img) From 10100767da16cfdaf9397ee6559482c925399e66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 27 Nov 2013 03:19:23 +0100 Subject: [PATCH 77/82] backups: hide VM names in encrypted backup Even when encrypted backup is selected, file list isn't encrypted. Do not leak VM names in the filenames. --- core/backup.py | 65 ++++++++++++++++++++++++++------------------ qvm-tools/qvm-backup | 5 +++- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/core/backup.py b/core/backup.py index 823e9d42..b3a74f8f 100644 --- a/core/backup.py +++ b/core/backup.py @@ -50,19 +50,23 @@ def get_disk_usage(file_or_dir): return sz -def file_to_backup (file_path, sz = None): - if sz is None: - sz = get_disk_usage (file_path) +def file_to_backup (file_path, subdir = None): + sz = get_disk_usage (file_path) - abs_file_path = os.path.abspath (file_path) - abs_base_dir = os.path.abspath (system_path["qubes_base_dir"]) + '/' - abs_file_dir = os.path.dirname (abs_file_path) + '/' - (nothing, dir, subdir) = abs_file_dir.partition (abs_base_dir) - assert nothing == "" - assert dir == abs_base_dir + if subdir is None: + abs_file_path = os.path.abspath (file_path) + abs_base_dir = os.path.abspath (system_path["qubes_base_dir"]) + '/' + abs_file_dir = os.path.dirname (abs_file_path) + '/' + (nothing, dir, subdir) = abs_file_dir.partition (abs_base_dir) + assert nothing == "" + assert dir == abs_base_dir + else: + if len(subdir) > 0 and not subdir.endswith('/'): + subdir += '/' return [ { "path" : file_path, "size": sz, "subdir": subdir} ] -def backup_prepare(vms_list = None, exclude_list = [], print_callback = print_stdout): +def backup_prepare(vms_list = None, exclude_list = [], + print_callback = print_stdout, hide_vm_names=True): """If vms = None, include all (sensible) VMs; exclude_list is always applied""" files_to_backup = file_to_backup (system_path["qubes_store_filename"]) @@ -119,33 +123,36 @@ def backup_prepare(vms_list = None, exclude_list = [], print_callback = print_st # handle templates later continue + if hide_vm_names: + subdir = 'vm%d' % vm.qid + else: + subdir = None + if vm.private_img is not None: - vm_sz = vm.get_disk_usage (vm.private_img) - files_to_backup += file_to_backup(vm.private_img, vm_sz ) + files_to_backup += file_to_backup(vm.private_img, subdir) if vm.is_appvm(): - files_to_backup += file_to_backup(vm.icon_path) + files_to_backup += file_to_backup(vm.icon_path, subdir) if vm.updateable: if os.path.exists(vm.dir_path + "/apps.templates"): # template - files_to_backup += file_to_backup(vm.dir_path + "/apps.templates") + files_to_backup += file_to_backup(vm.dir_path + "/apps.templates", subdir) else: # standaloneVM - files_to_backup += file_to_backup(vm.dir_path + "/apps") + files_to_backup += file_to_backup(vm.dir_path + "/apps", subdir) if os.path.exists(vm.dir_path + "/kernels"): - files_to_backup += file_to_backup(vm.dir_path + "/kernels") + files_to_backup += file_to_backup(vm.dir_path + "/kernels", subdir) if os.path.exists (vm.firewall_conf): - files_to_backup += file_to_backup(vm.firewall_conf) + files_to_backup += file_to_backup(vm.firewall_conf, subdir) if 'appmenus_whitelist' in vm_files and \ os.path.exists(os.path.join(vm.dir_path, vm_files['appmenus_whitelist'])): files_to_backup += file_to_backup( - os.path.join(vm.dir_path, vm_files['appmenus_whitelist'])) + os.path.join(vm.dir_path, vm_files['appmenus_whitelist']), + subdir) if vm.updateable: - sz = vm.get_disk_usage(vm.root_img) - files_to_backup += file_to_backup(vm.root_img, sz) - vm_sz += sz + files_to_backup += file_to_backup(vm.root_img, subdir) s = "" fmt="{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1) @@ -158,7 +165,7 @@ def backup_prepare(vms_list = None, exclude_list = [], print_callback = print_st s += fmt.format("AppVM" + (" + Sys" if vm.updateable else "")) fmt="{{0:>{0}}} |".format(fields_to_display[2]["width"] + 1) - s += fmt.format(size_to_human(vm_sz)) + s += fmt.format(size_to_human(vm.get_disk_utilization())) if vm.is_running(): s += " <-- The VM is running, please shut it down before proceeding with the backup!" @@ -171,9 +178,12 @@ def backup_prepare(vms_list = None, exclude_list = [], print_callback = print_st # already handled continue vm_sz = vm.get_disk_utilization() - template_subdir = os.path.relpath( - vm.dir_path, - system_path["qubes_base_dir"]) + '/' + if hide_vm_names: + template_subdir = 'vm%d' % vm.qid + else: + template_subdir = os.path.relpath( + vm.dir_path, + system_path["qubes_base_dir"]) + '/' template_to_backup = [ { "path": vm.dir_path + '/.', "size": vm_sz, @@ -204,7 +214,10 @@ def backup_prepare(vms_list = None, exclude_list = [], print_callback = print_st if vm.qid in vms_for_backup_qid: vm.backup_content = True vm.backup_size = vm.get_disk_utilization() - vm.backup_path = os.path.relpath(vm.dir_path, system_path["qubes_base_dir"]) + if hide_vm_names: + vm.backup_path = 'vm%d' % vm.qid + else: + vm.backup_path = os.path.relpath(vm.dir_path, system_path["qubes_base_dir"]) # Dom0 user home if not 'dom0' in exclude_list: diff --git a/qvm-tools/qvm-backup b/qvm-tools/qvm-backup index ae264a69..dcddee48 100755 --- a/qvm-tools/qvm-backup +++ b/qvm-tools/qvm-backup @@ -72,7 +72,10 @@ def main(): files_to_backup = None try: - files_to_backup = backup_prepare(vms_list=vms, exclude_list=options.exclude_list) + files_to_backup = backup_prepare( + vms_list=vms, + exclude_list=options.exclude_list, + hide_vm_names=options.encrypt) except QubesException as e: print >>sys.stderr, "ERROR: %s" % str(e) exit(1) From 105428accbea59e692322633c6a75260761debf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 27 Nov 2013 03:20:26 +0100 Subject: [PATCH 78/82] backups: fix encrypted backup restore --- core/backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/backup.py b/core/backup.py index b3a74f8f..09be708d 100644 --- a/core/backup.py +++ b/core/backup.py @@ -714,7 +714,7 @@ class Extract_Worker(Process): # Start decrypt encryptor = subprocess.Popen (["openssl", "enc", "-d", "-aes-256-cbc", - "-pass", "pass:"+passphrase], + "-pass", "pass:"+self.passphrase], stdin=open(filename,'rb'), stdout=subprocess.PIPE) From c8b8cd0d1f9fdb82e8b4d10a786187b91836e359 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 27 Nov 2013 03:20:45 +0100 Subject: [PATCH 79/82] backups: fix restore in non-debug mode --- core/backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/backup.py b/core/backup.py index 09be708d..5bdce1d1 100644 --- a/core/backup.py +++ b/core/backup.py @@ -791,7 +791,7 @@ def restore_vm_dirs (backup_source, restore_tmpdir, passphrase, vms_dirs, vms, backup_stdin = open(backup_source,'rb') tar1_command = ['tar', - '-ix%sf' % ("v" if BACKUP_DEBUG else ""), backup_source, + '-ixvf', backup_source, '-C', restore_tmpdir] + vms_dirs tar1_env = os.environ.copy() From eaebf04b3424b4124a31ebbbe061695b8728ea4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 27 Nov 2013 03:21:17 +0100 Subject: [PATCH 80/82] backups: remove some old unneeded code --- core/backup.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/core/backup.py b/core/backup.py index 5bdce1d1..d141e06e 100644 --- a/core/backup.py +++ b/core/backup.py @@ -853,8 +853,6 @@ def restore_vm_dirs (backup_source, restore_tmpdir, passphrase, vms_dirs, vms, "ERROR: unable to read the qubes backup {0} " \ "because of a VM error: {1}".format( backup_source, vmproc.stderr.read())) - if BACKUP_DEBUG: - print "Extraction process status:", extract_proc.exitcode to_extract.put("FINISHED") if BACKUP_DEBUG: @@ -895,12 +893,8 @@ def backup_restore_header(source, passphrase, print_callback = print_stdout, error_callback = print_stderr, encrypted=False, appvm=None): - # Simulate dd if=backup_file count=10 | file - - # Simulate dd if=backup_file count=10 | gpg2 -d | tar xzv -O - # analysis = subprocess.Popen() vmproc = None - import tempfile feedback_file = tempfile.NamedTemporaryFile() restore_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/restore_") From 3d7040277807712d3c3932485537d80ae9555e9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 29 Nov 2013 03:25:41 +0100 Subject: [PATCH 81/82] backups: add support for restoring old backup format Actually the code is quite similar, so just add few "if" instead of copying the whole functions. --- core/backup.py | 154 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 110 insertions(+), 44 deletions(-) diff --git a/core/backup.py b/core/backup.py index d141e06e..2fedbaa6 100644 --- a/core/backup.py +++ b/core/backup.py @@ -770,7 +770,7 @@ def restore_vm_dirs (backup_source, restore_tmpdir, passphrase, vms_dirs, vms, if BACKUP_DEBUG: print_callback("Working in temporary dir:"+restore_tmpdir) - print_callback(str(vms_size)+" bytes to restore") + print_callback("Extracting data: " + size_to_human(vms_size)+" to restore") vmproc = None if appvm != None: @@ -889,15 +889,27 @@ def load_hmac(hmac): return hmac +def backup_detect_format_version(backup_location): + if os.path.exists(os.path.join(backup_location, 'qubes.xml')): + return 1 + else: + return 2 + def backup_restore_header(source, passphrase, print_callback = print_stdout, error_callback = print_stderr, - encrypted=False, appvm=None): + encrypted=False, appvm=None, format_version = None): vmproc = None feedback_file = tempfile.NamedTemporaryFile() restore_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/restore_") + if format_version == None: + format_version = backup_detect_format_version(source) + + if format_version == 1: + return (restore_tmpdir, os.path.join(source, 'qubes.xml')) + os.chdir(restore_tmpdir) if BACKUP_DEBUG: @@ -923,13 +935,23 @@ def backup_restore_header(source, passphrase, return (restore_tmpdir, "qubes.xml") -def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, - host_collection = None, encrypt=False, appvm=None): +def backup_restore_prepare(backup_location, qubes_xml, passphrase, options = {}, + host_collection = None, encrypt=False, appvm=None, format_version=None): # Defaults backup_restore_set_defaults(options) #### Private functions begin - def is_vm_included_in_backup (backup_dir, vm): + def is_vm_included_in_backup_v1 (backup_dir, vm): + if vm.qid == 0: + return os.path.exists(os.path.join(backup_dir,'dom0-home')) + + backup_vm_dir_path = vm.dir_path.replace (system_path["qubes_base_dir"], backup_dir) + + if os.path.exists (backup_vm_dir_path): + return True + else: + return False + def is_vm_included_in_backup_v2 (backup_dir, vm): if vm.backup_content: return True else: @@ -944,6 +966,21 @@ def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, return template #### Private functions end + + # Format versions: + # 1 - Qubes R1, Qubes R2 beta1, beta2 + # 2 - Qubes R2 beta3 + + if format_version is None: + format_version = backup_detect_format_version(backup_location) + + if format_version == 1: + is_vm_included_in_backup = is_vm_included_in_backup_v1 + elif format_version == 2: + is_vm_included_in_backup = is_vm_included_in_backup_v2 + else: + raise QubesException("Unknown backup format version: %s" % str(format_version)) + if BACKUP_DEBUG: print "Loading file", qubes_xml backup_collection = QubesVmCollection(store_filename = qubes_xml) @@ -970,7 +1007,7 @@ def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, if vm.qid == 0: # Handle dom0 as special case later continue - if is_vm_included_in_backup (backup_dir, vm): + if is_vm_included_in_backup (backup_location, vm): if BACKUP_DEBUG: print vm.name,"is included in backup" @@ -997,7 +1034,7 @@ def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, # Maybe the (custom) template is in the backup? template_vm_on_backup = backup_collection.get_vm_by_name (templatevm_name) if template_vm_on_backup is None or not \ - (is_vm_included_in_backup(backup_dir, template_vm_on_backup) and \ + (is_vm_included_in_backup(backup_location, template_vm_on_backup) and \ template_vm_on_backup.is_template()): if options['use-default-template']: vms_to_restore[vm.name]['orig-template'] = templatevm_name @@ -1026,7 +1063,7 @@ def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, netvm_on_backup = backup_collection.get_vm_by_name (netvm_name) if not ((netvm_on_backup is not None) and \ netvm_on_backup.is_netvm() and \ - is_vm_included_in_backup(backup_dir, netvm_on_backup)): + is_vm_included_in_backup(backup_location, netvm_on_backup)): if options['use-default-netvm']: vms_to_restore[vm.name]['netvm'] = host_collection.get_default_netvm().name vm.uses_default_netvm = True @@ -1041,14 +1078,19 @@ def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, # ...and dom0 home if options['dom0-home'] and \ - is_vm_included_in_backup(backup_dir, backup_collection[0]): + is_vm_included_in_backup(backup_location, backup_collection[0]): vm = backup_collection[0] vms_to_restore['dom0'] = {} - vms_to_restore['dom0']['subdir'] = vm.backup_path - vms_to_restore['dom0']['size'] = vm.backup_size + if format_version == 1: + vms_to_restore['dom0']['subdir'] = \ + os.listdir(os.path.join(backup_location, 'dom0-home'))[0] + vms_to_restore['dom0']['size'] = 0 # unknown + else: + vms_to_restore['dom0']['subdir'] = vm.backup_path + vms_to_restore['dom0']['size'] = vm.backup_size local_user = grp.getgrnam('qubes').gr_mem[0] - dom0_home = vm.backup_path + dom0_home = vms_to_restore['dom0']['subdir'] vms_to_restore['dom0']['username'] = os.path.basename(dom0_home) if vms_to_restore['dom0']['username'] != local_user: @@ -1060,7 +1102,8 @@ def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, vms_to_restore['dom0']['good-to-go'] = True # Not needed - all the data stored in vms_to_restore - os.unlink(qubes_xml) + if format_version == 2: + os.unlink(qubes_xml) return vms_to_restore def backup_restore_print_summary(restore_info, print_callback = print_stdout): @@ -1162,10 +1205,24 @@ def backup_restore_print_summary(restore_info, print_callback = print_stdout): print_callback(s) -def backup_restore_do(backup_dir, restore_tmpdir, passphrase, restore_info, +def backup_restore_do(backup_location, restore_tmpdir, passphrase, restore_info, host_collection = None, print_callback = print_stdout, error_callback = print_stderr, progress_callback = None, - encrypted=False, appvm=None): + encrypted=False, appvm=None, format_version = None): + + ### Private functions begin + def restore_vm_dir_v1 (backup_dir, src_dir, dst_dir): + + backup_src_dir = src_dir.replace (system_path["qubes_base_dir"], backup_dir) + + # We prefer to use Linux's cp, because it nicely handles sparse files + retcode = subprocess.call (["cp", "-rp", backup_src_dir, dst_dir]) + if retcode != 0: + raise QubesException("*** Error while copying file {0} to {1}".format(backup_src_dir, dest_dir)) + ### Private functions end + + if format_version is None: + format_version = backup_detect_format_version(backup_location) lock_obtained = False if host_collection is None: @@ -1175,34 +1232,35 @@ def backup_restore_do(backup_dir, restore_tmpdir, passphrase, restore_info, lock_obtained = True # Perform VM restoration in backup order - vms_dirs = [] - vms_size = 0 - vms = {} - for vm_info in restore_info.values(): - if not vm_info['good-to-go']: - continue - if 'vm' not in vm_info: - continue - vm = vm_info['vm'] - vms_size += vm.backup_size - vms_dirs.append(vm.backup_path) - vms[vm.name] = vm + if format_version == 2: + vms_dirs = [] + vms_size = 0 + vms = {} + for vm_info in restore_info.values(): + if not vm_info['good-to-go']: + continue + if 'vm' not in vm_info: + continue + vm = vm_info['vm'] + vms_size += vm.backup_size + vms_dirs.append(vm.backup_path) + vms[vm.name] = vm - if 'dom0' in restore_info.keys() and restore_info['dom0']['good-to-go']: - vms_dirs.append('dom0-home') - vms_size += restore_info['dom0']['size'] + if 'dom0' in restore_info.keys() and restore_info['dom0']['good-to-go']: + vms_dirs.append('dom0-home') + vms_size += restore_info['dom0']['size'] - restore_vm_dirs (backup_dir, - restore_tmpdir, - passphrase=passphrase, - vms_dirs=vms_dirs, - vms=vms, - vms_size=vms_size, - print_callback=print_callback, - error_callback=error_callback, - progress_callback=progress_callback, - encrypted=encrypted, - appvm=appvm) + restore_vm_dirs (backup_location, + restore_tmpdir, + passphrase=passphrase, + vms_dirs=vms_dirs, + vms=vms, + vms_size=vms_size, + print_callback=print_callback, + error_callback=error_callback, + progress_callback=progress_callback, + encrypted=encrypted, + appvm=appvm) # Add VM in right order for (vm_class_name, vm_class) in sorted(QubesVmClasses.items(), @@ -1236,8 +1294,13 @@ def backup_restore_do(backup_dir, restore_tmpdir, passphrase, restore_info, template=template, installed_by_rpm=False) - shutil.move(os.path.join(restore_tmpdir, vm.backup_path), - new_vm.dir_path) + if format_version == 1: + restore_vm_dir_v1(backup_location, + vm.dir_path, + os.path.dirname(new_vm.dir_path)) + elif format_version == 2: + shutil.move(os.path.join(restore_tmpdir, vm.backup_path), + new_vm.dir_path) new_vm.verify_files() except Exception as err: @@ -1293,7 +1356,10 @@ def backup_restore_do(backup_dir, restore_tmpdir, passphrase, restore_info, home_file = home_dir + '/' + f if os.path.exists(home_file): os.rename(home_file, home_dir + '/' + restore_home_backupdir + '/' + f) - shutil.move(backup_dom0_home_dir + '/' + f, home_file) + if format_version == 1: + retcode = subprocess.call (["cp", "-nrp", backup_dom0_home_dir + '/' + f, home_file]) + elif format_version == 2: + shutil.move(backup_dom0_home_dir + '/' + f, home_file) retcode = subprocess.call(['sudo', 'chown', '-R', local_user, home_dir]) if retcode != 0: error_callback("*** Error while setting home directory owner") From b73970c62d053ace4923d51ca066d322e3ea38d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 29 Nov 2013 03:42:56 +0100 Subject: [PATCH 82/82] core: rename QubesDom0NetVm to QubesAdminVm This is somehow related to #757, but only first (easier) step. Actual change of QubesAdminVm base class requires somehow more changes, for example qvm-ls needs to know how to display this type of VM (none of template, appvm, netvm). Make this first step change now, because starting with R2Beta3 dom0 will be stored in qubes.xml (for new backups purposes) so this rename would be complicated later. --- core-modules/{006QubesDom0NetVm.py => 006QubesAdminVm.py} | 8 ++++---- core/qubes.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) rename core-modules/{006QubesDom0NetVm.py => 006QubesAdminVm.py} (93%) diff --git a/core-modules/006QubesDom0NetVm.py b/core-modules/006QubesAdminVm.py similarity index 93% rename from core-modules/006QubesDom0NetVm.py rename to core-modules/006QubesAdminVm.py index 43251e48..24274f91 100644 --- a/core-modules/006QubesDom0NetVm.py +++ b/core-modules/006QubesAdminVm.py @@ -25,13 +25,13 @@ from qubes.qubes import QubesNetVm,register_qubes_vm_class,xl_ctx,xc from qubes.qubes import defaults from qubes.qubes import QubesException,dry_run -class QubesDom0NetVm(QubesNetVm): +class QubesAdminVm(QubesNetVm): # In which order load this VM type from qubes.xml load_order = 10 def __init__(self, **kwargs): - super(QubesDom0NetVm, self).__init__(qid=0, name="dom0", netid=0, + super(QubesAdminVm, self).__init__(qid=0, name="dom0", netid=0, dir_path=None, private_img = None, template = None, @@ -41,7 +41,7 @@ class QubesDom0NetVm(QubesNetVm): @property def type(self): - return "Dom0NetVM" + return "AdminVM" def is_running(self): return True @@ -94,4 +94,4 @@ class QubesDom0NetVm(QubesNetVm): def verify_files(self): return True -register_qubes_vm_class(QubesDom0NetVm) +register_qubes_vm_class(QubesAdminVm) diff --git a/core/qubes.py b/core/qubes.py index 0532b1de..743ae15f 100755 --- a/core/qubes.py +++ b/core/qubes.py @@ -660,7 +660,7 @@ class QubesVmCollection(dict): # Add dom0 if wasn't present in qubes.xml if not 0 in self.keys(): - dom0vm = QubesDom0NetVm (collection=self) + dom0vm = QubesAdminVm (collection=self) self[dom0vm.qid] = dom0vm return True