From 99b001502a80c410a67f6b99eac67ab6dc93c448 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 2 Dec 2013 14:05:41 +0100 Subject: [PATCH] backups: compression support --- core/backup.py | 39 +++++++++++++++++++++++++++++------- qvm-tools/qvm-backup | 3 +++ qvm-tools/qvm-backup-restore | 8 +++++++- 3 files changed, 42 insertions(+), 8 deletions(-) diff --git a/core/backup.py b/core/backup.py index 059a40e7..8c9d310d 100644 --- a/core/backup.py +++ b/core/backup.py @@ -326,8 +326,9 @@ class Send_Worker(Process): if BACKUP_DEBUG: print "Finished sending thread" -def backup_do(base_backup_dir, files_to_backup, passphrase,\ - progress_callback = None, encrypt=False, appvm=None): +def backup_do(base_backup_dir, files_to_backup, passphrase, + progress_callback = None, encrypt=False, appvm=None, + compress = False): total_backup_sz = 0 for file in files_to_backup: total_backup_sz += file["size"] @@ -454,12 +455,20 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,\ # If no cipher is provided, the data is forwarded unencrypted !!! encryptor = subprocess.Popen (["openssl", "enc", "-e", "-aes-256-cbc", - "-pass", "pass:"+passphrase], + "-pass", "pass:"+passphrase] + + (["-z"] if compress else []), stdin=pipe, stdout=subprocess.PIPE) run_error = wait_backup_feedback( progress_callback=compute_progress, in_stream=encryptor.stdout, streamproc=encryptor, **common_args) + elif compress: + compressor = subprocess.Popen (["gzip"], + stdin=pipe, stdout=subprocess.PIPE) + run_error = wait_backup_feedback( + progress_callback=compute_progress, + in_stream=compressor.stdout, streamproc=compressor, + **common_args) else: run_error = wait_backup_feedback( progress_callback=compute_progress, @@ -638,12 +647,14 @@ def verify_hmac(filename, hmacfile, passphrase): class Extract_Worker(Process): def __init__(self, queue, base_dir, passphrase, encrypted, total_size, - print_callback, error_callback, progress_callback, vmproc=None): + print_callback, error_callback, progress_callback, vmproc=None, + compressed = False): super(Extract_Worker, self).__init__() self.queue = queue self.base_dir = base_dir self.passphrase = passphrase self.encrypted = encrypted + self.compressed = compressed self.total_size = total_size self.blocks_backedup = 0 self.tar2_command = None @@ -719,6 +730,7 @@ class Extract_Worker(Process): encryptor = subprocess.Popen (["openssl", "enc", "-d", "-aes-256-cbc", "-pass", "pass:"+self.passphrase], + (["-z"] if compressed else []), stdin=open(filename,'rb'), stdout=subprocess.PIPE) @@ -726,6 +738,15 @@ class Extract_Worker(Process): progress_callback=self.compute_progress, in_stream=encryptor.stdout, streamproc=encryptor, **common_args) + elif self.compressed: + compressor = subprocess.Popen (["gzip", "-d"], + stdin=open(filename,'rb'), + stdout=subprocess.PIPE) + + run_error = wait_backup_feedback( + progress_callback=self.compute_progress, + in_stream=compressor.stdout, streamproc=compressor, + **common_args) else: run_error = wait_backup_feedback( progress_callback=self.compute_progress, @@ -754,7 +775,8 @@ class Extract_Worker(Process): def restore_vm_dirs (backup_source, restore_tmpdir, passphrase, vms_dirs, vms, vms_size, print_callback=None, error_callback=None, - progress_callback=None, encrypted=False, appvm=None): + progress_callback=None, encrypted=False, appvm=None, + compressed = False): # Setup worker to extract encrypted data chunks to the restore dirs if progress_callback == None: @@ -766,6 +788,7 @@ def restore_vm_dirs (backup_source, restore_tmpdir, passphrase, vms_dirs, vms, base_dir=restore_tmpdir, passphrase=passphrase, encrypted=encrypted, + compressed=compressed, total_size=vms_size, print_callback=print_callback, error_callback=error_callback, @@ -901,7 +924,7 @@ def backup_detect_format_version(backup_location): def backup_restore_header(source, passphrase, print_callback = print_stdout, error_callback = print_stderr, - encrypted=False, appvm=None, format_version = None): + encrypted=False, appvm=None, compressed = False, format_version = None): vmproc = None @@ -935,6 +958,7 @@ def backup_restore_header(source, passphrase, error_callback=error_callback, progress_callback=None, encrypted=encrypted, + compressed=compressed, appvm=appvm) return (restore_tmpdir, "qubes.xml") @@ -1212,7 +1236,7 @@ def backup_restore_print_summary(restore_info, print_callback = print_stdout): def backup_restore_do(backup_location, restore_tmpdir, passphrase, restore_info, host_collection = None, print_callback = print_stdout, error_callback = print_stderr, progress_callback = None, - encrypted=False, appvm=None, format_version = None): + encrypted=False, appvm=None, compressed = False, format_version = None): ### Private functions begin def restore_vm_dir_v1 (backup_dir, src_dir, dst_dir): @@ -1264,6 +1288,7 @@ def backup_restore_do(backup_location, restore_tmpdir, passphrase, restore_info, error_callback=error_callback, progress_callback=progress_callback, encrypted=encrypted, + compressed=compressed, appvm=appvm) # Add VM in right order diff --git a/qvm-tools/qvm-backup b/qvm-tools/qvm-backup index dcddee48..ce7fc281 100755 --- a/qvm-tools/qvm-backup +++ b/qvm-tools/qvm-backup @@ -44,6 +44,8 @@ def main(): help="The AppVM to send backups to") parser.add_option ("-e", "--encrypt", action="store_true", dest="encrypt", default=False, help="Encrypts the backup") + parser.add_option ("-z", "--compress", action="store_true", dest="compress", default=False, + help="Compress the backup") (options, args) = parser.parse_args () @@ -122,6 +124,7 @@ def main(): backup_do(base_backup_dir, files_to_backup, passphrase, progress_callback=print_progress, encrypt=options.encrypt, + compress=options.compress, appvm=appvm) except QubesException as e: print >>sys.stderr, "ERROR: %s" % str(e) diff --git a/qvm-tools/qvm-backup-restore b/qvm-tools/qvm-backup-restore index 31eb5313..05da7812 100755 --- a/qvm-tools/qvm-backup-restore +++ b/qvm-tools/qvm-backup-restore @@ -66,6 +66,9 @@ def main(): parser.add_option ("-e", "--encrypted", action="store_true", dest="decrypt", default=False, help="The backup is encrypted") + parser.add_option ("-z", "--compressed", action="store_true", dest="compressed", default=False, + help="The backup is compressed") + (options, args) = parser.parse_args () if (len (args) != 1): @@ -105,7 +108,9 @@ def main(): passphrase = getpass.getpass("Please enter the pass phrase that will be used to decrypt/verify the backup: ") print >> sys.stderr, "Checking backup content..." - restore_tmpdir,qubes_xml = backup_restore_header(backup_dir, passphrase, encrypted=options.decrypt, appvm=appvm) + restore_tmpdir,qubes_xml = backup_restore_header(backup_dir, passphrase, + encrypted=options.decrypt, compressed=options.compressed, + appvm=appvm) restore_info = None try: @@ -211,6 +216,7 @@ def main(): restore_info, host_collection=host_collection, encrypted=options.decrypt, + compressed=options.compressed, appvm=appvm) host_collection.unlock_db()