From c781a522d86d9fbfe7e93e26340bb7aa4bb8ae3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Mon, 25 Nov 2013 05:41:13 +0100 Subject: [PATCH] backups: move backup code to separate file Also some major cleanups: Reduce some more code duplication (verify_hmac, simplify backup_restore_prepare). Rename backup_dir/backup_tmpdir variables to better match its purpose. Rename backup_do_copy back to backup_do. Require QubesVm object (instead of VM name) as appvm param. --- core/backup.py | 1279 ++++++++++++++++++++++++++++++ core/qubesutils.py | 1452 ---------------------------------- qvm-tools/qvm-backup | 14 +- qvm-tools/qvm-backup-restore | 35 +- rpm_spec/core-dom0.spec | 5 + 5 files changed, 1322 insertions(+), 1463 deletions(-) create mode 100644 core/backup.py diff --git a/core/backup.py b/core/backup.py new file mode 100644 index 00000000..6ea883e7 --- /dev/null +++ b/core/backup.py @@ -0,0 +1,1279 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# The Qubes OS Project, http://www.qubes-os.org +# +# Copyright (C) 2013 Marek Marczykowski-Górecki +# Copyright (C) 2013 Olivier Médoc +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# + +from qubes import QubesException,QubesVmCollection +from qubes import QubesVmClasses +from qubes import system_path,vm_files +from qubesutils import size_to_human, print_stdout, print_stderr +import sys +import os +import subprocess +import re +import shutil +import time +import grp,pwd +from multiprocessing import Queue,Process + +BACKUP_DEBUG = True + +def get_disk_usage(file_or_dir): + if not os.path.exists(file_or_dir): + return 0 + + p = subprocess.Popen (["du", "-s", "--block-size=1", file_or_dir], + stdout=subprocess.PIPE) + result = p.communicate() + m = re.match(r"^(\d+)\s.*", result[0]) + sz = int(m.group(1)) if m is not None else 0 + return sz + + +def file_to_backup (file_path, sz = None): + if sz is None: + sz = os.path.getsize (system_path["qubes_store_filename"]) + + abs_file_path = os.path.abspath (file_path) + abs_base_dir = os.path.abspath (system_path["qubes_base_dir"]) + '/' + abs_file_dir = os.path.dirname (abs_file_path) + '/' + (nothing, dir, subdir) = abs_file_dir.partition (abs_base_dir) + assert nothing == "" + assert dir == abs_base_dir + return [ { "path" : file_path, "size": sz, "subdir": subdir} ] + +def backup_prepare(vms_list = None, exclude_list = [], print_callback = print_stdout): + """If vms = None, include all (sensible) VMs; exclude_list is always applied""" + files_to_backup = file_to_backup (system_path["qubes_store_filename"]) + + if exclude_list is None: + exclude_list = [] + + qvm_collection = QubesVmCollection() + qvm_collection.lock_db_for_writing() + qvm_collection.load() + + if vms_list is None: + all_vms = [vm for vm in qvm_collection.values()] + selected_vms = [vm for vm in all_vms if vm.include_in_backups] + appvms_to_backup = [vm for vm in selected_vms if vm.is_appvm() and not vm.internal] + netvms_to_backup = [vm for vm in selected_vms if vm.is_netvm() and not vm.qid == 0] + template_vms_worth_backingup = [vm for vm in selected_vms if (vm.is_template() and not vm.installed_by_rpm)] + + vms_list = appvms_to_backup + netvms_to_backup + template_vms_worth_backingup + + vms_for_backup = vms_list + # Apply exclude list + if exclude_list: + vms_for_backup = [vm for vm in vms_list if vm.name not in exclude_list] + + no_vms = len (vms_for_backup) + + there_are_running_vms = False + + fields_to_display = [ + { "name": "VM", "width": 16}, + { "name": "type","width": 12 }, + { "name": "size", "width": 12} + ] + + # Display the header + s = "" + for f in fields_to_display: + fmt="{{0:-^{0}}}-+".format(f["width"] + 1) + s += fmt.format('-') + print_callback(s) + s = "" + for f in fields_to_display: + fmt="{{0:>{0}}} |".format(f["width"] + 1) + s += fmt.format(f["name"]) + print_callback(s) + s = "" + for f in fields_to_display: + fmt="{{0:-^{0}}}-+".format(f["width"] + 1) + s += fmt.format('-') + print_callback(s) + + for vm in vms_for_backup: + if vm.is_template(): + # handle templates later + continue + + if vm.private_img is not None: + vm_sz = vm.get_disk_usage (vm.private_img) + files_to_backup += file_to_backup(vm.private_img, vm_sz ) + + if vm.is_appvm(): + files_to_backup += file_to_backup(vm.icon_path) + if vm.updateable: + if os.path.exists(vm.dir_path + "/apps.templates"): + # template + files_to_backup += file_to_backup(vm.dir_path + "/apps.templates") + else: + # standaloneVM + files_to_backup += file_to_backup(vm.dir_path + "/apps") + + if os.path.exists(vm.dir_path + "/kernels"): + files_to_backup += file_to_backup(vm.dir_path + "/kernels") + if os.path.exists (vm.firewall_conf): + files_to_backup += file_to_backup(vm.firewall_conf) + if 'appmenus_whitelist' in vm_files and \ + os.path.exists(vm.dir_path + vm_files['appmenus_whitelist']): + files_to_backup += file_to_backup(vm.dir_path + vm_files['appmenus_whitelist']) + + if vm.updateable: + sz = vm.get_disk_usage(vm.root_img) + files_to_backup += file_to_backup(vm.root_img, sz) + vm_sz += sz + + s = "" + fmt="{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1) + s += fmt.format(vm.name) + + fmt="{{0:>{0}}} |".format(fields_to_display[1]["width"] + 1) + if vm.is_netvm(): + s += fmt.format("NetVM" + (" + Sys" if vm.updateable else "")) + else: + s += fmt.format("AppVM" + (" + Sys" if vm.updateable else "")) + + fmt="{{0:>{0}}} |".format(fields_to_display[2]["width"] + 1) + s += fmt.format(size_to_human(vm_sz)) + + if vm.is_running(): + s += " <-- The VM is running, please shut it down before proceeding with the backup!" + there_are_running_vms = True + + print_callback(s) + + for vm in vms_for_backup: + if not vm.is_template(): + # already handled + continue + vm_sz = vm.get_disk_utilization() + files_to_backup += file_to_backup (vm.dir_path, vm_sz) + + s = "" + fmt="{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1) + s += fmt.format(vm.name) + + fmt="{{0:>{0}}} |".format(fields_to_display[1]["width"] + 1) + s += fmt.format("Template VM") + + fmt="{{0:>{0}}} |".format(fields_to_display[2]["width"] + 1) + s += fmt.format(size_to_human(vm_sz)) + + if vm.is_running(): + s += " <-- The VM is running, please shut it down before proceeding with the backup!" + there_are_running_vms = True + + print_callback(s) + + # Initialize backup flag on all VMs + vms_for_backup_qid = [vm.qid for vm in vms_for_backup] + for vm in qvm_collection.values(): + vm.backup_content = False + + if vm.qid in vms_for_backup_qid: + vm.backup_content = True + vm.backup_size = vm.get_disk_utilization() + vm.backup_path = os.path.relpath(vm.dir_path, system_path["qubes_base_dir"]) + + # Dom0 user home + if not 'dom0' in exclude_list: + local_user = grp.getgrnam('qubes').gr_mem[0] + home_dir = pwd.getpwnam(local_user).pw_dir + # Home dir should have only user-owned files, so fix it now to prevent + # permissions problems - some root-owned files can left after + # 'sudo bash' and similar commands + subprocess.check_call(['sudo', 'chown', '-R', local_user, home_dir]) + + home_sz = get_disk_usage(home_dir) + home_to_backup = [ { "path" : home_dir, "size": home_sz, "subdir": 'dom0-home/'} ] + files_to_backup += home_to_backup + + vm = qvm_collection[0] + vm.backup_content = True + vm.backup_size = home_sz + vm.backup_path = os.path.join('dom0-home', os.path.basename(home_dir)) + + s = "" + fmt="{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1) + s += fmt.format('Dom0') + + fmt="{{0:>{0}}} |".format(fields_to_display[1]["width"] + 1) + s += fmt.format("User home") + + fmt="{{0:>{0}}} |".format(fields_to_display[2]["width"] + 1) + s += fmt.format(size_to_human(home_sz)) + + print_callback(s) + + qvm_collection.save() + # FIXME: should be after backup completed + qvm_collection.unlock_db() + + total_backup_sz = 0 + for file in files_to_backup: + total_backup_sz += file["size"] + + s = "" + for f in fields_to_display: + fmt="{{0:-^{0}}}-+".format(f["width"] + 1) + s += fmt.format('-') + print_callback(s) + + s = "" + fmt="{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1) + s += fmt.format("Total size:") + fmt="{{0:>{0}}} |".format(fields_to_display[1]["width"] + 1 + 2 + fields_to_display[2]["width"] + 1) + s += fmt.format(size_to_human(total_backup_sz)) + print_callback(s) + + s = "" + for f in fields_to_display: + fmt="{{0:-^{0}}}-+".format(f["width"] + 1) + s += fmt.format('-') + print_callback(s) + + if (there_are_running_vms): + raise QubesException("Please shutdown all VMs before proceeding.") + + return files_to_backup + +class Send_Worker(Process): + def __init__(self, queue, base_dir, backup_stdout): + super(Send_Worker, self).__init__() + self.queue = queue + self.base_dir = base_dir + self.backup_stdout = backup_stdout + + def run(self): + if BACKUP_DEBUG: + print "Started sending thread" + + if BACKUP_DEBUG: + print "Moving to temporary dir", self.base_dir + os.chdir(self.base_dir) + + for filename in iter(self.queue.get,None): + if filename == "FINISHED": + break + + if BACKUP_DEBUG: + print "Sending file", filename + # This tar used for sending data out need to be as simple, as + # simple, as featureless as possible. It will not be + # verified before untaring. + tar_final_cmd = ["tar", "-cO", "--posix", + "-C", self.base_dir, filename] + final_proc = subprocess.Popen (tar_final_cmd, + stdin=subprocess.PIPE, stdout=self.backup_stdout) + final_proc.wait() + + # Delete the file as we don't need it anymore + if BACKUP_DEBUG: + print "Removing file", filename + os.remove(filename) + + if BACKUP_DEBUG: + print "Finished sending thread" + +def backup_do(base_backup_dir, files_to_backup, passphrase,\ + progress_callback = None, encrypt=False, appvm=None): + total_backup_sz = 0 + for file in files_to_backup: + total_backup_sz += file["size"] + + vmproc = None + if appvm != None: + # Prepare the backup target (Qubes service call) + backup_target = "QUBESRPC qubes.Backup none" + + # If APPVM, STDOUT is a PIPE + vmproc = appvm.run(command = backup_target, passio_popen = True) + vmproc.stdin.write(base_backup_dir.\ + replace("\r","").replace("\n","")+"\n") + backup_stdout = vmproc.stdin + else: + # Prepare the backup target (local file) + backup_target = base_backup_dir + "/qubes-{0}".\ + format (time.strftime("%Y-%m-%d-%H%M%S")) + + # Create the target directory + if not os.path.exists (base_backup_dir): + raise QubesException( + "ERROR: the backup directory {0} does not exists".\ + format(base_backup_dir)) + + # If not APPVM, STDOUT is a local file + backup_stdout = open(backup_target,'wb') + + global blocks_backedup + blocks_backedup = 0 + progress = blocks_backedup * 11 / total_backup_sz + progress_callback(progress) + + import tempfile + feedback_file = tempfile.NamedTemporaryFile() + backup_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/backup_") + + # Tar with tapelength does not deals well with stdout (close stdout between + # two tapes) + # For this reason, we will use named pipes instead + if BACKUP_DEBUG: + print "Working in", backup_tmpdir + + backup_pipe = os.path.join(backup_tmpdir,"backup_pipe") + if BACKUP_DEBUG: + print "Creating pipe in:", backup_pipe + os.mkfifo(backup_pipe) + + if BACKUP_DEBUG: + print "Will backup:", files_to_backup + + # Setup worker to send encrypted data chunks to the backup_target + + def compute_progress(new_size, total_backup_sz): + global blocks_backedup + blocks_backedup += new_size + progress = blocks_backedup / float(total_backup_sz) + progress_callback(int(round(progress*100,2))) + + to_send = Queue(10) + send_proc = Send_Worker(to_send, backup_tmpdir, backup_stdout) + send_proc.start() + + for filename in files_to_backup: + if BACKUP_DEBUG: + print "Backing up", filename + + backup_tempfile = os.path.join(backup_tmpdir, + filename["subdir"], + os.path.basename(filename["path"])) + if BACKUP_DEBUG: + print "Using temporary location:", backup_tempfile + + # Ensure the temporary directory exists + if not os.path.isdir(os.path.dirname(backup_tempfile)): + os.makedirs(os.path.dirname(backup_tempfile)) + + # The first tar cmd can use any complex feature as we want. Files will + # be verified before untaring this. + # Prefix the path in archive with filename["subdir"] to have it verified during untar + tar_cmdline = ["tar", "-Pc", '--sparse', + "-f", backup_pipe, + '--tape-length', str(100000), + '-C', os.path.dirname(filename["path"]), + '--xform', 's:^[a-z]:%s\\0:' % filename["subdir"], + os.path.basename(filename["path"]) + ] + + if BACKUP_DEBUG: + print " ".join(tar_cmdline) + + # Tips: Popen(bufsize=0) + # Pipe: tar-sparse | encryptor [| hmac] | tar | backup_target + # Pipe: tar-sparse [| hmac] | tar | backup_target + tar_sparse = subprocess.Popen (tar_cmdline, stdin=subprocess.PIPE) + + # Wait for compressor (tar) process to finish or for any error of other + # subprocesses + i = 0 + run_error = "paused" + running = [] + while run_error == "paused": + + pipe = open(backup_pipe,'rb') + + # Start HMAC + hmac = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], + stdin=subprocess.PIPE, stdout=subprocess.PIPE) + + # Prepare a first chunk + chunkfile = backup_tempfile + "." + "%03d" % i + i += 1 + chunkfile_p = open(chunkfile,'wb') + + common_args = { + 'backup_target': chunkfile_p, + 'total_backup_sz': total_backup_sz, + 'hmac': hmac, + 'vmproc': vmproc, + 'addproc': tar_sparse + } + if encrypt: + # Start encrypt + # If no cipher is provided, the data is forwarded unencrypted !!! + encryptor = subprocess.Popen (["openssl", "enc", + "-e", "-aes-256-cbc", + "-pass", "pass:"+passphrase], + stdin=pipe, stdout=subprocess.PIPE) + run_error = wait_backup_feedback( + progress_callback=compute_progress, + in_stream=encryptor.stdout, streamproc=encryptor, + **common_args) + else: + run_error = wait_backup_feedback( + progress_callback=compute_progress, + in_stream=pipe, streamproc=None, + **common_args) + + chunkfile_p.close() + + if BACKUP_DEBUG: + print "Wait_backup_feedback returned:", run_error + + if len(run_error) > 0: + send_proc.terminate() + raise QubesException("Failed to perform backup: error with "+ \ + run_error) + + # Send the chunk to the backup target + to_send.put(os.path.relpath(chunkfile, backup_tmpdir)) + + # Close HMAC + hmac.stdin.close() + hmac.wait() + if BACKUP_DEBUG: + print "HMAC proc return code:", hmac.poll() + + # Write HMAC data next to the chunk file + hmac_data = hmac.stdout.read() + if BACKUP_DEBUG: + print "Writing hmac to", chunkfile+".hmac" + hmac_file = open(chunkfile+".hmac",'w') + hmac_file.write(hmac_data) + hmac_file.flush() + hmac_file.close() + + pipe.close() + + # Send the HMAC to the backup target + to_send.put(os.path.relpath(chunkfile, backup_tmpdir)+".hmac") + + if tar_sparse.poll() == None: + # Release the next chunk + if BACKUP_DEBUG: + print "Release next chunk for process:", tar_sparse.poll() + #tar_sparse.stdout = subprocess.PIPE + tar_sparse.stdin.write("\n") + tar_sparse.stdin.flush() + run_error="paused" + else: + if BACKUP_DEBUG: + print "Finished tar sparse with error", tar_sparse.poll() + + to_send.put("FINISHED") + send_proc.join() + + if send_proc.exitcode != 0: + raise QubesException("Failed to send backup: error in the sending process") + + if vmproc: + if BACKUP_DEBUG: + print "VMProc1 proc return code:", vmproc.poll() + print "Sparse1 proc return code:", tar_sparse.poll() + vmproc.stdin.close() + + shutil.rmtree(restore_tmpdir) + +''' +' Wait for backup chunk to finish +' - Monitor all the processes (streamproc, hmac, vmproc, addproc) for errors +' - Copy stdout of streamproc to backup_target and hmac stdin if available +' - Compute progress based on total_backup_sz and send progress to +' progress_callback function +' - Returns if +' - one of the monitored processes error out (streamproc, hmac, vmproc, +' addproc), along with the processe that failed +' - all of the monitored processes except vmproc finished successfully +' (vmproc termination is controlled by the python script) +' - streamproc does not delivers any data anymore (return with the error +' "") +''' +def wait_backup_feedback(progress_callback, in_stream, streamproc, + backup_target, total_backup_sz, hmac=None, vmproc=None, addproc=None, + remove_trailing_bytes=0): + + buffer_size = 409600 + + run_error = None + run_count = 1 + blocks_backedup = 0 + while run_count > 0 and run_error == None: + + buffer = in_stream.read(buffer_size) + progress_callback(len(buffer), total_backup_sz) + + run_count = 0 + if hmac: + retcode=hmac.poll() + if retcode != None: + if retcode != 0: + run_error = "hmac" + else: + run_count += 1 + + if addproc: + retcode=addproc.poll() + if retcode != None: + if retcode != 0: + run_error = "addproc" + else: + run_count += 1 + + if vmproc: + retcode = vmproc.poll() + if retcode != None: + if retcode != 0: + run_error = "VM" + if BACKUP_DEBUG: + print vmproc.stdout.read() + else: + # VM should run until the end + pass + + if streamproc: + retcode=streamproc.poll() + if retcode != None: + if retcode != 0: + run_error = "streamproc" + break + elif retcode == 0 and len(buffer) <= 0: + return "" + run_count += 1 + + else: + if len(buffer) <= 0: + return "" + + backup_target.write(buffer) + + if hmac: + hmac.stdin.write(buffer) + + return run_error + +def verify_hmac(filename, hmacfile, passphrase): + if BACKUP_DEBUG: + print "Verifying file "+filename + + if hmacfile != filename + ".hmac": + raise QubesException( + "ERROR: expected hmac for {}, but got {}".\ + format(filename, hmacfile)) + + hmac_proc = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], + stdin=open(filename,'rb'), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + hmac_stdout, hmac_stderr = hmac_proc.communicate() + + if len(hmac_stderr) > 0: + raise QubesException("ERROR: verify file {0}: {1}".format((filename, hmac_stderr))) + else: + if BACKUP_DEBUG: + print "Loading hmac for file " + filename + hmac = load_hmac(open(hmacfile,'r').read()) + + if len(hmac) > 0 and load_hmac(hmac_stdout) == hmac: + os.unlink(hmacfile) + if BACKUP_DEBUG: + print "File verification OK -> Sending file " + filename + return True + else: + raise QubesException( + "ERROR: invalid hmac for file {0}: {1}. " \ + "Is the passphrase correct?".\ + format(filename, load_hmac(hmac_stdout))) + # Not reachable + return False + +class Extract_Worker(Process): + def __init__(self, queue, base_dir, passphrase, encrypted, total_size, + print_callback, error_callback, progress_callback, vmproc=None): + super(Extract_Worker, self).__init__() + self.queue = queue + self.base_dir = base_dir + self.passphrase = passphrase + self.encrypted = encrypted + self.total_size = total_size + self.blocks_backedup = 0 + self.tar2_command = None + self.tar2_current_file = None + + self.print_callback = print_callback + self.error_callback = error_callback + self.progress_callback = progress_callback + + self.vmproc = vmproc + + self.restore_pipe = os.path.join(self.base_dir,"restore_pipe") + if BACKUP_DEBUG: + print "Creating pipe in:", self.restore_pipe + os.mkfifo(self.restore_pipe) + + def compute_progress(self, new_size, total_size): + self.blocks_backedup += new_size + progress = self.blocks_backedup / float(self.total_size) + progress = int(round(progress*100,2)) + self.progress_callback(progress) + + def run(self): + if BACKUP_DEBUG: + self.print_callback("Started sending thread") + self.print_callback("Moving to dir "+self.base_dir) + os.chdir(self.base_dir) + + for filename in iter(self.queue.get,None): + if filename == "FINISHED": + break + + if BACKUP_DEBUG: + self.print_callback("Extracting file "+filename) + + if filename.endswith('.000'): + # next file + if self.tar2_command != None: + if self.tar2_command.wait() != 0: + raise QubesException( + "ERROR: unable to extract files for {0}.".\ + format(self.tar2_current_file)) + else: + # Finished extracting the tar file + self.tar2_command = None + self.tar2_current_file = None + + tar2_cmdline = ['tar', + '-xMk%sf' % ("v" if BACKUP_DEBUG else ""), self.restore_pipe, + os.path.relpath(filename.rstrip('.000'))] + if BACKUP_DEBUG: + self.print_callback("Running command "+str(tar2_cmdline)) + self.tar2_command = subprocess.Popen(tar2_cmdline, + stdin=subprocess.PIPE, + stderr=(None if BACKUP_DEBUG else open('/dev/null', 'w'))) + else: + if BACKUP_DEBUG: + self.print_callback("Releasing next chunck") + self.tar2_command.stdin.write("\n") + self.tar2_command.stdin.flush() + self.tar2_current_file = filename + + pipe = open(self.restore_pipe,'wb') + common_args = { + 'backup_target': pipe, + 'total_backup_sz': self.total_size, + 'hmac': None, + 'vmproc': self.vmproc, + 'addproc': self.tar2_command + } + if self.encrypted: + # Start decrypt + encryptor = subprocess.Popen (["openssl", "enc", + "-d", "-aes-256-cbc", + "-pass", "pass:"+passphrase], + stdin=open(filename,'rb'), + stdout=subprocess.PIPE) + + run_error = wait_backup_feedback( + progress_callback=self.compute_progress, + in_stream=encryptor.stdout, streamproc=encryptor, + **common_args) + else: + run_error = wait_backup_feedback( + progress_callback=self.compute_progress, + in_stream=open(filename,"rb"), streamproc=None, + **common_args) + + pipe.close() + + # Delete the file as we don't need it anymore + if BACKUP_DEBUG: + self.print_callback("Removing file "+filename) + os.remove(filename) + + if self.tar2_command != None: + if self.tar2_command.wait() != 0: + raise QubesException( + "ERROR: unable to extract files for {0}.".\ + format(self.tar2_current_file)) + else: + # Finished extracting the tar file + self.tar2_command = None + + os.unlink(self.restore_pipe) + if BACKUP_DEBUG: + self.print_callback("Finished extracting thread") + +def restore_vm_dirs (backup_source, restore_tmpdir, passphrase, vms_dirs, vms, + vms_size, print_callback=None, error_callback=None, + progress_callback=None, encrypted=False, appvm=None): + + # Setup worker to extract encrypted data chunks to the restore dirs + if progress_callback == None: + def progress_callback(data): + pass + + to_extract = Queue() + extract_proc = Extract_Worker(queue=to_extract, + base_dir=restore_tmpdir, + passphrase=passphrase, + encrypted=encrypted, + total_size=vms_size, + print_callback=print_callback, + error_callback=error_callback, + progress_callback=progress_callback) + extract_proc.start() + + if BACKUP_DEBUG: + print_callback("Working in temporary dir:"+restore_tmpdir) + print_callback(str(vms_size)+" bytes to restore") + + vmproc = None + if appvm != None: + # Prepare the backup target (Qubes service call) + backup_target = "QUBESRPC qubes.Restore dom0" + + # If APPVM, STDOUT is a PIPE + vmproc = appvm.run(command = backup_target, passio_popen = True) + vmproc.stdin.write(backup_source.replace("\r","").replace("\n","")+"\n") + + # Send to tar2qfile the VMs that should be extracted + vmproc.stdin.write(" ".join(vms_dirs)+"\n") + + backup_stdin = vmproc.stdout + tar1_command = ['/usr/libexec/qubes/qfile-dom0-unpacker', + str(os.getuid()), restore_tmpdir, '-v'] + else: + backup_stdin = open(backup_source,'rb') + + tar1_command = ['tar', + '-ix%sf' % ("v" if BACKUP_DEBUG else ""), backup_source, + '-C', restore_tmpdir] + vms_dirs + + # Remove already processed qubes.xml.000, because qfile-dom0-unpacker will + # refuse to override files + tar1_env = os.environ.copy() + # TODO: add some safety margin? + tar1_env['UPDATES_MAX_BYTES'] = str(vms_size) + # Restoring only header + if len(vms_dirs) == 2 and vms_dirs[0] == 'qubes.xml.000': + tar1_env['UPDATES_MAX_FILES'] = '2' + else: + tar1_env['UPDATES_MAX_FILES'] = '0' + if BACKUP_DEBUG: + print_callback("Run command"+str(tar1_command)) + command = subprocess.Popen(tar1_command, + stdin=backup_stdin, + stdout=vmproc.stdin if vmproc else subprocess.PIPE, + stderr=subprocess.PIPE, + env=tar1_env) + + # qfile-dom0-unpacker output filelist on stderr (and have stdout connected + # to the VM), while tar output filelist on stdout + if appvm: + filelist_pipe = command.stderr + else: + filelist_pipe = command.stdout + + while True: + + filename = filelist_pipe.readline().strip(" \t\r\n") + + if BACKUP_DEBUG: + print_callback("Getting new file:"+filename) + + if not filename or filename=="EOF": + break + + hmacfile = filelist_pipe.readline().strip(" \t\r\n") + if BACKUP_DEBUG: + print_callback("Getting hmac:"+hmacfile) + + if not any(map(lambda x: filename.startswith(x), vms_dirs)): + if BACKUP_DEBUG: + print_callback("Ignoring VM not selected for restore") + os.unlink(os.path.join(restore_tmpdir, filename)) + os.unlink(os.path.join(restore_tmpdir, hmacfile)) + continue + + if verify_hmac(os.path.join(restore_tmpdir,filename), + os.path.join(restore_tmpdir,hmacfile), + passphrase): + to_extract.put(os.path.join(restore_tmpdir, filename)) + + if command.wait() != 0: + raise QubesException( + "ERROR: unable to read the qubes backup file {0} ({1}). " \ + "Is it really a backup?".format(backup_source, command.wait())) + if vmproc: + if vmproc.wait() != 0: + raise QubesException( + "ERROR: unable to read the qubes backup {0} " \ + "because of a VM error: {1}".format( + backup_source, vmproc.stderr.read())) + if BACKUP_DEBUG: + print "Extraction process status:", extract_proc.exitcode + + to_extract.put("FINISHED") + if BACKUP_DEBUG: + print_callback("Waiting for the extraction process to finish...") + extract_proc.join() + if BACKUP_DEBUG: + print_callback("Extraction process finished with code:" + \ + str(extract_proc.exitcode)) + if extract_proc.exitcode != 0: + raise QubesException( + "ERROR: unable to extract the qubes backup. " \ + "Check extracting process errors.") + +def backup_restore_set_defaults(options): + if 'use-default-netvm' not in options: + options['use-default-netvm'] = False + if 'use-none-netvm' not in options: + options['use-none-netvm'] = False + if 'use-default-template' not in options: + options['use-default-template'] = False + if 'dom0-home' not in options: + options['dom0-home'] = True + if 'replace-template' not in options: + options['replace-template'] = [] + + return options + +def load_hmac(hmac): + hmac = hmac.strip(" \t\r\n").split("=") + if len(hmac) > 1: + hmac = hmac[1].strip() + else: + raise QubesException("ERROR: invalid hmac file content") + + return hmac + +def backup_restore_header(source, passphrase, + print_callback = print_stdout, error_callback = print_stderr, + encrypted=False, appvm=None): + + # Simulate dd if=backup_file count=10 | file - + # Simulate dd if=backup_file count=10 | gpg2 -d | tar xzv -O + # analysis = subprocess.Popen() + vmproc = None + + import tempfile + feedback_file = tempfile.NamedTemporaryFile() + restore_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/restore_") + + os.chdir(restore_tmpdir) + + if BACKUP_DEBUG: + print "Working in", restore_tmpdir + + restore_vm_dirs (source, + restore_tmpdir, + passphrase=passphrase, + vms_dirs=['qubes.xml.000', 'qubes.xml.000.hmac'], + vms=None, + vms_size=40000, + print_callback=print_callback, + error_callback=error_callback, + progress_callback=None, + encrypted=encrypted, + appvm=appvm) + + return (restore_tmpdir, "qubes.xml") + +def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, + host_collection = None, encrypt=False, appvm=None): + # Defaults + backup_restore_set_defaults(options) + + #### Private functions begin + def is_vm_included_in_backup (backup_dir, vm): + if vm.backup_content: + return True + else: + return False + + def find_template_name(template, replaces): + rx_replace = re.compile("(.*):(.*)") + for r in replaces: + m = rx_replace.match(r) + if m.group(1) == template: + return m.group(2) + + return template + #### Private functions end + if BACKUP_DEBUG: + print "Loading file", qubes_xml + backup_collection = QubesVmCollection(store_filename = qubes_xml) + backup_collection.lock_db_for_reading() + backup_collection.load() + + if host_collection is None: + host_collection = QubesVmCollection() + host_collection.lock_db_for_reading() + host_collection.load() + host_collection.unlock_db() + + backup_vms_list = [vm for vm in backup_collection.values()] + host_vms_list = [vm for vm in host_collection.values()] + vms_to_restore = {} + + there_are_conflicting_vms = False + there_are_missing_templates = False + there_are_missing_netvms = False + dom0_username_mismatch = False + restore_home = False + # ... and the actual data + for vm in backup_vms_list: + if vm.qid == 0: + # Handle dom0 as special case later + continue + if is_vm_included_in_backup (backup_dir, vm): + if BACKUP_DEBUG: + print vm.name,"is included in backup" + + vms_to_restore[vm.name] = {} + vms_to_restore[vm.name]['vm'] = vm; + if 'exclude' in options.keys(): + vms_to_restore[vm.name]['excluded'] = vm.name in options['exclude'] + vms_to_restore[vm.name]['good-to-go'] = False + + if host_collection.get_vm_by_name (vm.name) is not None: + vms_to_restore[vm.name]['already-exists'] = True + vms_to_restore[vm.name]['good-to-go'] = False + + if vm.template is None: + vms_to_restore[vm.name]['template'] = None + else: + templatevm_name = find_template_name(vm.template.name, options['replace-template']) + vms_to_restore[vm.name]['template'] = templatevm_name + template_vm_on_host = host_collection.get_vm_by_name (templatevm_name) + + # No template on the host? + if not ((template_vm_on_host is not None) and template_vm_on_host.is_template()): + # Maybe the (custom) template is in the backup? + template_vm_on_backup = backup_collection.get_vm_by_name (templatevm_name) + if template_vm_on_backup is None or not \ + (is_vm_included_in_backup(backup_dir, template_vm_on_backup) and \ + template_vm_on_backup.is_template()): + if options['use-default-template']: + vms_to_restore[vm.name]['orig-template'] = templatevm_name + vms_to_restore[vm.name]['template'] = host_collection.get_default_template().name + else: + vms_to_restore[vm.name]['missing-template'] = True + vms_to_restore[vm.name]['good-to-go'] = False + + if vm.netvm is None: + vms_to_restore[vm.name]['netvm'] = None + else: + netvm_name = vm.netvm.name + vms_to_restore[vm.name]['netvm'] = netvm_name + # Set to None to not confuse QubesVm object from backup + # collection with host collection (further in clone_attrs). Set + # directly _netvm to suppress setter action, especially + # modifying firewall + vm._netvm = None + + netvm_on_host = host_collection.get_vm_by_name (netvm_name) + + # No netvm on the host? + if not ((netvm_on_host is not None) and netvm_on_host.is_netvm()): + + # Maybe the (custom) netvm is in the backup? + netvm_on_backup = backup_collection.get_vm_by_name (netvm_name) + if not ((netvm_on_backup is not None) and \ + netvm_on_backup.is_netvm() and \ + is_vm_included_in_backup(backup_dir, netvm_on_backup)): + if options['use-default-netvm']: + vms_to_restore[vm.name]['netvm'] = host_collection.get_default_netvm().name + vm.uses_default_netvm = True + elif options['use-none-netvm']: + vms_to_restore[vm.name]['netvm'] = None + else: + vms_to_restore[vm.name]['missing-netvm'] = True + vms_to_restore[vm.name]['good-to-go'] = False + + if 'good-to-go' not in vms_to_restore[vm.name].keys(): + vms_to_restore[vm.name]['good-to-go'] = True + + # ...and dom0 home + if options['dom0-home'] and \ + is_vm_included_in_backup(backup_dir, backup_collection[0]): + vm = backup_collection[0] + vms_to_restore['dom0'] = {} + vms_to_restore['dom0']['subdir'] = vm.backup_path + vms_to_restore['dom0']['size'] = vm.backup_size + local_user = grp.getgrnam('qubes').gr_mem[0] + + dom0_home = vm.backup_path + + vms_to_restore['dom0']['username'] = os.path.basename(dom0_home) + if vms_to_restore['dom0']['username'] != local_user: + vms_to_restore['dom0']['username-mismatch'] = True + if not options['ignore-dom0-username-mismatch']: + vms_to_restore['dom0']['good-to-go'] = False + + if 'good-to-go' not in vms_to_restore['dom0']: + vms_to_restore['dom0']['good-to-go'] = True + + # Not needed - all the data stored in vms_to_restore + os.unlink(qubes_xml) + return vms_to_restore + +def backup_restore_print_summary(restore_info, print_callback = print_stdout): + fields = { + "qid": {"func": "vm.qid"}, + + "name": {"func": "('[' if vm.is_template() else '')\ + + ('{' if vm.is_netvm() else '')\ + + vm.name \ + + (']' if vm.is_template() else '')\ + + ('}' if vm.is_netvm() else '')"}, + + "type": {"func": "'Tpl' if vm.is_template() else \ + 'HVM' if vm.type == 'HVM' else \ + vm.type.replace('VM','')"}, + + "updbl" : {"func": "'Yes' if vm.updateable else ''"}, + + "template": {"func": "'n/a' if vm.is_template() or vm.template is None else\ + vm_info['template']"}, + + "netvm": {"func": "'n/a' if vm.is_netvm() and not vm.is_proxyvm() else\ + ('*' if vm.uses_default_netvm else '') +\ + vm_info['netvm'] if vm_info['netvm'] is not None else '-'"}, + + "label" : {"func" : "vm.label.name"}, + } + + fields_to_display = ["name", "type", "template", "updbl", "netvm", "label" ] + + # First calculate the maximum width of each field we want to display + total_width = 0; + for f in fields_to_display: + fields[f]["max_width"] = len(f) + for vm_info in restore_info.values(): + if 'vm' in vm_info.keys(): + vm = vm_info['vm'] + l = len(str(eval(fields[f]["func"]))) + if l > fields[f]["max_width"]: + fields[f]["max_width"] = l + total_width += fields[f]["max_width"] + + print_callback("") + print_callback("The following VMs are included in the backup:") + print_callback("") + + # Display the header + s = "" + for f in fields_to_display: + fmt="{{0:-^{0}}}-+".format(fields[f]["max_width"] + 1) + s += fmt.format('-') + print_callback(s) + s = "" + for f in fields_to_display: + fmt="{{0:>{0}}} |".format(fields[f]["max_width"] + 1) + s += fmt.format(f) + print_callback(s) + s = "" + for f in fields_to_display: + fmt="{{0:-^{0}}}-+".format(fields[f]["max_width"] + 1) + s += fmt.format('-') + print_callback(s) + + for vm_info in restore_info.values(): + # Skip non-VM here + if not 'vm' in vm_info: + continue + vm = vm_info['vm'] + s = "" + for f in fields_to_display: + fmt="{{0:>{0}}} |".format(fields[f]["max_width"] + 1) + s += fmt.format(eval(fields[f]["func"])) + + if 'excluded' in vm_info and vm_info['excluded']: + s += " <-- Excluded from restore" + elif 'already-exists' in vm_info: + s += " <-- A VM with the same name already exists on the host!" + elif 'missing-template' in vm_info: + s += " <-- No matching template on the host or in the backup found!" + elif 'missing-netvm' in vm_info: + s += " <-- No matching netvm on the host or in the backup found!" + elif 'orig-template' in vm_info: + s += " <-- Original template was '%s'" % (vm_info['orig-template']) + + print_callback(s) + + if 'dom0' in restore_info.keys(): + s = "" + for f in fields_to_display: + fmt="{{0:>{0}}} |".format(fields[f]["max_width"] + 1) + if f == "name": + s += fmt.format("Dom0") + elif f == "type": + s += fmt.format("Home") + else: + s += fmt.format("") + if 'username-mismatch' in restore_info['dom0']: + s += " <-- username in backup and dom0 mismatch" + + print_callback(s) + +def backup_restore_do(backup_dir, restore_tmpdir, passphrase, restore_info, + host_collection = None, print_callback = print_stdout, + error_callback = print_stderr, progress_callback = None, + encrypted=False, appvm=None): + + lock_obtained = False + if host_collection is None: + host_collection = QubesVmCollection() + host_collection.lock_db_for_writing() + host_collection.load() + lock_obtained = True + + # Perform VM restoration in backup order + vms_dirs = [] + vms_size = 0 + vms = {} + for vm_info in restore_info.values(): + if not vm_info['good-to-go']: + continue + if 'vm' not in vm_info: + continue + vm = vm_info['vm'] + vms_size += vm.backup_size + vms_dirs.append(vm.backup_path) + vms[vm.name] = vm + + if 'dom0' in restore_info.keys() and restore_info['dom0']['good-to-go']: + vms_dirs.append('dom0-home') + vms_size += restore_info['dom0']['size'] + + restore_vm_dirs (backup_dir, + restore_tmpdir, + passphrase=passphrase, + vms_dirs=vms_dirs, + vms=vms, + vms_size=vms_size, + print_callback=print_callback, + error_callback=error_callback, + progress_callback=progress_callback, + encrypted=encrypted, + appvm=appvm) + + # Add VM in right order + for (vm_class_name, vm_class) in sorted(QubesVmClasses.items(), + key=lambda _x: _x[1].load_order): + for vm_info in restore_info.values(): + if not vm_info['good-to-go']: + continue + if 'vm' not in vm_info: + continue + vm = vm_info['vm'] + if not vm.__class__ == vm_class: + continue + print_callback("-> Restoring {type} {0}...".format(vm.name, type=vm_class_name)) + retcode = subprocess.call (["mkdir", "-p", os.path.dirname(vm.dir_path)]) + if retcode != 0: + error_callback("*** Cannot create directory: {0}?!".format(dest_dir)) + error_callback("Skipping...") + continue + + template = None + if vm.template is not None: + template_name = vm_info['template'] + template = host_collection.get_vm_by_name(template_name) + + new_vm = None + + try: + new_vm = host_collection.add_new_vm(vm_class_name, name=vm.name, + conf_file=vm.conf_file, + dir_path=vm.dir_path, + template=template, + installed_by_rpm=False) + + shutil.move(os.path.join(restore_tmpdir, vm.backup_path), + new_vm.dir_path) + + new_vm.verify_files() + except Exception as err: + error_callback("ERROR: {0}".format(err)) + error_callback("*** Skipping VM: {0}".format(vm.name)) + if new_vm: + host_collection.pop(new_vm.qid) + continue + + try: + new_vm.clone_attrs(vm) + except Exception as err: + error_callback("ERROR: {0}".format(err)) + error_callback("*** Some VM property will not be restored") + + try: + new_vm.appmenus_create(verbose=True) + except Exception as err: + error_callback("ERROR during appmenu restore: {0}".format(err)) + error_callback("*** VM '{0}' will not have appmenus".format(vm.name)) + + # Set network dependencies - only non-default netvm setting + for vm_info in restore_info.values(): + if not vm_info['good-to-go']: + continue + if 'vm' not in vm_info: + continue + vm = vm_info['vm'] + host_vm = host_collection.get_vm_by_name(vm.name) + if host_vm is None: + # Failed/skipped VM + continue + + if not vm.uses_default_netvm: + host_vm.netvm = host_collection.get_vm_by_name (vm_info['netvm']) if vm_info['netvm'] is not None else None + + host_collection.save() + if lock_obtained: + host_collection.unlock_db() + + # ... and dom0 home as last step + if 'dom0' in restore_info.keys() and restore_info['dom0']['good-to-go']: + backup_path = restore_info['dom0']['subdir'] + local_user = grp.getgrnam('qubes').gr_mem[0] + home_dir = pwd.getpwnam(local_user).pw_dir + backup_dom0_home_dir = os.path.join(restore_tmpdir, backup_path) + restore_home_backupdir = "home-pre-restore-{0}".format (time.strftime("%Y-%m-%d-%H%M%S")) + + print_callback("-> Restoring home of user '{0}'...".format(local_user)) + print_callback("--> Existing files/dirs backed up in '{0}' dir".format(restore_home_backupdir)) + os.mkdir(home_dir + '/' + restore_home_backupdir) + for f in os.listdir(backup_dom0_home_dir): + home_file = home_dir + '/' + f + if os.path.exists(home_file): + os.rename(home_file, home_dir + '/' + restore_home_backupdir + '/' + f) + shutil.move(backup_dom0_home_dir + '/' + f, home_file) + retcode = subprocess.call(['sudo', 'chown', '-R', local_user, home_dir]) + if retcode != 0: + error_callback("*** Error while setting home directory owner") + + shutil.rmtree(restore_tmpdir) + +# vim:sw=4:et: diff --git a/core/qubesutils.py b/core/qubesutils.py index 304e5a24..9034540e 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -38,8 +38,6 @@ import xen.lowlevel.xc import xen.lowlevel.xl import xen.lowlevel.xs -BACKUP_DEBUG = True - def mbytes_to_kmg(size): if size > 1024: return "%d GiB" % (size/1024) @@ -754,1454 +752,4 @@ class QubesWatch(object): while True: self.watch_single() -######## Backups ######### - -def get_disk_usage(file_or_dir): - if not os.path.exists(file_or_dir): - return 0 - - p = subprocess.Popen (["du", "-s", "--block-size=1", file_or_dir], - stdout=subprocess.PIPE) - result = p.communicate() - m = re.match(r"^(\d+)\s.*", result[0]) - sz = int(m.group(1)) if m is not None else 0 - return sz - - -def file_to_backup (file_path, sz = None): - if sz is None: - sz = os.path.getsize (system_path["qubes_store_filename"]) - - abs_file_path = os.path.abspath (file_path) - abs_base_dir = os.path.abspath (system_path["qubes_base_dir"]) + '/' - abs_file_dir = os.path.dirname (abs_file_path) + '/' - (nothing, dir, subdir) = abs_file_dir.partition (abs_base_dir) - assert nothing == "" - assert dir == abs_base_dir - return [ { "path" : file_path, "size": sz, "subdir": subdir} ] - -def backup_prepare(vms_list = None, exclude_list = [], print_callback = print_stdout): - """If vms = None, include all (sensible) VMs; exclude_list is always applied""" - files_to_backup = file_to_backup (system_path["qubes_store_filename"]) - - if exclude_list is None: - exclude_list = [] - - qvm_collection = QubesVmCollection() - qvm_collection.lock_db_for_writing() - qvm_collection.load() - - if vms_list is None: - all_vms = [vm for vm in qvm_collection.values()] - selected_vms = [vm for vm in all_vms if vm.include_in_backups] - appvms_to_backup = [vm for vm in selected_vms if vm.is_appvm() and not vm.internal] - netvms_to_backup = [vm for vm in selected_vms if vm.is_netvm() and not vm.qid == 0] - template_vms_worth_backingup = [vm for vm in selected_vms if (vm.is_template() and not vm.installed_by_rpm)] - - vms_list = appvms_to_backup + netvms_to_backup + template_vms_worth_backingup - - vms_for_backup = vms_list - # Apply exclude list - if exclude_list: - vms_for_backup = [vm for vm in vms_list if vm.name not in exclude_list] - - no_vms = len (vms_for_backup) - - there_are_running_vms = False - - fields_to_display = [ - { "name": "VM", "width": 16}, - { "name": "type","width": 12 }, - { "name": "size", "width": 12} - ] - - # Display the header - s = "" - for f in fields_to_display: - fmt="{{0:-^{0}}}-+".format(f["width"] + 1) - s += fmt.format('-') - print_callback(s) - s = "" - for f in fields_to_display: - fmt="{{0:>{0}}} |".format(f["width"] + 1) - s += fmt.format(f["name"]) - print_callback(s) - s = "" - for f in fields_to_display: - fmt="{{0:-^{0}}}-+".format(f["width"] + 1) - s += fmt.format('-') - print_callback(s) - - for vm in vms_for_backup: - if vm.is_template(): - # handle templates later - continue - - if vm.private_img is not None: - vm_sz = vm.get_disk_usage (vm.private_img) - files_to_backup += file_to_backup(vm.private_img, vm_sz ) - - if vm.is_appvm(): - files_to_backup += file_to_backup(vm.icon_path) - if vm.updateable: - if os.path.exists(vm.dir_path + "/apps.templates"): - # template - files_to_backup += file_to_backup(vm.dir_path + "/apps.templates") - else: - # standaloneVM - files_to_backup += file_to_backup(vm.dir_path + "/apps") - - if os.path.exists(vm.dir_path + "/kernels"): - files_to_backup += file_to_backup(vm.dir_path + "/kernels") - if os.path.exists (vm.firewall_conf): - files_to_backup += file_to_backup(vm.firewall_conf) - if 'appmenus_whitelist' in vm_files and \ - os.path.exists(vm.dir_path + vm_files['appmenus_whitelist']): - files_to_backup += file_to_backup(vm.dir_path + vm_files['appmenus_whitelist']) - - if vm.updateable: - sz = vm.get_disk_usage(vm.root_img) - files_to_backup += file_to_backup(vm.root_img, sz) - vm_sz += sz - - s = "" - fmt="{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1) - s += fmt.format(vm.name) - - fmt="{{0:>{0}}} |".format(fields_to_display[1]["width"] + 1) - if vm.is_netvm(): - s += fmt.format("NetVM" + (" + Sys" if vm.updateable else "")) - else: - s += fmt.format("AppVM" + (" + Sys" if vm.updateable else "")) - - fmt="{{0:>{0}}} |".format(fields_to_display[2]["width"] + 1) - s += fmt.format(size_to_human(vm_sz)) - - if vm.is_running(): - s += " <-- The VM is running, please shut it down before proceeding with the backup!" - there_are_running_vms = True - - print_callback(s) - - for vm in vms_for_backup: - if not vm.is_template(): - # already handled - continue - vm_sz = vm.get_disk_utilization() - files_to_backup += file_to_backup (vm.dir_path, vm_sz) - - s = "" - fmt="{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1) - s += fmt.format(vm.name) - - fmt="{{0:>{0}}} |".format(fields_to_display[1]["width"] + 1) - s += fmt.format("Template VM") - - fmt="{{0:>{0}}} |".format(fields_to_display[2]["width"] + 1) - s += fmt.format(size_to_human(vm_sz)) - - if vm.is_running(): - s += " <-- The VM is running, please shut it down before proceeding with the backup!" - there_are_running_vms = True - - print_callback(s) - - # Initialize backup flag on all VMs - vms_for_backup_qid = [vm.qid for vm in vms_for_backup] - for vm in qvm_collection.values(): - vm.backup_content = False - - if vm.qid in vms_for_backup_qid: - vm.backup_content = True - vm.backup_size = vm.get_disk_utilization() - vm.backup_path = os.path.relpath(vm.dir_path, system_path["qubes_base_dir"]) - - # Dom0 user home - if not 'dom0' in exclude_list: - local_user = grp.getgrnam('qubes').gr_mem[0] - home_dir = pwd.getpwnam(local_user).pw_dir - # Home dir should have only user-owned files, so fix it now to prevent - # permissions problems - some root-owned files can left after - # 'sudo bash' and similar commands - subprocess.check_call(['sudo', 'chown', '-R', local_user, home_dir]) - - home_sz = get_disk_usage(home_dir) - home_to_backup = [ { "path" : home_dir, "size": home_sz, "subdir": 'dom0-home/'} ] - files_to_backup += home_to_backup - - vm = qvm_collection[0] - vm.backup_content = True - vm.backup_size = home_sz - vm.backup_path = os.path.join('dom0-home', os.path.basename(home_dir)) - - s = "" - fmt="{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1) - s += fmt.format('Dom0') - - fmt="{{0:>{0}}} |".format(fields_to_display[1]["width"] + 1) - s += fmt.format("User home") - - fmt="{{0:>{0}}} |".format(fields_to_display[2]["width"] + 1) - s += fmt.format(size_to_human(home_sz)) - - print_callback(s) - - qvm_collection.save() - # FIXME: should be after backup completed - qvm_collection.unlock_db() - - total_backup_sz = 0 - for file in files_to_backup: - total_backup_sz += file["size"] - - s = "" - for f in fields_to_display: - fmt="{{0:-^{0}}}-+".format(f["width"] + 1) - s += fmt.format('-') - print_callback(s) - - s = "" - fmt="{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1) - s += fmt.format("Total size:") - fmt="{{0:>{0}}} |".format(fields_to_display[1]["width"] + 1 + 2 + fields_to_display[2]["width"] + 1) - s += fmt.format(size_to_human(total_backup_sz)) - print_callback(s) - - s = "" - for f in fields_to_display: - fmt="{{0:-^{0}}}-+".format(f["width"] + 1) - s += fmt.format('-') - print_callback(s) - - if (there_are_running_vms): - raise QubesException("Please shutdown all VMs before proceeding.") - - return files_to_backup - -def backup_do(base_backup_dir, files_to_backup, progress_callback = None): - - total_backup_sz = 0 - for file in files_to_backup: - total_backup_sz += file["size"] - - backup_dir = base_backup_dir + "/qubes-{0}".format (time.strftime("%Y-%m-%d-%H%M%S")) - if os.path.exists (backup_dir): - raise QubesException("ERROR: the path {0} already exists?!".format(backup_dir)) - - os.mkdir (backup_dir) - - if not os.path.exists (backup_dir): - raise QubesException("Strange: couldn't create backup dir: {0}?!".format(backup_dir)) - - bytes_backedup = 0 - for file in files_to_backup: - # We prefer to use Linux's cp, because it nicely handles sparse files - progress = bytes_backedup * 100 / total_backup_sz - progress_callback(progress) - dest_dir = backup_dir + '/' + file["subdir"] - if file["subdir"] != "": - retcode = subprocess.call (["mkdir", "-p", dest_dir]) - if retcode != 0: - raise QubesException("Cannot create directory: {0}?!".format(dest_dir)) - - retcode = subprocess.call (["cp", "-rp", file["path"], dest_dir]) - if retcode != 0: - raise QubesException("Error while copying file {0} to {1}".format(file["path"], dest_dir)) - - bytes_backedup += file["size"] - progress = bytes_backedup * 100 / total_backup_sz - progress_callback(progress) - -def backup_do_copy(base_backup_dir, files_to_backup, passphrase,\ - progress_callback = None, encrypt=False, appvm=None): - total_backup_sz = 0 - for file in files_to_backup: - total_backup_sz += file["size"] - - vmproc = None - if appvm != None: - # Prepare the backup target (Qubes service call) - backup_target = "QUBESRPC qubes.Backup none" - - # does the vm exist? - qvm_collection = QubesVmCollection() - qvm_collection.lock_db_for_reading() - qvm_collection.load() - - vm = qvm_collection.get_vm_by_name(appvm) - if vm is None or vm.qid not in qvm_collection: - raise QubesException("VM {0} does not exist".format(appvm)) - - qvm_collection.unlock_db() - - # If APPVM, STDOUT is a PIPE - vmproc = vm.run(command = backup_target, passio_popen = True) - vmproc.stdin.write(base_backup_dir.\ - replace("\r","").replace("\n","")+"\n") - backup_stdout = vmproc.stdin - - else: - # Prepare the backup target (local file) - backup_target = base_backup_dir + "/qubes-{0}".\ - format (time.strftime("%Y-%m-%d-%H%M%S")) - - # Create the target directory - if not os.path.exists (base_backup_dir): - raise QubesException( - "ERROR: the backup directory {0} does not exists".\ - format(base_backup_dir)) - - # If not APPVM, STDOUT is a local file - backup_stdout = open(backup_target,'wb') - - global blocks_backedup - blocks_backedup = 0 - progress = blocks_backedup * 11 / total_backup_sz - progress_callback(progress) - - import tempfile - feedback_file = tempfile.NamedTemporaryFile() - backup_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/backup_") - - # Tar with tapelength does not deals well with stdout (close stdout between - # two tapes) - # For this reason, we will use named pipes instead - if BACKUP_DEBUG: - print "Working in", backup_tmpdir - - backup_pipe = os.path.join(backup_tmpdir,"backup_pipe") - if BACKUP_DEBUG: - print "Creating pipe in:", backup_pipe - os.mkfifo(backup_pipe) - - if BACKUP_DEBUG: - print "Will backup:", files_to_backup - - # Setup worker to send encrypted data chunks to the backup_target - from multiprocessing import Queue,Process - class Send_Worker(Process): - def __init__(self, queue, base_dir, backup_stdout): - super(Send_Worker, self).__init__() - self.queue = queue - self.base_dir = base_dir - self.backup_stdout = backup_stdout - - def run(self): - if BACKUP_DEBUG: - print "Started sending thread" - - if BACKUP_DEBUG: - print "Moving to temporary dir", self.base_dir - os.chdir(self.base_dir) - - for filename in iter(self.queue.get,None): - if filename == "FINISHED": - break - - if BACKUP_DEBUG: - print "Sending file", filename - # This tar used for sending data out need to be as simple, as - # simple, as featureless as possible. It will not be - # verified before untaring. - tar_final_cmd = ["tar", "-cO", "--posix", - "-C", self.base_dir, filename] - final_proc = subprocess.Popen (tar_final_cmd, - stdin=subprocess.PIPE, stdout=self.backup_stdout) - final_proc.wait() - - # Delete the file as we don't need it anymore - if BACKUP_DEBUG: - print "Removing file", filename - os.remove(filename) - - if BACKUP_DEBUG: - print "Finished sending thread" - - def compute_progress(new_size, total_backup_sz): - global blocks_backedup - blocks_backedup += new_size - progress = blocks_backedup / float(total_backup_sz) - progress_callback(int(round(progress*100,2))) - - to_send = Queue(10) - send_proc = Send_Worker(to_send, backup_tmpdir, backup_stdout) - send_proc.start() - - for filename in files_to_backup: - if BACKUP_DEBUG: - print "Backing up", filename - - backup_tempfile = os.path.join(backup_tmpdir, - filename["subdir"], - os.path.basename(filename["path"])) - if BACKUP_DEBUG: - print "Using temporary location:", backup_tempfile - - # Ensure the temporary directory exists - - if not os.path.isdir(os.path.dirname(backup_tempfile)): - os.makedirs(os.path.dirname(backup_tempfile)) - - # The first tar cmd can use any complex feature as we want. Files will - # be verified before untaring this. - # Prefix the path in archive with filename["subdir"] to have it verified during untar - tar_cmdline = ["tar", "-Pc", '--sparse', - "-f", backup_pipe, - '--tape-length', str(100000), - '-C', os.path.dirname(filename["path"]), - '--xform', 's:^[a-z]:%s\\0:' % filename["subdir"], - os.path.basename(filename["path"]) - ] - - if BACKUP_DEBUG: - print " ".join(tar_cmdline) - - # Tips: Popen(bufsize=0) - # Pipe: tar-sparse | encryptor [| hmac] | tar | backup_target - # Pipe: tar-sparse [| hmac] | tar | backup_target - tar_sparse = subprocess.Popen (tar_cmdline, stdin=subprocess.PIPE) - - # Wait for compressor (tar) process to finish or for any error of other - # subprocesses - i = 0 - run_error = "paused" - running = [] - while run_error == "paused": - - pipe = open(backup_pipe,'rb') - - # Start HMAC - hmac = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], - stdin=subprocess.PIPE, stdout=subprocess.PIPE) - - # Prepare a first chunk - chunkfile = backup_tempfile + "." + "%03d" % i - i += 1 - chunkfile_p = open(chunkfile,'wb') - - if encrypt: - # Start encrypt - # If no cipher is provided, the data is forwarded unencrypted !!! - # Also note that the - encryptor = subprocess.Popen (["openssl", "enc", - "-e", "-aes-256-cbc", - "-pass", "pass:"+passphrase], - stdin=pipe, stdout=subprocess.PIPE) - run_error = wait_backup_feedback( - progress_callback=compute_progress, - in_stream=encryptor.stdout, - streamproc=encryptor, - backup_target=chunkfile_p, - total_backup_sz=total_backup_sz, - hmac=hmac, - vmproc=vmproc, - addproc=tar_sparse) - else: - run_error = wait_backup_feedback( - progress_callback=compute_progress, - in_stream=pipe, - streamproc=None, - backup_target=chunkfile_p, - total_backup_sz=total_backup_sz, - hmac=hmac, - vmproc=vmproc, - addproc=tar_sparse) - - chunkfile_p.close() - - if BACKUP_DEBUG: - print "Wait_backup_feedback returned:", run_error - - if len(run_error) > 0: - send_proc.terminate() - raise QubesException("Failed to perform backup: error with "+ \ - run_error) - - # Send the chunk to the backup target - to_send.put(os.path.relpath(chunkfile, backup_tmpdir)) - - # Close HMAC - hmac.stdin.close() - hmac.wait() - if BACKUP_DEBUG: - print "HMAC proc return code:", hmac.poll() - - # Write HMAC data next to the chunk file - hmac_data = hmac.stdout.read() - if BACKUP_DEBUG: - print "Writing hmac to", chunkfile+".hmac" - hmac_file = open(chunkfile+".hmac",'w') - hmac_file.write(hmac_data) - hmac_file.flush() - hmac_file.close() - - pipe.close() - - # Send the HMAC to the backup target - to_send.put(os.path.relpath(chunkfile, backup_tmpdir)+".hmac") - - if tar_sparse.poll() == None: - # Release the next chunk - if BACKUP_DEBUG: - print "Release next chunk for process:", tar_sparse.poll() - #tar_sparse.stdout = subprocess.PIPE - tar_sparse.stdin.write("\n") - run_error="paused" - else: - if BACKUP_DEBUG: - print "Finished tar sparse with error", tar_sparse.poll() - - to_send.put("FINISHED") - send_proc.join() - - if send_proc.exitcode != 0: - raise QubesException("Failed to send backup: error in the sending process") - - if vmproc: - if BACKUP_DEBUG: - print "VMProc1 proc return code:", vmproc.poll() - print "Sparse1 proc return code:", tar_sparse.poll() - vmproc.stdin.close() - -''' -' Wait for backup chunk to finish -' - Monitor all the processes (streamproc, hmac, vmproc, addproc) for errors -' - Copy stdout of streamproc to backup_target and hmac stdin if available -' - Compute progress based on total_backup_sz and send progress to -' progress_callback function -' - Returns if -' - one of the monitored processes error out (streamproc, hmac, vmproc, -' addproc), along with the processe that failed -' - all of the monitored processes except vmproc finished successfully -' (vmproc termination is controlled by the python script) -' - streamproc does not delivers any data anymore (return with the error -' "") -''' -def wait_backup_feedback(progress_callback, in_stream, streamproc, - backup_target, total_backup_sz, hmac=None, vmproc=None, addproc=None, - remove_trailing_bytes=0): - - buffer_size = 409600 - - run_error = None - run_count = 1 - blocks_backedup = 0 - while run_count > 0 and run_error == None: - - buffer = in_stream.read(buffer_size) - progress_callback(len(buffer), total_backup_sz) - - run_count = 0 - if hmac: - retcode=hmac.poll() - if retcode != None: - if retcode != 0: - run_error = "hmac" - else: - run_count += 1 - - if addproc: - retcode=addproc.poll() - #print "Tar proc status:", retcode - if retcode != None: - if retcode != 0: - run_error = "addproc" - else: - run_count += 1 - - if vmproc: - retcode = vmproc.poll() - if retcode != None: - if retcode != 0: - run_error = "VM" - #FIXME: ?? if BACKUP_DEBUG: - print vmproc.stdout.read() - else: - # VM should run until the end - pass - - if streamproc: - retcode=streamproc.poll() - if retcode != None: - if retcode != 0: - run_error = "streamproc" - elif retcode == 0 and len(buffer) <= 0: - return "" - else: - #print "INFO: last packet" - #if remove_trailing_bytes > 0: - # print buffer.encode("hex") - # buffer = buffer[:-remove_trailing_bytes] - # print buffer.encode("hex") - - backup_target.write(buffer) - - if hmac: - hmac.stdin.write(buffer) - - run_count += 1 - else: - #print "Process running:", len(buffer) - # Process still running - backup_target.write(buffer) - - if hmac: - hmac.stdin.write(buffer) - - run_count += 1 - - else: - if len(buffer) <= 0: - return "" - else: - backup_target.write(buffer) - - if hmac: - hmac.stdin.write(buffer) - - - return run_error - -def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, - vms_size, print_callback=None, error_callback=None, - progress_callback=None, encrypted=False, appvm=None): - - # Setup worker to extract encrypted data chunks to the restore dirs - from multiprocessing import Queue,Process - class Extract_Worker(Process): - def __init__(self, queue, base_dir, passphrase, encrypted, total_size, - print_callback, error_callback, progress_callback, vmproc=None): - super(Extract_Worker, self).__init__() - self.queue = queue - self.base_dir = base_dir - self.passphrase = passphrase - self.encrypted = encrypted - self.total_size = total_size - self.blocks_backedup = 0 - self.tar2_command = None - self.tar2_current_file = None - - self.print_callback = print_callback - self.error_callback = error_callback - self.progress_callback = progress_callback - - self.vmproc = vmproc - - self.restore_pipe = os.path.join(self.base_dir,"restore_pipe") - if BACKUP_DEBUG: - print "Creating pipe in:", self.restore_pipe - os.mkfifo(self.restore_pipe) - - def compute_progress(self, new_size, total_size): - self.blocks_backedup += new_size - progress = self.blocks_backedup / float(self.total_size) - progress = int(round(progress*100,2)) - self.progress_callback(progress) - - def run(self): - if BACKUP_DEBUG: - self.print_callback("Started sending thread") - self.print_callback("Moving to dir "+self.base_dir) - os.chdir(self.base_dir) - - for filename in iter(self.queue.get,None): - if filename == "FINISHED": - break - - if BACKUP_DEBUG: - self.print_callback("Extracting file "+filename) - - if filename.endswith('.000'): - # next file - if self.tar2_command != None: - if self.tar2_command.wait() != 0: - raise QubesException( - "ERROR: unable to extract files for {0}.".\ - format(self.tar2_current_file)) - else: - # Finished extracting the tar file - self.tar2_command = None - self.tar2_current_file = None - - tar2_cmdline = ['tar', - '-xMk%sf' % ("v" if BACKUP_DEBUG else ""), self.restore_pipe, - os.path.relpath(filename.rstrip('.000'))] - if BACKUP_DEBUG: - self.print_callback("Running command "+str(tar2_cmdline)) - self.tar2_command = subprocess.Popen(tar2_cmdline, - stdin=subprocess.PIPE, - stderr=(None if BACKUP_DEBUG else open('/dev/null', 'w'))) - else: - if BACKUP_DEBUG: - self.print_callback("Releasing next chunck") - self.tar2_command.stdin.write("\n") - self.tar2_command.stdin.flush() - self.tar2_current_file = filename - - pipe = open(self.restore_pipe,'wb') - common_args = { - 'backup_target': pipe, - 'total_backup_sz': self.total_size, - 'hmac': None, - 'vmproc': self.vmproc, - 'addproc': self.tar2_command - } - if self.encrypted: - # Start decrypt - encryptor = subprocess.Popen (["openssl", "enc", - "-d", "-aes-256-cbc", - "-pass", "pass:"+passphrase], - stdin=open(filename,'rb'), - stdout=subprocess.PIPE) - - run_error = wait_backup_feedback( - progress_callback=self.compute_progress, - in_stream=encryptor.stdout, streamproc=encryptor, - **common_args) - else: - run_error = wait_backup_feedback( - progress_callback=self.compute_progress, - in_stream=open(filename,"rb"), streamproc=None, - **common_args) - - pipe.close() - - # Delete the file as we don't need it anymore - if BACKUP_DEBUG: - self.print_callback("Removing file "+filename) - os.remove(filename) - - if self.tar2_command != None: - if self.tar2_command.wait() != 0: - raise QubesException( - "ERROR: unable to extract files for {0}.".\ - format(self.tar2_current_file)) - else: - # Finished extracting the tar file - self.tar2_command = None - - if BACKUP_DEBUG: - self.print_callback("Finished extracting thread") - - if progress_callback == None: - def progress_callback(data): - pass - - to_extract = Queue() - extract_proc = Extract_Worker(queue=to_extract, - base_dir=backup_tmpdir, - passphrase=passphrase, - encrypted=encrypted, - total_size=vms_size, - print_callback=print_callback, - error_callback=error_callback, - progress_callback=progress_callback) - extract_proc.start() - - if BACKUP_DEBUG: - print_callback("Working in temporary dir:"+backup_tmpdir) - print_callback(str(vms_size)+" bytes to restore") - - vmproc = None - if appvm != None: - # Prepare the backup target (Qubes service call) - backup_target = "QUBESRPC qubes.Restore dom0" - - # does the vm exist? - qvm_collection = QubesVmCollection() - qvm_collection.lock_db_for_reading() - qvm_collection.load() - - vm = qvm_collection.get_vm_by_name(appvm) - if vm is None or vm.qid not in qvm_collection: - raise QubesException("VM {0} does not exist".format(appvm)) - - qvm_collection.unlock_db() - - # If APPVM, STDOUT is a PIPE - vmproc = vm.run(command = backup_target, passio_popen = True) - vmproc.stdin.write(backup_dir.replace("\r","").replace("\n","")+"\n") - - # Send to tar2qfile the VMs that should be extracted - vmproc.stdin.write(" ".join(vms_dirs)+"\n") - - backup_stdin = vmproc.stdout - tar1_command = ['/usr/libexec/qubes/qfile-dom0-unpacker', - str(os.getuid()), backup_tmpdir, '-v'] - else: - backup_stdin = open(backup_dir,'rb') - - tar1_command = ['tar', - '-ix%sf' % ("v" if BACKUP_DEBUG else ""), backup_dir, - '-C', backup_tmpdir] + vms_dirs - - # Remove already processed qubes.xml.000, because qfile-dom0-unpacker will - # refuse to override files - os.unlink(os.path.join(backup_tmpdir,'qubes.xml.000')) - os.unlink(os.path.join(backup_tmpdir,'qubes.xml.000.hmac')) - tar1_env = os.environ.copy() - # TODO: add some safety margin? - tar1_env['UPDATES_MAX_BYTES'] = str(vms_size) - tar1_env['UPDATES_MAX_FILES'] = '0' - if BACKUP_DEBUG: - print_callback("Run command"+str(tar1_command)) - command = subprocess.Popen(tar1_command, - stdin=backup_stdin, - stdout=vmproc.stdin if vmproc else subprocess.PIPE, - stderr=subprocess.PIPE, - env=tar1_env) - - # qfile-dom0-unpacker output filelist on stderr (and have stdout connected - # to the VM), while tar output filelist on stdout - if appvm: - filelist_pipe = command.stderr - else: - filelist_pipe = command.stdout - - while True: - - filename = filelist_pipe.readline().strip(" \t\r\n") - - if BACKUP_DEBUG: - print_callback("Getting new file:"+filename) - - if not filename or filename=="EOF": - break - - hmacfile = filelist_pipe.readline().strip(" \t\r\n") - if BACKUP_DEBUG: - print_callback("Getting hmac:"+hmacfile) - - if hmacfile != filename + ".hmac": - raise QubesException( - "ERROR: expected hmac for {}, but got {}".\ - format(filename, hmacfile)) - - # skip qubes.xml after receiving its hmac to skip both of them - if filename == 'qubes.xml.000': - if BACKUP_DEBUG: - print_callback("Ignoring already processed qubes.xml") - continue - - if not any(map(lambda x: filename.startswith(x), vms_dirs)): - if BACKUP_DEBUG: - print_callback("Ignoring VM not selected for restore") - os.unlink(filename) - os.unlink(hmacfile) - continue - - if BACKUP_DEBUG: - print_callback("Verifying file "+filename) - - if BACKUP_DEBUG: - print os.path.join(backup_tmpdir, filename) - hmac_proc = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], - stdin=open(os.path.join(backup_tmpdir, filename),'rb'), - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = hmac_proc.communicate() - - if len(stderr) > 0: - raise QubesException("ERROR: verify file {0}: {1}".format((filename, stderr))) - else: - if BACKUP_DEBUG: - print_callback("Loading hmac for file"+filename) - hmac = load_hmac( - open(os.path.join(backup_tmpdir, - filename+".hmac"),'r').read()) - - if len(hmac) > 0 and load_hmac(stdout) == hmac: - if BACKUP_DEBUG: - print_callback("File verification OK -> Sending file " + \ - filename+" for extraction") - # Send the chunk to the backup target - to_extract.put(os.path.join(backup_tmpdir, filename)) - - else: - raise QubesException( - "ERROR: invalid hmac for file {0}: {1}. " \ - "Is the passphrase correct?".\ - format(filename, load_hmac(stdout))) - - if command.wait() != 0: - raise QubesException( - "ERROR: unable to read the qubes backup file {0} ({1}). " \ - "Is it really a backup?".format(backup_dir, command.wait())) - if vmproc: - if vmproc.wait() != 0: - raise QubesException( - "ERROR: unable to read the qubes backup {0} " \ - "because of a VM error: {1}".format( - backup_dir, vmproc.stderr.read())) - if BACKUP_DEBUG: - print "Extraction process status:", extract_proc.exitcode - - to_extract.put("FINISHED") - if BACKUP_DEBUG: - print_callback("Waiting for the extraction process to finish...") - extract_proc.join() - if BACKUP_DEBUG: - print_callback("Extraction process finished with code:" + \ - str(extract_proc.exitcode)) - if extract_proc.exitcode != 0: - raise QubesException( - "ERROR: unable to extract the qubes backup. " \ - "Check extracting process errors.") - -def backup_restore_set_defaults(options): - if 'use-default-netvm' not in options: - options['use-default-netvm'] = False - if 'use-none-netvm' not in options: - options['use-none-netvm'] = False - if 'use-default-template' not in options: - options['use-default-template'] = False - if 'dom0-home' not in options: - options['dom0-home'] = True - if 'replace-template' not in options: - options['replace-template'] = [] - - return options - -def load_hmac(hmac): - hmac = hmac.strip(" \t\r\n").split("=") - if len(hmac) > 1: - hmac = hmac[1].strip() - else: - raise QubesException("ERROR: invalid hmac file content") - - return hmac - -def backup_restore_header(restore_target, passphrase, - encrypt=False, appvm=None): - # Simulate dd if=backup_file count=10 | file - - # Simulate dd if=backup_file count=10 | gpg2 -d | tar xzv -O - # analysis = subprocess.Popen() - vmproc = None - - import tempfile - feedback_file = tempfile.NamedTemporaryFile() - backup_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/restore_") - - os.chdir(backup_tmpdir) - - # Tar with tapelength does not deals well with stdout (close stdout between - # two tapes) - # For this reason, we will use named pipes instead - if BACKUP_DEBUG: - print "Working in", backup_tmpdir - - - tar1_env = os.environ.copy() - if appvm != None: - # Prepare the backup target (Qubes service call) - restore_command = "QUBESRPC qubes.Restore dom0" - - # does the vm exist? - qvm_collection = QubesVmCollection() - qvm_collection.lock_db_for_reading() - qvm_collection.load() - - vm = qvm_collection.get_vm_by_name(appvm) - if vm is None or vm.qid not in qvm_collection: - raise QubesException("VM {0} does not exist".format(appvm)) - - qvm_collection.unlock_db() - - # If APPVM, STDOUT is a PIPE - vmproc = vm.run(command = restore_command, passio_popen = True, - passio_stderr = True) - vmproc.stdin.write(restore_target.replace("\r","").replace("\n","")+"\n") - - # Ask to tar2qfile to only extract qubes.xml.* - vmproc.stdin.write("qubes.xml\n") - - tar1_command = ['/usr/libexec/qubes/qfile-dom0-unpacker', - str(os.getuid()), backup_tmpdir, '-v'] - # Ask qfile-dom0-unpacker to extract only qubes.xml.000 and qubes.xml.000.hmac - tar1_env['UPDATES_MAX_FILES'] = '2' - else: - # Check source file - if not os.path.exists (restore_target): - raise QubesException( - "ERROR: the backup directory {0} does not exists".\ - format(restore_target)) - - # TODO: perhaps pass only first 40kB here? Tar uses seek to skip files, - # so not a big problem, but still it might save some time - tar1_command = ['tar', - '-xi%sf' % ("v" if BACKUP_DEBUG else ""), restore_target, - '-C', backup_tmpdir, 'qubes.xml.000', 'qubes.xml.000.hmac'] - - command = subprocess.Popen(tar1_command, - stdin=vmproc.stdout if vmproc else None, - stdout=vmproc.stdin if vmproc else subprocess.PIPE, - stderr=subprocess.PIPE, - env=tar1_env) - - if vmproc and vmproc.poll() != None: - error = vmproc.stderr.read() - if BACKUP_DEBUG: - print error - print vmproc.poll(), command.poll() - raise QubesException( - "ERROR: Immediate VM error while retrieving backup headers:{0}".\ - format(error)) - - filename = "qubes.xml.000" - - command.wait() - - if vmproc and vmproc.wait() != 0: - error = vmproc.stderr.read() - if BACKUP_DEBUG: - print error - print vmproc.poll(), command.poll() - raise QubesException( - "ERROR: AppVM error retrieving backup headers: {0}".\ - format(error)) - elif command.returncode not in [0,-15,122]: - error = command.stderr.read() - if BACKUP_DEBUG: - print error - print vmproc.poll(), command.poll() - raise QubesException("ERROR: retrieving backup headers: {0}".\ - format(error)) - - if not os.path.exists(os.path.join(backup_tmpdir, filename+".hmac")): - raise QubesException("ERROR: header not extracted correctly: {0}".\ - format(os.path.join(backup_tmpdir, filename+".hmac"))) - - if vmproc and vmproc.poll() == None: - vmproc.terminate() - vmproc.wait() - - if BACKUP_DEBUG: - print "Loading hmac for file", filename - hmac = load_hmac( - open(os.path.join(backup_tmpdir, filename+".hmac"),'r').read()) - - if BACKUP_DEBUG: - print "Successfully retrieved headers" - - if BACKUP_DEBUG: - print "Verifying file", filename - hmac_proc = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], - stdin=open(os.path.join(backup_tmpdir, filename),'rb'), - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = hmac_proc.communicate() - if len(stderr) > 0: - raise QubesException("ERROR: verify file {0}: {1}".format((filename, stderr))) - else: - if len(hmac) > 0 and load_hmac(stdout) == hmac: - if BACKUP_DEBUG: - print "File verification OK -> Extracting archive", filename - if encrypt: - if BACKUP_DEBUG: - print "Starting decryption process" - encryptor = subprocess.Popen (["openssl", "enc", - "-d", "-aes-256-cbc", - "-pass", "pass:"+passphrase], - stdin=open(os.path.join(backup_tmpdir, filename),'rb'), - stdout=subprocess.PIPE) - tarhead_command = subprocess.Popen(['tar', - '-xM%s' % ("v" if BACKUP_DEBUG else "")], - stdin=encryptor.stdout) - else: - if BACKUP_DEBUG: - print "No decryption process required" - encryptor = None - tarhead_command = subprocess.Popen(['tar', - '-xM%sf' % ("v" if BACKUP_DEBUG else ""), - os.path.join(backup_tmpdir, filename)]) - - if encryptor: - if encryptor.wait() != 0: - raise QubesException( - "ERROR: unable to decrypt file {0}. " \ - "Bad password or unencrypted archive?".\ - format(filename)) - if tarhead_command.wait() != 0: - raise QubesException( - "ERROR: unable to extract the qubes.xml file. " \ - "Is archive encrypted?") - - return (backup_tmpdir, "qubes.xml") - else: - raise QubesException( - "ERROR: unable to verify the qubes.xml file. " \ - "Is the passphrase correct?") - - return None - -def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, - host_collection = None, encrypt=False, appvm=None): - # Defaults - backup_restore_set_defaults(options) - - #### Private functions begin - def is_vm_included_in_backup (backup_dir, vm): - if vm.backup_content: - return True - else: - return False - - def find_template_name(template, replaces): - rx_replace = re.compile("(.*):(.*)") - for r in replaces: - m = rx_replace.match(r) - if m.group(1) == template: - return m.group(2) - - return template - #### Private functions end - if BACKUP_DEBUG: - print "Loading file", qubes_xml - backup_collection = QubesVmCollection(store_filename = qubes_xml) - backup_collection.lock_db_for_reading() - backup_collection.load() - - if host_collection is None: - host_collection = QubesVmCollection() - host_collection.lock_db_for_reading() - host_collection.load() - host_collection.unlock_db() - - backup_vms_list = [vm for vm in backup_collection.values()] - host_vms_list = [vm for vm in host_collection.values()] - vms_to_restore = {} - - there_are_conflicting_vms = False - there_are_missing_templates = False - there_are_missing_netvms = False - dom0_username_mismatch = False - restore_home = False - # ... and the actual data - for vm in backup_vms_list: - if vm.qid == 0: - # Handle dom0 as special case later - continue - if is_vm_included_in_backup (backup_dir, vm): - if BACKUP_DEBUG: - print vm.name,"is included in backup" - - vms_to_restore[vm.name] = {} - vms_to_restore[vm.name]['vm'] = vm; - if 'exclude' in options.keys(): - vms_to_restore[vm.name]['excluded'] = vm.name in options['exclude'] - vms_to_restore[vm.name]['good-to-go'] = False - - if host_collection.get_vm_by_name (vm.name) is not None: - vms_to_restore[vm.name]['already-exists'] = True - vms_to_restore[vm.name]['good-to-go'] = False - - if vm.template is None: - vms_to_restore[vm.name]['template'] = None - else: - templatevm_name = find_template_name(vm.template.name, options['replace-template']) - vms_to_restore[vm.name]['template'] = templatevm_name - template_vm_on_host = host_collection.get_vm_by_name (templatevm_name) - - # No template on the host? - if not ((template_vm_on_host is not None) and template_vm_on_host.is_template()): - # Maybe the (custom) template is in the backup? - template_vm_on_backup = backup_collection.get_vm_by_name (templatevm_name) - if template_vm_on_backup is None or not \ - (is_vm_included_in_backup(backup_dir, template_vm_on_backup) and \ - template_vm_on_backup.is_template()): - if options['use-default-template']: - vms_to_restore[vm.name]['orig-template'] = templatevm_name - vms_to_restore[vm.name]['template'] = host_collection.get_default_template().name - else: - vms_to_restore[vm.name]['missing-template'] = True - vms_to_restore[vm.name]['good-to-go'] = False - - if vm.netvm is None: - vms_to_restore[vm.name]['netvm'] = None - else: - netvm_name = vm.netvm.name - vms_to_restore[vm.name]['netvm'] = netvm_name - # Set to None to not confuse QubesVm object from backup - # collection with host collection (further in clone_attrs). Set - # directly _netvm to suppress setter action, especially - # modifying firewall - vm._netvm = None - - netvm_on_host = host_collection.get_vm_by_name (netvm_name) - - # No netvm on the host? - if not ((netvm_on_host is not None) and netvm_on_host.is_netvm()): - - # Maybe the (custom) netvm is in the backup? - netvm_on_backup = backup_collection.get_vm_by_name (netvm_name) - if not ((netvm_on_backup is not None) and \ - netvm_on_backup.is_netvm() and \ - is_vm_included_in_backup(backup_dir, netvm_on_backup)): - if options['use-default-netvm']: - vms_to_restore[vm.name]['netvm'] = host_collection.get_default_netvm().name - vm.uses_default_netvm = True - elif options['use-none-netvm']: - vms_to_restore[vm.name]['netvm'] = None - else: - vms_to_restore[vm.name]['missing-netvm'] = True - vms_to_restore[vm.name]['good-to-go'] = False - - if 'good-to-go' not in vms_to_restore[vm.name].keys(): - vms_to_restore[vm.name]['good-to-go'] = True - - # ...and dom0 home - if options['dom0-home'] and \ - is_vm_included_in_backup(backup_dir, backup_collection[0]): - vm = backup_collection[0] - vms_to_restore['dom0'] = {} - vms_to_restore['dom0']['subdir'] = vm.backup_path - vms_to_restore['dom0']['size'] = vm.backup_size - local_user = grp.getgrnam('qubes').gr_mem[0] - - dom0_home = vm.backup_path - - vms_to_restore['dom0']['username'] = os.path.basename(dom0_home) - if vms_to_restore['dom0']['username'] != local_user: - vms_to_restore['dom0']['username-mismatch'] = True - if not options['ignore-dom0-username-mismatch']: - vms_to_restore['dom0']['good-to-go'] = False - - if 'good-to-go' not in vms_to_restore['dom0']: - vms_to_restore['dom0']['good-to-go'] = True - - return vms_to_restore - -def backup_restore_print_summary(restore_info, print_callback = print_stdout): - fields = { - "qid": {"func": "vm.qid"}, - - "name": {"func": "('[' if vm.is_template() else '')\ - + ('{' if vm.is_netvm() else '')\ - + vm.name \ - + (']' if vm.is_template() else '')\ - + ('}' if vm.is_netvm() else '')"}, - - "type": {"func": "'Tpl' if vm.is_template() else \ - 'HVM' if vm.type == 'HVM' else \ - vm.type.replace('VM','')"}, - - "updbl" : {"func": "'Yes' if vm.updateable else ''"}, - - "template": {"func": "'n/a' if vm.is_template() or vm.template is None else\ - vm_info['template']"}, - - "netvm": {"func": "'n/a' if vm.is_netvm() and not vm.is_proxyvm() else\ - ('*' if vm.uses_default_netvm else '') +\ - vm_info['netvm'] if vm_info['netvm'] is not None else '-'"}, - - "label" : {"func" : "vm.label.name"}, - } - - fields_to_display = ["name", "type", "template", "updbl", "netvm", "label" ] - - # First calculate the maximum width of each field we want to display - total_width = 0; - for f in fields_to_display: - fields[f]["max_width"] = len(f) - for vm_info in restore_info.values(): - if 'vm' in vm_info.keys(): - vm = vm_info['vm'] - l = len(str(eval(fields[f]["func"]))) - if l > fields[f]["max_width"]: - fields[f]["max_width"] = l - total_width += fields[f]["max_width"] - - print_callback("") - print_callback("The following VMs are included in the backup:") - print_callback("") - - # Display the header - s = "" - for f in fields_to_display: - fmt="{{0:-^{0}}}-+".format(fields[f]["max_width"] + 1) - s += fmt.format('-') - print_callback(s) - s = "" - for f in fields_to_display: - fmt="{{0:>{0}}} |".format(fields[f]["max_width"] + 1) - s += fmt.format(f) - print_callback(s) - s = "" - for f in fields_to_display: - fmt="{{0:-^{0}}}-+".format(fields[f]["max_width"] + 1) - s += fmt.format('-') - print_callback(s) - - for vm_info in restore_info.values(): - # Skip non-VM here - if not 'vm' in vm_info: - continue - vm = vm_info['vm'] - s = "" - for f in fields_to_display: - fmt="{{0:>{0}}} |".format(fields[f]["max_width"] + 1) - s += fmt.format(eval(fields[f]["func"])) - - if 'excluded' in vm_info and vm_info['excluded']: - s += " <-- Excluded from restore" - elif 'already-exists' in vm_info: - s += " <-- A VM with the same name already exists on the host!" - elif 'missing-template' in vm_info: - s += " <-- No matching template on the host or in the backup found!" - elif 'missing-netvm' in vm_info: - s += " <-- No matching netvm on the host or in the backup found!" - elif 'orig-template' in vm_info: - s += " <-- Original template was '%s'" % (vm_info['orig-template']) - - print_callback(s) - - if 'dom0' in restore_info.keys(): - s = "" - for f in fields_to_display: - fmt="{{0:>{0}}} |".format(fields[f]["max_width"] + 1) - if f == "name": - s += fmt.format("Dom0") - elif f == "type": - s += fmt.format("Home") - else: - s += fmt.format("") - if 'username-mismatch' in restore_info['dom0']: - s += " <-- username in backup and dom0 mismatch" - - print_callback(s) - -def backup_restore_do(backup_dir, restore_tmpdir, passphrase, restore_info, - host_collection = None, print_callback = print_stdout, - error_callback = print_stderr, progress_callback = None, - encrypted=False, appvm=None): - - lock_obtained = False - if host_collection is None: - host_collection = QubesVmCollection() - host_collection.lock_db_for_writing() - host_collection.load() - lock_obtained = True - - # Perform VM restoration in backup order - vms_dirs = [] - vms_size = 0 - vms = {} - for vm_info in restore_info.values(): - if not vm_info['good-to-go']: - continue - if 'vm' not in vm_info: - continue - vm = vm_info['vm'] - vms_size += vm.backup_size - vms_dirs.append(vm.backup_path) - vms[vm.name] = vm - - if 'dom0' in restore_info.keys() and restore_info['dom0']['good-to-go']: - vms_dirs.append('dom0-home') - vms_size += restore_info['dom0']['size'] - - restore_vm_dirs (backup_dir, - restore_tmpdir, - passphrase=passphrase, - vms_dirs=vms_dirs, - vms=vms, - vms_size=vms_size, - print_callback=print_callback, - error_callback=error_callback, - progress_callback=progress_callback, - encrypted=encrypted, - appvm=appvm) - - # Add VM in right order - for (vm_class_name, vm_class) in sorted(QubesVmClasses.items(), - key=lambda _x: _x[1].load_order): - for vm_info in restore_info.values(): - if not vm_info['good-to-go']: - continue - if 'vm' not in vm_info: - continue - vm = vm_info['vm'] - if not vm.__class__ == vm_class: - continue - print_callback("-> Restoring {type} {0}...".format(vm.name, type=vm_class_name)) - retcode = subprocess.call (["mkdir", "-p", os.path.dirname(vm.dir_path)]) - if retcode != 0: - error_callback("*** Cannot create directory: {0}?!".format(dest_dir)) - error_callback("Skipping...") - continue - - template = None - if vm.template is not None: - template_name = vm_info['template'] - template = host_collection.get_vm_by_name(template_name) - - new_vm = None - - try: - new_vm = host_collection.add_new_vm(vm_class_name, name=vm.name, - conf_file=vm.conf_file, - dir_path=vm.dir_path, - template=template, - installed_by_rpm=False) - - shutil.move(os.path.join(restore_tmpdir, vm.backup_path), - new_vm.dir_path) - - new_vm.verify_files() - except Exception as err: - error_callback("ERROR: {0}".format(err)) - error_callback("*** Skipping VM: {0}".format(vm.name)) - if new_vm: - host_collection.pop(new_vm.qid) - continue - - try: - new_vm.clone_attrs(vm) - except Exception as err: - error_callback("ERROR: {0}".format(err)) - error_callback("*** Some VM property will not be restored") - - try: - new_vm.appmenus_create(verbose=True) - except Exception as err: - error_callback("ERROR during appmenu restore: {0}".format(err)) - error_callback("*** VM '{0}' will not have appmenus".format(vm.name)) - - # Set network dependencies - only non-default netvm setting - for vm_info in restore_info.values(): - if not vm_info['good-to-go']: - continue - if 'vm' not in vm_info: - continue - vm = vm_info['vm'] - host_vm = host_collection.get_vm_by_name(vm.name) - if host_vm is None: - # Failed/skipped VM - continue - - if not vm.uses_default_netvm: - host_vm.netvm = host_collection.get_vm_by_name (vm_info['netvm']) if vm_info['netvm'] is not None else None - - host_collection.save() - if lock_obtained: - host_collection.unlock_db() - - # ... and dom0 home as last step - if 'dom0' in restore_info.keys() and restore_info['dom0']['good-to-go']: - backup_path = restore_info['dom0']['subdir'] - local_user = grp.getgrnam('qubes').gr_mem[0] - home_dir = pwd.getpwnam(local_user).pw_dir - backup_dom0_home_dir = os.path.join(restore_tmpdir, backup_path) - restore_home_backupdir = "home-pre-restore-{0}".format (time.strftime("%Y-%m-%d-%H%M%S")) - - print_callback("-> Restoring home of user '{0}'...".format(local_user)) - print_callback("--> Existing files/dirs backed up in '{0}' dir".format(restore_home_backupdir)) - os.mkdir(home_dir + '/' + restore_home_backupdir) - for f in os.listdir(backup_dom0_home_dir): - home_file = home_dir + '/' + f - if os.path.exists(home_file): - os.rename(home_file, home_dir + '/' + restore_home_backupdir + '/' + f) - shutil.move(backup_dom0_home_dir + '/' + f, home_file) - retcode = subprocess.call(['sudo', 'chown', '-R', local_user, home_dir]) - if retcode != 0: - error_callback("*** Error while setting home directory owner") - # vim:sw=4:et: diff --git a/qvm-tools/qvm-backup b/qvm-tools/qvm-backup index 601bd009..108875eb 100755 --- a/qvm-tools/qvm-backup +++ b/qvm-tools/qvm-backup @@ -22,7 +22,8 @@ from qubes.qubes import QubesVmCollection from qubes.qubes import QubesException -from qubes.qubesutils import backup_prepare, backup_do_copy, size_to_human +from qubes.backup import backup_prepare, backup_do +from qubes.qubesutils import size_to_human from optparse import OptionParser import os import sys @@ -80,6 +81,8 @@ def main(): files_to_backup, 0) if not options.appvm: + appvm = None + stat = os.statvfs(base_backup_dir) backup_fs_free_sz = stat.f_bsize * stat.f_bavail print @@ -88,6 +91,11 @@ def main(): exit(1) print "-> Available space: {0}".format(size_to_human(backup_fs_free_sz)) + else: + appvm = qvm_collection.get_vm_by_name(options.appvm) + if appvm is None: + print >>sys.stderr, "ERROR: VM {0} does not exist".format(options.appvm) + exit(1) prompt = raw_input ("Do you want to proceed? [y/N] ") if not (prompt == "y" or prompt == "Y"): @@ -96,10 +104,10 @@ def main(): passphrase = getpass.getpass("Please enter the pass phrase that will be used to encrypt/verify the backup: ") try: - backup_do_copy(base_backup_dir, files_to_backup, passphrase, + backup_do(base_backup_dir, files_to_backup, passphrase, progress_callback=print_progress, encrypt=options.encrypt, - appvm=options.appvm) + appvm=appvm) except QubesException as e: print >>sys.stderr, "ERROR: %s" % str(e) exit(1) diff --git a/qvm-tools/qvm-backup-restore b/qvm-tools/qvm-backup-restore index 6ac22088..506646f6 100755 --- a/qvm-tools/qvm-backup-restore +++ b/qvm-tools/qvm-backup-restore @@ -22,10 +22,10 @@ from qubes.qubes import QubesVmCollection from qubes.qubes import QubesException -from qubes.qubesutils import backup_restore_header -from qubes.qubesutils import backup_restore_prepare -from qubes.qubesutils import backup_restore_print_summary -from qubes.qubesutils import backup_restore_do +from qubes.backup import backup_restore_header +from qubes.backup import backup_restore_prepare +from qubes.backup import backup_restore_print_summary +from qubes.backup import backup_restore_do from optparse import OptionParser import os @@ -95,15 +95,28 @@ def main(): if options.exclude: restore_options['exclude'] = options.exclude + appvm = None + if options.appvm is not None: + appvm = qvm_collection.get_vm_by_name(options.appvm) + if appvm is None: + print >>sys.stderr, "ERROR: VM {0} does not exist".format(options.appvm) + exit(1) passphrase = getpass.getpass("Please enter the pass phrase that will be used to decrypt/verify the backup: ") print >> sys.stderr, "Checking backup content..." - restore_tmpdir,qubes_xml = backup_restore_header(backup_dir, passphrase, options.decrypt, appvm=options.appvm) + restore_tmpdir,qubes_xml = backup_restore_header(backup_dir, passphrase, encrypted=options.decrypt, appvm=options.appvm) restore_info = None try: - restore_info = backup_restore_prepare(backup_dir,os.path.join(restore_tmpdir, qubes_xml), passphrase, options=restore_options, host_collection=host_collection, encrypt=options.decrypt, appvm=options.appvm) + restore_info = backup_restore_prepare( + backup_dir, + os.path.join(restore_tmpdir, qubes_xml), + passphrase, + options=restore_options, + host_collection=host_collection, + encrypt=options.decrypt, + appvm=appvm) except QubesException as e: print >> sys.stderr, "ERROR: %s" % str(e) exit(1) @@ -147,7 +160,7 @@ def main(): print "The above VMs will be copied and added to your system." print "Exisiting VMs will not be removed." - + if there_are_missing_templates: print >> sys.stderr, "*** One or more template VM is missing on the host! ***" if not (options.skip_broken or options.ignore_missing): @@ -192,7 +205,13 @@ def main(): exit (0) - backup_restore_do(backup_dir,restore_tmpdir, passphrase, restore_info, host_collection=host_collection, encrypted=options.decrypt, appvm=options.appvm) + backup_restore_do(backup_dir, + restore_tmpdir, + passphrase, + restore_info, + host_collection=host_collection, + encrypted=options.decrypt, + appvm=appvm) host_collection.unlock_db() diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec index a31aaac1..b5d56b84 100644 --- a/rpm_spec/core-dom0.spec +++ b/rpm_spec/core-dom0.spec @@ -109,6 +109,8 @@ cp core/qubesutils.py $RPM_BUILD_ROOT%{python_sitearch}/qubes cp core/qubesutils.py[co] $RPM_BUILD_ROOT%{python_sitearch}/qubes cp core/guihelpers.py $RPM_BUILD_ROOT%{python_sitearch}/qubes cp core/guihelpers.py[co] $RPM_BUILD_ROOT%{python_sitearch}/qubes +cp core/backup.py $RPM_BUILD_ROOT%{python_sitearch}/qubes +cp core/backup.py[co] $RPM_BUILD_ROOT%{python_sitearch}/qubes cp core/__init__.py $RPM_BUILD_ROOT%{python_sitearch}/qubes cp core/__init__.py[co] $RPM_BUILD_ROOT%{python_sitearch}/qubes cp qmemman/qmemman*py $RPM_BUILD_ROOT%{python_sitearch}/qubes @@ -272,6 +274,9 @@ fi %{python_sitearch}/qubes/guihelpers.py %{python_sitearch}/qubes/guihelpers.pyc %{python_sitearch}/qubes/guihelpers.pyo +%{python_sitearch}/qubes/backup.py +%{python_sitearch}/qubes/backup.pyc +%{python_sitearch}/qubes/backup.pyo %{python_sitearch}/qubes/__init__.py %{python_sitearch}/qubes/__init__.pyc %{python_sitearch}/qubes/__init__.pyo