diff --git a/core/backup.py b/core/backup.py deleted file mode 100644 index 7ba8808f..00000000 --- a/core/backup.py +++ /dev/null @@ -1,2308 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# -# The Qubes OS Project, http://www.qubes-os.org -# -# Copyright (C) 2013-2015 Marek Marczykowski-Górecki -# -# Copyright (C) 2013 Olivier Médoc -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# -from __future__ import unicode_literals -from qubes import QubesException, QubesVmCollection -from qubes import QubesVmClasses -from qubes import system_path, vm_files -from qubesutils import size_to_human, print_stdout, print_stderr, get_disk_usage -import sys -import os -import fcntl -import subprocess -import re -import shutil -import tempfile -import time -import grp -import pwd -import errno -import datetime -from multiprocessing import Queue, Process - -BACKUP_DEBUG = False - -HEADER_FILENAME = 'backup-header' -DEFAULT_CRYPTO_ALGORITHM = 'aes-256-cbc' -DEFAULT_HMAC_ALGORITHM = 'SHA512' -DEFAULT_COMPRESSION_FILTER = 'gzip' -CURRENT_BACKUP_FORMAT_VERSION = '3' -# Maximum size of error message get from process stderr (including VM process) -MAX_STDERR_BYTES = 1024 -# header + qubes.xml max size -HEADER_QUBES_XML_MAX_SIZE = 1024 * 1024 - -# global state for backup_cancel() -running_backup_operation = None - - -class BackupOperationInfo: - def __init__(self): - self.canceled = False - self.processes_to_kill_on_cancel = [] - self.tmpdir_to_remove = None - - -class BackupCanceledError(QubesException): - def __init__(self, msg, tmpdir=None): - super(BackupCanceledError, self).__init__(msg) - self.tmpdir = tmpdir - - -class BackupHeader: - version = 'version' - encrypted = 'encrypted' - compressed = 'compressed' - compression_filter = 'compression-filter' - crypto_algorithm = 'crypto-algorithm' - hmac_algorithm = 'hmac-algorithm' - bool_options = ['encrypted', 'compressed'] - int_options = ['version'] - - -def file_to_backup(file_path, subdir=None): - sz = get_disk_usage(file_path) - - if subdir is None: - abs_file_path = os.path.abspath(file_path) - abs_base_dir = os.path.abspath(system_path["qubes_base_dir"]) + '/' - abs_file_dir = os.path.dirname(abs_file_path) + '/' - (nothing, directory, subdir) = abs_file_dir.partition(abs_base_dir) - assert nothing == "" - assert directory == abs_base_dir - else: - if len(subdir) > 0 and not subdir.endswith('/'): - subdir += '/' - return [{"path": file_path, "size": sz, "subdir": subdir}] - - -def backup_cancel(): - """ - Cancel currently running backup/restore operation - - @return: True if any operation was signaled - """ - if running_backup_operation is None: - return False - - running_backup_operation.canceled = True - for proc in running_backup_operation.processes_to_kill_on_cancel: - try: - proc.terminate() - except: - pass - return True - - -def backup_prepare(vms_list=None, exclude_list=None, - print_callback=print_stdout, hide_vm_names=True): - """ - If vms = None, include all (sensible) VMs; - exclude_list is always applied - """ - files_to_backup = file_to_backup(system_path["qubes_store_filename"]) - - if exclude_list is None: - exclude_list = [] - - qvm_collection = QubesVmCollection() - qvm_collection.lock_db_for_writing() - qvm_collection.load() - - if vms_list is None: - all_vms = [vm for vm in qvm_collection.values()] - selected_vms = [vm for vm in all_vms if vm.include_in_backups] - appvms_to_backup = [vm for vm in selected_vms if - vm.is_appvm() and not vm.internal] - netvms_to_backup = [vm for vm in selected_vms if - vm.is_netvm() and not vm.qid == 0] - template_vms_worth_backingup = [vm for vm in selected_vms if ( - vm.is_template() and vm.include_in_backups)] - dom0 = [qvm_collection[0]] - - vms_list = appvms_to_backup + netvms_to_backup + \ - template_vms_worth_backingup + dom0 - - vms_for_backup = vms_list - # Apply exclude list - if exclude_list: - vms_for_backup = [vm for vm in vms_list if vm.name not in exclude_list] - - there_are_running_vms = False - - fields_to_display = [ - {"name": "VM", "width": 16}, - {"name": "type", "width": 12}, - {"name": "size", "width": 12} - ] - - # Display the header - s = "" - for f in fields_to_display: - fmt = "{{0:-^{0}}}-+".format(f["width"] + 1) - s += fmt.format('-') - print_callback(s) - s = "" - for f in fields_to_display: - fmt = "{{0:>{0}}} |".format(f["width"] + 1) - s += fmt.format(f["name"]) - print_callback(s) - s = "" - for f in fields_to_display: - fmt = "{{0:-^{0}}}-+".format(f["width"] + 1) - s += fmt.format('-') - print_callback(s) - - files_to_backup_index = 0 - for vm in vms_for_backup: - if vm.is_template(): - # handle templates later - continue - if vm.qid == 0: - # handle dom0 later - continue - - if hide_vm_names: - subdir = 'vm%d/' % vm.qid - else: - subdir = None - - if vm.private_img is not None: - files_to_backup += file_to_backup(vm.private_img, subdir) - - if vm.is_appvm(): - files_to_backup += file_to_backup(vm.icon_path, subdir) - if vm.updateable: - if os.path.exists(vm.dir_path + "/apps.templates"): - # template - files_to_backup += file_to_backup( - vm.dir_path + "/apps.templates", subdir) - else: - # standaloneVM - files_to_backup += file_to_backup(vm.dir_path + "/apps", subdir) - - if os.path.exists(vm.dir_path + "/kernels"): - files_to_backup += file_to_backup(vm.dir_path + "/kernels", - subdir) - if os.path.exists(vm.firewall_conf): - files_to_backup += file_to_backup(vm.firewall_conf, subdir) - if 'appmenus_whitelist' in vm_files and \ - os.path.exists(os.path.join(vm.dir_path, - vm_files['appmenus_whitelist'])): - files_to_backup += file_to_backup( - os.path.join(vm.dir_path, vm_files['appmenus_whitelist']), - subdir) - - if vm.updateable: - files_to_backup += file_to_backup(vm.root_img, subdir) - - s = "" - fmt = "{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1) - s += fmt.format(vm.name) - - fmt = "{{0:>{0}}} |".format(fields_to_display[1]["width"] + 1) - if vm.is_netvm(): - s += fmt.format("NetVM" + (" + Sys" if vm.updateable else "")) - else: - s += fmt.format("AppVM" + (" + Sys" if vm.updateable else "")) - - vm_size = reduce(lambda x, y: x + y["size"], - files_to_backup[files_to_backup_index:], - 0) - files_to_backup_index = len(files_to_backup) - - fmt = "{{0:>{0}}} |".format(fields_to_display[2]["width"] + 1) - s += fmt.format(size_to_human(vm_size)) - - if vm.is_running(): - s += " <-- The VM is running, please shut it down before proceeding " \ - "with the backup!" - there_are_running_vms = True - - print_callback(s) - - for vm in vms_for_backup: - if not vm.is_template(): - # already handled - continue - if vm.qid == 0: - # handle dom0 later - continue - vm_sz = vm.get_disk_utilization() - if hide_vm_names: - template_subdir = 'vm%d/' % vm.qid - else: - template_subdir = os.path.relpath( - vm.dir_path, - system_path["qubes_base_dir"]) + '/' - template_to_backup = [{"path": vm.dir_path + '/.', - "size": vm_sz, - "subdir": template_subdir}] - files_to_backup += template_to_backup - - s = "" - fmt = "{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1) - s += fmt.format(vm.name) - - fmt = "{{0:>{0}}} |".format(fields_to_display[1]["width"] + 1) - s += fmt.format("Template VM") - - fmt = "{{0:>{0}}} |".format(fields_to_display[2]["width"] + 1) - s += fmt.format(size_to_human(vm_sz)) - - if vm.is_running(): - s += " <-- The VM is running, please shut it down before proceeding " \ - "with the backup!" - there_are_running_vms = True - - print_callback(s) - - # Initialize backup flag on all VMs - vms_for_backup_qid = [vm.qid for vm in vms_for_backup] - for vm in qvm_collection.values(): - vm.backup_content = False - if vm.qid == 0: - # handle dom0 later - continue - - if vm.qid in vms_for_backup_qid: - vm.backup_content = True - vm.backup_size = vm.get_disk_utilization() - if hide_vm_names: - vm.backup_path = 'vm%d' % vm.qid - else: - vm.backup_path = os.path.relpath(vm.dir_path, - system_path["qubes_base_dir"]) - - # Dom0 user home - if 0 in vms_for_backup_qid: - local_user = grp.getgrnam('qubes').gr_mem[0] - home_dir = pwd.getpwnam(local_user).pw_dir - # Home dir should have only user-owned files, so fix it now to prevent - # permissions problems - some root-owned files can left after - # 'sudo bash' and similar commands - subprocess.check_call(['sudo', 'chown', '-R', local_user, home_dir]) - - home_sz = get_disk_usage(home_dir) - home_to_backup = [ - {"path": home_dir, "size": home_sz, "subdir": 'dom0-home/'}] - files_to_backup += home_to_backup - - vm = qvm_collection[0] - vm.backup_content = True - vm.backup_size = home_sz - vm.backup_path = os.path.join('dom0-home', os.path.basename(home_dir)) - - s = "" - fmt = "{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1) - s += fmt.format('Dom0') - - fmt = "{{0:>{0}}} |".format(fields_to_display[1]["width"] + 1) - s += fmt.format("User home") - - fmt = "{{0:>{0}}} |".format(fields_to_display[2]["width"] + 1) - s += fmt.format(size_to_human(home_sz)) - - print_callback(s) - - qvm_collection.save() - # FIXME: should be after backup completed - qvm_collection.unlock_db() - - total_backup_sz = 0 - for f in files_to_backup: - total_backup_sz += f["size"] - - s = "" - for f in fields_to_display: - fmt = "{{0:-^{0}}}-+".format(f["width"] + 1) - s += fmt.format('-') - print_callback(s) - - s = "" - fmt = "{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1) - s += fmt.format("Total size:") - fmt = "{{0:>{0}}} |".format( - fields_to_display[1]["width"] + 1 + 2 + fields_to_display[2][ - "width"] + 1) - s += fmt.format(size_to_human(total_backup_sz)) - print_callback(s) - - s = "" - for f in fields_to_display: - fmt = "{{0:-^{0}}}-+".format(f["width"] + 1) - s += fmt.format('-') - print_callback(s) - - vms_not_for_backup = [vm.name for vm in qvm_collection.values() - if not vm.backup_content] - print_callback("VMs not selected for backup: %s" % " ".join( - vms_not_for_backup)) - - if there_are_running_vms: - raise QubesException("Please shutdown all VMs before proceeding.") - - for fileinfo in files_to_backup: - assert len(fileinfo["subdir"]) == 0 or fileinfo["subdir"][-1] == '/', \ - "'subdir' must ends with a '/': %s" % unicode(fileinfo) - - return files_to_backup - - -class SendWorker(Process): - def __init__(self, queue, base_dir, backup_stdout): - super(SendWorker, self).__init__() - self.queue = queue - self.base_dir = base_dir - self.backup_stdout = backup_stdout - - def run(self): - if BACKUP_DEBUG: - print "Started sending thread" - - if BACKUP_DEBUG: - print "Moving to temporary dir", self.base_dir - os.chdir(self.base_dir) - - for filename in iter(self.queue.get, None): - if filename == "FINISHED" or filename == "ERROR": - break - - if BACKUP_DEBUG: - print "Sending file", filename - # This tar used for sending data out need to be as simple, as - # simple, as featureless as possible. It will not be - # verified before untaring. - tar_final_cmd = ["tar", "-cO", "--posix", - "-C", self.base_dir, filename] - final_proc = subprocess.Popen(tar_final_cmd, - stdin=subprocess.PIPE, - stdout=self.backup_stdout) - if final_proc.wait() >= 2: - if self.queue.full(): - # if queue is already full, remove some entry to wake up - # main thread, so it will be able to notice error - self.queue.get() - # handle only exit code 2 (tar fatal error) or - # greater (call failed?) - raise QubesException( - "ERROR: Failed to write the backup, out of disk space? " - "Check console output or ~/.xsession-errors for details.") - - # Delete the file as we don't need it anymore - if BACKUP_DEBUG: - print "Removing file", filename - os.remove(filename) - - if BACKUP_DEBUG: - print "Finished sending thread" - - -def prepare_backup_header(target_directory, passphrase, compressed=False, - encrypted=False, - hmac_algorithm=DEFAULT_HMAC_ALGORITHM, - crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM, - compression_filter=None): - header_file_path = os.path.join(target_directory, HEADER_FILENAME) - with open(header_file_path, "w") as f: - f.write(str("%s=%s\n" % (BackupHeader.version, - CURRENT_BACKUP_FORMAT_VERSION))) - f.write(str("%s=%s\n" % (BackupHeader.hmac_algorithm, hmac_algorithm))) - f.write(str("%s=%s\n" % (BackupHeader.crypto_algorithm, - crypto_algorithm))) - f.write(str("%s=%s\n" % (BackupHeader.encrypted, str(encrypted)))) - f.write(str("%s=%s\n" % (BackupHeader.compressed, str(compressed)))) - if compressed: - f.write(str("%s=%s\n" % (BackupHeader.compression_filter, - str(compression_filter)))) - - hmac = subprocess.Popen(["openssl", "dgst", - "-" + hmac_algorithm, "-hmac", passphrase], - stdin=open(header_file_path, "r"), - stdout=open(header_file_path + ".hmac", "w")) - if hmac.wait() != 0: - raise QubesException("Failed to compute hmac of header file") - return HEADER_FILENAME, HEADER_FILENAME + ".hmac" - - -def backup_do(base_backup_dir, files_to_backup, passphrase, - progress_callback=None, encrypted=False, appvm=None, - compressed=False, hmac_algorithm=DEFAULT_HMAC_ALGORITHM, - crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM): - global running_backup_operation - - def queue_put_with_check(proc, vmproc, queue, element): - if queue.full(): - if not proc.is_alive(): - if vmproc: - message = ("Failed to write the backup, VM output:\n" + - vmproc.stderr.read()) - else: - message = "Failed to write the backup. Out of disk space?" - raise QubesException(message) - queue.put(element) - - total_backup_sz = 0 - passphrase = passphrase.encode('utf-8') - for f in files_to_backup: - total_backup_sz += f["size"] - - if isinstance(compressed, str): - compression_filter = compressed - else: - compression_filter = DEFAULT_COMPRESSION_FILTER - - running_backup_operation = BackupOperationInfo() - vmproc = None - tar_sparse = None - if appvm is not None: - # Prepare the backup target (Qubes service call) - backup_target = "QUBESRPC qubes.Backup dom0" - - # If APPVM, STDOUT is a PIPE - vmproc = appvm.run(command=backup_target, passio_popen=True, - passio_stderr=True) - vmproc.stdin.write(base_backup_dir. - replace("\r", "").replace("\n", "") + "\n") - backup_stdout = vmproc.stdin - running_backup_operation.processes_to_kill_on_cancel.append(vmproc) - else: - # Prepare the backup target (local file) - if os.path.isdir(base_backup_dir): - backup_target = base_backup_dir + "/qubes-{0}". \ - format(time.strftime("%Y-%m-%dT%H%M%S")) - else: - backup_target = base_backup_dir - - # Create the target directory - if not os.path.exists(os.path.dirname(base_backup_dir)): - raise QubesException( - "ERROR: the backup directory for {0} does not exists". - format(base_backup_dir)) - - # If not APPVM, STDOUT is a local file - backup_stdout = open(backup_target, 'wb') - - global blocks_backedup - blocks_backedup = 0 - if callable(progress_callback): - progress = blocks_backedup * 11 / total_backup_sz - progress_callback(progress) - - backup_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/backup_") - running_backup_operation.tmpdir_to_remove = backup_tmpdir - - # Tar with tape length does not deals well with stdout (close stdout between - # two tapes) - # For this reason, we will use named pipes instead - if BACKUP_DEBUG: - print "Working in", backup_tmpdir - - backup_pipe = os.path.join(backup_tmpdir, "backup_pipe") - if BACKUP_DEBUG: - print "Creating pipe in:", backup_pipe - os.mkfifo(backup_pipe) - - if BACKUP_DEBUG: - print "Will backup:", files_to_backup - - header_files = prepare_backup_header(backup_tmpdir, passphrase, - compressed=bool(compressed), - encrypted=encrypted, - hmac_algorithm=hmac_algorithm, - crypto_algorithm=crypto_algorithm, - compression_filter=compression_filter) - - # Setup worker to send encrypted data chunks to the backup_target - def compute_progress(new_size, total_backup_size): - global blocks_backedup - blocks_backedup += new_size - if callable(progress_callback): - this_progress = blocks_backedup / float(total_backup_size) - progress_callback(int(round(this_progress * 100, 2))) - - to_send = Queue(10) - send_proc = SendWorker(to_send, backup_tmpdir, backup_stdout) - send_proc.start() - - for f in header_files: - to_send.put(f) - - for filename in files_to_backup: - if BACKUP_DEBUG: - print "Backing up", filename - - backup_tempfile = os.path.join(backup_tmpdir, - filename["subdir"], - os.path.basename(filename["path"])) - if BACKUP_DEBUG: - print "Using temporary location:", backup_tempfile - - # Ensure the temporary directory exists - if not os.path.isdir(os.path.dirname(backup_tempfile)): - os.makedirs(os.path.dirname(backup_tempfile)) - - # The first tar cmd can use any complex feature as we want. Files will - # be verified before untaring this. - # Prefix the path in archive with filename["subdir"] to have it - # verified during untar - tar_cmdline = (["tar", "-Pc", '--sparse', - "-f", backup_pipe, - '-C', os.path.dirname(filename["path"])] + - (['--dereference'] if filename["subdir"] != "dom0-home/" - else []) + - ['--xform', 's:^%s:%s\\0:' % ( - os.path.basename(filename["path"]), - filename["subdir"]), - os.path.basename(filename["path"]) - ]) - if compressed: - tar_cmdline.insert(-1, - "--use-compress-program=%s" % compression_filter) - - if BACKUP_DEBUG: - print " ".join(tar_cmdline) - - # Tips: Popen(bufsize=0) - # Pipe: tar-sparse | encryptor [| hmac] | tar | backup_target - # Pipe: tar-sparse [| hmac] | tar | backup_target - tar_sparse = subprocess.Popen(tar_cmdline, stdin=subprocess.PIPE, - stderr=(open(os.devnull, 'w') - if not BACKUP_DEBUG - else None)) - running_backup_operation.processes_to_kill_on_cancel.append(tar_sparse) - - # Wait for compressor (tar) process to finish or for any error of other - # subprocesses - i = 0 - run_error = "paused" - encryptor = None - if encrypted: - # Start encrypt - # If no cipher is provided, the data is forwarded unencrypted !!! - encryptor = subprocess.Popen(["openssl", "enc", - "-e", "-" + crypto_algorithm, - "-pass", "pass:" + passphrase], - stdin=open(backup_pipe, 'rb'), - stdout=subprocess.PIPE) - pipe = encryptor.stdout - else: - pipe = open(backup_pipe, 'rb') - while run_error == "paused": - - # Start HMAC - hmac = subprocess.Popen(["openssl", "dgst", - "-" + hmac_algorithm, "-hmac", passphrase], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE) - - # Prepare a first chunk - chunkfile = backup_tempfile + "." + "%03d" % i - i += 1 - chunkfile_p = open(chunkfile, 'wb') - - common_args = { - 'backup_target': chunkfile_p, - 'total_backup_sz': total_backup_sz, - 'hmac': hmac, - 'vmproc': vmproc, - 'addproc': tar_sparse, - 'progress_callback': compute_progress, - 'size_limit': 100 * 1024 * 1024, - } - run_error = wait_backup_feedback( - in_stream=pipe, streamproc=encryptor, - **common_args) - chunkfile_p.close() - - if BACKUP_DEBUG: - print "Wait_backup_feedback returned:", run_error - - if running_backup_operation.canceled: - try: - tar_sparse.terminate() - except: - pass - try: - hmac.terminate() - except: - pass - tar_sparse.wait() - hmac.wait() - to_send.put("ERROR") - send_proc.join() - shutil.rmtree(backup_tmpdir) - running_backup_operation = None - raise BackupCanceledError("Backup canceled") - if run_error and run_error != "size_limit": - send_proc.terminate() - if run_error == "VM" and vmproc: - raise QubesException( - "Failed to write the backup, VM output:\n" + - vmproc.stderr.read(MAX_STDERR_BYTES)) - else: - raise QubesException("Failed to perform backup: error in " + - run_error) - - # Send the chunk to the backup target - queue_put_with_check( - send_proc, vmproc, to_send, - os.path.relpath(chunkfile, backup_tmpdir)) - - # Close HMAC - hmac.stdin.close() - hmac.wait() - if BACKUP_DEBUG: - print "HMAC proc return code:", hmac.poll() - - # Write HMAC data next to the chunk file - hmac_data = hmac.stdout.read() - if BACKUP_DEBUG: - print "Writing hmac to", chunkfile + ".hmac" - hmac_file = open(chunkfile + ".hmac", 'w') - hmac_file.write(hmac_data) - hmac_file.flush() - hmac_file.close() - - # Send the HMAC to the backup target - queue_put_with_check( - send_proc, vmproc, to_send, - os.path.relpath(chunkfile, backup_tmpdir) + ".hmac") - - if tar_sparse.poll() is None or run_error == "size_limit": - run_error = "paused" - else: - running_backup_operation.processes_to_kill_on_cancel.remove( - tar_sparse) - if BACKUP_DEBUG: - print "Finished tar sparse with exit code", tar_sparse \ - .poll() - pipe.close() - - queue_put_with_check(send_proc, vmproc, to_send, "FINISHED") - send_proc.join() - shutil.rmtree(backup_tmpdir) - - if running_backup_operation.canceled: - running_backup_operation = None - raise BackupCanceledError("Backup canceled") - - running_backup_operation = None - - if send_proc.exitcode != 0: - raise QubesException( - "Failed to send backup: error in the sending process") - - if vmproc: - if BACKUP_DEBUG: - print "VMProc1 proc return code:", vmproc.poll() - if tar_sparse is not None: - print "Sparse1 proc return code:", tar_sparse.poll() - vmproc.stdin.close() - - # Save date of last backup - qvm_collection = QubesVmCollection() - qvm_collection.lock_db_for_writing() - qvm_collection.load() - - for vm in qvm_collection.values(): - if vm.backup_content: - vm.backup_timestamp = datetime.datetime.now() - - qvm_collection.save() - qvm_collection.unlock_db() - - -''' -' Wait for backup chunk to finish -' - Monitor all the processes (streamproc, hmac, vmproc, addproc) for errors -' - Copy stdout of streamproc to backup_target and hmac stdin if available -' - Compute progress based on total_backup_sz and send progress to -' progress_callback function -' - Returns if -' - one of the monitored processes error out (streamproc, hmac, vmproc, -' addproc), along with the processe that failed -' - all of the monitored processes except vmproc finished successfully -' (vmproc termination is controlled by the python script) -' - streamproc does not delivers any data anymore (return with the error -' "") -' - size_limit is provided and is about to be exceeded -''' - - -def wait_backup_feedback(progress_callback, in_stream, streamproc, - backup_target, total_backup_sz, hmac=None, vmproc=None, - addproc=None, - size_limit=None): - buffer_size = 409600 - - run_error = None - run_count = 1 - bytes_copied = 0 - while run_count > 0 and run_error is None: - - if size_limit and bytes_copied + buffer_size > size_limit: - return "size_limit" - buf = in_stream.read(buffer_size) - progress_callback(len(buf), total_backup_sz) - bytes_copied += len(buf) - - run_count = 0 - if hmac: - retcode = hmac.poll() - if retcode is not None: - if retcode != 0: - run_error = "hmac" - else: - run_count += 1 - - if addproc: - retcode = addproc.poll() - if retcode is not None: - if retcode != 0: - run_error = "addproc" - else: - run_count += 1 - - if vmproc: - retcode = vmproc.poll() - if retcode is not None: - if retcode != 0: - run_error = "VM" - if BACKUP_DEBUG: - print vmproc.stdout.read() - else: - # VM should run until the end - pass - - if streamproc: - retcode = streamproc.poll() - if retcode is not None: - if retcode != 0: - run_error = "streamproc" - break - elif retcode == 0 and len(buf) <= 0: - return "" - run_count += 1 - - else: - if len(buf) <= 0: - return "" - - try: - backup_target.write(buf) - except IOError as e: - if e.errno == errno.EPIPE: - run_error = "target" - else: - raise - - if hmac: - hmac.stdin.write(buf) - - return run_error - - -def verify_hmac(filename, hmacfile, passphrase, algorithm): - if BACKUP_DEBUG: - print "Verifying file " + filename - - if hmacfile != filename + ".hmac": - raise QubesException( - "ERROR: expected hmac for {}, but got {}". - format(filename, hmacfile)) - - hmac_proc = subprocess.Popen(["openssl", "dgst", "-" + algorithm, - "-hmac", passphrase], - stdin=open(filename, 'rb'), - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - hmac_stdout, hmac_stderr = hmac_proc.communicate() - - if len(hmac_stderr) > 0: - raise QubesException( - "ERROR: verify file {0}: {1}".format(filename, hmac_stderr)) - else: - if BACKUP_DEBUG: - print "Loading hmac for file " + filename - hmac = load_hmac(open(hmacfile, 'r').read()) - - if len(hmac) > 0 and load_hmac(hmac_stdout) == hmac: - os.unlink(hmacfile) - if BACKUP_DEBUG: - print "File verification OK -> Sending file " + filename - return True - else: - raise QubesException( - "ERROR: invalid hmac for file {0}: {1}. " - "Is the passphrase correct?". - format(filename, load_hmac(hmac_stdout))) - # Not reachable - return False - - -class ExtractWorker2(Process): - def __init__(self, queue, base_dir, passphrase, encrypted, total_size, - print_callback, error_callback, progress_callback, vmproc=None, - compressed=False, crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM, - verify_only=False): - super(ExtractWorker2, self).__init__() - self.queue = queue - self.base_dir = base_dir - self.passphrase = passphrase - self.encrypted = encrypted - self.compressed = compressed - self.crypto_algorithm = crypto_algorithm - self.verify_only = verify_only - self.total_size = total_size - self.blocks_backedup = 0 - self.tar2_process = None - self.tar2_current_file = None - self.decompressor_process = None - self.decryptor_process = None - - self.print_callback = print_callback - self.error_callback = error_callback - self.progress_callback = progress_callback - - self.vmproc = vmproc - - self.restore_pipe = os.path.join(self.base_dir, "restore_pipe") - if BACKUP_DEBUG: - print "Creating pipe in:", self.restore_pipe - os.mkfifo(self.restore_pipe) - - self.stderr_encoding = sys.stderr.encoding or 'utf-8' - - def compute_progress(self, new_size, _): - if self.progress_callback: - self.blocks_backedup += new_size - progress = self.blocks_backedup / float(self.total_size) - progress = int(round(progress * 100, 2)) - self.progress_callback(progress) - - def collect_tar_output(self): - if not self.tar2_process.stderr: - return - - if self.tar2_process.poll() is None: - try: - new_lines = self.tar2_process.stderr \ - .read(MAX_STDERR_BYTES).splitlines() - except IOError as e: - if e.errno == errno.EAGAIN: - return - else: - raise - else: - new_lines = self.tar2_process.stderr.readlines() - - new_lines = map(lambda x: x.decode(self.stderr_encoding), new_lines) - - if not BACKUP_DEBUG: - msg_re = re.compile(r".*#[0-9].*restore_pipe") - new_lines = filter(lambda x: not msg_re.match(x), new_lines) - - self.tar2_stderr += new_lines - - def run(self): - try: - self.__run__() - except Exception as e: - exc_type, exc_value, exc_traceback = sys.exc_info() - # Cleanup children - for process in [self.decompressor_process, - self.decryptor_process, - self.tar2_process]: - if process: - # FIXME: kill()? - try: - process.terminate() - except OSError: - pass - process.wait() - self.error_callback("ERROR: " + unicode(e)) - raise e, None, exc_traceback - - def __run__(self): - if BACKUP_DEBUG and callable(self.print_callback): - self.print_callback("Started sending thread") - self.print_callback("Moving to dir " + self.base_dir) - os.chdir(self.base_dir) - - filename = None - - for filename in iter(self.queue.get, None): - if filename == "FINISHED" or filename == "ERROR": - break - - if BACKUP_DEBUG and callable(self.print_callback): - self.print_callback("Extracting file " + filename) - - if filename.endswith('.000'): - # next file - if self.tar2_process is not None: - if self.tar2_process.wait() != 0: - self.collect_tar_output() - self.error_callback( - "ERROR: unable to extract files for {0}, tar " - "output:\n {1}". - format(self.tar2_current_file, - "\n ".join(self.tar2_stderr))) - else: - # Finished extracting the tar file - self.tar2_process = None - self.tar2_current_file = None - - tar2_cmdline = ['tar', - '-%sMk%sf' % ("t" if self.verify_only else "x", - "v" if BACKUP_DEBUG else ""), - self.restore_pipe, - os.path.relpath(filename.rstrip('.000'))] - if BACKUP_DEBUG and callable(self.print_callback): - self.print_callback("Running command " + - unicode(tar2_cmdline)) - self.tar2_process = subprocess.Popen(tar2_cmdline, - stdin=subprocess.PIPE, - stderr=subprocess.PIPE) - fcntl.fcntl(self.tar2_process.stderr.fileno(), fcntl.F_SETFL, - fcntl.fcntl(self.tar2_process.stderr.fileno(), - fcntl.F_GETFL) | os.O_NONBLOCK) - self.tar2_stderr = [] - elif not self.tar2_process: - # Extracting of the current archive failed, skip to the next - # archive - if not BACKUP_DEBUG: - os.remove(filename) - continue - else: - self.collect_tar_output() - if BACKUP_DEBUG and callable(self.print_callback): - self.print_callback("Releasing next chunck") - self.tar2_process.stdin.write("\n") - self.tar2_process.stdin.flush() - self.tar2_current_file = filename - - pipe = open(self.restore_pipe, 'wb') - common_args = { - 'backup_target': pipe, - 'total_backup_sz': self.total_size, - 'hmac': None, - 'vmproc': self.vmproc, - 'addproc': self.tar2_process - } - if self.encrypted: - # Start decrypt - self.decryptor_process = subprocess.Popen( - ["openssl", "enc", - "-d", - "-" + self.crypto_algorithm, - "-pass", - "pass:" + self.passphrase] + - (["-z"] if self.compressed else []), - stdin=open(filename, 'rb'), - stdout=subprocess.PIPE) - - run_error = wait_backup_feedback( - progress_callback=self.compute_progress, - in_stream=self.decryptor_process.stdout, - streamproc=self.decryptor_process, - **common_args) - elif self.compressed: - self.decompressor_process = subprocess.Popen( - ["gzip", "-d"], - stdin=open(filename, 'rb'), - stdout=subprocess.PIPE) - - run_error = wait_backup_feedback( - progress_callback=self.compute_progress, - in_stream=self.decompressor_process.stdout, - streamproc=self.decompressor_process, - **common_args) - else: - run_error = wait_backup_feedback( - progress_callback=self.compute_progress, - in_stream=open(filename, "rb"), streamproc=None, - **common_args) - - try: - pipe.close() - except IOError as e: - if e.errno == errno.EPIPE: - if BACKUP_DEBUG: - self.error_callback( - "Got EPIPE while closing pipe to " - "the inner tar process") - # ignore the error - else: - raise - if len(run_error): - if run_error == "target": - self.collect_tar_output() - details = "\n".join(self.tar2_stderr) - else: - details = "%s failed" % run_error - self.tar2_process.terminate() - self.tar2_process.wait() - self.tar2_process = None - self.error_callback("Error while processing '%s': %s " % - (self.tar2_current_file, details)) - - # Delete the file as we don't need it anymore - if BACKUP_DEBUG and callable(self.print_callback): - self.print_callback("Removing file " + filename) - os.remove(filename) - - os.unlink(self.restore_pipe) - - if self.tar2_process is not None: - if filename == "ERROR": - self.tar2_process.terminate() - self.tar2_process.wait() - elif self.tar2_process.wait() != 0: - self.collect_tar_output() - raise QubesException( - "unable to extract files for {0}.{1} Tar command " - "output: %s". - format(self.tar2_current_file, - (" Perhaps the backup is encrypted?" - if not self.encrypted else "", - "\n".join(self.tar2_stderr)))) - else: - # Finished extracting the tar file - self.tar2_process = None - - if BACKUP_DEBUG and callable(self.print_callback): - self.print_callback("Finished extracting thread") - - -class ExtractWorker3(ExtractWorker2): - def __init__(self, queue, base_dir, passphrase, encrypted, total_size, - print_callback, error_callback, progress_callback, vmproc=None, - compressed=False, crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM, - compression_filter=None, verify_only=False): - super(ExtractWorker3, self).__init__(queue, base_dir, passphrase, - encrypted, total_size, - print_callback, error_callback, - progress_callback, vmproc, - compressed, crypto_algorithm, - verify_only) - self.compression_filter = compression_filter - os.unlink(self.restore_pipe) - - def __run__(self): - if BACKUP_DEBUG and callable(self.print_callback): - self.print_callback("Started sending thread") - self.print_callback("Moving to dir " + self.base_dir) - os.chdir(self.base_dir) - - filename = None - - input_pipe = None - for filename in iter(self.queue.get, None): - if filename == "FINISHED" or filename == "ERROR": - break - - if BACKUP_DEBUG and callable(self.print_callback): - self.print_callback("Extracting file " + filename) - - if filename.endswith('.000'): - # next file - if self.tar2_process is not None: - input_pipe.close() - if self.tar2_process.wait() != 0: - self.collect_tar_output() - self.error_callback( - "ERROR: unable to extract files for {0}, tar " - "output:\n {1}". - format(self.tar2_current_file, - "\n ".join(self.tar2_stderr))) - else: - # Finished extracting the tar file - self.tar2_process = None - self.tar2_current_file = None - - tar2_cmdline = ['tar', - '-%sk%s' % ("t" if self.verify_only else "x", - "v" if BACKUP_DEBUG else ""), - os.path.relpath(filename.rstrip('.000'))] - if self.compressed: - if self.compression_filter: - tar2_cmdline.insert(-1, - "--use-compress-program=%s" % - self.compression_filter) - else: - tar2_cmdline.insert(-1, "--use-compress-program=%s" % - DEFAULT_COMPRESSION_FILTER) - - if BACKUP_DEBUG and callable(self.print_callback): - self.print_callback("Running command " + - unicode(tar2_cmdline)) - if self.encrypted: - # Start decrypt - self.decryptor_process = subprocess.Popen( - ["openssl", "enc", - "-d", - "-" + self.crypto_algorithm, - "-pass", - "pass:" + self.passphrase], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE) - - self.tar2_process = subprocess.Popen( - tar2_cmdline, - stdin=self.decryptor_process.stdout, - stderr=subprocess.PIPE) - input_pipe = self.decryptor_process.stdin - else: - self.tar2_process = subprocess.Popen( - tar2_cmdline, - stdin=subprocess.PIPE, - stderr=subprocess.PIPE) - input_pipe = self.tar2_process.stdin - - fcntl.fcntl(self.tar2_process.stderr.fileno(), fcntl.F_SETFL, - fcntl.fcntl(self.tar2_process.stderr.fileno(), - fcntl.F_GETFL) | os.O_NONBLOCK) - self.tar2_stderr = [] - elif not self.tar2_process: - # Extracting of the current archive failed, skip to the next - # archive - if not BACKUP_DEBUG: - os.remove(filename) - continue - else: - if BACKUP_DEBUG and callable(self.print_callback): - self.print_callback("Releasing next chunck") - self.tar2_current_file = filename - - common_args = { - 'backup_target': input_pipe, - 'total_backup_sz': self.total_size, - 'hmac': None, - 'vmproc': self.vmproc, - 'addproc': self.tar2_process - } - - run_error = wait_backup_feedback( - progress_callback=self.compute_progress, - in_stream=open(filename, "rb"), streamproc=None, - **common_args) - - if len(run_error): - if run_error == "target": - self.collect_tar_output() - details = "\n".join(self.tar2_stderr) - else: - details = "%s failed" % run_error - if self.decryptor_process: - self.decryptor_process.terminate() - self.decryptor_process.wait() - self.decryptor_process = None - self.tar2_process.terminate() - self.tar2_process.wait() - self.tar2_process = None - self.error_callback("Error while processing '%s': %s " % - (self.tar2_current_file, details)) - - # Delete the file as we don't need it anymore - if BACKUP_DEBUG and callable(self.print_callback): - self.print_callback("Removing file " + filename) - os.remove(filename) - - if self.tar2_process is not None: - input_pipe.close() - if filename == "ERROR": - if self.decryptor_process: - self.decryptor_process.terminate() - self.decryptor_process.wait() - self.decryptor_process = None - self.tar2_process.terminate() - self.tar2_process.wait() - elif self.tar2_process.wait() != 0: - self.collect_tar_output() - raise QubesException( - "unable to extract files for {0}.{1} Tar command " - "output: %s". - format(self.tar2_current_file, - (" Perhaps the backup is encrypted?" - if not self.encrypted else "", - "\n".join(self.tar2_stderr)))) - else: - # Finished extracting the tar file - self.tar2_process = None - - if BACKUP_DEBUG and callable(self.print_callback): - self.print_callback("Finished extracting thread") - - -def get_supported_hmac_algo(hmac_algorithm): - # Start with provided default - if hmac_algorithm: - yield hmac_algorithm - proc = subprocess.Popen(['openssl', 'list-message-digest-algorithms'], - stdout=subprocess.PIPE) - for algo in proc.stdout.readlines(): - if '=>' in algo: - continue - yield algo.strip() - proc.wait() - - -def parse_backup_header(filename): - header_data = {} - with open(filename, 'r') as f: - for line in f.readlines(): - if line.count('=') != 1: - raise QubesException("Invalid backup header (line %s)" % line) - (key, value) = line.strip().split('=') - if not any([key == getattr(BackupHeader, attr) for attr in dir( - BackupHeader)]): - # Ignoring unknown option - continue - if key in BackupHeader.bool_options: - value = value.lower() in ["1", "true", "yes"] - elif key in BackupHeader.int_options: - value = int(value) - header_data[key] = value - return header_data - - -def restore_vm_dirs(backup_source, restore_tmpdir, passphrase, vms_dirs, vms, - vms_size, print_callback=None, error_callback=None, - progress_callback=None, encrypted=False, appvm=None, - compressed=False, hmac_algorithm=DEFAULT_HMAC_ALGORITHM, - crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM, - verify_only=False, - format_version=CURRENT_BACKUP_FORMAT_VERSION, - compression_filter=None): - global running_backup_operation - - if callable(print_callback): - if BACKUP_DEBUG: - print_callback("Working in temporary dir:" + restore_tmpdir) - print_callback( - "Extracting data: " + size_to_human(vms_size) + " to restore") - - passphrase = passphrase.encode('utf-8') - header_data = None - vmproc = None - if appvm is not None: - # Prepare the backup target (Qubes service call) - backup_target = "QUBESRPC qubes.Restore dom0" - - # If APPVM, STDOUT is a PIPE - vmproc = appvm.run(command=backup_target, passio_popen=True, - passio_stderr=True) - vmproc.stdin.write( - backup_source.replace("\r", "").replace("\n", "") + "\n") - - # Send to tar2qfile the VMs that should be extracted - vmproc.stdin.write(" ".join(vms_dirs) + "\n") - if running_backup_operation: - running_backup_operation.processes_to_kill_on_cancel.append(vmproc) - - backup_stdin = vmproc.stdout - tar1_command = ['/usr/libexec/qubes/qfile-dom0-unpacker', - str(os.getuid()), restore_tmpdir, '-v'] - else: - backup_stdin = open(backup_source, 'rb') - - tar1_command = ['tar', - '-ixvf', backup_source, - '-C', restore_tmpdir] + vms_dirs - - tar1_env = os.environ.copy() - # TODO: add some safety margin? - tar1_env['UPDATES_MAX_BYTES'] = str(vms_size) - # Restoring only header - if vms_dirs and vms_dirs[0] == HEADER_FILENAME: - # backup-header, backup-header.hmac, qubes-xml.000, qubes-xml.000.hmac - tar1_env['UPDATES_MAX_FILES'] = '4' - else: - # Currently each VM consists of at most 7 archives (count - # file_to_backup calls in backup_prepare()), but add some safety - # margin for further extensions. Each archive is divided into 100MB - # chunks. Additionally each file have own hmac file. So assume upper - # limit as 2*(10*COUNT_OF_VMS+TOTAL_SIZE/100MB) - tar1_env['UPDATES_MAX_FILES'] = str(2 * (10 * len(vms_dirs) + - int(vms_size / - (100 * 1024 * 1024)))) - if BACKUP_DEBUG and callable(print_callback): - print_callback("Run command" + unicode(tar1_command)) - command = subprocess.Popen( - tar1_command, - stdin=backup_stdin, - stdout=vmproc.stdin if vmproc else subprocess.PIPE, - stderr=subprocess.PIPE, - env=tar1_env) - if running_backup_operation: - running_backup_operation.processes_to_kill_on_cancel.append(command) - - # qfile-dom0-unpacker output filelist on stderr (and have stdout connected - # to the VM), while tar output filelist on stdout - if appvm: - filelist_pipe = command.stderr - # let qfile-dom0-unpacker hold the only open FD to the write end of - # pipe, otherwise qrexec-client will not receive EOF when - # qfile-dom0-unpacker terminates - vmproc.stdin.close() - else: - filelist_pipe = command.stdout - - expect_tar_error = False - - to_extract = Queue() - nextfile = None - - # If want to analyze backup header, do it now - if vms_dirs and vms_dirs[0] == HEADER_FILENAME: - filename = filelist_pipe.readline().strip() - hmacfile = filelist_pipe.readline().strip() - if not appvm: - nextfile = filelist_pipe.readline().strip() - - if BACKUP_DEBUG and callable(print_callback): - print_callback("Got backup header and hmac: %s, %s" % (filename, - hmacfile)) - - if not filename or filename == "EOF" or \ - not hmacfile or hmacfile == "EOF": - if appvm: - vmproc.wait() - proc_error_msg = vmproc.stderr.read(MAX_STDERR_BYTES) - else: - command.wait() - proc_error_msg = command.stderr.read(MAX_STDERR_BYTES) - raise QubesException("Premature end of archive while receiving " - "backup header. Process output:\n" + - proc_error_msg) - filename = os.path.join(restore_tmpdir, filename) - hmacfile = os.path.join(restore_tmpdir, hmacfile) - file_ok = False - for hmac_algo in get_supported_hmac_algo(hmac_algorithm): - try: - if verify_hmac(filename, hmacfile, passphrase, hmac_algo): - file_ok = True - hmac_algorithm = hmac_algo - break - except QubesException: - # Ignore exception here, try the next algo - pass - if not file_ok: - raise QubesException("Corrupted backup header (hmac verification " - "failed). Is the password correct?") - if os.path.basename(filename) == HEADER_FILENAME: - header_data = parse_backup_header(filename) - if BackupHeader.version in header_data: - format_version = header_data[BackupHeader.version] - if BackupHeader.crypto_algorithm in header_data: - crypto_algorithm = header_data[BackupHeader.crypto_algorithm] - if BackupHeader.hmac_algorithm in header_data: - hmac_algorithm = header_data[BackupHeader.hmac_algorithm] - if BackupHeader.compressed in header_data: - compressed = header_data[BackupHeader.compressed] - if BackupHeader.encrypted in header_data: - encrypted = header_data[BackupHeader.encrypted] - if BackupHeader.compression_filter in header_data: - compression_filter = header_data[ - BackupHeader.compression_filter] - os.unlink(filename) - else: - # if no header found, create one with guessed HMAC algo - header_data = {BackupHeader.hmac_algorithm: hmac_algorithm} - # If this isn't backup header, pass it to ExtractWorker - to_extract.put(filename) - # when tar do not find expected file in archive, it exit with - # code 2. This will happen because we've requested backup-header - # file, but the archive do not contain it. Ignore this particular - # error. - if not appvm: - expect_tar_error = True - - # Setup worker to extract encrypted data chunks to the restore dirs - # Create the process here to pass it options extracted from backup header - extractor_params = { - 'queue': to_extract, - 'base_dir': restore_tmpdir, - 'passphrase': passphrase, - 'encrypted': encrypted, - 'compressed': compressed, - 'crypto_algorithm': crypto_algorithm, - 'verify_only': verify_only, - 'total_size': vms_size, - 'print_callback': print_callback, - 'error_callback': error_callback, - 'progress_callback': progress_callback, - } - if format_version == 2: - extract_proc = ExtractWorker2(**extractor_params) - elif format_version == 3: - extractor_params['compression_filter'] = compression_filter - extract_proc = ExtractWorker3(**extractor_params) - else: - raise NotImplemented( - "Backup format version %d not supported" % format_version) - extract_proc.start() - - try: - filename = None - while True: - if running_backup_operation and running_backup_operation.canceled: - break - if not extract_proc.is_alive(): - command.terminate() - command.wait() - expect_tar_error = True - if vmproc: - vmproc.terminate() - vmproc.wait() - vmproc = None - break - if nextfile is not None: - filename = nextfile - else: - filename = filelist_pipe.readline().strip() - - if BACKUP_DEBUG and callable(print_callback): - print_callback("Getting new file:" + filename) - - if not filename or filename == "EOF": - break - - hmacfile = filelist_pipe.readline().strip() - - if running_backup_operation and running_backup_operation.canceled: - break - # if reading archive directly with tar, wait for next filename - - # tar prints filename before processing it, so wait for - # the next one to be sure that whole file was extracted - if not appvm: - nextfile = filelist_pipe.readline().strip() - - if BACKUP_DEBUG and callable(print_callback): - print_callback("Getting hmac:" + hmacfile) - if not hmacfile or hmacfile == "EOF": - # Premature end of archive, either of tar1_command or - # vmproc exited with error - break - - if not any(map(lambda x: filename.startswith(x), vms_dirs)): - if BACKUP_DEBUG and callable(print_callback): - print_callback("Ignoring VM not selected for restore") - os.unlink(os.path.join(restore_tmpdir, filename)) - os.unlink(os.path.join(restore_tmpdir, hmacfile)) - continue - - if verify_hmac(os.path.join(restore_tmpdir, filename), - os.path.join(restore_tmpdir, hmacfile), - passphrase, hmac_algorithm): - to_extract.put(os.path.join(restore_tmpdir, filename)) - - if running_backup_operation and running_backup_operation.canceled: - raise BackupCanceledError("Restore canceled", - tmpdir=restore_tmpdir) - - if command.wait() != 0 and not expect_tar_error: - raise QubesException( - "unable to read the qubes backup file {0} ({1}). " - "Is it really a backup?".format(backup_source, command.wait())) - if vmproc: - if vmproc.wait() != 0: - raise QubesException( - "unable to read the qubes backup {0} " - "because of a VM error: {1}".format( - backup_source, vmproc.stderr.read(MAX_STDERR_BYTES))) - - if filename and filename != "EOF": - raise QubesException( - "Premature end of archive, the last file was %s" % filename) - except: - to_extract.put("ERROR") - extract_proc.join() - raise - else: - to_extract.put("FINISHED") - - if BACKUP_DEBUG and callable(print_callback): - print_callback("Waiting for the extraction process to finish...") - extract_proc.join() - if BACKUP_DEBUG and callable(print_callback): - print_callback("Extraction process finished with code:" + - str(extract_proc.exitcode)) - if extract_proc.exitcode != 0: - raise QubesException( - "unable to extract the qubes backup. " - "Check extracting process errors.") - - return header_data - - -def backup_restore_set_defaults(options): - if 'use-default-netvm' not in options: - options['use-default-netvm'] = False - if 'use-none-netvm' not in options: - options['use-none-netvm'] = False - if 'use-default-template' not in options: - options['use-default-template'] = False - if 'dom0-home' not in options: - options['dom0-home'] = True - if 'replace-template' not in options: - options['replace-template'] = [] - if 'ignore-username-mismatch' not in options: - options['ignore-username-mismatch'] = False - if 'verify-only' not in options: - options['verify-only'] = False - if 'rename-conflicting' not in options: - options['rename-conflicting'] = False - - return options - - -def load_hmac(hmac): - hmac = hmac.strip().split("=") - if len(hmac) > 1: - hmac = hmac[1].strip() - else: - raise QubesException("ERROR: invalid hmac file content") - - return hmac - - -def backup_detect_format_version(backup_location): - if os.path.exists(os.path.join(backup_location, 'qubes.xml')): - return 1 - else: - # this could mean also 3, but not distinguishable until backup header - # is read - return 2 - - -def backup_restore_header(source, passphrase, - print_callback=print_stdout, - error_callback=print_stderr, - encrypted=False, appvm=None, compressed=False, - format_version=None, - hmac_algorithm=DEFAULT_HMAC_ALGORITHM, - crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM): - global running_backup_operation - running_backup_operation = None - - restore_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/restore_") - - if format_version is None: - format_version = backup_detect_format_version(source) - - if format_version == 1: - return restore_tmpdir, os.path.join(source, 'qubes.xml'), None - - # tar2qfile matches only beginnings, while tar full path - if appvm: - extract_filter = [HEADER_FILENAME, 'qubes.xml.000'] - else: - extract_filter = [HEADER_FILENAME, HEADER_FILENAME + '.hmac', - 'qubes.xml.000', 'qubes.xml.000.hmac'] - - header_data = restore_vm_dirs(source, - restore_tmpdir, - passphrase=passphrase, - vms_dirs=extract_filter, - vms=None, - vms_size=HEADER_QUBES_XML_MAX_SIZE, - format_version=format_version, - hmac_algorithm=hmac_algorithm, - crypto_algorithm=crypto_algorithm, - print_callback=print_callback, - error_callback=error_callback, - progress_callback=None, - encrypted=encrypted, - compressed=compressed, - appvm=appvm) - - return (restore_tmpdir, os.path.join(restore_tmpdir, "qubes.xml"), - header_data) - -def generate_new_name_for_conflicting_vm(orig_name, host_collection, - restore_info): - number = 1 - if len(orig_name) > 29: - orig_name = orig_name[0:29] - new_name = orig_name - while (new_name in restore_info.keys() or - new_name in map(lambda x: x.get('rename_to', None), - restore_info.values()) or - host_collection.get_vm_by_name(new_name)): - new_name = str('{}{}'.format(orig_name, number)) - number += 1 - if number == 100: - # give up - return None - return new_name - -def restore_info_verify(restore_info, host_collection): - options = restore_info['$OPTIONS$'] - for vm in restore_info.keys(): - if vm in ['$OPTIONS$', 'dom0']: - continue - - vm_info = restore_info[vm] - - vm_info.pop('excluded', None) - if 'exclude' in options.keys(): - if vm in options['exclude']: - vm_info['excluded'] = True - - vm_info.pop('already-exists', None) - if not options['verify-only'] and \ - host_collection.get_vm_by_name(vm) is not None: - if options['rename-conflicting']: - new_name = generate_new_name_for_conflicting_vm( - vm, host_collection, restore_info - ) - if new_name is not None: - vm_info['rename-to'] = new_name - else: - vm_info['already-exists'] = True - else: - vm_info['already-exists'] = True - - # check template - vm_info.pop('missing-template', None) - if vm_info['template']: - template_name = vm_info['template'] - host_template = host_collection.get_vm_by_name(template_name) - if not host_template or not host_template.is_template(): - # Maybe the (custom) template is in the backup? - if not (template_name in restore_info.keys() and - restore_info[template_name]['vm'].is_template()): - if options['use-default-template']: - if 'orig-template' not in vm_info.keys(): - vm_info['orig-template'] = template_name - vm_info['template'] = host_collection \ - .get_default_template().name - else: - vm_info['missing-template'] = True - - # check netvm - vm_info.pop('missing-netvm', None) - if vm_info['netvm']: - netvm_name = vm_info['netvm'] - - netvm_on_host = host_collection.get_vm_by_name(netvm_name) - - # No netvm on the host? - if not ((netvm_on_host is not None) and netvm_on_host.is_netvm()): - - # Maybe the (custom) netvm is in the backup? - if not (netvm_name in restore_info.keys() and - restore_info[netvm_name]['vm'].is_netvm()): - if options['use-default-netvm']: - vm_info['netvm'] = host_collection \ - .get_default_netvm().name - vm_info['vm'].uses_default_netvm = True - elif options['use-none-netvm']: - vm_info['netvm'] = None - else: - vm_info['missing-netvm'] = True - - vm_info['good-to-go'] = not any([(prop in vm_info.keys()) for - prop in ['missing-netvm', - 'missing-template', - 'already-exists', - 'excluded']]) - - # update references to renamed VMs: - for vm in restore_info.keys(): - if vm in ['$OPTIONS$', 'dom0']: - continue - vm_info = restore_info[vm] - template_name = vm_info['template'] - if (template_name in restore_info and - restore_info[template_name]['good-to-go'] and - 'rename-to' in restore_info[template_name]): - vm_info['template'] = restore_info[template_name]['rename-to'] - netvm_name = vm_info['netvm'] - if (netvm_name in restore_info and - restore_info[netvm_name]['good-to-go'] and - 'rename-to' in restore_info[netvm_name]): - vm_info['netvm'] = restore_info[netvm_name]['rename-to'] - - return restore_info - - -def backup_restore_prepare(backup_location, passphrase, options=None, - host_collection=None, encrypted=False, appvm=None, - compressed=False, print_callback=print_stdout, - error_callback=print_stderr, - format_version=None, - hmac_algorithm=DEFAULT_HMAC_ALGORITHM, - crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM): - if options is None: - options = {} - # Defaults - backup_restore_set_defaults(options) - # Options introduced in backup format 3+, which always have a header, - # so no need for fallback in function parameter - compression_filter = DEFAULT_COMPRESSION_FILTER - - # Private functions begin - def is_vm_included_in_backup_v1(backup_dir, check_vm): - if check_vm.qid == 0: - return os.path.exists(os.path.join(backup_dir, 'dom0-home')) - - # DisposableVM - if check_vm.dir_path is None: - return False - - backup_vm_dir_path = check_vm.dir_path.replace( - system_path["qubes_base_dir"], backup_dir) - - if os.path.exists(backup_vm_dir_path): - return True - else: - return False - - def is_vm_included_in_backup_v2(_, check_vm): - if check_vm.backup_content: - return True - else: - return False - - def find_template_name(template, replaces): - rx_replace = re.compile("(.*):(.*)") - for r in replaces: - m = rx_replace.match(r) - if m.group(1) == template: - return m.group(2) - - return template - - # Private functions end - - # Format versions: - # 1 - Qubes R1, Qubes R2 beta1, beta2 - # 2 - Qubes R2 beta3+ - - if format_version is None: - format_version = backup_detect_format_version(backup_location) - - if format_version == 1: - is_vm_included_in_backup = is_vm_included_in_backup_v1 - elif format_version in [2, 3]: - is_vm_included_in_backup = is_vm_included_in_backup_v2 - if not appvm: - if not os.path.isfile(backup_location): - raise QubesException("Invalid backup location (not a file or " - "directory with qubes.xml)" - ": %s" % unicode(backup_location)) - else: - raise QubesException( - "Unknown backup format version: %s" % str(format_version)) - - (restore_tmpdir, qubes_xml, header_data) = backup_restore_header( - backup_location, - passphrase, - encrypted=encrypted, - appvm=appvm, - compressed=compressed, - hmac_algorithm=hmac_algorithm, - crypto_algorithm=crypto_algorithm, - print_callback=print_callback, - error_callback=error_callback, - format_version=format_version) - - if header_data: - if BackupHeader.version in header_data: - format_version = header_data[BackupHeader.version] - if BackupHeader.crypto_algorithm in header_data: - crypto_algorithm = header_data[BackupHeader.crypto_algorithm] - if BackupHeader.hmac_algorithm in header_data: - hmac_algorithm = header_data[BackupHeader.hmac_algorithm] - if BackupHeader.compressed in header_data: - compressed = header_data[BackupHeader.compressed] - if BackupHeader.encrypted in header_data: - encrypted = header_data[BackupHeader.encrypted] - if BackupHeader.compression_filter in header_data: - compression_filter = header_data[BackupHeader.compression_filter] - - if BACKUP_DEBUG: - print "Loading file", qubes_xml - backup_collection = QubesVmCollection(store_filename=qubes_xml) - backup_collection.lock_db_for_reading() - backup_collection.load() - - if host_collection is None: - host_collection = QubesVmCollection() - host_collection.lock_db_for_reading() - host_collection.load() - host_collection.unlock_db() - - backup_vms_list = [vm for vm in backup_collection.values()] - vms_to_restore = {} - - # ... and the actual data - for vm in backup_vms_list: - if vm.qid == 0: - # Handle dom0 as special case later - continue - if is_vm_included_in_backup(backup_location, vm): - if BACKUP_DEBUG: - print vm.name, "is included in backup" - - vms_to_restore[vm.name] = {} - vms_to_restore[vm.name]['vm'] = vm - - if vm.template is None: - vms_to_restore[vm.name]['template'] = None - else: - templatevm_name = find_template_name(vm.template.name, options[ - 'replace-template']) - vms_to_restore[vm.name]['template'] = templatevm_name - - if vm.netvm is None: - vms_to_restore[vm.name]['netvm'] = None - else: - netvm_name = vm.netvm.name - vms_to_restore[vm.name]['netvm'] = netvm_name - # Set to None to not confuse QubesVm object from backup - # collection with host collection (further in clone_attrs). Set - # directly _netvm to suppress setter action, especially - # modifying firewall - vm._netvm = None - - # Store restore parameters - options['location'] = backup_location - options['restore_tmpdir'] = restore_tmpdir - options['passphrase'] = passphrase - options['encrypted'] = encrypted - options['compressed'] = compressed - options['compression_filter'] = compression_filter - options['hmac_algorithm'] = hmac_algorithm - options['crypto_algorithm'] = crypto_algorithm - options['appvm'] = appvm - options['format_version'] = format_version - vms_to_restore['$OPTIONS$'] = options - - vms_to_restore = restore_info_verify(vms_to_restore, host_collection) - - # ...and dom0 home - if options['dom0-home'] and \ - is_vm_included_in_backup(backup_location, backup_collection[0]): - vm = backup_collection[0] - vms_to_restore['dom0'] = {} - if format_version == 1: - vms_to_restore['dom0']['subdir'] = \ - os.listdir(os.path.join(backup_location, 'dom0-home'))[0] - vms_to_restore['dom0']['size'] = 0 # unknown - else: - vms_to_restore['dom0']['subdir'] = vm.backup_path - vms_to_restore['dom0']['size'] = vm.backup_size - local_user = grp.getgrnam('qubes').gr_mem[0] - - dom0_home = vms_to_restore['dom0']['subdir'] - - vms_to_restore['dom0']['username'] = os.path.basename(dom0_home) - if vms_to_restore['dom0']['username'] != local_user: - vms_to_restore['dom0']['username-mismatch'] = True - if options['ignore-username-mismatch']: - vms_to_restore['dom0']['ignore-username-mismatch'] = True - else: - vms_to_restore['dom0']['good-to-go'] = False - - if 'good-to-go' not in vms_to_restore['dom0']: - vms_to_restore['dom0']['good-to-go'] = True - - # Not needed - all the data stored in vms_to_restore - if format_version >= 2: - os.unlink(qubes_xml) - return vms_to_restore - - -def backup_restore_print_summary(restore_info, print_callback=print_stdout): - fields = { - "qid": {"func": "vm.qid"}, - - "name": {"func": "('[' if vm.is_template() else '')\ - + ('{' if vm.is_netvm() else '')\ - + vm.name \ - + (']' if vm.is_template() else '')\ - + ('}' if vm.is_netvm() else '')"}, - - "type": {"func": "'Tpl' if vm.is_template() else \ - 'HVM' if vm.type == 'HVM' else \ - vm.type.replace('VM','')"}, - - "updbl": {"func": "'Yes' if vm.updateable else ''"}, - - "template": {"func": "'n/a' if vm.is_template() or vm.template is None else\ - vm_info['template']"}, - - "netvm": {"func": "'n/a' if vm.is_netvm() and not vm.is_proxyvm() else\ - ('*' if vm.uses_default_netvm else '') +\ - vm_info['netvm'] if vm_info['netvm'] is not None else '-'"}, - - "label": {"func": "vm.label.name"}, - } - - fields_to_display = ["name", "type", "template", "updbl", "netvm", "label"] - - # First calculate the maximum width of each field we want to display - total_width = 0 - for f in fields_to_display: - fields[f]["max_width"] = len(f) - for vm_info in restore_info.values(): - if 'vm' in vm_info.keys(): - # noinspection PyUnusedLocal - vm = vm_info['vm'] - l = len(unicode(eval(fields[f]["func"]))) - if l > fields[f]["max_width"]: - fields[f]["max_width"] = l - total_width += fields[f]["max_width"] - - print_callback("") - print_callback("The following VMs are included in the backup:") - print_callback("") - - # Display the header - s = "" - for f in fields_to_display: - fmt = "{{0:-^{0}}}-+".format(fields[f]["max_width"] + 1) - s += fmt.format('-') - print_callback(s) - s = "" - for f in fields_to_display: - fmt = "{{0:>{0}}} |".format(fields[f]["max_width"] + 1) - s += fmt.format(f) - print_callback(s) - s = "" - for f in fields_to_display: - fmt = "{{0:-^{0}}}-+".format(fields[f]["max_width"] + 1) - s += fmt.format('-') - print_callback(s) - - for vm_info in restore_info.values(): - # Skip non-VM here - if 'vm' not in vm_info: - continue - # noinspection PyUnusedLocal - vm = vm_info['vm'] - s = "" - for f in fields_to_display: - fmt = "{{0:>{0}}} |".format(fields[f]["max_width"] + 1) - s += fmt.format(eval(fields[f]["func"])) - - if 'excluded' in vm_info and vm_info['excluded']: - s += " <-- Excluded from restore" - elif 'already-exists' in vm_info: - s += " <-- A VM with the same name already exists on the host!" - elif 'missing-template' in vm_info: - s += " <-- No matching template on the host or in the backup found!" - elif 'missing-netvm' in vm_info: - s += " <-- No matching netvm on the host or in the backup found!" - else: - if 'orig-template' in vm_info: - s += " <-- Original template was '%s'" % (vm_info['orig-template']) - if 'rename-to' in vm_info: - s += " <-- Will be renamed to '%s'" % vm_info['rename-to'] - - print_callback(s) - - if 'dom0' in restore_info.keys(): - s = "" - for f in fields_to_display: - fmt = "{{0:>{0}}} |".format(fields[f]["max_width"] + 1) - if f == "name": - s += fmt.format("Dom0") - elif f == "type": - s += fmt.format("Home") - else: - s += fmt.format("") - if 'username-mismatch' in restore_info['dom0']: - s += " <-- username in backup and dom0 mismatch" - if 'ignore-username-mismatch' in restore_info['dom0']: - s += " (ignored)" - - print_callback(s) - - -def backup_restore_do(restore_info, - host_collection=None, print_callback=print_stdout, - error_callback=print_stderr, progress_callback=None, - ): - global running_backup_operation - - # Private functions begin - def restore_vm_dir_v1(backup_dir, src_dir, dst_dir): - - backup_src_dir = src_dir.replace(system_path["qubes_base_dir"], - backup_dir) - - # We prefer to use Linux's cp, because it nicely handles sparse files - cp_retcode = subprocess.call(["cp", "-rp", "--reflink=auto", backup_src_dir, dst_dir]) - if cp_retcode != 0: - raise QubesException( - "*** Error while copying file {0} to {1}".format(backup_src_dir, - dst_dir)) - - # Private functions end - - options = restore_info['$OPTIONS$'] - backup_location = options['location'] - restore_tmpdir = options['restore_tmpdir'] - passphrase = options['passphrase'] - encrypted = options['encrypted'] - compressed = options['compressed'] - compression_filter = options['compression_filter'] - hmac_algorithm = options['hmac_algorithm'] - crypto_algorithm = options['crypto_algorithm'] - verify_only = options['verify-only'] - appvm = options['appvm'] - format_version = options['format_version'] - - if format_version is None: - format_version = backup_detect_format_version(backup_location) - - lock_obtained = False - if host_collection is None: - host_collection = QubesVmCollection() - host_collection.lock_db_for_writing() - host_collection.load() - lock_obtained = True - - # Perform VM restoration in backup order - vms_dirs = [] - vms_size = 0 - vms = {} - for vm_info in restore_info.values(): - if 'vm' not in vm_info: - continue - if not vm_info['good-to-go']: - continue - vm = vm_info['vm'] - if format_version >= 2: - vms_size += vm.backup_size - vms_dirs.append(vm.backup_path) - vms[vm.name] = vm - - running_backup_operation = BackupOperationInfo() - - if format_version >= 2: - if 'dom0' in restore_info.keys() and restore_info['dom0']['good-to-go']: - vms_dirs.append(os.path.dirname(restore_info['dom0']['subdir'])) - vms_size += restore_info['dom0']['size'] - - try: - restore_vm_dirs(backup_location, - restore_tmpdir, - passphrase=passphrase, - vms_dirs=vms_dirs, - vms=vms, - vms_size=vms_size, - format_version=format_version, - hmac_algorithm=hmac_algorithm, - crypto_algorithm=crypto_algorithm, - verify_only=verify_only, - print_callback=print_callback, - error_callback=error_callback, - progress_callback=progress_callback, - encrypted=encrypted, - compressed=compressed, - compression_filter=compression_filter, - appvm=appvm) - except QubesException: - if verify_only: - raise - else: - if callable(print_callback): - print_callback( - "Some errors occurred during data extraction, " - "continuing anyway to restore at least some " - "VMs") - else: - if verify_only: - if callable(print_callback): - print_callback("WARNING: Backup verification not supported for " - "this backup format.") - - if verify_only: - shutil.rmtree(restore_tmpdir) - return - - # Add VM in right order - for (vm_class_name, vm_class) in sorted(QubesVmClasses.items(), - key=lambda _x: _x[1].load_order): - if running_backup_operation.canceled: - break - for vm in vms.values(): - if running_backup_operation.canceled: - # only break the loop to save qubes.xml with already restored - # VMs - break - if not vm.__class__ == vm_class: - continue - if callable(print_callback): - print_callback("-> Restoring {type} {0}...". - format(vm.name, type=vm_class_name)) - retcode = subprocess.call( - ["mkdir", "-p", os.path.dirname(vm.dir_path)]) - if retcode != 0: - error_callback("*** Cannot create directory: {0}?!".format( - vm.dir_path)) - error_callback("Skipping...") - continue - - template = None - if vm.template is not None: - template_name = restore_info[vm.name]['template'] - template = host_collection.get_vm_by_name(template_name) - - new_vm = None - vm_name = vm.name - if 'rename-to' in restore_info[vm.name]: - vm_name = restore_info[vm.name]['rename-to'] - - try: - new_vm = host_collection.add_new_vm(vm_class_name, name=vm_name, - template=template, - installed_by_rpm=False) - if os.path.exists(new_vm.dir_path): - move_to_path = tempfile.mkdtemp('', os.path.basename( - new_vm.dir_path), os.path.dirname(new_vm.dir_path)) - try: - os.rename(new_vm.dir_path, move_to_path) - error_callback( - "*** Directory {} already exists! It has " - "been moved to {}".format(new_vm.dir_path, - move_to_path)) - except OSError: - error_callback( - "*** Directory {} already exists and " - "cannot be moved!".format(new_vm.dir_path)) - error_callback("Skipping...") - continue - - if format_version == 1: - restore_vm_dir_v1(backup_location, - vm.dir_path, - os.path.dirname(new_vm.dir_path)) - elif format_version >= 2: - shutil.move(os.path.join(restore_tmpdir, vm.backup_path), - new_vm.dir_path) - - new_vm.verify_files() - except Exception as err: - error_callback("ERROR: {0}".format(err)) - error_callback("*** Skipping VM: {0}".format(vm.name)) - if new_vm: - host_collection.pop(new_vm.qid) - continue - - # FIXME: cannot check for 'kernel' property, because it is always - # defined - accessing it touches non-existent '_kernel' - if not isinstance(vm, QubesVmClasses['QubesHVm']): - # TODO: add a setting for this? - if vm.kernel and vm.kernel not in \ - os.listdir(system_path['qubes_kernels_base_dir']): - if callable(print_callback): - print_callback("WARNING: Kernel %s not installed, " - "using default one" % vm.kernel) - vm.uses_default_kernel = True - vm.kernel = host_collection.get_default_kernel() - try: - new_vm.clone_attrs(vm) - except Exception as err: - error_callback("ERROR: {0}".format(err)) - error_callback("*** Some VM property will not be restored") - - try: - new_vm.appmenus_create(verbose=callable(print_callback)) - except Exception as err: - error_callback("ERROR during appmenu restore: {0}".format(err)) - error_callback( - "*** VM '{0}' will not have appmenus".format(vm.name)) - - # Set network dependencies - only non-default netvm setting - for vm in vms.values(): - vm_name = vm.name - if 'rename-to' in restore_info[vm.name]: - vm_name = restore_info[vm.name]['rename-to'] - host_vm = host_collection.get_vm_by_name(vm_name) - - if host_vm is None: - # Failed/skipped VM - continue - - if not vm.uses_default_netvm: - if restore_info[vm.name]['netvm'] is not None: - host_vm.netvm = host_collection.get_vm_by_name( - restore_info[vm.name]['netvm']) - else: - host_vm.netvm = None - - host_collection.save() - if lock_obtained: - host_collection.unlock_db() - - if running_backup_operation.canceled: - if format_version >= 2: - raise BackupCanceledError("Restore canceled", - tmpdir=restore_tmpdir) - else: - raise BackupCanceledError("Restore canceled") - - # ... and dom0 home as last step - if 'dom0' in restore_info.keys() and restore_info['dom0']['good-to-go']: - backup_path = restore_info['dom0']['subdir'] - local_user = grp.getgrnam('qubes').gr_mem[0] - home_dir = pwd.getpwnam(local_user).pw_dir - if format_version == 1: - backup_dom0_home_dir = os.path.join(backup_location, backup_path) - else: - backup_dom0_home_dir = os.path.join(restore_tmpdir, backup_path) - restore_home_backupdir = "home-pre-restore-{0}".format( - time.strftime("%Y-%m-%d-%H%M%S")) - - if callable(print_callback): - print_callback( - "-> Restoring home of user '{0}'...".format(local_user)) - print_callback( - "--> Existing files/dirs backed up in '{0}' dir".format( - restore_home_backupdir)) - os.mkdir(home_dir + '/' + restore_home_backupdir) - for f in os.listdir(backup_dom0_home_dir): - home_file = home_dir + '/' + f - if os.path.exists(home_file): - os.rename(home_file, - home_dir + '/' + restore_home_backupdir + '/' + f) - if format_version == 1: - subprocess.call( - ["cp", "-nrp", "--reflink=auto", backup_dom0_home_dir + '/' + f, home_file]) - elif format_version >= 2: - shutil.move(backup_dom0_home_dir + '/' + f, home_file) - retcode = subprocess.call(['sudo', 'chown', '-R', local_user, home_dir]) - if retcode != 0: - error_callback("*** Error while setting home directory owner") - - shutil.rmtree(restore_tmpdir) - -# vim:sw=4:et: diff --git a/qubes/__init__.py b/qubes/__init__.py index 1a805cc6..e8758668 100644 --- a/qubes/__init__.py +++ b/qubes/__init__.py @@ -1224,6 +1224,9 @@ class Qubes(PropertyHolder): self.events_enabled = True + @__builtin__.property + def store(self): + return self._store def load(self): '''Open qubes.xml diff --git a/qubes/backup.py b/qubes/backup.py new file mode 100644 index 00000000..506ed63e --- /dev/null +++ b/qubes/backup.py @@ -0,0 +1,2261 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# The Qubes OS Project, http://www.qubes-os.org +# +# Copyright (C) 2013-2015 Marek Marczykowski-Górecki +# +# Copyright (C) 2013 Olivier Médoc +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +# +# +from __future__ import unicode_literals +import itertools +import logging +from qubes.utils import size_to_human +import sys +import os +import fcntl +import subprocess +import re +import shutil +import tempfile +import time +import grp +import pwd +import errno +import datetime +from multiprocessing import Queue, Process +import qubes +import qubes.core2migration +import qubes.storage + +QUEUE_ERROR = "ERROR" + +QUEUE_FINISHED = "FINISHED" + +HEADER_FILENAME = 'backup-header' +DEFAULT_CRYPTO_ALGORITHM = 'aes-256-cbc' +DEFAULT_HMAC_ALGORITHM = 'SHA512' +DEFAULT_COMPRESSION_FILTER = 'gzip' +CURRENT_BACKUP_FORMAT_VERSION = '4' +# Maximum size of error message get from process stderr (including VM process) +MAX_STDERR_BYTES = 1024 +# header + qubes.xml max size +HEADER_QUBES_XML_MAX_SIZE = 1024 * 1024 + +BLKSIZE = 512 + +_re_alphanum = re.compile(r'^[A-Za-z0-9-]*$') + +class BackupCanceledError(qubes.exc.QubesException): + def __init__(self, msg, tmpdir=None): + super(BackupCanceledError, self).__init__(msg) + self.tmpdir = tmpdir + + +class BackupHeader(object): + header_keys = { + 'version': 'version', + 'encrypted': 'encrypted', + 'compressed': 'compressed', + 'compression-filter': 'compression_filter', + 'crypto-algorithm': 'crypto_algorithm', + 'hmac-algorithm': 'hmac_algorithm', + } + bool_options = ['encrypted', 'compressed'] + int_options = ['version'] + + def __init__(self, + header_data=None, + version=None, + encrypted=None, + compressed=None, + compression_filter=None, + hmac_algorithm=None, + crypto_algorithm=None): + # repeat the list to help code completion... + self.version = version + self.encrypted = encrypted + self.compressed = compressed + # Options introduced in backup format 3+, which always have a header, + # so no need for fallback in function parameter + self.compression_filter = compression_filter + self.hmac_algorithm = hmac_algorithm + self.crypto_algorithm = crypto_algorithm + + if header_data is not None: + self.load(header_data) + + def load(self, untrusted_header_text): + """Parse backup header file. + + :param untrusted_header_text: header content + :type untrusted_header_text: basestring + .. warning:: + This function may be exposed to not yet verified header, + so is security critical. + """ + try: + untrusted_header_text = untrusted_header_text.decode('ascii') + except UnicodeDecodeError: + raise qubes.exc.QubesException( + "Non-ASCII characters in backup header") + for untrusted_line in untrusted_header_text.splitlines(): + if untrusted_line.count('=') != 1: + raise qubes.exc.QubesException("Invalid backup header") + key, value = untrusted_line.strip().split('=', 1) + if not _re_alphanum.match(key): + raise qubes.exc.QubesException("Invalid backup header (key)") + if key not in self.header_keys.keys(): + # Ignoring unknown option + continue + if not _re_alphanum.match(value): + raise qubes.exc.QubesException("Invalid backup header (value)") + if getattr(self, self.header_keys[key]) is not None: + raise qubes.exc.QubesException( + "Duplicated header line: {}".format(key)) + if key in self.bool_options: + value = value.lower() in ["1", "true", "yes"] + elif key in self.int_options: + value = int(value) + setattr(self, self.header_keys[key], value) + + self.validate() + + def validate(self): + if self.version == 1: + # header not really present + pass + elif self.version in [2, 3, 4]: + expected_attrs = ['version', 'encrypted', 'compressed', + 'hmac_algorithm'] + if self.encrypted: + expected_attrs += ['crypto_algorithm'] + if self.version >= 3 and self.compressed: + expected_attrs += ['compression_filter'] + for key in expected_attrs: + if getattr(self, key) is None: + raise qubes.exc.QubesException( + "Backup header lack '{}' info".format(key)) + else: + raise qubes.exc.QubesException( + "Unsupported backup version {}".format(self.version)) + + def save(self, filename): + with open(filename, "w") as f: + # make sure 'version' is the first key + f.write('version={}\n'.format(self.version)) + for key, attr in self.header_keys.iteritems(): + if key == 'version': + continue + if getattr(self, attr) is None: + continue + f.write("{!s}={!s}\n".format(key, getattr(self, attr))) + + +class SendWorker(Process): + def __init__(self, queue, base_dir, backup_stdout): + super(SendWorker, self).__init__() + self.queue = queue + self.base_dir = base_dir + self.backup_stdout = backup_stdout + self.log = logging.getLogger('qubes.backup') + + def run(self): + self.log.debug("Started sending thread") + + self.log.debug("Moving to temporary dir".format(self.base_dir)) + os.chdir(self.base_dir) + + for filename in iter(self.queue.get, None): + if filename in (QUEUE_FINISHED, QUEUE_ERROR): + break + + self.log.debug("Sending file {}".format(filename)) + # This tar used for sending data out need to be as simple, as + # simple, as featureless as possible. It will not be + # verified before untaring. + tar_final_cmd = ["tar", "-cO", "--posix", + "-C", self.base_dir, filename] + final_proc = subprocess.Popen(tar_final_cmd, + stdin=subprocess.PIPE, + stdout=self.backup_stdout) + if final_proc.wait() >= 2: + if self.queue.full(): + # if queue is already full, remove some entry to wake up + # main thread, so it will be able to notice error + self.queue.get() + # handle only exit code 2 (tar fatal error) or + # greater (call failed?) + raise qubes.exc.QubesException( + "ERROR: Failed to write the backup, out of disk space? " + "Check console output or ~/.xsession-errors for details.") + + # Delete the file as we don't need it anymore + self.log.debug("Removing file {}".format(filename)) + os.remove(filename) + + self.log.debug("Finished sending thread") + + +class Backup(object): + class FileToBackup(object): + def __init__(self, file_path, subdir=None): + sz = qubes.storage.get_disk_usage(file_path) + + if subdir is None: + abs_file_path = os.path.abspath(file_path) + abs_base_dir = os.path.abspath( + qubes.config.system_path["qubes_base_dir"]) + '/' + abs_file_dir = os.path.dirname(abs_file_path) + '/' + (nothing, directory, subdir) = abs_file_dir.partition(abs_base_dir) + assert nothing == "" + assert directory == abs_base_dir + else: + if len(subdir) > 0 and not subdir.endswith('/'): + subdir += '/' + + self.path = file_path + self.size = sz + self.subdir = subdir + + class VMToBackup(object): + def __init__(self, vm, files, subdir): + self.vm = vm + self.files = files + self.subdir = subdir + + @property + def size(self): + return reduce(lambda x, y: x + y.size, self.files, 0) + + def __init__(self, app, vms_list=None, exclude_list=None, **kwargs): + """ + If vms = None, include all (sensible) VMs; + exclude_list is always applied + """ + super(Backup, self).__init__() + + #: progress of the backup - bytes handled of the current VM + self.chunk_size = 100 * 1024 * 1024 + self._current_vm_bytes = 0 + #: progress of the backup - bytes handled of finished VMs + self._done_vms_bytes = 0 + #: total backup size (set by :py:meth:`get_files_to_backup`) + self.total_backup_bytes = 0 + #: application object + self.app = app + #: directory for temporary files - set after creating the directory + self.tmpdir = None + + # Backup settings - defaults + #: should the backup be encrypted? + self.encrypted = True + #: should the backup be compressed? + self.compressed = True + #: what passphrase should be used to intergrity protect (and encrypt) + #: the backup; required + self.passphrase = None + #: custom hmac algorithm + self.hmac_algorithm = DEFAULT_HMAC_ALGORITHM + #: custom encryption algorithm + self.crypto_algorithm = DEFAULT_CRYPTO_ALGORITHM + #: custom compression filter; a program which process stdin to stdout + self.compression_filter = DEFAULT_COMPRESSION_FILTER + #: VM to which backup should be sent (if any) + self.target_vm = None + #: directory to save backup in (either in dom0 or target VM, + #: depending on :py:attr:`target_vm` + self.target_dir = None + #: callback for progress reporting. Will be called with one argument + #: - progress in percents + self.progress_callback = None + + for key, value in kwargs.iteritems(): + if hasattr(self, key): + setattr(self, key, value) + else: + raise AttributeError(key) + + #: whether backup was canceled + self.canceled = False + #: list of PIDs to kill on backup cancel + self.processes_to_kill_on_cancel = [] + + self.log = logging.getLogger('qubes.backup') + + # FIXME: drop this legacy feature? + if isinstance(self.compressed, basestring): + self.compression_filter = self.compressed + self.compressed = True + else: + self.compression_filter = DEFAULT_COMPRESSION_FILTER + + if exclude_list is None: + exclude_list = [] + + if vms_list is None: + vms_list = [vm for vm in app.domains if vm.include_in_backups] + + # Apply exclude list + self.vms_for_backup = [vm for vm in vms_list + if vm.name not in exclude_list] + + def __del__(self): + if self.tmpdir and os.path.exists(self.tmpdir): + shutil.rmtree(self.tmpdir) + + def cancel(self): + """Cancel running backup operation. Can be called from another thread. + """ + self.canceled = True + for proc in self.processes_to_kill_on_cancel: + try: + proc.terminate() + except OSError: + pass + + + def get_files_to_backup(self): + files_to_backup = {} + for vm in self.vms_for_backup: + if vm.qid == 0: + # handle dom0 later + continue + + if self.encrypted: + subdir = 'vm%d/' % vm.qid + else: + subdir = None + + vm_files = [] + if vm.private_img is not None: + vm_files.append(self.FileToBackup(vm.private_img, subdir)) + + vm_files.append(self.FileToBackup(vm.icon_path, subdir)) + vm_files.extend(self.FileToBackup(i, subdir) + for i in vm.fire_event('backup-get-files')) + + # TODO: drop after merging firewall.xml into qubes.xml + firewall_conf = os.path.join(vm.dir_path, vm.firewall_conf) + if os.path.exists(firewall_conf): + vm_files.append(self.FileToBackup(firewall_conf, subdir)) + + if vm.updateable: + vm_files.append(self.FileToBackup(vm.root_img, subdir)) + files_to_backup[vm.qid] = self.VMToBackup(vm, vm_files, subdir) + + # Dom0 user home + if 0 in [vm.qid for vm in self.vms_for_backup]: + local_user = grp.getgrnam('qubes').gr_mem[0] + home_dir = pwd.getpwnam(local_user).pw_dir + # Home dir should have only user-owned files, so fix it now + # to prevent permissions problems - some root-owned files can + # left after 'sudo bash' and similar commands + subprocess.check_call(['sudo', 'chown', '-R', local_user, home_dir]) + + home_to_backup = [ + self.FileToBackup(home_dir, 'dom0-home/')] + vm_files = home_to_backup + + files_to_backup[0] = self.VMToBackup(self.app.domains[0], + vm_files, + os.path.join('dom0-home', os.path.basename(home_dir))) + + self.total_backup_bytes = reduce( + lambda x, y: x + y.size, files_to_backup.values(), 0) + return files_to_backup + + + def get_backup_summary(self): + summary = "" + + fields_to_display = [ + {"name": "VM", "width": 16}, + {"name": "type", "width": 12}, + {"name": "size", "width": 12} + ] + + # Display the header + for f in fields_to_display: + fmt = "{{0:-^{0}}}-+".format(f["width"] + 1) + summary += fmt.format('-') + summary += "\n" + for f in fields_to_display: + fmt = "{{0:>{0}}} |".format(f["width"] + 1) + summary += fmt.format(f["name"]) + summary += "\n" + for f in fields_to_display: + fmt = "{{0:-^{0}}}-+".format(f["width"] + 1) + summary += fmt.format('-') + summary += "\n" + + files_to_backup = self.get_files_to_backup() + + for qid, vm_info in files_to_backup.iteritems(): + s = "" + fmt = "{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1) + s += fmt.format(vm_info['vm'].name) + + fmt = "{{0:>{0}}} |".format(fields_to_display[1]["width"] + 1) + if qid == 0: + s += fmt.format("User home") + elif vm_info['vm'].is_template(): + s += fmt.format("Template VM") + else: + s += fmt.format("VM" + (" + Sys" if vm_info['vm'].updateable + else "")) + + vm_size = vm_info['size'] + + fmt = "{{0:>{0}}} |".format(fields_to_display[2]["width"] + 1) + s += fmt.format(size_to_human(vm_size)) + + if qid != 0 and vm_info['vm'].is_running(): + s += " <-- The VM is running, please shut it down before proceeding " \ + "with the backup!" + + summary += s + "\n" + + for f in fields_to_display: + fmt = "{{0:-^{0}}}-+".format(f["width"] + 1) + summary += fmt.format('-') + summary += "\n" + + fmt = "{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1) + summary += fmt.format("Total size:") + fmt = "{{0:>{0}}} |".format( + fields_to_display[1]["width"] + 1 + 2 + fields_to_display[2][ + "width"] + 1) + summary += fmt.format(size_to_human(self.total_backup_bytes)) + summary += "\n" + + for f in fields_to_display: + fmt = "{{0:-^{0}}}-+".format(f["width"] + 1) + summary += fmt.format('-') + summary += "\n" + + vms_not_for_backup = [vm.name for vm in self.app.domains + if vm not in self.vms_for_backup] + summary += "VMs not selected for backup:\n - " + "\n - ".join( + vms_not_for_backup) + + return summary + + def prepare_backup_header(self): + header_file_path = os.path.join(self.tmpdir, HEADER_FILENAME) + backup_header = BackupHeader( + version=CURRENT_BACKUP_FORMAT_VERSION, + hmac_algorithm=self.hmac_algorithm, + crypto_algorithm=self.crypto_algorithm, + encrypted=self.encrypted, + compressed=self.compressed, + compression_filter=self.compression_filter, + ) + backup_header.save(header_file_path) + + hmac = subprocess.Popen( + ["openssl", "dgst", "-" + self.hmac_algorithm, + "-hmac", self.passphrase], + stdin=open(header_file_path, "r"), + stdout=open(header_file_path + ".hmac", "w")) + if hmac.wait() != 0: + raise qubes.exc.QubesException( + "Failed to compute hmac of header file") + return HEADER_FILENAME, HEADER_FILENAME + ".hmac" + + + @staticmethod + def _queue_put_with_check(proc, vmproc, queue, element): + if queue.full(): + if not proc.is_alive(): + if vmproc: + message = ("Failed to write the backup, VM output:\n" + + vmproc.stderr.read()) + else: + message = "Failed to write the backup. Out of disk space?" + raise qubes.exc.QubesException(message) + queue.put(element) + + def _send_progress_update(self): + if callable(self.progress_callback): + progress = ( + 100 * (self._done_vms_bytes + self._current_vm_bytes) / + self.total_backup_bytes) + self.progress_callback(progress) + + def _add_vm_progress(self, bytes_done): + self._current_vm_bytes += bytes_done + self._send_progress_update() + + def backup_do(self): + if self.passphrase is None: + raise qubes.exc.QubesException("No passphrase set") + qubes_xml = self.app.store + self.tmpdir = tempfile.mkdtemp() + shutil.copy(qubes_xml, os.path.join(self.tmpdir, 'qubes.xml')) + qubes_xml = os.path.join(self.tmpdir, 'qubes.xml') + backup_app = qubes.Qubes(qubes_xml) + + # FIXME: cache it earlier? + files_to_backup = self.get_files_to_backup() + # make sure backup_content isn't set initially + for vm in backup_app.domains: + vm.features['backup-content'] = False + + for qid, vm_info in files_to_backup.iteritems(): + if qid != 0 and vm_info.vm.is_running(): + raise qubes.exc.QubesVMNotHaltedError(vm_info.vm) + # VM is included in the backup + backup_app.domains[qid].features['backup-content'] = True + backup_app.domains[qid].features['backup-path'] = vm_info.subdir + backup_app.domains[qid].features['backup-size'] = vm_info.size + backup_app.save() + + passphrase = self.passphrase.encode('utf-8') + + vmproc = None + tar_sparse = None + if self.target_vm is not None: + # Prepare the backup target (Qubes service call) + # If APPVM, STDOUT is a PIPE + vmproc = self.target_vm.run_service('qubes.Backup', + passio_popen=True, passio_stderr=True) + vmproc.stdin.write(self.target_dir. + replace("\r", "").replace("\n", "") + "\n") + backup_stdout = vmproc.stdin + self.processes_to_kill_on_cancel.append(vmproc) + else: + # Prepare the backup target (local file) + if os.path.isdir(self.target_dir): + backup_target = self.target_dir + "/qubes-{0}". \ + format(time.strftime("%Y-%m-%dT%H%M%S")) + else: + backup_target = self.target_dir + + # Create the target directory + if not os.path.exists(os.path.dirname(self.target_dir)): + raise qubes.exc.QubesException( + "ERROR: the backup directory for {0} does not exists". + format(self.target_dir)) + + # If not APPVM, STDOUT is a local file + backup_stdout = open(backup_target, 'wb') + + # Tar with tape length does not deals well with stdout + # (close stdout between two tapes) + # For this reason, we will use named pipes instead + self.log.debug("Working in {}".format(self.tmpdir)) + + backup_pipe = os.path.join(self.tmpdir, "backup_pipe") + self.log.debug("Creating pipe in: {}".format(backup_pipe)) + os.mkfifo(backup_pipe) + + self.log.debug("Will backup: {}".format(files_to_backup)) + + header_files = self.prepare_backup_header() + + # Setup worker to send encrypted data chunks to the backup_target + to_send = Queue(10) + send_proc = SendWorker(to_send, self.tmpdir, backup_stdout) + send_proc.start() + + for f in header_files: + to_send.put(f) + + vm_files_to_backup = self.get_files_to_backup() + qubes_xml_info = self.VMToBackup( + None, + [self.FileToBackup(qubes_xml, '')], + '' + ) + for vm_info in itertools.chain([qubes_xml_info], + vm_files_to_backup.itervalues()): + for file_info in vm_info.files: + + self.log.debug("Backing up {}".format(file_info)) + + backup_tempfile = os.path.join( + self.tmpdir, file_info.subdir, + os.path.basename(file_info.path)) + self.log.debug("Using temporary location: {}".format( + backup_tempfile)) + + # Ensure the temporary directory exists + if not os.path.isdir(os.path.dirname(backup_tempfile)): + os.makedirs(os.path.dirname(backup_tempfile)) + + # The first tar cmd can use any complex feature as we want. + # Files will be verified before untaring this. + # Prefix the path in archive with filename["subdir"] to have it + # verified during untar + tar_cmdline = (["tar", "-Pc", '--sparse', + "-f", backup_pipe, + '-C', os.path.dirname(file_info.path)] + + (['--dereference'] if + file_info.subdir != "dom0-home/" else []) + + ['--xform', 's:^%s:%s\\0:' % ( + os.path.basename(file_info.path), + file_info.subdir), + os.path.basename(file_info.path) + ]) + if self.compressed: + tar_cmdline.insert(-1, + "--use-compress-program=%s" % self.compression_filter) + + self.log.debug(" ".join(tar_cmdline)) + + # Tips: Popen(bufsize=0) + # Pipe: tar-sparse | encryptor [| hmac] | tar | backup_target + # Pipe: tar-sparse [| hmac] | tar | backup_target + # TODO: log handle stderr + tar_sparse = subprocess.Popen( + tar_cmdline, stdin=subprocess.PIPE) + self.processes_to_kill_on_cancel.append(tar_sparse) + + # Wait for compressor (tar) process to finish or for any + # error of other subprocesses + i = 0 + run_error = "paused" + encryptor = None + if self.encrypted: + # Start encrypt + # If no cipher is provided, + # the data is forwarded unencrypted !!! + encryptor = subprocess.Popen([ + "openssl", "enc", + "-e", "-" + self.crypto_algorithm, + "-pass", "pass:" + passphrase], + stdin=open(backup_pipe, 'rb'), + stdout=subprocess.PIPE) + pipe = encryptor.stdout + else: + pipe = open(backup_pipe, 'rb') + while run_error == "paused": + + # Start HMAC + hmac = subprocess.Popen([ + "openssl", "dgst", "-" + self.hmac_algorithm, + "-hmac", passphrase], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + + # Prepare a first chunk + chunkfile = backup_tempfile + "." + "%03d" % i + i += 1 + chunkfile_p = open(chunkfile, 'wb') + + common_args = { + 'backup_target': chunkfile_p, + 'hmac': hmac, + 'vmproc': vmproc, + 'addproc': tar_sparse, + 'progress_callback': self._add_vm_progress, + 'size_limit': self.chunk_size, + } + run_error = wait_backup_feedback( + in_stream=pipe, streamproc=encryptor, + **common_args) + chunkfile_p.close() + + self.log.debug( + "Wait_backup_feedback returned: {}".format(run_error)) + + if self.canceled: + try: + tar_sparse.terminate() + except OSError: + pass + try: + hmac.terminate() + except OSError: + pass + tar_sparse.wait() + hmac.wait() + to_send.put(QUEUE_ERROR) + send_proc.join() + shutil.rmtree(self.tmpdir) + raise BackupCanceledError("Backup canceled") + if run_error and run_error != "size_limit": + send_proc.terminate() + if run_error == "VM" and vmproc: + raise qubes.exc.QubesException( + "Failed to write the backup, VM output:\n" + + vmproc.stderr.read(MAX_STDERR_BYTES)) + else: + raise qubes.exc.QubesException( + "Failed to perform backup: error in " + + run_error) + + # Send the chunk to the backup target + self._queue_put_with_check( + send_proc, vmproc, to_send, + os.path.relpath(chunkfile, self.tmpdir)) + + # Close HMAC + hmac.stdin.close() + hmac.wait() + self.log.debug("HMAC proc return code: {}".format( + hmac.poll())) + + # Write HMAC data next to the chunk file + hmac_data = hmac.stdout.read() + self.log.debug( + "Writing hmac to {}.hmac".format(chunkfile)) + with open(chunkfile + ".hmac", 'w') as hmac_file: + hmac_file.write(hmac_data) + + # Send the HMAC to the backup target + self._queue_put_with_check( + send_proc, vmproc, to_send, + os.path.relpath(chunkfile, self.tmpdir) + ".hmac") + + if tar_sparse.poll() is None or run_error == "size_limit": + run_error = "paused" + else: + self.processes_to_kill_on_cancel.remove(tar_sparse) + self.log.debug( + "Finished tar sparse with exit code {}".format( + tar_sparse.poll())) + pipe.close() + + # This VM done, update progress + self._done_vms_bytes += vm_info.size + self._current_vm_bytes = 0 + self._send_progress_update() + # Save date of last backup + if vm_info.vm: + vm_info.vm.backup_timestamp = datetime.datetime.now() + + self._queue_put_with_check(send_proc, vmproc, to_send, QUEUE_FINISHED) + send_proc.join() + shutil.rmtree(self.tmpdir) + + if self.canceled: + raise BackupCanceledError("Backup canceled") + + if send_proc.exitcode != 0: + raise qubes.exc.QubesException( + "Failed to send backup: error in the sending process") + + if vmproc: + self.log.debug("VMProc1 proc return code: {}".format(vmproc.poll())) + if tar_sparse is not None: + self.log.debug("Sparse1 proc return code: {}".format( + tar_sparse.poll())) + vmproc.stdin.close() + + self.app.save() + + +''' +' Wait for backup chunk to finish +' - Monitor all the processes (streamproc, hmac, vmproc, addproc) for errors +' - Copy stdout of streamproc to backup_target and hmac stdin if available +' - Compute progress based on total_backup_sz and send progress to +' progress_callback function +' - Returns if +' - one of the monitored processes error out (streamproc, hmac, vmproc, +' addproc), along with the processe that failed +' - all of the monitored processes except vmproc finished successfully +' (vmproc termination is controlled by the python script) +' - streamproc does not delivers any data anymore (return with the error +' "") +' - size_limit is provided and is about to be exceeded +''' + + +def wait_backup_feedback(progress_callback, in_stream, streamproc, + backup_target, hmac=None, vmproc=None, + addproc=None, + size_limit=None): + buffer_size = 409600 + + run_error = None + run_count = 1 + bytes_copied = 0 + log = logging.getLogger('qubes.backup') + while run_count > 0 and run_error is None: + + if size_limit and bytes_copied + buffer_size > size_limit: + return "size_limit" + buf = in_stream.read(buffer_size) + if callable(progress_callback): + progress_callback(len(buf)) + bytes_copied += len(buf) + + run_count = 0 + if hmac: + retcode = hmac.poll() + if retcode is not None: + if retcode != 0: + run_error = "hmac" + else: + run_count += 1 + + if addproc: + retcode = addproc.poll() + if retcode is not None: + if retcode != 0: + run_error = "addproc" + else: + run_count += 1 + + if vmproc: + retcode = vmproc.poll() + if retcode is not None: + if retcode != 0: + run_error = "VM" + log.debug(vmproc.stdout.read()) + else: + # VM should run until the end + pass + + if streamproc: + retcode = streamproc.poll() + if retcode is not None: + if retcode != 0: + run_error = "streamproc" + break + elif retcode == 0 and len(buf) <= 0: + return "" + run_count += 1 + + else: + if len(buf) <= 0: + return "" + + try: + backup_target.write(buf) + except IOError as e: + if e.errno == errno.EPIPE: + run_error = "target" + else: + raise + + if hmac: + hmac.stdin.write(buf) + + return run_error + + +class ExtractWorker2(Process): + def __init__(self, queue, base_dir, passphrase, encrypted, + progress_callback, vmproc=None, + compressed=False, crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM, + verify_only=False): + super(ExtractWorker2, self).__init__() + self.queue = queue + self.base_dir = base_dir + self.passphrase = passphrase + self.encrypted = encrypted + self.compressed = compressed + self.crypto_algorithm = crypto_algorithm + self.verify_only = verify_only + self.blocks_backedup = 0 + self.tar2_process = None + self.tar2_current_file = None + self.decompressor_process = None + self.decryptor_process = None + + self.progress_callback = progress_callback + + self.vmproc = vmproc + + self.restore_pipe = os.path.join(self.base_dir, "restore_pipe") + + self.log = logging.getLogger('qubes.backup.extract') + self.log.debug("Creating pipe in: {}".format(self.restore_pipe)) + os.mkfifo(self.restore_pipe) + + self.stderr_encoding = sys.stderr.encoding or 'utf-8' + + def collect_tar_output(self): + if not self.tar2_process.stderr: + return + + if self.tar2_process.poll() is None: + try: + new_lines = self.tar2_process.stderr \ + .read(MAX_STDERR_BYTES).splitlines() + except IOError as e: + if e.errno == errno.EAGAIN: + return + else: + raise + else: + new_lines = self.tar2_process.stderr.readlines() + + new_lines = map(lambda x: x.decode(self.stderr_encoding), new_lines) + + msg_re = re.compile(r".*#[0-9].*restore_pipe") + debug_msg = filter(msg_re.match, new_lines) + self.log.debug('tar2_stderr: {}'.format('\n'.join(debug_msg))) + new_lines = filter(lambda x: not msg_re.match(x), new_lines) + + self.tar2_stderr += new_lines + + def run(self): + try: + self.__run__() + except Exception as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + # Cleanup children + for process in [self.decompressor_process, + self.decryptor_process, + self.tar2_process]: + if process: + # FIXME: kill()? + try: + process.terminate() + except OSError: + pass + process.wait() + self.log.error("ERROR: " + unicode(e)) + raise e, None, exc_traceback + + def __run__(self): + self.log.debug("Started sending thread") + self.log.debug("Moving to dir " + self.base_dir) + os.chdir(self.base_dir) + + filename = None + + for filename in iter(self.queue.get, None): + if filename in (QUEUE_FINISHED, QUEUE_ERROR): + break + + self.log.debug("Extracting file " + filename) + + if filename.endswith('.000'): + # next file + if self.tar2_process is not None: + if self.tar2_process.wait() != 0: + self.collect_tar_output() + self.log.error( + "ERROR: unable to extract files for {0}, tar " + "output:\n {1}". + format(self.tar2_current_file, + "\n ".join(self.tar2_stderr))) + else: + # Finished extracting the tar file + self.tar2_process = None + self.tar2_current_file = None + + tar2_cmdline = ['tar', + '-%sMkvf' % ("t" if self.verify_only else "x"), + self.restore_pipe, + os.path.relpath(filename.rstrip('.000'))] + self.log.debug("Running command " + unicode(tar2_cmdline)) + self.tar2_process = subprocess.Popen(tar2_cmdline, + stdin=subprocess.PIPE, + stderr=subprocess.PIPE) + fcntl.fcntl(self.tar2_process.stderr.fileno(), fcntl.F_SETFL, + fcntl.fcntl(self.tar2_process.stderr.fileno(), + fcntl.F_GETFL) | os.O_NONBLOCK) + self.tar2_stderr = [] + elif not self.tar2_process: + # Extracting of the current archive failed, skip to the next + # archive + # TODO: some debug option to preserve it? + os.remove(filename) + continue + else: + self.collect_tar_output() + self.log.debug("Releasing next chunck") + self.tar2_process.stdin.write("\n") + self.tar2_process.stdin.flush() + self.tar2_current_file = filename + + pipe = open(self.restore_pipe, 'wb') + common_args = { + 'backup_target': pipe, + 'hmac': None, + 'vmproc': self.vmproc, + 'addproc': self.tar2_process + } + if self.encrypted: + # Start decrypt + self.decryptor_process = subprocess.Popen( + ["openssl", "enc", + "-d", + "-" + self.crypto_algorithm, + "-pass", + "pass:" + self.passphrase] + + (["-z"] if self.compressed else []), + stdin=open(filename, 'rb'), + stdout=subprocess.PIPE) + + run_error = wait_backup_feedback( + progress_callback=self.progress_callback, + in_stream=self.decryptor_process.stdout, + streamproc=self.decryptor_process, + **common_args) + elif self.compressed: + self.decompressor_process = subprocess.Popen( + ["gzip", "-d"], + stdin=open(filename, 'rb'), + stdout=subprocess.PIPE) + + run_error = wait_backup_feedback( + progress_callback=self.progress_callback, + in_stream=self.decompressor_process.stdout, + streamproc=self.decompressor_process, + **common_args) + else: + run_error = wait_backup_feedback( + progress_callback=self.progress_callback, + in_stream=open(filename, "rb"), streamproc=None, + **common_args) + + try: + pipe.close() + except IOError as e: + if e.errno == errno.EPIPE: + self.log.debug( + "Got EPIPE while closing pipe to " + "the inner tar process") + # ignore the error + else: + raise + if len(run_error): + if run_error == "target": + self.collect_tar_output() + details = "\n".join(self.tar2_stderr) + else: + details = "%s failed" % run_error + self.tar2_process.terminate() + self.tar2_process.wait() + self.tar2_process = None + self.log.error("Error while processing '{}': {}".format( + self.tar2_current_file, details)) + + # Delete the file as we don't need it anymore + self.log.debug("Removing file " + filename) + os.remove(filename) + + os.unlink(self.restore_pipe) + + if self.tar2_process is not None: + if filename == QUEUE_ERROR: + self.tar2_process.terminate() + self.tar2_process.wait() + elif self.tar2_process.wait() != 0: + self.collect_tar_output() + raise qubes.exc.QubesException( + "unable to extract files for {0}.{1} Tar command " + "output: %s". + format(self.tar2_current_file, + (" Perhaps the backup is encrypted?" + if not self.encrypted else "", + "\n".join(self.tar2_stderr)))) + else: + # Finished extracting the tar file + self.tar2_process = None + + self.log.debug("Finished extracting thread") + + +class ExtractWorker3(ExtractWorker2): + def __init__(self, queue, base_dir, passphrase, encrypted, + progress_callback, vmproc=None, + compressed=False, crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM, + compression_filter=None, verify_only=False): + super(ExtractWorker3, self).__init__(queue, base_dir, passphrase, + encrypted, + progress_callback, vmproc, + compressed, crypto_algorithm, + verify_only) + self.compression_filter = compression_filter + os.unlink(self.restore_pipe) + + def __run__(self): + self.log.debug("Started sending thread") + self.log.debug("Moving to dir " + self.base_dir) + os.chdir(self.base_dir) + + filename = None + + input_pipe = None + for filename in iter(self.queue.get, None): + if filename in (QUEUE_FINISHED, QUEUE_ERROR): + break + + self.log.debug("Extracting file " + filename) + + if filename.endswith('.000'): + # next file + if self.tar2_process is not None: + input_pipe.close() + if self.tar2_process.wait() != 0: + self.collect_tar_output() + self.log.error( + "ERROR: unable to extract files for {0}, tar " + "output:\n {1}". + format(self.tar2_current_file, + "\n ".join(self.tar2_stderr))) + else: + # Finished extracting the tar file + self.tar2_process = None + self.tar2_current_file = None + + tar2_cmdline = ['tar', + '-%sk' % ("t" if self.verify_only else "x"), + os.path.relpath(filename.rstrip('.000'))] + if self.compressed: + if self.compression_filter: + tar2_cmdline.insert(-1, + "--use-compress-program=%s" % + self.compression_filter) + else: + tar2_cmdline.insert(-1, "--use-compress-program=%s" % + DEFAULT_COMPRESSION_FILTER) + + self.log.debug("Running command " + unicode(tar2_cmdline)) + if self.encrypted: + # Start decrypt + self.decryptor_process = subprocess.Popen( + ["openssl", "enc", + "-d", + "-" + self.crypto_algorithm, + "-pass", + "pass:" + self.passphrase], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + + self.tar2_process = subprocess.Popen( + tar2_cmdline, + stdin=self.decryptor_process.stdout, + stderr=subprocess.PIPE) + input_pipe = self.decryptor_process.stdin + else: + self.tar2_process = subprocess.Popen( + tar2_cmdline, + stdin=subprocess.PIPE, + stderr=subprocess.PIPE) + input_pipe = self.tar2_process.stdin + + fcntl.fcntl(self.tar2_process.stderr.fileno(), fcntl.F_SETFL, + fcntl.fcntl(self.tar2_process.stderr.fileno(), + fcntl.F_GETFL) | os.O_NONBLOCK) + self.tar2_stderr = [] + elif not self.tar2_process: + # Extracting of the current archive failed, skip to the next + # archive + # TODO: some debug option to preserve it? + os.remove(filename) + continue + else: + self.log.debug("Releasing next chunck") + self.tar2_current_file = filename + + common_args = { + 'backup_target': input_pipe, + 'hmac': None, + 'vmproc': self.vmproc, + 'addproc': self.tar2_process + } + + run_error = wait_backup_feedback( + progress_callback=self.progress_callback, + in_stream=open(filename, "rb"), streamproc=None, + **common_args) + + if len(run_error): + if run_error == "target": + self.collect_tar_output() + details = "\n".join(self.tar2_stderr) + else: + details = "%s failed" % run_error + if self.decryptor_process: + self.decryptor_process.terminate() + self.decryptor_process.wait() + self.decryptor_process = None + self.tar2_process.terminate() + self.tar2_process.wait() + self.tar2_process = None + self.log.error("Error while processing '{}': {}".format( + self.tar2_current_file, details)) + + # Delete the file as we don't need it anymore + self.log.debug("Removing file " + filename) + os.remove(filename) + + if self.tar2_process is not None: + input_pipe.close() + if filename == QUEUE_ERROR: + if self.decryptor_process: + self.decryptor_process.terminate() + self.decryptor_process.wait() + self.decryptor_process = None + self.tar2_process.terminate() + self.tar2_process.wait() + elif self.tar2_process.wait() != 0: + self.collect_tar_output() + raise qubes.exc.QubesException( + "unable to extract files for {0}.{1} Tar command " + "output: %s". + format(self.tar2_current_file, + (" Perhaps the backup is encrypted?" + if not self.encrypted else "", + "\n".join(self.tar2_stderr)))) + else: + # Finished extracting the tar file + self.tar2_process = None + + self.log.debug("Finished extracting thread") + + +def get_supported_hmac_algo(hmac_algorithm=None): + # Start with provided default + if hmac_algorithm: + yield hmac_algorithm + proc = subprocess.Popen(['openssl', 'list-message-digest-algorithms'], + stdout=subprocess.PIPE) + for algo in proc.stdout.readlines(): + if '=>' in algo: + continue + yield algo.strip() + proc.wait() + + +class BackupRestoreOptions(object): + def __init__(self): + #: use default NetVM if the one referenced in backup do not exists on + # the host + self.use_default_netvm = True + #: set NetVM to "none" if the one referenced in backup do not exists + # on the host + self.use_none_netvm = False + #: set template to default if the one referenced in backup do not + # exists on the host + self.use_default_template = True + #: restore dom0 home + self.dom0_home = True + #: dictionary how what templates should be used instead of those + # referenced in backup + self.replace_template = {} + #: restore dom0 home even if username is different + self.ignore_username_mismatch = False + #: do not restore data, only verify backup integrity + self.verify_only = False + #: automatically rename VM during restore, when it would conflict + # with existing one + self.rename_conflicting = True + #: list of VM names to exclude + self.exclude = [] + + +class BackupRestore(object): + """Usage: + >>> restore_op = BackupRestore(...) + >>> # adjust restore_op.options here + >>> restore_info = restore_op.get_restore_info() + >>> # manipulate restore_info to select VMs to restore here + >>> restore_op.restore_do(restore_info) + """ + + class VMToRestore(object): + #: VM excluded from restore by user + EXCLUDED = object() + #: VM with such name already exists on the host + ALREADY_EXISTS = object() + #: NetVM used by the VM does not exists on the host + MISSING_NETVM = object() + #: TemplateVM used by the VM does not exists on the host + MISSING_TEMPLATE = object() + + def __init__(self, vm): + self.vm = vm + if 'backup-path' in vm.features: + self.subdir = vm.features['backup-path'] + else: + self.subdir = None + if 'backup-size' in vm.features and vm.features['backup-size']: + self.size = int(vm.features['backup-size']) + else: + self.size = 0 + self.problems = set() + if hasattr(vm, 'template') and vm.template: + self.template = vm.template.name + else: + self.template = None + if vm.netvm: + self.netvm = vm.netvm.name + else: + self.netvm = None + self.name = vm.name + self.orig_template = None + + @property + def good_to_go(self): + return len(self.problems) == 0 + + class Dom0ToRestore(VMToRestore): + #: backup was performed on system with different dom0 username + USERNAME_MISMATCH = object() + + def __init__(self, vm, subdir=None): + super(BackupRestore.Dom0ToRestore, self).__init__(vm) + if subdir: + self.subdir = subdir + self.username = os.path.basename(subdir) + + def __init__(self, app, backup_location, backup_vm, passphrase): + super(BackupRestore, self).__init__() + + #: qubes.Qubes instance + self.app = app + + #: options how the backup should be restored + self.options = BackupRestoreOptions() + + #: VM from which backup should be retrieved + self.backup_vm = backup_vm + if backup_vm and backup_vm.qid == 0: + self.backup_vm = None + + #: backup path, inside VM pointed by :py:attr:`backup_vm` + self.backup_location = backup_location + + #: passphrase protecting backup integrity and optionally decryption + self.passphrase = passphrase + + #: temporary directory used to extract the data before moving to the + # final location; should be on the same filesystem as /var/lib/qubes + self.tmpdir = tempfile.mkdtemp(prefix="restore", dir="/var/tmp") + + #: list of processes (Popen objects) to kill on cancel + self.processes_to_kill_on_cancel = [] + + #: is the backup operation canceled + self.canceled = False + + #: report restore progress, called with one argument - percents of + # data restored + # FIXME: convert to float [0,1] + self.progress_callback = None + + self.log = logging.getLogger('qubes.backup') + + #: basic information about the backup + self.header_data = self._retrieve_backup_header() + + #: VMs included in the backup + self.backup_app = self._process_qubes_xml() + + def cancel(self): + """Cancel running backup operation. Can be called from another thread. + """ + self.canceled = True + for proc in self.processes_to_kill_on_cancel: + try: + proc.terminate() + except OSError: + pass + + def _start_retrieval_process(self, filelist, limit_count, limit_bytes): + """Retrieve backup stream and extract it to :py:attr:`tmpdir` + + :param filelist: list of files to extract; listing directory name + will extract the whole directory; use empty list to extract the whole + archive + :param limit_count: maximum number of files to extract + :param limit_bytes: maximum size of extracted data + :return: a touple of (Popen object of started process, file-like + object for reading extracted files list, file-like object for reading + errors) + """ + + vmproc = None + if self.backup_vm is not None: + # If APPVM, STDOUT is a PIPE + vmproc = self.backup_vm.run_service('qubes.Restore', + passio_popen=True, passio_stderr=True) + vmproc.stdin.write( + self.backup_location.replace("\r", "").replace("\n", "") + "\n") + + # Send to tar2qfile the VMs that should be extracted + vmproc.stdin.write(" ".join(filelist) + "\n") + self.processes_to_kill_on_cancel.append(vmproc) + + backup_stdin = vmproc.stdout + tar1_command = ['/usr/libexec/qubes/qfile-dom0-unpacker', + str(os.getuid()), self.tmpdir, '-v'] + else: + backup_stdin = open(self.backup_location, 'rb') + + tar1_command = ['tar', + '-ixv', + '-C', self.tmpdir] + filelist + + tar1_env = os.environ.copy() + tar1_env['UPDATES_MAX_BYTES'] = str(limit_bytes) + tar1_env['UPDATES_MAX_FILES'] = str(limit_count) + self.log.debug("Run command" + unicode(tar1_command)) + command = subprocess.Popen( + tar1_command, + stdin=backup_stdin, + stdout=vmproc.stdin if vmproc else subprocess.PIPE, + stderr=subprocess.PIPE, + env=tar1_env) + self.processes_to_kill_on_cancel.append(command) + + # qfile-dom0-unpacker output filelist on stderr + # and have stdout connected to the VM), while tar output filelist + # on stdout + if self.backup_vm: + filelist_pipe = command.stderr + # let qfile-dom0-unpacker hold the only open FD to the write end of + # pipe, otherwise qrexec-client will not receive EOF when + # qfile-dom0-unpacker terminates + vmproc.stdin.close() + else: + filelist_pipe = command.stdout + + if self.backup_vm: + error_pipe = vmproc.stderr + else: + error_pipe = command.stderr + return command, filelist_pipe, error_pipe + + def _verify_hmac(self, filename, hmacfile, algorithm=None): + def load_hmac(hmac_text): + hmac_text = hmac_text.strip().split("=") + if len(hmac_text) > 1: + hmac_text = hmac_text[1].strip() + else: + raise qubes.exc.QubesException( + "ERROR: invalid hmac file content") + + return hmac_text + if algorithm is None: + algorithm = self.header_data.hmac_algorithm + passphrase = self.passphrase.encode('utf-8') + self.log.debug("Verifying file {}".format(filename)) + + if hmacfile != filename + ".hmac": + raise qubes.exc.QubesException( + "ERROR: expected hmac for {}, but got {}". + format(filename, hmacfile)) + + hmac_proc = subprocess.Popen( + ["openssl", "dgst", "-" + algorithm, "-hmac", passphrase], + stdin=open(os.path.join(self.tmpdir, filename), 'rb'), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + hmac_stdout, hmac_stderr = hmac_proc.communicate() + + if len(hmac_stderr) > 0: + raise qubes.exc.QubesException( + "ERROR: verify file {0}: {1}".format(filename, hmac_stderr)) + else: + self.log.debug("Loading hmac for file {}".format(filename)) + hmac = load_hmac(open(os.path.join(self.tmpdir, hmacfile), + 'r').read()) + + if len(hmac) > 0 and load_hmac(hmac_stdout) == hmac: + os.unlink(os.path.join(self.tmpdir, hmacfile)) + self.log.debug( + "File verification OK -> Sending file {}".format(filename)) + return True + else: + raise qubes.exc.QubesException( + "ERROR: invalid hmac for file {0}: {1}. " + "Is the passphrase correct?". + format(filename, load_hmac(hmac_stdout))) + + def _retrieve_backup_header(self): + """Retrieve backup header and qubes.xml. Only backup header is + analyzed, qubes.xml is left as-is + (not even verified/decrypted/uncompressed) + + :return header_data + :rtype :py:class:`BackupHeader` + """ + + if not self.backup_vm and os.path.exists( + os.path.join(self.backup_location, 'qubes.xml')): + # backup format version 1 doesn't have header + header_data = BackupHeader() + header_data.version = 1 + return header_data + + (retrieve_proc, filelist_pipe, error_pipe) = \ + self._start_retrieval_process( + ['backup-header', 'backup-header.hmac', + 'qubes.xml.000', 'qubes.xml.000.hmac'], 4, 1024 * 1024) + + expect_tar_error = False + + filename = filelist_pipe.readline().strip() + hmacfile = filelist_pipe.readline().strip() + # tar output filename before actually extracting it, so wait for the + # next one before trying to access it + if not self.backup_vm: + filelist_pipe.readline().strip() + + self.log.debug("Got backup header and hmac: {}, {}".format( + filename, hmacfile)) + + if not filename or filename == "EOF" or \ + not hmacfile or hmacfile == "EOF": + retrieve_proc.wait() + proc_error_msg = error_pipe.read(MAX_STDERR_BYTES) + raise qubes.exc.QubesException( + "Premature end of archive while receiving " + "backup header. Process output:\n" + proc_error_msg) + file_ok = False + hmac_algorithm = DEFAULT_HMAC_ALGORITHM + for hmac_algo in get_supported_hmac_algo(hmac_algorithm): + try: + if self._verify_hmac(filename, hmacfile, hmac_algo): + file_ok = True + hmac_algorithm = hmac_algo + break + except qubes.exc.QubesException: + # Ignore exception here, try the next algo + pass + if not file_ok: + raise qubes.exc.QubesException( + "Corrupted backup header (hmac verification " + "failed). Is the password correct?") + if os.path.basename(filename) == HEADER_FILENAME: + filename = os.path.join(self.tmpdir, filename) + header_data = BackupHeader(open(filename, 'r').read()) + os.unlink(filename) + else: + # if no header found, create one with guessed HMAC algo + header_data = BackupHeader( + version=2, + hmac_algorithm=hmac_algorithm, + # place explicitly this value, because it is what format_version + # 2 have + crypto_algorithm='aes-256-cbc', + # TODO: set encrypted to something... + ) + # when tar do not find expected file in archive, it exit with + # code 2. This will happen because we've requested backup-header + # file, but the archive do not contain it. Ignore this particular + # error. + if not self.backup_vm: + expect_tar_error = True + + if retrieve_proc.wait() != 0 and not expect_tar_error: + raise qubes.exc.QubesException( + "unable to read the qubes backup file {0} ({1}): {2}".format( + self.backup_location, + retrieve_proc.wait(), + error_pipe.read(MAX_STDERR_BYTES) + )) + if retrieve_proc in self.processes_to_kill_on_cancel: + self.processes_to_kill_on_cancel.remove(retrieve_proc) + # wait for other processes (if any) + for proc in self.processes_to_kill_on_cancel: + if proc.wait() != 0: + raise qubes.exc.QubesException( + "Backup header retrieval failed (exit code {})".format( + proc.wait()) + ) + return header_data + + def _start_inner_extraction_worker(self, queue): + """Start a worker process, extracting inner layer of bacup archive, + extract them to :py:attr:`tmpdir`. + End the data by pushing QUEUE_FINISHED or QUEUE_ERROR to the queue. + + :param queue :py:class:`Queue` object to handle files from + """ + + # Setup worker to extract encrypted data chunks to the restore dirs + # Create the process here to pass it options extracted from + # backup header + extractor_params = { + 'queue': queue, + 'base_dir': self.tmpdir, + 'passphrase': self.passphrase, + 'encrypted': self.header_data.encrypted, + 'compressed': self.header_data.compressed, + 'crypto_algorithm': self.header_data.crypto_algorithm, + 'verify_only': self.options.verify_only, + 'progress_callback': self.progress_callback, + } + format_version = self.header_data.version + if format_version == 2: + extract_proc = ExtractWorker2(**extractor_params) + elif format_version in [3, 4]: + extractor_params['compression_filter'] = \ + self.header_data.compression_filter + extract_proc = ExtractWorker3(**extractor_params) + else: + raise NotImplementedError( + "Backup format version %d not supported" % format_version) + extract_proc.start() + return extract_proc + + def _process_qubes_xml(self): + """Verify, unpack and load qubes.xml. Possibly convert its format if + necessary. It expect that :py:attr:`header_data` is already populated, + and :py:meth:`retrieve_backup_header` was called. + """ + if self.header_data.version == 1: + backup_app = qubes.core2migration.Core2Qubes( + os.path.join(self.backup_location, 'qubes.xml')) + return backup_app + else: + self._verify_hmac("qubes.xml.000", "qubes.xml.000.hmac") + queue = Queue() + queue.put("qubes.xml.000") + queue.put(QUEUE_FINISHED) + + extract_proc = self._start_inner_extraction_worker(queue) + extract_proc.join() + if extract_proc.exitcode != 0: + raise qubes.exc.QubesException( + "unable to extract the qubes backup. " + "Check extracting process errors.") + + if self.header_data.version in [2, 3]: + backup_app = qubes.core2migration.Core2Qubes( + os.path.join(self.tmpdir, 'qubes.xml')) + else: + backup_app = qubes.Qubes(os.path.join(self.tmpdir, 'qubes.xml')) + # Not needed anymore - all the data stored in backup_app + os.unlink(os.path.join(self.tmpdir, 'qubes.xml')) + return backup_app + + def _restore_vm_dirs(self, vms_dirs, vms_size): + # Currently each VM consists of at most 7 archives (count + # file_to_backup calls in backup_prepare()), but add some safety + # margin for further extensions. Each archive is divided into 100MB + # chunks. Additionally each file have own hmac file. So assume upper + # limit as 2*(10*COUNT_OF_VMS+TOTAL_SIZE/100MB) + limit_count = str(2 * (10 * len(vms_dirs) + + int(vms_size / (100 * 1024 * 1024)))) + + self.log.debug("Working in temporary dir:" + self.tmpdir) + self.log.info( + "Extracting data: " + size_to_human(vms_size) + " to restore") + + # retrieve backup from the backup stream (either VM, or dom0 file) + # TODO: add some safety margin in vms_size? + (retrieve_proc, filelist_pipe, error_pipe) = \ + self._start_retrieval_process(vms_dirs, limit_count, vms_size) + + to_extract = Queue() + + # extract data retrieved by retrieve_proc + extract_proc = self._start_inner_extraction_worker(to_extract) + + try: + filename = None + nextfile = None + while True: + if self.canceled: + break + if not extract_proc.is_alive(): + retrieve_proc.terminate() + retrieve_proc.wait() + if retrieve_proc in self.processes_to_kill_on_cancel: + self.processes_to_kill_on_cancel.remove(retrieve_proc) + # wait for other processes (if any) + for proc in self.processes_to_kill_on_cancel: + proc.wait() + break + if nextfile is not None: + filename = nextfile + else: + filename = filelist_pipe.readline().strip() + + self.log.debug("Getting new file:" + filename) + + if not filename or filename == "EOF": + break + + hmacfile = filelist_pipe.readline().strip() + + if self.canceled: + break + # if reading archive directly with tar, wait for next filename - + # tar prints filename before processing it, so wait for + # the next one to be sure that whole file was extracted + if not self.backup_vm: + nextfile = filelist_pipe.readline().strip() + + self.log.debug("Getting hmac:" + hmacfile) + if not hmacfile or hmacfile == "EOF": + # Premature end of archive, either of tar1_command or + # vmproc exited with error + break + + if not any(map(lambda x: filename.startswith(x), vms_dirs)): + self.log.debug("Ignoring VM not selected for restore") + os.unlink(os.path.join(self.tmpdir, filename)) + os.unlink(os.path.join(self.tmpdir, hmacfile)) + continue + + if self._verify_hmac(filename, hmacfile): + to_extract.put(os.path.join(self.tmpdir, filename)) + + if self.canceled: + raise BackupCanceledError("Restore canceled", + tmpdir=self.tmpdir) + + if retrieve_proc.wait() != 0: + raise qubes.exc.QubesException( + "unable to read the qubes backup file {0} ({1}): {2}" + .format(self.backup_location, error_pipe.read( + MAX_STDERR_BYTES))) + # wait for other processes (if any) + for proc in self.processes_to_kill_on_cancel: + # FIXME check 'vmproc' exit code? + proc.wait() + + if filename and filename != "EOF": + raise qubes.exc.QubesException( + "Premature end of archive, the last file was %s" % filename) + except: + to_extract.put(QUEUE_ERROR) + extract_proc.join() + raise + else: + to_extract.put(QUEUE_FINISHED) + + self.log.debug("Waiting for the extraction process to finish...") + extract_proc.join() + self.log.debug("Extraction process finished with code: {}".format( + extract_proc.exitcode)) + if extract_proc.exitcode != 0: + raise qubes.exc.QubesException( + "unable to extract the qubes backup. " + "Check extracting process errors.") + + def generate_new_name_for_conflicting_vm(self, orig_name, restore_info): + number = 1 + if len(orig_name) > 29: + orig_name = orig_name[0:29] + new_name = orig_name + while (new_name in restore_info.keys() or + new_name in map(lambda x: x.name, + restore_info.values()) or + new_name in self.app.domains): + new_name = str('{}{}'.format(orig_name, number)) + number += 1 + if number == 100: + # give up + return None + return new_name + + def restore_info_verify(self, restore_info): + for vm in restore_info.keys(): + if vm in ['dom0']: + continue + + vm_info = restore_info[vm] + assert isinstance(vm_info, self.VMToRestore) + + vm_info.problems.clear() + if vm in self.options.exclude: + vm_info.problems.add(self.VMToRestore.EXCLUDED) + + if not self.options.verify_only and \ + vm in self.app.domains: + if self.options.rename_conflicting: + new_name = self.generate_new_name_for_conflicting_vm( + vm, restore_info + ) + if new_name is not None: + vm_info.name = new_name + else: + vm_info.problems.add(self.VMToRestore.ALREADY_EXISTS) + else: + vm_info.problems.add(self.VMToRestore.ALREADY_EXISTS) + + # check template + if vm_info.template: + template_name = vm_info.template + try: + host_template = self.app.domains[template_name] + except KeyError: + host_template = None + if not host_template or not host_template.is_template(): + # Maybe the (custom) template is in the backup? + if not (template_name in restore_info.keys() and + restore_info[template_name].good_to_go and + restore_info[template_name].vm.is_template()): + if self.options.use_default_template and \ + self.app.default_template: + if vm_info.orig_template is None: + vm_info.orig_template = template_name + vm_info.template = self.app.default_template.name + else: + vm_info.problems.add( + self.VMToRestore.MISSING_TEMPLATE) + + # check netvm + if not vm_info.vm.property_is_default('netvm') and vm_info.netvm: + netvm_name = vm_info.netvm + + try: + netvm_on_host = self.app.domains[netvm_name] + except KeyError: + netvm_on_host = None + # No netvm on the host? + if not ((netvm_on_host is not None) + and netvm_on_host.provides_network): + + # Maybe the (custom) netvm is in the backup? + if not (netvm_name in restore_info.keys() and + restore_info[netvm_name].good_to_go and + restore_info[netvm_name].vm.provides_network): + if self.options.use_default_netvm: + vm_info.vm.netvm = qubes.property.DEFAULT + elif self.options.use_none_netvm: + vm_info.netvm = None + else: + vm_info.problems.add(self.VMToRestore.MISSING_NETVM) + + return restore_info + + def _is_vm_included_in_backup_v1(self, check_vm): + if check_vm.qid == 0: + return os.path.exists( + os.path.join(self.backup_location, 'dom0-home')) + + # DisposableVM + if check_vm.dir_path is None: + return False + + backup_vm_dir_path = check_vm.dir_path.replace( + qubes.config.system_path["qubes_base_dir"], self.backup_location) + + if os.path.exists(backup_vm_dir_path): + return True + else: + return False + + @staticmethod + def _is_vm_included_in_backup_v2(check_vm): + if 'backup-content' in check_vm.features: + return check_vm.features['backup-content'] + else: + return False + + def _find_template_name(self, template): + if template in self.options.replace_template: + return self.options.replace_template[template] + return template + + def _is_vm_included_in_backup(self, vm): + if self.header_data.version == 1: + return self._is_vm_included_in_backup_v1(vm) + elif self.header_data.version in [2, 3, 4]: + return self._is_vm_included_in_backup_v2(vm) + else: + raise qubes.exc.QubesException( + "Unknown backup format version: {}".format( + self.header_data.version)) + + def get_restore_info(self): + # Format versions: + # 1 - Qubes R1, Qubes R2 beta1, beta2 + # 2 - Qubes R2 beta3+ + + vms_to_restore = {} + + for vm in self.backup_app.domains: + if vm.qid == 0: + # Handle dom0 as special case later + continue + if self._is_vm_included_in_backup(vm): + self.log.debug("{} is included in backup".format(vm.name)) + + vms_to_restore[vm.name] = self.VMToRestore(vm) + + if hasattr(vm, 'template'): + templatevm_name = self._find_template_name( + vm.template.name) + vms_to_restore[vm.name].template = templatevm_name + + # Set to None to not confuse QubesVm object from backup + # collection with host collection (further in clone_attrs). + vm.netvm = None + + vms_to_restore = self.restore_info_verify(vms_to_restore) + + # ...and dom0 home + if self.options.dom0_home and \ + self._is_vm_included_in_backup(self.backup_app.domains[0]): + vm = self.backup_app.domains[0] + if self.header_data.version == 1: + subdir = os.listdir(os.path.join(self.backup_location, + 'dom0-home'))[0] + else: + subdir = None + vms_to_restore['dom0'] = self.Dom0ToRestore(vm, subdir) + local_user = grp.getgrnam('qubes').gr_mem[0] + + if vms_to_restore['dom0'].username != local_user: + if not self.options.ignore_username_mismatch: + vms_to_restore['dom0'].problems.add( + self.Dom0ToRestore.USERNAME_MISMATCH) + + return vms_to_restore + + @staticmethod + def get_restore_summary(restore_info): + fields = { + "qid": {"func": "vm.qid"}, + + "name": {"func": "('[' if vm.is_template() else '')\ + + ('{' if vm.is_netvm() else '')\ + + vm.name \ + + (']' if vm.is_template() else '')\ + + ('}' if vm.is_netvm() else '')"}, + + "type": {"func": "'Tpl' if vm.is_template() else \ + 'App' if isinstance(vm, qubes.vm.appvm.AppVM) else \ + vm.__class__.__name__.replace('VM','')"}, + + "updbl": {"func": "'Yes' if vm.updateable else ''"}, + + "template": {"func": "'n/a' if not hasattr(vm, 'template') is None " + "else vm_info.template"}, + + "netvm": {"func": "'n/a' if vm.is_netvm() and not vm.is_proxyvm() else\ + ('*' if vm.property_is_default('netvm') else '') +\ + vm_info.netvm if vm_info.netvm is not None " + "else '-'"}, + + "label": {"func": "vm.label.name"}, + } + + fields_to_display = ["name", "type", "template", "updbl", + "netvm", "label"] + + # First calculate the maximum width of each field we want to display + total_width = 0 + for f in fields_to_display: + fields[f]["max_width"] = len(f) + for vm_info in restore_info.values(): + if vm_info.vm: + # noinspection PyUnusedLocal + vm = vm_info.vm + l = len(unicode(eval(fields[f]["func"]))) + if l > fields[f]["max_width"]: + fields[f]["max_width"] = l + total_width += fields[f]["max_width"] + + summary = "" + summary += "The following VMs are included in the backup:\n" + summary += "\n" + + # Display the header + for f in fields_to_display: + # noinspection PyTypeChecker + fmt = "{{0:-^{0}}}-+".format(fields[f]["max_width"] + 1) + summary += fmt.format('-') + summary += "\n" + for f in fields_to_display: + # noinspection PyTypeChecker + fmt = "{{0:>{0}}} |".format(fields[f]["max_width"] + 1) + summary += fmt.format(f) + summary += "\n" + for f in fields_to_display: + # noinspection PyTypeChecker + fmt = "{{0:-^{0}}}-+".format(fields[f]["max_width"] + 1) + summary += fmt.format('-') + summary += "\n" + + for vm_info in restore_info.values(): + assert isinstance(vm_info, BackupRestore.VMToRestore) + # Skip non-VM here + if not vm_info.vm: + continue + # noinspection PyUnusedLocal + vm = vm_info.vm + s = "" + for f in fields_to_display: + # noinspection PyTypeChecker + fmt = "{{0:>{0}}} |".format(fields[f]["max_width"] + 1) + s += fmt.format(eval(fields[f]["func"])) + + if BackupRestore.VMToRestore.EXCLUDED in vm_info.problems: + s += " <-- Excluded from restore" + elif BackupRestore.VMToRestore.ALREADY_EXISTS in vm_info.problems: + s += " <-- A VM with the same name already exists on the host!" + elif BackupRestore.VMToRestore.MISSING_TEMPLATE in \ + vm_info.problems: + s += " <-- No matching template on the host " \ + "or in the backup found!" + elif BackupRestore.VMToRestore.MISSING_NETVM in \ + vm_info.problems: + s += " <-- No matching netvm on the host " \ + "or in the backup found!" + else: + if vm_info.orig_template: + s += " <-- Original template was '{}'".format( + vm_info.orig_template) + if vm_info.name != vm_info.vm.name: + s += " <-- Will be renamed to '{}'".format( + vm_info.name) + + summary += s + "\n" + + if 'dom0' in restore_info.keys(): + s = "" + for f in fields_to_display: + # noinspection PyTypeChecker + fmt = "{{0:>{0}}} |".format(fields[f]["max_width"] + 1) + if f == "name": + s += fmt.format("Dom0") + elif f == "type": + s += fmt.format("Home") + else: + s += fmt.format("") + if BackupRestore.Dom0ToRestore.USERNAME_MISMATCH in \ + restore_info['dom0'].problems: + s += " <-- username in backup and dom0 mismatch" + + summary += s + "\n" + + return summary + + def _restore_vm_dir_v1(self, src_dir, dst_dir): + + backup_src_dir = src_dir.replace( + qubes.config.system_path["qubes_base_dir"], self.backup_location) + + # We prefer to use Linux's cp, because it nicely handles sparse files + cp_retcode = subprocess.call( + ["cp", "-rp", "--reflink=auto", backup_src_dir, dst_dir]) + if cp_retcode != 0: + raise qubes.exc.QubesException( + "*** Error while copying file {0} to {1}".format(backup_src_dir, + dst_dir)) + + def restore_do(self, restore_info): + # FIXME handle locking + + # Perform VM restoration in backup order + vms_dirs = [] + vms_size = 0 + vms = {} + for vm_info in restore_info.values(): + assert isinstance(vm_info, self.VMToRestore) + if not vm_info.vm: + continue + if not vm_info.good_to_go: + continue + vm = vm_info.vm + if self.header_data.version >= 2: + if vm.features['backup-size']: + vms_size += int(vm.features['backup-size']) + vms_dirs.append(vm.features['backup-path']) + vms[vm.name] = vm + + if self.header_data.version >= 2: + if 'dom0' in restore_info.keys() and \ + restore_info['dom0'].good_to_go: + vms_dirs.append(os.path.dirname(restore_info['dom0'].subdir)) + vms_size += restore_info['dom0'].size + + try: + self._restore_vm_dirs(vms_dirs=vms_dirs, vms_size=vms_size) + except qubes.exc.QubesException: + if self.options.verify_only: + raise + else: + self.log.warning( + "Some errors occurred during data extraction, " + "continuing anyway to restore at least some " + "VMs") + else: + if self.options.verify_only: + self.log.warning( + "Backup verification not supported for this backup format.") + + if self.options.verify_only: + shutil.rmtree(self.tmpdir) + return + + # First load templates, then other VMs + for vm in sorted(vms.values(), key=lambda x: x.is_template(), + reverse=True): + if self.canceled: + # only break the loop to save qubes.xml + # with already restored VMs + break + self.log.info("-> Restoring {0}...".format(vm.name)) + retcode = subprocess.call( + ["mkdir", "-p", os.path.dirname(vm.dir_path)]) + if retcode != 0: + self.log.error("*** Cannot create directory: {0}?!".format( + vm.dir_path)) + self.log.warning("Skipping VM {}...".format(vm.name)) + continue + + kwargs = {} + if hasattr(vm, 'template'): + template = restore_info[vm.name].template + # handle potentially renamed template + if template in restore_info \ + and restore_info[template].good_to_go: + template = restore_info[template].name + kwargs['template'] = template + + new_vm = None + vm_name = restore_info[vm.name].name + + try: + # first only minimal set, later clone_properties + # will be called + new_vm = self.app.add_new_vm( + vm.__class__, + name=vm_name, + label=vm.label, + installed_by_rpm=False, + **kwargs) + if os.path.exists(new_vm.dir_path): + move_to_path = tempfile.mkdtemp('', os.path.basename( + new_vm.dir_path), os.path.dirname(new_vm.dir_path)) + try: + os.rename(new_vm.dir_path, move_to_path) + self.log.warning( + "*** Directory {} already exists! It has " + "been moved to {}".format(new_vm.dir_path, + move_to_path)) + except OSError: + self.log.error( + "*** Directory {} already exists and " + "cannot be moved!".format(new_vm.dir_path)) + self.log.warning("Skipping VM {}...".format( + vm.name)) + continue + + if self.header_data.version == 1: + self._restore_vm_dir_v1(vm.dir_path, + os.path.dirname(new_vm.dir_path)) + else: + shutil.move(os.path.join(self.tmpdir, + vm.features['backup-path']), + new_vm.dir_path) + + new_vm.verify_files() + except Exception as err: + self.log.error("ERROR: {0}".format(err)) + self.log.warning("*** Skipping VM: {0}".format(vm.name)) + if new_vm: + del self.app.domains[new_vm.qid] + continue + + if hasattr(vm, 'kernel'): + # TODO: add a setting for this? + if not vm.property_is_default('kernel') and vm.kernel and \ + vm.kernel not in \ + os.listdir(os.path.join(qubes.config.qubes_base_dir, + qubes.config.system_path[ + 'qubes_kernels_base_dir'])): + self.log.warning("Kernel %s not installed, " + "using default one" % vm.kernel) + vm.kernel = qubes.property.DEFAULT + # remove no longer needed backup metadata + if 'backup-content' in vm.features: + del vm.features['backup-content'] + del vm.features['backup-size'] + del vm.features['backup-path'] + try: + # exclude VM references - handled manually according to + # restore options + proplist = [prop for prop in new_vm.property_list() + if prop.clone and prop.__name__ not in + ['template', 'netvm', 'dispvm_netvm']] + new_vm.clone_properties(vm, proplist=proplist) + except Exception as err: + self.log.error("ERROR: {0}".format(err)) + self.log.warning("*** Some VM property will not be " + "restored") + + try: + new_vm.fire_event('domain-restore') + except Exception as err: + self.log.error("ERROR during appmenu restore: " + "{0}".format(err)) + self.log.warning( + "*** VM '{0}' will not have appmenus".format(vm.name)) + + # Set network dependencies - only non-default netvm setting + for vm in vms.values(): + vm_info = restore_info[vm.name] + vm_name = vm_info.name + try: + host_vm = self.app.domains[vm_name] + except KeyError: + # Failed/skipped VM + continue + + if not vm.property_is_default('netvm'): + if vm_info.netvm in restore_info: + host_vm.netvm = restore_info[vm_info.netvm].name + else: + host_vm.netvm = vm_info.netvm + + self.app.save() + + if self.canceled: + if self.header_data.version >= 2: + raise BackupCanceledError("Restore canceled", + tmpdir=self.tmpdir) + else: + raise BackupCanceledError("Restore canceled") + + # ... and dom0 home as last step + if 'dom0' in restore_info.keys() and restore_info['dom0'].good_to_go: + backup_path = restore_info['dom0'].subdir + local_user = grp.getgrnam('qubes').gr_mem[0] + home_dir = pwd.getpwnam(local_user).pw_dir + if self.header_data.version == 1: + backup_dom0_home_dir = os.path.join(self.backup_location, + backup_path) + else: + backup_dom0_home_dir = os.path.join(self.tmpdir, backup_path) + restore_home_backupdir = "home-pre-restore-{0}".format( + time.strftime("%Y-%m-%d-%H%M%S")) + + self.log.info( + "Restoring home of user '{0}'...".format(local_user)) + self.log.info( + "Existing files/dirs backed up in '{0}' dir".format( + restore_home_backupdir)) + os.mkdir(home_dir + '/' + restore_home_backupdir) + for f in os.listdir(backup_dom0_home_dir): + home_file = home_dir + '/' + f + if os.path.exists(home_file): + os.rename(home_file, + home_dir + '/' + restore_home_backupdir + '/' + f) + if self.header_data.version == 1: + subprocess.call( + ["cp", "-nrp", "--reflink=auto", + backup_dom0_home_dir + '/' + f, home_file]) + elif self.header_data.version >= 2: + shutil.move(backup_dom0_home_dir + '/' + f, home_file) + retcode = subprocess.call(['sudo', 'chown', '-R', + local_user, home_dir]) + if retcode != 0: + self.log.error("*** Error while setting home directory owner") + + shutil.rmtree(self.tmpdir) + +# vim:sw=4:et: diff --git a/qubes/core2migration.py b/qubes/core2migration.py new file mode 100644 index 00000000..f19fb3c0 --- /dev/null +++ b/qubes/core2migration.py @@ -0,0 +1,223 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# The Qubes OS Project, http://www.qubes-os.org +# +# Copyright (C) 2016 Marek Marczykowski-Górecki +# +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +# +# +import os +import sys +import qubes +import qubes.vm.appvm +import qubes.vm.standalonevm +import qubes.vm.templatevm +import qubes.vm.adminvm +import qubes.ext.r3compatibility +import lxml.etree +import xml.parsers.expat + + +class AppVM(qubes.vm.appvm.AppVM): + """core2 compatibility AppVM class, with variable dir_path""" + dir_path = qubes.property('dir_path', + default=(lambda self: self.storage.vmdir), + saver=qubes.property.dontsave, + doc="VM storage directory", + ) + + def is_running(self): + return False + +class StandaloneVM(qubes.vm.standalonevm.StandaloneVM): + """core2 compatibility StandaloneVM class, with variable dir_path""" + dir_path = qubes.property('dir_path', + default=(lambda self: self.storage.vmdir), + saver=qubes.property.dontsave, + doc="VM storage directory") + + def is_running(self): + return False + + +class Core2Qubes(qubes.Qubes): + + def __init__(self, store=None, load=True, **kwargs): + if store is None: + raise ValueError("store path required") + super(Core2Qubes, self).__init__(store, load, **kwargs) + + def load_globals(self, element): + default_template = element.get("default_template") + self.default_template = int(default_template) \ + if default_template.lower() != "none" else None + + default_netvm = element.get("default_netvm") + if default_netvm is not None: + self.default_netvm = int(default_netvm) \ + if default_netvm != "None" else None + + default_fw_netvm = element.get("default_fw_netvm") + if default_fw_netvm is not None: + self.default_fw_netvm = int(default_fw_netvm) \ + if default_fw_netvm != "None" else None + + updatevm = element.get("updatevm") + if updatevm is not None: + self.updatevm = int(updatevm) \ + if updatevm != "None" else None + + clockvm = element.get("clockvm") + if clockvm is not None: + self.clockvm = int(clockvm) \ + if clockvm != "None" else None + + self.default_kernel = element.get("default_kernel") + + def set_netvm_dependency(self, element): + kwargs = {} + attr_list = ("qid", "uses_default_netvm", "netvm_qid") + + for attribute in attr_list: + kwargs[attribute] = element.get(attribute) + + vm = self.domains[int(kwargs["qid"])] + + if element.get("uses_default_netvm") is None: + uses_default_netvm = True + else: + uses_default_netvm = ( + True if element.get("uses_default_netvm") == "True" else False) + if not uses_default_netvm: + netvm_qid = element.get("netvm_qid") + if netvm_qid is None or netvm_qid == "none": + vm.netvm = None + else: + vm.netvm = int(netvm_qid) + + # TODO: dispvm_netvm + + def load(self): + qubes_store_file = open(self._store, 'r') + + try: + qubes_store_file.seek(0) + tree = lxml.etree.parse(qubes_store_file) + except (EnvironmentError, + xml.parsers.expat.ExpatError) as err: + self.log.error(err) + return False + + self.labels = { + 1: qubes.Label(1, '0xcc0000', 'red'), + 2: qubes.Label(2, '0xf57900', 'orange'), + 3: qubes.Label(3, '0xedd400', 'yellow'), + 4: qubes.Label(4, '0x73d216', 'green'), + 5: qubes.Label(5, '0x555753', 'gray'), + 6: qubes.Label(6, '0x3465a4', 'blue'), + 7: qubes.Label(7, '0x75507b', 'purple'), + 8: qubes.Label(8, '0x000000', 'black'), + } + + self.domains.add(qubes.vm.adminvm.AdminVM( + self, None, qid=0, name='dom0')) + + vm_classes = ["TemplateVm", "TemplateHVm", + "AppVm", "HVm", "NetVm", "ProxyVm"] + for (vm_class_name) in vm_classes: + vms_of_class = tree.findall("Qubes" + vm_class_name) + # first non-template based, then template based + sorted_vms_of_class = sorted(vms_of_class, + key=lambda x: str(x.get('template_qid')).lower() != "none") + for element in sorted_vms_of_class: + try: + kwargs = {} + if vm_class_name in ["TemplateVm", "TemplateHVm"]: + vm_class = qubes.vm.templatevm.TemplateVM + elif element.get('template_qid').lower() == "none": + kwargs['dir_path'] = element.get('dir_path') + vm_class = StandaloneVM + else: + kwargs['dir_path'] = element.get('dir_path') + kwargs['template'] = int(element.get('template_qid')) + vm_class = AppVM + # simple attributes + for attr in ['installed_by_rpm', 'include_in_backups', + 'qrexec_timeout', 'internal', 'label', 'name', + 'vcpus', 'memory', 'maxmem', 'default_user', + 'debug', 'pci_strictreset', 'mac', 'autostart']: + value = element.get(attr) + if value: + kwargs[attr] = value + # attributes with default value + for attr in ["kernel", "kernelopts"]: + value = element.get(attr) + if value and value.lower() == "none": + value = None + value_is_default = element.get( + "uses_default_{}".format(attr)) + if value_is_default and value_is_default.lower() != \ + "true": + kwargs[attr] = value + kwargs['hvm'] = "HVm" in vm_class_name + vm = self.add_new_vm(vm_class, + qid=int(element.get('qid')), **kwargs) + services = element.get('services') + if services: + services = eval(services) + else: + services = {} + for service, value in services.iteritems(): + feature = service + for repl_feature, repl_service in \ + qubes.ext.r3compatibility.\ + R3Compatibility.features_to_services.\ + iteritems(): + if repl_service == service: + feature = repl_feature + vm.features[feature] = value + for attr in ['backup_content', 'backup_path', + 'backup_size']: + value = element.get(attr) + vm.features[attr.replace('_', '-')] = value + pcidevs = element.get('pcidevs') + if pcidevs: + pcidevs = eval(pcidevs) + for pcidev in pcidevs: + try: + vm.devices["pci"].attach(pcidev) + except qubes.exc.QubesException as e: + self.log.error("VM {}: {}".format(vm.name, str(e))) + except (ValueError, LookupError) as err: + self.log.error("import error ({1}): {2}".format( + vm_class_name, err)) + if 'vm' in locals(): + del self.domains[vm] + + # After importing all VMs, set netvm references, in the same order + for vm_class_name in vm_classes: + for element in tree.findall("Qubes" + vm_class_name): + try: + self.set_netvm_dependency(element) + except (ValueError, LookupError) as err: + self.log.error("VM {}: failed to set netvm dependency: {}". + format(element.get('name'), err)) + + self.load_globals(tree.getroot()) + + def save(self): + raise NotImplementedError("Saving old qubes.xml not supported") diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index 60d54247..e95ae3d4 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -40,6 +40,9 @@ import time import qubes.config import qubes.events +import qubes.backup +import qubes.exc +import qubes.vm.standalonevm XMLPATH = '/var/lib/qubes/qubes-test.xml' CLASS_XMLPATH = '/var/lib/qubes/qubes-class-test.xml' @@ -482,7 +485,7 @@ class SystemTestsMixin(object): except (AttributeError, libvirt.libvirtError): pass - del app.domains[vm] + del app.domains[vm.qid] del vm app.save() @@ -628,6 +631,14 @@ class SystemTestsMixin(object): # noinspection PyAttributeOutsideInit class BackupTestsMixin(SystemTestsMixin): + class BackupErrorHandler(logging.Handler): + def __init__(self, errors_queue, level=logging.NOTSET): + super(BackupTestsMixin.BackupErrorHandler, self).__init__(level) + self.errors_queue = errors_queue + + def emit(self, record): + self.errors_queue.put(record.getMessage()) + def setUp(self): super(BackupTestsMixin, self).setUp() self.init_default_template() @@ -642,22 +653,17 @@ class BackupTestsMixin(SystemTestsMixin): shutil.rmtree(self.backupdir) os.mkdir(self.backupdir) + self.error_handler = self.BackupErrorHandler(self.error_detected, + level=logging.WARNING) + backup_log = logging.getLogger('qubes.backup') + backup_log.addHandler(self.error_handler) + def tearDown(self): super(BackupTestsMixin, self).tearDown() shutil.rmtree(self.backupdir) - def print_progress(self, progress): - if self.verbose: - print >> sys.stderr, "\r-> Backing up files: {0}%...".format(progress) - - def error_callback(self, message): - self.error_detected.put(message) - if self.verbose: - print >> sys.stderr, "ERROR: {0}".format(message) - - def print_callback(self, msg): - if self.verbose: - print msg + backup_log = logging.getLogger('qubes.backup') + backup_log.removeHandler(self.error_handler) def fill_image(self, path, size=None, sparse=False): block_size = 4096 @@ -686,8 +692,8 @@ class BackupTestsMixin(SystemTestsMixin): if self.verbose: print >>sys.stderr, "-> Creating %s" % vmname testnet = self.app.add_new_vm(qubes.vm.appvm.AppVM, - name=vmname, template=template, provides_network=True) - testnet.create_on_disk(verbose=self.verbose) + name=vmname, template=template, provides_network=True, label='red') + testnet.create_on_disk() vms.append(testnet) self.fill_image(testnet.private_img, 20*1024*1024) @@ -695,55 +701,68 @@ class BackupTestsMixin(SystemTestsMixin): if self.verbose: print >>sys.stderr, "-> Creating %s" % vmname testvm1 = self.app.add_new_vm(qubes.vm.appvm.AppVM, - name=vmname, template=template) + name=vmname, template=template, label='red') testvm1.uses_default_netvm = False testvm1.netvm = testnet - testvm1.create_on_disk(verbose=self.verbose) + testvm1.create_on_disk() vms.append(testvm1) self.fill_image(testvm1.private_img, 100*1024*1024) vmname = self.make_vm_name('testhvm1') if self.verbose: print >>sys.stderr, "-> Creating %s" % vmname - testvm2 = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=vmname, - hvm=True) - testvm2.create_on_disk(verbose=self.verbose) + testvm2 = self.app.add_new_vm(qubes.vm.standalonevm.StandaloneVM, + name=vmname, + hvm=True, label='red') + testvm2.create_on_disk() self.fill_image(testvm2.root_img, 1024*1024*1024, True) vms.append(testvm2) + vmname = self.make_vm_name('template') + if self.verbose: + print >>sys.stderr, "-> Creating %s" % vmname + testvm3 = self.app.add_new_vm(qubes.vm.templatevm.TemplateVM, + name=vmname, label='red') + testvm3.create_on_disk() + self.fill_image(testvm3.root_img, 100*1024*1024, True) + vms.append(testvm3) + + vmname = self.make_vm_name('custom') + if self.verbose: + print >>sys.stderr, "-> Creating %s" % vmname + testvm4 = self.app.add_new_vm(qubes.vm.appvm.AppVM, + name=vmname, template=testvm3, label='red') + testvm4.create_on_disk() + vms.append(testvm4) + self.app.save() return vms - def make_backup(self, vms, prepare_kwargs=dict(), do_kwargs=dict(), - target=None, expect_failure=False): - # XXX: bakup_prepare and backup_do don't support host_collection - # self.qc.unlock_db() + def make_backup(self, vms, target=None, expect_failure=False, **kwargs): if target is None: target = self.backupdir try: - files_to_backup = \ - qubes.backup.backup_prepare(vms, - print_callback=self.print_callback, - **prepare_kwargs) - except qubes.qubes.QubesException as e: + backup = qubes.backup.Backup(self.app, vms, **kwargs) + except qubes.exc.QubesException as e: if not expect_failure: self.fail("QubesException during backup_prepare: %s" % str(e)) else: raise + backup.passphrase = 'qubes' + backup.target_dir = target + try: - qubes.backup.backup_do(target, files_to_backup, "qubes", - progress_callback=self.print_progress, - **do_kwargs) - except qubes.qubes.QubesException as e: + backup.backup_do() + except qubes.exc.QubesException as e: if not expect_failure: self.fail("QubesException during backup_do: %s" % str(e)) else: raise # FIXME why? - self.reload_db() + #self.reload_db() def restore_backup(self, source=None, appvm=None, options=None, expect_errors=None): @@ -753,23 +772,18 @@ class BackupTestsMixin(SystemTestsMixin): else: backupfile = source - with self.assertNotRaises(qubes.qubes.QubesException): - backup_info = qubes.backup.backup_restore_prepare( - backupfile, "qubes", - host_collection=self.app, - print_callback=self.print_callback, - appvm=appvm, - options=options or {}) - + with self.assertNotRaises(qubes.exc.QubesException): + restore_op = qubes.backup.BackupRestore( + self.app, backupfile, appvm, "qubes") + if options: + for key, value in options.iteritems(): + setattr(restore_op.options, key, value) + restore_info = restore_op.get_restore_info() if self.verbose: - qubes.backup.backup_restore_print_summary(backup_info) + print restore_op.get_restore_summary(restore_info) - with self.assertNotRaises(qubes.qubes.QubesException): - qubes.backup.backup_restore_do( - backup_info, - host_collection=self.app, - print_callback=self.print_callback if self.verbose else None, - error_callback=self.error_callback) + with self.assertNotRaises(qubes.exc.QubesException): + restore_op.restore_do(restore_info) # maybe someone forgot to call .save() self.reload_db() @@ -777,6 +791,9 @@ class BackupTestsMixin(SystemTestsMixin): errors = [] if expect_errors is None: expect_errors = [] + else: + self.assertFalse(self.error_detected.empty(), + "Restore errors expected, but none detected") while not self.error_detected.empty(): current_error = self.error_detected.get() if any(map(current_error.startswith, expect_errors)): @@ -821,8 +838,8 @@ def load_tests(loader, tests, pattern): # pylint: disable=unused-argument 'qubes.tests.int.dom0_update', 'qubes.tests.int.network', # 'qubes.tests.vm_qrexec_gui', -# 'qubes.tests.backup', -# 'qubes.tests.backupcompatibility', + 'qubes.tests.int.backup', + 'qubes.tests.int.backupcompatibility', # 'qubes.tests.regressions', # tool tests diff --git a/tests/backup.py b/qubes/tests/int/backup.py similarity index 64% rename from tests/backup.py rename to qubes/tests/int/backup.py index 72463c91..c11b0948 100644 --- a/tests/backup.py +++ b/qubes/tests/int/backup.py @@ -28,40 +28,44 @@ import os import unittest import sys -from qubes.qubes import QubesException, QubesTemplateVm +import qubes +import qubes.exc import qubes.tests +import qubes.vm.appvm +import qubes.vm.templatevm class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase): def test_000_basic_backup(self): vms = self.create_backup_vms() self.make_backup(vms) - self.remove_vms(vms) + self.remove_vms(reversed(vms)) self.restore_backup() - self.remove_vms(vms) + for vm in vms: + self.assertIn(vm.name, self.app.domains) def test_001_compressed_backup(self): vms = self.create_backup_vms() - self.make_backup(vms, do_kwargs={'compressed': True}) - self.remove_vms(vms) + self.make_backup(vms, compressed=True) + self.remove_vms(reversed(vms)) self.restore_backup() - self.remove_vms(vms) + for vm in vms: + self.assertIn(vm.name, self.app.domains) def test_002_encrypted_backup(self): vms = self.create_backup_vms() - self.make_backup(vms, do_kwargs={'encrypted': True}) - self.remove_vms(vms) + self.make_backup(vms, encrypted=True) + self.remove_vms(reversed(vms)) self.restore_backup() - self.remove_vms(vms) + for vm in vms: + self.assertIn(vm.name, self.app.domains) def test_003_compressed_encrypted_backup(self): vms = self.create_backup_vms() - self.make_backup(vms, - do_kwargs={ - 'compressed': True, - 'encrypted': True}) - self.remove_vms(vms) + self.make_backup(vms, compressed=True, encrypted=True) + self.remove_vms(reversed(vms)) self.restore_backup() - self.remove_vms(vms) + for vm in vms: + self.assertIn(vm.name, self.app.domains) def test_004_sparse_multipart(self): vms = [] @@ -70,29 +74,36 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase): if self.verbose: print >>sys.stderr, "-> Creating %s" % vmname - hvmtemplate = self.qc.add_new_vm("QubesTemplateHVm", name=vmname) - hvmtemplate.create_on_disk(verbose=self.verbose) + hvmtemplate = self.app.add_new_vm( + qubes.vm.templatevm.TemplateVM, name=vmname, hvm=True, label='red') + hvmtemplate.create_on_disk() self.fill_image(os.path.join(hvmtemplate.dir_path, '00file'), 195*1024*1024-4096*3) self.fill_image(hvmtemplate.private_img, 195*1024*1024-4096*3) self.fill_image(hvmtemplate.root_img, 1024*1024*1024, sparse=True) vms.append(hvmtemplate) - self.qc.save() + self.app.save() self.make_backup(vms) - self.remove_vms(vms) + self.remove_vms(reversed(vms)) self.restore_backup() - self.remove_vms(vms) + for vm in vms: + self.assertIn(vm.name, self.app.domains) + # TODO check vm.backup_timestamp def test_005_compressed_custom(self): vms = self.create_backup_vms() - self.make_backup(vms, do_kwargs={'compressed': "bzip2"}) - self.remove_vms(vms) + self.make_backup(vms, compressed="bzip2") + self.remove_vms(reversed(vms)) self.restore_backup() - self.remove_vms(vms) + for vm in vms: + self.assertIn(vm.name, self.app.domains) def test_100_backup_dom0_no_restore(self): - self.make_backup([self.qc[0]]) + # do not write it into dom0 home itself... + os.mkdir('/var/tmp/test-backup') + self.backupdir = '/var/tmp/test-backup' + self.make_backup([self.app.domains[0]]) # TODO: think of some safe way to test restore... def test_200_restore_over_existing_directory(self): @@ -102,7 +113,7 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase): """ vms = self.create_backup_vms() self.make_backup(vms) - self.remove_vms(vms) + self.remove_vms(reversed(vms)) test_dir = vms[0].dir_path os.mkdir(test_dir) with open(os.path.join(test_dir, 'some-file.txt'), 'w') as f: @@ -112,7 +123,6 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase): '*** Directory {} already exists! It has been moved'.format( test_dir) ]) - self.remove_vms(vms) def test_210_auto_rename(self): """ @@ -122,58 +132,51 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase): vms = self.create_backup_vms() self.make_backup(vms) self.restore_backup(options={ - 'rename-conflicting': True + 'rename_conflicting': True }) for vm in vms: - self.assertIsNotNone(self.qc.get_vm_by_name(vm.name+'1')) - restored_vm = self.qc.get_vm_by_name(vm.name+'1') - if vm.netvm and not vm.uses_default_netvm: - self.assertEqual(restored_vm.netvm.name, vm.netvm.name+'1') + with self.assertNotRaises( + (qubes.exc.QubesVMNotFoundError, KeyError)): + restored_vm = self.app.domains[vm.name + '1'] + if vm.netvm and not vm.property_is_default('netvm'): + self.assertEqual(restored_vm.netvm.name, vm.netvm.name + '1') - self.remove_vms(vms) class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin): def setUp(self): super(TC_10_BackupVMMixin, self).setUp() - self.backupvm = self.qc.add_new_vm( - "QubesAppVm", + self.backupvm = self.app.add_new_vm( + qubes.vm.appvm.AppVM, + label='red', name=self.make_vm_name('backupvm'), - template=self.qc.get_vm_by_name(self.template) + template=self.template ) - self.backupvm.create_on_disk(verbose=self.verbose) + self.backupvm.create_on_disk() def test_100_send_to_vm_file_with_spaces(self): vms = self.create_backup_vms() self.backupvm.start() self.backupvm.run("mkdir '/var/tmp/backup directory'", wait=True) - self.make_backup(vms, - do_kwargs={ - 'appvm': self.backupvm, - 'compressed': True, - 'encrypted': True}, - target='/var/tmp/backup directory') - self.remove_vms(vms) + self.make_backup(vms, target_vm=self.backupvm, + compressed=True, encrypted=True, + target='/var/tmp/backup directory') + self.remove_vms(reversed(vms)) p = self.backupvm.run("ls /var/tmp/backup*/qubes-backup*", passio_popen=True) (backup_path, _) = p.communicate() backup_path = backup_path.strip() self.restore_backup(source=backup_path, appvm=self.backupvm) - self.remove_vms(vms) def test_110_send_to_vm_command(self): vms = self.create_backup_vms() self.backupvm.start() - self.make_backup(vms, - do_kwargs={ - 'appvm': self.backupvm, - 'compressed': True, - 'encrypted': True}, - target='dd of=/var/tmp/backup-test') - self.remove_vms(vms) + self.make_backup(vms, target_vm=self.backupvm, + compressed=True, encrypted=True, + target='dd of=/var/tmp/backup-test') + self.remove_vms(reversed(vms)) self.restore_backup(source='dd if=/var/tmp/backup-test', appvm=self.backupvm) - self.remove_vms(vms) def test_110_send_to_vm_no_space(self): """ @@ -192,27 +195,18 @@ class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin): user="root", wait=True) if retcode != 0: raise RuntimeError("Failed to prepare backup directory") - with self.assertRaises(QubesException): - self.make_backup(vms, - do_kwargs={ - 'appvm': self.backupvm, - 'compressed': False, - 'encrypted': True}, - target='/home/user/backup', - expect_failure=True) - self.qc.lock_db_for_writing() - self.qc.load() - self.remove_vms(vms) + with self.assertRaises(qubes.exc.QubesException): + self.make_backup(vms, target_vm=self.backupvm, + compressed=False, encrypted=True, + target='/home/user/backup', + expect_failure=True) def load_tests(loader, tests, pattern): try: - qc = qubes.qubes.QubesVmCollection() - qc.lock_db_for_reading() - qc.load() - qc.unlock_db() - templates = [vm.name for vm in qc.values() if - isinstance(vm, QubesTemplateVm)] + app = qubes.Qubes() + templates = [vm.name for vm in app.domains if + isinstance(vm, qubes.vm.templatevm.TemplateVM)] except OSError: templates = [] for template in templates: diff --git a/tests/backupcompatibility.py b/qubes/tests/int/backupcompatibility.py similarity index 93% rename from tests/backupcompatibility.py rename to qubes/tests/int/backupcompatibility.py index 775db92f..939e6eaf 100644 --- a/tests/backupcompatibility.py +++ b/qubes/tests/int/backupcompatibility.py @@ -29,8 +29,6 @@ import subprocess import unittest import sys import re -from qubes.qubes import QubesVmCollection, QubesException -from qubes import backup import qubes.tests @@ -146,6 +144,11 @@ compression-filter=gzip ''' class TC_00_BackupCompatibility(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase): + + def tearDown(self): + self.remove_test_vms(prefix="test-") + super(TC_00_BackupCompatibility, self).tearDown() + def create_whitelisted_appmenus(self, filename): f = open(filename, "w") f.write("gnome-terminal.desktop\n") @@ -401,19 +404,22 @@ class TC_00_BackupCompatibility(qubes.tests.BackupTestsMixin, qubes.tests.QubesT f.write(QUBESXML_R1) f.close() - self.restore_backup(self.backupdir, options={ - 'use-default-template': True, - 'use-default-netvm': True, - }) - self.assertIsNotNone(self.qc.get_vm_by_name("test-template-clone")) - self.assertIsNotNone(self.qc.get_vm_by_name("test-testproxy")) - self.assertIsNotNone(self.qc.get_vm_by_name("test-work")) - self.assertIsNotNone(self.qc.get_vm_by_name("test-standalonevm")) - self.assertIsNotNone(self.qc.get_vm_by_name( - "test-custom-template-appvm")) - self.assertEqual(self.qc.get_vm_by_name("test-custom-template-appvm") + self.restore_backup(self.backupdir, + options={ + 'use-default-template': True, + 'use-default-netvm': True, + }, + expect_errors=['Kernel None not installed, using default one'] + ) + with self.assertNotRaises(KeyError): + vm = self.app.domains["test-template-clone"] + vm = self.app.domains["test-testproxy"] + vm = self.app.domains["test-work"] + vm = self.app.domains["test-standalonevm"] + vm = self.app.domains["test-custom-template-appvm"] + self.assertEqual(self.app.domains["test-custom-template-appvm"] .template, - self.qc.get_vm_by_name("test-template-clone")) + self.app.domains["test-template-clone"]) def test_200_r2b2(self): self.create_v1_files(r2b2=True) @@ -425,16 +431,16 @@ class TC_00_BackupCompatibility(qubes.tests.BackupTestsMixin, qubes.tests.QubesT self.restore_backup(self.backupdir, options={ 'use-default-template': True, }) - self.assertIsNotNone(self.qc.get_vm_by_name("test-template-clone")) - self.assertIsNotNone(self.qc.get_vm_by_name("test-testproxy")) - self.assertIsNotNone(self.qc.get_vm_by_name("test-work")) - self.assertIsNotNone(self.qc.get_vm_by_name("test-testhvm")) - self.assertIsNotNone(self.qc.get_vm_by_name("test-standalonevm")) - self.assertIsNotNone(self.qc.get_vm_by_name( - "test-custom-template-appvm")) - self.assertEqual(self.qc.get_vm_by_name("test-custom-template-appvm") + with self.assertNotRaises(KeyError): + vm = self.app.domains["test-template-clone"] + vm = self.app.domains["test-testproxy"] + vm = self.app.domains["test-work"] + vm = self.app.domains["test-testhvm"] + vm = self.app.domains["test-standalonevm"] + vm = self.app.domains["test-custom-template-appvm"] + self.assertEqual(self.app.domains["test-custom-template-appvm"] .template, - self.qc.get_vm_by_name("test-template-clone")) + self.app.domains["test-template-clone"]) def test_210_r2(self): self.create_v3_backup(False) @@ -443,16 +449,16 @@ class TC_00_BackupCompatibility(qubes.tests.BackupTestsMixin, qubes.tests.QubesT 'use-default-template': True, 'use-default-netvm': True, }) - self.assertIsNotNone(self.qc.get_vm_by_name("test-template-clone")) - self.assertIsNotNone(self.qc.get_vm_by_name("test-testproxy")) - self.assertIsNotNone(self.qc.get_vm_by_name("test-work")) - self.assertIsNotNone(self.qc.get_vm_by_name("test-testhvm")) - self.assertIsNotNone(self.qc.get_vm_by_name("test-standalonevm")) - self.assertIsNotNone(self.qc.get_vm_by_name( - "test-custom-template-appvm")) - self.assertEqual(self.qc.get_vm_by_name("test-custom-template-appvm") + with self.assertNotRaises(KeyError): + vm = self.app.domains["test-template-clone"] + vm = self.app.domains["test-testproxy"] + vm = self.app.domains["test-work"] + vm = self.app.domains["test-testhvm"] + vm = self.app.domains["test-standalonevm"] + vm = self.app.domains["test-custom-template-appvm"] + self.assertEqual(self.app.domains["test-custom-template-appvm"] .template, - self.qc.get_vm_by_name("test-template-clone")) + self.app.domains["test-template-clone"]) def test_220_r2_encrypted(self): self.create_v3_backup(True) @@ -461,13 +467,13 @@ class TC_00_BackupCompatibility(qubes.tests.BackupTestsMixin, qubes.tests.QubesT 'use-default-template': True, 'use-default-netvm': True, }) - self.assertIsNotNone(self.qc.get_vm_by_name("test-template-clone")) - self.assertIsNotNone(self.qc.get_vm_by_name("test-testproxy")) - self.assertIsNotNone(self.qc.get_vm_by_name("test-work")) - self.assertIsNotNone(self.qc.get_vm_by_name("test-testhvm")) - self.assertIsNotNone(self.qc.get_vm_by_name("test-standalonevm")) - self.assertIsNotNone(self.qc.get_vm_by_name( - "test-custom-template-appvm")) - self.assertEqual(self.qc.get_vm_by_name("test-custom-template-appvm") + with self.assertNotRaises(KeyError): + vm = self.app.domains["test-template-clone"] + vm = self.app.domains["test-testproxy"] + vm = self.app.domains["test-work"] + vm = self.app.domains["test-testhvm"] + vm = self.app.domains["test-standalonevm"] + vm = self.app.domains["test-custom-template-appvm"] + self.assertEqual(self.app.domains["test-custom-template-appvm"] .template, - self.qc.get_vm_by_name("test-template-clone")) + self.app.domains["test-template-clone"]) diff --git a/qubes/tests/vm/qubesvm.py b/qubes/tests/vm/qubesvm.py index e9041aba..95c0a6f6 100644 --- a/qubes/tests/vm/qubesvm.py +++ b/qubes/tests/vm/qubesvm.py @@ -639,22 +639,6 @@ class TC_90_QubesVM(qubes.tests.QubesTestCase): vm = self.get_vm() self._test_generic_bool_property(vm, 'pci_strictreset') - def test_380_backup_size(self): - vm = self.get_vm() - self.assertPropertyDefaultValue(vm, 'backup_size', 0) - self.assertPropertyValue(vm, 'backup_size', 0, 0, '0') - del vm.backup_size - self.assertPropertyDefaultValue(vm, 'backup_size', 0) - self.assertPropertyValue(vm, 'backup_size', '0', 0, '0') - self.assertPropertyValue(vm, 'backup_size', 300, 300, '300') - - def test_390_backup_path(self): - vm = self.get_vm() - self.assertPropertyDefaultValue(vm, 'backup_path', '') - self.assertPropertyValue(vm, 'backup_path', 'some/dir', 'some/dir') - del vm.backup_path - self.assertPropertyDefaultValue(vm, 'backup_path', '') - def test_400_backup_timestamp(self): vm = self.get_vm() timestamp = datetime.datetime(2016, 1, 1, 12, 14, 2) diff --git a/qubes/utils.py b/qubes/utils.py index 08646d73..9b9384ef 100644 --- a/qubes/utils.py +++ b/qubes/utils.py @@ -90,9 +90,12 @@ def format_doc(docstring): # maybe adapt https://code.activestate.com/recipes/578019 def parse_size(size): units = [ - ('K', 1024), ('KB', 1024), - ('M', 1024*1024), ('MB', 1024*1024), - ('G', 1024*1024*1024), ('GB', 1024*1024*1024), + ('K', 1000), ('KB', 1000), + ('M', 1000 * 1000), ('MB', 1000 * 1000), + ('G', 1000 * 1000 * 1000), ('GB', 1000 * 1000 * 1000), + ('Ki', 1024), ('KiB', 1024), + ('Mi', 1024 * 1024), ('MiB', 1024 * 1024), + ('Gi', 1024 * 1024 * 1024), ('GiB', 1024 * 1024 * 1024), ] size = size.strip().upper() @@ -102,10 +105,43 @@ def parse_size(size): for unit, multiplier in units: if size.endswith(unit): size = size[:-len(unit)].strip() - return int(size)*multiplier + return int(size) * multiplier raise qubes.exc.QubesException("Invalid size: {0}.".format(size)) +def mbytes_to_kmg(size): + if size > 1024: + return "%d GiB" % (size / 1024) + else: + return "%d MiB" % size + + +def kbytes_to_kmg(size): + if size > 1024: + return mbytes_to_kmg(size / 1024) + else: + return "%d KiB" % size + + +def bytes_to_kmg(size): + if size > 1024: + return kbytes_to_kmg(size / 1024) + else: + return "%d B" % size + + +def size_to_human(size): + """Humane readable size, with 1/10 precision""" + if size < 1024: + return str(size) + elif size < 1024 * 1024: + return str(round(size / 1024.0, 1)) + ' KiB' + elif size < 1024 * 1024 * 1024: + return str(round(size / (1024.0 * 1024), 1)) + ' MiB' + else: + return str(round(size / (1024.0 * 1024 * 1024), 1)) + ' GiB' + + def urandom(size): rand = os.urandom(size) if rand is None: diff --git a/qubes/vm/qubesvm.py b/qubes/vm/qubesvm.py index 52e44c43..0015d6c1 100644 --- a/qubes/vm/qubesvm.py +++ b/qubes/vm/qubesvm.py @@ -262,22 +262,11 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM): doc='''Setting this to `True` means that VM should be autostarted on dom0 boot.''') - # XXX I don't understand backups - include_in_backups = qubes.property('include_in_backups', default=True, + include_in_backups = qubes.property('include_in_backups', + default=(lambda self: not self.internal), type=bool, setter=qubes.property.bool, doc='If this domain is to be included in default backup.') - backup_content = qubes.property('backup_content', default=False, - type=bool, setter=qubes.property.bool, - doc='FIXME') - - backup_size = qubes.property('backup_size', type=int, default=0, - doc='FIXME') - - # TODO default=None? - backup_path = qubes.property('backup_path', type=str, default='', - doc='FIXME') - # format got changed from %s to str(datetime.datetime) backup_timestamp = qubes.property('backup_timestamp', default=None, setter=(lambda self, prop, value: diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec index ffbe42d3..488e80eb 100644 --- a/rpm_spec/core-dom0.spec +++ b/rpm_spec/core-dom0.spec @@ -201,7 +201,9 @@ fi %dir %{python_sitelib}/qubes %{python_sitelib}/qubes/__init__.py* +%{python_sitelib}/qubes/backup.py* %{python_sitelib}/qubes/config.py* +%{python_sitelib}/qubes/core2migration.py* %{python_sitelib}/qubes/devices.py* %{python_sitelib}/qubes/dochelpers.py* %{python_sitelib}/qubes/events.py* @@ -272,6 +274,8 @@ fi %dir %{python_sitelib}/qubes/tests/int %{python_sitelib}/qubes/tests/int/__init__.py* +%{python_sitelib}/qubes/tests/int/backup.py* +%{python_sitelib}/qubes/tests/int/backupcompatibility.py* %{python_sitelib}/qubes/tests/int/basic.py* %{python_sitelib}/qubes/tests/int/dom0_update.py* %{python_sitelib}/qubes/tests/int/network.py*