diff --git a/qubes/backup.py b/qubes/backup.py index d3637903..fd200059 100644 --- a/qubes/backup.py +++ b/qubes/backup.py @@ -901,484 +901,6 @@ def handle_streams(stream_in, streams_out, processes, size_limit=None, return name -class ExtractWorker2(Process): - def __init__(self, queue, base_dir, passphrase, encrypted, - progress_callback, vmproc=None, - compressed=False, crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM, - verify_only=False, relocate=None): - super(ExtractWorker2, self).__init__() - #: queue with files to extract - self.queue = queue - #: paths on the queue are relative to this dir - self.base_dir = base_dir - #: passphrase to decrypt/authenticate data - self.passphrase = passphrase - #: extract those files/directories to alternative locations (truncate, - # but not unlink target beforehand); if specific file is in the map, - # redirect it accordingly, otherwise check if the whole directory is - # there - self.relocate = relocate - #: is the backup encrypted? - self.encrypted = encrypted - #: is the backup compressed? - self.compressed = compressed - #: what crypto algorithm is used for encryption? - self.crypto_algorithm = crypto_algorithm - #: only verify integrity, don't extract anything - self.verify_only = verify_only - #: progress - self.blocks_backedup = 0 - #: inner tar layer extraction (subprocess.Popen instance) - self.tar2_process = None - #: current inner tar archive name - self.tar2_current_file = None - #: set size of this file when tar report it on stderr (adjust LVM - # volume size) - self.adjust_output_size = None - #: decompressor subprocess.Popen instance - self.decompressor_process = None - #: decryptor subprocess.Popen instance - self.decryptor_process = None - #: callback reporting progress to UI - self.progress_callback = progress_callback - #: process (subprocess.Popen instance) feeding the data into - # extraction tool - self.vmproc = vmproc - - #: pipe to feed the data into tar (use pipe instead of stdin, - # as stdin is used for tar control commands) - self.restore_pipe = os.path.join(self.base_dir, "restore_pipe") - - self.log = logging.getLogger('qubes.backup.extract') - self.log.debug("Creating pipe in: {}".format(self.restore_pipe)) - os.mkfifo(self.restore_pipe) - - self.stderr_encoding = sys.stderr.encoding or 'utf-8' - - def collect_tar_output(self): - if not self.tar2_process.stderr: - return - - if self.tar2_process.poll() is None: - try: - new_lines = self.tar2_process.stderr \ - .read(MAX_STDERR_BYTES).splitlines() - except IOError as e: - if e.errno == errno.EAGAIN: - return - else: - raise - else: - new_lines = self.tar2_process.stderr.readlines() - - new_lines = map(lambda x: x.decode(self.stderr_encoding), new_lines) - - msg_re = re.compile(r".*#[0-9].*restore_pipe") - debug_msg = filter(msg_re.match, new_lines) - self.log.debug('tar2_stderr: {}'.format('\n'.join(debug_msg))) - new_lines = filter(lambda x: not msg_re.match(x), new_lines) - if self.adjust_output_size: - # search for first file size reported by tar, after setting - # self.adjust_output_size (so don't look at self.tar2_stderr) - # this is used only when extracting single-file archive, so don't - # bother with checking file name - file_size_re = re.compile(r"^[^ ]+ [^ ]+/[^ ]+ *([0-9]+) .*") - for line in new_lines: - match = file_size_re.match(line) - if match: - file_size = match.groups()[0] - self.resize_lvm(self.adjust_output_size, file_size) - self.adjust_output_size = None - self.tar2_stderr += new_lines - - def resize_lvm(self, dev, size): - # FIXME: HACK - try: - subprocess.check_call( - ['sudo', 'lvresize', '-f', '-L', str(size) + 'B', dev], - stdout=open(os.devnull, 'w'), stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - if e.returncode == 3: - # already at the right size - pass - else: - raise - - def run(self): - try: - self.__run__() - except Exception as e: - # Cleanup children - for process in [self.decompressor_process, - self.decryptor_process, - self.tar2_process]: - if process: - try: - process.terminate() - except OSError: - pass - process.wait() - self.log.error("ERROR: " + str(e)) - raise - - def handle_dir_relocations(self, dirname): - ''' Relocate files in given director when it's already extracted - - :param dirname: directory path to handle (relative to backup root), - without trailing slash - ''' - - for old, new in self.relocate: - if not old.startswith(dirname + '/'): - continue - # if directory is relocated too (most likely is), the file - # is extracted there - if dirname in self.relocate: - old = old.replace(dirname, self.relocate[dirname], 1) - try: - stat_buf = os.stat(new) - if stat.S_ISBLK(stat_buf.st_mode): - # output file is block device (LVM) - adjust its - # size, otherwise it may fail - # from lack of space - self.resize_lvm(new, stat_buf.st_size) - except OSError: # ENOENT - pass - subprocess.check_call( - ['dd', 'if='+old, 'of='+new, 'conv=sparse']) - os.unlink(old) - - def cleanup_tar2(self, wait=True, terminate=False): - if self.tar2_process is None: - return - if terminate: - self.tar2_process.terminate() - if wait: - self.tar2_process.wait() - elif self.tar2_process.poll() is None: - return - if self.tar2_process.returncode != 0: - self.collect_tar_output() - self.log.error( - "ERROR: unable to extract files for {0}, tar " - "output:\n {1}". - format(self.tar2_current_file, - "\n ".join(self.tar2_stderr))) - else: - # Finished extracting the tar file - self.collect_tar_output() - self.tar2_process = None - # if that was whole-directory archive, handle - # relocated files now - inner_name = os.path.splitext(self.tar2_current_file)[0]\ - .replace(self.base_dir + '/', '') - if os.path.basename(inner_name) == '.': - self.handle_dir_relocations( - os.path.dirname(inner_name)) - self.tar2_current_file = None - self.adjust_output_size = None - - def __run__(self): - self.log.debug("Started sending thread") - self.log.debug("Moving to dir " + self.base_dir) - os.chdir(self.base_dir) - - filename = None - - for filename in iter(self.queue.get, None): - if filename in (QUEUE_FINISHED, QUEUE_ERROR): - break - - self.log.debug("Extracting file " + filename) - - if filename.endswith('.000'): - # next file - self.cleanup_tar2(wait=True, terminate=False) - - inner_name = filename.rstrip('.000').replace( - self.base_dir + '/', '') - redirect_stdout = None - if self.relocate and inner_name in self.relocate: - # TODO: add `dd conv=sparse` when removing tar layer - tar2_cmdline = ['tar', - '-%sMvvOf' % ("t" if self.verify_only else "x"), - self.restore_pipe, - inner_name] - output_file = self.relocate[inner_name] - try: - stat_buf = os.stat(output_file) - if stat.S_ISBLK(stat_buf.st_mode): - # output file is block device (LVM) - adjust its - # size during extraction, otherwise it may fail - # from lack of space - self.adjust_output_size = output_file - except OSError: # ENOENT - pass - redirect_stdout = open(output_file, 'w') - elif self.relocate and \ - os.path.dirname(inner_name) in self.relocate: - tar2_cmdline = ['tar', - '-%sMf' % ("t" if self.verify_only else "x"), - self.restore_pipe, - '-C', self.relocate[os.path.dirname(inner_name)], - # strip all directories - leave only final filename - '--strip-components', str(inner_name.count(os.sep)), - inner_name] - - else: - tar2_cmdline = ['tar', - '-%sMkf' % ("t" if self.verify_only else "x"), - self.restore_pipe, - inner_name] - - self.log.debug("Running command " + str(tar2_cmdline)) - self.tar2_process = subprocess.Popen(tar2_cmdline, - stdin=subprocess.PIPE, stderr=subprocess.PIPE, - stdout=redirect_stdout) - fcntl.fcntl(self.tar2_process.stderr.fileno(), fcntl.F_SETFL, - fcntl.fcntl(self.tar2_process.stderr.fileno(), - fcntl.F_GETFL) | os.O_NONBLOCK) - self.tar2_stderr = [] - elif not self.tar2_process: - # Extracting of the current archive failed, skip to the next - # archive - os.remove(filename) - continue - else: - self.collect_tar_output() - self.log.debug("Releasing next chunck") - self.tar2_process.stdin.write("\n") - self.tar2_process.stdin.flush() - self.tar2_current_file = filename - - pipe = open(self.restore_pipe, 'wb') - monitor_processes = { - 'vmproc': self.vmproc, - 'addproc': self.tar2_process, - } - if self.encrypted: - # Start decrypt - self.decryptor_process = subprocess.Popen( - ["openssl", "enc", - "-d", - "-" + self.crypto_algorithm, - "-pass", - "pass:" + self.passphrase] + - (["-z"] if self.compressed else []), - stdin=open(filename, 'rb'), - stdout=subprocess.PIPE) - in_stream = self.decryptor_process.stdout - monitor_processes['decryptor'] = self.decryptor_process - elif self.compressed: - self.decompressor_process = subprocess.Popen( - ["gzip", "-d"], - stdin=open(filename, 'rb'), - stdout=subprocess.PIPE) - in_stream = self.decompressor_process.stdout - monitor_processes['decompresor'] = self.decompressor_process - else: - in_stream = open(filename, 'rb') - - run_error = handle_streams( - in_stream, - {'target': pipe}, - monitor_processes, - progress_callback=self.progress_callback) - - try: - pipe.close() - except IOError as e: - if e.errno == errno.EPIPE: - self.log.debug( - "Got EPIPE while closing pipe to " - "the inner tar process") - # ignore the error - else: - raise - if run_error: - if run_error == "target": - self.collect_tar_output() - details = "\n".join(self.tar2_stderr) - else: - details = "%s failed" % run_error - self.log.error("Error while processing '{}': {}".format( - self.tar2_current_file, details)) - self.cleanup_tar2(wait=True, terminate=True) - - # Delete the file as we don't need it anymore - self.log.debug("Removing file " + filename) - os.remove(filename) - - os.unlink(self.restore_pipe) - - self.cleanup_tar2(wait=True, terminate=(filename == QUEUE_ERROR)) - self.log.debug("Finished extracting thread") - - -class ExtractWorker3(ExtractWorker2): - def __init__(self, queue, base_dir, passphrase, encrypted, - progress_callback, vmproc=None, - compressed=False, crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM, - compression_filter=None, verify_only=False, relocate=None): - super(ExtractWorker3, self).__init__(queue, base_dir, passphrase, - encrypted, - progress_callback, vmproc, - compressed, crypto_algorithm, - verify_only, relocate) - self.compression_filter = compression_filter - os.unlink(self.restore_pipe) - - def __run__(self): - self.log.debug("Started sending thread") - self.log.debug("Moving to dir " + self.base_dir) - os.chdir(self.base_dir) - - filename = None - - input_pipe = None - for filename in iter(self.queue.get, None): - if filename in (QUEUE_FINISHED, QUEUE_ERROR): - break - - self.log.debug("Extracting file " + filename) - - if filename.endswith('.000'): - # next file - if self.tar2_process is not None: - input_pipe.close() - self.cleanup_tar2(wait=True, terminate=False) - - inner_name = filename.rstrip('.000').replace( - self.base_dir + '/', '') - redirect_stdout = None - if self.relocate and inner_name in self.relocate: - # TODO: add dd conv=sparse when removing tar layer - tar2_cmdline = ['tar', - '-%svvO' % ("t" if self.verify_only else "x"), - inner_name] - output_file = self.relocate[inner_name] - try: - stat_buf = os.stat(output_file) - if stat.S_ISBLK(stat_buf.st_mode): - # output file is block device (LVM) - adjust its - # size during extraction, otherwise it may fail - # from lack of space - self.adjust_output_size = output_file - except OSError: # ENOENT - pass - redirect_stdout = open(output_file, 'w') - elif self.relocate and \ - os.path.dirname(inner_name) in self.relocate: - tar2_cmdline = ['tar', - '-%s' % ("t" if self.verify_only else "x"), - '-C', self.relocate[os.path.dirname(inner_name)], - # strip all directories - leave only final filename - '--strip-components', str(inner_name.count(os.sep)), - inner_name] - else: - tar2_cmdline = ['tar', - '-%sk' % ("t" if self.verify_only else "x"), - inner_name] - - if self.compressed: - if self.compression_filter: - tar2_cmdline.insert(-1, - "--use-compress-program=%s" % - self.compression_filter) - else: - tar2_cmdline.insert(-1, "--use-compress-program=%s" % - DEFAULT_COMPRESSION_FILTER) - - self.log.debug("Running command " + str(tar2_cmdline)) - if self.encrypted: - # Start decrypt - self.decryptor_process = subprocess.Popen( - ["openssl", "enc", - "-d", - "-" + self.crypto_algorithm, - "-pass", - "pass:" + self.passphrase], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE) - - self.tar2_process = subprocess.Popen( - tar2_cmdline, - stdin=self.decryptor_process.stdout, - stdout=redirect_stdout, - stderr=subprocess.PIPE) - input_pipe = self.decryptor_process.stdin - else: - self.tar2_process = subprocess.Popen( - tar2_cmdline, - stdin=subprocess.PIPE, - stdout=redirect_stdout, - stderr=subprocess.PIPE) - input_pipe = self.tar2_process.stdin - - fcntl.fcntl(self.tar2_process.stderr.fileno(), fcntl.F_SETFL, - fcntl.fcntl(self.tar2_process.stderr.fileno(), - fcntl.F_GETFL) | os.O_NONBLOCK) - self.tar2_stderr = [] - elif not self.tar2_process: - # Extracting of the current archive failed, skip to the next - # archive - os.remove(filename) - continue - else: - (basename, ext) = os.path.splitext(self.tar2_current_file) - previous_chunk_number = int(ext[1:]) - expected_filename = basename + '.%03d' % ( - previous_chunk_number+1) - if expected_filename != filename: - self.cleanup_tar2(wait=True, terminate=True) - self.log.error( - 'Unexpected file in archive: {}, expected {}'.format( - filename, expected_filename)) - os.remove(filename) - continue - self.log.debug("Releasing next chunck") - - self.tar2_current_file = filename - - run_error = handle_streams( - open(filename, 'rb'), - {'target': input_pipe}, - {'vmproc': self.vmproc, - 'addproc': self.tar2_process, - 'decryptor': self.decryptor_process, - }, - progress_callback=self.progress_callback) - - if run_error: - if run_error == "target": - self.collect_tar_output() - details = "\n".join(self.tar2_stderr) - else: - details = "%s failed" % run_error - if self.decryptor_process: - self.decryptor_process.terminate() - self.decryptor_process.wait() - self.decryptor_process = None - self.log.error("Error while processing '{}': {}".format( - self.tar2_current_file, details)) - self.cleanup_tar2(wait=True, terminate=True) - - # Delete the file as we don't need it anymore - self.log.debug("Removing file " + filename) - os.remove(filename) - - if self.tar2_process is not None: - input_pipe.close() - if filename == QUEUE_ERROR: - if self.decryptor_process: - self.decryptor_process.terminate() - self.decryptor_process.wait() - self.decryptor_process = None - self.cleanup_tar2(terminate=(filename == QUEUE_ERROR)) - - self.log.debug("Finished extracting thread") - - def get_supported_hmac_algo(hmac_algorithm=None): # Start with provided default if hmac_algorithm: @@ -1394,1231 +916,4 @@ def get_supported_hmac_algo(hmac_algorithm=None): yield algo.strip() proc.wait() - -class BackupRestoreOptions(object): - def __init__(self): - #: use default NetVM if the one referenced in backup do not exists on - # the host - self.use_default_netvm = True - #: set NetVM to "none" if the one referenced in backup do not exists - # on the host - self.use_none_netvm = False - #: set template to default if the one referenced in backup do not - # exists on the host - self.use_default_template = True - #: use default kernel if the one referenced in backup do not exists - # on the host - self.use_default_kernel = True - #: restore dom0 home - self.dom0_home = True - #: dictionary how what templates should be used instead of those - # referenced in backup - self.replace_template = {} - #: restore dom0 home even if username is different - self.ignore_username_mismatch = False - #: do not restore data, only verify backup integrity - self.verify_only = False - #: automatically rename VM during restore, when it would conflict - # with existing one - self.rename_conflicting = True - #: list of VM names to exclude - self.exclude = [] - #: restore VMs into selected storage pool - self.override_pool = None - - -class BackupRestore(object): - """Usage: - - >>> restore_op = BackupRestore(...) - >>> # adjust restore_op.options here - >>> restore_info = restore_op.get_restore_info() - >>> # manipulate restore_info to select VMs to restore here - >>> restore_op.restore_do(restore_info) - """ - - class VMToRestore(object): - #: VM excluded from restore by user - EXCLUDED = object() - #: VM with such name already exists on the host - ALREADY_EXISTS = object() - #: NetVM used by the VM does not exists on the host - MISSING_NETVM = object() - #: TemplateVM used by the VM does not exists on the host - MISSING_TEMPLATE = object() - #: Kernel used by the VM does not exists on the host - MISSING_KERNEL = object() - - def __init__(self, vm): - self.vm = vm - if 'backup-path' in vm.features: - self.subdir = vm.features['backup-path'] - else: - self.subdir = None - if 'backup-size' in vm.features and vm.features['backup-size']: - self.size = int(vm.features['backup-size']) - else: - self.size = 0 - self.problems = set() - if hasattr(vm, 'template') and vm.template: - self.template = vm.template.name - else: - self.template = None - if vm.netvm: - self.netvm = vm.netvm.name - else: - self.netvm = None - self.name = vm.name - self.orig_template = None - self.restored_vm = None - - @property - def good_to_go(self): - return len(self.problems) == 0 - - class Dom0ToRestore(VMToRestore): - #: backup was performed on system with different dom0 username - USERNAME_MISMATCH = object() - - def __init__(self, vm, subdir=None): - super(BackupRestore.Dom0ToRestore, self).__init__(vm) - if subdir: - self.subdir = subdir - self.username = os.path.basename(subdir) - - def __init__(self, app, backup_location, backup_vm, passphrase): - super(BackupRestore, self).__init__() - - #: qubes.Qubes instance - self.app = app - - #: options how the backup should be restored - self.options = BackupRestoreOptions() - - #: VM from which backup should be retrieved - self.backup_vm = backup_vm - if backup_vm and backup_vm.qid == 0: - self.backup_vm = None - - #: backup path, inside VM pointed by :py:attr:`backup_vm` - self.backup_location = backup_location - - #: passphrase protecting backup integrity and optionally decryption - self.passphrase = passphrase - - #: temporary directory used to extract the data before moving to the - # final location; should be on the same filesystem as /var/lib/qubes - self.tmpdir = tempfile.mkdtemp(prefix="restore", dir="/var/tmp") - - #: list of processes (Popen objects) to kill on cancel - self.processes_to_kill_on_cancel = [] - - #: is the backup operation canceled - self.canceled = False - - #: report restore progress, called with one argument - percents of - # data restored - # FIXME: convert to float [0,1] - self.progress_callback = None - - self.log = logging.getLogger('qubes.backup') - - #: basic information about the backup - self.header_data = self._retrieve_backup_header() - - #: VMs included in the backup - self.backup_app = self._process_qubes_xml() - - def cancel(self): - """Cancel running backup operation. Can be called from another thread. - """ - self.canceled = True - for proc in self.processes_to_kill_on_cancel: - try: - proc.terminate() - except OSError: - pass - - def _start_retrieval_process(self, filelist, limit_count, limit_bytes): - """Retrieve backup stream and extract it to :py:attr:`tmpdir` - - :param filelist: list of files to extract; listing directory name - will extract the whole directory; use empty list to extract the whole - archive - :param limit_count: maximum number of files to extract - :param limit_bytes: maximum size of extracted data - :return: a touple of (Popen object of started process, file-like - object for reading extracted files list, file-like object for reading - errors) - """ - - vmproc = None - if self.backup_vm is not None: - # If APPVM, STDOUT is a PIPE - vmproc = self.backup_vm.run_service('qubes.Restore', - passio_popen=True, passio_stderr=True) - vmproc.stdin.write( - (self.backup_location.replace("\r", "").replace("\n", - "") + "\n").encode()) - vmproc.stdin.flush() - - # Send to tar2qfile the VMs that should be extracted - vmproc.stdin.write((" ".join(filelist) + "\n").encode()) - vmproc.stdin.flush() - self.processes_to_kill_on_cancel.append(vmproc) - - backup_stdin = vmproc.stdout - tar1_command = ['/usr/libexec/qubes/qfile-dom0-unpacker', - str(os.getuid()), self.tmpdir, '-v'] - else: - backup_stdin = open(self.backup_location, 'rb') - - tar1_command = ['tar', - '-ixv', - '-C', self.tmpdir] + filelist - - tar1_env = os.environ.copy() - tar1_env['UPDATES_MAX_BYTES'] = str(limit_bytes) - tar1_env['UPDATES_MAX_FILES'] = str(limit_count) - self.log.debug("Run command" + str(tar1_command)) - command = subprocess.Popen( - tar1_command, - stdin=backup_stdin, - stdout=vmproc.stdin if vmproc else subprocess.PIPE, - stderr=subprocess.PIPE, - env=tar1_env) - self.processes_to_kill_on_cancel.append(command) - - # qfile-dom0-unpacker output filelist on stderr - # and have stdout connected to the VM), while tar output filelist - # on stdout - if self.backup_vm: - filelist_pipe = command.stderr - # let qfile-dom0-unpacker hold the only open FD to the write end of - # pipe, otherwise qrexec-client will not receive EOF when - # qfile-dom0-unpacker terminates - vmproc.stdin.close() - else: - filelist_pipe = command.stdout - - if self.backup_vm: - error_pipe = vmproc.stderr - else: - error_pipe = command.stderr - return command, filelist_pipe, error_pipe - - def _verify_hmac(self, filename, hmacfile, algorithm=None): - def load_hmac(hmac_text): - if any(ord(x) not in range(128) for x in hmac_text): - raise qubes.exc.QubesException( - "Invalid content of {}".format(hmacfile)) - hmac_text = hmac_text.strip().split("=") - if len(hmac_text) > 1: - hmac_text = hmac_text[1].strip() - else: - raise qubes.exc.QubesException( - "ERROR: invalid hmac file content") - - return hmac_text - if algorithm is None: - algorithm = self.header_data.hmac_algorithm - passphrase = self.passphrase.encode('utf-8') - self.log.debug("Verifying file {}".format(filename)) - - if os.stat(os.path.join(self.tmpdir, hmacfile)).st_size > \ - HMAC_MAX_SIZE: - raise qubes.exc.QubesException('HMAC file {} too large'.format( - hmacfile)) - - if hmacfile != filename + ".hmac": - raise qubes.exc.QubesException( - "ERROR: expected hmac for {}, but got {}". - format(filename, hmacfile)) - - if algorithm == 'scrypt': - # in case of 'scrypt' _verify_hmac is only used for backup header - assert filename == HEADER_FILENAME - self._verify_and_decrypt(hmacfile, HEADER_FILENAME + '.dec') - if open(os.path.join(self.tmpdir, filename), 'rb').read() != \ - open(os.path.join(self.tmpdir, filename + '.dec'), - 'rb').read(): - raise qubes.exc.QubesException( - 'Invalid hmac on {}'.format(filename)) - else: - return True - - hmac_proc = subprocess.Popen( - ["openssl", "dgst", "-" + algorithm, "-hmac", passphrase], - stdin=open(os.path.join(self.tmpdir, filename), 'rb'), - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - hmac_stdout, hmac_stderr = hmac_proc.communicate() - - if len(hmac_stderr) > 0: - raise qubes.exc.QubesException( - "ERROR: verify file {0}: {1}".format(filename, hmac_stderr)) - else: - self.log.debug("Loading hmac for file {}".format(filename)) - hmac = load_hmac(open(os.path.join(self.tmpdir, hmacfile), - 'r', encoding='ascii').read()) - - if len(hmac) > 0 and load_hmac(hmac_stdout.decode('ascii')) == hmac: - os.unlink(os.path.join(self.tmpdir, hmacfile)) - self.log.debug( - "File verification OK -> Sending file {}".format(filename)) - return True - else: - raise qubes.exc.QubesException( - "ERROR: invalid hmac for file {0}: {1}. " - "Is the passphrase correct?". - format(filename, load_hmac(hmac_stdout.decode('ascii')))) - - def _verify_and_decrypt(self, filename, output=None): - assert filename.endswith('.enc') or filename.endswith('.hmac') - fullname = os.path.join(self.tmpdir, filename) - (origname, _) = os.path.splitext(filename) - if output: - fulloutput = os.path.join(self.tmpdir, output) - else: - fulloutput = os.path.join(self.tmpdir, origname) - if origname == HEADER_FILENAME: - passphrase = u'{filename}!{passphrase}'.format( - filename=origname, - passphrase=self.passphrase) - else: - passphrase = u'{backup_id}!{filename}!{passphrase}'.format( - backup_id=self.header_data.backup_id, - filename=origname, - passphrase=self.passphrase) - p = launch_scrypt('dec', fullname, fulloutput, passphrase) - (_, stderr) = p.communicate() - if p.returncode != 0: - os.unlink(fulloutput) - raise qubes.exc.QubesException('failed to decrypt {}: {}'.format( - fullname, stderr)) - # encrypted file is no longer needed - os.unlink(fullname) - return origname - - def _retrieve_backup_header_files(self, files, allow_none=False): - (retrieve_proc, filelist_pipe, error_pipe) = \ - self._start_retrieval_process( - files, len(files), 1024 * 1024) - filelist = filelist_pipe.read() - retrieve_proc_returncode = retrieve_proc.wait() - if retrieve_proc in self.processes_to_kill_on_cancel: - self.processes_to_kill_on_cancel.remove(retrieve_proc) - extract_stderr = error_pipe.read(MAX_STDERR_BYTES) - - # wait for other processes (if any) - for proc in self.processes_to_kill_on_cancel: - if proc.wait() != 0: - raise qubes.exc.QubesException( - "Backup header retrieval failed (exit code {})".format( - proc.wait()) - ) - - if retrieve_proc_returncode != 0: - if not filelist and 'Not found in archive' in extract_stderr: - if allow_none: - return None - else: - raise qubes.exc.QubesException( - "unable to read the qubes backup file {0} ({1}): {2}".format( - self.backup_location, - retrieve_proc.wait(), - extract_stderr - )) - actual_files = filelist.decode('ascii').splitlines() - if sorted(actual_files) != sorted(files): - raise qubes.exc.QubesException( - 'unexpected files in archive: got {!r}, expected {!r}'.format( - actual_files, files - )) - for f in files: - if not os.path.exists(os.path.join(self.tmpdir, f)): - if allow_none: - return None - else: - raise qubes.exc.QubesException( - 'Unable to retrieve file {} from backup {}: {}'.format( - f, self.backup_location, extract_stderr - ) - ) - return files - - def _retrieve_backup_header(self): - """Retrieve backup header and qubes.xml. Only backup header is - analyzed, qubes.xml is left as-is - (not even verified/decrypted/uncompressed) - - :return header_data - :rtype :py:class:`BackupHeader` - """ - - if not self.backup_vm and os.path.exists( - os.path.join(self.backup_location, 'qubes.xml')): - # backup format version 1 doesn't have header - header_data = BackupHeader() - header_data.version = 1 - return header_data - - header_files = self._retrieve_backup_header_files( - ['backup-header', 'backup-header.hmac'], allow_none=True) - - if not header_files: - # R2-Beta3 didn't have backup header, so if none is found, - # assume it's version=2 and use values present at that time - header_data = BackupHeader( - version=2, - # place explicitly this value, because it is what format_version - # 2 have - hmac_algorithm='SHA1', - crypto_algorithm='aes-256-cbc', - # TODO: set encrypted to something... - ) - else: - filename = HEADER_FILENAME - hmacfile = HEADER_FILENAME + '.hmac' - self.log.debug("Got backup header and hmac: {}, {}".format( - filename, hmacfile)) - - file_ok = False - hmac_algorithm = DEFAULT_HMAC_ALGORITHM - for hmac_algo in get_supported_hmac_algo(hmac_algorithm): - try: - if self._verify_hmac(filename, hmacfile, hmac_algo): - file_ok = True - break - except qubes.exc.QubesException as e: - self.log.debug( - 'Failed to verify {} using {}: {}'.format( - hmacfile, hmac_algo, str(e))) - # Ignore exception here, try the next algo - pass - if not file_ok: - raise qubes.exc.QubesException( - "Corrupted backup header (hmac verification " - "failed). Is the password correct?") - filename = os.path.join(self.tmpdir, filename) - header_data = BackupHeader(open(filename, 'rb').read()) - os.unlink(filename) - - return header_data - - def _start_inner_extraction_worker(self, queue, relocate): - """Start a worker process, extracting inner layer of bacup archive, - extract them to :py:attr:`tmpdir`. - End the data by pushing QUEUE_FINISHED or QUEUE_ERROR to the queue. - - :param queue :py:class:`Queue` object to handle files from - """ - - # Setup worker to extract encrypted data chunks to the restore dirs - # Create the process here to pass it options extracted from - # backup header - extractor_params = { - 'queue': queue, - 'base_dir': self.tmpdir, - 'passphrase': self.passphrase, - 'encrypted': self.header_data.encrypted, - 'compressed': self.header_data.compressed, - 'crypto_algorithm': self.header_data.crypto_algorithm, - 'verify_only': self.options.verify_only, - 'progress_callback': self.progress_callback, - 'relocate': relocate, - } - self.log.debug('Starting extraction worker in {}, file relocation ' - 'map: {!r}'.format(self.tmpdir, relocate)) - format_version = self.header_data.version - if format_version == 2: - extract_proc = ExtractWorker2(**extractor_params) - elif format_version in [3, 4]: - extractor_params['compression_filter'] = \ - self.header_data.compression_filter - if format_version == 4: - # encryption already handled - extractor_params['encrypted'] = False - extract_proc = ExtractWorker3(**extractor_params) - else: - raise NotImplementedError( - "Backup format version %d not supported" % format_version) - extract_proc.start() - return extract_proc - - def _process_qubes_xml(self): - """Verify, unpack and load qubes.xml. Possibly convert its format if - necessary. It expect that :py:attr:`header_data` is already populated, - and :py:meth:`retrieve_backup_header` was called. - """ - if self.header_data.version == 1: - backup_app = qubes.core2migration.Core2Qubes( - os.path.join(self.backup_location, 'qubes.xml'), - offline_mode=True) - return backup_app - else: - if self.header_data.version in [2, 3]: - self._retrieve_backup_header_files( - ['qubes.xml.000', 'qubes.xml.000.hmac']) - self._verify_hmac("qubes.xml.000", "qubes.xml.000.hmac") - else: - self._retrieve_backup_header_files(['qubes.xml.000.enc']) - self._verify_and_decrypt('qubes.xml.000.enc') - - queue = Queue() - queue.put("qubes.xml.000") - queue.put(QUEUE_FINISHED) - - extract_proc = self._start_inner_extraction_worker(queue, None) - extract_proc.join() - if extract_proc.exitcode != 0: - raise qubes.exc.QubesException( - "unable to extract the qubes backup. " - "Check extracting process errors.") - - if self.header_data.version in [2, 3]: - backup_app = qubes.core2migration.Core2Qubes( - os.path.join(self.tmpdir, 'qubes.xml'), offline_mode=True) - else: - backup_app = qubes.Qubes(os.path.join(self.tmpdir, 'qubes.xml'), - offline_mode=True) - # Not needed anymore - all the data stored in backup_app - os.unlink(os.path.join(self.tmpdir, 'qubes.xml')) - return backup_app - - def _restore_vm_dirs(self, vms_dirs, vms_size, relocate): - # Currently each VM consists of at most 7 archives (count - # file_to_backup calls in backup_prepare()), but add some safety - # margin for further extensions. Each archive is divided into 100MB - # chunks. Additionally each file have own hmac file. So assume upper - # limit as 2*(10*COUNT_OF_VMS+TOTAL_SIZE/100MB) - limit_count = str(2 * (10 * len(vms_dirs) + - int(vms_size / (100 * 1024 * 1024)))) - - self.log.debug("Working in temporary dir:" + self.tmpdir) - self.log.info( - "Extracting data: " + size_to_human(vms_size) + " to restore") - - # retrieve backup from the backup stream (either VM, or dom0 file) - (retrieve_proc, filelist_pipe, error_pipe) = \ - self._start_retrieval_process( - vms_dirs, limit_count, vms_size) - - to_extract = Queue() - - # extract data retrieved by retrieve_proc - extract_proc = self._start_inner_extraction_worker( - to_extract, relocate) - - try: - filename = None - hmacfile = None - nextfile = None - while True: - if self.canceled: - break - if not extract_proc.is_alive(): - retrieve_proc.terminate() - retrieve_proc.wait() - if retrieve_proc in self.processes_to_kill_on_cancel: - self.processes_to_kill_on_cancel.remove(retrieve_proc) - # wait for other processes (if any) - for proc in self.processes_to_kill_on_cancel: - proc.wait() - break - if nextfile is not None: - filename = nextfile - else: - filename = filelist_pipe.readline().decode('ascii').strip() - - self.log.debug("Getting new file:" + filename) - - if not filename or filename == "EOF": - break - - # if reading archive directly with tar, wait for next filename - - # tar prints filename before processing it, so wait for - # the next one to be sure that whole file was extracted - if not self.backup_vm: - nextfile = filelist_pipe.readline().decode('ascii').strip() - - if self.header_data.version in [2, 3]: - if not self.backup_vm: - hmacfile = nextfile - nextfile = filelist_pipe.readline().\ - decode('ascii').strip() - else: - hmacfile = filelist_pipe.readline().\ - decode('ascii').strip() - - if self.canceled: - break - - self.log.debug("Getting hmac:" + hmacfile) - if not hmacfile or hmacfile == "EOF": - # Premature end of archive, either of tar1_command or - # vmproc exited with error - break - else: # self.header_data.version == 4 - if not filename.endswith('.enc'): - raise qubes.exc.QubesException( - 'Invalid file extension found in archive: {}'. - format(filename)) - - if not any(map(lambda x: filename.startswith(x), vms_dirs)): - self.log.debug("Ignoring VM not selected for restore") - os.unlink(os.path.join(self.tmpdir, filename)) - if hmacfile: - os.unlink(os.path.join(self.tmpdir, hmacfile)) - continue - - if self.header_data.version in [2, 3]: - self._verify_hmac(filename, hmacfile) - else: - # _verify_and_decrypt will write output to a file with - # '.enc' extension cut off. This is safe because: - # - `scrypt` tool will override output, so if the file was - # already there (received from the VM), it will be removed - # - incoming archive extraction will refuse to override - # existing file, so if `scrypt` already created one, - # it can not be manipulated by the VM - # - when the file is retrieved from the VM, it appears at - # the final form - if it's visible, VM have no longer - # influence over its content - # - # This all means that if the file was correctly verified - # + decrypted, we will surely access the right file - filename = self._verify_and_decrypt(filename) - to_extract.put(os.path.join(self.tmpdir, filename)) - - if self.canceled: - raise BackupCanceledError("Restore canceled", - tmpdir=self.tmpdir) - - if retrieve_proc.wait() != 0: - raise qubes.exc.QubesException( - "unable to read the qubes backup file {0}: {1}" - .format(self.backup_location, error_pipe.read( - MAX_STDERR_BYTES))) - # wait for other processes (if any) - for proc in self.processes_to_kill_on_cancel: - proc.wait() - if proc.returncode != 0: - raise qubes.exc.QubesException( - "Backup completed, but VM receiving it reported an error " - "(exit code {})".format(proc.returncode)) - - if filename and filename != "EOF": - raise qubes.exc.QubesException( - "Premature end of archive, the last file was %s" % filename) - except: - to_extract.put(QUEUE_ERROR) - extract_proc.join() - raise - else: - to_extract.put(QUEUE_FINISHED) - - self.log.debug("Waiting for the extraction process to finish...") - extract_proc.join() - self.log.debug("Extraction process finished with code: {}".format( - extract_proc.exitcode)) - if extract_proc.exitcode != 0: - raise qubes.exc.QubesException( - "unable to extract the qubes backup. " - "Check extracting process errors.") - - def generate_new_name_for_conflicting_vm(self, orig_name, restore_info): - number = 1 - if len(orig_name) > 29: - orig_name = orig_name[0:29] - new_name = orig_name - while (new_name in restore_info.keys() or - new_name in map(lambda x: x.name, - restore_info.values()) or - new_name in self.app.domains): - new_name = str('{}{}'.format(orig_name, number)) - number += 1 - if number == 100: - # give up - return None - return new_name - - def restore_info_verify(self, restore_info): - for vm in restore_info.keys(): - if vm in ['dom0']: - continue - - vm_info = restore_info[vm] - assert isinstance(vm_info, self.VMToRestore) - - vm_info.problems.clear() - if vm in self.options.exclude: - vm_info.problems.add(self.VMToRestore.EXCLUDED) - - if not self.options.verify_only and \ - vm_info.name in self.app.domains: - if self.options.rename_conflicting: - new_name = self.generate_new_name_for_conflicting_vm( - vm, restore_info - ) - if new_name is not None: - vm_info.name = new_name - else: - vm_info.problems.add(self.VMToRestore.ALREADY_EXISTS) - else: - vm_info.problems.add(self.VMToRestore.ALREADY_EXISTS) - - # check template - if vm_info.template: - template_name = vm_info.template - try: - host_template = self.app.domains[template_name] - except KeyError: - host_template = None - if not host_template \ - or not isinstance(host_template, - qubes.vm.templatevm.TemplateVM): - # Maybe the (custom) template is in the backup? - if not (template_name in restore_info.keys() and - restore_info[template_name].good_to_go and - isinstance(restore_info[template_name].vm, - qubes.vm.templatevm.TemplateVM)): - if self.options.use_default_template and \ - self.app.default_template: - if vm_info.orig_template is None: - vm_info.orig_template = template_name - vm_info.template = self.app.default_template.name - else: - vm_info.problems.add( - self.VMToRestore.MISSING_TEMPLATE) - - # check netvm - if not vm_info.vm.property_is_default('netvm') and vm_info.netvm: - netvm_name = vm_info.netvm - - try: - netvm_on_host = self.app.domains[netvm_name] - except KeyError: - netvm_on_host = None - # No netvm on the host? - if not ((netvm_on_host is not None) - and netvm_on_host.provides_network): - - # Maybe the (custom) netvm is in the backup? - if not (netvm_name in restore_info.keys() and - restore_info[netvm_name].good_to_go and - restore_info[netvm_name].vm.provides_network): - if self.options.use_default_netvm: - vm_info.vm.netvm = qubes.property.DEFAULT - elif self.options.use_none_netvm: - vm_info.netvm = None - else: - vm_info.problems.add(self.VMToRestore.MISSING_NETVM) - - # check kernel - if hasattr(vm_info.vm, 'kernel'): - installed_kernels = os.listdir(os.path.join( - qubes.config.qubes_base_dir, - qubes.config.system_path['qubes_kernels_base_dir'])) - # if uses default kernel - do not validate it - # allow kernel=None only for HVM, - # otherwise require valid kernel - if not (vm_info.vm.property_is_default('kernel') - or (not vm_info.vm.kernel and vm_info.vm.virt_mode == 'hvm') - or vm_info.vm.kernel in installed_kernels): - if self.options.use_default_kernel: - vm_info.vm.kernel = qubes.property.DEFAULT - else: - vm_info.problems.add(self.VMToRestore.MISSING_KERNEL) - - return restore_info - - def _is_vm_included_in_backup_v1(self, check_vm): - if check_vm.qid == 0: - return os.path.exists( - os.path.join(self.backup_location, 'dom0-home')) - - # DisposableVM - if check_vm.dir_path is None: - return False - - backup_vm_dir_path = check_vm.dir_path.replace( - qubes.config.system_path["qubes_base_dir"], self.backup_location) - - if os.path.exists(backup_vm_dir_path): - return True - else: - return False - - @staticmethod - def _is_vm_included_in_backup_v2(check_vm): - if 'backup-content' in check_vm.features: - return check_vm.features['backup-content'] - else: - return False - - def _find_template_name(self, template): - if template in self.options.replace_template: - return self.options.replace_template[template] - return template - - def _is_vm_included_in_backup(self, vm): - if self.header_data.version == 1: - return self._is_vm_included_in_backup_v1(vm) - elif self.header_data.version in [2, 3, 4]: - return self._is_vm_included_in_backup_v2(vm) - else: - raise qubes.exc.QubesException( - "Unknown backup format version: {}".format( - self.header_data.version)) - - def get_restore_info(self): - # Format versions: - # 1 - Qubes R1, Qubes R2 beta1, beta2 - # 2 - Qubes R2 beta3+ - - vms_to_restore = {} - - for vm in self.backup_app.domains: - if vm.qid == 0: - # Handle dom0 as special case later - continue - if self._is_vm_included_in_backup(vm): - self.log.debug("{} is included in backup".format(vm.name)) - - vms_to_restore[vm.name] = self.VMToRestore(vm) - - if hasattr(vm, 'template'): - templatevm_name = self._find_template_name( - vm.template.name) - vms_to_restore[vm.name].template = templatevm_name - - # Set to None to not confuse QubesVm object from backup - # collection with host collection (further in clone_attrs). - vm.netvm = None - - vms_to_restore = self.restore_info_verify(vms_to_restore) - - # ...and dom0 home - if self.options.dom0_home and \ - self._is_vm_included_in_backup(self.backup_app.domains[0]): - vm = self.backup_app.domains[0] - if self.header_data.version == 1: - subdir = os.listdir(os.path.join(self.backup_location, - 'dom0-home'))[0] - else: - subdir = None - vms_to_restore['dom0'] = self.Dom0ToRestore(vm, subdir) - local_user = grp.getgrnam('qubes').gr_mem[0] - - if vms_to_restore['dom0'].username != local_user: - if not self.options.ignore_username_mismatch: - vms_to_restore['dom0'].problems.add( - self.Dom0ToRestore.USERNAME_MISMATCH) - - return vms_to_restore - - @staticmethod - def get_restore_summary(restore_info): - fields = { - "qid": {"func": "vm.qid"}, - - "name": {"func": "('[' if isinstance(vm, qubes.vm.templatevm.TemplateVM) else '')\ - + ('{' if vm.provides_network else '')\ - + vm.name \ - + (']' if isinstance(vm, qubes.vm.templatevm.TemplateVM) else '')\ - + ('}' if vm.provides_network else '')"}, - - "type": {"func": "'Tpl' if isinstance(vm, qubes.vm.templatevm.TemplateVM) else \ - 'App' if isinstance(vm, qubes.vm.appvm.AppVM) else \ - vm.__class__.__name__.replace('VM','')"}, - - "updbl": {"func": "'Yes' if vm.updateable else ''"}, - - "template": {"func": "'n/a' if not hasattr(vm, 'template') " - "else vm_info.template"}, - - "netvm": {"func": "('*' if vm.property_is_default('netvm') else '') +\ - vm_info.netvm if vm_info.netvm is not None " - "else '-'"}, - - "label": {"func": "vm.label.name"}, - } - - fields_to_display = ["name", "type", "template", "updbl", - "netvm", "label"] - - # First calculate the maximum width of each field we want to display - total_width = 0 - for f in fields_to_display: - fields[f]["max_width"] = len(f) - for vm_info in restore_info.values(): - if vm_info.vm: - # noinspection PyUnusedLocal - vm = vm_info.vm - l = len(str(eval(fields[f]["func"]))) - if l > fields[f]["max_width"]: - fields[f]["max_width"] = l - total_width += fields[f]["max_width"] - - summary = "" - summary += "The following VMs are included in the backup:\n" - summary += "\n" - - # Display the header - for f in fields_to_display: - # noinspection PyTypeChecker - fmt = "{{0:-^{0}}}-+".format(fields[f]["max_width"] + 1) - summary += fmt.format('-') - summary += "\n" - for f in fields_to_display: - # noinspection PyTypeChecker - fmt = "{{0:>{0}}} |".format(fields[f]["max_width"] + 1) - summary += fmt.format(f) - summary += "\n" - for f in fields_to_display: - # noinspection PyTypeChecker - fmt = "{{0:-^{0}}}-+".format(fields[f]["max_width"] + 1) - summary += fmt.format('-') - summary += "\n" - - for vm_info in restore_info.values(): - assert isinstance(vm_info, BackupRestore.VMToRestore) - # Skip non-VM here - if not vm_info.vm: - continue - # noinspection PyUnusedLocal - vm = vm_info.vm - s = "" - for f in fields_to_display: - # noinspection PyTypeChecker - fmt = "{{0:>{0}}} |".format(fields[f]["max_width"] + 1) - s += fmt.format(eval(fields[f]["func"])) - - if BackupRestore.VMToRestore.EXCLUDED in vm_info.problems: - s += " <-- Excluded from restore" - elif BackupRestore.VMToRestore.ALREADY_EXISTS in vm_info.problems: - s += " <-- A VM with the same name already exists on the host!" - elif BackupRestore.VMToRestore.MISSING_TEMPLATE in \ - vm_info.problems: - s += " <-- No matching template on the host " \ - "or in the backup found!" - elif BackupRestore.VMToRestore.MISSING_NETVM in \ - vm_info.problems: - s += " <-- No matching netvm on the host " \ - "or in the backup found!" - else: - if vm_info.orig_template: - s += " <-- Original template was '{}'".format( - vm_info.orig_template) - if vm_info.name != vm_info.vm.name: - s += " <-- Will be renamed to '{}'".format( - vm_info.name) - - summary += s + "\n" - - if 'dom0' in restore_info.keys(): - s = "" - for f in fields_to_display: - # noinspection PyTypeChecker - fmt = "{{0:>{0}}} |".format(fields[f]["max_width"] + 1) - if f == "name": - s += fmt.format("Dom0") - elif f == "type": - s += fmt.format("Home") - else: - s += fmt.format("") - if BackupRestore.Dom0ToRestore.USERNAME_MISMATCH in \ - restore_info['dom0'].problems: - s += " <-- username in backup and dom0 mismatch" - - summary += s + "\n" - - return summary - - def _restore_vm_dir_v1(self, src_dir, dst_dir): - - backup_src_dir = src_dir.replace( - qubes.config.system_path["qubes_base_dir"], self.backup_location) - - # We prefer to use Linux's cp, because it nicely handles sparse files - cp_retcode = subprocess.call( - ["cp", "-rp", "--reflink=auto", backup_src_dir, dst_dir]) - if cp_retcode != 0: - raise qubes.exc.QubesException( - "*** Error while copying file {0} to {1}".format(backup_src_dir, - dst_dir)) - - @staticmethod - def _templates_first(vms): - def key_function(instance): - if isinstance(instance, qubes.vm.BaseVM): - return isinstance(instance, qubes.vm.templatevm.TemplateVM) - elif hasattr(instance, 'vm'): - return key_function(instance.vm) - else: - return 0 - return sorted(vms, - key=key_function, - reverse=True) - - def restore_do(self, restore_info): - ''' - - - High level workflow: - 1. Create VMs object in host collection (qubes.xml) - 2. Create them on disk (vm.create_on_disk) - 3. Restore VM data, overriding/converting VM files - 4. Apply possible fixups and save qubes.xml - - :param restore_info: - :return: - ''' - - # FIXME handle locking - - restore_info = self.restore_info_verify(restore_info) - - self._restore_vms_metadata(restore_info) - - # Perform VM restoration in backup order - vms_dirs = [] - relocate = {} - vms_size = 0 - for vm_info in self._templates_first(restore_info.values()): - vm = vm_info.restored_vm - if vm: - vms_size += int(vm_info.size) - vms_dirs.append(vm_info.subdir) - relocate[vm_info.subdir.rstrip('/')] = vm.dir_path - for name, volume in vm.volumes.items(): - if not volume.save_on_stop: - continue - export_path = vm.storage.export(name) - backup_path = os.path.join( - vm_info.vm.dir_path, name + '.img') - if backup_path != export_path: - relocate[ - os.path.join(vm_info.subdir, name + '.img')] = \ - export_path - - if self.header_data.version >= 2: - if 'dom0' in restore_info.keys() and \ - restore_info['dom0'].good_to_go: - vms_dirs.append(os.path.dirname(restore_info['dom0'].subdir)) - vms_size += restore_info['dom0'].size - - try: - self._restore_vm_dirs(vms_dirs=vms_dirs, vms_size=vms_size, - relocate=relocate) - except qubes.exc.QubesException: - if self.options.verify_only: - raise - else: - self.log.warning( - "Some errors occurred during data extraction, " - "continuing anyway to restore at least some " - "VMs") - else: - for vm_info in self._templates_first(restore_info.values()): - vm = vm_info.restored_vm - if vm: - try: - self._restore_vm_dir_v1(vm_info.vm.dir_path, - os.path.dirname(vm.dir_path)) - except qubes.exc.QubesException as e: - if self.options.verify_only: - raise - else: - self.log.error( - "Failed to restore VM '{}': {}".format( - vm.name, str(e))) - vm.remove_from_disk() - del self.app.domains[vm] - - if self.options.verify_only: - self.log.warning( - "Backup verification not supported for this backup format.") - - if self.options.verify_only: - shutil.rmtree(self.tmpdir) - return - - for vm_info in self._templates_first(restore_info.values()): - if not vm_info.restored_vm: - continue - try: - vm_info.restored_vm.fire_event('domain-restore') - except Exception as err: - self.log.error("ERROR during appmenu restore: " - "{0}".format(err)) - self.log.warning( - "*** VM '{0}' will not have appmenus".format(vm_info.name)) - - try: - vm_info.restored_vm.storage.verify() - except Exception as err: - self.log.error("ERROR: {0}".format(err)) - if vm_info.restored_vm: - vm_info.restored_vm.remove_from_disk() - del self.app.domains[vm_info.restored_vm] - - self.app.save() - - if self.canceled: - if self.header_data.version >= 2: - raise BackupCanceledError("Restore canceled", - tmpdir=self.tmpdir) - else: - raise BackupCanceledError("Restore canceled") - - # ... and dom0 home as last step - if 'dom0' in restore_info.keys() and restore_info['dom0'].good_to_go: - backup_path = restore_info['dom0'].subdir - local_user = grp.getgrnam('qubes').gr_mem[0] - home_dir = pwd.getpwnam(local_user).pw_dir - if self.header_data.version == 1: - backup_dom0_home_dir = os.path.join(self.backup_location, - backup_path) - else: - backup_dom0_home_dir = os.path.join(self.tmpdir, backup_path) - restore_home_backupdir = "home-pre-restore-{0}".format( - time.strftime("%Y-%m-%d-%H%M%S")) - - self.log.info( - "Restoring home of user '{0}'...".format(local_user)) - self.log.info( - "Existing files/dirs backed up in '{0}' dir".format( - restore_home_backupdir)) - os.mkdir(home_dir + '/' + restore_home_backupdir) - for f in os.listdir(backup_dom0_home_dir): - home_file = home_dir + '/' + f - if os.path.exists(home_file): - os.rename(home_file, - home_dir + '/' + restore_home_backupdir + '/' + f) - if self.header_data.version == 1: - subprocess.call( - ["cp", "-nrp", "--reflink=auto", - backup_dom0_home_dir + '/' + f, home_file]) - elif self.header_data.version >= 2: - shutil.move(backup_dom0_home_dir + '/' + f, home_file) - retcode = subprocess.call(['sudo', 'chown', '-R', - local_user, home_dir]) - if retcode != 0: - self.log.error("*** Error while setting home directory owner") - - shutil.rmtree(self.tmpdir) - self.log.info("-> Done. Please install updates for all the restored " - "templates.") - - def _restore_vms_metadata(self, restore_info): - vms = {} - for vm_info in restore_info.values(): - assert isinstance(vm_info, self.VMToRestore) - if not vm_info.vm: - continue - if not vm_info.good_to_go: - continue - vm = vm_info.vm - vms[vm.name] = vm - - # First load templates, then other VMs - for vm in self._templates_first(vms.values()): - if self.canceled: - # only break the loop to save qubes.xml - # with already restored VMs - break - self.log.info("-> Restoring {0}...".format(vm.name)) - kwargs = {} - if hasattr(vm, 'template'): - template = restore_info[vm.name].template - # handle potentially renamed template - if template in restore_info \ - and restore_info[template].good_to_go: - template = restore_info[template].name - kwargs['template'] = template - - new_vm = None - vm_name = restore_info[vm.name].name - - try: - # first only minimal set, later clone_properties - # will be called - cls = self.app.get_vm_class(vm.__class__.__name__) - new_vm = self.app.add_new_vm( - cls, - name=vm_name, - label=vm.label, - installed_by_rpm=False, - **kwargs) - if os.path.exists(new_vm.dir_path): - move_to_path = tempfile.mkdtemp('', os.path.basename( - new_vm.dir_path), os.path.dirname(new_vm.dir_path)) - try: - os.rename(new_vm.dir_path, move_to_path) - self.log.warning( - "*** Directory {} already exists! It has " - "been moved to {}".format(new_vm.dir_path, - move_to_path)) - except OSError: - self.log.error( - "*** Directory {} already exists and " - "cannot be moved!".format(new_vm.dir_path)) - self.log.warning("Skipping VM {}...".format( - vm.name)) - continue - except Exception as err: - self.log.error("ERROR: {0}".format(err)) - self.log.warning("*** Skipping VM: {0}".format(vm.name)) - if new_vm: - del self.app.domains[new_vm.qid] - continue - - # remove no longer needed backup metadata - if 'backup-content' in vm.features: - del vm.features['backup-content'] - del vm.features['backup-size'] - del vm.features['backup-path'] - try: - # exclude VM references - handled manually according to - # restore options - proplist = [prop for prop in new_vm.property_list() - if prop.clone and prop.__name__ not in - ['template', 'netvm', 'dispvm_netvm']] - new_vm.clone_properties(vm, proplist=proplist) - except Exception as err: - self.log.error("ERROR: {0}".format(err)) - self.log.warning("*** Some VM property will not be " - "restored") - - if not self.options.verify_only: - try: - # have it here, to (maybe) patch storage config before - # creating child VMs (template first) - # TODO: adjust volumes config - especially size - new_vm.create_on_disk(pool=self.options.override_pool) - except qubes.exc.QubesException as e: - self.log.warning("Failed to create VM {}: {}".format( - vm.name, str(e))) - del self.app.domains[new_vm] - continue - - restore_info[vm.name].restored_vm = new_vm - - # Set network dependencies - only non-default netvm setting - for vm in vms.values(): - vm_info = restore_info[vm.name] - vm_name = vm_info.name - try: - host_vm = self.app.domains[vm_name] - except KeyError: - # Failed/skipped VM - continue - - if not vm.property_is_default('netvm'): - if vm_info.netvm in restore_info: - host_vm.netvm = restore_info[vm_info.netvm].name - else: - host_vm.netvm = vm_info.netvm - # vim:sw=4:et: diff --git a/qubes/tools/qvm_backup_restore.py b/qubes/tools/qvm_backup_restore.py deleted file mode 100644 index d8cc87f0..00000000 --- a/qubes/tools/qvm_backup_restore.py +++ /dev/null @@ -1,258 +0,0 @@ -# -# The Qubes OS Project, http://www.qubes-os.org -# -# Copyright (C) 2016 Marek Marczykowski-Górecki -# -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -from __future__ import print_function - -import getpass -import locale -import sys - -import qubes.backup -import qubes.tools -import qubes.utils - -parser = qubes.tools.QubesArgumentParser(want_force_root=True) - -parser.add_argument("--verify-only", action="store_true", - dest="verify_only", default=False, - help="Verify backup integrity without restoring any " - "data") - -parser.add_argument("--skip-broken", action="store_true", dest="skip_broken", - default=False, - help="Do not restore VMs that have missing TemplateVMs " - "or NetVMs") - -parser.add_argument("--ignore-missing", action="store_true", - dest="ignore_missing", default=False, - help="Restore VMs even if their associated TemplateVMs " - "and NetVMs are missing") - -parser.add_argument("--skip-conflicting", action="store_true", - dest="skip_conflicting", default=False, - help="Do not restore VMs that are already present on " - "the host") - -parser.add_argument("--rename-conflicting", action="store_true", - dest="rename_conflicting", default=False, - help="Restore VMs that are already present on the host " - "under different names") - -parser.add_argument("--replace-template", action="append", - dest="replace_template", default=[], - help="Restore VMs using another TemplateVM; syntax: " - "old-template-name:new-template-name (may be " - "repeated)") - -parser.add_argument("-x", "--exclude", action="append", dest="exclude", - default=[], - help="Skip restore of specified VM (may be repeated)") - -parser.add_argument("--skip-dom0-home", action="store_false", dest="dom0_home", - default=True, - help="Do not restore dom0 user home directory") - -parser.add_argument("--ignore-username-mismatch", action="store_true", - dest="ignore_username_mismatch", default=False, - help="Ignore dom0 username mismatch when restoring home " - "directory") - -parser.add_argument("-d", "--dest-vm", action="store", dest="appvm", - help="Specify VM containing the backup to be restored") - -parser.add_argument("-p", "--passphrase-file", action="store", - dest="pass_file", default=None, - help="Read passphrase from file, or use '-' to read from stdin") - -parser.add_argument('backup_location', action='store', - help="Backup directory name, or command to pipe from") - -parser.add_argument('vms', nargs='*', action='store', default='[]', - help='Restore only those VMs') - - -def handle_broken(app, args, restore_info): - there_are_conflicting_vms = False - there_are_missing_templates = False - there_are_missing_netvms = False - dom0_username_mismatch = False - - for vm_info in restore_info.values(): - assert isinstance(vm_info, qubes.backup.BackupRestore.VMToRestore) - if qubes.backup.BackupRestore.VMToRestore.EXCLUDED in vm_info.problems: - continue - if qubes.backup.BackupRestore.VMToRestore.MISSING_TEMPLATE in \ - vm_info.problems: - there_are_missing_templates = True - if qubes.backup.BackupRestore.VMToRestore.MISSING_NETVM in \ - vm_info.problems: - there_are_missing_netvms = True - if qubes.backup.BackupRestore.VMToRestore.ALREADY_EXISTS in \ - vm_info.problems: - there_are_conflicting_vms = True - if qubes.backup.BackupRestore.Dom0ToRestore.USERNAME_MISMATCH in \ - vm_info.problems: - dom0_username_mismatch = True - - - if there_are_conflicting_vms: - app.log.error( - "*** There are VMs with conflicting names on the host! ***") - if args.skip_conflicting: - app.log.error( - "Those VMs will not be restored. " - "The host VMs will NOT be overwritten.") - else: - raise qubes.exc.QubesException( - "Remove VMs with conflicting names from the host " - "before proceeding.\n" - "Or use --skip-conflicting to restore only those VMs that " - "do not exist on the host.\n" - "Or use --rename-conflicting to restore those VMs under " - "modified names (with numbers at the end).") - - app.log.info("The above VMs will be copied and added to your system.") - app.log.info("Exisiting VMs will NOT be removed.") - - if there_are_missing_templates: - app.log.warning("*** One or more TemplateVMs are missing on the " - "host! ***") - if not (args.skip_broken or args.ignore_missing): - raise qubes.exc.QubesException( - "Install them before proceeding with the restore." - "Or pass: --skip-broken or --ignore-missing.") - elif args.skip_broken: - app.log.warning("Skipping broken entries: VMs that depend on " - "missing TemplateVMs will NOT be restored.") - elif args.ignore_missing: - app.log.warning("Ignoring missing entries: VMs that depend " - "on missing TemplateVMs will NOT be restored.") - else: - raise qubes.exc.QubesException( - "INTERNAL ERROR! Please report this to the Qubes OS team!") - - if there_are_missing_netvms: - app.log.warning("*** One or more NetVMs are missing on the " - "host! ***") - if not (args.skip_broken or args.ignore_missing): - raise qubes.exc.QubesException( - "Install them before proceeding with the restore." - "Or pass: --skip-broken or --ignore-missing.") - elif args.skip_broken: - app.log.warning("Skipping broken entries: VMs that depend on " - "missing NetVMs will NOT be restored.") - elif args.ignore_missing: - app.log.warning("Ignoring missing entries: VMs that depend " - "on missing NetVMs will NOT be restored.") - else: - raise qubes.exc.QubesException( - "INTERNAL ERROR! Please report this to the Qubes OS team!") - - if 'dom0' in restore_info.keys() and args.dom0_home: - if dom0_username_mismatch: - app.log.warning("*** Dom0 username mismatch! This can break " - "some settings! ***") - if not args.ignore_username_mismatch: - raise qubes.exc.QubesException( - "Skip restoring the dom0 home directory " - "(--skip-dom0-home), or pass " - "--ignore-username-mismatch to continue anyway.") - else: - app.log.warning("Continuing as directed.") - app.log.warning("NOTE: Before restoring the dom0 home directory, " - "a new directory named " - "'home-pre-restore-' will be " - "created inside the dom0 home directory. If any " - "restored files conflict with existing files, " - "the existing files will be moved to this new " - "directory.") - -def main(args=None): - # pylint: disable=too-many-return-statements - args = parser.parse_args(args) - - appvm = None - if args.appvm: - try: - appvm = args.app.domains[args.appvm] - except KeyError: - parser.error('no such domain: {!r}'.format(args.appvm)) - - if args.pass_file is not None: - pass_f = open(args.pass_file) if args.pass_file != "-" else sys.stdin - passphrase = pass_f.readline().rstrip() - if pass_f is not sys.stdin: - pass_f.close() - else: - passphrase = getpass.getpass("Please enter the passphrase to verify " - "and (if encrypted) decrypt the backup: ") - - encoding = sys.stdin.encoding or locale.getpreferredencoding() - passphrase = passphrase.decode(encoding) - - args.app.log.info("Checking backup content...") - - try: - backup = qubes.backup.BackupRestore(args.app, args.backup_location, - appvm, passphrase) - except qubes.exc.QubesException as e: - parser.error_runtime(str(e)) - # unreachable - error_runtime will raise SystemExit - return 1 - - if args.ignore_missing: - backup.options.use_default_template = True - backup.options.use_default_netvm = True - if args.replace_template: - backup.options.replace_template = args.replace_template - if args.rename_conflicting: - backup.options.rename_conflicting = True - if not args.dom0_home: - backup.options.dom0_home = False - if args.ignore_username_mismatch: - backup.options.ignore_username_mismatch = True - if args.exclude: - backup.options.exclude = args.exclude - if args.verify_only: - backup.options.verify_only = True - - restore_info = None - try: - restore_info = backup.get_restore_info() - except qubes.exc.QubesException as e: - parser.error_runtime(str(e)) - - print(backup.get_restore_summary(restore_info)) - - try: - handle_broken(args.app, args, restore_info) - except qubes.exc.QubesException as e: - parser.error_runtime(str(e)) - - if args.pass_file is None: - if input("Do you want to proceed? [y/N] ").upper() != "Y": - exit(0) - - try: - backup.restore_do(restore_info) - except qubes.exc.QubesException as e: - parser.error_runtime(str(e)) - -if __name__ == '__main__': - main() diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec index fa6ec560..ef7fff15 100644 --- a/rpm_spec/core-dom0.spec +++ b/rpm_spec/core-dom0.spec @@ -278,7 +278,6 @@ fi %{python3_sitelib}/qubes/tools/qubesd.py %{python3_sitelib}/qubes/tools/qubesd_query.py %{python3_sitelib}/qubes/tools/qvm_backup.py -%{python3_sitelib}/qubes/tools/qvm_backup_restore.py %dir %{python3_sitelib}/qubes/ext %dir %{python3_sitelib}/qubes/ext/__pycache__