diff --git a/qubes/backup.py b/qubes/backup.py index a4c79ea6..e5d5619d 100644 --- a/qubes/backup.py +++ b/qubes/backup.py @@ -22,8 +22,8 @@ from __future__ import unicode_literals import itertools import logging - import functools +import termios from qubes.utils import size_to_human import sys @@ -51,7 +51,9 @@ QUEUE_FINISHED = "FINISHED" HEADER_FILENAME = 'backup-header' DEFAULT_CRYPTO_ALGORITHM = 'aes-256-cbc' -DEFAULT_HMAC_ALGORITHM = 'SHA512' +# 'scrypt' is not exactly HMAC algorithm, but a tool we use to +# integrity-protect the data +DEFAULT_HMAC_ALGORITHM = 'scrypt' DEFAULT_COMPRESSION_FILTER = 'gzip' CURRENT_BACKUP_FORMAT_VERSION = '4' # Maximum size of error message get from process stderr (including VM process) @@ -219,6 +221,63 @@ class SendWorker(Process): self.log.debug("Finished sending thread") +def launch_proc_with_pty(args, stdin=None, stdout=None, stderr=None, echo=True): + """Similar to pty.fork, but handle stdin/stdout according to parameters + instead of connecting to the pty + + :return tuple (subprocess.Popen, pty_master) + """ + + def set_ctty(ctty_fd, master_fd): + os.setsid() + os.close(master_fd) + fcntl.ioctl(ctty_fd, termios.TIOCSCTTY, 0) + if not echo: + termios_p = termios.tcgetattr(ctty_fd) + # termios_p.c_lflags + termios_p[3] &= ~termios.ECHO + termios.tcsetattr(ctty_fd, termios.TCSANOW, termios_p) + (pty_master, pty_slave) = os.openpty() + p = subprocess.Popen(args, stdin=stdin, stdout=stdout, stderr=stderr, + preexec_fn=lambda: set_ctty(pty_slave, pty_master)) + os.close(pty_slave) + return p, os.fdopen(pty_master, 'w+') + + +def launch_scrypt(action, input_name, output_name, passphrase): + ''' + Launch 'scrypt' process, pass passphrase to it and return + subprocess.Popen object. + + :param action: 'enc' or 'dec' + :param input_name: input path or '-' for stdin + :param output_name: output path or '-' for stdout + :param passphrase: passphrase + :return: subprocess.Popen object + ''' + command_line = ['scrypt', action, input_name, output_name] + (p, pty) = launch_proc_with_pty(command_line, + stdin=subprocess.PIPE if input_name == '-' else None, + stdout=subprocess.PIPE if output_name == '-' else None, + stderr=subprocess.PIPE, + echo=False) + if action == 'enc': + prompts = ('Please enter passphrase: ', 'Please confirm passphrase: ') + else: + prompts = ('Please enter passphrase: ',) + for prompt in prompts: + actual_prompt = p.stderr.read(len(prompt)) + if actual_prompt != prompt: + raise qubes.exc.QubesException( + 'Unexpected prompt from scrypt: {}'.format(actual_prompt)) + pty.write(passphrase.encode('utf-8') + b'\n') + pty.flush() + # save it here, so garbage collector would not close it (which would kill + # the child) + p.pty = pty + return p + + class Backup(object): class FileToBackup(object): def __init__(self, file_path, subdir=None, name=None): @@ -489,15 +548,18 @@ class Backup(object): backup_id=self.backup_id, ) backup_header.save(header_file_path) + # Start encrypt, scrypt will also handle integrity + # protection + scrypt_passphrase = u'{filename}!{passphrase}'.format( + filename=HEADER_FILENAME, passphrase=self.passphrase) + scrypt = launch_scrypt( + 'enc', header_file_path, header_file_path + '.hmac', + scrypt_passphrase) - hmac = subprocess.Popen( - ["openssl", "dgst", "-" + self.hmac_algorithm, - "-hmac", self.passphrase], - stdin=open(header_file_path, "r"), - stdout=open(header_file_path + ".hmac", "w")) - if hmac.wait() != 0: + if scrypt.wait() != 0: raise qubes.exc.QubesException( - "Failed to compute hmac of header file") + "Failed to compute hmac of header file: " + + scrypt.stderr.read()) return HEADER_FILENAME, HEADER_FILENAME + ".hmac" @@ -651,60 +713,43 @@ class Backup(object): self.log.debug(" ".join(tar_cmdline)) - # Tips: Popen(bufsize=0) - # Pipe: tar-sparse | encryptor [| hmac] | tar | backup_target - # Pipe: tar-sparse [| hmac] | tar | backup_target + # Pipe: tar-sparse | scrypt | tar | backup_target # TODO: log handle stderr tar_sparse = subprocess.Popen( - tar_cmdline, stdin=subprocess.PIPE) + tar_cmdline) self.processes_to_kill_on_cancel.append(tar_sparse) # Wait for compressor (tar) process to finish or for any # error of other subprocesses i = 0 + pipe = open(backup_pipe, 'rb') run_error = "paused" - encryptor = None - if self.encrypted: - # Start encrypt - # If no cipher is provided, - # the data is forwarded unencrypted !!! - encryptor = subprocess.Popen([ - "openssl", "enc", - "-e", "-" + self.crypto_algorithm, - "-pass", "pass:" + passphrase], - stdin=open(backup_pipe, 'rb'), - stdout=subprocess.PIPE) - pipe = encryptor.stdout - else: - pipe = open(backup_pipe, 'rb') while run_error == "paused": - - # Start HMAC - hmac = subprocess.Popen([ - "openssl", "dgst", "-" + self.hmac_algorithm, - "-hmac", passphrase], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE) - # Prepare a first chunk - chunkfile = backup_tempfile + "." + "%03d" % i + chunkfile = backup_tempfile + ".%03d.enc" % i i += 1 - chunkfile_p = open(chunkfile, 'wb') + + # Start encrypt, scrypt will also handle integrity + # protection + scrypt_passphrase = \ + u'{backup_id}!{filename}!{passphrase}'.format( + backup_id=self.backup_id, + filename=os.path.relpath(chunkfile[:-4], + self.tmpdir), + passphrase=self.passphrase) + scrypt = launch_scrypt( + "enc", "-", chunkfile, scrypt_passphrase) run_error = handle_streams( pipe, - {'hmac_data': hmac.stdin, - 'backup_target': chunkfile_p, - }, - {'hmac': hmac, - 'vmproc': vmproc, + {'backup_target': scrypt.stdin}, + {'vmproc': vmproc, 'addproc': tar_sparse, - 'streamproc': encryptor, + 'scrypt': scrypt, }, self.chunk_size, self._add_vm_progress ) - chunkfile_p.close() self.log.debug( "12 returned: {}".format(run_error)) @@ -714,12 +759,7 @@ class Backup(object): tar_sparse.terminate() except OSError: pass - try: - hmac.terminate() - except OSError: - pass tar_sparse.wait() - hmac.wait() to_send.put(QUEUE_ERROR) send_proc.join() shutil.rmtree(self.tmpdir) @@ -735,29 +775,16 @@ class Backup(object): "Failed to perform backup: error in " + run_error) + scrypt.stdin.close() + scrypt.wait() + self.log.debug("scrypt return code: {}".format( + scrypt.poll())) + # Send the chunk to the backup target self._queue_put_with_check( send_proc, vmproc, to_send, os.path.relpath(chunkfile, self.tmpdir)) - # Close HMAC - hmac.stdin.close() - hmac.wait() - self.log.debug("HMAC proc return code: {}".format( - hmac.poll())) - - # Write HMAC data next to the chunk file - hmac_data = hmac.stdout.read() - self.log.debug( - "Writing hmac to {}.hmac".format(chunkfile)) - with open(chunkfile + ".hmac", 'w') as hmac_file: - hmac_file.write(hmac_data) - - # Send the HMAC to the backup target - self._queue_put_with_check( - send_proc, vmproc, to_send, - os.path.relpath(chunkfile, self.tmpdir) + ".hmac") - if tar_sparse.poll() is None or run_error == "size_limit": run_error = "paused" else: @@ -1333,6 +1360,8 @@ def get_supported_hmac_algo(hmac_algorithm=None): # Start with provided default if hmac_algorithm: yield hmac_algorithm + if hmac_algorithm != 'scrypt': + yield 'scrypt' proc = subprocess.Popen(['openssl', 'list-message-digest-algorithms'], stdout=subprocess.PIPE) for algo in proc.stdout.readlines(): @@ -1552,6 +1581,10 @@ class BackupRestore(object): def _verify_hmac(self, filename, hmacfile, algorithm=None): def load_hmac(hmac_text): + if filter(lambda x: ord(x) not in range(128), + hmac_text): + raise qubes.exc.QubesException( + "Invalid content of {}".format(hmacfile)) hmac_text = hmac_text.strip().split("=") if len(hmac_text) > 1: hmac_text = hmac_text[1].strip() @@ -1570,6 +1603,17 @@ class BackupRestore(object): "ERROR: expected hmac for {}, but got {}". format(filename, hmacfile)) + if algorithm == 'scrypt': + # in case of 'scrypt' _verify_hmac is only used for backup header + assert filename == HEADER_FILENAME + self._verify_and_decrypt(hmacfile, HEADER_FILENAME + '.dec') + if open(os.path.join(self.tmpdir, filename)).read() != \ + open(os.path.join(self.tmpdir, filename + '.dec')).read(): + raise qubes.exc.QubesException( + 'Invalid hmac on {}'.format(filename)) + else: + return True + hmac_proc = subprocess.Popen( ["openssl", "dgst", "-" + algorithm, "-hmac", passphrase], stdin=open(os.path.join(self.tmpdir, filename), 'rb'), @@ -1595,6 +1639,80 @@ class BackupRestore(object): "Is the passphrase correct?". format(filename, load_hmac(hmac_stdout.decode('ascii')))) + def _verify_and_decrypt(self, filename, output=None): + assert filename.endswith('.enc') or filename.endswith('.hmac') + fullname = os.path.join(self.tmpdir, filename) + (origname, _) = os.path.splitext(filename) + if output: + fulloutput = os.path.join(self.tmpdir, output) + else: + fulloutput = os.path.join(self.tmpdir, origname) + if origname == HEADER_FILENAME: + passphrase = u'{filename}!{passphrase}'.format( + filename=origname, + passphrase=self.passphrase) + else: + passphrase = u'{backup_id}!{filename}!{passphrase}'.format( + backup_id=self.header_data.backup_id, + filename=origname, + passphrase=self.passphrase) + p = launch_scrypt('dec', fullname, fulloutput, passphrase) + (_, stderr) = p.communicate() + if p.returncode != 0: + os.unlink(fulloutput) + raise qubes.exc.QubesException('failed to decrypt {}: {}'.format( + fullname, stderr)) + # encrypted file is no longer needed + os.unlink(fullname) + return origname + + def _retrieve_backup_header_files(self, files, allow_none=False): + (retrieve_proc, filelist_pipe, error_pipe) = \ + self._start_retrieval_process( + files, len(files), 1024 * 1024) + filelist = filelist_pipe.read() + retrieve_proc_returncode = retrieve_proc.wait() + if retrieve_proc in self.processes_to_kill_on_cancel: + self.processes_to_kill_on_cancel.remove(retrieve_proc) + extract_stderr = error_pipe.read(MAX_STDERR_BYTES) + + # wait for other processes (if any) + for proc in self.processes_to_kill_on_cancel: + if proc.wait() != 0: + raise qubes.exc.QubesException( + "Backup header retrieval failed (exit code {})".format( + proc.wait()) + ) + + if retrieve_proc_returncode != 0: + if not filelist and 'Not found in archive' in extract_stderr: + if allow_none: + return None + else: + raise qubes.exc.QubesException( + "unable to read the qubes backup file {0} ({1}): {2}".format( + self.backup_location, + retrieve_proc.wait(), + extract_stderr + )) + actual_files = filelist.splitlines() + if sorted(actual_files) != sorted(files): + raise qubes.exc.QubesException( + 'unexpected files in archive: got {!r}, expeced {!r}'.format( + actual_files, files + )) + for f in files: + if not os.path.exists(os.path.join(self.tmpdir, f)): + if allow_none: + return None + else: + raise qubes.exc.QubesException( + 'Unable to retrieve file {} from backup {}: {}'.format( + f, self.backup_location, extract_stderr + ) + ) + return files + def _retrieve_backup_header(self): """Retrieve backup header and qubes.xml. Only backup header is analyzed, qubes.xml is left as-is @@ -1611,82 +1729,47 @@ class BackupRestore(object): header_data.version = 1 return header_data - (retrieve_proc, filelist_pipe, error_pipe) = \ - self._start_retrieval_process( - ['backup-header', 'backup-header.hmac', - 'qubes.xml.000', 'qubes.xml.000.hmac'], 4, 1024 * 1024) + header_files = self._retrieve_backup_header_files( + ['backup-header', 'backup-header.hmac'], allow_none=True) - expect_tar_error = False - - filename = filelist_pipe.readline().strip().decode('ascii') - hmacfile = filelist_pipe.readline().strip().decode('ascii') - # tar output filename before actually extracting it, so wait for the - # next one before trying to access it - if not self.backup_vm: - filelist_pipe.readline().strip() - - self.log.debug("Got backup header and hmac: {}, {}".format( - filename, hmacfile)) - - if not filename or filename == "EOF" or \ - not hmacfile or hmacfile == "EOF": - retrieve_proc.wait() - proc_error_msg = error_pipe.read(MAX_STDERR_BYTES) - raise qubes.exc.QubesException( - "Premature end of archive while receiving " - "backup header. Process output:\n" + proc_error_msg) - file_ok = False - hmac_algorithm = DEFAULT_HMAC_ALGORITHM - for hmac_algo in get_supported_hmac_algo(hmac_algorithm): - try: - if self._verify_hmac(filename, hmacfile, hmac_algo): - file_ok = True - hmac_algorithm = hmac_algo - break - except qubes.exc.QubesException: - # Ignore exception here, try the next algo - pass - if not file_ok: - raise qubes.exc.QubesException( - "Corrupted backup header (hmac verification " - "failed). Is the password correct?") - if os.path.basename(filename) == HEADER_FILENAME: - filename = os.path.join(self.tmpdir, filename) - header_data = BackupHeader(open(filename, 'rb').read()) - os.unlink(filename) - else: - # if no header found, create one with guessed HMAC algo + if not header_files: + # R2-Beta3 didn't have backup header, so if none is found, + # assume it's version=2 and use values present at that time header_data = BackupHeader( version=2, - hmac_algorithm=hmac_algorithm, # place explicitly this value, because it is what format_version # 2 have + hmac_algorithm='SHA1', crypto_algorithm='aes-256-cbc', # TODO: set encrypted to something... ) - # when tar do not find expected file in archive, it exit with - # code 2. This will happen because we've requested backup-header - # file, but the archive do not contain it. Ignore this particular - # error. - if not self.backup_vm: - expect_tar_error = True + else: + filename = HEADER_FILENAME + hmacfile = HEADER_FILENAME + '.hmac' + self.log.debug("Got backup header and hmac: {}, {}".format( + filename, hmacfile)) - if retrieve_proc.wait() != 0 and not expect_tar_error: - raise qubes.exc.QubesException( - "unable to read the qubes backup file {0} ({1}): {2}".format( - self.backup_location, - retrieve_proc.wait(), - error_pipe.read(MAX_STDERR_BYTES) - )) - if retrieve_proc in self.processes_to_kill_on_cancel: - self.processes_to_kill_on_cancel.remove(retrieve_proc) - # wait for other processes (if any) - for proc in self.processes_to_kill_on_cancel: - if proc.wait() != 0: + file_ok = False + hmac_algorithm = DEFAULT_HMAC_ALGORITHM + for hmac_algo in get_supported_hmac_algo(hmac_algorithm): + try: + if self._verify_hmac(filename, hmacfile, hmac_algo): + file_ok = True + break + except qubes.exc.QubesException as e: + self.log.debug( + 'Failed to verify {} using {}: {}'.format( + hmacfile, hmac_algo, str(e))) + # Ignore exception here, try the next algo + pass + if not file_ok: raise qubes.exc.QubesException( - "Backup header retrieval failed (exit code {})".format( - proc.wait()) - ) + "Corrupted backup header (hmac verification " + "failed). Is the password correct?") + filename = os.path.join(self.tmpdir, filename) + header_data = BackupHeader(open(filename, 'r').read()) + os.unlink(filename) + return header_data def _start_inner_extraction_worker(self, queue, relocate): @@ -1719,6 +1802,9 @@ class BackupRestore(object): elif format_version in [3, 4]: extractor_params['compression_filter'] = \ self.header_data.compression_filter + if format_version == 4: + # encryption already handled + extractor_params['encrypted'] = False extract_proc = ExtractWorker3(**extractor_params) else: raise NotImplementedError( @@ -1737,7 +1823,14 @@ class BackupRestore(object): offline_mode=True) return backup_app else: - self._verify_hmac("qubes.xml.000", "qubes.xml.000.hmac") + if self.header_data.version in [2, 3]: + self._retrieve_backup_header_files( + ['qubes.xml.000', 'qubes.xml.000.hmac']) + self._verify_hmac("qubes.xml.000", "qubes.xml.000.hmac") + else: + self._retrieve_backup_header_files(['qubes.xml.000.enc']) + self._verify_and_decrypt('qubes.xml.000.enc') + queue = Queue() queue.put("qubes.xml.000") queue.put(QUEUE_FINISHED) @@ -1785,6 +1878,7 @@ class BackupRestore(object): try: filename = None + hmacfile = None nextfile = None while True: if self.canceled: @@ -1808,30 +1902,58 @@ class BackupRestore(object): if not filename or filename == "EOF": break - hmacfile = filelist_pipe.readline().strip() - - if self.canceled: - break # if reading archive directly with tar, wait for next filename - # tar prints filename before processing it, so wait for # the next one to be sure that whole file was extracted if not self.backup_vm: nextfile = filelist_pipe.readline().strip() - self.log.debug("Getting hmac:" + hmacfile) - if not hmacfile or hmacfile == "EOF": - # Premature end of archive, either of tar1_command or - # vmproc exited with error - break + if self.header_data.version in [2, 3]: + if not self.backup_vm: + hmacfile = nextfile + nextfile = filelist_pipe.readline().strip() + else: + hmacfile = filelist_pipe.readline().strip() + + if self.canceled: + break + + self.log.debug("Getting hmac:" + hmacfile) + if not hmacfile or hmacfile == "EOF": + # Premature end of archive, either of tar1_command or + # vmproc exited with error + break + else: # self.header_data.version == 4 + if not filename.endswith('.enc'): + raise qubes.exc.QubesException( + 'Invalid file extension found in archive: {}'. + format(filename)) if not any(map(lambda x: filename.startswith(x), vms_dirs)): self.log.debug("Ignoring VM not selected for restore") os.unlink(os.path.join(self.tmpdir, filename)) - os.unlink(os.path.join(self.tmpdir, hmacfile)) + if hmacfile: + os.unlink(os.path.join(self.tmpdir, hmacfile)) continue - if self._verify_hmac(filename, hmacfile): - to_extract.put(os.path.join(self.tmpdir, filename)) + if self.header_data.version in [2, 3]: + self._verify_hmac(filename, hmacfile) + else: + # _verify_and_decrypt will write output to a file with + # '.enc' extension cut off. This is safe because: + # - `scrypt` tool will override output, so if the file was + # already there (received from the VM), it will be removed + # - incoming archive extraction will refuse to override + # existing file, so if `scrypt` already created one, + # it can not be manipulated by the VM + # - when the file is retrieved from the VM, it appears at + # the final form - if it's visible, VM have no longer + # influence over its content + # + # This all means that if the file was correctly verified + # + decrypted, we will surely access the right file + filename = self._verify_and_decrypt(filename) + to_extract.put(os.path.join(self.tmpdir, filename)) if self.canceled: raise BackupCanceledError("Restore canceled", diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec index abece061..1cf8ef79 100644 --- a/rpm_spec/core-dom0.spec +++ b/rpm_spec/core-dom0.spec @@ -86,6 +86,7 @@ Requires: gnome-packagekit Requires: cronie Requires: bsdtar Requires: python3-jinja2 +Requires: scrypt # for qubes-hcl-report Requires: dmidecode Requires: PyQt4