Merge remote-tracking branch 'marmarek/core3-backup2' into core3-devel

qubes/backup.py

@@ -24,6 +24,8 @@
 from __future__ import unicode_literals
 import itertools
 import logging
+import termios
+
 from qubes.utils import size_to_human
 import sys
 import stat
@@ -50,7 +52,9 @@ QUEUE_FINISHED = "FINISHED"
 
 HEADER_FILENAME = 'backup-header'
 DEFAULT_CRYPTO_ALGORITHM = 'aes-256-cbc'
-DEFAULT_HMAC_ALGORITHM = 'SHA512'
+# 'scrypt' is not exactly HMAC algorithm, but a tool we use to
+# integrity-protect the data
+DEFAULT_HMAC_ALGORITHM = 'scrypt'
 DEFAULT_COMPRESSION_FILTER = 'gzip'
 CURRENT_BACKUP_FORMAT_VERSION = '4'
 # Maximum size of error message get from process stderr (including VM process)
@@ -76,6 +80,7 @@ class BackupHeader(object):
         'compression-filter': 'compression_filter',
         'crypto-algorithm': 'crypto_algorithm',
         'hmac-algorithm': 'hmac_algorithm',
+        'backup-id': 'backup_id'
     }
     bool_options = ['encrypted', 'compressed']
     int_options = ['version']
@@ -87,7 +92,8 @@ class BackupHeader(object):
             compressed=None,
             compression_filter=None,
             hmac_algorithm=None,
-            crypto_algorithm=None):
+            crypto_algorithm=None,
+            backup_id=None):
         # repeat the list to help code completion...
         self.version = version
         self.encrypted = encrypted
@@ -97,6 +103,7 @@ class BackupHeader(object):
         self.compression_filter = compression_filter
         self.hmac_algorithm = hmac_algorithm
         self.crypto_algorithm = crypto_algorithm
+        self.backup_id = backup_id
 
         if header_data is not None:
             self.load(header_data)
@@ -148,6 +155,8 @@ class BackupHeader(object):
                 expected_attrs += ['crypto_algorithm']
             if self.version >= 3 and self.compressed:
                 expected_attrs += ['compression_filter']
+            if self.version >= 4:
+                expected_attrs += ['backup_id']
             for key in expected_attrs:
                 if getattr(self, key) is None:
                     raise qubes.exc.QubesException(
@@ -213,6 +222,63 @@ class SendWorker(Process):
         self.log.debug("Finished sending thread")
 
 
+def launch_proc_with_pty(args, stdin=None, stdout=None, stderr=None, echo=True):
+    """Similar to pty.fork, but handle stdin/stdout according to parameters
+    instead of connecting to the pty
+
+    :return tuple (subprocess.Popen, pty_master)
+    """
+
+    def set_ctty(ctty_fd, master_fd):
+        os.setsid()
+        os.close(master_fd)
+        fcntl.ioctl(ctty_fd, termios.TIOCSCTTY, 0)
+        if not echo:
+            termios_p = termios.tcgetattr(ctty_fd)
+            # termios_p.c_lflags
+            termios_p[3] &= ~termios.ECHO
+            termios.tcsetattr(ctty_fd, termios.TCSANOW, termios_p)
+    (pty_master, pty_slave) = os.openpty()
+    p = subprocess.Popen(args, stdin=stdin, stdout=stdout, stderr=stderr,
+        preexec_fn=lambda: set_ctty(pty_slave, pty_master))
+    os.close(pty_slave)
+    return p, os.fdopen(pty_master, 'w+')
+
+
+def launch_scrypt(action, input_name, output_name, passphrase):
+    '''
+    Launch 'scrypt' process, pass passphrase to it and return
+    subprocess.Popen object.
+
+    :param action: 'enc' or 'dec'
+    :param input_name: input path or '-' for stdin
+    :param output_name: output path or '-' for stdout
+    :param passphrase: passphrase
+    :return: subprocess.Popen object
+    '''
+    command_line = ['scrypt', action, input_name, output_name]
+    (p, pty) = launch_proc_with_pty(command_line,
+        stdin=subprocess.PIPE if input_name == '-' else None,
+        stdout=subprocess.PIPE if output_name == '-' else None,
+        stderr=subprocess.PIPE,
+        echo=False)
+    if action == 'enc':
+        prompts = ('Please enter passphrase: ', 'Please confirm passphrase: ')
+    else:
+        prompts = ('Please enter passphrase: ',)
+    for prompt in prompts:
+        actual_prompt = p.stderr.read(len(prompt))
+        if actual_prompt != prompt:
+            raise qubes.exc.QubesException(
+                'Unexpected prompt from scrypt: {}'.format(actual_prompt))
+        pty.write(passphrase.encode('utf-8') + b'\n')
+        pty.flush()
+    # save it here, so garbage collector would not close it (which would kill
+    #  the child)
+    p.pty = pty
+    return p
+
+
 class Backup(object):
     class FileToBackup(object):
         def __init__(self, file_path, subdir=None, name=None):
@@ -292,6 +358,10 @@ class Backup(object):
         #: callback for progress reporting. Will be called with one argument
         #: - progress in percents
         self.progress_callback = None
+        #: backup ID, needs to be unique (for a given user),
+        #: not necessary unpredictable; automatically generated
+        self.backup_id = datetime.datetime.now().strftime(
+            '%Y%m%dT%H%M%S-' + str(os.getpid()))
 
         for key, value in kwargs.iteritems():
             if hasattr(self, key):
@@ -306,7 +376,9 @@ class Backup(object):
 
         self.log = logging.getLogger('qubes.backup')
 
-        self.compression_filter = DEFAULT_COMPRESSION_FILTER
+        if not self.encrypted:
+            self.log.warning('\'encrypted\' option is ignored, backup is '
+                             'always encrypted')
 
         if exclude_list is None:
             exclude_list = []
@@ -474,17 +546,21 @@ class Backup(object):
             encrypted=self.encrypted,
             compressed=self.compressed,
             compression_filter=self.compression_filter,
+            backup_id=self.backup_id,
         )
         backup_header.save(header_file_path)
+        # Start encrypt, scrypt will also handle integrity
+        # protection
+        scrypt_passphrase = u'{filename}!{passphrase}'.format(
+            filename=HEADER_FILENAME, passphrase=self.passphrase)
+        scrypt = launch_scrypt(
+            'enc', header_file_path, header_file_path + '.hmac',
+            scrypt_passphrase)
 
-        hmac = subprocess.Popen(
+        if scrypt.wait() != 0:
-            ["openssl", "dgst", "-" + self.hmac_algorithm,
-                "-hmac", self.passphrase],
-            stdin=open(header_file_path, "r"),
-            stdout=open(header_file_path + ".hmac", "w"))
-        if hmac.wait() != 0:
             raise qubes.exc.QubesException(
-                "Failed to compute hmac of header file")
+                "Failed to compute hmac of header file: "
+                + scrypt.stderr.read())
         return HEADER_FILENAME, HEADER_FILENAME + ".hmac"
 
 
@@ -534,8 +610,6 @@ class Backup(object):
             backup_app.domains[qid].features['backup-size'] = vm_info.size
         backup_app.save()
 
-        passphrase = self.passphrase.encode('utf-8')
-
         vmproc = None
         tar_sparse = None
         if self.target_vm is not None:
@@ -640,73 +714,53 @@ class Backup(object):
 
                 self.log.debug(" ".join(tar_cmdline))
 
-                # Tips: Popen(bufsize=0)
+                # Pipe: tar-sparse | scrypt | tar | backup_target
-                # Pipe: tar-sparse | encryptor [| hmac] | tar | backup_target
-                # Pipe: tar-sparse [| hmac] | tar | backup_target
                 # TODO: log handle stderr
                 tar_sparse = subprocess.Popen(
-                    tar_cmdline, stdin=subprocess.PIPE)
+                    tar_cmdline)
                 self.processes_to_kill_on_cancel.append(tar_sparse)
 
                 # Wait for compressor (tar) process to finish or for any
                 # error of other subprocesses
                 i = 0
-                run_error = "paused"
-                encryptor = None
-                if self.encrypted:
-                    # Start encrypt
-                    # If no cipher is provided,
-                    # the data is forwarded unencrypted !!!
-                    encryptor = subprocess.Popen([
-                        "openssl", "enc",
-                        "-e", "-" + self.crypto_algorithm,
-                        "-pass", "pass:" + passphrase],
-                        stdin=open(backup_pipe, 'rb'),
-                        stdout=subprocess.PIPE)
-                    pipe = encryptor.stdout
-                else:
                 pipe = open(backup_pipe, 'rb')
+                run_error = "paused"
                 while run_error == "paused":
-
-                    # Start HMAC
-                    hmac = subprocess.Popen([
-                        "openssl", "dgst", "-" + self.hmac_algorithm,
-                        "-hmac", passphrase],
-                        stdin=subprocess.PIPE,
-                        stdout=subprocess.PIPE)
-
                     # Prepare a first chunk
-                    chunkfile = backup_tempfile + "." + "%03d" % i
+                    chunkfile = backup_tempfile + ".%03d.enc" % i
                     i += 1
-                    chunkfile_p = open(chunkfile, 'wb')
 
-                    common_args = {
+                    # Start encrypt, scrypt will also handle integrity
-                        'backup_target': chunkfile_p,
+                    # protection
-                        'hmac': hmac,
+                    scrypt_passphrase = \
-                        'vmproc': vmproc,
+                        u'{backup_id}!{filename}!{passphrase}'.format(
+                            backup_id=self.backup_id,
+                            filename=os.path.relpath(chunkfile[:-4],
+                                self.tmpdir),
+                            passphrase=self.passphrase)
+                    scrypt = launch_scrypt(
+                        "enc", "-", chunkfile, scrypt_passphrase)
+
+                    run_error = handle_streams(
+                        pipe,
+                        {'backup_target': scrypt.stdin},
+                        {'vmproc': vmproc,
                          'addproc': tar_sparse,
-                        'progress_callback': self._add_vm_progress,
+                         'scrypt': scrypt,
-                        'size_limit': self.chunk_size,
+                        },
-                    }
+                        self.chunk_size,
-                    run_error = wait_backup_feedback(
+                        self._add_vm_progress
-                        in_stream=pipe, streamproc=encryptor,
+                    )
-                        **common_args)
-                    chunkfile_p.close()
 
                     self.log.debug(
-                        "Wait_backup_feedback returned: {}".format(run_error))
+                        "12 returned: {}".format(run_error))
 
                     if self.canceled:
                         try:
                             tar_sparse.terminate()
                         except OSError:
                             pass
-                        try:
-                            hmac.terminate()
-                        except OSError:
-                            pass
                         tar_sparse.wait()
-                        hmac.wait()
                         to_send.put(QUEUE_ERROR)
                         send_proc.join()
                         shutil.rmtree(self.tmpdir)
@@ -722,29 +776,16 @@ class Backup(object):
                                 "Failed to perform backup: error in " +
                                 run_error)
 
+                    scrypt.stdin.close()
+                    scrypt.wait()
+                    self.log.debug("scrypt return code: {}".format(
+                        scrypt.poll()))
+
                     # Send the chunk to the backup target
                     self._queue_put_with_check(
                         send_proc, vmproc, to_send,
                         os.path.relpath(chunkfile, self.tmpdir))
 
-                    # Close HMAC
-                    hmac.stdin.close()
-                    hmac.wait()
-                    self.log.debug("HMAC proc return code: {}".format(
-                        hmac.poll()))
-
-                    # Write HMAC data next to the chunk file
-                    hmac_data = hmac.stdout.read()
-                    self.log.debug(
-                        "Writing hmac to {}.hmac".format(chunkfile))
-                    with open(chunkfile + ".hmac", 'w') as hmac_file:
-                        hmac_file.write(hmac_data)
-
-                    # Send the HMAC to the backup target
-                    self._queue_put_with_check(
-                        send_proc, vmproc, to_send,
-                        os.path.relpath(chunkfile, self.tmpdir) + ".hmac")
-
                     if tar_sparse.poll() is None or run_error == "size_limit":
                         run_error = "paused"
                     else:
@@ -783,96 +824,52 @@ class Backup(object):
         self.app.save()
 
 
-
+def handle_streams(stream_in, streams_out, processes, size_limit=None,
-
+        progress_callback=None):
-def wait_backup_feedback(progress_callback, in_stream, streamproc,
-                         backup_target, hmac=None, vmproc=None,
-                         addproc=None,
-                         size_limit=None):
-    '''
-    Wait for backup chunk to finish
-    - Monitor all the processes (streamproc, hmac, vmproc, addproc) for errors
-    - Copy stdout of streamproc to backup_target and hmac stdin if available
-    - Compute progress based on total_backup_sz and send progress to
-      progress_callback function
-    - Returns if
-    -     one of the monitored processes error out (streamproc, hmac, vmproc,
-          addproc), along with the processe that failed
-    -     all of the monitored processes except vmproc finished successfully
-          (vmproc termination is controlled by the python script)
-    -     streamproc does not delivers any data anymore (return with the error
-          "")
-    -     size_limit is provided and is about to be exceeded
     '''
+    Copy stream_in to all streams_out and monitor all mentioned processes.
+    If any of them terminate with non-zero code, interrupt the process. Copy
+    at most `size_limit` data (if given).
 
+    :param stream_in: file-like object to read data from
+    :param streams_out: dict of file-like objects to write data to
+    :param processes: dict of subprocess.Popen objects to monitor
+    :param size_limit: int maximum data amount to process
+    :param progress_callback: callable function to report progress, will be
+    given copied data size (it should accumulate internally)
+    :return: failed process name, failed stream name, "size_limit" or None (
+    no error)
+    '''
     buffer_size = 409600
-    run_error = None
-    run_count = 1
     bytes_copied = 0
-    log = logging.getLogger('qubes.backup')
+    while True:
-
+        if size_limit:
-    while run_count > 0 and run_error is None:
+            to_copy = min(buffer_size, size_limit - bytes_copied)
-        if size_limit and bytes_copied + buffer_size > size_limit:
+            if to_copy <= 0:
                 return "size_limit"
+        else:
+            to_copy = buffer_size
+        buf = stream_in.read(to_copy)
+        if not len(buf):
+            # done
+            return None
 
-        buf = in_stream.read(buffer_size)
         if callable(progress_callback):
             progress_callback(len(buf))
+        for name, stream in streams_out.items():
+            if stream is None:
+                continue
+            try:
+                stream.write(buf)
+            except IOError:
+                return name
         bytes_copied += len(buf)
 
-        run_count = 0
+        for name, proc in processes.items():
-        if hmac:
+            if proc is None:
-            retcode = hmac.poll()
+                continue
-            if retcode is not None:
+            if proc.poll():
-                if retcode != 0:
+                return name
-                    run_error = "hmac"
-            else:
-                run_count += 1
-
-        if addproc:
-            retcode = addproc.poll()
-            if retcode is not None:
-                if retcode != 0:
-                    run_error = "addproc"
-            else:
-                run_count += 1
-
-        if vmproc:
-            retcode = vmproc.poll()
-            if retcode is not None:
-                if retcode != 0:
-                    run_error = "VM"
-                    log.debug(vmproc.stdout.read())
-            else:
-                # VM should run until the end
-                pass
-
-        if streamproc:
-            retcode = streamproc.poll()
-            if retcode is not None:
-                if retcode != 0:
-                    run_error = "streamproc"
-                    break
-                elif retcode == 0 and len(buf) <= 0:
-                    return ""
-            run_count += 1
-
-        else:
-            if len(buf) <= 0:
-                return ""
-
-        try:
-            backup_target.write(buf)
-        except IOError as e:
-            if e.errno == errno.EPIPE:
-                run_error = "target"
-            else:
-                raise
-
-        if hmac:
-            hmac.stdin.write(buf)
-
-    return run_error
 
 
 class ExtractWorker2(Process):
@@ -1127,6 +1124,10 @@ class ExtractWorker2(Process):
             self.tar2_current_file = filename
 
             pipe = open(self.restore_pipe, 'wb')
+            monitor_processes = {
+                'vmproc': self.vmproc,
+                'addproc': self.tar2_process,
+            }
             common_args = {
                 'backup_target': pipe,
                 'hmac': None,
@@ -1144,28 +1145,23 @@ class ExtractWorker2(Process):
                     (["-z"] if self.compressed else []),
                     stdin=open(filename, 'rb'),
                     stdout=subprocess.PIPE)
-
+                in_stream = self.decryptor_process.stdout
-                run_error = wait_backup_feedback(
+                monitor_processes['decryptor'] = self.decryptor_process
-                    progress_callback=self.progress_callback,
-                    in_stream=self.decryptor_process.stdout,
-                    streamproc=self.decryptor_process,
-                    **common_args)
             elif self.compressed:
                 self.decompressor_process = subprocess.Popen(
                     ["gzip", "-d"],
                     stdin=open(filename, 'rb'),
                     stdout=subprocess.PIPE)
-
+                in_stream = self.decompressor_process.stdout
-                run_error = wait_backup_feedback(
+                monitor_processes['decompresor'] = self.decompressor_process
-                    progress_callback=self.progress_callback,
-                    in_stream=self.decompressor_process.stdout,
-                    streamproc=self.decompressor_process,
-                    **common_args)
             else:
-                run_error = wait_backup_feedback(
+                in_stream = open(filename, 'rb')
-                    progress_callback=self.progress_callback,
+
-                    in_stream=open(filename, "rb"), streamproc=None,
+            run_error = handle_streams(
-                    **common_args)
+                in_stream,
+                {'target': pipe},
+                monitor_processes,
+                progress_callback=self.progress_callback)
 
             try:
                 pipe.close()
@@ -1177,7 +1173,7 @@ class ExtractWorker2(Process):
                     # ignore the error
                 else:
                     raise
-            if len(run_error):
+            if run_error:
                 if run_error == "target":
                     self.collect_tar_output()
                     details = "\n".join(self.tar2_stderr)
@@ -1307,22 +1303,31 @@ class ExtractWorker3(ExtractWorker2):
                 os.remove(filename)
                 continue
             else:
+                (basename, ext) = os.path.splitext(self.tar2_current_file)
+                previous_chunk_number = int(ext[1:])
+                expected_filename = basename + '.%03d' % (
+                    previous_chunk_number+1)
+                if expected_filename != filename:
+                    self.cleanup_tar2(wait=True, terminate=True)
+                    self.log.error(
+                        'Unexpected file in archive: {}, expected {}'.format(
+                            filename, expected_filename))
+                    os.remove(filename)
+                    continue
                 self.log.debug("Releasing next chunck")
+
             self.tar2_current_file = filename
 
-            common_args = {
+            run_error = handle_streams(
-                'backup_target': input_pipe,
+                open(filename, 'rb'),
-                'hmac': None,
+                {'target': input_pipe},
-                'vmproc': self.vmproc,
+                {'vmproc': self.vmproc,
-                'addproc': self.tar2_process
+                 'addproc': self.tar2_process,
-            }
+                 'decryptor': self.decryptor_process,
+                },
+                progress_callback=self.progress_callback)
 
-            run_error = wait_backup_feedback(
+            if run_error:
-                progress_callback=self.progress_callback,
-                in_stream=open(filename, "rb"), streamproc=None,
-                **common_args)
-
-            if len(run_error):
                 if run_error == "target":
                     self.collect_tar_output()
                     details = "\n".join(self.tar2_stderr)
@@ -1356,6 +1361,8 @@ def get_supported_hmac_algo(hmac_algorithm=None):
     # Start with provided default
     if hmac_algorithm:
         yield hmac_algorithm
+    if hmac_algorithm != 'scrypt':
+        yield 'scrypt'
     proc = subprocess.Popen(['openssl', 'list-message-digest-algorithms'],
                             stdout=subprocess.PIPE)
     for algo in proc.stdout.readlines():
@@ -1575,6 +1582,10 @@ class BackupRestore(object):
 
     def _verify_hmac(self, filename, hmacfile, algorithm=None):
         def load_hmac(hmac_text):
+            if filter(lambda x: ord(x) not in range(128),
+                    hmac_text):
+                raise qubes.exc.QubesException(
+                    "Invalid content of {}".format(hmacfile))
             hmac_text = hmac_text.strip().split("=")
             if len(hmac_text) > 1:
                 hmac_text = hmac_text[1].strip()
@@ -1593,6 +1604,17 @@ class BackupRestore(object):
                 "ERROR: expected hmac for {}, but got {}".
                 format(filename, hmacfile))
 
+        if algorithm == 'scrypt':
+            # in case of 'scrypt' _verify_hmac is only used for backup header
+            assert filename == HEADER_FILENAME
+            self._verify_and_decrypt(hmacfile, HEADER_FILENAME + '.dec')
+            if open(os.path.join(self.tmpdir, filename)).read() != \
+                    open(os.path.join(self.tmpdir, filename + '.dec')).read():
+                raise qubes.exc.QubesException(
+                    'Invalid hmac on {}'.format(filename))
+            else:
+                return True
+
         hmac_proc = subprocess.Popen(
             ["openssl", "dgst", "-" + algorithm, "-hmac", passphrase],
             stdin=open(os.path.join(self.tmpdir, filename), 'rb'),
@@ -1618,6 +1640,80 @@ class BackupRestore(object):
                     "Is the passphrase correct?".
                     format(filename, load_hmac(hmac_stdout)))
 
+    def _verify_and_decrypt(self, filename, output=None):
+        assert filename.endswith('.enc') or filename.endswith('.hmac')
+        fullname = os.path.join(self.tmpdir, filename)
+        (origname, _) = os.path.splitext(filename)
+        if output:
+            fulloutput = os.path.join(self.tmpdir, output)
+        else:
+            fulloutput = os.path.join(self.tmpdir, origname)
+        if origname == HEADER_FILENAME:
+            passphrase = u'{filename}!{passphrase}'.format(
+                filename=origname,
+                passphrase=self.passphrase)
+        else:
+            passphrase = u'{backup_id}!{filename}!{passphrase}'.format(
+                backup_id=self.header_data.backup_id,
+                filename=origname,
+                passphrase=self.passphrase)
+        p = launch_scrypt('dec', fullname, fulloutput, passphrase)
+        (_, stderr) = p.communicate()
+        if p.returncode != 0:
+            os.unlink(fulloutput)
+            raise qubes.exc.QubesException('failed to decrypt {}: {}'.format(
+                fullname, stderr))
+        # encrypted file is no longer needed
+        os.unlink(fullname)
+        return origname
+
+    def _retrieve_backup_header_files(self, files, allow_none=False):
+        (retrieve_proc, filelist_pipe, error_pipe) = \
+            self._start_retrieval_process(
+                files, len(files), 1024 * 1024)
+        filelist = filelist_pipe.read()
+        retrieve_proc_returncode = retrieve_proc.wait()
+        if retrieve_proc in self.processes_to_kill_on_cancel:
+            self.processes_to_kill_on_cancel.remove(retrieve_proc)
+        extract_stderr = error_pipe.read(MAX_STDERR_BYTES)
+
+        # wait for other processes (if any)
+        for proc in self.processes_to_kill_on_cancel:
+            if proc.wait() != 0:
+                raise qubes.exc.QubesException(
+                    "Backup header retrieval failed (exit code {})".format(
+                        proc.wait())
+                )
+
+        if retrieve_proc_returncode != 0:
+            if not filelist and 'Not found in archive' in extract_stderr:
+                if allow_none:
+                    return None
+                else:
+                    raise qubes.exc.QubesException(
+                        "unable to read the qubes backup file {0} ({1}): {2}".format(
+                            self.backup_location,
+                            retrieve_proc.wait(),
+                            extract_stderr
+                        ))
+        actual_files = filelist.splitlines()
+        if sorted(actual_files) != sorted(files):
+            raise qubes.exc.QubesException(
+                'unexpected files in archive: got {!r}, expeced {!r}'.format(
+                    actual_files, files
+                ))
+        for f in files:
+            if not os.path.exists(os.path.join(self.tmpdir, f)):
+                if allow_none:
+                    return None
+                else:
+                    raise qubes.exc.QubesException(
+                        'Unable to retrieve file {} from backup {}: {}'.format(
+                            f, self.backup_location, extract_stderr
+                        )
+                    )
+        return files
+
     def _retrieve_backup_header(self):
         """Retrieve backup header and qubes.xml. Only backup header is
         analyzed, qubes.xml is left as-is
@@ -1634,82 +1730,47 @@ class BackupRestore(object):
             header_data.version = 1
             return header_data
 
-        (retrieve_proc, filelist_pipe, error_pipe) = \
+        header_files = self._retrieve_backup_header_files(
-            self._start_retrieval_process(
+            ['backup-header', 'backup-header.hmac'], allow_none=True)
-                ['backup-header', 'backup-header.hmac',
-                'qubes.xml.000', 'qubes.xml.000.hmac'], 4, 1024 * 1024)
-
-        expect_tar_error = False
-
-        filename = filelist_pipe.readline().strip()
-        hmacfile = filelist_pipe.readline().strip()
-        # tar output filename before actually extracting it, so wait for the
-        # next one before trying to access it
-        if not self.backup_vm:
-            filelist_pipe.readline().strip()
 
+        if not header_files:
+            # R2-Beta3 didn't have backup header, so if none is found,
+            # assume it's version=2 and use values present at that time
+            header_data = BackupHeader(
+                version=2,
+                # place explicitly this value, because it is what format_version
+                # 2 have
+                hmac_algorithm='SHA1',
+                crypto_algorithm='aes-256-cbc',
+                # TODO: set encrypted to something...
+            )
+        else:
+            filename = HEADER_FILENAME
+            hmacfile = HEADER_FILENAME + '.hmac'
             self.log.debug("Got backup header and hmac: {}, {}".format(
                 filename, hmacfile))
 
-        if not filename or filename == "EOF" or \
-                not hmacfile or hmacfile == "EOF":
-            retrieve_proc.wait()
-            proc_error_msg = error_pipe.read(MAX_STDERR_BYTES)
-            raise qubes.exc.QubesException(
-                "Premature end of archive while receiving "
-                "backup header. Process output:\n" + proc_error_msg)
             file_ok = False
             hmac_algorithm = DEFAULT_HMAC_ALGORITHM
             for hmac_algo in get_supported_hmac_algo(hmac_algorithm):
                 try:
                     if self._verify_hmac(filename, hmacfile, hmac_algo):
                         file_ok = True
-                    hmac_algorithm = hmac_algo
                         break
-            except qubes.exc.QubesException:
+                except qubes.exc.QubesException as e:
+                    self.log.debug(
+                        'Failed to verify {} using {}: {}'.format(
+                            hmacfile, hmac_algo, str(e)))
                     # Ignore exception here, try the next algo
                     pass
             if not file_ok:
                 raise qubes.exc.QubesException(
                     "Corrupted backup header (hmac verification "
                     "failed). Is the password correct?")
-        if os.path.basename(filename) == HEADER_FILENAME:
             filename = os.path.join(self.tmpdir, filename)
             header_data = BackupHeader(open(filename, 'r').read())
             os.unlink(filename)
-        else:
-            # if no header found, create one with guessed HMAC algo
-            header_data = BackupHeader(
-                version=2,
-                hmac_algorithm=hmac_algorithm,
-                # place explicitly this value, because it is what format_version
-                # 2 have
-                crypto_algorithm='aes-256-cbc',
-                # TODO: set encrypted to something...
-            )
-            # when tar do not find expected file in archive, it exit with
-            # code 2. This will happen because we've requested backup-header
-            # file, but the archive do not contain it. Ignore this particular
-            # error.
-            if not self.backup_vm:
-                expect_tar_error = True
 
-        if retrieve_proc.wait() != 0 and not expect_tar_error:
-            raise qubes.exc.QubesException(
-                "unable to read the qubes backup file {0} ({1}): {2}".format(
-                    self.backup_location,
-                    retrieve_proc.wait(),
-                    error_pipe.read(MAX_STDERR_BYTES)
-                ))
-        if retrieve_proc in self.processes_to_kill_on_cancel:
-            self.processes_to_kill_on_cancel.remove(retrieve_proc)
-        # wait for other processes (if any)
-        for proc in self.processes_to_kill_on_cancel:
-            if proc.wait() != 0:
-                raise qubes.exc.QubesException(
-                    "Backup header retrieval failed (exit code {})".format(
-                        proc.wait())
-                )
         return header_data
 
     def _start_inner_extraction_worker(self, queue, relocate):
@@ -1742,6 +1803,9 @@ class BackupRestore(object):
         elif format_version in [3, 4]:
             extractor_params['compression_filter'] = \
                 self.header_data.compression_filter
+            if format_version == 4:
+                # encryption already handled
+                extractor_params['encrypted'] = False
             extract_proc = ExtractWorker3(**extractor_params)
         else:
             raise NotImplementedError(
@@ -1760,7 +1824,14 @@ class BackupRestore(object):
                 offline_mode=True)
             return backup_app
         else:
+            if self.header_data.version in [2, 3]:
+                self._retrieve_backup_header_files(
+                    ['qubes.xml.000', 'qubes.xml.000.hmac'])
                 self._verify_hmac("qubes.xml.000", "qubes.xml.000.hmac")
+            else:
+                self._retrieve_backup_header_files(['qubes.xml.000.enc'])
+                self._verify_and_decrypt('qubes.xml.000.enc')
+
             queue = Queue()
             queue.put("qubes.xml.000")
             queue.put(QUEUE_FINISHED)
@@ -1808,6 +1879,7 @@ class BackupRestore(object):
 
         try:
             filename = None
+            hmacfile = None
             nextfile = None
             while True:
                 if self.canceled:
@@ -1831,29 +1903,57 @@ class BackupRestore(object):
                 if not filename or filename == "EOF":
                     break
 
-                hmacfile = filelist_pipe.readline().strip()
-
-                if self.canceled:
-                    break
                 # if reading archive directly with tar, wait for next filename -
                 # tar prints filename before processing it, so wait for
                 # the next one to be sure that whole file was extracted
                 if not self.backup_vm:
                     nextfile = filelist_pipe.readline().strip()
 
+                if self.header_data.version in [2, 3]:
+                    if not self.backup_vm:
+                        hmacfile = nextfile
+                        nextfile = filelist_pipe.readline().strip()
+                    else:
+                        hmacfile = filelist_pipe.readline().strip()
+
+                    if self.canceled:
+                        break
+
                     self.log.debug("Getting hmac:" + hmacfile)
                     if not hmacfile or hmacfile == "EOF":
                         # Premature end of archive, either of tar1_command or
                         # vmproc exited with error
                         break
+                else:  # self.header_data.version == 4
+                    if not filename.endswith('.enc'):
+                        raise qubes.exc.QubesException(
+                            'Invalid file extension found in archive: {}'.
+                            format(filename))
 
                 if not any(map(lambda x: filename.startswith(x), vms_dirs)):
                     self.log.debug("Ignoring VM not selected for restore")
                     os.unlink(os.path.join(self.tmpdir, filename))
+                    if hmacfile:
                         os.unlink(os.path.join(self.tmpdir, hmacfile))
                     continue
 
-                if self._verify_hmac(filename, hmacfile):
+                if self.header_data.version in [2, 3]:
+                    self._verify_hmac(filename, hmacfile)
+                else:
+                    # _verify_and_decrypt will write output to a file with
+                    # '.enc' extension cut off. This is safe because:
+                    # - `scrypt` tool will override output, so if the file was
+                    # already there (received from the VM), it will be removed
+                    # - incoming archive extraction will refuse to override
+                    # existing file, so if `scrypt` already created one,
+                    # it can not be manipulated by the VM
+                    # - when the file is retrieved from the VM, it appears at
+                    # the final form - if it's visible, VM have no longer
+                    # influence over its content
+                    #
+                    # This all means that if the file was correctly verified
+                    # + decrypted, we will surely access the right file
+                    filename = self._verify_and_decrypt(filename)
                 to_extract.put(os.path.join(self.tmpdir, filename))
 
             if self.canceled:
@@ -1921,7 +2021,7 @@ class BackupRestore(object):
                 vm_info.problems.add(self.VMToRestore.EXCLUDED)
 
             if not self.options.verify_only and \
-                    vm in self.app.domains:
+                    vm_info.name in self.app.domains:
                 if self.options.rename_conflicting:
                     new_name = self.generate_new_name_for_conflicting_vm(
                         vm, restore_info
@@ -2243,6 +2343,8 @@ class BackupRestore(object):
 
         # FIXME handle locking
 
+        restore_info = self.restore_info_verify(restore_info)
+
         self._restore_vms_metadata(restore_info)
 
         # Perform VM restoration in backup order

qubes/core2migration.py

@@ -163,12 +163,23 @@ class Core2Qubes(qubes.Qubes):
                     'template_qid'))]
                 vm_class = AppVM
             # simple attributes
-            for attr in ['installed_by_rpm', 'include_in_backups',
+            for attr, default in {
-                    'qrexec_timeout', 'internal', 'label', 'name',
+                'installed_by_rpm': 'False',
-                    'vcpus', 'memory', 'maxmem', 'default_user',
+                'include_in_backups': 'True',
-                    'debug', 'pci_strictreset', 'mac', 'autostart']:
+                'qrexec_timeout': '60',
+                'internal': 'False',
+                'label': None,
+                'name': None,
+                'vcpus': '2',
+                'memory': '400',
+                'maxmem': '4000',
+                'default_user': 'user',
+                'debug': 'False',
+                'pci_strictreset': 'True',
+                'mac': None,
+                'autostart': 'False'}.items():
                 value = element.get(attr)
-                if value:
+                if value and value != default:
                     kwargs[attr] = value
             # attributes with default value
             for attr in ["kernel", "kernelopts"]:

qubes/tests/__init__.py

@@ -63,6 +63,17 @@ VMPREFIX = 'test-inst-'
 CLSVMPREFIX = 'test-cls-'
 
 
+if 'DEFAULT_LVM_POOL' in os.environ.keys():
+    DEFAULT_LVM_POOL = os.environ['DEFAULT_LVM_POOL']
+else:
+    DEFAULT_LVM_POOL = 'qubes_dom0/pool00'
+
+
+POOL_CONF = {'name': 'test-lvm',
+             'driver': 'lvm_thin',
+             'volume_group': DEFAULT_LVM_POOL.split('/')[0],
+             'thin_pool': DEFAULT_LVM_POOL.split('/')[1]}
+
 #: :py:obj:`True` if running in dom0, :py:obj:`False` otherwise
 in_dom0 = False
 
@@ -528,6 +539,30 @@ class SystemTestsMixin(object):
         )
         self.app.default_netvm = netvm_clone
 
+
+    def _find_pool(self, volume_group, thin_pool):
+        ''' Returns the pool matching the specified ``volume_group`` &
+            ``thin_pool``, or None.
+        '''
+        pools = [p for p in self.app.pools
+                 if issubclass(p.__class__, qubes.storage.lvm.ThinPool)]
+        for pool in pools:
+            if pool.volume_group == volume_group \
+                    and pool.thin_pool == thin_pool:
+                return pool
+        return None
+
+    def init_lvm_pool(self):
+        volume_group, thin_pool = DEFAULT_LVM_POOL.split('/', 1)
+        path = "/dev/mapper/{!s}-{!s}".format(volume_group, thin_pool)
+        if not os.path.exists(path):
+            self.skipTest('LVM thin pool {!r} does not exist'.
+                format(DEFAULT_LVM_POOL))
+        self.pool = self._find_pool(volume_group, thin_pool)
+        if not self.pool:
+            self.pool = self.app.add_pool(**POOL_CONF)
+            self.created_pool = True
+
     def reload_db(self):
         self.app = qubes.Qubes(qubes.tests.XMLPATH)
 

qubes/tests/int/backup.py

@@ -161,6 +161,7 @@ class BackupTestsMixin(qubes.tests.SystemTestsMixin):
             else:
                 raise
 
+        if 'passphrase' not in kwargs:
             backup.passphrase = 'qubes'
         backup.target_dir = target
 
@@ -176,7 +177,8 @@ class BackupTestsMixin(qubes.tests.SystemTestsMixin):
         #self.reload_db()
 
     def restore_backup(self, source=None, appvm=None, options=None,
-                       expect_errors=None):
+                       expect_errors=None, manipulate_restore_info=None,
+                       passphrase='qubes'):
         if source is None:
             backupfile = os.path.join(self.backupdir,
                                       sorted(os.listdir(self.backupdir))[-1])
@@ -185,11 +187,13 @@ class BackupTestsMixin(qubes.tests.SystemTestsMixin):
 
         with self.assertNotRaises(qubes.exc.QubesException):
             restore_op = qubes.backup.BackupRestore(
-                self.app, backupfile, appvm, "qubes")
+                self.app, backupfile, appvm, passphrase)
             if options:
                 for key, value in options.items():
                     setattr(restore_op.options, key, value)
             restore_info = restore_op.get_restore_info()
+        if callable(manipulate_restore_info):
+            restore_info = manipulate_restore_info(restore_info)
         self.log.debug(restore_op.get_restore_summary(restore_info))
 
         with self.assertNotRaises(qubes.exc.QubesException):
@@ -357,7 +361,36 @@ class TC_00_Backup(BackupTestsMixin, qubes.tests.QubesTestCase):
         # create backup with internal dependencies (template, netvm etc)
         # try restoring only AppVMs (but not templates, netvms) - should
         # handle according to options set
-        self.skipTest('test not implemented')
+        exclude = [
+            self.make_vm_name('test-net'),
+            self.make_vm_name('template')
+        ]
+        def exclude_some(restore_info):
+            for name in exclude:
+                restore_info.pop(name)
+            return restore_info
+        vms = self.create_backup_vms()
+        orig_hashes = self.vm_checksum(vms)
+        self.make_backup(vms, compression_filter="bzip2")
+        self.remove_vms(reversed(vms))
+        self.restore_backup(manipulate_restore_info=exclude_some)
+        for vm in vms:
+            if vm.name == self.make_vm_name('test1'):
+                # netvm was set to 'test-inst-test-net' - excluded
+                vm.netvm = qubes.property.DEFAULT
+            elif vm.name == self.make_vm_name('custom'):
+                # template was set to 'test-inst-template' - excluded
+                vm.template = self.app.default_template
+        vms = [vm for vm in vms if vm.name not in exclude]
+        self.assertCorrectlyRestored(vms, orig_hashes)
+
+    def test_020_encrypted_backup_non_ascii(self):
+        vms = self.create_backup_vms()
+        orig_hashes = self.vm_checksum(vms)
+        self.make_backup(vms, encrypted=True, passphrase=u'zażółć gęślą jaźń')
+        self.remove_vms(reversed(vms))
+        self.restore_backup(passphrase=u'zażółć gęślą jaźń')
+        self.assertCorrectlyRestored(vms, orig_hashes)
 
     def test_100_backup_dom0_no_restore(self):
         # do not write it into dom0 home itself...

qubes/tests/int/backupcompatibility.py

@@ -405,6 +405,28 @@ class TC_00_BackupCompatibility(
 
         output.close()
 
+    def assertRestored(self, name, **kwargs):
+        with self.assertNotRaises((KeyError, qubes.exc.QubesException)):
+            vm = self.app.domains[name]
+            vm.storage.verify()
+            for prop, value in kwargs.items():
+                if prop == 'klass':
+                    self.assertIsInstance(vm, value)
+                elif value is qubes.property.DEFAULT:
+                    self.assertTrue(vm.property_is_default(prop),
+                        'VM {} - property {} not default'.format(vm.name, prop))
+                else:
+                    actual_value = getattr(vm, prop)
+                    if isinstance(actual_value, qubes.vm.BaseVM):
+                        self.assertEqual(value, actual_value.name,
+                            'VM {} - property {}'.format(vm.name, prop))
+                    elif isinstance(actual_value, qubes.Label):
+                        self.assertEqual(value, actual_value.name,
+                            'VM {} - property {}'.format(vm.name, prop))
+                    else:
+                        self.assertEqual(value, actual_value,
+                            'VM {} - property {}'.format(vm.name, prop))
+
     def test_100_r1(self):
         self.create_v1_files(r2b2=False)
 
@@ -418,15 +440,49 @@ class TC_00_BackupCompatibility(
                 'use-default-netvm': True,
             },
         )
-        with self.assertNotRaises(KeyError):
+        common_props = {
-            vm = self.app.domains["test-template-clone"]
+            'installed_by_rpm': False,
-            vm = self.app.domains["test-testproxy"]
+            'kernel': qubes.property.DEFAULT,
-            vm = self.app.domains["test-work"]
+            'kernelopts': qubes.property.DEFAULT,
-            vm = self.app.domains["test-standalonevm"]
+            'qrexec_timeout': qubes.property.DEFAULT,
-            vm = self.app.domains["test-custom-template-appvm"]
+            'netvm': qubes.property.DEFAULT,
-        self.assertEqual(self.app.domains["test-custom-template-appvm"]
+            'default_user': qubes.property.DEFAULT,
-                         .template,
+            'internal': qubes.property.DEFAULT,
-                         self.app.domains["test-template-clone"])
+            'include_in_backups': True,
+            'debug': False,
+            'maxmem': 4000,  # 4063 caped by 10*400
+            'memory': 400,
+        }
+        self.assertRestored("test-template-clone",
+            klass=qubes.vm.templatevm.TemplateVM,
+            label='gray',
+            provides_network=False,
+            **common_props)
+        testproxy_props = common_props.copy()
+        testproxy_props.update(
+            label='yellow',
+            provides_network=True,
+            memory=200,
+            maxmem=2000,
+            template=self.app.default_template.name,
+        )
+        self.assertRestored("test-testproxy",
+            klass=qubes.vm.appvm.AppVM,
+            **testproxy_props)
+        self.assertRestored("test-work",
+            klass=qubes.vm.appvm.AppVM,
+            template=self.app.default_template.name,
+            label='green',
+            **common_props)
+        self.assertRestored("test-standalonevm",
+            klass=qubes.vm.standalonevm.StandaloneVM,
+            label='red',
+            **common_props)
+        self.assertRestored("test-custom-template-appvm",
+            klass=qubes.vm.appvm.AppVM,
+            template='test-template-clone',
+            label='yellow',
+            **common_props)
 
     def test_200_r2b2(self):
         self.create_v1_files(r2b2=True)
@@ -439,16 +495,51 @@ class TC_00_BackupCompatibility(
             'use-default-template': True,
             'use-default-netvm': True,
         })
-        with self.assertNotRaises(KeyError):
+        common_props = {
-            vm = self.app.domains["test-template-clone"]
+            'installed_by_rpm': False,
-            vm = self.app.domains["test-testproxy"]
+            'kernel': qubes.property.DEFAULT,
-            vm = self.app.domains["test-work"]
+            'kernelopts': qubes.property.DEFAULT,
-            vm = self.app.domains["test-testhvm"]
+            'qrexec_timeout': qubes.property.DEFAULT,
-            vm = self.app.domains["test-standalonevm"]
+            'netvm': qubes.property.DEFAULT,
-            vm = self.app.domains["test-custom-template-appvm"]
+            'default_user': qubes.property.DEFAULT,
-        self.assertEqual(self.app.domains["test-custom-template-appvm"]
+            'internal': qubes.property.DEFAULT,
-                         .template,
+            'include_in_backups': True,
-                         self.app.domains["test-template-clone"])
+            'debug': False,
+            'maxmem': 1535,
+            'memory': 400,
+        }
+        template_clone_props = common_props.copy()
+        template_clone_props.update(
+            label='green',
+            provides_network=False,
+        )
+        self.assertRestored("test-template-clone",
+            klass=qubes.vm.templatevm.TemplateVM,
+            **template_clone_props)
+        testproxy_props = common_props.copy()
+        testproxy_props.update(
+            label='red',
+            provides_network=True,
+            memory=200,
+            template=self.app.default_template.name,
+        )
+        self.assertRestored("test-testproxy",
+            klass=qubes.vm.appvm.AppVM,
+            **testproxy_props)
+        self.assertRestored("test-work",
+            klass=qubes.vm.appvm.AppVM,
+            template=self.app.default_template.name,
+            label='green',
+            **common_props)
+        self.assertRestored("test-standalonevm",
+            klass=qubes.vm.standalonevm.StandaloneVM,
+            label='blue',
+            **common_props)
+        self.assertRestored("test-custom-template-appvm",
+            klass=qubes.vm.appvm.AppVM,
+            template='test-template-clone',
+            label='yellow',
+            **common_props)
 
     def test_210_r2(self):
         self.create_v3_backup(False)
@@ -457,16 +548,48 @@ class TC_00_BackupCompatibility(
             'use-default-template': True,
             'use-default-netvm': True,
         })
-        with self.assertNotRaises(KeyError):
+        common_props = {
-            vm = self.app.domains["test-template-clone"]
+            'installed_by_rpm': False,
-            vm = self.app.domains["test-testproxy"]
+            'kernel': qubes.property.DEFAULT,
-            vm = self.app.domains["test-work"]
+            'kernelopts': qubes.property.DEFAULT,
-            vm = self.app.domains["test-testhvm"]
+            'qrexec_timeout': qubes.property.DEFAULT,
-            vm = self.app.domains["test-standalonevm"]
+            'netvm': qubes.property.DEFAULT,
-            vm = self.app.domains["test-custom-template-appvm"]
+            'default_user': qubes.property.DEFAULT,
-        self.assertEqual(self.app.domains["test-custom-template-appvm"]
+            'internal': qubes.property.DEFAULT,
-                         .template,
+            'include_in_backups': True,
-                         self.app.domains["test-template-clone"])
+            'debug': False,
+            'maxmem': 1535,
+            'memory': 400,
+        }
+        self.assertRestored("test-template-clone",
+            klass=qubes.vm.templatevm.TemplateVM,
+            label='green',
+            provides_network=False,
+            **common_props)
+        testproxy_props = common_props.copy()
+        testproxy_props.update(
+            label='red',
+            provides_network=True,
+            memory=200,
+            template=self.app.default_template.name,
+        )
+        self.assertRestored("test-testproxy",
+            klass=qubes.vm.appvm.AppVM,
+            **testproxy_props)
+        self.assertRestored("test-work",
+            klass=qubes.vm.appvm.AppVM,
+            template=self.app.default_template.name,
+            label='green',
+            **common_props)
+        self.assertRestored("test-standalonevm",
+            klass=qubes.vm.standalonevm.StandaloneVM,
+            label='blue',
+            **common_props)
+        self.assertRestored("test-custom-template-appvm",
+            klass=qubes.vm.appvm.AppVM,
+            template='test-template-clone',
+            label='yellow',
+            **common_props)
 
     def test_220_r2_encrypted(self):
         self.create_v3_backup(True)
@@ -475,13 +598,59 @@ class TC_00_BackupCompatibility(
             'use-default-template': True,
             'use-default-netvm': True,
         })
-        with self.assertNotRaises(KeyError):
+        common_props = {
-            vm = self.app.domains["test-template-clone"]
+            'installed_by_rpm': False,
-            vm = self.app.domains["test-testproxy"]
+            'kernel': qubes.property.DEFAULT,
-            vm = self.app.domains["test-work"]
+            'kernelopts': qubes.property.DEFAULT,
-            vm = self.app.domains["test-testhvm"]
+            'qrexec_timeout': qubes.property.DEFAULT,
-            vm = self.app.domains["test-standalonevm"]
+            'netvm': qubes.property.DEFAULT,
-            vm = self.app.domains["test-custom-template-appvm"]
+            'default_user': qubes.property.DEFAULT,
-        self.assertEqual(self.app.domains["test-custom-template-appvm"]
+            'internal': qubes.property.DEFAULT,
-                         .template,
+            'include_in_backups': True,
-                         self.app.domains["test-template-clone"])
+            'debug': False,
+            'maxmem': 1535,  # 4063 caped by 10*400
+            'memory': 400,
+        }
+        self.assertRestored("test-template-clone",
+            klass=qubes.vm.templatevm.TemplateVM,
+            label='green',
+            provides_network=False,
+            **common_props)
+        testproxy_props = common_props.copy()
+        testproxy_props.update(
+            label='red',
+            provides_network=True,
+            memory=200,
+            template=self.app.default_template.name,
+        )
+        self.assertRestored("test-testproxy",
+            klass=qubes.vm.appvm.AppVM,
+            **testproxy_props)
+        self.assertRestored("test-work",
+            klass=qubes.vm.appvm.AppVM,
+            template=self.app.default_template.name,
+            label='green',
+            **common_props)
+        self.assertRestored("test-standalonevm",
+            klass=qubes.vm.standalonevm.StandaloneVM,
+            label='blue',
+            **common_props)
+        self.assertRestored("test-custom-template-appvm",
+            klass=qubes.vm.appvm.AppVM,
+            template='test-template-clone',
+            label='yellow',
+            **common_props)
+
+
+class TC_01_BackupCompatibilityIntoLVM(TC_00_BackupCompatibility):
+    def setUp(self):
+        super(TC_01_BackupCompatibilityIntoLVM, self).setUp()
+        self.init_lvm_pool()
+
+    def restore_backup(self, source=None, appvm=None, options=None,
+            expect_errors=None, manipulate_restore_info=None):
+        if options is None:
+            options = {}
+        options['override_pool'] = self.pool.name
+        super(TC_01_BackupCompatibilityIntoLVM, self).restore_backup(source,
+            appvm, options, expect_errors, manipulate_restore_info)

rpm_spec/core-dom0.spec

@@ -83,6 +83,7 @@ Requires:       createrepo
 Requires:       gnome-packagekit
 Requires:       cronie
 Requires:       bsdtar
+Requires:       scrypt
 # for qubes-hcl-report
 Requires:       dmidecode
 Requires:       PyQt4