tools: add qvm-backup tool

New qvm-backup tool can either use pre-existing backup profile
(--profile), or - when running in dom0 - can create new one based on
used options (--save-profile).

This commit add a tool itself, update its man page, and add tests for
it.

Fixes QubesOS/qubes-issues#2931

ci/requirements.txt

@@ -7,3 +7,4 @@ codecov
 python-daemon
 mock
 lxml
+PyYAML

doc/manpages/qvm-backup.rst

@@ -3,26 +3,29 @@
 :program:`qvm-backup` -- Create a backup of Qubes
 =================================================
 
-.. warning::
-
-   This page was autogenerated from command-line parser. It shouldn't be 1:1
-   conversion, because it would add little value. Please revise it and add
-   more descriptive help, which normally won't fit in standard ``--help``
-   option.
-
-   After rewrite, please remove this admonition.
-
 Synopsis
 --------
 
-:command:`qvm-backup` [-h] [--verbose] [--quiet] [--force-root] [--exclude EXCLUDE_LIST] [--dest-vm *APPVM*] [--encrypt] [--no-encrypt] [--passphrase-file PASS_FILE] [--enc-algo CRYPTO_ALGORITHM] [--hmac-algo HMAC_ALGORITHM] [--compress] [--compress-filter COMPRESS_FILTER] [--tmpdir *TMPDIR*] backup_location [vms [vms ...]]
+:command:`qvm-backup` [-h] [--verbose] [--quiet] [--profile *PROFILE*] [--exclude EXCLUDE_LIST] [--dest-vm *APPVM*] [--encrypt] [--passphrase-file PASSPHRASE_FILE] [--compress] [--compress-filter *COMPRESSION*] [--save-profile SAVE_PROFILE] backup_location [vms [vms ...]]
+
 
 Options
 -------
 
+.. option:: --profile
+
+   Specify backup profile to use. This option is mutually exclusive with all
+   other options. This is also the only working mode when running from non-dom0.
+
+.. option:: --save-profile
+
+   Save backup profile based on given options. This is possible only when
+   running in dom0. Otherwise, prepared profile is printed on standard output
+   and user needs to manually place it into /etc/qubes/backup in dom0.
+
 .. option:: --help, -h
 
-   show this help message and exit
+   show help message and exit
 
 .. option:: --verbose, -v
 
@@ -32,10 +35,6 @@ Options
 
    decrease verbosity
 
-.. option:: --force-root
-
-   force to run as root
-
 .. option:: --exclude, -x
 
    Exclude the specified VM from the backup (may be repeated)
@@ -46,24 +45,12 @@ Options
 
 .. option:: --encrypt, -e
 
-   Encrypt the backup
+   Ignored, backup is always encrypted
-
-.. option:: --no-encrypt
-
-   Skip encryption even if sending the backup to a VM
 
 .. option:: --passphrase-file, -p
 
    Read passphrase from a file, or use '-' to read from stdin
 
-.. option:: --enc-algo, -E
-
-   Specify a non-default encryption algorithm. For a list of supported algorithms, execute 'openssl list-cipher-algorithms' (implies -e)
-
-.. option:: --hmac-algo, -H
-
-   Specify a non-default HMAC algorithm. For a list of supported algorithms, execute 'openssl list-message-digest-algorithms'
-
 .. option:: --compress, -z
 
    Compress the backup
@@ -72,17 +59,12 @@ Options
 
    Specify a non-default compression filter program (default: gzip)
 
-.. option:: --tmpdir
-
-   Specify a temporary directory (if you have at least 1GB free RAM in dom0, use of /tmp is advised) (default: /var/tmp)
-
 Arguments
 ---------
 
 The first positional parameter is the backup location (directory path, or
 command to pipe backup to). After that you may specify the qubes you'd like to
-backup. If not specified, all qubes with `include_in_backups` property set are
+backup. If not specified, all qubes are included.
-included.
 
 Authors
 -------

qubesadmin/tests/tools/qvm_backup.py

@@ -0,0 +1,237 @@
+# -*- encoding: utf8 -*-
+#
+# The Qubes OS Project, http://www.qubes-os.org
+#
+# Copyright (C) 2017 Marek Marczykowski-Górecki
+#                               <marmarek@invisiblethingslab.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 2.1 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with this program; if not, see <http://www.gnu.org/licenses/>.
+import io
+import os
+import unittest.mock as mock
+
+import asyncio
+
+import qubesadmin.tests
+import qubesadmin.tests.tools
+import qubesadmin.tools.qvm_backup as qvm_backup
+
+class TC_00_qvm_backup(qubesadmin.tests.QubesTestCase):
+    def test_000_write_backup_profile(self):
+        args = qvm_backup.parser.parse_args(['/var/tmp'], app=self.app)
+        profile = io.StringIO()
+        qvm_backup.write_backup_profile(profile, args)
+        expected_profile = (
+            'destination_path: /var/tmp\n'
+            'destination_vm: dom0\n'
+            'include: [\'$type:AppVM\', \'$type:TemplateVM\', '
+            '\'$type:StandaloneVM\']\n'
+        )
+        self.assertEqual(profile.getvalue(), expected_profile)
+
+    def test_001_write_backup_profile_include(self):
+        self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \
+            b'0\0dom0 class=AdminVM state=Running\n' \
+            b'vm1 class=AppVM state=Halted\n' \
+            b'vm2 class=AppVM state=Halted\n' \
+            b'vm3 class=AppVM state=Halted\n'
+        args = qvm_backup.parser.parse_args(['/var/tmp', 'vm1', 'vm2'],
+            app=self.app)
+        profile = io.StringIO()
+        qvm_backup.write_backup_profile(profile, args)
+        expected_profile = (
+            'destination_path: /var/tmp\n'
+            'destination_vm: dom0\n'
+            'include: [vm1, vm2]\n'
+        )
+        self.assertEqual(profile.getvalue(), expected_profile)
+        self.assertAllCalled()
+
+    def test_002_write_backup_profile_exclude(self):
+        args = qvm_backup.parser.parse_args([
+            '-x', 'vm1', '-x', 'vm2', '/var/tmp'], app=self.app)
+        profile = io.StringIO()
+        qvm_backup.write_backup_profile(profile, args)
+        expected_profile = (
+            'destination_path: /var/tmp\n'
+            'destination_vm: dom0\n'
+            'exclude: [vm1, vm2]\n'
+            'include: [\'$type:AppVM\', \'$type:TemplateVM\', '
+            '\'$type:StandaloneVM\']\n'
+        )
+        self.assertEqual(profile.getvalue(), expected_profile)
+
+    def test_003_write_backup_with_passphrase(self):
+        args = qvm_backup.parser.parse_args(['/var/tmp'], app=self.app)
+        profile = io.StringIO()
+        qvm_backup.write_backup_profile(profile, args, passphrase='test123')
+        expected_profile = (
+            'destination_path: /var/tmp\n'
+            'destination_vm: dom0\n'
+            'include: [\'$type:AppVM\', \'$type:TemplateVM\', '
+            '\'$type:StandaloneVM\']\n'
+            'passphrase_text: test123\n'
+        )
+        self.assertEqual(profile.getvalue(), expected_profile)
+
+    @mock.patch('qubesadmin.tools.qvm_backup.backup_profile_dir', '/tmp')
+    @mock.patch('qubesadmin.tools.qvm_backup.input', create=True)
+    @mock.patch('getpass.getpass')
+    def test_010_main_save_profile_cancel(self, mock_getpass, mock_input):
+        asyncio.set_event_loop(asyncio.new_event_loop())
+        mock_input.return_value = 'n'
+        mock_getpass.return_value = 'some password'
+        self.app.qubesd_connection_type = 'socket'
+        self.app.expected_calls[('dom0', 'admin.backup.Info', 'test-profile',
+                None)] = \
+            b'0\0backup summary'
+        profile_path = '/tmp/test-profile.conf'
+        try:
+            qvm_backup.main(['--save-profile', 'test-profile', '/var/tmp'],
+                app=self.app)
+            expected_profile = (
+                'destination_path: /var/tmp\n'
+                'destination_vm: dom0\n'
+                'include: [\'$type:AppVM\', \'$type:TemplateVM\', '
+                '\'$type:StandaloneVM\']\n'
+            )
+            with open(profile_path) as f:
+                self.assertEqual(expected_profile, f.read())
+        finally:
+            os.unlink(profile_path)
+
+    @mock.patch('qubesadmin.tools.qvm_backup.backup_profile_dir', '/tmp')
+    @mock.patch('qubesadmin.tools.qvm_backup.input', create=True)
+    @mock.patch('getpass.getpass')
+    def test_011_main_save_profile_confirm(self, mock_getpass, mock_input):
+        asyncio.set_event_loop(asyncio.new_event_loop())
+        mock_input.return_value = 'y'
+        mock_getpass.return_value = 'some password'
+        self.app.qubesd_connection_type = 'socket'
+        self.app.expected_calls[('dom0', 'admin.backup.Info', 'test-profile',
+                None)] = \
+            b'0\0backup summary'
+        self.app.expected_calls[('dom0', 'admin.backup.Execute', 'test-profile',
+                None)] = \
+            b'0\0'
+        profile_path = '/tmp/test-profile.conf'
+        try:
+            qvm_backup.main(['--save-profile', 'test-profile', '/var/tmp'],
+                app=self.app)
+            expected_profile = (
+                'destination_path: /var/tmp\n'
+                'destination_vm: dom0\n'
+                'include: [\'$type:AppVM\', \'$type:TemplateVM\', '
+                '\'$type:StandaloneVM\']\n'
+                'passphrase_text: some password\n'
+            )
+            with open(profile_path) as f:
+                self.assertEqual(expected_profile, f.read())
+        finally:
+            os.unlink(profile_path)
+
+    @mock.patch('qubesadmin.tools.qvm_backup.backup_profile_dir', '/tmp')
+    @mock.patch('qubesadmin.tools.qvm_backup.input', create=True)
+    @mock.patch('getpass.getpass')
+    def test_012_main_existing_profile(self, mock_getpass, mock_input):
+        asyncio.set_event_loop(asyncio.new_event_loop())
+        mock_input.return_value = 'y'
+        mock_getpass.return_value = 'some password'
+        self.app.qubesd_connection_type = 'socket'
+        self.app.expected_calls[('dom0', 'admin.backup.Info', 'test-profile',
+                None)] = \
+            b'0\0backup summary'
+        self.app.expected_calls[('dom0', 'admin.backup.Execute', 'test-profile',
+                None)] = \
+            b'0\0'
+        self.app.expected_calls[('dom0', 'admin.Events', None,
+                None)] = \
+            b'0\0'
+        try:
+            patch = mock.patch(
+                'qubesadmin.events.EventsDispatcher._get_events_reader')
+            mock_events = patch.start()
+            self.addCleanup(patch.stop)
+            mock_events.side_effect = qubesadmin.tests.tools.MockEventsReader([
+                b'1\0\0connection-established\0\0',
+                b'1\0\0backup-progress\0backup_profile\0test-profile\0progress\x000'
+                b'.25\0\0',
+                ])
+        except ImportError:
+            pass
+
+        qvm_backup.main(['--profile', 'test-profile'],
+            app=self.app)
+        self.assertFalse(os.path.exists('/tmp/test-profile.conf'))
+        self.assertFalse(mock_getpass.called)
+
+    @mock.patch('qubesadmin.tools.qvm_backup.backup_profile_dir', '/tmp')
+    @mock.patch('qubesadmin.tools.qvm_backup.input', create=True)
+    @mock.patch('getpass.getpass')
+    def test_013_main_new_profile_vm(self, mock_getpass, mock_input):
+        asyncio.set_event_loop(asyncio.new_event_loop())
+        mock_input.return_value = 'y'
+        mock_getpass.return_value = 'some password'
+        self.app.qubesd_connection_type = 'qrexec'
+        with qubesadmin.tests.tools.StdoutBuffer() as stdout:
+            qvm_backup.main(['-x', 'vm1', '/var/tmp'],
+                app=self.app)
+        expected_output = (
+            'To perform the backup according to selected options, create '
+            'backup profile (/tmp/profile_name.conf) in dom0 with following '
+            'content:\n'
+            'destination_path: /var/tmp\n'
+            'destination_vm: dom0\n'
+            'exclude: [vm1]\n'
+            'include: [\'$type:AppVM\', \'$type:TemplateVM\', '
+            '\'$type:StandaloneVM\']\n'
+            '# specify backup passphrase below\n'
+            'passphrase_text: ...\n'
+        )
+        self.assertEqual(stdout.getvalue(), expected_output)
+
+    @mock.patch('qubesadmin.tools.qvm_backup.backup_profile_dir', '/tmp')
+    @mock.patch('qubesadmin.tools.qvm_backup.input', create=True)
+    @mock.patch('getpass.getpass')
+    def test_014_main_passphrase_file(self, mock_getpass, mock_input):
+        asyncio.set_event_loop(asyncio.new_event_loop())
+        mock_input.return_value = 'y'
+        mock_getpass.return_value = 'some password'
+        self.app.qubesd_connection_type = 'socket'
+        self.app.expected_calls[('dom0', 'admin.backup.Info', 'test-profile',
+                None)] = \
+            b'0\0backup summary'
+        self.app.expected_calls[('dom0', 'admin.backup.Execute', 'test-profile',
+                None)] = \
+            b'0\0'
+        profile_path = '/tmp/test-profile.conf'
+        try:
+            stdin = io.StringIO()
+            stdin.write('other passphrase\n')
+            stdin.seek(0)
+            with mock.patch('sys.stdin', stdin):
+                qvm_backup.main(['--passphrase-file', '-', '--save-profile',
+                    'test-profile', '/var/tmp'],
+                    app=self.app)
+            expected_profile = (
+                'destination_path: /var/tmp\n'
+                'destination_vm: dom0\n'
+                'include: [\'$type:AppVM\', \'$type:TemplateVM\', '
+                '\'$type:StandaloneVM\']\n'
+                'passphrase_text: other passphrase\n'
+            )
+            with open(profile_path) as f:
+                self.assertEqual(expected_profile, f.read())
+        finally:
+            os.unlink(profile_path)

qubesadmin/tools/qvm_backup.py

@@ -0,0 +1,221 @@
+# -*- encoding: utf8 -*-
+#
+# The Qubes OS Project, http://www.qubes-os.org
+#
+# Copyright (C) 2017 Marek Marczykowski-Górecki
+#                               <marmarek@invisiblethingslab.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 2.1 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with this program; if not, see <http://www.gnu.org/licenses/>.
+
+'''qvm-backup tool'''
+
+import asyncio
+import functools
+import getpass
+import os
+import signal
+import sys
+import yaml
+
+try:
+    import qubesadmin.events
+    have_events = True
+except ImportError:
+    have_events = False
+import qubesadmin.tools
+from qubesadmin.exc import QubesException
+
+backup_profile_dir = '/etc/qubes/backup'
+
+parser = qubesadmin.tools.QubesArgumentParser()
+
+parser.add_argument("--yes", "-y", action="store_true",
+    dest="yes", default=False,
+    help="Do not ask for confirmation")
+
+group = parser.add_mutually_exclusive_group()
+group.add_argument('--profile', action='store',
+    help='Perform backup defined by a given profile')
+no_profile = group.add_argument_group('Profile setup',
+    'Manually specify profile options')
+no_profile.add_argument("--exclude", "-x", action="append",
+    dest="exclude_list", default=[],
+    help="Exclude the specified VM from the backup (may be "
+         "repeated)")
+no_profile.add_argument("--dest-vm", "-d", action="store",
+    dest="appvm", default=None,
+    help="Specify the destination VM to which the backup "
+         "will be sent (implies -e)")
+no_profile.add_argument("--encrypt", "-e", action="store_true",
+    dest="encrypted", default=True,
+    help="Ignored, backup is always encrypted")
+no_profile.add_argument("--passphrase-file", "-p", action="store",
+    dest="passphrase_file", default=None,
+    help="Read passphrase from a file, or use '-' to read "
+         "from stdin")
+no_profile.add_argument("--compress", "-z", action="store_true",
+    dest="compression", default=False,
+    help="Compress the backup")
+no_profile.add_argument("--compress-filter", "-Z", action="store",
+    dest="compression",
+    help="Specify a non-default compression filter program "
+         "(default: gzip)")
+no_profile.add_argument('--save-profile', action='store',
+    help='Save profile under selected name for further use.'
+         'Available only in dom0.')
+
+no_profile.add_argument("backup_location", action="store", default=None,
+    nargs='?',
+    help="Backup location (directory path, or command to pipe backup to)")
+
+no_profile.add_argument("vms", nargs="*", action=qubesadmin.tools.VmNameAction,
+    help="Backup only those VMs")
+
+
+def write_backup_profile(output_stream, args, passphrase=None):
+    '''Format backup profile and print it to *output_stream* (a file or
+    stdout)
+
+    :param output_stream: file-like object ro print the profile to
+    :param args: parsed arguments
+    :param passphrase: passphrase to use
+    '''
+
+    profile_data = {}
+    if args.vms:
+        profile_data['include'] = args.vms
+    else:
+        profile_data['include'] = [
+            '$type:AppVM', '$type:TemplateVM', '$type:StandaloneVM']
+    if args.exclude_list:
+        profile_data['exclude'] = args.exclude_list
+    if passphrase:
+        profile_data['passphrase_text'] = passphrase
+    if args.compression:
+        profile_data['compression'] = args.compression
+    if args.appvm:
+        profile_data['destination_vm'] = args.appvm
+    else:
+        profile_data['destination_vm'] = 'dom0'
+    profile_data['destination_path'] = args.backup_location
+
+    yaml.safe_dump(profile_data, output_stream)
+
+
+def print_progress(expected_profile, _subject, _event, backup_profile,
+        progress):
+    '''Event handler for reporting backup progress'''
+    if backup_profile != expected_profile:
+        return
+    sys.stderr.write('\rMaking a backup... {:.02f}%'.format(float(progress)))
+
+def main(args=None, app=None):
+    '''Main function of qvm-backup tool'''
+    args = parser.parse_args(args, app=app)
+    profile_path = None
+    if args.profile is None:
+        if args.backup_location is None:
+            parser.error('either --profile or \'backup_location\' is required')
+        if args.app.qubesd_connection_type == 'socket':
+            # when running in dom0, we can create backup profile, including
+            # passphrase
+            if args.save_profile:
+                profile_name = args.save_profile
+            else:
+                # don't care about collisions because only the user in dom0 can
+                # trigger this, and qrexec policy should not allow random VM
+                # to execute the same backup in the meantime
+                profile_name = 'backup-run-{}'.format(os.getpid())
+            # first write the backup profile without passphrase, to display
+            # summary
+            profile_path = os.path.join(
+                backup_profile_dir, profile_name + '.conf')
+            with open(profile_path, 'w') as f_profile:
+                write_backup_profile(f_profile, args)
+        else:
+            if args.save_profile:
+                parser.error(
+                    'Cannot save backup profile when running not in dom0')
+                # unreachable - parser.error terminate the process
+                return 1
+            print('To perform the backup according to selected options, '
+                'create backup profile ({}) in dom0 with following '
+                  'content:'.format(
+                    os.path.join(backup_profile_dir, 'profile_name.conf')))
+            write_backup_profile(sys.stdout, args)
+            print('# specify backup passphrase below')
+            print('passphrase_text: ...')
+            return 1
+    else:
+        profile_name = args.profile
+
+    backup_summary = args.app.qubesd_call(
+        'dom0', 'admin.backup.Info', profile_name)
+    print(backup_summary.decode())
+
+    if not args.yes:
+        if input("Do you want to proceed? [y/N] ").upper() != "Y":
+            if args.profile is None and not args.save_profile:
+                os.unlink(profile_path)
+            return 0
+
+    if args.profile is None:
+        if args.passphrase_file is not None:
+            pass_f = open(args.passphrase_file) \
+                if args.passphrase_file != "-" else sys.stdin
+            passphrase = pass_f.readline().rstrip()
+            if pass_f is not sys.stdin:
+                pass_f.close()
+        else:
+            prompt = ("Please enter the passphrase that will be used to "
+                      "encrypt and verify the backup: ")
+            passphrase = getpass.getpass(prompt)
+
+            if getpass.getpass("Enter again for verification: ") != passphrase:
+                parser.error_runtime("Passphrase mismatch!")
+
+        with open(profile_path, 'w') as f_profile:
+            write_backup_profile(f_profile, args, passphrase)
+
+    loop = asyncio.get_event_loop()
+    if have_events:
+        # pylint: disable=no-member
+        events_dispatcher = qubesadmin.events.EventsDispatcher(args.app)
+        events_dispatcher.add_handler('backup-progress',
+            functools.partial(print_progress, profile_name))
+        events_task = asyncio.ensure_future(
+            events_dispatcher.listen_for_events())
+    loop.add_signal_handler(signal.SIGINT,
+        args.app.qubesd_call, 'dom0', 'admin.backup.Cancel', profile_name)
+    try:
+        loop.run_until_complete(loop.run_in_executor(None,
+            args.app.qubesd_call, 'dom0', 'admin.backup.Execute', profile_name))
+    except QubesException as err:
+        print('\nBackup error: {}'.format(err), file=sys.stderr)
+        return 1
+    finally:
+        if have_events:
+            events_task.cancel()
+            try:
+                loop.run_until_complete(events_task)
+            except asyncio.CancelledError:
+                pass
+        loop.close()
+        if args.profile is None and not args.save_profile:
+            os.unlink(profile_path)
+        print('\n')
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())