Merge remote-tracking branch 'qubesos/pr/52'

* qubesos/pr/52:
  backup: don't crash when no 'qubes' group is present
  tests: dom0 backup restore, both v3 and v4
  backup: add support for openssl 1.1.0 options
  backup: skip dom0's properties while restoring core2 backup
  unused variable
  style issues
  Fix dom0 restore
  Fix dom0 handling
  Fix AdminVm class name

qubesadmin/backup/core2.py

@@ -256,6 +256,9 @@ class Core2Qubes(qubesadmin.backup.BackupApp):
         kwargs = {}
         if vm_class_name in ["QubesTemplateVm", "QubesTemplateHVm"]:
             vm.klass = "TemplateVM"
+        elif element.get('qid') == '0':
+            kwargs['dir_path'] = element.get('dir_path')
+            vm.klass = "AdminVM"
         elif element.get('template_qid').lower() == "none":
             kwargs['dir_path'] = element.get('dir_path')
             vm.klass = "StandaloneVM"
@@ -264,6 +267,15 @@ class Core2Qubes(qubesadmin.backup.BackupApp):
             vm.template = \
                 self.qid_map[int(element.get('template_qid'))]
             vm.klass = "AppVM"
+
+        vm.backup_content = element.get('backup_content', False) == 'True'
+        vm.backup_path = element.get('backup_path', None)
+        vm.size = element.get('backup_size', 0)
+
+        if vm.klass == 'AdminVM':
+            # don't set any other dom0 property
+            return
+
         # simple attributes
         for attr, default in {
             #'installed_by_rpm': 'False',
@@ -321,10 +333,6 @@ class Core2Qubes(qubesadmin.backup.BackupApp):
                     feature = repl_feature
             vm.features[feature] = value
 
-        vm.backup_content = element.get('backup_content', False) == 'True'
-        vm.backup_path = element.get('backup_path', None)
-        vm.size = element.get('backup_size', 0)
-
         pci_strictreset = element.get('pci_strictreset', True)
         pcidevs = element.get('pcidevs')
         if pcidevs:
@@ -348,7 +356,7 @@ class Core2Qubes(qubesadmin.backup.BackupApp):
 
         self.globals['default_kernel'] = tree.getroot().get("default_kernel")
 
-        vm_classes = ["AdminVM", "TemplateVm", "TemplateHVm",
+        vm_classes = ["AdminVm", "TemplateVm", "TemplateHVm",
             "AppVm", "HVm", "NetVm", "ProxyVm"]
 
         # First build qid->name map

qubesadmin/backup/restore.py

@@ -23,6 +23,7 @@
 import errno
 import fcntl
 import functools
+import getpass
 import grp
 import logging
 import multiprocessing
@@ -564,6 +565,10 @@ class ExtractWorker3(Process):
                     else:
                         # ignore this directory
                         tar2_cmdline = None
+                elif os.path.dirname(inner_name) == "dom0-home":
+                    tar2_cmdline = ['cat']
+                    redirect_stdout = subprocess.PIPE
+
                 elif inner_name in self.handlers:
                     tar2_cmdline = ['tar',
                         '-%svvO' % ("t" if self.verify_only else "x"),
@@ -578,14 +583,18 @@ class ExtractWorker3(Process):
                     os.remove(filename)
                     continue
 
+                tar_compress_cmd = None
                 if self.compressed:
                     if self.compression_filter:
-                        tar2_cmdline.insert(-1,
+                        tar_compress_cmd = self.compression_filter
-                                            "--use-compress-program=%s" %
+                    else:
-                                            self.compression_filter)
+                        tar_compress_cmd = DEFAULT_COMPRESSION_FILTER
+                    if os.path.dirname(inner_name) == "dom0-home":
+                        # Replaces 'cat' for compressed dom0-home!
+                        tar2_cmdline = [tar_compress_cmd, "-d"]
                     else:
                         tar2_cmdline.insert(-1, "--use-compress-program=%s " %
-                                            DEFAULT_COMPRESSION_FILTER)
+                                            tar_compress_cmd)
 
                 self.log.debug("Running command %s", str(tar2_cmdline))
                 if self.encrypted:
@@ -647,7 +656,7 @@ class ExtractWorker3(Process):
                     os.remove(filename)
                     continue
 
-                self.log.debug("Releasing next chunck")
+                self.log.debug("Releasing next chunk")
                 self.feed_tar2(filename, input_pipe)
 
             self.tar2_current_file = filename
@@ -694,8 +703,12 @@ def get_supported_hmac_algo(hmac_algorithm=None):
         yield hmac_algorithm
     if hmac_algorithm != 'scrypt':
         yield 'scrypt'
-    proc = subprocess.Popen(['openssl', 'list-message-digest-algorithms'],
+    proc = subprocess.Popen(
-                            stdout=subprocess.PIPE)
+        'openssl list-message-digest-algorithms || '
+        'openssl list -digest-algorithms',
+        shell=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.DEVNULL)
     try:
         for algo in proc.stdout.readlines():
             algo = algo.decode('ascii')
@@ -1540,7 +1553,11 @@ class BackupRestore(object):
             vm = self.backup_app.domains['dom0']
             vms_to_restore['dom0'] = self.Dom0ToRestore(vm,
                 self.backup_app.domains['dom0'].backup_path)
+            try:
                 local_user = grp.getgrnam('qubes').gr_mem[0]
+            except KeyError:
+                # if no qubes group is present, assume username matches
+                local_user = vms_to_restore['dom0'].username
 
             if vms_to_restore['dom0'].username != local_user:
                 if not self.options.ignore_username_mismatch:
@@ -1660,33 +1677,29 @@ class BackupRestore(object):
             reverse=True)
 
 
-    def _handle_dom0(self, backup_path):
+    def _handle_dom0(self, stream):
         '''Extract dom0 home'''
+        try:
             local_user = grp.getgrnam('qubes').gr_mem[0]
             home_dir = pwd.getpwnam(local_user).pw_dir
-        backup_dom0_home_dir = os.path.join(self.tmpdir, backup_path)
+        except KeyError:
-        restore_home_backupdir = "home-pre-restore-{0}".format(
+            home_dir = os.path.expanduser('~')
+            local_user = getpass.getuser()
+        restore_home_backupdir = "home-restore-{0}".format(
             time.strftime("%Y-%m-%d-%H%M%S"))
 
-        self.log.info("Restoring home of user '%s'...", local_user)
+        self.log.info("Restoring home of user '%s' to '%s' directory...",
-        self.log.info("Existing files/dirs backed up in '%s' dir",
+                     local_user, restore_home_backupdir)
-            restore_home_backupdir)
+        os.mkdir(os.path.join(home_dir, restore_home_backupdir))
-        os.mkdir(home_dir + '/' + restore_home_backupdir)
+        tar3_cmdline = ['tar', '-C',
-        for f_name in os.listdir(backup_dom0_home_dir):
+                        os.path.join(home_dir, restore_home_backupdir), '-x']
-            home_file = home_dir + '/' + f_name
+        retcode = subprocess.call(tar3_cmdline, stdin=stream)
-            if os.path.exists(home_file):
-                os.rename(home_file,
-                    home_dir + '/' + restore_home_backupdir + '/' + f_name)
-            if self.header_data.version == 1:
-                subprocess.call(
-                    ["cp", "-nrp", "--reflink=auto",
-                        backup_dom0_home_dir + '/' + f_name, home_file])
-            elif self.header_data.version >= 2:
-                shutil.move(backup_dom0_home_dir + '/' + f_name, home_file)
-        retcode = subprocess.call(['sudo', 'chown', '-R',
-            local_user, home_dir])
         if retcode != 0:
-            self.log.error("*** Error while setting home directory owner")
+            raise QubesException("Inner tar error for dom0-home")
+        retcode = subprocess.call(['sudo', 'chown', '-R',
+            local_user, os.path.join(home_dir, restore_home_backupdir)])
+        if retcode != 0:
+            self.log.error("*** Error while setting restore directory owner")
 
     def _handle_appmenus_list(self, vm, stream):
         '''Handle whitelisted-appmenus.list file'''
@@ -1827,7 +1840,8 @@ class BackupRestore(object):
             new_vm = None
             vm_name = restore_info[vm.name].name
 
-            if self.options.verify_only:
+            if self.options.verify_only or vm.name == 'dom0':
+                # can't create vm, but need backup info
                 new_vm = self.backup_app.domains[vm_name]
             else:
                 try:

qubesadmin/tests/backup/backupcompatibility.py

@@ -19,6 +19,7 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
 import functools
+import shutil
 import tempfile
 import unittest
 from distutils import spawn
@@ -29,6 +30,8 @@ import subprocess
 
 import logging
 
+import time
+
 try:
     import unittest.mock as mock
 except ImportError:
@@ -147,8 +150,8 @@ parsed_qubes_xml_r2 = {
             'tags': set(),
             'features': {},
             'template': None,
-            'backup_path': None,
+            'backup_path': 'dom0-home/user',
-            'included_in_backup': False,
+            'included_in_backup': True,
         },
         'fedora-20-x64': {
             'klass': 'TemplateVM',
@@ -426,8 +429,8 @@ parsed_qubes_xml_v4 = {
             'tags': set(),
             'features': {},
             'template': None,
-            'backup_path': None,
+            'backup_path': 'dom0-home/user',
-            'included_in_backup': False,
+            'included_in_backup': True,
         },
         'fedora-25': {
             'klass': 'TemplateVM',
@@ -967,6 +970,45 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
                 "vm-templates/test-template-clone")),
             appmenus_list)
 
+        self.create_dom0_files()
+
+    dom0_dirs = ('Downloads', 'Pictures', 'Documents', '.config', '.local')
+    dom0_files = ('.bash_history', 'some-file.txt',
+        'Pictures/another-file.png')
+
+    def create_dom0_files(self):
+        # dom0 files
+        os.mkdir(self.fullpath('dom0-home'))
+        os.mkdir(self.fullpath('dom0-home/user'))
+        for d in self.dom0_dirs:
+            os.mkdir(self.fullpath('dom0-home/user/' + d))
+        for f in self.dom0_files:
+            with open(self.fullpath('dom0-home/user/' + f), 'w') as ff:
+                ff.write('some content')
+
+    def assertDirectoryExists(self, path):
+        if not os.path.exists(path):
+            self.fail(path + ' missing')
+        if not os.path.isdir(path):
+            self.fail(path + ' is not a directory')
+
+    def assertFileExists(self, path):
+        if not os.path.exists(path):
+            self.fail(path + ' missing')
+        if not os.path.isfile(path):
+            self.fail(path + ' is not a file')
+
+    def assertDom0Restored(self, timestamp):
+        expected_dir = os.path.expanduser(
+            '~/home-restore-' + timestamp + '/dom0-home/user')
+        self.assertTrue(os.path.exists(expected_dir))
+        for d in self.dom0_dirs:
+            self.assertDirectoryExists(os.path.join(expected_dir, d))
+        for f in self.dom0_files:
+            self.assertFileExists(os.path.join(expected_dir, f))
+        # cleanup
+        shutil.rmtree(expected_dir)
+
     def create_v4_files(self):
         appmenus_list = [
             "firefox", "gnome-terminal", "evince", "evolution",
@@ -1041,6 +1083,8 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
                 "vm-templates/test-fedora-25-clone")),
             appmenus_list)
 
+        self.create_dom0_files()
+
     def scrypt_encrypt(self, f_name, output_name=None, password='qubes',
             basedir=None):
         if basedir is None:
@@ -1094,6 +1138,7 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
             data = encryptor.stdout
         else:
             data = tar.stdout
+            encryptor = None
 
         stage1_dir = self.fullpath(os.path.join("stage1", subdir))
         if not os.path.exists(stage1_dir):
@@ -1105,6 +1150,9 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
                                             os.path.basename(f_name+"."))],
                               stdin=data)
         data.close()
+        tar.wait()
+        if encryptor:
+            encryptor.wait()
 
         for part in sorted(os.listdir(stage1_dir)):
             if not re.match(
@@ -1142,6 +1190,7 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
                                             os.path.basename(f_name+"."))],
                               stdin=data)
         data.close()
+        tar.wait()
 
         for part in sorted(os.listdir(stage1_dir)):
             if not re.match(
@@ -1211,6 +1260,12 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
                 compressed=compressed,
                 encrypted=encrypted)
 
+        self.handle_v3_file(
+            'dom0-home/user',
+            'dom0-home/', output,
+            compressed=compressed,
+            encrypted=encrypted)
+
         output.close()
 
     def create_v4_backup(self, compressed=True):
@@ -1249,6 +1304,11 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
                         os.path.join(vm_dir, f_name),
                         subdir+'/', output, compressed=compressed)
 
+        self.handle_v4_file(
+            'dom0-home/user',
+            'dom0-home/', output,
+            compressed=compressed)
+
         output.close()
 
     def setup_expected_calls(self, parsed_qubes_xml, templates_map=None):
@@ -1260,6 +1320,9 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
             if not vm['included_in_backup']:
                 continue
 
+            if name == 'dom0':
+                continue
+
             if self.storage_pool:
                 self.app.expected_calls[
                     ('dom0', 'admin.vm.CreateInPool.' + vm['klass'],
@@ -1426,6 +1489,7 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
             firewall_data.encode())] = b'0\0'
         qubesd_calls_queue = multiprocessing.Queue()
 
+        dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
         patches = [
             mock.patch('qubesadmin.storage.Volume',
                 functools.partial(MockVolume, qubesd_calls_queue)),
@@ -1435,6 +1499,9 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
             mock.patch(
                 'qubesadmin.firewall.Firewall',
                 functools.partial(MockFirewall, qubesd_calls_queue)),
+            mock.patch(
+                'time.strftime',
+                return_value=dummy_timestamp)
         ]
         for patch in patches:
             patch.start()
@@ -1455,6 +1522,8 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
 
         self.assertAllCalled()
 
+        self.assertDom0Restored(dummy_timestamp)
+
     def test_220_r2_encrypted(self):
         self.create_v3_backup(True)
 
@@ -1488,6 +1557,7 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
 
         qubesd_calls_queue = multiprocessing.Queue()
 
+        dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
         patches = [
             mock.patch('qubesadmin.storage.Volume',
                 functools.partial(MockVolume, qubesd_calls_queue)),
@@ -1497,6 +1567,9 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
             mock.patch(
                 'qubesadmin.firewall.Firewall',
                 functools.partial(MockFirewall, qubesd_calls_queue)),
+            mock.patch(
+                'time.strftime',
+                return_value=dummy_timestamp)
         ]
         for patch in patches:
             patch.start()
@@ -1517,6 +1590,8 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
 
         self.assertAllCalled()
 
+        self.assertDom0Restored(dummy_timestamp)
+
     def test_230_r2_uncompressed(self):
         self.create_v3_backup(False, False)
         self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = (
@@ -1549,6 +1624,7 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
 
         qubesd_calls_queue = multiprocessing.Queue()
 
+        dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
         patches = [
             mock.patch('qubesadmin.storage.Volume',
                 functools.partial(MockVolume, qubesd_calls_queue)),
@@ -1558,6 +1634,9 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
             mock.patch(
                 'qubesadmin.firewall.Firewall',
                 functools.partial(MockFirewall, qubesd_calls_queue)),
+            mock.patch(
+                'time.strftime',
+                return_value=dummy_timestamp)
         ]
         for patch in patches:
             patch.start()
@@ -1578,6 +1657,8 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
 
         self.assertAllCalled()
 
+        self.assertDom0Restored(dummy_timestamp)
+
     @unittest.skipUnless(spawn.find_executable('scrypt'),
         "scrypt not installed")
     def test_230_r4(self):
@@ -1612,6 +1693,7 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
 
         qubesd_calls_queue = multiprocessing.Queue()
 
+        dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
         patches = [
             mock.patch('qubesadmin.storage.Volume',
                 functools.partial(MockVolume, qubesd_calls_queue)),
@@ -1621,6 +1703,9 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
             mock.patch(
                 'qubesadmin.firewall.Firewall',
                 functools.partial(MockFirewall, qubesd_calls_queue)),
+            mock.patch(
+                'time.strftime',
+                return_value=dummy_timestamp)
         ]
         for patch in patches:
             patch.start()
@@ -1641,6 +1726,8 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
 
         self.assertAllCalled()
 
+        self.assertDom0Restored(dummy_timestamp)
+
     @unittest.skipUnless(spawn.find_executable('scrypt'),
         "scrypt not installed")
     def test_230_r4_compressed(self):
@@ -1676,6 +1763,7 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
 
         qubesd_calls_queue = multiprocessing.Queue()
 
+        dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
         patches = [
             mock.patch('qubesadmin.storage.Volume',
                 functools.partial(MockVolume, qubesd_calls_queue)),
@@ -1685,6 +1773,9 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
             mock.patch(
                 'qubesadmin.firewall.Firewall',
                 functools.partial(MockFirewall, qubesd_calls_queue)),
+            mock.patch(
+                'time.strftime',
+                return_value=dummy_timestamp)
         ]
         for patch in patches:
             patch.start()
@@ -1705,6 +1796,8 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
 
         self.assertAllCalled()
 
+        self.assertDom0Restored(dummy_timestamp)
+
 
 class TC_11_BackupCompatibilityIntoLVM(TC_10_BackupCompatibility):
     storage_pool = 'some-pool'

qubesadmin/tests/backup/v3-qubes.xml

@@ -1,4 +1,5 @@
 <QubesVmCollection updatevm="3" default_kernel="3.7.6-2" default_netvm="3" default_fw_netvm="2" default_template="1" clockvm="2">
+  <QubesAdminVm autostart="False" backup_content="True" backup_path="dom0-home/user" backup_size="837570560" backup_timestamp="1470279173" conf_file="dom0.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/servicevms/dom0" dispvm_netvm="none" firewall_conf="firewall.xml" include_in_backups="True" installed_by_rpm="False" internal="False" label="black" maxmem="0" memory="300" name="dom0" netid="0" pci_e820_host="True" pci_strictreset="True" pcidevs="[]" pool_name="default" qid="0" qrexec_timeout="60" services="{'meminfo-writer': True}" template_qid="none" uses_default_dispvm_netvm="True" vcpus="0"/>
   <QubesTemplateVm installed_by_rpm="True" kernel="3.7.6-2" uses_default_kernelopts="True" qid="1" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="fedora-20-x64.conf" label="black" template_qid="none" kernelopts="" memory="400" default_user="user" netvm_qid="3" uses_default_netvm="True" volatile_img="volatile.img" services="{ 'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="fedora-20-x64" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/vm-templates/fedora-20-x64"/>
   <QubesNetVm installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="2" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="netvm.conf" label="red" template_qid="1" kernelopts="iommu=soft swiotlb=4096" memory="200" default_user="user" volatile_img="volatile.img" services="{'ntpd': False, 'meminfo-writer': False}" maxmem="1535" pcidevs="['02:00.0', '03:00.0']" name="netvm" netid="1" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/servicevms/netvm"/>
   <QubesProxyVm installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="3" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="firewallvm.conf" label="green" template_qid="1" kernelopts="" memory="200" default_user="user" netvm_qid="2" volatile_img="volatile.img" services="{'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="firewallvm" netid="2" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/servicevms/firewallvm"/>

qubesadmin/tests/backup/v4-qubes.xml

@@ -105,7 +105,11 @@
       <properties>
         <property name="label">label-8</property>
       </properties>
-      <features/>
+      <features>
+        <feature name="backup-content">True</feature>
+        <feature name="backup-path">dom0-home/user</feature>
+        <feature name="backup-size">20971520</feature>
+      </features>
       <devices class="pci"/>
       <devices class="block"/>
       <devices class="usb"/>

qubesadmin/tools/qvm_backup_restore.py

@@ -188,13 +188,12 @@ def handle_broken(app, args, restore_info):
                     "--ignore-username-mismatch to continue anyway.")
             else:
                 app.log.warning("Continuing as directed.")
-        app.log.warning("NOTE: Before restoring the dom0 home directory, "
+        app.log.warning("NOTE: The archived dom0 home directory "
-            "a new directory named "
+            "will be restored to a new directory "
-            "'home-pre-restore-<current-time>' will be "
+            "'home-restore-<current-time>' "
-            "created inside the dom0 home directory. If any "
+            "created inside the dom0 home directory. Restored "
-            "restored files conflict with existing files, "
+            "files should be copied or moved out of the new "
-            "the existing files will be moved to this new "
+            "directory before using them.")
-            "directory.")
 
 def main(args=None, app=None):
     '''Main function of qvm-backup-restore'''