Qubes/core-admin-client
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge remote-tracking branch 'qubesos/pr/52'

Browse Source 
* qubesos/pr/52:
  backup: don't crash when no 'qubes' group is present
  tests: dom0 backup restore, both v3 and v4
  backup: add support for openssl 1.1.0 options
  backup: skip dom0's properties while restoring core2 backup
  unused variable
  style issues
  Fix dom0 restore
  Fix dom0 handling
  Fix AdminVm class name
This commit is contained in:
Marek Marczykowski-Górecki 2018-02-14 04:45:12 +01:00
commit d07475427f
No known key found for this signature in database
GPG Key ID: 063938BA42CFA724
6 changed files with 169 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
qubesadmin/backup/core2.py
View File

@ -256,6 +256,9 @@ class Core2Qubes(qubesadmin.backup.BackupApp):
kwargs = {}
if vm_class_name in ["QubesTemplateVm", "QubesTemplateHVm"]:
vm.klass = "TemplateVM"
elif element.get('qid') == '0':
kwargs['dir_path'] = element.get('dir_path')
vm.klass = "AdminVM"
elif element.get('template_qid').lower() == "none":
kwargs['dir_path'] = element.get('dir_path')
vm.klass = "StandaloneVM"
@ -264,6 +267,15 @@ class Core2Qubes(qubesadmin.backup.BackupApp):
vm.template = \
self.qid_map[int(element.get('template_qid'))]
vm.klass = "AppVM"
vm.backup_content = element.get('backup_content', False) == 'True'
vm.backup_path = element.get('backup_path', None)
vm.size = element.get('backup_size', 0)
if vm.klass == 'AdminVM':
# don't set any other dom0 property
return
# simple attributes
for attr, default in {
#'installed_by_rpm': 'False',
@ -321,10 +333,6 @@ class Core2Qubes(qubesadmin.backup.BackupApp):
feature = repl_feature
vm.features[feature] = value
vm.backup_content = element.get('backup_content', False) == 'True'
vm.backup_path = element.get('backup_path', None)
vm.size = element.get('backup_size', 0)
pci_strictreset = element.get('pci_strictreset', True)
pcidevs = element.get('pcidevs')
if pcidevs:
@ -348,7 +356,7 @@ class Core2Qubes(qubesadmin.backup.BackupApp):
self.globals['default_kernel'] = tree.getroot().get("default_kernel")
vm_classes = ["AdminVM", "TemplateVm", "TemplateHVm",
vm_classes = ["AdminVm", "TemplateVm", "TemplateHVm",
"AppVm", "HVm", "NetVm", "ProxyVm"]
# First build qid->name map

80
qubesadmin/backup/restore.py
View File

@ -23,6 +23,7 @@
import errno
import fcntl
import functools
import getpass
import grp
import logging
import multiprocessing
@ -564,6 +565,10 @@ class ExtractWorker3(Process):
else:
# ignore this directory
tar2_cmdline = None
elif os.path.dirname(inner_name) == "dom0-home":
tar2_cmdline = ['cat']
redirect_stdout = subprocess.PIPE
elif inner_name in self.handlers:
tar2_cmdline = ['tar',
'-%svvO' % ("t" if self.verify_only else "x"),
@ -578,14 +583,18 @@ class ExtractWorker3(Process):
os.remove(filename)
continue
tar_compress_cmd = None
if self.compressed:
if self.compression_filter:
tar2_cmdline.insert(-1,
"--use-compress-program=%s" %
self.compression_filter)
tar_compress_cmd = self.compression_filter
else:
tar2_cmdline.insert(-1, "--use-compress-program=%s" %
DEFAULT_COMPRESSION_FILTER)
tar_compress_cmd = DEFAULT_COMPRESSION_FILTER
if os.path.dirname(inner_name) == "dom0-home":
# Replaces 'cat' for compressed dom0-home!
tar2_cmdline = [tar_compress_cmd, "-d"]
else:
tar2_cmdline.insert(-1, "--use-compress-program=%s " %
tar_compress_cmd)
self.log.debug("Running command %s", str(tar2_cmdline))
if self.encrypted:
@ -647,7 +656,7 @@ class ExtractWorker3(Process):
os.remove(filename)
continue
self.log.debug("Releasing next chunck")
self.log.debug("Releasing next chunk")
self.feed_tar2(filename, input_pipe)
self.tar2_current_file = filename
@ -694,8 +703,12 @@ def get_supported_hmac_algo(hmac_algorithm=None):
yield hmac_algorithm
if hmac_algorithm != 'scrypt':
yield 'scrypt'
proc = subprocess.Popen(['openssl', 'list-message-digest-algorithms'],
stdout=subprocess.PIPE)
proc = subprocess.Popen(
'openssl list-message-digest-algorithms || '
'openssl list -digest-algorithms',
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL)
try:
for algo in proc.stdout.readlines():
algo = algo.decode('ascii')
@ -1540,7 +1553,11 @@ class BackupRestore(object):
vm = self.backup_app.domains['dom0']
vms_to_restore['dom0'] = self.Dom0ToRestore(vm,
self.backup_app.domains['dom0'].backup_path)
local_user = grp.getgrnam('qubes').gr_mem[0]
try:
local_user = grp.getgrnam('qubes').gr_mem[0]
except KeyError:
# if no qubes group is present, assume username matches
local_user = vms_to_restore['dom0'].username
if vms_to_restore['dom0'].username != local_user:
if not self.options.ignore_username_mismatch:
@ -1660,33 +1677,29 @@ class BackupRestore(object):
reverse=True)
def _handle_dom0(self, backup_path):
def _handle_dom0(self, stream):
'''Extract dom0 home'''
local_user = grp.getgrnam('qubes').gr_mem[0]
home_dir = pwd.getpwnam(local_user).pw_dir
backup_dom0_home_dir = os.path.join(self.tmpdir, backup_path)
restore_home_backupdir = "home-pre-restore-{0}".format(
try:
local_user = grp.getgrnam('qubes').gr_mem[0]
home_dir = pwd.getpwnam(local_user).pw_dir
except KeyError:
home_dir = os.path.expanduser('~')
local_user = getpass.getuser()
restore_home_backupdir = "home-restore-{0}".format(
time.strftime("%Y-%m-%d-%H%M%S"))
self.log.info("Restoring home of user '%s'...", local_user)
self.log.info("Existing files/dirs backed up in '%s' dir",
restore_home_backupdir)
os.mkdir(home_dir + '/' + restore_home_backupdir)
for f_name in os.listdir(backup_dom0_home_dir):
home_file = home_dir + '/' + f_name
if os.path.exists(home_file):
os.rename(home_file,
home_dir + '/' + restore_home_backupdir + '/' + f_name)
if self.header_data.version == 1:
subprocess.call(
["cp", "-nrp", "--reflink=auto",
backup_dom0_home_dir + '/' + f_name, home_file])
elif self.header_data.version >= 2:
shutil.move(backup_dom0_home_dir + '/' + f_name, home_file)
retcode = subprocess.call(['sudo', 'chown', '-R',
local_user, home_dir])
self.log.info("Restoring home of user '%s' to '%s' directory...",
local_user, restore_home_backupdir)
os.mkdir(os.path.join(home_dir, restore_home_backupdir))
tar3_cmdline = ['tar', '-C',
os.path.join(home_dir, restore_home_backupdir), '-x']
retcode = subprocess.call(tar3_cmdline, stdin=stream)
if retcode != 0:
self.log.error("*** Error while setting home directory owner")
raise QubesException("Inner tar error for dom0-home")
retcode = subprocess.call(['sudo', 'chown', '-R',
local_user, os.path.join(home_dir, restore_home_backupdir)])
if retcode != 0:
self.log.error("*** Error while setting restore directory owner")
def _handle_appmenus_list(self, vm, stream):
'''Handle whitelisted-appmenus.list file'''
@ -1827,7 +1840,8 @@ class BackupRestore(object):
new_vm = None
vm_name = restore_info[vm.name].name
if self.options.verify_only:
if self.options.verify_only or vm.name == 'dom0':
# can't create vm, but need backup info
new_vm = self.backup_app.domains[vm_name]
else:
try:

101
qubesadmin/tests/backup/backupcompatibility.py
View File

@ -19,6 +19,7 @@
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
import functools
import shutil
import tempfile
import unittest
from distutils import spawn
@ -29,6 +30,8 @@ import subprocess
import logging
import time
try:
import unittest.mock as mock
except ImportError:
@ -147,8 +150,8 @@ parsed_qubes_xml_r2 = {
'tags': set(),
'features': {},
'template': None,
'backup_path': None,
'included_in_backup': False,
'backup_path': 'dom0-home/user',
'included_in_backup': True,
},
'fedora-20-x64': {
'klass': 'TemplateVM',
@ -426,8 +429,8 @@ parsed_qubes_xml_v4 = {
'tags': set(),
'features': {},
'template': None,
'backup_path': None,
'included_in_backup': False,
'backup_path': 'dom0-home/user',
'included_in_backup': True,
},
'fedora-25': {
'klass': 'TemplateVM',
@ -967,6 +970,45 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
"vm-templates/test-template-clone")),
appmenus_list)
self.create_dom0_files()
dom0_dirs = ('Downloads', 'Pictures', 'Documents', '.config', '.local')
dom0_files = ('.bash_history', 'some-file.txt',
'Pictures/another-file.png')
def create_dom0_files(self):
# dom0 files
os.mkdir(self.fullpath('dom0-home'))
os.mkdir(self.fullpath('dom0-home/user'))
for d in self.dom0_dirs:
os.mkdir(self.fullpath('dom0-home/user/' + d))
for f in self.dom0_files:
with open(self.fullpath('dom0-home/user/' + f), 'w') as ff:
ff.write('some content')
def assertDirectoryExists(self, path):
if not os.path.exists(path):
self.fail(path + ' missing')
if not os.path.isdir(path):
self.fail(path + ' is not a directory')
def assertFileExists(self, path):
if not os.path.exists(path):
self.fail(path + ' missing')
if not os.path.isfile(path):
self.fail(path + ' is not a file')
def assertDom0Restored(self, timestamp):
expected_dir = os.path.expanduser(
'~/home-restore-' + timestamp + '/dom0-home/user')
self.assertTrue(os.path.exists(expected_dir))
for d in self.dom0_dirs:
self.assertDirectoryExists(os.path.join(expected_dir, d))
for f in self.dom0_files:
self.assertFileExists(os.path.join(expected_dir, f))
# cleanup
shutil.rmtree(expected_dir)
def create_v4_files(self):
appmenus_list = [
"firefox", "gnome-terminal", "evince", "evolution",
@ -1041,6 +1083,8 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
"vm-templates/test-fedora-25-clone")),
appmenus_list)
self.create_dom0_files()
def scrypt_encrypt(self, f_name, output_name=None, password='qubes',
basedir=None):
if basedir is None:
@ -1094,6 +1138,7 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
data = encryptor.stdout
else:
data = tar.stdout
encryptor = None
stage1_dir = self.fullpath(os.path.join("stage1", subdir))
if not os.path.exists(stage1_dir):
@ -1105,6 +1150,9 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
os.path.basename(f_name+"."))],
stdin=data)
data.close()
tar.wait()
if encryptor:
encryptor.wait()
for part in sorted(os.listdir(stage1_dir)):
if not re.match(
@ -1142,6 +1190,7 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
os.path.basename(f_name+"."))],
stdin=data)
data.close()
tar.wait()
for part in sorted(os.listdir(stage1_dir)):
if not re.match(
@ -1211,6 +1260,12 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
compressed=compressed,
encrypted=encrypted)
self.handle_v3_file(
'dom0-home/user',
'dom0-home/', output,
compressed=compressed,
encrypted=encrypted)
output.close()
def create_v4_backup(self, compressed=True):
@ -1249,6 +1304,11 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
os.path.join(vm_dir, f_name),
subdir+'/', output, compressed=compressed)
self.handle_v4_file(
'dom0-home/user',
'dom0-home/', output,
compressed=compressed)
output.close()
def setup_expected_calls(self, parsed_qubes_xml, templates_map=None):
@ -1260,6 +1320,9 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
if not vm['included_in_backup']:
continue
if name == 'dom0':
continue
if self.storage_pool:
self.app.expected_calls[
('dom0', 'admin.vm.CreateInPool.' + vm['klass'],
@ -1426,6 +1489,7 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
firewall_data.encode())] = b'0\0'
qubesd_calls_queue = multiprocessing.Queue()
dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
patches = [
mock.patch('qubesadmin.storage.Volume',
functools.partial(MockVolume, qubesd_calls_queue)),
@ -1435,6 +1499,9 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
mock.patch(
'qubesadmin.firewall.Firewall',
functools.partial(MockFirewall, qubesd_calls_queue)),
mock.patch(
'time.strftime',
return_value=dummy_timestamp)
]
for patch in patches:
patch.start()
@ -1455,6 +1522,8 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
self.assertAllCalled()
self.assertDom0Restored(dummy_timestamp)
def test_220_r2_encrypted(self):
self.create_v3_backup(True)
@ -1488,6 +1557,7 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
qubesd_calls_queue = multiprocessing.Queue()
dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
patches = [
mock.patch('qubesadmin.storage.Volume',
functools.partial(MockVolume, qubesd_calls_queue)),
@ -1497,6 +1567,9 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
mock.patch(
'qubesadmin.firewall.Firewall',
functools.partial(MockFirewall, qubesd_calls_queue)),
mock.patch(
'time.strftime',
return_value=dummy_timestamp)
]
for patch in patches:
patch.start()
@ -1517,6 +1590,8 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
self.assertAllCalled()
self.assertDom0Restored(dummy_timestamp)
def test_230_r2_uncompressed(self):
self.create_v3_backup(False, False)
self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = (
@ -1549,6 +1624,7 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
qubesd_calls_queue = multiprocessing.Queue()
dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
patches = [
mock.patch('qubesadmin.storage.Volume',
functools.partial(MockVolume, qubesd_calls_queue)),
@ -1558,6 +1634,9 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
mock.patch(
'qubesadmin.firewall.Firewall',
functools.partial(MockFirewall, qubesd_calls_queue)),
mock.patch(
'time.strftime',
return_value=dummy_timestamp)
]
for patch in patches:
patch.start()
@ -1578,6 +1657,8 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
self.assertAllCalled()
self.assertDom0Restored(dummy_timestamp)
@unittest.skipUnless(spawn.find_executable('scrypt'),
"scrypt not installed")
def test_230_r4(self):
@ -1612,6 +1693,7 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
qubesd_calls_queue = multiprocessing.Queue()
dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
patches = [
mock.patch('qubesadmin.storage.Volume',
functools.partial(MockVolume, qubesd_calls_queue)),
@ -1621,6 +1703,9 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
mock.patch(
'qubesadmin.firewall.Firewall',
functools.partial(MockFirewall, qubesd_calls_queue)),
mock.patch(
'time.strftime',
return_value=dummy_timestamp)
]
for patch in patches:
patch.start()
@ -1641,6 +1726,8 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
self.assertAllCalled()
self.assertDom0Restored(dummy_timestamp)
@unittest.skipUnless(spawn.find_executable('scrypt'),
"scrypt not installed")
def test_230_r4_compressed(self):
@ -1676,6 +1763,7 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
qubesd_calls_queue = multiprocessing.Queue()
dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
patches = [
mock.patch('qubesadmin.storage.Volume',
functools.partial(MockVolume, qubesd_calls_queue)),
@ -1685,6 +1773,9 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
mock.patch(
'qubesadmin.firewall.Firewall',
functools.partial(MockFirewall, qubesd_calls_queue)),
mock.patch(
'time.strftime',
return_value=dummy_timestamp)
]
for patch in patches:
patch.start()
@ -1705,6 +1796,8 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
self.assertAllCalled()
self.assertDom0Restored(dummy_timestamp)
class TC_11_BackupCompatibilityIntoLVM(TC_10_BackupCompatibility):
storage_pool = 'some-pool'

1
qubesadmin/tests/backup/v3-qubes.xml
View File

@ -1,4 +1,5 @@
<QubesVmCollection updatevm="3" default_kernel="3.7.6-2" default_netvm="3" default_fw_netvm="2" default_template="1" clockvm="2">
<QubesAdminVm autostart="False" backup_content="True" backup_path="dom0-home/user" backup_size="837570560" backup_timestamp="1470279173" conf_file="dom0.conf" debug="False" default_user="user" dir_path="/var/lib/qubes/servicevms/dom0" dispvm_netvm="none" firewall_conf="firewall.xml" include_in_backups="True" installed_by_rpm="False" internal="False" label="black" maxmem="0" memory="300" name="dom0" netid="0" pci_e820_host="True" pci_strictreset="True" pcidevs="[]" pool_name="default" qid="0" qrexec_timeout="60" services="{'meminfo-writer': True}" template_qid="none" uses_default_dispvm_netvm="True" vcpus="0"/>
<QubesTemplateVm installed_by_rpm="True" kernel="3.7.6-2" uses_default_kernelopts="True" qid="1" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="fedora-20-x64.conf" label="black" template_qid="none" kernelopts="" memory="400" default_user="user" netvm_qid="3" uses_default_netvm="True" volatile_img="volatile.img" services="{ 'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="fedora-20-x64" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/vm-templates/fedora-20-x64"/>
<QubesNetVm installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="2" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="netvm.conf" label="red" template_qid="1" kernelopts="iommu=soft swiotlb=4096" memory="200" default_user="user" volatile_img="volatile.img" services="{'ntpd': False, 'meminfo-writer': False}" maxmem="1535" pcidevs="['02:00.0', '03:00.0']" name="netvm" netid="1" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/servicevms/netvm"/>
<QubesProxyVm installed_by_rpm="False" kernel="3.7.6-2" uses_default_kernelopts="True" qid="3" include_in_backups="True" uses_default_kernel="True" qrexec_timeout="60" internal="False" conf_file="firewallvm.conf" label="green" template_qid="1" kernelopts="" memory="200" default_user="user" netvm_qid="2" volatile_img="volatile.img" services="{'meminfo-writer': True}" maxmem="1535" pcidevs="[]" name="firewallvm" netid="2" private_img="private.img" vcpus="2" root_img="root.img" debug="False" dir_path="/var/lib/qubes/servicevms/firewallvm"/>

6
qubesadmin/tests/backup/v4-qubes.xml
View File

@ -105,7 +105,11 @@
<properties>
<property name="label">label-8</property>
</properties>
<features/>
<features>
<feature name="backup-content">True</feature>
<feature name="backup-path">dom0-home/user</feature>
<feature name="backup-size">20971520</feature>
</features>
<devices class="pci"/>
<devices class="block"/>
<devices class="usb"/>

13
qubesadmin/tools/qvm_backup_restore.py
View File

@ -188,13 +188,12 @@ def handle_broken(app, args, restore_info):
"--ignore-username-mismatch to continue anyway.")
else:
app.log.warning("Continuing as directed.")
app.log.warning("NOTE: Before restoring the dom0 home directory, "
"a new directory named "
"'home-pre-restore-<current-time>' will be "
"created inside the dom0 home directory. If any "
"restored files conflict with existing files, "
"the existing files will be moved to this new "
"directory.")
app.log.warning("NOTE: The archived dom0 home directory "
"will be restored to a new directory "
"'home-restore-<current-time>' "
"created inside the dom0 home directory. Restored "
"files should be copied or moved out of the new "
"directory before using them.")
def main(args=None, app=None):
'''Main function of qvm-backup-restore'''