Qubes/core-admin
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

backup: initial conversion to core3 API

Browse Source
This commit is contained in:
Marek Marczykowski-Górecki 2016-03-10 11:22:52 +01:00 committed by Wojtek Porczyk
parent 45d6ab3862
commit e0686e1e02
5 changed files with 230 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

248
core/backup.py → qubes/backup.py
View File

@ -23,10 +23,7 @@
#
#
from __future__ import unicode_literals
from qubes import QubesException, QubesVmCollection
from qubes import QubesVmClasses
from qubes import system_path, vm_files
from qubesutils import size_to_human, print_stdout, print_stderr, get_disk_usage
from qubes.utils import size_to_human
import sys
import os
import fcntl
@ -40,6 +37,7 @@ import pwd
import errno
import datetime
from multiprocessing import Queue, Process
import qubes
BACKUP_DEBUG = False
@ -53,9 +51,37 @@ MAX_STDERR_BYTES = 1024
# header + qubes.xml max size
HEADER_QUBES_XML_MAX_SIZE = 1024 * 1024
BLKSIZE = 512
# global state for backup_cancel()
running_backup_operation = None
def print_stdout(text):
print (text)
def print_stderr(text):
print >> sys.stderr, (text)
def get_disk_usage_one(st):
try:
return st.st_blocks * BLKSIZE
except AttributeError:
return st.st_size
def get_disk_usage(path):
try:
st = os.lstat(path)
except OSError:
return 0
ret = get_disk_usage_one(st)
# if path is not a directory, this is skipped
for dirpath, dirnames, filenames in os.walk(path):
for name in dirnames + filenames:
ret += get_disk_usage_one(os.lstat(os.path.join(dirpath, name)))
return ret
class BackupOperationInfo:
def __init__(self):
@ -64,7 +90,7 @@ class BackupOperationInfo:
self.tmpdir_to_remove = None
class BackupCanceledError(QubesException):
class BackupCanceledError(qubes.exc.QubesException):
def __init__(self, msg, tmpdir=None):
super(BackupCanceledError, self).__init__(msg)
self.tmpdir = tmpdir
@ -86,7 +112,7 @@ def file_to_backup(file_path, subdir=None):
if subdir is None:
abs_file_path = os.path.abspath(file_path)
abs_base_dir = os.path.abspath(system_path["qubes_base_dir"]) + '/'
abs_base_dir = os.path.abspath(qubes.config.system_path["qubes_base_dir"]) + '/'
abs_file_dir = os.path.dirname(abs_file_path) + '/'
(nothing, directory, subdir) = abs_file_dir.partition(abs_base_dir)
assert nothing == ""
@ -115,33 +141,35 @@ def backup_cancel():
return True
def backup_prepare(vms_list=None, exclude_list=None,
def backup_prepare(app, vms_list=None, exclude_list=None,
print_callback=print_stdout, hide_vm_names=True):
"""
If vms = None, include all (sensible) VMs;
exclude_list is always applied
"""
files_to_backup = file_to_backup(system_path["qubes_store_filename"])
# XXX hack for tests, where store filename is qubes-test.xml
qubes_xml = app._store
if os.path.basename(qubes_xml) != 'qubes.xml':
dir = tempfile.mkdtemp()
shutil.copy(qubes_xml, os.path.join(dir, 'qubes.xml'))
qubes_xml = os.path.join(dir, 'qubes.xml')
# FIXME cleanup tempdir later
files_to_backup = file_to_backup(qubes_xml, '')
if exclude_list is None:
exclude_list = []
qvm_collection = QubesVmCollection()
qvm_collection.lock_db_for_writing()
qvm_collection.load()
if vms_list is None:
all_vms = [vm for vm in qvm_collection.values()]
all_vms = [vm for vm in app.domains]
selected_vms = [vm for vm in all_vms if vm.include_in_backups]
appvms_to_backup = [vm for vm in selected_vms if
vm.is_appvm() and not vm.internal]
netvms_to_backup = [vm for vm in selected_vms if
vm.is_netvm() and not vm.qid == 0]
template_vms_worth_backingup = [vm for vm in selected_vms if (
vm.is_template() and vm.include_in_backups)]
dom0 = [qvm_collection[0]]
dom0 = [app.domains[0]]
vms_list = appvms_to_backup + netvms_to_backup + \
vms_list = appvms_to_backup + \
template_vms_worth_backingup + dom0
vms_for_backup = vms_list
@ -207,11 +235,13 @@ def backup_prepare(vms_list=None, exclude_list=None,
subdir)
if os.path.exists(vm.firewall_conf):
files_to_backup += file_to_backup(vm.firewall_conf, subdir)
if 'appmenus_whitelist' in vm_files and \
if 'appmenus_whitelist' in qubes.config.vm_files and \
os.path.exists(os.path.join(vm.dir_path,
vm_files['appmenus_whitelist'])):
qubes.config.vm_files[
'appmenus_whitelist'])):
files_to_backup += file_to_backup(
os.path.join(vm.dir_path, vm_files['appmenus_whitelist']),
os.path.join(vm.dir_path, qubes.config.vm_files[
'appmenus_whitelist']),
subdir)
if vm.updateable:
@ -255,7 +285,7 @@ def backup_prepare(vms_list=None, exclude_list=None,
else:
template_subdir = os.path.relpath(
vm.dir_path,
system_path["qubes_base_dir"]) + '/'
qubes.config.system_path["qubes_base_dir"]) + '/'
template_to_backup = [{"path": vm.dir_path + '/.',
"size": vm_sz,
"subdir": template_subdir}]
@ -280,7 +310,7 @@ def backup_prepare(vms_list=None, exclude_list=None,
# Initialize backup flag on all VMs
vms_for_backup_qid = [vm.qid for vm in vms_for_backup]
for vm in qvm_collection.values():
for vm in app.domains:
vm.backup_content = False
if vm.qid == 0:
# handle dom0 later
@ -292,8 +322,9 @@ def backup_prepare(vms_list=None, exclude_list=None,
if hide_vm_names:
vm.backup_path = 'vm%d' % vm.qid
else:
vm.backup_path = os.path.relpath(vm.dir_path,
system_path["qubes_base_dir"])
vm.backup_path = os.path.relpath(
vm.dir_path,
qubes.config.system_path["qubes_base_dir"])
# Dom0 user home
if 0 in vms_for_backup_qid:
@ -309,7 +340,7 @@ def backup_prepare(vms_list=None, exclude_list=None,
{"path": home_dir, "size": home_sz, "subdir": 'dom0-home/'}]
files_to_backup += home_to_backup
vm = qvm_collection[0]
vm = app.domains[0]
vm.backup_content = True
vm.backup_size = home_sz
vm.backup_path = os.path.join('dom0-home', os.path.basename(home_dir))
@ -326,9 +357,8 @@ def backup_prepare(vms_list=None, exclude_list=None,
print_callback(s)
qvm_collection.save()
app.save()
# FIXME: should be after backup completed
qvm_collection.unlock_db()
total_backup_sz = 0
for f in files_to_backup:
@ -355,13 +385,13 @@ def backup_prepare(vms_list=None, exclude_list=None,
s += fmt.format('-')
print_callback(s)
vms_not_for_backup = [vm.name for vm in qvm_collection.values()
vms_not_for_backup = [vm.name for vm in app.domains
if not vm.backup_content]
print_callback("VMs not selected for backup: %s" % " ".join(
vms_not_for_backup))
if there_are_running_vms:
raise QubesException("Please shutdown all VMs before proceeding.")
raise qubes.exc.QubesException("Please shutdown all VMs before proceeding.")
for fileinfo in files_to_backup:
assert len(fileinfo["subdir"]) == 0 or fileinfo["subdir"][-1] == '/', \
@ -406,7 +436,7 @@ class SendWorker(Process):
self.queue.get()
# handle only exit code 2 (tar fatal error) or
# greater (call failed?)
raise QubesException(
raise qubes.exc.QubesException(
"ERROR: Failed to write the backup, out of disk space? "
"Check console output or ~/.xsession-errors for details.")
@ -442,11 +472,11 @@ def prepare_backup_header(target_directory, passphrase, compressed=False,
stdin=open(header_file_path, "r"),
stdout=open(header_file_path + ".hmac", "w"))
if hmac.wait() != 0:
raise QubesException("Failed to compute hmac of header file")
raise qubes.exc.QubesException("Failed to compute hmac of header file")
return HEADER_FILENAME, HEADER_FILENAME + ".hmac"
def backup_do(base_backup_dir, files_to_backup, passphrase,
def backup_do(app, base_backup_dir, files_to_backup, passphrase,
progress_callback=None, encrypted=False, appvm=None,
compressed=False, hmac_algorithm=DEFAULT_HMAC_ALGORITHM,
crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM):
@ -460,7 +490,7 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
vmproc.stderr.read())
else:
message = "Failed to write the backup. Out of disk space?"
raise QubesException(message)
raise qubes.exc.QubesException(message)
queue.put(element)
total_backup_sz = 0
@ -497,7 +527,7 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
# Create the target directory
if not os.path.exists(os.path.dirname(base_backup_dir)):
raise QubesException(
raise qubes.exc.QubesException(
"ERROR: the backup directory for {0} does not exists".
format(base_backup_dir))
@ -658,11 +688,11 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
if run_error and run_error != "size_limit":
send_proc.terminate()
if run_error == "VM" and vmproc:
raise QubesException(
raise qubes.exc.QubesException(
"Failed to write the backup, VM output:\n" +
vmproc.stderr.read(MAX_STDERR_BYTES))
else:
raise QubesException("Failed to perform backup: error in " +
raise qubes.exc.QubesException("Failed to perform backup: error in " +
run_error)
# Send the chunk to the backup target
@ -711,7 +741,7 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
running_backup_operation = None
if send_proc.exitcode != 0:
raise QubesException(
raise qubes.exc.QubesException(
"Failed to send backup: error in the sending process")
if vmproc:
@ -722,16 +752,11 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
vmproc.stdin.close()
# Save date of last backup
qvm_collection = QubesVmCollection()
qvm_collection.lock_db_for_writing()
qvm_collection.load()
for vm in qvm_collection.values():
for vm in app.domains:
if vm.backup_content:
vm.backup_timestamp = datetime.datetime.now()
qvm_collection.save()
qvm_collection.unlock_db()
app.save()
'''
@ -829,7 +854,7 @@ def verify_hmac(filename, hmacfile, passphrase, algorithm):
print "Verifying file " + filename
if hmacfile != filename + ".hmac":
raise QubesException(
raise qubes.exc.QubesException(
"ERROR: expected hmac for {}, but got {}".
format(filename, hmacfile))
@ -840,7 +865,7 @@ def verify_hmac(filename, hmacfile, passphrase, algorithm):
hmac_stdout, hmac_stderr = hmac_proc.communicate()
if len(hmac_stderr) > 0:
raise QubesException(
raise qubes.exc.QubesException(
"ERROR: verify file {0}: {1}".format(filename, hmac_stderr))
else:
if BACKUP_DEBUG:
@ -853,7 +878,7 @@ def verify_hmac(filename, hmacfile, passphrase, algorithm):
print "File verification OK -> Sending file " + filename
return True
else:
raise QubesException(
raise qubes.exc.QubesException(
"ERROR: invalid hmac for file {0}: {1}. "
"Is the passphrase correct?".
format(filename, load_hmac(hmac_stdout)))
@ -1081,7 +1106,7 @@ class ExtractWorker2(Process):
self.tar2_process.wait()
elif self.tar2_process.wait() != 0:
self.collect_tar_output()
raise QubesException(
raise qubes.exc.QubesException(
"unable to extract files for {0}.{1} Tar command "
"output: %s".
format(self.tar2_current_file,
@ -1241,7 +1266,7 @@ class ExtractWorker3(ExtractWorker2):
self.tar2_process.wait()
elif self.tar2_process.wait() != 0:
self.collect_tar_output()
raise QubesException(
raise qubes.exc.QubesException(
"unable to extract files for {0}.{1} Tar command "
"output: %s".
format(self.tar2_current_file,
@ -1274,7 +1299,7 @@ def parse_backup_header(filename):
with open(filename, 'r') as f:
for line in f.readlines():
if line.count('=') != 1:
raise QubesException("Invalid backup header (line %s)" % line)
raise qubes.exc.QubesException("Invalid backup header (line %s)" % line)
(key, value) = line.strip().split('=')
if not any([key == getattr(BackupHeader, attr) for attr in dir(
BackupHeader)]):
@ -1394,7 +1419,7 @@ def restore_vm_dirs(backup_source, restore_tmpdir, passphrase, vms_dirs, vms,
else:
command.wait()
proc_error_msg = command.stderr.read(MAX_STDERR_BYTES)
raise QubesException("Premature end of archive while receiving "
raise qubes.exc.QubesException("Premature end of archive while receiving "
"backup header. Process output:\n" +
proc_error_msg)
filename = os.path.join(restore_tmpdir, filename)
@ -1406,11 +1431,11 @@ def restore_vm_dirs(backup_source, restore_tmpdir, passphrase, vms_dirs, vms,
file_ok = True
hmac_algorithm = hmac_algo
break
except QubesException:
except qubes.exc.QubesException:
# Ignore exception here, try the next algo
pass
if not file_ok:
raise QubesException("Corrupted backup header (hmac verification "
raise qubes.exc.QubesException("Corrupted backup header (hmac verification "
"failed). Is the password correct?")
if os.path.basename(filename) == HEADER_FILENAME:
header_data = parse_backup_header(filename)
@ -1524,18 +1549,18 @@ def restore_vm_dirs(backup_source, restore_tmpdir, passphrase, vms_dirs, vms,
tmpdir=restore_tmpdir)
if command.wait() != 0 and not expect_tar_error:
raise QubesException(
raise qubes.exc.QubesException(
"unable to read the qubes backup file {0} ({1}). "
"Is it really a backup?".format(backup_source, command.wait()))
if vmproc:
if vmproc.wait() != 0:
raise QubesException(
raise qubes.exc.QubesException(
"unable to read the qubes backup {0} "
"because of a VM error: {1}".format(
backup_source, vmproc.stderr.read(MAX_STDERR_BYTES)))
if filename and filename != "EOF":
raise QubesException(
raise qubes.exc.QubesException(
"Premature end of archive, the last file was %s" % filename)
except:
to_extract.put("ERROR")
@ -1551,7 +1576,7 @@ def restore_vm_dirs(backup_source, restore_tmpdir, passphrase, vms_dirs, vms,
print_callback("Extraction process finished with code:" +
str(extract_proc.exitcode))
if extract_proc.exitcode != 0:
raise QubesException(
raise qubes.exc.QubesException(
"unable to extract the qubes backup. "
"Check extracting process errors.")
@ -1584,7 +1609,7 @@ def load_hmac(hmac):
if len(hmac) > 1:
hmac = hmac[1].strip()
else:
raise QubesException("ERROR: invalid hmac file content")
raise qubes.exc.QubesException("ERROR: invalid hmac file content")
return hmac
@ -1642,6 +1667,7 @@ def backup_restore_header(source, passphrase,
return (restore_tmpdir, os.path.join(restore_tmpdir, "qubes.xml"),
header_data)
def generate_new_name_for_conflicting_vm(orig_name, host_collection,
restore_info):
number = 1
@ -1651,7 +1677,7 @@ def generate_new_name_for_conflicting_vm(orig_name, host_collection,
while (new_name in restore_info.keys() or
new_name in map(lambda x: x.get('rename_to', None),
restore_info.values()) or
host_collection.get_vm_by_name(new_name)):
new_name in host_collection.domains):
new_name = str('{}{}'.format(orig_name, number))
number += 1
if number == 100:
@ -1660,6 +1686,7 @@ def generate_new_name_for_conflicting_vm(orig_name, host_collection,
return new_name
def restore_info_verify(restore_info, host_collection):
assert isinstance(host_collection, qubes.Qubes)
options = restore_info['$OPTIONS$']
for vm in restore_info.keys():
if vm in ['$OPTIONS$', 'dom0']:
@ -1674,7 +1701,7 @@ def restore_info_verify(restore_info, host_collection):
vm_info.pop('already-exists', None)
if not options['verify-only'] and \
host_collection.get_vm_by_name(vm) is not None:
vm in host_collection.domains:
if options['rename-conflicting']:
new_name = generate_new_name_for_conflicting_vm(
vm, host_collection, restore_info
@ -1690,7 +1717,7 @@ def restore_info_verify(restore_info, host_collection):
vm_info.pop('missing-template', None)
if vm_info['template']:
template_name = vm_info['template']
host_template = host_collection.get_vm_by_name(template_name)
host_template = host_collection.domains.get(template_name, None)
if not host_template or not host_template.is_template():
# Maybe the (custom) template is in the backup?
if not (template_name in restore_info.keys() and
@ -1699,7 +1726,7 @@ def restore_info_verify(restore_info, host_collection):
if 'orig-template' not in vm_info.keys():
vm_info['orig-template'] = template_name
vm_info['template'] = host_collection \
.get_default_template().name
.default_template.name
else:
vm_info['missing-template'] = True
@ -1708,7 +1735,7 @@ def restore_info_verify(restore_info, host_collection):
if vm_info['netvm']:
netvm_name = vm_info['netvm']
netvm_on_host = host_collection.get_vm_by_name(netvm_name)
netvm_on_host = host_collection.domains.get(netvm_name, None)
# No netvm on the host?
if not ((netvm_on_host is not None) and netvm_on_host.is_netvm()):
@ -1718,7 +1745,7 @@ def restore_info_verify(restore_info, host_collection):
restore_info[netvm_name]['vm'].is_netvm()):
if options['use-default-netvm']:
vm_info['netvm'] = host_collection \
.get_default_netvm().name
.default_netvm.name
vm_info['vm'].uses_default_netvm = True
elif options['use-none-netvm']:
vm_info['netvm'] = None
@ -1775,7 +1802,7 @@ def backup_restore_prepare(backup_location, passphrase, options=None,
return False
backup_vm_dir_path = check_vm.dir_path.replace(
system_path["qubes_base_dir"], backup_dir)
qubes.config.system_path["qubes_base_dir"], backup_dir)
if os.path.exists(backup_vm_dir_path):
return True
@ -1812,11 +1839,11 @@ def backup_restore_prepare(backup_location, passphrase, options=None,
is_vm_included_in_backup = is_vm_included_in_backup_v2
if not appvm:
if not os.path.isfile(backup_location):
raise QubesException("Invalid backup location (not a file or "
raise qubes.exc.QubesException("Invalid backup location (not a file or "
"directory with qubes.xml)"
": %s" % unicode(backup_location))
else:
raise QubesException(
raise qubes.exc.QubesException(
"Unknown backup format version: %s" % str(format_version))
(restore_tmpdir, qubes_xml, header_data) = backup_restore_header(
@ -1847,17 +1874,12 @@ def backup_restore_prepare(backup_location, passphrase, options=None,
if BACKUP_DEBUG:
print "Loading file", qubes_xml
backup_collection = QubesVmCollection(store_filename=qubes_xml)
backup_collection.lock_db_for_reading()
backup_collection.load()
backup_collection = qubes.Qubes(qubes_xml)
if host_collection is None:
host_collection = QubesVmCollection()
host_collection.lock_db_for_reading()
host_collection.load()
host_collection.unlock_db()
host_collection = qubes.Qubes()
backup_vms_list = [vm for vm in backup_collection.values()]
backup_vms_list = [vm for vm in backup_collection.domains]
vms_to_restore = {}
# ... and the actual data
@ -1907,8 +1929,9 @@ def backup_restore_prepare(backup_location, passphrase, options=None,
# ...and dom0 home
if options['dom0-home'] and \
is_vm_included_in_backup(backup_location, backup_collection[0]):
vm = backup_collection[0]
is_vm_included_in_backup(backup_location,
backup_collection.domains[0]):
vm = backup_collection.domains[0]
vms_to_restore['dom0'] = {}
if format_version == 1:
vms_to_restore['dom0']['subdir'] = \
@ -2054,13 +2077,13 @@ def backup_restore_do(restore_info,
# Private functions begin
def restore_vm_dir_v1(backup_dir, src_dir, dst_dir):
backup_src_dir = src_dir.replace(system_path["qubes_base_dir"],
backup_dir)
backup_src_dir = src_dir.replace(
qubes.config.system_path["qubes_base_dir"], backup_dir)
# We prefer to use Linux's cp, because it nicely handles sparse files
cp_retcode = subprocess.call(["cp", "-rp", "--reflink=auto", backup_src_dir, dst_dir])
if cp_retcode != 0:
raise QubesException(
raise qubes.exc.QubesException(
"*** Error while copying file {0} to {1}".format(backup_src_dir,
dst_dir))
@ -2084,9 +2107,8 @@ def backup_restore_do(restore_info,
lock_obtained = False
if host_collection is None:
host_collection = QubesVmCollection()
host_collection.lock_db_for_writing()
host_collection.load()
host_collection = qubes.Qubes()
# FIXME
lock_obtained = True
# Perform VM restoration in backup order
@ -2129,7 +2151,7 @@ def backup_restore_do(restore_info,
compressed=compressed,
compression_filter=compression_filter,
appvm=appvm)
except QubesException:
except qubes.exc.QubesException:
if verify_only:
raise
else:
@ -2148,9 +2170,8 @@ def backup_restore_do(restore_info,
shutil.rmtree(restore_tmpdir)
return
# Add VM in right order
for (vm_class_name, vm_class) in sorted(QubesVmClasses.items(),
key=lambda _x: _x[1].load_order):
# First load templates, then other VMs
for do_templates in (True, False):
if running_backup_operation.canceled:
break
for vm in vms.values():
@ -2158,11 +2179,10 @@ def backup_restore_do(restore_info,
# only break the loop to save qubes.xml with already restored
# VMs
break
if not vm.__class__ == vm_class:
if vm.is_template != do_templates:
continue
if callable(print_callback):
print_callback("-> Restoring {type} {0}...".
format(vm.name, type=vm_class_name))
print_callback("-> Restoring {0}...".format(vm.name))
retcode = subprocess.call(
["mkdir", "-p", os.path.dirname(vm.dir_path)])
if retcode != 0:
@ -2171,10 +2191,12 @@ def backup_restore_do(restore_info,
error_callback("Skipping...")
continue
template = None
kwargs = {}
if hasattr(vm, 'template'):
if vm.template is not None:
template_name = restore_info[vm.name]['template']
template = host_collection.get_vm_by_name(template_name)
kwargs['template'] = restore_info[vm.name]['template']
else:
kwargs['template'] = None
new_vm = None
vm_name = vm.name
@ -2182,9 +2204,13 @@ def backup_restore_do(restore_info,
vm_name = restore_info[vm.name]['rename-to']
try:
new_vm = host_collection.add_new_vm(vm_class_name, name=vm_name,
template=template,
installed_by_rpm=False)
# first only minimal set, later clone_properties will be called
new_vm = host_collection.add_new_vm(
vm.__class__,
name=vm_name,
label=vm.label,
installed_by_rpm=False,
**kwargs)
if os.path.exists(new_vm.dir_path):
move_to_path = tempfile.mkdtemp('', os.path.basename(
new_vm.dir_path), os.path.dirname(new_vm.dir_path))
@ -2214,20 +2240,19 @@ def backup_restore_do(restore_info,
error_callback("ERROR: {0}".format(err))
error_callback("*** Skipping VM: {0}".format(vm.name))
if new_vm:
host_collection.pop(new_vm.qid)
del host_collection.domains[new_vm.qid]
continue
# FIXME: cannot check for 'kernel' property, because it is always
# defined - accessing it touches non-existent '_kernel'
if not isinstance(vm, QubesVmClasses['QubesHVm']):
if hasattr(vm, 'kernel'):
# TODO: add a setting for this?
if vm.kernel and vm.kernel not in \
os.listdir(system_path['qubes_kernels_base_dir']):
if not vm.is_property_default('kernel') and \
vm.kernel not in \
os.listdir(qubes.config.system_path[
'qubes_kernels_base_dir']):
if callable(print_callback):
print_callback("WARNING: Kernel %s not installed, "
"using default one" % vm.kernel)
vm.uses_default_kernel = True
vm.kernel = host_collection.get_default_kernel()
vm.kernel = qubes.property.DEFAULT
try:
new_vm.clone_attrs(vm)
except Exception as err:
@ -2235,7 +2260,7 @@ def backup_restore_do(restore_info,
error_callback("*** Some VM property will not be restored")
try:
new_vm.appmenus_create(verbose=callable(print_callback))
new_vm.appmenus_create()
except Exception as err:
error_callback("ERROR during appmenu restore: {0}".format(err))
error_callback(
@ -2246,22 +2271,19 @@ def backup_restore_do(restore_info,
vm_name = vm.name
if 'rename-to' in restore_info[vm.name]:
vm_name = restore_info[vm.name]['rename-to']
host_vm = host_collection.get_vm_by_name(vm_name)
if host_vm is None:
try:
host_vm = host_collection.domains[vm_name]
except KeyError:
# Failed/skipped VM
continue
if not vm.uses_default_netvm:
if not vm.is_property_default('netvm'):
if restore_info[vm.name]['netvm'] is not None:
host_vm.netvm = host_collection.get_vm_by_name(
restore_info[vm.name]['netvm'])
host_vm.netvm = restore_info[vm.name]['netvm']
else:
host_vm.netvm = None
host_collection.save()
if lock_obtained:
host_collection.unlock_db()
if running_backup_operation.canceled:
if format_version >= 2:

34
qubes/tests/__init__.py
View File

@ -40,6 +40,8 @@ import time
import qubes.config
import qubes.events
import qubes.backup
import qubes.exc
XMLPATH = '/var/lib/qubes/qubes-test.xml'
CLASS_XMLPATH = '/var/lib/qubes/qubes-class-test.xml'
@ -482,7 +484,7 @@ class SystemTestsMixin(object):
except (AttributeError, libvirt.libvirtError):
pass
del app.domains[vm]
del app.domains[vm.qid]
del vm
app.save()
@ -686,8 +688,8 @@ class BackupTestsMixin(SystemTestsMixin):
if self.verbose:
print >>sys.stderr, "-> Creating %s" % vmname
testnet = self.app.add_new_vm(qubes.vm.appvm.AppVM,
name=vmname, template=template, provides_network=True)
testnet.create_on_disk(verbose=self.verbose)
name=vmname, template=template, provides_network=True, label='red')
testnet.create_on_disk()
vms.append(testnet)
self.fill_image(testnet.private_img, 20*1024*1024)
@ -695,10 +697,10 @@ class BackupTestsMixin(SystemTestsMixin):
if self.verbose:
print >>sys.stderr, "-> Creating %s" % vmname
testvm1 = self.app.add_new_vm(qubes.vm.appvm.AppVM,
name=vmname, template=template)
name=vmname, template=template, label='red')
testvm1.uses_default_netvm = False
testvm1.netvm = testnet
testvm1.create_on_disk(verbose=self.verbose)
testvm1.create_on_disk()
vms.append(testvm1)
self.fill_image(testvm1.private_img, 100*1024*1024)
@ -706,8 +708,8 @@ class BackupTestsMixin(SystemTestsMixin):
if self.verbose:
print >>sys.stderr, "-> Creating %s" % vmname
testvm2 = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=vmname,
hvm=True)
testvm2.create_on_disk(verbose=self.verbose)
hvm=True, label='red')
testvm2.create_on_disk()
self.fill_image(testvm2.root_img, 1024*1024*1024, True)
vms.append(testvm2)
@ -717,33 +719,31 @@ class BackupTestsMixin(SystemTestsMixin):
def make_backup(self, vms, prepare_kwargs=dict(), do_kwargs=dict(),
target=None, expect_failure=False):
# XXX: bakup_prepare and backup_do don't support host_collection
# self.qc.unlock_db()
if target is None:
target = self.backupdir
try:
files_to_backup = \
qubes.backup.backup_prepare(vms,
qubes.backup.backup_prepare(self.app, vms,
print_callback=self.print_callback,
**prepare_kwargs)
except qubes.qubes.QubesException as e:
except qubes.exc.QubesException as e:
if not expect_failure:
self.fail("QubesException during backup_prepare: %s" % str(e))
else:
raise
try:
qubes.backup.backup_do(target, files_to_backup, "qubes",
qubes.backup.backup_do(self.app, target, files_to_backup, "qubes",
progress_callback=self.print_progress,
**do_kwargs)
except qubes.qubes.QubesException as e:
except qubes.exc.QubesException as e:
if not expect_failure:
self.fail("QubesException during backup_do: %s" % str(e))
else:
raise
# FIXME why?
self.reload_db()
#self.reload_db()
def restore_backup(self, source=None, appvm=None, options=None,
expect_errors=None):
@ -753,7 +753,7 @@ class BackupTestsMixin(SystemTestsMixin):
else:
backupfile = source
with self.assertNotRaises(qubes.qubes.QubesException):
with self.assertNotRaises(qubes.exc.QubesException):
backup_info = qubes.backup.backup_restore_prepare(
backupfile, "qubes",
host_collection=self.app,
@ -764,7 +764,7 @@ class BackupTestsMixin(SystemTestsMixin):
if self.verbose:
qubes.backup.backup_restore_print_summary(backup_info)
with self.assertNotRaises(qubes.qubes.QubesException):
with self.assertNotRaises(qubes.exc.QubesException):
qubes.backup.backup_restore_do(
backup_info,
host_collection=self.app,
@ -821,7 +821,7 @@ def load_tests(loader, tests, pattern): # pylint: disable=unused-argument
'qubes.tests.int.dom0_update',
'qubes.tests.int.network',
# 'qubes.tests.vm_qrexec_gui',
# 'qubes.tests.backup',
'qubes.tests.int.backup',
# 'qubes.tests.backupcompatibility',
# 'qubes.tests.regressions',

66
tests/backup.py → qubes/tests/int/backup.py
View File

@ -28,8 +28,11 @@ import os
import unittest
import sys
from qubes.qubes import QubesException, QubesTemplateVm
import qubes
import qubes.exc
import qubes.tests
import qubes.vm.appvm
import qubes.vm.templatevm
class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
def test_000_basic_backup(self):
@ -37,21 +40,24 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
self.make_backup(vms)
self.remove_vms(vms)
self.restore_backup()
self.remove_vms(vms)
for vm in vms:
self.assertIn(vm.name, self.app.domains)
def test_001_compressed_backup(self):
vms = self.create_backup_vms()
self.make_backup(vms, do_kwargs={'compressed': True})
self.remove_vms(vms)
self.restore_backup()
self.remove_vms(vms)
for vm in vms:
self.assertIn(vm.name, self.app.domains)
def test_002_encrypted_backup(self):
vms = self.create_backup_vms()
self.make_backup(vms, do_kwargs={'encrypted': True})
self.remove_vms(vms)
self.restore_backup()
self.remove_vms(vms)
for vm in vms:
self.assertIn(vm.name, self.app.domains)
def test_003_compressed_encrypted_backup(self):
vms = self.create_backup_vms()
@ -61,7 +67,8 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
'encrypted': True})
self.remove_vms(vms)
self.restore_backup()
self.remove_vms(vms)
for vm in vms:
self.assertIn(vm.name, self.app.domains)
def test_004_sparse_multipart(self):
vms = []
@ -70,29 +77,32 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
if self.verbose:
print >>sys.stderr, "-> Creating %s" % vmname
hvmtemplate = self.qc.add_new_vm("QubesTemplateHVm", name=vmname)
hvmtemplate.create_on_disk(verbose=self.verbose)
hvmtemplate = self.app.add_new_vm(
qubes.vm.templatevm.TemplateVM, name=vmname, hvm=True, label='red')
hvmtemplate.create_on_disk()
self.fill_image(os.path.join(hvmtemplate.dir_path, '00file'),
195*1024*1024-4096*3)
self.fill_image(hvmtemplate.private_img, 195*1024*1024-4096*3)
self.fill_image(hvmtemplate.root_img, 1024*1024*1024, sparse=True)
vms.append(hvmtemplate)
self.qc.save()
self.app.save()
self.make_backup(vms)
self.remove_vms(vms)
self.restore_backup()
self.remove_vms(vms)
for vm in vms:
self.assertIn(vm.name, self.app.domains)
def test_005_compressed_custom(self):
vms = self.create_backup_vms()
self.make_backup(vms, do_kwargs={'compressed': "bzip2"})
self.remove_vms(vms)
self.restore_backup()
self.remove_vms(vms)
for vm in vms:
self.assertIn(vm.name, self.app.domains)
def test_100_backup_dom0_no_restore(self):
self.make_backup([self.qc[0]])
self.make_backup([self.app.domains[0]])
# TODO: think of some safe way to test restore...
def test_200_restore_over_existing_directory(self):
@ -112,7 +122,6 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
'*** Directory {} already exists! It has been moved'.format(
test_dir)
])
self.remove_vms(vms)
def test_210_auto_rename(self):
"""
@ -125,22 +134,23 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase):
'rename-conflicting': True
})
for vm in vms:
self.assertIsNotNone(self.qc.get_vm_by_name(vm.name+'1'))
restored_vm = self.qc.get_vm_by_name(vm.name+'1')
if vm.netvm and not vm.uses_default_netvm:
with self.assertNotRaises(qubes.exc.QubesVMNotFoundError):
restored_vm = self.app.domains[vm.name + '1']
if vm.netvm and not vm.property_is_default('netvm'):
self.assertEqual(restored_vm.netvm.name, vm.netvm.name + '1')
self.remove_vms(vms)
class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin):
def setUp(self):
super(TC_10_BackupVMMixin, self).setUp()
self.backupvm = self.qc.add_new_vm(
"QubesAppVm",
self.backupvm = self.app.add_new_vm(
qubes.vm.appvm.AppVM,
label='red',
name=self.make_vm_name('backupvm'),
template=self.qc.get_vm_by_name(self.template)
template=self.template
)
self.backupvm.create_on_disk(verbose=self.verbose)
self.backupvm.create_on_disk()
def test_100_send_to_vm_file_with_spaces(self):
vms = self.create_backup_vms()
@ -159,7 +169,6 @@ class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin):
backup_path = backup_path.strip()
self.restore_backup(source=backup_path,
appvm=self.backupvm)
self.remove_vms(vms)
def test_110_send_to_vm_command(self):
vms = self.create_backup_vms()
@ -173,7 +182,6 @@ class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin):
self.remove_vms(vms)
self.restore_backup(source='dd if=/var/tmp/backup-test',
appvm=self.backupvm)
self.remove_vms(vms)
def test_110_send_to_vm_no_space(self):
"""
@ -192,7 +200,7 @@ class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin):
user="root", wait=True)
if retcode != 0:
raise RuntimeError("Failed to prepare backup directory")
with self.assertRaises(QubesException):
with self.assertRaises(qubes.exc.QubesException):
self.make_backup(vms,
do_kwargs={
'appvm': self.backupvm,
@ -200,19 +208,13 @@ class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin):
'encrypted': True},
target='/home/user/backup',
expect_failure=True)
self.qc.lock_db_for_writing()
self.qc.load()
self.remove_vms(vms)
def load_tests(loader, tests, pattern):
try:
qc = qubes.qubes.QubesVmCollection()
qc.lock_db_for_reading()
qc.load()
qc.unlock_db()
templates = [vm.name for vm in qc.values() if
isinstance(vm, QubesTemplateVm)]
app = qubes.Qubes()
templates = [vm.name for vm in app.domains if
isinstance(vm, qubes.vm.templatevm.TemplateVM)]
except OSError:
templates = []
for template in templates:

42
qubes/utils.py
View File

@ -90,9 +90,12 @@ def format_doc(docstring):
# maybe adapt https://code.activestate.com/recipes/578019
def parse_size(size):
units = [
('K', 1024), ('KB', 1024),
('M', 1024*1024), ('MB', 1024*1024),
('G', 1024*1024*1024), ('GB', 1024*1024*1024),
('K', 1000), ('KB', 1000),
('M', 1000 * 1000), ('MB', 1000 * 1000),
('G', 1000 * 1000 * 1000), ('GB', 1000 * 1000 * 1000),
('Ki', 1024), ('KiB', 1024),
('Mi', 1024 * 1024), ('MiB', 1024 * 1024),
('Gi', 1024 * 1024 * 1024), ('GiB', 1024 * 1024 * 1024),
]
size = size.strip().upper()
@ -106,6 +109,39 @@ def parse_size(size):
raise qubes.exc.QubesException("Invalid size: {0}.".format(size))
def mbytes_to_kmg(size):
if size > 1024:
return "%d GiB" % (size / 1024)
else:
return "%d MiB" % size
def kbytes_to_kmg(size):
if size > 1024:
return mbytes_to_kmg(size / 1024)
else:
return "%d KiB" % size
def bytes_to_kmg(size):
if size > 1024:
return kbytes_to_kmg(size / 1024)
else:
return "%d B" % size
def size_to_human(size):
"""Humane readable size, with 1/10 precision"""
if size < 1024:
return str(size)
elif size < 1024 * 1024:
return str(round(size / 1024.0, 1)) + ' KiB'
elif size < 1024 * 1024 * 1024:
return str(round(size / (1024.0 * 1024), 1)) + ' MiB'
else:
return str(round(size / (1024.0 * 1024 * 1024), 1)) + ' GiB'
def urandom(size):
rand = os.urandom(size)
if rand is None:

2
rpm_spec/core-dom0.spec
View File

@ -201,6 +201,7 @@ fi
%dir %{python_sitelib}/qubes
%{python_sitelib}/qubes/__init__.py*
%{python_sitelib}/qubes/backup.py*
%{python_sitelib}/qubes/config.py*
%{python_sitelib}/qubes/devices.py*
%{python_sitelib}/qubes/dochelpers.py*
@ -272,6 +273,7 @@ fi
%dir %{python_sitelib}/qubes/tests/int
%{python_sitelib}/qubes/tests/int/__init__.py*
%{python_sitelib}/qubes/tests/int/backup.py*
%{python_sitelib}/qubes/tests/int/basic.py*
%{python_sitelib}/qubes/tests/int/dom0_update.py*
%{python_sitelib}/qubes/tests/int/network.py*