backup: code style fixes, no functional change (part 1)

Indentation, break long lines, use is/is not None instead of ==/!=.
This commit is contained in:
Marek Marczykowski-Górecki 2015-05-03 14:45:01 +02:00
parent e2d6ff653a
commit 9ec0580840

View File

@ -3,7 +3,8 @@
#
# The Qubes OS Project, http://www.qubes-os.org
#
# Copyright (C) 2013 Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
# Copyright (C) 2013-2015 Marek Marczykowski-Górecki
# <marmarek@invisiblethingslab.com>
# Copyright (C) 2013 Olivier Médoc <o_medoc@yahoo.fr>
#
# This program is free software; you can redistribute it and/or
@ -34,7 +35,8 @@ import re
import shutil
import tempfile
import time
import grp,pwd
import grp
import pwd
import errno
import datetime
from multiprocessing import Queue, Process
@ -54,17 +56,20 @@ HEADER_QUBES_XML_MAX_SIZE = 1024 * 1024
# global state for backup_cancel()
running_backup_operation = None
class BackupOperationInfo:
def __init__(self):
self.canceled = False
self.processes_to_kill_on_cancel = []
self.tmpdir_to_remove = None
class BackupCanceledError(QubesException):
def __init__(self, msg, tmpdir=None):
super(BackupCanceledError, self).__init__(msg)
self.tmpdir = tmpdir
class BackupHeader:
version = 'version'
encrypted = 'encrypted'
@ -91,6 +96,7 @@ def file_to_backup (file_path, subdir = None):
subdir += '/'
return [{"path": file_path, "size": sz, "subdir": subdir}]
def backup_cancel():
"""
Cancel currently running backup/restore operation
@ -108,9 +114,13 @@ def backup_cancel():
pass
return True
def backup_prepare(vms_list=None, exclude_list=None,
print_callback=print_stdout, hide_vm_names=True):
"""If vms = None, include all (sensible) VMs; exclude_list is always applied"""
"""
If vms = None, include all (sensible) VMs;
exclude_list is always applied
"""
files_to_backup = file_to_backup(system_path["qubes_store_filename"])
if exclude_list is None:
@ -123,13 +133,16 @@ def backup_prepare(vms_list = None, exclude_list = None,
if vms_list is None:
all_vms = [vm for vm in qvm_collection.values()]
selected_vms = [vm for vm in all_vms if vm.include_in_backups]
appvms_to_backup = [vm for vm in selected_vms if vm.is_appvm() and not vm.internal]
netvms_to_backup = [vm for vm in selected_vms if vm.is_netvm() and not vm.qid == 0]
appvms_to_backup = [vm for vm in selected_vms if
vm.is_appvm() and not vm.internal]
netvms_to_backup = [vm for vm in selected_vms if
vm.is_netvm() and not vm.qid == 0]
template_vms_worth_backingup = [vm for vm in selected_vms if (
vm.is_template() and vm.include_in_backups)]
dom0 = [qvm_collection[0]]
vms_list = appvms_to_backup + netvms_to_backup + template_vms_worth_backingup + dom0
vms_list = appvms_to_backup + netvms_to_backup + \
template_vms_worth_backingup + dom0
vms_for_backup = vms_list
# Apply exclude list
@ -185,17 +198,20 @@ def backup_prepare(vms_list = None, exclude_list = None,
if vm.updateable:
if os.path.exists(vm.dir_path + "/apps.templates"):
# template
files_to_backup += file_to_backup(vm.dir_path + "/apps.templates", subdir)
files_to_backup += file_to_backup(
vm.dir_path + "/apps.templates", subdir)
else:
# standaloneVM
files_to_backup += file_to_backup(vm.dir_path + "/apps", subdir)
if os.path.exists(vm.dir_path + "/kernels"):
files_to_backup += file_to_backup(vm.dir_path + "/kernels", subdir)
files_to_backup += file_to_backup(vm.dir_path + "/kernels",
subdir)
if os.path.exists(vm.firewall_conf):
files_to_backup += file_to_backup(vm.firewall_conf, subdir)
if 'appmenus_whitelist' in vm_files and \
os.path.exists(os.path.join(vm.dir_path, vm_files['appmenus_whitelist'])):
os.path.exists(os.path.join(vm.dir_path,
vm_files['appmenus_whitelist'])):
files_to_backup += file_to_backup(
os.path.join(vm.dir_path, vm_files['appmenus_whitelist']),
subdir)
@ -222,7 +238,8 @@ def backup_prepare(vms_list = None, exclude_list = None,
s += fmt.format(size_to_human(vm_size))
if vm.is_running():
s += " <-- The VM is running, please shut it down before proceeding with the backup!"
s += " <-- The VM is running, please shut it down before proceeding " \
"with the backup!"
there_are_running_vms = True
print_callback(s)
@ -241,8 +258,7 @@ def backup_prepare(vms_list = None, exclude_list = None,
template_subdir = os.path.relpath(
vm.dir_path,
system_path["qubes_base_dir"]) + '/'
template_to_backup = [ {
"path": vm.dir_path + '/.',
template_to_backup = [{"path": vm.dir_path + '/.',
"size": vm_sz,
"subdir": template_subdir}]
files_to_backup += template_to_backup
@ -258,7 +274,8 @@ def backup_prepare(vms_list = None, exclude_list = None,
s += fmt.format(size_to_human(vm_sz))
if vm.is_running():
s += " <-- The VM is running, please shut it down before proceeding with the backup!"
s += " <-- The VM is running, please shut it down before proceeding " \
"with the backup!"
there_are_running_vms = True
print_callback(s)
@ -277,7 +294,8 @@ def backup_prepare(vms_list = None, exclude_list = None,
if hide_vm_names:
vm.backup_path = 'vm%d' % vm.qid
else:
vm.backup_path = os.path.relpath(vm.dir_path, system_path["qubes_base_dir"])
vm.backup_path = os.path.relpath(vm.dir_path,
system_path["qubes_base_dir"])
# Dom0 user home
if 0 in vms_for_backup_qid:
@ -289,7 +307,8 @@ def backup_prepare(vms_list = None, exclude_list = None,
subprocess.check_call(['sudo', 'chown', '-R', local_user, home_dir])
home_sz = get_disk_usage(home_dir)
home_to_backup = [ { "path" : home_dir, "size": home_sz, "subdir": 'dom0-home/'} ]
home_to_backup = [
{"path": home_dir, "size": home_sz, "subdir": 'dom0-home/'}]
files_to_backup += home_to_backup
vm = qvm_collection[0]
@ -314,8 +333,8 @@ def backup_prepare(vms_list = None, exclude_list = None,
qvm_collection.unlock_db()
total_backup_sz = 0
for file in files_to_backup:
total_backup_sz += file["size"]
for f in files_to_backup:
total_backup_sz += f["size"]
s = ""
for f in fields_to_display:
@ -326,7 +345,9 @@ def backup_prepare(vms_list = None, exclude_list = None,
s = ""
fmt = "{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1)
s += fmt.format("Total size:")
fmt="{{0:>{0}}} |".format(fields_to_display[1]["width"] + 1 + 2 + fields_to_display[2]["width"] + 1)
fmt = "{{0:>{0}}} |".format(
fields_to_display[1]["width"] + 1 + 2 + fields_to_display[2][
"width"] + 1)
s += fmt.format(size_to_human(total_backup_sz))
print_callback(s)
@ -336,12 +357,12 @@ def backup_prepare(vms_list = None, exclude_list = None,
s += fmt.format('-')
print_callback(s)
vms_not_for_backup = [vm.name for vm in qvm_collection.values() if not vm
.backup_content]
vms_not_for_backup = [vm.name for vm in qvm_collection.values()
if not vm.backup_content]
print_callback("VMs not selected for backup: %s" % " ".join(
vms_not_for_backup))
if (there_are_running_vms):
if there_are_running_vms:
raise QubesException("Please shutdown all VMs before proceeding.")
for fileinfo in files_to_backup:
@ -350,6 +371,7 @@ def backup_prepare(vms_list = None, exclude_list = None,
return files_to_backup
class SendWorker(Process):
def __init__(self, queue, base_dir, backup_stdout):
super(SendWorker, self).__init__()
@ -377,10 +399,13 @@ class SendWorker(Process):
tar_final_cmd = ["tar", "-cO", "--posix",
"-C", self.base_dir, filename]
final_proc = subprocess.Popen(tar_final_cmd,
stdin=subprocess.PIPE, stdout=self.backup_stdout)
stdin=subprocess.PIPE,
stdout=self.backup_stdout)
if final_proc.wait() >= 2:
# handle only exit code 2 (tar fatal error) or greater (call failed?)
raise QubesException("ERROR: Failed to write the backup, out of disk space? "
# handle only exit code 2 (tar fatal error) or
# greater (call failed?)
raise QubesException(
"ERROR: Failed to write the backup, out of disk space? "
"Check console output or ~/.xsession-errors for details.")
# Delete the file as we don't need it anymore
@ -391,6 +416,7 @@ class SendWorker(Process):
if BACKUP_DEBUG:
print "Finished sending thread"
def prepare_backup_header(target_directory, passphrase, compressed=False,
encrypted=False,
hmac_algorithm=DEFAULT_HMAC_ALGORITHM,
@ -415,7 +441,8 @@ def prepare_backup_header(target_directory, passphrase, compressed=False,
stdout=open(header_file_path + ".hmac", "w"))
if hmac.wait() != 0:
raise QubesException("Failed to compute hmac of header file")
return (HEADER_FILENAME, HEADER_FILENAME+".hmac")
return HEADER_FILENAME, HEADER_FILENAME + ".hmac"
def backup_do(base_backup_dir, files_to_backup, passphrase,
progress_callback=None, encrypted=False, appvm=None,
@ -425,8 +452,8 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
total_backup_sz = 0
passphrase = passphrase.encode('utf-8')
for file in files_to_backup:
total_backup_sz += file["size"]
for f in files_to_backup:
total_backup_sz += f["size"]
if isinstance(compressed, str):
compression_filter = compressed
@ -435,7 +462,7 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
running_backup_operation = BackupOperationInfo()
vmproc = None
if appvm != None:
if appvm is not None:
# Prepare the backup target (Qubes service call)
backup_target = "QUBESRPC qubes.Backup dom0"
@ -524,7 +551,8 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
# The first tar cmd can use any complex feature as we want. Files will
# be verified before untaring this.
# Prefix the path in archive with filename["subdir"] to have it verified during untar
# Prefix the path in archive with filename["subdir"] to have it
# verified during untar
tar_cmdline = ["tar", "-Pc", '--sparse',
"-f", backup_pipe,
'-C', os.path.dirname(filename["path"]),
@ -534,7 +562,8 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
os.path.basename(filename["path"])
]
if compressed:
tar_cmdline.insert(-1, "--use-compress-program=%s" % compression_filter)
tar_cmdline.insert(-1,
"--use-compress-program=%s" % compression_filter)
if BACKUP_DEBUG:
print " ".join(tar_cmdline)
@ -543,7 +572,9 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
# Pipe: tar-sparse | encryptor [| hmac] | tar | backup_target
# Pipe: tar-sparse [| hmac] | tar | backup_target
tar_sparse = subprocess.Popen(tar_cmdline, stdin=subprocess.PIPE,
stderr=(open(os.devnull, 'w') if not BACKUP_DEBUG else None))
stderr=(open(os.devnull, 'w')
if not BACKUP_DEBUG
else None))
running_backup_operation.processes_to_kill_on_cancel.append(tar_sparse)
# Wait for compressor (tar) process to finish or for any error of other
@ -557,7 +588,8 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
encryptor = subprocess.Popen(["openssl", "enc",
"-e", "-" + crypto_algorithm,
"-pass", "pass:" + passphrase],
stdin=open(backup_pipe,'rb'), stdout=subprocess.PIPE)
stdin=open(backup_pipe, 'rb'),
stdout=subprocess.PIPE)
pipe = encryptor.stdout
else:
pipe = open(backup_pipe, 'rb')
@ -566,7 +598,8 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
# Start HMAC
hmac = subprocess.Popen(["openssl", "dgst",
"-" + hmac_algorithm, "-hmac", passphrase],
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
# Prepare a first chunk
chunkfile = backup_tempfile + "." + "%03d" % i
@ -609,10 +642,11 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
if run_error and run_error != "size_limit":
send_proc.terminate()
if run_error == "VM" and vmproc:
raise QubesException("Failed to write the backup, VM output:\n" +
raise QubesException(
"Failed to write the backup, VM output:\n" +
vmproc.stderr.read(MAX_STDERR_BYTES))
else:
raise QubesException("Failed to perform backup: error in "+ \
raise QubesException("Failed to perform backup: error in " +
run_error)
# Send the chunk to the backup target
@ -646,7 +680,6 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
.poll()
pipe.close()
to_send.put("FINISHED")
send_proc.join()
shutil.rmtree(backup_tmpdir)
@ -658,7 +691,8 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
running_backup_operation = None
if send_proc.exitcode != 0:
raise QubesException("Failed to send backup: error in the sending process")
raise QubesException(
"Failed to send backup: error in the sending process")
if vmproc:
if BACKUP_DEBUG:
@ -678,6 +712,7 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
qvm_collection.save()
qvm_collection.unlock_db()
'''
' Wait for backup chunk to finish
' - Monitor all the processes (streamproc, hmac, vmproc, addproc) for errors
@ -693,16 +728,18 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
' "")
' - size_limit is provided and is about to be exceeded
'''
def wait_backup_feedback(progress_callback, in_stream, streamproc,
backup_target, total_backup_sz, hmac=None, vmproc=None, addproc=None,
remove_trailing_bytes=0, size_limit=None):
def wait_backup_feedback(progress_callback, in_stream, streamproc,
backup_target, total_backup_sz, hmac=None, vmproc=None,
addproc=None,
remove_trailing_bytes=0, size_limit=None):
buffer_size = 409600
run_error = None
run_count = 1
bytes_copied = 0
while run_count > 0 and run_error == None:
while run_count > 0 and run_error is None:
if size_limit and bytes_copied + buffer_size > size_limit:
return "size_limit"
@ -713,7 +750,7 @@ def wait_backup_feedback(progress_callback, in_stream, streamproc,
run_count = 0
if hmac:
retcode = hmac.poll()
if retcode != None:
if retcode is not None:
if retcode != 0:
run_error = "hmac"
else:
@ -721,7 +758,7 @@ def wait_backup_feedback(progress_callback, in_stream, streamproc,
if addproc:
retcode = addproc.poll()
if retcode != None:
if retcode is not None:
if retcode != 0:
run_error = "addproc"
else:
@ -729,7 +766,7 @@ def wait_backup_feedback(progress_callback, in_stream, streamproc,
if vmproc:
retcode = vmproc.poll()
if retcode != None:
if retcode is not None:
if retcode != 0:
run_error = "VM"
if BACKUP_DEBUG:
@ -740,7 +777,7 @@ def wait_backup_feedback(progress_callback, in_stream, streamproc,
if streamproc:
retcode = streamproc.poll()
if retcode != None:
if retcode is not None:
if retcode != 0:
run_error = "streamproc"
break
@ -765,13 +802,14 @@ def wait_backup_feedback(progress_callback, in_stream, streamproc,
return run_error
def verify_hmac(filename, hmacfile, passphrase, algorithm):
if BACKUP_DEBUG:
print "Verifying file " + filename
if hmacfile != filename + ".hmac":
raise QubesException(
"ERROR: expected hmac for {}, but got {}".\
"ERROR: expected hmac for {}, but got {}".
format(filename, hmacfile))
hmac_proc = subprocess.Popen(["openssl", "dgst", "-" + algorithm,
@ -781,7 +819,8 @@ def verify_hmac(filename, hmacfile, passphrase, algorithm):
hmac_stdout, hmac_stderr = hmac_proc.communicate()
if len(hmac_stderr) > 0:
raise QubesException("ERROR: verify file {0}: {1}".format(filename, hmac_stderr))
raise QubesException(
"ERROR: verify file {0}: {1}".format(filename, hmac_stderr))
else:
if BACKUP_DEBUG:
print "Loading hmac for file " + filename
@ -794,8 +833,8 @@ def verify_hmac(filename, hmacfile, passphrase, algorithm):
return True
else:
raise QubesException(
"ERROR: invalid hmac for file {0}: {1}. " \
"Is the passphrase correct?".\
"ERROR: invalid hmac for file {0}: {1}. "
"Is the passphrase correct?".
format(filename, load_hmac(hmac_stdout)))
# Not reachable
return False
@ -901,12 +940,12 @@ class ExtractWorker2(Process):
if filename.endswith('.000'):
# next file
if self.tar2_process != None:
if self.tar2_process is not None:
if self.tar2_process.wait() != 0:
self.collect_tar_output()
self.error_callback(
"ERROR: unable to extract files for {0}, tar "
"output:\n {1}".\
"output:\n {1}".
format(self.tar2_current_file,
"\n ".join(self.tar2_stderr)))
else:
@ -953,9 +992,12 @@ class ExtractWorker2(Process):
}
if self.encrypted:
# Start decrypt
self.decryptor_process = subprocess.Popen (["openssl", "enc",
"-d", "-" + self.crypto_algorithm,
"-pass", "pass:"+self.passphrase] +
self.decryptor_process = subprocess.Popen(
["openssl", "enc",
"-d",
"-" + self.crypto_algorithm,
"-pass",
"pass:" + self.passphrase] +
(["-z"] if self.compressed else []),
stdin=open(filename, 'rb'),
stdout=subprocess.PIPE)
@ -966,7 +1008,8 @@ class ExtractWorker2(Process):
streamproc=self.decryptor_process,
**common_args)
elif self.compressed:
self.decompressor_process = subprocess.Popen (["gzip", "-d"],
self.decompressor_process = subprocess.Popen(
["gzip", "-d"],
stdin=open(filename, 'rb'),
stdout=subprocess.PIPE)
@ -986,7 +1029,9 @@ class ExtractWorker2(Process):
except IOError as e:
if e.errno == errno.EPIPE:
if BACKUP_DEBUG:
self.error_callback("Got EPIPE while closing pipe to the inner tar process")
self.error_callback(
"Got EPIPE while closing pipe to "
"the inner tar process")
# ignore the error
else:
raise
@ -999,7 +1044,7 @@ class ExtractWorker2(Process):
self.tar2_process.terminate()
self.tar2_process.wait()
self.tar2_process = None
self.error_callback("Error while processing '%s': %s " % \
self.error_callback("Error while processing '%s': %s " %
(self.tar2_current_file, details))
# Delete the file as we don't need it anymore
@ -1029,9 +1074,8 @@ class ExtractWorker2(Process):
if BACKUP_DEBUG and callable(self.print_callback):
self.print_callback("Finished extracting thread")
class ExtractWorker3(ExtractWorker2):
def __init__(self, queue, base_dir, passphrase, encrypted, total_size,
print_callback, error_callback, progress_callback, vmproc=None,
compressed=False, crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM,
@ -1063,13 +1107,13 @@ class ExtractWorker3(ExtractWorker2):
if filename.endswith('.000'):
# next file
if self.tar2_process != None:
if self.tar2_process is not None:
input_pipe.close()
if self.tar2_process.wait() != 0:
self.collect_tar_output()
self.error_callback(
"ERROR: unable to extract files for {0}, tar "
"output:\n {1}".\
"output:\n {1}".
format(self.tar2_current_file,
"\n ".join(self.tar2_stderr)))
else:
@ -1095,18 +1139,23 @@ class ExtractWorker3(ExtractWorker2):
unicode(tar2_cmdline))
if self.encrypted:
# Start decrypt
self.decryptor_process = subprocess.Popen (["openssl", "enc",
"-d", "-" + self.crypto_algorithm,
"-pass", "pass:"+self.passphrase],
self.decryptor_process = subprocess.Popen(
["openssl", "enc",
"-d",
"-" + self.crypto_algorithm,
"-pass",
"pass:" + self.passphrase],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
self.tar2_process = subprocess.Popen(tar2_cmdline,
self.tar2_process = subprocess.Popen(
tar2_cmdline,
stdin=self.decryptor_process.stdout,
stderr=subprocess.PIPE)
input_pipe = self.decryptor_process.stdin
else:
self.tar2_process = subprocess.Popen(tar2_cmdline,
self.tar2_process = subprocess.Popen(
tar2_cmdline,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE)
input_pipe = self.tar2_process.stdin
@ -1152,7 +1201,7 @@ class ExtractWorker3(ExtractWorker2):
self.tar2_process.terminate()
self.tar2_process.wait()
self.tar2_process = None
self.error_callback("Error while processing '%s': %s " % \
self.error_callback("Error while processing '%s': %s " %
(self.tar2_current_file, details))
# Delete the file as we don't need it anymore
@ -1198,6 +1247,7 @@ def get_supported_hmac_algo(hmac_algorithm):
yield algo.strip()
proc.wait()
def parse_backup_header(filename):
header_data = {}
with open(filename, 'r') as f:
@ -1216,31 +1266,35 @@ def parse_backup_header(filename):
header_data[key] = value
return header_data
def restore_vm_dirs(backup_source, restore_tmpdir, passphrase, vms_dirs, vms,
vms_size, print_callback=None, error_callback=None,
progress_callback=None, encrypted=False, appvm=None,
compressed=False, hmac_algorithm=DEFAULT_HMAC_ALGORITHM,
crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM,
verify_only=False, format_version = CURRENT_BACKUP_FORMAT_VERSION,
verify_only=False,
format_version=CURRENT_BACKUP_FORMAT_VERSION,
compression_filter=None):
global running_backup_operation
if callable(print_callback):
if BACKUP_DEBUG:
print_callback("Working in temporary dir:" + restore_tmpdir)
print_callback("Extracting data: " + size_to_human(vms_size)+" to restore")
print_callback(
"Extracting data: " + size_to_human(vms_size) + " to restore")
passphrase = passphrase.encode('utf-8')
header_data = None
vmproc = None
if appvm != None:
if appvm is not None:
# Prepare the backup target (Qubes service call)
backup_target = "QUBESRPC qubes.Restore dom0"
# If APPVM, STDOUT is a PIPE
vmproc = appvm.run(command = backup_target, passio_popen = True, passio_stderr=True)
vmproc.stdin.write(backup_source.replace("\r","").replace("\n","")+"\n")
vmproc = appvm.run(command=backup_target, passio_popen=True,
passio_stderr=True)
vmproc.stdin.write(
backup_source.replace("\r", "").replace("\n", "") + "\n")
# Send to tar2qfile the VMs that should be extracted
vmproc.stdin.write(" ".join(vms_dirs) + "\n")
@ -1271,10 +1325,12 @@ def restore_vm_dirs (backup_source, restore_tmpdir, passphrase, vms_dirs, vms,
# chunks. Additionally each file have own hmac file. So assume upper
# limit as 2*(10*COUNT_OF_VMS+TOTAL_SIZE/100MB)
tar1_env['UPDATES_MAX_FILES'] = str(2 * (10 * len(vms_dirs) +
int(vms_size/(100*1024*1024))))
int(vms_size /
(100 * 1024 * 1024))))
if BACKUP_DEBUG and callable(print_callback):
print_callback("Run command" + unicode(tar1_command))
command = subprocess.Popen(tar1_command,
command = subprocess.Popen(
tar1_command,
stdin=backup_stdin,
stdout=vmproc.stdin if vmproc else subprocess.PIPE,
stderr=subprocess.PIPE,
@ -1348,7 +1404,8 @@ def restore_vm_dirs (backup_source, restore_tmpdir, passphrase, vms_dirs, vms,
if BackupHeader.encrypted in header_data:
encrypted = header_data[BackupHeader.encrypted]
if BackupHeader.compression_filter in header_data:
compression_filter = header_data[BackupHeader.compression_filter]
compression_filter = header_data[
BackupHeader.compression_filter]
os.unlink(filename)
else:
# if no header found, create one with guessed HMAC algo
@ -1383,7 +1440,8 @@ def restore_vm_dirs (backup_source, restore_tmpdir, passphrase, vms_dirs, vms,
extractor_params['compression_filter'] = compression_filter
extract_proc = ExtractWorker3(**extractor_params)
else:
raise NotImplemented("Backup format version %d not supported" % format_version)
raise NotImplemented(
"Backup format version %d not supported" % format_version)
extract_proc.start()
try:
@ -1416,15 +1474,16 @@ def restore_vm_dirs (backup_source, restore_tmpdir, passphrase, vms_dirs, vms,
if running_backup_operation and running_backup_operation.canceled:
break
# if reading archive directly with tar, wait for next filename -
# tar prints filename before processing it, so wait for the next one to be
# sure that whole file was extracted
# tar prints filename before processing it, so wait for
# the next one to be sure that whole file was extracted
if not appvm:
nextfile = filelist_pipe.readline().strip()
if BACKUP_DEBUG and callable(print_callback):
print_callback("Getting hmac:" + hmacfile)
if not hmacfile or hmacfile == "EOF":
# Premature end of archive, either of tar1_command or vmproc exited with error
# Premature end of archive, either of tar1_command or
# vmproc exited with error
break
if not any(map(lambda x: filename.startswith(x), vms_dirs)):
@ -1445,17 +1504,18 @@ def restore_vm_dirs (backup_source, restore_tmpdir, passphrase, vms_dirs, vms,
if command.wait() != 0 and not expect_tar_error:
raise QubesException(
"unable to read the qubes backup file {0} ({1}). " \
"unable to read the qubes backup file {0} ({1}). "
"Is it really a backup?".format(backup_source, command.wait()))
if vmproc:
if vmproc.wait() != 0:
raise QubesException(
"unable to read the qubes backup {0} " \
"unable to read the qubes backup {0} "
"because of a VM error: {1}".format(
backup_source, vmproc.stderr.read(MAX_STDERR_BYTES)))
if filename and filename != "EOF":
raise QubesException("Premature end of archive, the last file was %s" % filename)
raise QubesException(
"Premature end of archive, the last file was %s" % filename)
except:
to_extract.put("ERROR")
extract_proc.join()
@ -1467,15 +1527,16 @@ def restore_vm_dirs (backup_source, restore_tmpdir, passphrase, vms_dirs, vms,
print_callback("Waiting for the extraction process to finish...")
extract_proc.join()
if BACKUP_DEBUG and callable(print_callback):
print_callback("Extraction process finished with code:" + \
print_callback("Extraction process finished with code:" +
str(extract_proc.exitcode))
if extract_proc.exitcode != 0:
raise QubesException(
"unable to extract the qubes backup. " \
"unable to extract the qubes backup. "
"Check extracting process errors.")
return header_data
def backup_restore_set_defaults(options):
if 'use-default-netvm' not in options:
options['use-default-netvm'] = False
@ -1494,6 +1555,7 @@ def backup_restore_set_defaults(options):
return options
def load_hmac(hmac):
hmac = hmac.strip().split("=")
if len(hmac) > 1:
@ -1503,6 +1565,7 @@ def load_hmac(hmac):
return hmac
def backup_detect_format_version(backup_location):
if os.path.exists(os.path.join(backup_location, 'qubes.xml')):
return 1
@ -1511,23 +1574,25 @@ def backup_detect_format_version(backup_location):
# is read
return 2
def backup_restore_header(source, passphrase,
print_callback = print_stdout, error_callback = print_stderr,
encrypted=False, appvm=None, compressed = False, format_version = None,
print_callback=print_stdout,
error_callback=print_stderr,
encrypted=False, appvm=None, compressed=False,
format_version=None,
hmac_algorithm=DEFAULT_HMAC_ALGORITHM,
crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM):
global running_backup_operation
vmproc = None
running_backup_operation = None
restore_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/restore_")
if format_version == None:
if format_version is None:
format_version = backup_detect_format_version(source)
if format_version == 1:
return (restore_tmpdir, os.path.join(source, 'qubes.xml'), None)
return restore_tmpdir, os.path.join(source, 'qubes.xml'), None
# tar2qfile matches only beginnings, while tar full path
if appvm:
@ -1555,6 +1620,7 @@ def backup_restore_header(source, passphrase,
return (restore_tmpdir, os.path.join(restore_tmpdir, "qubes.xml"),
header_data)
def restore_info_verify(restore_info, host_collection):
options = restore_info['$OPTIONS$']
for vm in restore_info.keys():
@ -1601,7 +1667,7 @@ def restore_info_verify(restore_info, host_collection):
if not ((netvm_on_host is not None) and netvm_on_host.is_netvm()):
# Maybe the (custom) netvm is in the backup?
if not (netvm_name in restore_info.keys() and \
if not (netvm_name in restore_info.keys() and
restore_info[netvm_name]['vm'].is_netvm()):
if options['use-default-netvm']:
vm_info['netvm'] = host_collection \
@ -1620,10 +1686,13 @@ def restore_info_verify(restore_info, host_collection):
return restore_info
def backup_restore_prepare(backup_location, passphrase, options=None,
host_collection=None, encrypted=False, appvm=None,
compressed = False, print_callback = print_stdout, error_callback = print_stderr,
format_version=None, hmac_algorithm=DEFAULT_HMAC_ALGORITHM,
compressed=False, print_callback=print_stdout,
error_callback=print_stderr,
format_version=None,
hmac_algorithm=DEFAULT_HMAC_ALGORITHM,
crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM):
if options is None:
options = {}
@ -1633,17 +1702,19 @@ def backup_restore_prepare(backup_location, passphrase, options = None,
# so no need for fallback in function parameter
compression_filter = DEFAULT_COMPRESSION_FILTER
#### Private functions begin
# Private functions begin
def is_vm_included_in_backup_v1(backup_dir, vm):
if vm.qid == 0:
return os.path.exists(os.path.join(backup_dir, 'dom0-home'))
backup_vm_dir_path = vm.dir_path.replace (system_path["qubes_base_dir"], backup_dir)
backup_vm_dir_path = vm.dir_path.replace(system_path["qubes_base_dir"],
backup_dir)
if os.path.exists(backup_vm_dir_path):
return True
else:
return False
def is_vm_included_in_backup_v2(backup_dir, vm):
if vm.backup_content:
return True
@ -1658,7 +1729,8 @@ def backup_restore_prepare(backup_location, passphrase, options = None,
return m.group(2)
return template
#### Private functions end
# Private functions end
# Format versions:
# 1 - Qubes R1, Qubes R2 beta1, beta2
@ -1675,10 +1747,10 @@ def backup_restore_prepare(backup_location, passphrase, options = None,
if not os.path.isfile(backup_location):
raise QubesException("Invalid backup location (not a file or "
"directory with qubes.xml)"
": %s" % unicode(
backup_location))
": %s" % unicode(backup_location))
else:
raise QubesException("Unknown backup format version: %s" % str(format_version))
raise QubesException(
"Unknown backup format version: %s" % str(format_version))
(restore_tmpdir, qubes_xml, header_data) = backup_restore_header(
backup_location,
@ -1736,7 +1808,8 @@ def backup_restore_prepare(backup_location, passphrase, options = None,
if vm.template is None:
vms_to_restore[vm.name]['template'] = None
else:
templatevm_name = find_template_name(vm.template.name, options['replace-template'])
templatevm_name = find_template_name(vm.template.name, options[
'replace-template'])
vms_to_restore[vm.name]['template'] = templatevm_name
if vm.netvm is None:
@ -1797,6 +1870,7 @@ def backup_restore_prepare(backup_location, passphrase, options = None,
os.unlink(qubes_xml)
return vms_to_restore
def backup_restore_print_summary(restore_info, print_callback=print_stdout):
fields = {
"qid": {"func": "vm.qid"},
@ -1860,7 +1934,7 @@ def backup_restore_print_summary(restore_info, print_callback = print_stdout):
for vm_info in restore_info.values():
# Skip non-VM here
if not 'vm' in vm_info:
if 'vm' not in vm_info:
continue
vm = vm_info['vm']
s = ""
@ -1898,17 +1972,18 @@ def backup_restore_print_summary(restore_info, print_callback = print_stdout):
print_callback(s)
def backup_restore_do(restore_info,
host_collection=None, print_callback=print_stdout,
error_callback=print_stderr, progress_callback=None,
):
global running_backup_operation
### Private functions begin
# Private functions begin
def restore_vm_dir_v1(backup_dir, src_dir, dst_dir):
backup_src_dir = src_dir.replace (system_path["qubes_base_dir"], backup_dir)
backup_src_dir = src_dir.replace(system_path["qubes_base_dir"],
backup_dir)
# We prefer to use Linux's cp, because it nicely handles sparse files
retcode = subprocess.call(["cp", "-rp", backup_src_dir, dst_dir])
@ -1916,7 +1991,8 @@ def backup_restore_do(restore_info,
raise QubesException(
"*** Error while copying file {0} to {1}".format(backup_src_dir,
dst_dir))
### Private functions end
# Private functions end
options = restore_info['$OPTIONS$']
backup_location = options['location']
@ -1986,7 +2062,8 @@ def backup_restore_do(restore_info,
raise
else:
if callable(print_callback):
print_callback("Some errors occurred during data extraction, "
print_callback(
"Some errors occurred during data extraction, "
"continuing anyway to restore at least some "
"VMs")
else:
@ -2012,8 +2089,10 @@ def backup_restore_do(restore_info,
if not vm.__class__ == vm_class:
continue
if callable(print_callback):
print_callback("-> Restoring {type} {0}...".format(vm.name, type=vm_class_name))
retcode = subprocess.call (["mkdir", "-p", os.path.dirname(vm.dir_path)])
print_callback("-> Restoring {type} {0}...".
format(vm.name, type=vm_class_name))
retcode = subprocess.call(
["mkdir", "-p", os.path.dirname(vm.dir_path)])
if retcode != 0:
error_callback("*** Cannot create directory: {0}?!".format(
vm.dir_path))
@ -2054,8 +2133,8 @@ def backup_restore_do(restore_info,
# defined - accessing it touches non-existent '_kernel'
if not isinstance(vm, QubesVmClasses['QubesHVm']):
# TODO: add a setting for this?
if vm.kernel and vm.kernel not in os.listdir(system_path[
'qubes_kernels_base_dir']):
if vm.kernel and vm.kernel not in \
os.listdir(system_path['qubes_kernels_base_dir']):
if callable(print_callback):
print_callback("WARNING: Kernel %s not installed, "
"using default one" % vm.kernel)
@ -2071,7 +2150,8 @@ def backup_restore_do(restore_info,
new_vm.appmenus_create(verbose=callable(print_callback))
except Exception as err:
error_callback("ERROR during appmenu restore: {0}".format(err))
error_callback("*** VM '{0}' will not have appmenus".format(vm.name))
error_callback(
"*** VM '{0}' will not have appmenus".format(vm.name))
# Set network dependencies - only non-default netvm setting
for vm in vms.values():
@ -2098,7 +2178,6 @@ def backup_restore_do(restore_info,
else:
raise BackupCanceledError("Restore canceled")
# ... and dom0 home as last step
if 'dom0' in restore_info.keys() and restore_info['dom0']['good-to-go']:
backup_path = restore_info['dom0']['subdir']
@ -2108,18 +2187,24 @@ def backup_restore_do(restore_info,
backup_dom0_home_dir = os.path.join(backup_location, backup_path)
else:
backup_dom0_home_dir = os.path.join(restore_tmpdir, backup_path)
restore_home_backupdir = "home-pre-restore-{0}".format (time.strftime("%Y-%m-%d-%H%M%S"))
restore_home_backupdir = "home-pre-restore-{0}".format(
time.strftime("%Y-%m-%d-%H%M%S"))
if callable(print_callback):
print_callback("-> Restoring home of user '{0}'...".format(local_user))
print_callback("--> Existing files/dirs backed up in '{0}' dir".format(restore_home_backupdir))
print_callback(
"-> Restoring home of user '{0}'...".format(local_user))
print_callback(
"--> Existing files/dirs backed up in '{0}' dir".format(
restore_home_backupdir))
os.mkdir(home_dir + '/' + restore_home_backupdir)
for f in os.listdir(backup_dom0_home_dir):
home_file = home_dir + '/' + f
if os.path.exists(home_file):
os.rename(home_file, home_dir + '/' + restore_home_backupdir + '/' + f)
os.rename(home_file,
home_dir + '/' + restore_home_backupdir + '/' + f)
if format_version == 1:
retcode = subprocess.call (["cp", "-nrp", backup_dom0_home_dir + '/' + f, home_file])
retcode = subprocess.call(
["cp", "-nrp", backup_dom0_home_dir + '/' + f, home_file])
elif format_version >= 2:
shutil.move(backup_dom0_home_dir + '/' + f, home_file)
retcode = subprocess.call(['sudo', 'chown', '-R', local_user, home_dir])