backups: minor fixes, mostly formatting
This commit is contained in:
parent
6b4c96d195
commit
b85cd0448f
120
core/backup.py
120
core/backup.py
@ -347,23 +347,24 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
|
|||||||
vmproc = None
|
vmproc = None
|
||||||
if appvm != None:
|
if appvm != None:
|
||||||
# Prepare the backup target (Qubes service call)
|
# Prepare the backup target (Qubes service call)
|
||||||
backup_target = "QUBESRPC qubes.Backup none"
|
backup_target = "QUBESRPC qubes.Backup dom0"
|
||||||
|
|
||||||
# If APPVM, STDOUT is a PIPE
|
# If APPVM, STDOUT is a PIPE
|
||||||
vmproc = appvm.run(command = backup_target, passio_popen = True, passio_stderr = True)
|
vmproc = appvm.run(command=backup_target, passio_popen=True,
|
||||||
vmproc.stdin.write(base_backup_dir.\
|
passio_stderr=True)
|
||||||
replace("\r","").replace("\n","")+"\n")
|
vmproc.stdin.write(base_backup_dir.
|
||||||
|
replace("\r", "").replace("\n", "")+"\n")
|
||||||
backup_stdout = vmproc.stdin
|
backup_stdout = vmproc.stdin
|
||||||
else:
|
else:
|
||||||
# Prepare the backup target (local file)
|
# Prepare the backup target (local file)
|
||||||
backup_target = base_backup_dir + "/qubes-{0}".\
|
backup_target = base_backup_dir + "/qubes-{0}". \
|
||||||
format (time.strftime("%Y-%m-%d-%H%M%S"))
|
format(time.strftime("%Y-%m-%d-%H%M%S"))
|
||||||
|
|
||||||
# Create the target directory
|
# Create the target directory
|
||||||
if not os.path.exists (base_backup_dir):
|
if not os.path.exists (base_backup_dir):
|
||||||
raise QubesException(
|
raise QubesException(
|
||||||
"ERROR: the backup directory {0} does not exists".\
|
"ERROR: the backup directory {0} does not exists".
|
||||||
format(base_backup_dir))
|
format(base_backup_dir))
|
||||||
|
|
||||||
# If not APPVM, STDOUT is a local file
|
# If not APPVM, STDOUT is a local file
|
||||||
backup_stdout = open(backup_target,'wb')
|
backup_stdout = open(backup_target,'wb')
|
||||||
@ -382,7 +383,7 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
|
|||||||
if BACKUP_DEBUG:
|
if BACKUP_DEBUG:
|
||||||
print "Working in", backup_tmpdir
|
print "Working in", backup_tmpdir
|
||||||
|
|
||||||
backup_pipe = os.path.join(backup_tmpdir,"backup_pipe")
|
backup_pipe = os.path.join(backup_tmpdir, "backup_pipe")
|
||||||
if BACKUP_DEBUG:
|
if BACKUP_DEBUG:
|
||||||
print "Creating pipe in:", backup_pipe
|
print "Creating pipe in:", backup_pipe
|
||||||
os.mkfifo(backup_pipe)
|
os.mkfifo(backup_pipe)
|
||||||
@ -398,7 +399,7 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
|
|||||||
progress = blocks_backedup / float(total_backup_sz)
|
progress = blocks_backedup / float(total_backup_sz)
|
||||||
progress_callback(int(round(progress*100,2)))
|
progress_callback(int(round(progress*100,2)))
|
||||||
|
|
||||||
to_send = Queue(10)
|
to_send = Queue(10)
|
||||||
send_proc = SendWorker(to_send, backup_tmpdir, backup_stdout)
|
send_proc = SendWorker(to_send, backup_tmpdir, backup_stdout)
|
||||||
send_proc.start()
|
send_proc.start()
|
||||||
|
|
||||||
@ -407,8 +408,8 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
|
|||||||
print "Backing up", filename
|
print "Backing up", filename
|
||||||
|
|
||||||
backup_tempfile = os.path.join(backup_tmpdir,
|
backup_tempfile = os.path.join(backup_tmpdir,
|
||||||
filename["subdir"],
|
filename["subdir"],
|
||||||
os.path.basename(filename["path"]))
|
os.path.basename(filename["path"]))
|
||||||
if BACKUP_DEBUG:
|
if BACKUP_DEBUG:
|
||||||
print "Using temporary location:", backup_tempfile
|
print "Using temporary location:", backup_tempfile
|
||||||
|
|
||||||
@ -420,12 +421,12 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
|
|||||||
# be verified before untaring this.
|
# be verified before untaring this.
|
||||||
# Prefix the path in archive with filename["subdir"] to have it verified during untar
|
# Prefix the path in archive with filename["subdir"] to have it verified during untar
|
||||||
tar_cmdline = ["tar", "-Pc", '--sparse',
|
tar_cmdline = ["tar", "-Pc", '--sparse',
|
||||||
"-f", backup_pipe,
|
"-f", backup_pipe,
|
||||||
'--tape-length', str(100000),
|
'--tape-length', str(100000),
|
||||||
'-C', os.path.dirname(filename["path"]),
|
'-C', os.path.dirname(filename["path"]),
|
||||||
'--xform', 's:^[^/]:%s\\0:' % filename["subdir"],
|
'--xform', 's:^[^/]:%s\\0:' % filename["subdir"],
|
||||||
os.path.basename(filename["path"])
|
os.path.basename(filename["path"])
|
||||||
]
|
]
|
||||||
|
|
||||||
if BACKUP_DEBUG:
|
if BACKUP_DEBUG:
|
||||||
print " ".join(tar_cmdline)
|
print " ".join(tar_cmdline)
|
||||||
@ -440,9 +441,7 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
|
|||||||
# subprocesses
|
# subprocesses
|
||||||
i = 0
|
i = 0
|
||||||
run_error = "paused"
|
run_error = "paused"
|
||||||
running = []
|
|
||||||
while run_error == "paused":
|
while run_error == "paused":
|
||||||
|
|
||||||
pipe = open(backup_pipe,'rb')
|
pipe = open(backup_pipe,'rb')
|
||||||
|
|
||||||
# Start HMAC
|
# Start HMAC
|
||||||
@ -459,7 +458,8 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
|
|||||||
'total_backup_sz': total_backup_sz,
|
'total_backup_sz': total_backup_sz,
|
||||||
'hmac': hmac,
|
'hmac': hmac,
|
||||||
'vmproc': vmproc,
|
'vmproc': vmproc,
|
||||||
'addproc': tar_sparse
|
'addproc': tar_sparse,
|
||||||
|
'progress_callback': compute_progress,
|
||||||
}
|
}
|
||||||
if encrypt:
|
if encrypt:
|
||||||
# Start encrypt
|
# Start encrypt
|
||||||
@ -470,19 +470,16 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
|
|||||||
(["-z"] if compress else []),
|
(["-z"] if compress else []),
|
||||||
stdin=pipe, stdout=subprocess.PIPE)
|
stdin=pipe, stdout=subprocess.PIPE)
|
||||||
run_error = wait_backup_feedback(
|
run_error = wait_backup_feedback(
|
||||||
progress_callback=compute_progress,
|
|
||||||
in_stream=encryptor.stdout, streamproc=encryptor,
|
in_stream=encryptor.stdout, streamproc=encryptor,
|
||||||
**common_args)
|
**common_args)
|
||||||
elif compress:
|
elif compress:
|
||||||
compressor = subprocess.Popen (["gzip"],
|
compressor = subprocess.Popen (["gzip"],
|
||||||
stdin=pipe, stdout=subprocess.PIPE)
|
stdin=pipe, stdout=subprocess.PIPE)
|
||||||
run_error = wait_backup_feedback(
|
run_error = wait_backup_feedback(
|
||||||
progress_callback=compute_progress,
|
|
||||||
in_stream=compressor.stdout, streamproc=compressor,
|
in_stream=compressor.stdout, streamproc=compressor,
|
||||||
**common_args)
|
**common_args)
|
||||||
else:
|
else:
|
||||||
run_error = wait_backup_feedback(
|
run_error = wait_backup_feedback(
|
||||||
progress_callback=compute_progress,
|
|
||||||
in_stream=pipe, streamproc=None,
|
in_stream=pipe, streamproc=None,
|
||||||
**common_args)
|
**common_args)
|
||||||
|
|
||||||
@ -494,7 +491,7 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
|
|||||||
if len(run_error) > 0:
|
if len(run_error) > 0:
|
||||||
send_proc.terminate()
|
send_proc.terminate()
|
||||||
if run_error == "VM" and vmproc:
|
if run_error == "VM" and vmproc:
|
||||||
raise QubesException("Failed to write the backup, the VM output:\n" +
|
raise QubesException("Failed to write the backup, VM output:\n" +
|
||||||
vmproc.stderr.read())
|
vmproc.stderr.read())
|
||||||
else:
|
else:
|
||||||
raise QubesException("Failed to perform backup: error in "+ \
|
raise QubesException("Failed to perform backup: error in "+ \
|
||||||
@ -523,7 +520,7 @@ def backup_do(base_backup_dir, files_to_backup, passphrase,
|
|||||||
# Send the HMAC to the backup target
|
# Send the HMAC to the backup target
|
||||||
to_send.put(os.path.relpath(chunkfile, backup_tmpdir)+".hmac")
|
to_send.put(os.path.relpath(chunkfile, backup_tmpdir)+".hmac")
|
||||||
|
|
||||||
if tar_sparse.poll() == None:
|
if tar_sparse.poll() is None:
|
||||||
# Release the next chunk
|
# Release the next chunk
|
||||||
if BACKUP_DEBUG:
|
if BACKUP_DEBUG:
|
||||||
print "Release next chunk for process:", tar_sparse.poll()
|
print "Release next chunk for process:", tar_sparse.poll()
|
||||||
@ -641,7 +638,7 @@ def verify_hmac(filename, hmacfile, passphrase):
|
|||||||
hmac_stdout, hmac_stderr = hmac_proc.communicate()
|
hmac_stdout, hmac_stderr = hmac_proc.communicate()
|
||||||
|
|
||||||
if len(hmac_stderr) > 0:
|
if len(hmac_stderr) > 0:
|
||||||
raise QubesException("ERROR: verify file {0}: {1}".format((filename, hmac_stderr)))
|
raise QubesException("ERROR: verify file {0}: {1}".format(filename, hmac_stderr))
|
||||||
else:
|
else:
|
||||||
if BACKUP_DEBUG:
|
if BACKUP_DEBUG:
|
||||||
print "Loading hmac for file " + filename
|
print "Loading hmac for file " + filename
|
||||||
@ -690,10 +687,11 @@ class ExtractWorker(Process):
|
|||||||
os.mkfifo(self.restore_pipe)
|
os.mkfifo(self.restore_pipe)
|
||||||
|
|
||||||
def compute_progress(self, new_size, total_size):
|
def compute_progress(self, new_size, total_size):
|
||||||
self.blocks_backedup += new_size
|
if self.progress_callback:
|
||||||
progress = self.blocks_backedup / float(self.total_size)
|
self.blocks_backedup += new_size
|
||||||
progress = int(round(progress*100,2))
|
progress = self.blocks_backedup / float(self.total_size)
|
||||||
self.progress_callback(progress)
|
progress = int(round(progress*100,2))
|
||||||
|
self.progress_callback(progress)
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
try:
|
try:
|
||||||
@ -810,13 +808,15 @@ class ExtractWorker(Process):
|
|||||||
|
|
||||||
os.unlink(self.restore_pipe)
|
os.unlink(self.restore_pipe)
|
||||||
|
|
||||||
if self.tar2_process != None:
|
if self.tar2_process is not None:
|
||||||
if filename == "ERROR":
|
if filename == "ERROR":
|
||||||
self.tar2_process.terminate()
|
self.tar2_process.terminate()
|
||||||
if self.tar2_process.wait() != 0:
|
if self.tar2_process.wait() != 0:
|
||||||
raise QubesException(
|
raise QubesException(
|
||||||
"ERROR: unable to extract files for {0}.".\
|
"ERROR: unable to extract files for {0}.{1}".
|
||||||
format(self.tar2_current_file))
|
format(self.tar2_current_file,
|
||||||
|
(" Perhaps the backup is encrypted?"
|
||||||
|
if not self.encrypted else "")))
|
||||||
else:
|
else:
|
||||||
# Finished extracting the tar file
|
# Finished extracting the tar file
|
||||||
self.tar2_process = None
|
self.tar2_process = None
|
||||||
@ -830,10 +830,6 @@ def restore_vm_dirs (backup_source, restore_tmpdir, passphrase, vms_dirs, vms,
|
|||||||
compressed = False):
|
compressed = False):
|
||||||
|
|
||||||
# Setup worker to extract encrypted data chunks to the restore dirs
|
# Setup worker to extract encrypted data chunks to the restore dirs
|
||||||
if progress_callback == None:
|
|
||||||
def progress_callback(data):
|
|
||||||
pass
|
|
||||||
|
|
||||||
to_extract = Queue()
|
to_extract = Queue()
|
||||||
extract_proc = ExtractWorker(queue=to_extract,
|
extract_proc = ExtractWorker(queue=to_extract,
|
||||||
base_dir=restore_tmpdir,
|
base_dir=restore_tmpdir,
|
||||||
@ -896,6 +892,7 @@ def restore_vm_dirs (backup_source, restore_tmpdir, passphrase, vms_dirs, vms,
|
|||||||
filelist_pipe = command.stdout
|
filelist_pipe = command.stdout
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
filename = None
|
||||||
while True:
|
while True:
|
||||||
filename = filelist_pipe.readline().strip(" \t\r\n")
|
filename = filelist_pipe.readline().strip(" \t\r\n")
|
||||||
|
|
||||||
@ -1120,7 +1117,7 @@ def backup_restore_prepare(backup_location, passphrase, options = {},
|
|||||||
|
|
||||||
# Format versions:
|
# Format versions:
|
||||||
# 1 - Qubes R1, Qubes R2 beta1, beta2
|
# 1 - Qubes R1, Qubes R2 beta1, beta2
|
||||||
# 2 - Qubes R2 beta3
|
# 2 - Qubes R2 beta3+
|
||||||
|
|
||||||
if format_version is None:
|
if format_version is None:
|
||||||
format_version = backup_detect_format_version(backup_location)
|
format_version = backup_detect_format_version(backup_location)
|
||||||
@ -1364,20 +1361,21 @@ def backup_restore_do(restore_info,
|
|||||||
lock_obtained = True
|
lock_obtained = True
|
||||||
|
|
||||||
# Perform VM restoration in backup order
|
# Perform VM restoration in backup order
|
||||||
if format_version == 2:
|
vms_dirs = []
|
||||||
vms_dirs = []
|
vms_size = 0
|
||||||
vms_size = 0
|
vms = {}
|
||||||
vms = {}
|
for vm_info in restore_info.values():
|
||||||
for vm_info in restore_info.values():
|
if 'vm' not in vm_info:
|
||||||
if 'vm' not in vm_info:
|
continue
|
||||||
continue
|
if not vm_info['good-to-go']:
|
||||||
if not vm_info['good-to-go']:
|
continue
|
||||||
continue
|
vm = vm_info['vm']
|
||||||
vm = vm_info['vm']
|
if format_version == 2:
|
||||||
vms_size += vm.backup_size
|
vms_size += vm.backup_size
|
||||||
vms_dirs.append(vm.backup_path)
|
vms_dirs.append(vm.backup_path)
|
||||||
vms[vm.name] = vm
|
vms[vm.name] = vm
|
||||||
|
|
||||||
|
if format_version == 2:
|
||||||
if 'dom0' in restore_info.keys() and restore_info['dom0']['good-to-go']:
|
if 'dom0' in restore_info.keys() and restore_info['dom0']['good-to-go']:
|
||||||
vms_dirs.append('dom0-home')
|
vms_dirs.append('dom0-home')
|
||||||
vms_size += restore_info['dom0']['size']
|
vms_size += restore_info['dom0']['size']
|
||||||
@ -1398,12 +1396,7 @@ def backup_restore_do(restore_info,
|
|||||||
# Add VM in right order
|
# Add VM in right order
|
||||||
for (vm_class_name, vm_class) in sorted(QubesVmClasses.items(),
|
for (vm_class_name, vm_class) in sorted(QubesVmClasses.items(),
|
||||||
key=lambda _x: _x[1].load_order):
|
key=lambda _x: _x[1].load_order):
|
||||||
for vm_info in restore_info.values():
|
for vm in vms.values():
|
||||||
if 'vm' not in vm_info:
|
|
||||||
continue
|
|
||||||
if not vm_info['good-to-go']:
|
|
||||||
continue
|
|
||||||
vm = vm_info['vm']
|
|
||||||
if not vm.__class__ == vm_class:
|
if not vm.__class__ == vm_class:
|
||||||
continue
|
continue
|
||||||
print_callback("-> Restoring {type} {0}...".format(vm.name, type=vm_class_name))
|
print_callback("-> Restoring {type} {0}...".format(vm.name, type=vm_class_name))
|
||||||
@ -1416,7 +1409,7 @@ def backup_restore_do(restore_info,
|
|||||||
|
|
||||||
template = None
|
template = None
|
||||||
if vm.template is not None:
|
if vm.template is not None:
|
||||||
template_name = vm_info['template']
|
template_name = restore_info[vm.name]['template']
|
||||||
template = host_collection.get_vm_by_name(template_name)
|
template = host_collection.get_vm_by_name(template_name)
|
||||||
|
|
||||||
new_vm = None
|
new_vm = None
|
||||||
@ -1457,19 +1450,18 @@ def backup_restore_do(restore_info,
|
|||||||
error_callback("*** VM '{0}' will not have appmenus".format(vm.name))
|
error_callback("*** VM '{0}' will not have appmenus".format(vm.name))
|
||||||
|
|
||||||
# Set network dependencies - only non-default netvm setting
|
# Set network dependencies - only non-default netvm setting
|
||||||
for vm_info in restore_info.values():
|
for vm in vms.values():
|
||||||
if 'vm' not in vm_info:
|
|
||||||
continue
|
|
||||||
if not vm_info['good-to-go']:
|
|
||||||
continue
|
|
||||||
vm = vm_info['vm']
|
|
||||||
host_vm = host_collection.get_vm_by_name(vm.name)
|
host_vm = host_collection.get_vm_by_name(vm.name)
|
||||||
if host_vm is None:
|
if host_vm is None:
|
||||||
# Failed/skipped VM
|
# Failed/skipped VM
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if not vm.uses_default_netvm:
|
if not vm.uses_default_netvm:
|
||||||
host_vm.netvm = host_collection.get_vm_by_name (vm_info['netvm']) if vm_info['netvm'] is not None else None
|
if restore_info[vm.name]['netvm'] is not None:
|
||||||
|
host_vm.netvm = host_collection.get_vm_by_name (
|
||||||
|
restore_info[vm.name]['netvm'])
|
||||||
|
else:
|
||||||
|
host_vm.netvm = None
|
||||||
|
|
||||||
host_collection.save()
|
host_collection.save()
|
||||||
if lock_obtained:
|
if lock_obtained:
|
||||||
|
Loading…
Reference in New Issue
Block a user