Qubes/core-admin
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

backup: implement header restoration for the new backup format

Browse Source
This commit is contained in:
Olivier MEDOC 2013-09-25 09:51:17 +02:00
parent 361741b8aa
commit c805ff6aeb
3 changed files with 87 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
dom0/qvm-core/qubes.py
View File

@ -285,6 +285,9 @@ class QubesVm(object):
# for backward compatibility (or another rare case): kernel=None -> kernel in VM dir # for backward compatibility (or another rare case): kernel=None -> kernel in VM dir
'self.dir_path + "/" + default_kernels_subdir' }, 'self.dir_path + "/" + default_kernels_subdir' },
"_start_guid_first": { 'eval': 'False' }, "_start_guid_first": { 'eval': 'False' },
"backup_content" : { 'default': False },
"backup_size" : { 'default': 0, "eval": "int(value)" },
"backup_path" : { 'default': "" },
} }
### Mark attrs for XML inclusion ### Mark attrs for XML inclusion
@ -293,7 +296,8 @@ class QubesVm(object):
'uses_default_kernel', 'kernel', 'uses_default_kernelopts',\ 'uses_default_kernel', 'kernel', 'uses_default_kernelopts',\
'kernelopts', 'services', 'installed_by_rpm',\ 'kernelopts', 'services', 'installed_by_rpm',\
'uses_default_netvm', 'include_in_backups', 'debug',\ 'uses_default_netvm', 'include_in_backups', 'debug',\
'default_user', 'qrexec_timeout' ]: 'default_user', 'qrexec_timeout',
'backup_content', 'backup_size', 'backup_path' ]:
attrs[prop]['save'] = 'str(self.%s)' % prop attrs[prop]['save'] = 'str(self.%s)' % prop
# Simple paths # Simple paths
for prop in ['conf_file', 'root_img', 'volatile_img', 'private_img']: for prop in ['conf_file', 'root_img', 'volatile_img', 'private_img']:

137
dom0/qvm-core/qubesutils.py
View File

@ -789,12 +789,11 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca
if exclude_list is None: if exclude_list is None:
exclude_list = [] exclude_list = []
qvm_collection = None
if vms_list is None: if vms_list is None:
qvm_collection = QubesVmCollection() qvm_collection = QubesVmCollection()
qvm_collection.lock_db_for_reading() qvm_collection.lock_db_for_writing()
qvm_collection.load() qvm_collection.load()
# FIXME: should be after backup completed
qvm_collection.unlock_db()
all_vms = [vm for vm in qvm_collection.values()] all_vms = [vm for vm in qvm_collection.values()]
appvms_to_backup = [vm for vm in all_vms if vm.is_appvm() and not vm.internal] appvms_to_backup = [vm for vm in all_vms if vm.is_appvm() and not vm.internal]
@ -908,14 +907,13 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca
print_callback(s) print_callback(s)
# Build Backup VMs reference file vm.backup_content = True
# Format: vm_name:tarball_path\n vm.backup_size = vm.get_disk_utilization()
backup_reference_file = open(os.path.join(qubes_base_dir,"backup_targets"),'w') vm.backup_path = vm.dir_path.split(qubes_base_dir)[1]
for vm in vms_for_backup:
backup_reference_file.write(vm.name+":"+vm.dir_path.split(qubes_base_dir)[1]+":"+str(vm.get_disk_utilization())+"\n") qvm_collection.save()
backup_reference_file.flush() # FIXME: should be after backup completed
backup_reference_file.close() qvm_collection.unlock_db()
#files_to_backup = file_to_backup(backup_reference_file.name,os.stat(backup_reference_file.name).st_size) + files_to_backup
# Dom0 user home # Dom0 user home
if not 'dom0' in exclude_list: if not 'dom0' in exclude_list:
@ -1306,6 +1304,8 @@ def wait_backup_feedback(progress_callback, in_stream, streamproc, backup_target
def restore_vm_dir (backup_dir, src_dir, dst_dir, vm_spec, print_callback=None, error_callback=None, encrypted=False, appvm=None): def restore_vm_dir (backup_dir, src_dir, dst_dir, vm_spec, print_callback=None, error_callback=None, encrypted=False, appvm=None):
#backup_src_dir = src_dir.replace (qubes_base_dir, backup_dir) #backup_src_dir = src_dir.replace (qubes_base_dir, backup_dir)
print "Restore vm dir:",backup_dir, src_dir, dst_dir, vm_spec
vmproc = None vmproc = None
if appvm != None: if appvm != None:
@ -1378,11 +1378,29 @@ def backup_restore_set_defaults(options):
return options return options
def backup_restore_header(restore_target, encrypt=False, appvm=None): def load_hmac(hmac):
hmac = hmac.strip(" \t\r\n").split("=")
if len(hmac) > 1:
hmac = hmac[1].strip()
else:
raise QubesException("ERROR: invalid hmac file content")
return hmac
def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None):
# Simulate dd if=backup_file count=10 | file - # Simulate dd if=backup_file count=10 | file -
# Simulate dd if=backup_file count=10 | gpg2 -d | tar xzv -O # Simulate dd if=backup_file count=10 | gpg2 -d | tar xzv -O
# analysis = subprocess.Popen() # analysis = subprocess.Popen()
vmproc = None vmproc = None
import tempfile
feedback_file = tempfile.NamedTemporaryFile()
backup_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/restore_")
# Tar with tapelength does not deals well with stdout (close stdout between two tapes)
# For this reason, we will use named pipes instead
print "Working in",backup_tmpdir
if appvm != None: if appvm != None:
# Prepare the backup target (Qubes service call) # Prepare the backup target (Qubes service call)
restore_command = "QUBESRPC qubes.Restore none" restore_command = "QUBESRPC qubes.Restore none"
@ -1402,7 +1420,7 @@ def backup_restore_header(restore_target, encrypt=False, appvm=None):
vmproc = vm.run(command = restore_command, passio_popen = True) vmproc = vm.run(command = restore_command, passio_popen = True)
vmproc.stdin.write(restore_target.replace("\r","").replace("\n","")+"\n") vmproc.stdin.write(restore_target.replace("\r","").replace("\n","")+"\n")
headers = vmproc.stdout.read(4096*4) headers = vmproc.stdout.read(4096*64)
vmproc.terminate() vmproc.terminate()
if len(headers) <= 0: if len(headers) <= 0:
@ -1414,46 +1432,58 @@ def backup_restore_header(restore_target, encrypt=False, appvm=None):
raise QubesException("ERROR: the backup directory {0} does not exists".format(restore_target)) raise QubesException("ERROR: the backup directory {0} does not exists".format(restore_target))
fp = open(restore_target,'rb') fp = open(restore_target,'rb')
headers = fp.read(4096*4) headers = fp.read(4096*16)
if encrypt: # FIXME: Use a safer program such as cpio, modified uncompress.c, or try to extract it from the APPVM directly
command = subprocess.Popen(['gpg2','--decrypt'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) command = subprocess.Popen(['tar', '-i', '-xv', '-C', backup_tmpdir, 'qubes.xml.*'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
stdout,stderr = command.communicate(headers)
if len(stdout) > 0:
headers = stdout
else:
print stderr
raise QubesException("ERROR: unable to decrypt the backup {0}. Is it really encrypted?".format(restore_target))
command = subprocess.Popen(['tar', 'xzv', '-O', 'backup_targets','qubes.xml'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
headers,stderr = command.communicate(headers) headers,stderr = command.communicate(headers)
if len(headers) <= 0: if len(headers) <= 0:
print stderr print stderr
raise QubesException("ERROR: unable to read the qubes backup file {0}. Is it really a backup?".format(restore_target)) raise QubesException("ERROR: unable to read the qubes backup file {0}. Is it really a backup?".format(restore_target))
vms_in_backup = {} print "Retrieved headers",headers
for vm in headers.split("\n"):
match = re.match("^(?P<name>[^:]+):(?P<path>[^:]+):(?P<size>[0-9]+)$",vm,re.MULTILINE) for filename in headers.splitlines():
if match: print "Loading hmac for file",filename
item = match.groupdict() hmac = load_hmac(open(os.path.join(backup_tmpdir,filename+".hmac"),'r').read())
item["size"] = int(item["size"])
vms_in_backup[item["name"]] = item print "Verifying file",filename
headers = headers.replace(vm,"") hmac_proc = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], stdin=open(os.path.join(backup_tmpdir,filename),'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout,stderr = hmac_proc.communicate()
if len(stderr) > 0:
raise QubesException("ERROR: verify file {0}: {1}".format((filename,stderr)))
else: else:
break if len(hmac) > 0 and load_hmac(stdout) == hmac:
print "File verification OK -> Extracting archive",filename
# FIXME: implement function for encrypted file
command = subprocess.Popen(['tar', '-xvf', os.path.join(backup_tmpdir,filename), '-C', backup_tmpdir],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
headers,stderr = command.communicate(headers)
if len(headers) <= 0:
print stderr
raise QubesException("ERROR: unable to read the qubes backup file {0}. Is it really a backup?".format(restore_target))
else:
return os.path.join(backup_tmpdir,headers.strip(" \r\n\t"))
return vms_in_backup, headers else:
raise QubesException("ERROR: invalid hmac for file {0}: {1}. Is the passphrase correct?".format(filename,load_hmac(stdout)))
def backup_restore_prepare(backup_dir, backup_content, qubes_xml, options = {}, host_collection = None, encrypt=False, appvm=None): return None
def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, host_collection = None, encrypt=False, appvm=None):
# Defaults # Defaults
backup_restore_set_defaults(options) backup_restore_set_defaults(options)
#### Private functions begin #### Private functions begin
def is_vm_included_in_backup (backup_dir, vm): def is_vm_included_in_backup (backup_dir, vm):
for item in backup_content.keys(): if vm.qid == 0:
if vm.name == item: # Dom0 is not included, obviously
return True
return False return False
if vm.backup_content:
return True
else:
return False
def find_template_name(template, replaces): def find_template_name(template, replaces):
rx_replace = re.compile("(.*):(.*)") rx_replace = re.compile("(.*):(.*)")
for r in replaces: for r in replaces:
@ -1463,14 +1493,9 @@ def backup_restore_prepare(backup_dir, backup_content, qubes_xml, options = {},
return template return template
#### Private functions end #### Private functions end
print "Loading file",qubes_xml
backup_collection = QubesVmCollection() backup_collection = QubesVmCollection(store_filename = qubes_xml)
import StringIO backup_collection.lock_db_for_reading()
backup_collection.qubes_store_file=StringIO.StringIO(qubes_xml)
#backup_collection.lock_db_for_reading()
backup_collection.load() backup_collection.load()
if host_collection is None: if host_collection is None:
@ -1480,11 +1505,9 @@ def backup_restore_prepare(backup_dir, backup_content, qubes_xml, options = {},
host_collection.unlock_db() host_collection.unlock_db()
backup_vms_list = [vm for vm in backup_collection.values()] backup_vms_list = [vm for vm in backup_collection.values()]
host_vms_list = [vm for vm in host_collection.values()] host_vms_list = [vm for vm in host_collection.values()]
vms_to_restore = {} vms_to_restore = {}
there_are_conflicting_vms = False there_are_conflicting_vms = False
there_are_missing_templates = False there_are_missing_templates = False
there_are_missing_netvms = False there_are_missing_netvms = False
@ -1493,6 +1516,7 @@ def backup_restore_prepare(backup_dir, backup_content, qubes_xml, options = {},
# ... and the actual data # ... and the actual data
for vm in backup_vms_list: for vm in backup_vms_list:
if is_vm_included_in_backup (backup_dir, vm): if is_vm_included_in_backup (backup_dir, vm):
print vm.name,"is included in backup"
vms_to_restore[vm.name] = {} vms_to_restore[vm.name] = {}
vms_to_restore[vm.name]['vm'] = vm; vms_to_restore[vm.name]['vm'] = vm;
@ -1676,20 +1700,7 @@ def backup_restore_print_summary(restore_info, print_callback = print_stdout):
print_callback(s) print_callback(s)
def backup_restore_do(backup_dir, restore_info, restore_vms, host_collection = None, print_callback = print_stdout, error_callback = print_stderr, encrypted=False, appvm = None ): def backup_restore_do(backup_dir, restore_info, host_collection = None, print_callback = print_stdout, error_callback = print_stderr, encrypted=False, appvm = None ):
### Private functions begin
'''
def restore_vm_dir (backup_dir, src_dir, dst_dir, print_callback, error_callback, encrypted, appvm):
backup_src_dir = src_dir.replace (qubes_base_dir, backup_dir)
# We prefer to use Linux's cp, because it nicely handles sparse files
retcode = subprocess.call (["cp", "-rp", backup_src_dir, dst_dir])
if retcode != 0:
raise QubesException("*** Error while copying file {0} to {1}".format(backup_src_dir, dest_dir))
'''
### Private functions end
lock_obtained = False lock_obtained = False
if host_collection is None: if host_collection is None:
@ -1731,7 +1742,7 @@ def backup_restore_do(backup_dir, restore_info, restore_vms, host_collection = N
template=template, template=template,
installed_by_rpm=False) installed_by_rpm=False)
restore_vm_dir (backup_dir, vm.dir_path, os.path.dirname(new_vm.dir_path), restore_vms[vm.name], print_callback, error_callback, encrypted, appvm) restore_vm_dir (backup_dir, vm.dir_path, os.path.dirname(new_vm.dir_path), vm, print_callback, error_callback, encrypted, appvm)
new_vm.verify_files() new_vm.verify_files()
except Exception as err: except Exception as err:

9
dom0/qvm-tools/qvm-backup-restore
View File

@ -95,12 +95,15 @@ def main():
restore_options['exclude'] = options.exclude restore_options['exclude'] = options.exclude
passphrase = raw_input("Please enter the pass phrase that will be used to decrypt/verify the backup:\n")
passphrase = passphrase.replace("\r","").replace("\n","")
print >> sys.stderr, "Checking backup content..." print >> sys.stderr, "Checking backup content..."
encrypted, vms, qubes_xml = backup_restore_header(backup_dir, options.decrypt, appvm=options.appvm) qubes_xml = backup_restore_header(backup_dir, passphrase, options.decrypt, appvm=options.appvm)
restore_info = None restore_info = None
try: try:
restore_info = backup_restore_prepare(backup_dir, vms, qubes_xml, options=restore_options, host_collection=host_collection, encrypt=encrypted, appvm=options.appvm) restore_info = backup_restore_prepare(backup_dir, qubes_xml, passphrase, options=restore_options, host_collection=host_collection, encrypt=options.decrypt, appvm=options.appvm)
except QubesException as e: except QubesException as e:
print >> sys.stderr, "ERROR: %s" % str(e) print >> sys.stderr, "ERROR: %s" % str(e)
exit(1) exit(1)
@ -189,7 +192,7 @@ def main():
exit (0) exit (0)
backup_restore_do(backup_dir, restore_info, vms, host_collection=host_collection, encrypted=encrypted, appvm=options.appvm) backup_restore_do(backup_dir, restore_info, host_collection=host_collection, encrypted=options.decrypt, appvm=options.appvm)
host_collection.unlock_db() host_collection.unlock_db()