diff --git a/core/backup.py b/qubes/backup.py similarity index 93% rename from core/backup.py rename to qubes/backup.py index 7ba8808f..09ccb11f 100644 --- a/core/backup.py +++ b/qubes/backup.py @@ -23,10 +23,7 @@ # # from __future__ import unicode_literals -from qubes import QubesException, QubesVmCollection -from qubes import QubesVmClasses -from qubes import system_path, vm_files -from qubesutils import size_to_human, print_stdout, print_stderr, get_disk_usage +from qubes.utils import size_to_human import sys import os import fcntl @@ -40,6 +37,7 @@ import pwd import errno import datetime from multiprocessing import Queue, Process +import qubes BACKUP_DEBUG = False @@ -53,9 +51,37 @@ MAX_STDERR_BYTES = 1024 # header + qubes.xml max size HEADER_QUBES_XML_MAX_SIZE = 1024 * 1024 +BLKSIZE = 512 + # global state for backup_cancel() running_backup_operation = None +def print_stdout(text): + print (text) + +def print_stderr(text): + print >> sys.stderr, (text) + +def get_disk_usage_one(st): + try: + return st.st_blocks * BLKSIZE + except AttributeError: + return st.st_size + +def get_disk_usage(path): + try: + st = os.lstat(path) + except OSError: + return 0 + + ret = get_disk_usage_one(st) + + # if path is not a directory, this is skipped + for dirpath, dirnames, filenames in os.walk(path): + for name in dirnames + filenames: + ret += get_disk_usage_one(os.lstat(os.path.join(dirpath, name))) + + return ret class BackupOperationInfo: def __init__(self): @@ -64,7 +90,7 @@ class BackupOperationInfo: self.tmpdir_to_remove = None -class BackupCanceledError(QubesException): +class BackupCanceledError(qubes.exc.QubesException): def __init__(self, msg, tmpdir=None): super(BackupCanceledError, self).__init__(msg) self.tmpdir = tmpdir @@ -86,7 +112,7 @@ def file_to_backup(file_path, subdir=None): if subdir is None: abs_file_path = os.path.abspath(file_path) - abs_base_dir = os.path.abspath(system_path["qubes_base_dir"]) + '/' + abs_base_dir = os.path.abspath(qubes.config.system_path["qubes_base_dir"]) + '/' abs_file_dir = os.path.dirname(abs_file_path) + '/' (nothing, directory, subdir) = abs_file_dir.partition(abs_base_dir) assert nothing == "" @@ -115,33 +141,35 @@ def backup_cancel(): return True -def backup_prepare(vms_list=None, exclude_list=None, +def backup_prepare(app, vms_list=None, exclude_list=None, print_callback=print_stdout, hide_vm_names=True): """ If vms = None, include all (sensible) VMs; exclude_list is always applied """ - files_to_backup = file_to_backup(system_path["qubes_store_filename"]) + # XXX hack for tests, where store filename is qubes-test.xml + qubes_xml = app._store + if os.path.basename(qubes_xml) != 'qubes.xml': + dir = tempfile.mkdtemp() + shutil.copy(qubes_xml, os.path.join(dir, 'qubes.xml')) + qubes_xml = os.path.join(dir, 'qubes.xml') + # FIXME cleanup tempdir later + + files_to_backup = file_to_backup(qubes_xml, '') if exclude_list is None: exclude_list = [] - qvm_collection = QubesVmCollection() - qvm_collection.lock_db_for_writing() - qvm_collection.load() - if vms_list is None: - all_vms = [vm for vm in qvm_collection.values()] + all_vms = [vm for vm in app.domains] selected_vms = [vm for vm in all_vms if vm.include_in_backups] appvms_to_backup = [vm for vm in selected_vms if vm.is_appvm() and not vm.internal] - netvms_to_backup = [vm for vm in selected_vms if - vm.is_netvm() and not vm.qid == 0] template_vms_worth_backingup = [vm for vm in selected_vms if ( vm.is_template() and vm.include_in_backups)] - dom0 = [qvm_collection[0]] + dom0 = [app.domains[0]] - vms_list = appvms_to_backup + netvms_to_backup + \ + vms_list = appvms_to_backup + \ template_vms_worth_backingup + dom0 vms_for_backup = vms_list @@ -207,11 +235,13 @@ def backup_prepare(vms_list=None, exclude_list=None, subdir) if os.path.exists(vm.firewall_conf): files_to_backup += file_to_backup(vm.firewall_conf, subdir) - if 'appmenus_whitelist' in vm_files and \ + if 'appmenus_whitelist' in qubes.config.vm_files and \ os.path.exists(os.path.join(vm.dir_path, - vm_files['appmenus_whitelist'])): + qubes.config.vm_files[ + 'appmenus_whitelist'])): files_to_backup += file_to_backup( - os.path.join(vm.dir_path, vm_files['appmenus_whitelist']), + os.path.join(vm.dir_path, qubes.config.vm_files[ + 'appmenus_whitelist']), subdir) if vm.updateable: @@ -255,7 +285,7 @@ def backup_prepare(vms_list=None, exclude_list=None, else: template_subdir = os.path.relpath( vm.dir_path, - system_path["qubes_base_dir"]) + '/' + qubes.config.system_path["qubes_base_dir"]) + '/' template_to_backup = [{"path": vm.dir_path + '/.', "size": vm_sz, "subdir": template_subdir}] @@ -280,7 +310,7 @@ def backup_prepare(vms_list=None, exclude_list=None, # Initialize backup flag on all VMs vms_for_backup_qid = [vm.qid for vm in vms_for_backup] - for vm in qvm_collection.values(): + for vm in app.domains: vm.backup_content = False if vm.qid == 0: # handle dom0 later @@ -292,8 +322,9 @@ def backup_prepare(vms_list=None, exclude_list=None, if hide_vm_names: vm.backup_path = 'vm%d' % vm.qid else: - vm.backup_path = os.path.relpath(vm.dir_path, - system_path["qubes_base_dir"]) + vm.backup_path = os.path.relpath( + vm.dir_path, + qubes.config.system_path["qubes_base_dir"]) # Dom0 user home if 0 in vms_for_backup_qid: @@ -309,7 +340,7 @@ def backup_prepare(vms_list=None, exclude_list=None, {"path": home_dir, "size": home_sz, "subdir": 'dom0-home/'}] files_to_backup += home_to_backup - vm = qvm_collection[0] + vm = app.domains[0] vm.backup_content = True vm.backup_size = home_sz vm.backup_path = os.path.join('dom0-home', os.path.basename(home_dir)) @@ -326,9 +357,8 @@ def backup_prepare(vms_list=None, exclude_list=None, print_callback(s) - qvm_collection.save() + app.save() # FIXME: should be after backup completed - qvm_collection.unlock_db() total_backup_sz = 0 for f in files_to_backup: @@ -355,13 +385,13 @@ def backup_prepare(vms_list=None, exclude_list=None, s += fmt.format('-') print_callback(s) - vms_not_for_backup = [vm.name for vm in qvm_collection.values() + vms_not_for_backup = [vm.name for vm in app.domains if not vm.backup_content] print_callback("VMs not selected for backup: %s" % " ".join( vms_not_for_backup)) if there_are_running_vms: - raise QubesException("Please shutdown all VMs before proceeding.") + raise qubes.exc.QubesException("Please shutdown all VMs before proceeding.") for fileinfo in files_to_backup: assert len(fileinfo["subdir"]) == 0 or fileinfo["subdir"][-1] == '/', \ @@ -406,7 +436,7 @@ class SendWorker(Process): self.queue.get() # handle only exit code 2 (tar fatal error) or # greater (call failed?) - raise QubesException( + raise qubes.exc.QubesException( "ERROR: Failed to write the backup, out of disk space? " "Check console output or ~/.xsession-errors for details.") @@ -442,11 +472,11 @@ def prepare_backup_header(target_directory, passphrase, compressed=False, stdin=open(header_file_path, "r"), stdout=open(header_file_path + ".hmac", "w")) if hmac.wait() != 0: - raise QubesException("Failed to compute hmac of header file") + raise qubes.exc.QubesException("Failed to compute hmac of header file") return HEADER_FILENAME, HEADER_FILENAME + ".hmac" -def backup_do(base_backup_dir, files_to_backup, passphrase, +def backup_do(app, base_backup_dir, files_to_backup, passphrase, progress_callback=None, encrypted=False, appvm=None, compressed=False, hmac_algorithm=DEFAULT_HMAC_ALGORITHM, crypto_algorithm=DEFAULT_CRYPTO_ALGORITHM): @@ -460,7 +490,7 @@ def backup_do(base_backup_dir, files_to_backup, passphrase, vmproc.stderr.read()) else: message = "Failed to write the backup. Out of disk space?" - raise QubesException(message) + raise qubes.exc.QubesException(message) queue.put(element) total_backup_sz = 0 @@ -497,7 +527,7 @@ def backup_do(base_backup_dir, files_to_backup, passphrase, # Create the target directory if not os.path.exists(os.path.dirname(base_backup_dir)): - raise QubesException( + raise qubes.exc.QubesException( "ERROR: the backup directory for {0} does not exists". format(base_backup_dir)) @@ -658,11 +688,11 @@ def backup_do(base_backup_dir, files_to_backup, passphrase, if run_error and run_error != "size_limit": send_proc.terminate() if run_error == "VM" and vmproc: - raise QubesException( + raise qubes.exc.QubesException( "Failed to write the backup, VM output:\n" + vmproc.stderr.read(MAX_STDERR_BYTES)) else: - raise QubesException("Failed to perform backup: error in " + + raise qubes.exc.QubesException("Failed to perform backup: error in " + run_error) # Send the chunk to the backup target @@ -711,7 +741,7 @@ def backup_do(base_backup_dir, files_to_backup, passphrase, running_backup_operation = None if send_proc.exitcode != 0: - raise QubesException( + raise qubes.exc.QubesException( "Failed to send backup: error in the sending process") if vmproc: @@ -722,16 +752,11 @@ def backup_do(base_backup_dir, files_to_backup, passphrase, vmproc.stdin.close() # Save date of last backup - qvm_collection = QubesVmCollection() - qvm_collection.lock_db_for_writing() - qvm_collection.load() - - for vm in qvm_collection.values(): + for vm in app.domains: if vm.backup_content: vm.backup_timestamp = datetime.datetime.now() - qvm_collection.save() - qvm_collection.unlock_db() + app.save() ''' @@ -829,7 +854,7 @@ def verify_hmac(filename, hmacfile, passphrase, algorithm): print "Verifying file " + filename if hmacfile != filename + ".hmac": - raise QubesException( + raise qubes.exc.QubesException( "ERROR: expected hmac for {}, but got {}". format(filename, hmacfile)) @@ -840,7 +865,7 @@ def verify_hmac(filename, hmacfile, passphrase, algorithm): hmac_stdout, hmac_stderr = hmac_proc.communicate() if len(hmac_stderr) > 0: - raise QubesException( + raise qubes.exc.QubesException( "ERROR: verify file {0}: {1}".format(filename, hmac_stderr)) else: if BACKUP_DEBUG: @@ -853,7 +878,7 @@ def verify_hmac(filename, hmacfile, passphrase, algorithm): print "File verification OK -> Sending file " + filename return True else: - raise QubesException( + raise qubes.exc.QubesException( "ERROR: invalid hmac for file {0}: {1}. " "Is the passphrase correct?". format(filename, load_hmac(hmac_stdout))) @@ -1081,7 +1106,7 @@ class ExtractWorker2(Process): self.tar2_process.wait() elif self.tar2_process.wait() != 0: self.collect_tar_output() - raise QubesException( + raise qubes.exc.QubesException( "unable to extract files for {0}.{1} Tar command " "output: %s". format(self.tar2_current_file, @@ -1241,7 +1266,7 @@ class ExtractWorker3(ExtractWorker2): self.tar2_process.wait() elif self.tar2_process.wait() != 0: self.collect_tar_output() - raise QubesException( + raise qubes.exc.QubesException( "unable to extract files for {0}.{1} Tar command " "output: %s". format(self.tar2_current_file, @@ -1274,7 +1299,7 @@ def parse_backup_header(filename): with open(filename, 'r') as f: for line in f.readlines(): if line.count('=') != 1: - raise QubesException("Invalid backup header (line %s)" % line) + raise qubes.exc.QubesException("Invalid backup header (line %s)" % line) (key, value) = line.strip().split('=') if not any([key == getattr(BackupHeader, attr) for attr in dir( BackupHeader)]): @@ -1394,7 +1419,7 @@ def restore_vm_dirs(backup_source, restore_tmpdir, passphrase, vms_dirs, vms, else: command.wait() proc_error_msg = command.stderr.read(MAX_STDERR_BYTES) - raise QubesException("Premature end of archive while receiving " + raise qubes.exc.QubesException("Premature end of archive while receiving " "backup header. Process output:\n" + proc_error_msg) filename = os.path.join(restore_tmpdir, filename) @@ -1406,11 +1431,11 @@ def restore_vm_dirs(backup_source, restore_tmpdir, passphrase, vms_dirs, vms, file_ok = True hmac_algorithm = hmac_algo break - except QubesException: + except qubes.exc.QubesException: # Ignore exception here, try the next algo pass if not file_ok: - raise QubesException("Corrupted backup header (hmac verification " + raise qubes.exc.QubesException("Corrupted backup header (hmac verification " "failed). Is the password correct?") if os.path.basename(filename) == HEADER_FILENAME: header_data = parse_backup_header(filename) @@ -1524,18 +1549,18 @@ def restore_vm_dirs(backup_source, restore_tmpdir, passphrase, vms_dirs, vms, tmpdir=restore_tmpdir) if command.wait() != 0 and not expect_tar_error: - raise QubesException( + raise qubes.exc.QubesException( "unable to read the qubes backup file {0} ({1}). " "Is it really a backup?".format(backup_source, command.wait())) if vmproc: if vmproc.wait() != 0: - raise QubesException( + raise qubes.exc.QubesException( "unable to read the qubes backup {0} " "because of a VM error: {1}".format( backup_source, vmproc.stderr.read(MAX_STDERR_BYTES))) if filename and filename != "EOF": - raise QubesException( + raise qubes.exc.QubesException( "Premature end of archive, the last file was %s" % filename) except: to_extract.put("ERROR") @@ -1551,7 +1576,7 @@ def restore_vm_dirs(backup_source, restore_tmpdir, passphrase, vms_dirs, vms, print_callback("Extraction process finished with code:" + str(extract_proc.exitcode)) if extract_proc.exitcode != 0: - raise QubesException( + raise qubes.exc.QubesException( "unable to extract the qubes backup. " "Check extracting process errors.") @@ -1584,7 +1609,7 @@ def load_hmac(hmac): if len(hmac) > 1: hmac = hmac[1].strip() else: - raise QubesException("ERROR: invalid hmac file content") + raise qubes.exc.QubesException("ERROR: invalid hmac file content") return hmac @@ -1642,6 +1667,7 @@ def backup_restore_header(source, passphrase, return (restore_tmpdir, os.path.join(restore_tmpdir, "qubes.xml"), header_data) + def generate_new_name_for_conflicting_vm(orig_name, host_collection, restore_info): number = 1 @@ -1651,7 +1677,7 @@ def generate_new_name_for_conflicting_vm(orig_name, host_collection, while (new_name in restore_info.keys() or new_name in map(lambda x: x.get('rename_to', None), restore_info.values()) or - host_collection.get_vm_by_name(new_name)): + new_name in host_collection.domains): new_name = str('{}{}'.format(orig_name, number)) number += 1 if number == 100: @@ -1660,6 +1686,7 @@ def generate_new_name_for_conflicting_vm(orig_name, host_collection, return new_name def restore_info_verify(restore_info, host_collection): + assert isinstance(host_collection, qubes.Qubes) options = restore_info['$OPTIONS$'] for vm in restore_info.keys(): if vm in ['$OPTIONS$', 'dom0']: @@ -1674,7 +1701,7 @@ def restore_info_verify(restore_info, host_collection): vm_info.pop('already-exists', None) if not options['verify-only'] and \ - host_collection.get_vm_by_name(vm) is not None: + vm in host_collection.domains: if options['rename-conflicting']: new_name = generate_new_name_for_conflicting_vm( vm, host_collection, restore_info @@ -1690,7 +1717,7 @@ def restore_info_verify(restore_info, host_collection): vm_info.pop('missing-template', None) if vm_info['template']: template_name = vm_info['template'] - host_template = host_collection.get_vm_by_name(template_name) + host_template = host_collection.domains.get(template_name, None) if not host_template or not host_template.is_template(): # Maybe the (custom) template is in the backup? if not (template_name in restore_info.keys() and @@ -1699,7 +1726,7 @@ def restore_info_verify(restore_info, host_collection): if 'orig-template' not in vm_info.keys(): vm_info['orig-template'] = template_name vm_info['template'] = host_collection \ - .get_default_template().name + .default_template.name else: vm_info['missing-template'] = True @@ -1708,7 +1735,7 @@ def restore_info_verify(restore_info, host_collection): if vm_info['netvm']: netvm_name = vm_info['netvm'] - netvm_on_host = host_collection.get_vm_by_name(netvm_name) + netvm_on_host = host_collection.domains.get(netvm_name, None) # No netvm on the host? if not ((netvm_on_host is not None) and netvm_on_host.is_netvm()): @@ -1718,7 +1745,7 @@ def restore_info_verify(restore_info, host_collection): restore_info[netvm_name]['vm'].is_netvm()): if options['use-default-netvm']: vm_info['netvm'] = host_collection \ - .get_default_netvm().name + .default_netvm.name vm_info['vm'].uses_default_netvm = True elif options['use-none-netvm']: vm_info['netvm'] = None @@ -1775,7 +1802,7 @@ def backup_restore_prepare(backup_location, passphrase, options=None, return False backup_vm_dir_path = check_vm.dir_path.replace( - system_path["qubes_base_dir"], backup_dir) + qubes.config.system_path["qubes_base_dir"], backup_dir) if os.path.exists(backup_vm_dir_path): return True @@ -1812,11 +1839,11 @@ def backup_restore_prepare(backup_location, passphrase, options=None, is_vm_included_in_backup = is_vm_included_in_backup_v2 if not appvm: if not os.path.isfile(backup_location): - raise QubesException("Invalid backup location (not a file or " + raise qubes.exc.QubesException("Invalid backup location (not a file or " "directory with qubes.xml)" ": %s" % unicode(backup_location)) else: - raise QubesException( + raise qubes.exc.QubesException( "Unknown backup format version: %s" % str(format_version)) (restore_tmpdir, qubes_xml, header_data) = backup_restore_header( @@ -1847,17 +1874,12 @@ def backup_restore_prepare(backup_location, passphrase, options=None, if BACKUP_DEBUG: print "Loading file", qubes_xml - backup_collection = QubesVmCollection(store_filename=qubes_xml) - backup_collection.lock_db_for_reading() - backup_collection.load() + backup_collection = qubes.Qubes(qubes_xml) if host_collection is None: - host_collection = QubesVmCollection() - host_collection.lock_db_for_reading() - host_collection.load() - host_collection.unlock_db() + host_collection = qubes.Qubes() - backup_vms_list = [vm for vm in backup_collection.values()] + backup_vms_list = [vm for vm in backup_collection.domains] vms_to_restore = {} # ... and the actual data @@ -1907,8 +1929,9 @@ def backup_restore_prepare(backup_location, passphrase, options=None, # ...and dom0 home if options['dom0-home'] and \ - is_vm_included_in_backup(backup_location, backup_collection[0]): - vm = backup_collection[0] + is_vm_included_in_backup(backup_location, + backup_collection.domains[0]): + vm = backup_collection.domains[0] vms_to_restore['dom0'] = {} if format_version == 1: vms_to_restore['dom0']['subdir'] = \ @@ -2054,13 +2077,13 @@ def backup_restore_do(restore_info, # Private functions begin def restore_vm_dir_v1(backup_dir, src_dir, dst_dir): - backup_src_dir = src_dir.replace(system_path["qubes_base_dir"], - backup_dir) + backup_src_dir = src_dir.replace( + qubes.config.system_path["qubes_base_dir"], backup_dir) # We prefer to use Linux's cp, because it nicely handles sparse files cp_retcode = subprocess.call(["cp", "-rp", "--reflink=auto", backup_src_dir, dst_dir]) if cp_retcode != 0: - raise QubesException( + raise qubes.exc.QubesException( "*** Error while copying file {0} to {1}".format(backup_src_dir, dst_dir)) @@ -2084,9 +2107,8 @@ def backup_restore_do(restore_info, lock_obtained = False if host_collection is None: - host_collection = QubesVmCollection() - host_collection.lock_db_for_writing() - host_collection.load() + host_collection = qubes.Qubes() + # FIXME lock_obtained = True # Perform VM restoration in backup order @@ -2129,7 +2151,7 @@ def backup_restore_do(restore_info, compressed=compressed, compression_filter=compression_filter, appvm=appvm) - except QubesException: + except qubes.exc.QubesException: if verify_only: raise else: @@ -2148,9 +2170,8 @@ def backup_restore_do(restore_info, shutil.rmtree(restore_tmpdir) return - # Add VM in right order - for (vm_class_name, vm_class) in sorted(QubesVmClasses.items(), - key=lambda _x: _x[1].load_order): + # First load templates, then other VMs + for do_templates in (True, False): if running_backup_operation.canceled: break for vm in vms.values(): @@ -2158,11 +2179,10 @@ def backup_restore_do(restore_info, # only break the loop to save qubes.xml with already restored # VMs break - if not vm.__class__ == vm_class: + if vm.is_template != do_templates: continue if callable(print_callback): - print_callback("-> Restoring {type} {0}...". - format(vm.name, type=vm_class_name)) + print_callback("-> Restoring {0}...".format(vm.name)) retcode = subprocess.call( ["mkdir", "-p", os.path.dirname(vm.dir_path)]) if retcode != 0: @@ -2171,10 +2191,12 @@ def backup_restore_do(restore_info, error_callback("Skipping...") continue - template = None - if vm.template is not None: - template_name = restore_info[vm.name]['template'] - template = host_collection.get_vm_by_name(template_name) + kwargs = {} + if hasattr(vm, 'template'): + if vm.template is not None: + kwargs['template'] = restore_info[vm.name]['template'] + else: + kwargs['template'] = None new_vm = None vm_name = vm.name @@ -2182,9 +2204,13 @@ def backup_restore_do(restore_info, vm_name = restore_info[vm.name]['rename-to'] try: - new_vm = host_collection.add_new_vm(vm_class_name, name=vm_name, - template=template, - installed_by_rpm=False) + # first only minimal set, later clone_properties will be called + new_vm = host_collection.add_new_vm( + vm.__class__, + name=vm_name, + label=vm.label, + installed_by_rpm=False, + **kwargs) if os.path.exists(new_vm.dir_path): move_to_path = tempfile.mkdtemp('', os.path.basename( new_vm.dir_path), os.path.dirname(new_vm.dir_path)) @@ -2214,20 +2240,19 @@ def backup_restore_do(restore_info, error_callback("ERROR: {0}".format(err)) error_callback("*** Skipping VM: {0}".format(vm.name)) if new_vm: - host_collection.pop(new_vm.qid) + del host_collection.domains[new_vm.qid] continue - # FIXME: cannot check for 'kernel' property, because it is always - # defined - accessing it touches non-existent '_kernel' - if not isinstance(vm, QubesVmClasses['QubesHVm']): + if hasattr(vm, 'kernel'): # TODO: add a setting for this? - if vm.kernel and vm.kernel not in \ - os.listdir(system_path['qubes_kernels_base_dir']): + if not vm.is_property_default('kernel') and \ + vm.kernel not in \ + os.listdir(qubes.config.system_path[ + 'qubes_kernels_base_dir']): if callable(print_callback): print_callback("WARNING: Kernel %s not installed, " "using default one" % vm.kernel) - vm.uses_default_kernel = True - vm.kernel = host_collection.get_default_kernel() + vm.kernel = qubes.property.DEFAULT try: new_vm.clone_attrs(vm) except Exception as err: @@ -2235,7 +2260,7 @@ def backup_restore_do(restore_info, error_callback("*** Some VM property will not be restored") try: - new_vm.appmenus_create(verbose=callable(print_callback)) + new_vm.appmenus_create() except Exception as err: error_callback("ERROR during appmenu restore: {0}".format(err)) error_callback( @@ -2246,22 +2271,19 @@ def backup_restore_do(restore_info, vm_name = vm.name if 'rename-to' in restore_info[vm.name]: vm_name = restore_info[vm.name]['rename-to'] - host_vm = host_collection.get_vm_by_name(vm_name) - - if host_vm is None: + try: + host_vm = host_collection.domains[vm_name] + except KeyError: # Failed/skipped VM continue - if not vm.uses_default_netvm: + if not vm.is_property_default('netvm'): if restore_info[vm.name]['netvm'] is not None: - host_vm.netvm = host_collection.get_vm_by_name( - restore_info[vm.name]['netvm']) + host_vm.netvm = restore_info[vm.name]['netvm'] else: host_vm.netvm = None host_collection.save() - if lock_obtained: - host_collection.unlock_db() if running_backup_operation.canceled: if format_version >= 2: diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index 60d54247..3c13f701 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -40,6 +40,8 @@ import time import qubes.config import qubes.events +import qubes.backup +import qubes.exc XMLPATH = '/var/lib/qubes/qubes-test.xml' CLASS_XMLPATH = '/var/lib/qubes/qubes-class-test.xml' @@ -482,7 +484,7 @@ class SystemTestsMixin(object): except (AttributeError, libvirt.libvirtError): pass - del app.domains[vm] + del app.domains[vm.qid] del vm app.save() @@ -686,8 +688,8 @@ class BackupTestsMixin(SystemTestsMixin): if self.verbose: print >>sys.stderr, "-> Creating %s" % vmname testnet = self.app.add_new_vm(qubes.vm.appvm.AppVM, - name=vmname, template=template, provides_network=True) - testnet.create_on_disk(verbose=self.verbose) + name=vmname, template=template, provides_network=True, label='red') + testnet.create_on_disk() vms.append(testnet) self.fill_image(testnet.private_img, 20*1024*1024) @@ -695,10 +697,10 @@ class BackupTestsMixin(SystemTestsMixin): if self.verbose: print >>sys.stderr, "-> Creating %s" % vmname testvm1 = self.app.add_new_vm(qubes.vm.appvm.AppVM, - name=vmname, template=template) + name=vmname, template=template, label='red') testvm1.uses_default_netvm = False testvm1.netvm = testnet - testvm1.create_on_disk(verbose=self.verbose) + testvm1.create_on_disk() vms.append(testvm1) self.fill_image(testvm1.private_img, 100*1024*1024) @@ -706,8 +708,8 @@ class BackupTestsMixin(SystemTestsMixin): if self.verbose: print >>sys.stderr, "-> Creating %s" % vmname testvm2 = self.app.add_new_vm(qubes.vm.appvm.AppVM, name=vmname, - hvm=True) - testvm2.create_on_disk(verbose=self.verbose) + hvm=True, label='red') + testvm2.create_on_disk() self.fill_image(testvm2.root_img, 1024*1024*1024, True) vms.append(testvm2) @@ -717,33 +719,31 @@ class BackupTestsMixin(SystemTestsMixin): def make_backup(self, vms, prepare_kwargs=dict(), do_kwargs=dict(), target=None, expect_failure=False): - # XXX: bakup_prepare and backup_do don't support host_collection - # self.qc.unlock_db() if target is None: target = self.backupdir try: files_to_backup = \ - qubes.backup.backup_prepare(vms, + qubes.backup.backup_prepare(self.app, vms, print_callback=self.print_callback, **prepare_kwargs) - except qubes.qubes.QubesException as e: + except qubes.exc.QubesException as e: if not expect_failure: self.fail("QubesException during backup_prepare: %s" % str(e)) else: raise try: - qubes.backup.backup_do(target, files_to_backup, "qubes", + qubes.backup.backup_do(self.app, target, files_to_backup, "qubes", progress_callback=self.print_progress, **do_kwargs) - except qubes.qubes.QubesException as e: + except qubes.exc.QubesException as e: if not expect_failure: self.fail("QubesException during backup_do: %s" % str(e)) else: raise # FIXME why? - self.reload_db() + #self.reload_db() def restore_backup(self, source=None, appvm=None, options=None, expect_errors=None): @@ -753,7 +753,7 @@ class BackupTestsMixin(SystemTestsMixin): else: backupfile = source - with self.assertNotRaises(qubes.qubes.QubesException): + with self.assertNotRaises(qubes.exc.QubesException): backup_info = qubes.backup.backup_restore_prepare( backupfile, "qubes", host_collection=self.app, @@ -764,7 +764,7 @@ class BackupTestsMixin(SystemTestsMixin): if self.verbose: qubes.backup.backup_restore_print_summary(backup_info) - with self.assertNotRaises(qubes.qubes.QubesException): + with self.assertNotRaises(qubes.exc.QubesException): qubes.backup.backup_restore_do( backup_info, host_collection=self.app, @@ -821,7 +821,7 @@ def load_tests(loader, tests, pattern): # pylint: disable=unused-argument 'qubes.tests.int.dom0_update', 'qubes.tests.int.network', # 'qubes.tests.vm_qrexec_gui', -# 'qubes.tests.backup', + 'qubes.tests.int.backup', # 'qubes.tests.backupcompatibility', # 'qubes.tests.regressions', diff --git a/tests/backup.py b/qubes/tests/int/backup.py similarity index 83% rename from tests/backup.py rename to qubes/tests/int/backup.py index 72463c91..7f5fd096 100644 --- a/tests/backup.py +++ b/qubes/tests/int/backup.py @@ -28,8 +28,11 @@ import os import unittest import sys -from qubes.qubes import QubesException, QubesTemplateVm +import qubes +import qubes.exc import qubes.tests +import qubes.vm.appvm +import qubes.vm.templatevm class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase): def test_000_basic_backup(self): @@ -37,21 +40,24 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase): self.make_backup(vms) self.remove_vms(vms) self.restore_backup() - self.remove_vms(vms) + for vm in vms: + self.assertIn(vm.name, self.app.domains) def test_001_compressed_backup(self): vms = self.create_backup_vms() self.make_backup(vms, do_kwargs={'compressed': True}) self.remove_vms(vms) self.restore_backup() - self.remove_vms(vms) + for vm in vms: + self.assertIn(vm.name, self.app.domains) def test_002_encrypted_backup(self): vms = self.create_backup_vms() self.make_backup(vms, do_kwargs={'encrypted': True}) self.remove_vms(vms) self.restore_backup() - self.remove_vms(vms) + for vm in vms: + self.assertIn(vm.name, self.app.domains) def test_003_compressed_encrypted_backup(self): vms = self.create_backup_vms() @@ -61,7 +67,8 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase): 'encrypted': True}) self.remove_vms(vms) self.restore_backup() - self.remove_vms(vms) + for vm in vms: + self.assertIn(vm.name, self.app.domains) def test_004_sparse_multipart(self): vms = [] @@ -70,29 +77,32 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase): if self.verbose: print >>sys.stderr, "-> Creating %s" % vmname - hvmtemplate = self.qc.add_new_vm("QubesTemplateHVm", name=vmname) - hvmtemplate.create_on_disk(verbose=self.verbose) + hvmtemplate = self.app.add_new_vm( + qubes.vm.templatevm.TemplateVM, name=vmname, hvm=True, label='red') + hvmtemplate.create_on_disk() self.fill_image(os.path.join(hvmtemplate.dir_path, '00file'), 195*1024*1024-4096*3) self.fill_image(hvmtemplate.private_img, 195*1024*1024-4096*3) self.fill_image(hvmtemplate.root_img, 1024*1024*1024, sparse=True) vms.append(hvmtemplate) - self.qc.save() + self.app.save() self.make_backup(vms) self.remove_vms(vms) self.restore_backup() - self.remove_vms(vms) + for vm in vms: + self.assertIn(vm.name, self.app.domains) def test_005_compressed_custom(self): vms = self.create_backup_vms() self.make_backup(vms, do_kwargs={'compressed': "bzip2"}) self.remove_vms(vms) self.restore_backup() - self.remove_vms(vms) + for vm in vms: + self.assertIn(vm.name, self.app.domains) def test_100_backup_dom0_no_restore(self): - self.make_backup([self.qc[0]]) + self.make_backup([self.app.domains[0]]) # TODO: think of some safe way to test restore... def test_200_restore_over_existing_directory(self): @@ -112,7 +122,6 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase): '*** Directory {} already exists! It has been moved'.format( test_dir) ]) - self.remove_vms(vms) def test_210_auto_rename(self): """ @@ -125,22 +134,23 @@ class TC_00_Backup(qubes.tests.BackupTestsMixin, qubes.tests.QubesTestCase): 'rename-conflicting': True }) for vm in vms: - self.assertIsNotNone(self.qc.get_vm_by_name(vm.name+'1')) - restored_vm = self.qc.get_vm_by_name(vm.name+'1') - if vm.netvm and not vm.uses_default_netvm: - self.assertEqual(restored_vm.netvm.name, vm.netvm.name+'1') + with self.assertNotRaises(qubes.exc.QubesVMNotFoundError): + restored_vm = self.app.domains[vm.name + '1'] + if vm.netvm and not vm.property_is_default('netvm'): + self.assertEqual(restored_vm.netvm.name, vm.netvm.name + '1') + - self.remove_vms(vms) class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin): def setUp(self): super(TC_10_BackupVMMixin, self).setUp() - self.backupvm = self.qc.add_new_vm( - "QubesAppVm", + self.backupvm = self.app.add_new_vm( + qubes.vm.appvm.AppVM, + label='red', name=self.make_vm_name('backupvm'), - template=self.qc.get_vm_by_name(self.template) + template=self.template ) - self.backupvm.create_on_disk(verbose=self.verbose) + self.backupvm.create_on_disk() def test_100_send_to_vm_file_with_spaces(self): vms = self.create_backup_vms() @@ -159,7 +169,6 @@ class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin): backup_path = backup_path.strip() self.restore_backup(source=backup_path, appvm=self.backupvm) - self.remove_vms(vms) def test_110_send_to_vm_command(self): vms = self.create_backup_vms() @@ -173,7 +182,6 @@ class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin): self.remove_vms(vms) self.restore_backup(source='dd if=/var/tmp/backup-test', appvm=self.backupvm) - self.remove_vms(vms) def test_110_send_to_vm_no_space(self): """ @@ -192,7 +200,7 @@ class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin): user="root", wait=True) if retcode != 0: raise RuntimeError("Failed to prepare backup directory") - with self.assertRaises(QubesException): + with self.assertRaises(qubes.exc.QubesException): self.make_backup(vms, do_kwargs={ 'appvm': self.backupvm, @@ -200,19 +208,13 @@ class TC_10_BackupVMMixin(qubes.tests.BackupTestsMixin): 'encrypted': True}, target='/home/user/backup', expect_failure=True) - self.qc.lock_db_for_writing() - self.qc.load() - self.remove_vms(vms) def load_tests(loader, tests, pattern): try: - qc = qubes.qubes.QubesVmCollection() - qc.lock_db_for_reading() - qc.load() - qc.unlock_db() - templates = [vm.name for vm in qc.values() if - isinstance(vm, QubesTemplateVm)] + app = qubes.Qubes() + templates = [vm.name for vm in app.domains if + isinstance(vm, qubes.vm.templatevm.TemplateVM)] except OSError: templates = [] for template in templates: diff --git a/qubes/utils.py b/qubes/utils.py index 08646d73..9b9384ef 100644 --- a/qubes/utils.py +++ b/qubes/utils.py @@ -90,9 +90,12 @@ def format_doc(docstring): # maybe adapt https://code.activestate.com/recipes/578019 def parse_size(size): units = [ - ('K', 1024), ('KB', 1024), - ('M', 1024*1024), ('MB', 1024*1024), - ('G', 1024*1024*1024), ('GB', 1024*1024*1024), + ('K', 1000), ('KB', 1000), + ('M', 1000 * 1000), ('MB', 1000 * 1000), + ('G', 1000 * 1000 * 1000), ('GB', 1000 * 1000 * 1000), + ('Ki', 1024), ('KiB', 1024), + ('Mi', 1024 * 1024), ('MiB', 1024 * 1024), + ('Gi', 1024 * 1024 * 1024), ('GiB', 1024 * 1024 * 1024), ] size = size.strip().upper() @@ -102,10 +105,43 @@ def parse_size(size): for unit, multiplier in units: if size.endswith(unit): size = size[:-len(unit)].strip() - return int(size)*multiplier + return int(size) * multiplier raise qubes.exc.QubesException("Invalid size: {0}.".format(size)) +def mbytes_to_kmg(size): + if size > 1024: + return "%d GiB" % (size / 1024) + else: + return "%d MiB" % size + + +def kbytes_to_kmg(size): + if size > 1024: + return mbytes_to_kmg(size / 1024) + else: + return "%d KiB" % size + + +def bytes_to_kmg(size): + if size > 1024: + return kbytes_to_kmg(size / 1024) + else: + return "%d B" % size + + +def size_to_human(size): + """Humane readable size, with 1/10 precision""" + if size < 1024: + return str(size) + elif size < 1024 * 1024: + return str(round(size / 1024.0, 1)) + ' KiB' + elif size < 1024 * 1024 * 1024: + return str(round(size / (1024.0 * 1024), 1)) + ' MiB' + else: + return str(round(size / (1024.0 * 1024 * 1024), 1)) + ' GiB' + + def urandom(size): rand = os.urandom(size) if rand is None: diff --git a/rpm_spec/core-dom0.spec b/rpm_spec/core-dom0.spec index ffbe42d3..962b10ba 100644 --- a/rpm_spec/core-dom0.spec +++ b/rpm_spec/core-dom0.spec @@ -201,6 +201,7 @@ fi %dir %{python_sitelib}/qubes %{python_sitelib}/qubes/__init__.py* +%{python_sitelib}/qubes/backup.py* %{python_sitelib}/qubes/config.py* %{python_sitelib}/qubes/devices.py* %{python_sitelib}/qubes/dochelpers.py* @@ -272,6 +273,7 @@ fi %dir %{python_sitelib}/qubes/tests/int %{python_sitelib}/qubes/tests/int/__init__.py* +%{python_sitelib}/qubes/tests/int/backup.py* %{python_sitelib}/qubes/tests/int/basic.py* %{python_sitelib}/qubes/tests/int/dom0_update.py* %{python_sitelib}/qubes/tests/int/network.py*