diff --git a/core-modules/000QubesVm.py b/core-modules/000QubesVm.py index 4c8fdf8f..f89bf300 100644 --- a/core-modules/000QubesVm.py +++ b/core-modules/000QubesVm.py @@ -120,6 +120,9 @@ class QubesVm(object): "debug": { "default": False }, "default_user": { "default": "user" }, "qrexec_timeout": { "default": 60 }, + "backup_content" : { 'default': False }, + "backup_size" : { 'default': 0, "eval": "int(value)" }, + "backup_path" : { 'default': "" }, ##### Internal attributes - will be overriden in __init__ regardless of args "config_file_template": { "eval": 'system_path["config_template_pv"]' }, "icon_path": { "eval": 'os.path.join(self.dir_path, "icon.png") if self.dir_path is not None else None' }, @@ -137,7 +140,8 @@ class QubesVm(object): 'uses_default_kernel', 'kernel', 'uses_default_kernelopts',\ 'kernelopts', 'services', 'installed_by_rpm',\ 'uses_default_netvm', 'include_in_backups', 'debug',\ - 'default_user', 'qrexec_timeout' ]: + 'default_user', 'qrexec_timeout', + 'backup_content', 'backup_size', 'backup_path' ]: attrs[prop]['save'] = 'str(self.%s)' % prop # Simple paths for prop in ['conf_file', 'root_img', 'volatile_img', 'private_img']: diff --git a/core/qubesutils.py b/core/qubesutils.py index 61a3f96c..e96ce13d 100644 --- a/core/qubesutils.py +++ b/core/qubesutils.py @@ -779,21 +779,21 @@ def file_to_backup (file_path, sz = None): def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_callback = print_stdout): """If vms = None, include all (sensible) VMs; exclude_list is always applied""" - + ''' if not os.path.exists (base_backup_dir): raise QubesException("The target directory doesn't exist!") files_to_backup = file_to_backup (system_path["qubes_store_filename"]) + ''' if exclude_list is None: exclude_list = [] + qvm_collection = None if vms_list is None: qvm_collection = QubesVmCollection() - qvm_collection.lock_db_for_reading() + qvm_collection.lock_db_for_writing() qvm_collection.load() - # FIXME: should be after backup completed - qvm_collection.unlock_db() all_vms = [vm for vm in qvm_collection.values()] selected_vms = [vm for vm in all_vms if vm.include_in_backups] @@ -909,6 +909,19 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca print_callback(s) + # Initialize backup flag on all VMs + for vm in qvm_collection.values(): + vm.backup_content = False + + if vm in vms_for_backup: + vm.backup_content = True + vm.backup_size = vm.get_disk_utilization() + vm.backup_path = vm.dir_path.split(os.path.normpath(qubes_base_dir)+"/")[1] + + qvm_collection.save() + # FIXME: should be after backup completed + qvm_collection.unlock_db() + # Dom0 user home if not 'dom0' in exclude_list: local_user = grp.getgrnam('qubes').gr_mem[0] @@ -956,7 +969,7 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca fmt="{{0:-^{0}}}-+".format(f["width"] + 1) s += fmt.format('-') print_callback(s) - + ''' stat = os.statvfs(base_backup_dir) backup_fs_free_sz = stat.f_bsize * stat.f_bavail print_callback("") @@ -967,7 +980,7 @@ def backup_prepare(base_backup_dir, vms_list = None, exclude_list = [], print_ca raise QubesException("Please shutdown all VMs before proceeding.") print_callback("-> Available space: {0}".format(size_to_human(backup_fs_free_sz))) - + ''' return files_to_backup def backup_do(base_backup_dir, files_to_backup, progress_callback = None): @@ -1004,6 +1017,463 @@ def backup_do(base_backup_dir, files_to_backup, progress_callback = None): progress = bytes_backedup * 100 / total_backup_sz progress_callback(progress) +def backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callback = None, encrypt=False, appvm=None): + total_backup_sz = 0 + for file in files_to_backup: + total_backup_sz += file["size"] + + vmproc = None + if appvm != None: + # Prepare the backup target (Qubes service call) + backup_target = "QUBESRPC qubes.Backup none" + + # does the vm exist? + qvm_collection = QubesVmCollection() + qvm_collection.lock_db_for_reading() + qvm_collection.load() + + vm = qvm_collection.get_vm_by_name(appvm) + if vm is None or vm.qid not in qvm_collection: + raise QubesException("VM {0} does not exist".format(appvm)) + + qvm_collection.unlock_db() + + # If APPVM, STDOUT is a PIPE + vmproc = vm.run(command = backup_target, passio_popen = True) + vmproc.stdin.write(base_backup_dir.replace("\r","").replace("\n","")+"\n") + backup_stdout = vmproc.stdin + + else: + # Prepare the backup target (local file) + backup_target = base_backup_dir + "/qubes-{0}".format (time.strftime("%Y-%m-%d-%H%M%S")) + + # Create the target directory + if not os.path.exists (base_backup_dir): + raise QubesException("ERROR: the backup directory {0} does not exists".format(base_backup_dir)) + + # If not APPVM, STDOUT is a local file + backup_stdout = open(backup_target,'wb') + + blocks_backedup = 0 + progress = blocks_backedup * 11 / total_backup_sz + progress_callback(progress) + + import tempfile + feedback_file = tempfile.NamedTemporaryFile() + backup_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/backup_") + + # Tar with tapelength does not deals well with stdout (close stdout between two tapes) + # For this reason, we will use named pipes instead + print "Working in",backup_tmpdir + + backup_pipe = os.path.join(backup_tmpdir,"backup_pipe") + print "Creating pipe in:",backup_pipe + print os.mkfifo(backup_pipe) + + print "Will backup:",files_to_backup + + # Setup worker to send encrypted data chunks to the backup_target + from multiprocessing import Queue,Process + class Send_Worker(Process): + def __init__(self,queue,base_dir,backup_stdout): + super(Send_Worker, self).__init__() + self.queue = queue + self.base_dir = base_dir + self.backup_stdout = backup_stdout + + def run(self): + print "Started sending thread" + + print "Moving to temporary dir",self.base_dir + os.chdir(self.base_dir) + + for filename in iter(self.queue.get,None): + if filename == "FINISHED": + break + + print "Sending file",filename + # This tar used for sending data out need to be as simple, as simple, as featureless as possible. It will not be verified before untaring. + tar_final_cmd = ["tar", "-cO", "--posix", "-C", self.base_dir, filename] + final_proc = subprocess.Popen (tar_final_cmd, stdin=subprocess.PIPE, stdout=self.backup_stdout) + final_proc.wait() + + # Delete the file as we don't need it anymore + print "Removing file",filename + os.remove(filename) + + print "Finished sending thread" + + global blocks_backedup + blocks_backedup = 0 + def compute_progress(new_size, total_backup_sz): + global blocks_backedup + blocks_backedup += new_size + progress = blocks_backedup / float(total_backup_sz) + progress_callback(int(round(progress*100,2))) + + to_send = Queue() + send_proc = Send_Worker(to_send, backup_tmpdir, backup_stdout) + send_proc.start() + + for filename in files_to_backup: + print "Backing up",filename + + backup_tempfile = os.path.join(backup_tmpdir,filename["path"].split(os.path.normpath(qubes_base_dir)+"/")[1]) + print "Using temporary location:",backup_tempfile + + # Ensure the temporary directory exists + + if not os.path.isdir(os.path.dirname(backup_tempfile)): + os.makedirs(os.path.dirname(backup_tempfile)) + + # The first tar cmd can use any complex feature as we want. Files will be verified before untaring this. + tar_cmdline = ["tar", "-Pc", "-f", backup_pipe,'--sparse','--tape-length',str(1000000),'-C',qubes_base_dir, + filename["path"].split(os.path.normpath(qubes_base_dir)+"/")[1] + ] + + print " ".join(tar_cmdline) + + # Tips: Popen(bufsize=0) + # Pipe: tar-sparse | encryptor [| hmac] | tar | backup_target + # Pipe: tar-sparse [| hmac] | tar | backup_target + tar_sparse = subprocess.Popen (tar_cmdline,stdin=subprocess.PIPE) + + # Wait for compressor (tar) process to finish or for any error of other subprocesses + i=0 + run_error = "paused" + running = [] + while run_error == "paused": + + pipe = open(backup_pipe,'rb') + + # Start HMAC + hmac = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], stdin=subprocess.PIPE, stdout=subprocess.PIPE) + + # Prepare a first chunk + chunkfile = backup_tempfile + "." + "%03d" % i + i += 1 + chunkfile_p = open(chunkfile,'wb') + + if encrypt: + # Start encrypt + # If no cipher is provided, the data is forwarded unencrypted !!! + # Also note that the + encryptor = subprocess.Popen (["openssl", "enc", "-e", "-aes-256-cbc", "-pass", "pass:"+passphrase], stdin=pipe, stdout=subprocess.PIPE) + run_error = wait_backup_feedback(compute_progress, encryptor.stdout, encryptor, chunkfile_p, total_backup_sz, hmac=hmac, vmproc=vmproc, addproc=tar_sparse) + else: + run_error = wait_backup_feedback(compute_progress, pipe, None, chunkfile_p, total_backup_sz, hmac=hmac, vmproc=vmproc, addproc=tar_sparse) + + chunkfile_p.close() + + print "Wait_backup_feedback returned:",run_error + + if len(run_error) > 0: + send_proc.terminate() + raise QubesException("Failed to perform backup: error with "+run_error) + + # Send the chunk to the backup target + to_send.put(chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]) + + # Close HMAC + hmac.stdin.close() + hmac.wait() + print "HMAC proc return code:",hmac.poll() + + # Write HMAC data next to the chunk file + hmac_data = hmac.stdout.read() + print "Writing hmac to",chunkfile+".hmac" + hmac_file = open(chunkfile+".hmac",'w') + hmac_file.write(hmac_data) + hmac_file.flush() + hmac_file.close() + + # Send the HMAC to the backup target + to_send.put(chunkfile.split(os.path.normpath(backup_tmpdir)+"/")[1]+".hmac") + + if tar_sparse.poll() == None: + # Release the next chunk + print "Release next chunk for process:",tar_sparse.poll() + #tar_sparse.stdout = subprocess.PIPE + tar_sparse.stdin.write("\n") + run_error="paused" + else: + print "Finished tar sparse with error",tar_sparse.poll() + + pipe.close() + + # Close the backup target and wait for it to finish + #backup_stdout.close() + + to_send.put("FINISHED") + send_proc.join() + + if send_proc.exitcode != 0: + raise QubesException("Failed to send backup: error in the sending process") + + if vmproc: + print "VMProc1 proc return code:",vmproc.poll() + print "Sparse1 proc return code:",tar_sparse.poll() + vmproc.stdin.close() + +''' +' Wait for backup chunk to finish +' - Monitor all the processes (streamproc, hmac, vmproc, addproc) for errors +' - Copy stdout of streamproc to backup_target and hmac stdin if available +' - Compute progress based on total_backup_sz and send progress to progress_callback function +' - Returns if +' - one of the monitored processes error out (streamproc, hmac, vmproc, addproc), along with the processe that failed +' - all of the monitored processes except vmproc finished successfully (vmproc termination is controlled by the python script) +' - streamproc does not delivers any data anymore (return with the error "paused") +''' +def wait_backup_feedback(progress_callback, in_stream, streamproc, backup_target, total_backup_sz, hmac=None, vmproc=None, addproc=None, remove_trailing_bytes=0): + + buffer_size = 4096 + + run_error = None + run_count = 1 + blocks_backedup = 0 + while run_count > 0 and run_error == None: + + buffer = in_stream.read(buffer_size) + progress_callback(len(buffer),total_backup_sz) + + run_count = 0 + if hmac: + retcode=hmac.poll() + if retcode != None: + if retcode != 0: + run_error = "hmac" + else: + run_count += 1 + + if addproc: + retcode=addproc.poll() + #print "Tar proc status:",retcode + if retcode != None: + if retcode != 0: + run_error = "addproc" + else: + run_count += 1 + + if vmproc: + retcode = vmproc.poll() + if retcode != None: + if retcode != 0: + run_error = "VM" + print vmproc.stdout.read() + else: + # VM should run until the end + pass + + if streamproc: + retcode=streamproc.poll() + if retcode != None: + if retcode != 0: + run_error = "streamproc" + elif retcode == 0 and len(buffer) <= 0: + return "" + else: + #print "INFO: last packet" + #if remove_trailing_bytes > 0: + # print buffer.encode("hex") + # buffer = buffer[:-remove_trailing_bytes] + # print buffer.encode("hex") + + backup_target.write(buffer) + + if hmac: + hmac.stdin.write(buffer) + + run_count += 1 + else: + #print "Process running:",len(buffer) + # Process still running + backup_target.write(buffer) + + if hmac: + hmac.stdin.write(buffer) + + run_count += 1 + + else: + if len(buffer) <= 0: + return "" + else: + backup_target.write(buffer) + + if hmac: + hmac.stdin.write(buffer) + + + return run_error + +def restore_vm_dirs (backup_dir, backup_tmpdir, passphrase, vms_dirs, vms, vms_size, print_callback=None, error_callback=None, progress_callback=None, encrypted=False, appvm=None): + + # Setup worker to extract encrypted data chunks to the restore dirs + from multiprocessing import Queue,Process + class Extract_Worker(Process): + def __init__(self,queue,base_dir,passphrase,encrypted,total_size,print_callback,error_callback,progress_callback,vmproc=None): + super(Extract_Worker, self).__init__() + self.queue = queue + self.base_dir = base_dir + self.passphrase = passphrase + self.encrypted = encrypted + self.total_size = total_size + self.blocks_backedup = 0 + self.tar2_command = None + + self.print_callback = print_callback + self.error_callback = error_callback + self.progress_callback = progress_callback + + self.vmproc = vmproc + + self.restore_pipe = os.path.join(self.base_dir,"restore_pipe") + print "Creating pipe in:",self.restore_pipe + print os.mkfifo(self.restore_pipe) + + def compute_progress(self, new_size, total_size): + self.blocks_backedup += new_size + progress = self.blocks_backedup / float(self.total_size) + progress = int(round(progress*100,2)) + self.progress_callback(progress) + + def run(self): + self.print_callback("Started sending thread") + + self.print_callback("Moving to dir "+self.base_dir) + os.chdir(self.base_dir) + + for filename in iter(self.queue.get,None): + if filename == "FINISHED": + break + + self.print_callback("Extracting file "+filename+" to "+qubes_base_dir) + + if self.tar2_command == None: + # FIXME: Make the extraction safer by avoiding to erase other vms: + # - extracting directly to the target directory (based on the vm name and by using the --strip=2). + # - ensuring that the leading slashs are ignored when extracting (can also be obtained by running with --strip ?) + self.tar2_command = ['tar', '--tape-length','1000000', '-C', qubes_base_dir, '-xvf', self.restore_pipe] + self.print_callback("Running command "+str(self.tar2_command)) + self.tar2_command = subprocess.Popen(self.tar2_command,stdin=subprocess.PIPE) + + pipe = open(self.restore_pipe,'r+b') + if self.encrypted: + # Start decrypt + encryptor = subprocess.Popen (["openssl", "enc", "-d", "-aes-256-cbc", "-pass", "pass:"+passphrase], stdin=open(filename,'rb'), stdout=subprocess.PIPE) + + # progress_callback, in_stream, streamproc, backup_target, total_backup_sz, hmac=None, vmproc=None, addproc=None, remove_trailing_bytes=0): + run_error = wait_backup_feedback(self.compute_progress, encryptor.stdout, encryptor, pipe, self.total_size, hmac=None, vmproc=self.vmproc, addproc=self.tar2_command) + #print "End wait_backup_feedback",run_error,self.tar2_command.poll(),encryptor.poll() + else: + run_error = wait_backup_feedback(self.compute_progress, open(filename,"rb"), None, pipe, self.total_size, hmac=None, vmproc=self.vmproc, addproc=self.tar2_command) + + pipe.close() + + self.print_callback("Run error:"+run_error) + self.print_callback(str(self.tar2_command.poll())) + + if self.tar2_command.poll() != None: + if self.tar2_command.poll() != 0: + raise QubesException("ERROR: unable to extract files for {0}.".format(filename)) + else: + # Finished extracting the tar file + self.tar2_command = None + + else: + self.print_callback("Releasing next chunck") + self.tar2_command.stdin.write("\n") + + # Delete the file as we don't need it anymore + self.print_callback("Removing file "+filename) + os.remove(filename) + + self.print_callback("Finished extracting thread") + + if progress_callback == None: + def progress_callback(data): + pass + + to_extract = Queue() + extract_proc = Extract_Worker(to_extract, backup_tmpdir, passphrase, encrypted, vms_size, print_callback, error_callback, progress_callback) + extract_proc.start() + + print_callback("Working in temporary dir:"+backup_tmpdir) + print_callback(str(vms_size)+" bytes to restore") + + vmproc = None + if appvm != None: + # Prepare the backup target (Qubes service call) + backup_target = "QUBESRPC qubes.Restore none" + + # does the vm exist? + qvm_collection = QubesVmCollection() + qvm_collection.lock_db_for_reading() + qvm_collection.load() + + vm = qvm_collection.get_vm_by_name(appvm) + if vm is None or vm.qid not in qvm_collection: + raise QubesException("VM {0} does not exist".format(appvm)) + + qvm_collection.unlock_db() + + # If APPVM, STDOUT is a PIPE + vmproc = vm.run(command = backup_target, passio_popen = True) + vmproc.stdin.write(backup_dir.replace("\r","").replace("\n","")+"\n") + backup_stdin = vmproc.stdout + else: + backup_stdin = open(backup_dir,'rb') + + # FIXME: Use a safer program such as cpio, modified uncompress.c, or try to extract it from + tar1_command = ['tar', '-i', '-xv', '-C', backup_tmpdir] + tar1_command.extend(vms_dirs) + print_callback("Run command"+str(tar1_command)) + command = subprocess.Popen(tar1_command, stdin=backup_stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + while command.poll() == None and vmproc.poll() == None: + + filename = command.stdout.readline().strip(" \t\r\n") + + print_callback("Getting new file:"+filename) + + hmacfile = command.stdout.readline().strip(" \t\r\n") + + print_callback("Getting hmac:"+hmacfile) + + print_callback("Verifying file"+filename) + + hmac_proc = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], stdin=open(os.path.join(backup_tmpdir,filename),'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout,stderr = hmac_proc.communicate() + + if len(stderr) > 0: + raise QubesException("ERROR: verify file {0}: {1}".format((filename,stderr))) + else: + print_callback("Loading hmac for file"+filename) + hmac = load_hmac(open(os.path.join(backup_tmpdir,filename+".hmac"),'r').read()) + + if len(hmac) > 0 and load_hmac(stdout) == hmac: + print_callback("File verification OK -> Sending file "+filename+" for extraction") + # Send the chunk to the backup target + to_extract.put(os.path.join(backup_tmpdir,filename)) + + else: + raise QubesException("ERROR: invalid hmac for file {0}: {1}. Is the passphrase correct?".format(filename,load_hmac(stdout))) + + if command.poll() != 0: + raise QubesException("ERROR: unable to read the qubes backup file {0}. Is it really a backup?".format(restore_target)) + if vmproc.poll() != 0: + raise QubesException("ERROR: unable to read the qubes backup {0} because of a VM error: {1}".format(restore_target,vmproc.stderr.read())) + + print "Extraction process status:",extract_proc.exitcode + + to_extract.put("FINISHED") + print_callback("Waiting for the extraction process to finish...") + extract_proc.join() + print_callback("Extraction process finished with code:"+str(extract_proc.exitcode)) + if extract_proc.exitcode != 0: + raise QubesException("ERROR: unable to extract the qubes backup. Check extracting process errors.") + def backup_restore_set_defaults(options): if 'use-default-netvm' not in options: options['use-default-netvm'] = False @@ -1018,8 +1488,146 @@ def backup_restore_set_defaults(options): return options +def load_hmac(hmac): + hmac = hmac.strip(" \t\r\n").split("=") + if len(hmac) > 1: + hmac = hmac[1].strip() + else: + raise QubesException("ERROR: invalid hmac file content") -def backup_restore_prepare(backup_dir, options = {}, host_collection = None): + return hmac + +import struct +def get_qfile_error(buffer): + error = struct.unpack("I",buffer[0:4])[0] + error_msg = { 0: "COPY_FILE_OK", + 1: "COPY_FILE_READ_EOF", + 2: "COPY_FILE_READ_ERROR", + 3: "COPY_FILE_WRITE_ERROR", + } + + if error in error_msg.keys(): + return error_msg[error] + else: + return "UNKNOWN_ERROR_"+str(error) + +def backup_restore_header(restore_target, passphrase, encrypt=False, appvm=None): + # Simulate dd if=backup_file count=10 | file - + # Simulate dd if=backup_file count=10 | gpg2 -d | tar xzv -O + # analysis = subprocess.Popen() + vmproc = None + + import tempfile + feedback_file = tempfile.NamedTemporaryFile() + backup_tmpdir = tempfile.mkdtemp(prefix="/var/tmp/restore_") + + os.chdir(backup_tmpdir) + + # Tar with tapelength does not deals well with stdout (close stdout between two tapes) + # For this reason, we will use named pipes instead + print "Working in",backup_tmpdir + + if appvm != None: + # Prepare the backup target (Qubes service call) + restore_command = "QUBESRPC qubes.Restore none" + + # does the vm exist? + qvm_collection = QubesVmCollection() + qvm_collection.lock_db_for_reading() + qvm_collection.load() + + vm = qvm_collection.get_vm_by_name(appvm) + if vm is None or vm.qid not in qvm_collection: + raise QubesException("VM {0} does not exist".format(appvm)) + + qvm_collection.unlock_db() + + # If APPVM, STDOUT is a PIPE + vmproc = vm.run(command = restore_command, passio_popen = True, passio_stderr = True) + vmproc.stdin.write(restore_target.replace("\r","").replace("\n","")+"\n") + else: + # Create the target directory + if not os.path.exists (restore_target): + raise QubesException("ERROR: the backup directory {0} does not exists".format(restore_target)) + + fp = open(restore_target,'rb') + headers = fp.read(4096*16) + + tar1_command = ['/usr/lib/qubes/qfile-dom0-unpacker', str(os.getuid()), backup_tmpdir] + command = subprocess.Popen(tar1_command,stdin=vmproc.stdout,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + + result_header = command.stdout.read() + + if vmproc.poll() != None: + error = vmproc.stderr.read() + print error + print vmproc.poll(),command.poll() + raise QubesException("ERROR: Immediate VM error while retrieving backup headers:{0}".format(error)) + + filename = "qubes.xml.000" + + print result_header.encode("hex") + error_msg = get_qfile_error(result_header) + if error_msg != "COPY_FILE_OK": + print vmproc.stdout.read() + raise QubesException("ERROR: unpacking backup headers: {0}".format(error_msg)) + if not os.path.exists(os.path.join(backup_tmpdir,filename+".hmac")): + raise QubesException("ERROR: header not extracted correctly: {0}".format(os.path.join(backup_tmpdir,filename+".hmac"))) + + command.terminate() + command.wait() + + if vmproc.poll() != None and vmproc.poll() != 0: + error = vmproc.stderr.read() + print error + print vmproc.poll(),command.poll() + raise QubesException("ERROR: VM error retrieving backup headers") + elif command.poll() != None and command.poll() not in [0,-15]: + error = command.stderr.read() + print error + print vmproc.poll(),command.poll() + raise QubesException("ERROR: retrieving backup headers:{0}".format(error)) + + if vmproc.poll() == None: + vmproc.terminate() + vmproc.wait() + + print "Loading hmac for file",filename + hmac = load_hmac(open(os.path.join(backup_tmpdir,filename+".hmac"),'r').read()) + + print "Successfully retrieved headers" + + print "Verifying file",filename + hmac_proc = subprocess.Popen (["openssl", "dgst", "-hmac", passphrase], stdin=open(os.path.join(backup_tmpdir,filename),'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout,stderr = hmac_proc.communicate() + if len(stderr) > 0: + raise QubesException("ERROR: verify file {0}: {1}".format((filename,stderr))) + else: + if len(hmac) > 0 and load_hmac(stdout) == hmac: + print "File verification OK -> Extracting archive",filename + if encrypt: + print "Starting decryption process" + encryptor = subprocess.Popen (["openssl", "enc", "-d", "-aes-256-cbc", "-pass", "pass:"+passphrase], stdin=open(os.path.join(backup_tmpdir,filename),'rb'), stdout=subprocess.PIPE) + tarhead_command = subprocess.Popen(['tar', '--tape-length','1000000', '-xv'],stdin=encryptor.stdout) + else: + print "No decryption process required" + encryptor = None + tarhead_command = subprocess.Popen(['tar', '--tape-length','1000000', '-xvf', os.path.join(backup_tmpdir,filename)]) + + tarhead_command.wait() + if encryptor: + if encryptor.poll() != 0: + raise QubesException("ERROR: unable to decrypt file {0}".format(filename)) + if tarhead_command.poll() != 0: + raise QubesException("ERROR: unable to extract the qubes.xml file. Is archive encrypted?") + + return (backup_tmpdir,"qubes.xml") + else: + raise QubesException("ERROR: unable to verify the qubes.xml file. Is the passphrase correct?") + + return None + +def backup_restore_prepare(backup_dir, qubes_xml, passphrase, options = {}, host_collection = None, encrypt=False, appvm=None): # Defaults backup_restore_set_defaults(options) @@ -1029,9 +1637,7 @@ def backup_restore_prepare(backup_dir, options = {}, host_collection = None): # Dom0 is not included, obviously return False - backup_vm_dir_path = vm.dir_path.replace (system_path["qubes_base_dir"], backup_dir) - - if os.path.exists (backup_vm_dir_path): + if vm.backup_content: return True else: return False @@ -1044,16 +1650,12 @@ def backup_restore_prepare(backup_dir, options = {}, host_collection = None): return m.group(2) return template - #### Private functions end - - if not os.path.exists (backup_dir): - raise QubesException("The backup directory doesn't exist!") - - backup_collection = QubesVmCollection(store_filename = backup_dir + "/qubes.xml") + print "Loading file",qubes_xml + backup_collection = QubesVmCollection(store_filename = qubes_xml) backup_collection.lock_db_for_reading() backup_collection.load() - + if host_collection is None: host_collection = QubesVmCollection() host_collection.lock_db_for_reading() @@ -1072,6 +1674,7 @@ def backup_restore_prepare(backup_dir, options = {}, host_collection = None): # ... and the actual data for vm in backup_vms_list: if is_vm_included_in_backup (backup_dir, vm): + print vm.name,"is included in backup" vms_to_restore[vm.name] = {} vms_to_restore[vm.name]['vm'] = vm; @@ -1136,6 +1739,7 @@ def backup_restore_prepare(backup_dir, options = {}, host_collection = None): vms_to_restore[vm.name]['good-to-go'] = True # ...and dom0 home + # FIXME, replace this part of code to handle the new backup format using tar if options['dom0-home'] and os.path.exists(backup_dir + '/dom0-home'): vms_to_restore['dom0'] = {} local_user = grp.getgrnam('qubes').gr_mem[0] @@ -1254,18 +1858,7 @@ def backup_restore_print_summary(restore_info, print_callback = print_stdout): print_callback(s) -def backup_restore_do(backup_dir, restore_info, host_collection = None, print_callback = print_stdout, error_callback = print_stderr): - - ### Private functions begin - def restore_vm_dir (backup_dir, src_dir, dst_dir): - - backup_src_dir = src_dir.replace (system_path["qubes_base_dir"], backup_dir) - - # We prefer to use Linux's cp, because it nicely handles sparse files - retcode = subprocess.call (["cp", "-rp", backup_src_dir, dst_dir]) - if retcode != 0: - raise QubesException("*** Error while copying file {0} to {1}".format(backup_src_dir, dest_dir)) - ### Private functions end +def backup_restore_do(backup_dir, restore_tmpdir, passphrase, restore_info, host_collection = None, print_callback = print_stdout, error_callback = print_stderr, progress_callback = None, encrypted=False, appvm=None): lock_obtained = False if host_collection is None: @@ -1274,6 +1867,23 @@ def backup_restore_do(backup_dir, restore_info, host_collection = None, print_ca host_collection.load() lock_obtained = True + # Perform VM restoration in backup order + vms_dirs = [] + vms_size = 0 + vms = {} + for vm_info in restore_info.values(): + if not vm_info['good-to-go']: + continue + if 'vm' not in vm_info: + continue + vm = vm_info['vm'] + vms_size += vm.backup_size + vms_dirs.append(vm.backup_path+"*") + vms[vm.name] = vm + + restore_vm_dirs (backup_dir, restore_tmpdir, passphrase, vms_dirs, vms, vms_size, print_callback, error_callback, progress_callback, encrypted, appvm) + + # Add VM in right order for (vm_class_name, vm_class) in sorted(QubesVmClasses.items(), key=lambda _x: _x[1].load_order): @@ -1306,7 +1916,6 @@ def backup_restore_do(backup_dir, restore_info, host_collection = None, print_ca dir_path=vm.dir_path, template=template, installed_by_rpm=False) - restore_vm_dir (backup_dir, vm.dir_path, os.path.dirname(new_vm.dir_path)); new_vm.verify_files() except Exception as err: diff --git a/qubes-rpc/qubes.Backup b/qubes-rpc/qubes.Backup new file mode 100644 index 00000000..6e3f1d48 --- /dev/null +++ b/qubes-rpc/qubes.Backup @@ -0,0 +1,23 @@ +echo Starting Backupcopy +read args +echo Arguments: $args +if [ -d "$args" ] ; then + echo "Performing backup to directory $args" + TARGET="$args/qubes-backup-`date +'%Y-%d-%d-%H%M%S'`" + echo "Copying STDIN data to $TARGET" + cat > $TARGET +else + echo "Checking if arguments is matching a command" + COMMAND=`echo $args | cut -d ' ' -f 1` + TYPE=`type -t $COMMAND` + if [ "$TYPE" == "file" ] ; then + echo "Redirecting STDIN to $args" + # Parsing args to handle quotes correctly + # Dangerous method if args are uncontrolled + eval "set -- $args" + $@ + else + echo "Invalid command $COMMAND" + exit 1 + fi +fi diff --git a/qubes-rpc/qubes.Restore b/qubes-rpc/qubes.Restore new file mode 100644 index 00000000..96a76e53 --- /dev/null +++ b/qubes-rpc/qubes.Restore @@ -0,0 +1,23 @@ +echo Starting Restorecopy >2 +read args +echo Arguments: $args >2 +if [ -f "$args" ] ; then + echo "Performing restore from backup file $args" >2 + TARGET="$args" + echo "Copying $TARGET to STDOUT" >2 + cat $TARGET | /usr/lib/qubes/tar2qfile +else + echo "Checking if arguments is matching a command" >2 + COMMAND=`echo $args | cut -d ' ' -f 1` + TYPE=`type -t $COMMAND` + if [ "$TYPE" == "file" ] ; then + echo "Redirecting $args to STDOUT" >2 + # Parsing args to handle quotes correctly + # Dangerous method if args are uncontrolled + eval "set -- $args" + $@ + else + echo "Invalid command $COMMAND" >2 + exit 1 + fi +fi diff --git a/qvm-tools/qvm-backup b/qvm-tools/qvm-backup index c4b5bfe0..46d48096 100755 --- a/qvm-tools/qvm-backup +++ b/qvm-tools/qvm-backup @@ -22,7 +22,7 @@ from qubes.qubes import QubesVmCollection from qubes.qubes import QubesException -from qubes.qubesutils import backup_prepare, backup_do +from qubes.qubesutils import backup_prepare, backup_do_copy from optparse import OptionParser import os import sys @@ -38,6 +38,10 @@ def main(): help="Exclude the specified VM from backup (might be repeated)") parser.add_option ("--force-root", action="store_true", dest="force_root", default=False, help="Force to run, even with root privileges") + parser.add_option ("-d", "--dest-vm", action="store", dest="appvm", + help="The AppVM to send backups to") + parser.add_option ("-e", "--encrypt", action="store_true", dest="encrypt", default=False, + help="Encrypts the backup") (options, args) = parser.parse_args () @@ -75,8 +79,11 @@ def main(): if not (prompt == "y" or prompt == "Y"): exit (0) + passphrase = raw_input("Please enter the pass phrase that will be used to encrypt/verify the backup:\n") + passphrase = passphrase.replace("\r","").replace("\n","") + try: - backup_do(base_backup_dir, files_to_backup, progress_callback=print_progress) + backup_do_copy(base_backup_dir, files_to_backup, passphrase, progress_callback=print_progress, encrypt=options.encrypt,appvm=options.appvm) except QubesException as e: print >>sys.stderr, "ERROR: %s" % str(e) exit(1) diff --git a/qvm-tools/qvm-backup-restore b/qvm-tools/qvm-backup-restore index 6b1a590d..bf1c1253 100755 --- a/qvm-tools/qvm-backup-restore +++ b/qvm-tools/qvm-backup-restore @@ -22,7 +22,8 @@ from qubes.qubes import QubesVmCollection from qubes.qubes import QubesException -from qubes.qubesutils import backup_restore_prepare +from qubes.qubesutils import backup_restore_header +from qubes.qubesutils import backup_restore_prepare from qubes.qubesutils import backup_restore_print_summary from qubes.qubesutils import backup_restore_do from optparse import OptionParser @@ -58,6 +59,12 @@ def main(): parser.add_option ("--ignore-username-mismatch", action="store_true", dest="ignore_username_mismatch", default=False, help="Ignore dom0 username mismatch while restoring homedir") + parser.add_option ("-d", "--dest-vm", action="store", dest="appvm", + help="The AppVM to send backups to") + + parser.add_option ("-e", "--encrypted", action="store_true", dest="decrypt", default=False, + help="The backup is encrypted") + (options, args) = parser.parse_args () if (len (args) != 1): @@ -66,9 +73,9 @@ def main(): backup_dir = args[0] - if not os.path.exists (backup_dir): - print >> sys.stderr, "The backup directory doesn't exist!" - exit(1) + #if not os.path.exists (backup_dir): + # print >> sys.stderr, "The backup directory doesn't exist!" + # exit(1) host_collection = QubesVmCollection() host_collection.lock_db_for_writing() @@ -87,9 +94,16 @@ def main(): if options.exclude: restore_options['exclude'] = options.exclude + + passphrase = raw_input("Please enter the pass phrase that will be used to decrypt/verify the backup:\n") + passphrase = passphrase.replace("\r","").replace("\n","") + + print >> sys.stderr, "Checking backup content..." + restore_tmpdir,qubes_xml = backup_restore_header(backup_dir, passphrase, options.decrypt, appvm=options.appvm) + restore_info = None try: - restore_info = backup_restore_prepare(backup_dir, options=restore_options, host_collection=host_collection) + restore_info = backup_restore_prepare(backup_dir,os.path.join(restore_tmpdir, qubes_xml), passphrase, options=restore_options, host_collection=host_collection, encrypt=options.decrypt, appvm=options.appvm) except QubesException as e: print >> sys.stderr, "ERROR: %s" % str(e) exit(1) @@ -113,8 +127,6 @@ def main(): if 'username-mismatch' in vm_info.keys(): dom0_username_mismatch = True - print - if os.geteuid() == 0: print >> sys.stderr, "*** Running this tool as root is strongly discouraged, this will lead you in permissions problems." if options.force_root: @@ -179,7 +191,8 @@ def main(): if not (prompt == "y" or prompt == "Y"): exit (0) - backup_restore_do(backup_dir, restore_info, host_collection=host_collection) + + backup_restore_do(backup_dir,restore_tmpdir, passphrase, restore_info, host_collection=host_collection, encrypted=options.decrypt, appvm=options.appvm) host_collection.unlock_db()