Merge branch 'prebeta2' of git.qubes-os.org:/var/lib/qubes/git/rafal/core

appvm/qubes.Filecopy.policy

@@ -1 +1,6 @@
-anyvm	anyvm	ask,user=root
+## Note that policy parsing stops at the first match,
+## so adding anything below "$anyvm $anyvm action" line will have no effect
+
+## Please use a single # to start your custom comments
+
+$anyvm	$anyvm	ask,user=root

appvm/qubes.OpenInVM.policy

@@ -1,2 +1,7 @@
-anyvm	dispvm	allow
+## Note that policy parsing stops at the first match,
-anyvm	anyvm	ask
+## so adding anything below "$anyvm $anyvm action" line will have no effect
+
+## Please use a single # to start your custom comments
+
+$anyvm	$dispvm	allow
+$anyvm	$anyvm	ask

appvm/qvm-open-in-dvm2

@@ -25,4 +25,4 @@ if ! [ $# = 1 ] ; then
 	exit 1
 fi
 
-exec /usr/lib/qubes/qrexec_client_vm dispvm qubes.OpenInVM "/usr/lib/qubes/qopen-in-vm" "$1"
+exec /usr/lib/qubes/qrexec_client_vm '$dispvm' qubes.OpenInVM "/usr/lib/qubes/qopen-in-vm" "$1"

dom0/aux-tools/qubes.ReceiveUpdates.policy

@@ -1 +1,6 @@
-anyvm	dom0	allow
+## Note that policy parsing stops at the first match,
+## so adding anything below "$anyvm $anyvm action" line will have no effect
+
+## Please use a single # to start your custom comments
+
+$anyvm	dom0	allow

dom0/init.d/qubes_netvm

@@ -61,7 +61,7 @@ stop()
 
         echo -n $"Stopping NetVMs:"
         for VM in `get_running_netvms`; do
-            qvm-run -q --shutdown --wait $VM
+            qvm-run -q --force --shutdown --wait $VM
         done
 
     fi

dom0/qmemman/qmemman.py

@@ -22,6 +22,7 @@ class SystemState:
         self.BALOON_DELAY = 0.1
         self.XEN_FREE_MEM_LEFT = 50*1024*1024
         self.XEN_FREE_MEM_MIN = 25*1024*1024
+        self.ALL_PHYS_MEM = self.xc.physinfo()['total_memory']*1024
 
     def add_domain(self, id):
         self.domdict[id] = DomainState(id)
@@ -46,7 +47,13 @@ class SystemState:
                 self.domdict[id].memory_actual = domain['mem_kb']*1024
                 self.domdict[id].memory_maximum = self.xs.read('', '/local/domain/%s/memory/static-max' % str(id))
                 if not self.domdict[id].memory_maximum:
-                    self.domdict[id].memory_maximum = domain['maxmem_kb']*1024
+                    self.domdict[id].memory_maximum = self.ALL_PHYS_MEM
+# the previous line used to be                                   
+#                    self.domdict[id].memory_maximum = domain['maxmem_kb']*1024
+# but domain['maxmem_kb'] changes in self.mem_set as well, and this results in
+# the memory never increasing
+# in fact, the only possible case of nonexisting memory/static-max is dom0
+# see #307
 
 #the below works (and is fast), but then 'xm list' shows unchanged memory value
     def mem_set(self, id, val):
@@ -74,6 +81,15 @@ class SystemState:
         except XenAPI.Failure:
             pass
 
+# this is called at the end of ballooning, when we have Xen free mem already
+# make sure that past mem_set will not decrease Xen free mem
+    def inhibit_balloon_up(self):
+        for i in self.domdict.keys():
+            dom = self.domdict[i]
+            if dom.memory_actual is not None and dom.memory_actual + 200*1024 < dom.last_target:
+                print "Preventing balloon up to", dom.last_target 
+                self.mem_set(i, dom.memory_actual)
+
 #perform memory ballooning, across all domains, to add "memsize" to Xen free memory
     def do_balloon(self, memsize):
         MAX_TRIES = 20
@@ -81,12 +97,14 @@ class SystemState:
         prev_memory_actual = None
         for i in self.domdict.keys():
             self.domdict[i].no_progress = False
+        print "do_balloon start"
         while True:
+            self.refresh_memactual()
             xenfree = self.get_free_xen_memory()
             print 'got xenfree=', xenfree
             if xenfree >= memsize + self.XEN_FREE_MEM_MIN:
+                self.inhibit_balloon_up()
                 return True
-            self.refresh_memactual()
             if prev_memory_actual is not None:
                 for i in prev_memory_actual.keys():
                     if prev_memory_actual[i] == self.domdict[i].memory_actual:

dom0/qubes.SyncAppMenus.policy

@@ -1 +1,6 @@
-anyvm	dom0	allow
+## Note that policy parsing stops at the first match,
+## so adding anything below "$anyvm $anyvm action" line will have no effect
+
+## Please use a single # to start your custom comments
+
+$anyvm	dom0	allow

dom0/qvm-core/qubes.py

@@ -432,8 +432,8 @@ class QubesVm(object):
 
         dominfo = self.get_xl_dominfo()
         if dominfo:
-            uuid = uuid.UUID(''.join('%02x' % b for b in dominfo.uuid))
+            vmuuid = uuid.UUID(''.join('%02x' % b for b in dominfo.uuid))
-            return uuid
+            return vmuuid
         else:
             return None
 
@@ -742,6 +742,12 @@ class QubesVm(object):
         if source_template is None:
             source_template = self.template_vm
 
+        vmtype = None
+        if self.is_netvm():
+            vmtype = 'servicevms'
+        else:
+            vmtype = 'appvms'
+
         try:
             if source_template is not None:
                 subprocess.check_call ([qubes_appmenu_create_cmd, source_template.appmenus_templates_dir, self.name])
@@ -937,8 +943,6 @@ class QubesVm(object):
             subprocess.check_call(xl_cmdline)
         except:
             raise QubesException("Failed to load VM config")
-        finally:
-            qmemman_client.close() # let qmemman_daemon resume balancing
 
         xid = self.get_xid()
         self.xid = xid
@@ -970,6 +974,13 @@ class QubesVm(object):
                 self.force_shutdown()
                 raise OSError ("ERROR: Cannot execute qrexec_daemon!")
 
+# close() is not really needed, because the descriptor is close-on-exec
+# anyway, the reason to postpone close() is that possibly xl is not done
+# constructing the domain after its main process exits 
+# so we close() when we know the domain is up
+# the successful qrexec connect is a good indicator of it
+        qmemman_client.close()
+
         if preparing_dvm:
             if verbose:
                 print "--> Preparing config template for DispVM"
@@ -1093,11 +1104,6 @@ class QubesTemplateVm(QubesVm):
             print "--> Creating directory: {0}".format(self.dir_path)
         os.mkdir (self.dir_path)
 
-        if verbose:
-            print "--> Creating VM config file: {0}".\
-                    format(self.conf_file)
-        self.create_config_file(source_template=src_template_vm)
-
         if verbose:
             print "--> Copying the template's private image:\n{0} ==>\n{1}".\
                     format(src_template_vm.private_img, self.private_img)

dom0/qvm-tools/qvm-backup

@@ -87,7 +87,7 @@ def main():
 
     files_to_backup = file_to_backup (qubes_store_filename)
 
-    appvms_to_backup = [vm for vm in vms_list if vm.is_appvm()]
+    appvms_to_backup = [vm for vm in vms_list if vm.is_appvm() and not vm.name.endswith('-dvm')]
     there_are_running_vms = False
 
     fields_to_display = [ 
@@ -120,12 +120,12 @@ def main():
             files_to_backup += file_to_backup(vm.private_img, vm_sz )
 
             files_to_backup += file_to_backup(vm.icon_path)
-            files_to_backup += file_to_backup(vm.conf_file)
             if vm.is_updateable():
-                files_to_backup += file_to_backup(vm.dir_path + "/apps")
+                files_to_backup += file_to_backup(vm.dir_path + "/apps.templates")
-                files_to_backup += file_to_backup(vm.dir_path + "/kernels")
             if os.path.exists (vm.firewall_conf):
                 files_to_backup += file_to_backup(vm.firewall_conf)
+            if os.path.exists(vm.dir_path + '/whitelisted-appmenus.list'):
+                files_to_backup += file_to_backup(vm.dir_path + '/whitelisted-appmenus.list')
 
             if vm.is_updateable():
                 sz = vm.get_disk_usage(vm.root_img)

dom0/qvm-tools/qvm-backup-restore

@@ -130,9 +130,6 @@ def main():
     parser.add_option ("--force-root", action="store_true", dest="force_root", default=False,
                        help="Force to run, even with root privileges")
 
-    parser.add_option ("--recreate-conf-files", action="store_true", dest="recreate_conf", default=False,
-                       help="Recreate conf files after restore")
-
     parser.add_option ("--replace-template", action="append", dest="replace_template", default=[],
                        help="Restore VMs using another template, syntax: old-template-name:new-template-name (might be repeated)")
 
@@ -298,25 +295,16 @@ def main():
     if not (prompt == "y" or prompt == "Y"):
         exit (0)
 
-    for vm in vms_to_restore:
+    # Add templates...
-        print "-> Restoring: {0} ...".format(vm.name)
+    for vm in [ vm for vm in vms_to_restore if vm.is_template()]:
-
+        print "-> Restoring Template VM {0}...".format(vm.name)
         retcode = subprocess.call (["mkdir", "-p", vm.dir_path])
         if retcode != 0:
             print ("*** Cannot create directory: {0}?!".format(dest_dir))
             print ("Skiping...")
             continue
 
-        if vm.is_appvm():
-            restore_vm_dir (backup_dir, vm.dir_path, qubes_appvms_dir);
-        elif vm.is_template():
         restore_vm_dir (backup_dir, vm.dir_path, qubes_templates_dir);
-        else:
-            print "ERROR: VM '{0}', type='{1}': unsupported VM type!".format(vm.name, vm.type)
-
-    # Add templates...
-    for vm in [ vm for vm in vms_to_restore if vm.is_template()]:
-        print "-> Adding Template VM {0}...".format(vm.name)
         updateable = vm.updateable
         try:
             vm = host_collection.add_new_templatevm(vm.name, 
@@ -336,14 +324,19 @@ def main():
     # ... then appvms...
     for vm in [ vm for vm in vms_to_restore if vm.is_appvm()]:
 
-        print "-> Adding AppVM {0}...".format(vm.name)
+        print "-> Restoring AppVM {0}...".format(vm.name)
+        retcode = subprocess.call (["mkdir", "-p", vm.dir_path])
+        if retcode != 0:
+            print ("*** Cannot create directory: {0}?!".format(dest_dir))
+            print ("Skiping...")
+            continue
+
+        restore_vm_dir (backup_dir, vm.dir_path, qubes_appvms_dir);
+
         template_vm = None
-        recreate_conf = options.recreate_conf
         if vm.template_vm is not None:
             template_name = find_template_name(vm.template_vm.name, options.replace_template)
             template_vm = host_collection.get_vm_by_name(template_name)
-            if template_name != vm.template_vm.name:
-                recreate_conf = True
 
         if not vm.uses_default_netvm:
             uses_default_netvm = False
@@ -370,23 +363,15 @@ def main():
             vm.uses_default_netvm = False
             vm.netvm_vm = netvm_vm
 
-        try:
-            if template_vm is not None and recreate_conf:
-                print "--> Recreating config file..."
-                vm.create_config_file()
-        except QubesException as err:
-            print "ERROR xen config restore: {0}".format(err)
-            print "*** VM '{0}' will not boot until you manually fix it (or correctly restore this VM)!".format(vm.name)
-
         try:
             vm.create_appmenus(verbose=True)
-        except QubesException as err:
+        except Exception as err:
             print "ERROR during appmenu restore: {0}".format(err)
             print "*** VM '{0}' will not have appmenus".format(vm.name)
 
         try:
             vm.verify_files()
-        except QubesException as err:
+        except Exception as err:
             print "ERROR: {0}".format(err)
             print "*** Skiping VM: {0}".format(vm.name)
             host_collection.pop(vm.qid)

dom0/qvm-tools/qvm-clone-template

@@ -58,12 +58,6 @@ def main():
 
     try:
         dst_tvm.clone_disk_files (src_template_vm=src_tvm, verbose=options.verbose)
-
-        if options.verbose:
-            print "--> Adding to Xen Storage..."
- 
-        dst_tvm.add_to_xen_storage()
-
     except (IOError, OSError) as err:
         print "ERROR: {0}".format(err)
         qvm_collection.pop(dst_tvm.qid)

qrexec/qrexec_daemon.c

@@ -82,12 +82,14 @@ int create_qrexec_socket(int domid, char *domname)
 	return get_server_socket(socket_address);
 }
 
+#define MAX_STARTUP_TIME 120
 
 /* do the preparatory tasks, needed before entering the main event loop */
 void init(int xid)
 {
 	char qrexec_error_log_name[256];
 	int logfd;
+	int i;
 
 	if (xid <= 0) {
 		fprintf(stderr, "domain id=0?\n");
@@ -102,11 +104,12 @@ void init(int xid)
 		break;
 	default:
 		fprintf(stderr, "Waiting for VM's qrexec agent.");
-		for (;;) {
+		for (i=0;i<MAX_STARTUP_TIME;i++) {
 			sleep(1);
 			fprintf(stderr, ".");
 		}
-		exit(0);
+		fprintf(stderr, "Cannot connect to qrexec agent for %d seconds, giving up\n", MAX_STARTUP_TIME);
+		exit(1);
 	}
 	close(0);
 	snprintf(qrexec_error_log_name, sizeof(qrexec_error_log_name),
@@ -369,7 +372,7 @@ void sanitize_name(char * untrusted_s_signed)
                         continue;
                 if (*untrusted_s >= '0' && *untrusted_s <= '9')
                         continue;
-                if (*untrusted_s == '_' || *untrusted_s == '-' || *untrusted_s == '.' || *untrusted_s == ' ')
+                if (*untrusted_s == '$' || *untrusted_s == '_' || *untrusted_s == '-' || *untrusted_s == '.' || *untrusted_s == ' ')
                         continue;
                 *untrusted_s = '_';
         }

qrexec/qrexec_policy

@@ -12,6 +12,10 @@ def line_to_dict(line):
     tokens=line.split()
     if len(tokens) < 3:
         return None
+
+    if tokens[0][0] == '#':
+        return None  
+
     dict={}
     dict['source']=tokens[0]
     dict['dest']=tokens[1]
@@ -40,7 +44,7 @@ def read_policy_file(exec_index):
     return policy_list
 
 def is_match(item, config_term):
-    return (item is not "dom0" and config_term == "anyvm") or item == config_term
+    return (item is not "dom0" and config_term == "$anyvm") or item == config_term
 
 def get_default_policy():
     dict={}
@@ -76,7 +80,7 @@ def spawn_target_if_necessary(target):
 def do_execute(domain, target, user, exec_index, process_ident):
     if target == "dom0":
         cmd="/usr/lib/qubes/qubes_rpc_multiplexer "+exec_index + " " + domain
-    elif target == "dispvm":
+    elif target == "$dispvm":
         cmd = "/usr/lib/qubes/qfile-daemon-dvm " + exec_index + " " + domain + " " +user
     else:
     # see the previous commit why "qvm-run -a" is broken and dangerous