Merge branch 'master' of git.qubes-os.org:/var/lib/qubes/git/marmarek/core into spring-merge

appvm/.gitignore

@@ -1,3 +1,7 @@
 qubes_add_pendrive_script
 qubes_penctl
 qvm-open-in-dvm
+dvm_file_editor
+qfile-agent
+qfile-agent-dvm
+qfile-unpacker

dom0/aux-tools/create_apps_for_appvm.sh

@@ -31,10 +31,12 @@ if [ $# != 2 ]; then
 fi
 mkdir -p $APPSDIR
 
-echo "--> Converting Appmenu Templates..."
-find $SRCDIR -name "*.desktop" -exec /usr/lib/qubes/convert_apptemplate2vm.sh {} $APPSDIR $VMNAME $VMDIR \;
+if [ "$SRCDIR" != "none" ]; then
+    echo "--> Converting Appmenu Templates..."
+    find $SRCDIR -name "*.desktop" -exec /usr/lib/qubes/convert_apptemplate2vm.sh {} $APPSDIR $VMNAME $VMDIR \;
 
-/usr/lib/qubes/convert_dirtemplate2vm.sh $SRCDIR/qubes-vm.directory.template $APPSDIR/$VMNAME-vm.directory $VMNAME $VMDIR
+    /usr/lib/qubes/convert_dirtemplate2vm.sh $SRCDIR/qubes-vm.directory.template $APPSDIR/$VMNAME-vm.directory $VMNAME $VMDIR
+fi
 
 echo "--> Adding Apps to the Menu..."
 xdg-desktop-menu install $APPSDIR/*.directory $APPSDIR/*.desktop

dom0/aux-tools/reset_vm_configs.py

@@ -18,7 +18,7 @@ def main():
         print templ, 'is not a template'
         sys.exit(1)
     for vm in qvm_collection.values():
-        if vm.is_appvm() and vm.template_vm.qid == tvm.qid:
+        if vm.template_vm is not None and vm.template_vm.qid == tvm.qid:
             vm.create_config_file()
     
 main()

dom0/init.d/qubes_netvm

@@ -30,8 +30,8 @@ start()
     elif [ $NETVM = "dom0" ] ; then 
 
         echo -n $"Setting up net backend in Dom0:"
-        echo "NS1=10.0.0.1" > /var/run/qubes/qubes_ns
-        echo "NS2=10.0.255.254" >> /var/run/qubes/qubes_ns
+        echo "NS1=10.137.0.1" > /var/run/qubes/qubes_ns
+        echo "NS2=10.137.255.254" >> /var/run/qubes/qubes_ns
         /usr/lib/qubes/qubes_setup_dnat_to_ns
         echo "1" > /proc/sys/net/ipv4/ip_forward || exit 1
     else

dom0/qvm-core/qubes.py

@@ -54,9 +54,9 @@ qubes_templates_dir = qubes_base_dir + "/vm-templates"
 qubes_servicevms_dir = qubes_base_dir + "/servicevms"
 qubes_store_filename = qubes_base_dir + "/qubes.xml"
 
-qubes_max_qid = 254*254
+qubes_max_qid = 254
 qubes_max_netid = 254
-vm_default_netmask = "255.255.0.0"
+vm_default_netmask = "255.255.255.0"
 
 default_root_img = "root.img"
 default_rootcow_img = "root-cow.img"
@@ -199,6 +199,9 @@ class QubesVm(object):
         self.uses_default_netvm = uses_default_netvm
         self.netvm_vm = netvm_vm
 
+        # Create template_vm property - used in AppVM, NetVM, ProxyVM
+        self.template_vm = None
+
         # We use it in remove from disk to avoid removing rpm files (for templates)
         self.installed_by_rpm = installed_by_rpm
 
@@ -473,6 +476,11 @@ class QubesVm(object):
 
         return os.path.getsize(self.private_img)
 
+    def resize_private_img(self, size):
+        f_private = open (self.private_img, "a+b")
+        f_private.truncate (size)
+        f_private.close ()
+
     def create_xenstore_entries(self, xid):
         if dry_run:
             return
@@ -1127,11 +1135,11 @@ class QubesNetVm(QubesCowVm):
     def __init__(self, **kwargs):
         netid = kwargs.pop("netid")
         self.netid = netid
-        self.__network = "10.{0}.0.0".format(netid)
-        self.netprefix = "10.{0}.".format(netid)
+        self.__network = "10.137.{0}.0".format(netid)
+        self.netprefix = "10.137.{0}.".format(netid)
         self.__netmask = vm_default_netmask
-        self.__gateway = self.netprefix + "0.1"
-        self.__secondary_dns = self.netprefix + "255.254"
+        self.__gateway = self.netprefix + "1"
+        self.__secondary_dns = self.netprefix + "254"
 
         if "dir_path" not in kwargs or kwargs["dir_path"] is None:
             kwargs["dir_path"] = qubes_servicevms_dir + "/" + kwargs["name"]
@@ -1165,10 +1173,9 @@ class QubesNetVm(QubesCowVm):
         return self.__network
 
     def get_ip_for_vm(self, qid):
-        hi = qid / 253
         lo = qid % 253 + 2
-        assert hi >= 0 and hi <= 254 and lo >= 2 and lo <= 254, "Wrong IP address for VM"
-        return self.netprefix  + "{0}.{1}".format(hi,lo)
+        assert lo >= 2 and lo <= 254, "Wrong IP address for VM"
+        return self.netprefix  + "{0}".format(lo)
 
     def create_xenstore_entries(self, xid):
         if dry_run:
@@ -1492,7 +1499,11 @@ class QubesAppVm(QubesCowVm):
         self.create_appmenus (verbose)
 
     def create_appmenus(self, verbose):
-        subprocess.check_call ([qubes_appmenu_create_cmd, self.template_vm.appmenus_templates_dir, self.name])
+        if self.template_vm is not None:
+            subprocess.check_call ([qubes_appmenu_create_cmd, self.template_vm.appmenus_templates_dir, self.name])
+        else:
+            # Only add apps to menu
+            subprocess.check_call ([qubes_appmenu_create_cmd, "none", self.name])
 
     def write_firewall_conf(self, conf):
         root = xml.etree.ElementTree.Element(

dom0/qvm-tools/qvm-backup

@@ -121,19 +121,25 @@ def main():
 
             files_to_backup += file_to_backup(vm.icon_path)
             files_to_backup += file_to_backup(vm.conf_file)
-            #files_to_backup += file_to_backup(vm.dir_path + "/apps")
+            if vm.is_updateable():
+                files_to_backup += file_to_backup(vm.dir_path + "/apps")
+            if os.path.exists (vm.firewall_conf):
+                files_to_backup += file_to_backup(vm.firewall_conf)
 
             if vm.is_updateable():
                 sz = vm.get_disk_usage(vm.root_img)
                 files_to_backup += file_to_backup(vm.root_img, sz)
                 vm_sz += sz
+                sz = vm.get_disk_usage(vm.volatile_img)
+                files_to_backup += file_to_backup(vm.volatile_img, sz)
+                vm_sz += sz
 
             s = ""
             fmt="{{0:>{0}}} |".format(fields_to_display[0]["width"] + 1)
             s += fmt.format(vm.name) 
 
             fmt="{{0:>{0}}} |".format(fields_to_display[1]["width"] + 1)
-            s += fmt.format("AppVM" + (" + COW" if vm.is_updateable() else "")) 
+            s += fmt.format("AppVM" + (" + Sys" if vm.is_updateable() else ""))
 
             fmt="{{0:>{0}}} |".format(fields_to_display[2]["width"] + 1)
             s += fmt.format(size_to_human(vm_sz)) 

dom0/qvm-tools/qvm-backup-restore

@@ -25,12 +25,14 @@ from qubes.qubes import QubesException
 from qubes.qubes import qubes_store_filename
 from qubes.qubes import qubes_base_dir
 from qubes.qubes import qubes_templates_dir
+from qubes.qubes import qubes_appvms_dir
 from optparse import OptionParser
 
 import os
 import time
 import subprocess
 import sys
+import re
 
 def size_to_human (size):
     if size < 1024:
@@ -58,8 +60,9 @@ fields = {
 
     "updbl" : {"func": "'Yes' if vm.is_updateable() else ''"},
 
-    "template": {"func": "'n/a' if vm.is_template() or vm.is_netvm() else\
-                 backup_collection[vm.template_vm.qid].name"},
+    "template": {"func": "'n/a' if vm.is_template() or vm.template_vm is None else\
+                 find_template_name(backup_collection[vm.template_vm.qid].name,\
+                         options.replace_template)"},
 
     "netvm": {"func": "'n/a' if vm.is_netvm() else\
               ('*' if vm.uses_default_netvm else '') +\
@@ -102,6 +105,14 @@ def restore_vm_dir (backup_dir, src_dir, dst_dir):
         print "*** Error while copying file {0} to {1}".format(backup_src_dir, dest_dir)
         exit (1)
 
+def find_template_name(template, replaces):
+    rx_replace = re.compile("(.*):(.*)")
+    for r in replaces:
+        m = rx_replace.match(r)
+        if m.group(1) == template:
+            return m.group(2)
+
+    return template
 
 def main():
     usage = "usage: %prog [options] <backup-dir>"
@@ -116,6 +127,12 @@ def main():
     parser.add_option ("--skip-conflicting", action="store_true", dest="skip_conflicting", default=False,
                        help="Do not restore VMs that are already present on the host")
 
+    parser.add_option ("--recreate-conf-files", action="store_true", dest="recreate_conf", default=False,
+                       help="Recreate conf files after restore")
+
+    parser.add_option ("--replace-template", action="append", dest="replace_template", default=[],
+                       help="Restore VMs using another template, syntax: old-template-name:new-template-name (might be repeated)")
+
     (options, args) = parser.parse_args ()
 
     if (len (args) != 1):
@@ -191,12 +208,12 @@ def main():
                 there_are_conflicting_vms = True
                 good_to_go = False # Do not overwrite VMs on the host!
 
-            if vm.is_appvm():
-                templatevm_name = vm.template_vm.name
+            if vm.template_vm is not None:
+                templatevm_name = find_template_name(vm.template_vm.name, options.replace_template)
                 template_vm_on_host = host_collection.get_vm_by_name (templatevm_name)
 
                 # No template on the host?
-                if not ((template_vm_on_host is not None) and template_vm_on_host.is_template):
+                if not ((template_vm_on_host is not None) and template_vm_on_host.is_template()):
 
                     # Maybe the (custom) template is in the backup?
                     template_vm_on_backup = backup_collection.get_vm_by_name (templatevm_name)
@@ -279,14 +296,7 @@ def main():
             continue
 
         if vm.is_appvm():
-
-            restore_vm_file (backup_dir, vm.private_img)
-            restore_vm_file (backup_dir, vm.icon_path)
-            restore_vm_file (backup_dir, vm.conf_file)
-
-            if vm.is_updateable():
-                restore_vm_file (backup_dir, vm.root_img)
-
+            restore_vm_dir (backup_dir, vm.dir_path, qubes_appvms_dir);
         elif vm.is_template():
             restore_vm_dir (backup_dir, vm.dir_path, qubes_templates_dir);
         else:
@@ -323,7 +333,13 @@ def main():
     for vm in [ vm for vm in vms_to_restore if vm.is_appvm()]:
 
         print "-> Adding AppVM {0}...".format(vm.name)
-        template_vm = host_collection.get_vm_by_name(vm.template_vm.name)
+        template_vm = None
+        recreate_conf = options.recreate_conf
+        if vm.template_vm is not None:
+            template_name = find_template_name(vm.template_vm.name, options.replace_template)
+            template_vm = host_collection.get_vm_by_name(template_name)
+            if template_name != vm.template_vm.name:
+                recreate_conf = True
 
         if not vm.uses_default_netvm:
             uses_default_netvm = False
@@ -336,27 +352,30 @@ def main():
         vm = host_collection.add_new_appvm(vm.name, template_vm,
                                           conf_file=vm.conf_file,
                                           dir_path=vm.dir_path,
+                                          updateable=updateable,
                                           label=vm.label)
 
-        vm.updateable = updateable
         if not uses_default_netvm:
             vm.uses_default_netvm = False
             vm.netvm_vm = netvm_vm
 
+        if template_vm is not None and recreate_conf:
+            print "--> Recreating config file..."
+            vm.create_config_file()
 
         vm.create_appmenus(verbose=True)
         try:
             vm.verify_files()
         except QubesException as err:
             print "ERROR: {0}".format(err)
-            print "*** Skiping VM: {0}".vm.name
+            print "*** Skiping VM: {0}".format(vm.name)
             host_collection.pop(vm.qid)
 
         try:
             vm.add_to_xen_storage()
         except (IOError, OSError) as err:
             print "ERROR: {0}".format(err)
-            print "*** Skiping VM: {0}".vm.name
+            print "*** Skiping VM: {0}".format(vm.name)
             host_collection.pop(vm.qid)
 
 

dom0/qvm-tools/qvm-dom0-network-via-netvm

@@ -52,7 +52,7 @@ def bringup_eth0(netvm):
     resolv_conf.write('nameserver ' + netvm.gateway + '\n')
     resolv_conf.write('nameserver ' + netvm.secondary_dns + '\n')
     resolv_conf.close()
-    return os.system('ifconfig eth0 10.0.0.1 netmask 255.255.255.255 && route add default dev eth0') == 0
+    return os.system('ifconfig eth0 10.137.0.1 netmask 255.255.255.255 && route add default dev eth0') == 0
 
 def unpause_all(netvm_name):
      os.system('qvm-run --exclude=' + netvm_name + ' --all --unpause')
@@ -62,7 +62,7 @@ def netup():
     if os.path.isfile('/var/lock/subsys/NetworkManager'):
         os.system('/etc/init.d/NetworkManager stop')
     if not vif_eth0_exists():
-        cmd = 'modprobe xennet && xm network-attach 0 ip=10.0.0.1 backend='
+        cmd = 'modprobe xennet && xm network-attach 0 ip=10.137.0.1 backend='
         cmd += netvm.name
         cmd += ' script=vif-route-qubes'
         if os.system(cmd) != 0:

dom0/qvm-tools/qvm-grow-private

@@ -0,0 +1,109 @@
+#!/usr/bin/python2.6
+#
+# The Qubes OS Project, http://www.qubes-os.org
+#
+# Copyright (C) 2011  Marek Marczykowski <marmarek@mimuw.edu.pl>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+#
+
+from qubes.qubes import QubesVmCollection
+from qubes.qubes import QubesException
+from optparse import OptionParser
+import subprocess
+import os
+import re
+
+qvm_run_path = "/usr/bin/qvm-run"
+
+def parse_size(size):
+    units = [ ('K', 1024), ('KB', 1024),
+        ('M', 1024*1024), ('MB', 1024*1024),
+        ('G', 1024*1024*1024), ('GB', 1024*1024*1024),
+    ]
+
+    size = size.strip().upper()
+    if size.isdigit():
+        return size
+
+    for unit, multiplier in units:
+        if size.endswith(unit):
+            size = size[:-len(unit)].strip()
+            return int(size)*multiplier
+    
+    print "Invalid size: {0}.".format(size)
+    exit(1)
+
+def main():
+    usage = "usage: %prog <vm-name> <size>"
+    parser = OptionParser (usage)
+
+    (options, args) = parser.parse_args ()
+    if (len (args) != 2):
+        parser.error ("You must specify VM name and new size!")
+    vmname = args[0]
+    size = args[1]
+
+    qvm_collection = QubesVmCollection()
+    qvm_collection.lock_db_for_reading()
+    qvm_collection.load()
+    qvm_collection.unlock_db()
+
+    vm = qvm_collection.get_vm_by_name(vmname)
+    if vm is None:
+        print "A VM with the name '{0}' does not exist in the system.".format(vmname)
+        exit(1)
+
+    if vm.is_running() and os.geteuid() != 0:
+        print "You must be root to grow private.img on running VM."
+        exit(1)
+
+    size_bytes = parse_size(size)
+
+    if size_bytes < vm.get_private_img_sz():
+        print "Cannot shrink private.img ({0} < {1})".format(size_bytes, vm.get_private_img_sz())
+        exit(1)
+
+    try:
+        vm.resize_private_img(size_bytes)
+    except (IOError, OSError, QubesException) as err:
+        print "ERROR: {0}".format(err)
+        exit (1)
+
+    if vm.is_running():
+        # find loop device
+        p = subprocess.Popen (["losetup", "--associated", vm.private_img],
+                stdout=subprocess.PIPE)
+        result = p.communicate()
+        m = re.match(r"^(/dev/loop\d+):\s", result[0])
+        if m is None:
+            print "ERROR: Cannot find loop device!"
+            exit(1)
+
+        loop_dev = m.group(1)
+
+        # resize loop device
+        retcode = subprocess.check_call(["losetup", "--set-capacity", loop_dev])
+
+        retcode = subprocess.check_call([qvm_run_path, "-uroot", vmname, 
+                "resize2fs /dev/xvdb" ])
+    else:
+        retcode = subprocess.check_call(["resize2fs", "-f", vm.private_img])
+        
+	exit (0)
+
+
+main()

qrexec/.gitignore

@@ -0,0 +1,3 @@
+qrexec_agent
+qrexec_client
+qrexec_daemon

rpm_spec/core-appvm.spec

@@ -24,7 +24,7 @@
 
 Name:		qubes-core-appvm
 Version:	%{version}
-Release:	1
+Release:	1%{dist}
 Summary:	The Qubes core files for AppVM
 
 Group:		Qubes

rpm_spec/core-commonvm.spec

@@ -24,7 +24,7 @@
 
 Name:		qubes-core-commonvm
 Version:	%{version}
-Release:	1
+Release:	1%{dist}
 Summary:	The Qubes core files for any VM
 
 Group:		Qubes

rpm_spec/core-dom0.spec

@@ -29,7 +29,7 @@
 
 Name:		qubes-core-dom0
 Version:	%{version}
-Release:	1
+Release:	1%{dist}
 Summary:	The Qubes core files (Dom0-side)
 
 Group:		Qubes
@@ -51,9 +51,9 @@ python -m compileall qvm-core qmemman
 python -O -m compileall qvm-core qmemman
 make -C restore
 make -C ../common
-make -C ../qrexec
 make -C ../vchan
 make -C ../u2mfn
+make -C ../qrexec
 
 %install
 

rpm_spec/core-netvm.spec

@@ -24,7 +24,7 @@
 
 Name:		qubes-core-netvm
 Version:	%{version}
-Release:	1
+Release:	1%{dist}
 Summary:	The Qubes core files for NetVM
 
 Group:		Qubes

rpm_spec/core-proxyvm.spec

@@ -24,7 +24,7 @@
 
 Name:		qubes-core-proxyvm
 Version:	%{version}
-Release:	1
+Release:	1%{dist}
 Summary:	The Qubes core files for NetVM
 
 Group:		Qubes

u2mfn/.gitignore

@@ -0,0 +1 @@
+libu2mfn.so

vchan/.gitignore

@@ -0,0 +1 @@
+libvchan.so

version_dom0

@@ -1 +1 @@
-1.4.1
+1.5.1

version_vm

@@ -1 +1 @@
-1.4.1
+1.5.1