Merge branch 'hvm' of 10.141.1.101:/var/lib/qubes/git/marmarek/core into hvm

dom0/misc/qmemman.conf

@@ -0,0 +1,13 @@
+# The only section in this file
+[global]
+# vm-min-mem - give at least this amount of RAM for dynamically managed VM
+#  Default: 200M
+vm-min-mem = 200M
+
+# dom0-mem-boost - additional memory given to dom0 for disk caches etc
+#  Default: 350M
+dom0-mem-boost = 350M
+
+# cache-margin-factor - calculate VM preferred memory as (used memory)*cache-margin-factor
+#  Default: 1.3
+cache-margin-factor = 1.3

dom0/qmemman/qmemman_algo.py

@@ -1,5 +1,11 @@
 import string
 
+# This are only defaults - can be overriden by QMemmanServer with values from
+# config file
+CACHE_FACTOR = 1.3
+MIN_PREFMEM = 200*1024*1024
+DOM0_MEM_BOOST = 350*1024*1024
+
 #untrusted meminfo size is taken from xenstore key, thus its size is limited
 #so splits do not require excessive memory
 def parse_meminfo(untrusted_meminfo):
@@ -57,8 +63,6 @@ def refresh_meminfo_for_domain(domain, untrusted_xenstore_key):
         domain.mem_used =  domain.meminfo['MemTotal'] - domain.meminfo['MemFree'] - domain.meminfo['Cached'] - domain.meminfo['Buffers'] + domain.meminfo['SwapTotal'] - domain.meminfo['SwapFree']
                         
 def prefmem(domain):
-    CACHE_FACTOR = 1.3
-    MIN_PREFMEM = 200*1024*1024
 #dom0 is special, as it must have large cache, for vbds. Thus, give it a special boost
     if domain.id == '0':
         return min(domain.mem_used*CACHE_FACTOR + 350*1024*1024, domain.memory_maximum)

dom0/qmemman/qmemman_server.py

@@ -6,6 +6,12 @@ import xen.lowlevel.xs
 import sys
 import os
 from qmemman import SystemState
+import qmemman_algo
+from ConfigParser import SafeConfigParser
+from optparse import OptionParser
+from qubesutils import parse_size
+
+config_path = '/etc/qubes/qmemman.conf'
 
 system_state = SystemState()
 global_lock = thread.allocate_lock()
@@ -113,5 +119,23 @@ def start_server():
 class QMemmanServer:
     @staticmethod          
     def main():
+        usage = "usage: %prog [options]"
+        parser = OptionParser(usage)
+        parser.add_option("-c", "--config", action="store", dest="config", default=config_path)
+        (options, args) = parser.parse_args()
+
+        config = SafeConfigParser({
+                'vm-min-mem': str(qmemman_algo.MIN_PREFMEM),
+                'dom0-mem-boost': str(qmemman_algo.DOM0_MEM_BOOST),
+                'cache-margin-factor': str(qmemman_algo.CACHE_FACTOR)
+                })
+        config.read(options.config)
+        if config.has_section('global'):
+            qmemman_algo.MIN_PREFMEM = parse_size(config.get('global', 'vm-min-mem'))
+            qmemman_algo.DOM0_MEM_BOOST = parse_size(config.get('global', 'dom0-mem-boost'))
+            qmemman_algo.CACHE_FACTOR = config.getfloat('global', 'cache-margin-factor')
+
+        print "values: %s, %s, %s" % (str(qmemman_algo.MIN_PREFMEM), str(qmemman_algo.DOM0_MEM_BOOST), str(qmemman_algo.CACHE_FACTOR))
+
         thread.start_new_thread(start_server, tuple([]))
         XS_Watcher().watch_loop()

dom0/qvm-core/qubes.py

@@ -248,11 +248,10 @@ class QubesVm(object):
             # Internal VM (not shown in qubes-manager, doesn't create appmenus entries
             "internal": { "default": False },
             "vcpus": { "default": None },
-            "kernel": { "default": None, 'eval': \
+            "kernel": { "default": None, 'order': 30 },
-                'self.template.kernel if self.template is not None else value' },
+            "uses_default_kernel": { "default": True, 'order': 30 },
-            "uses_default_kernel": { "default": True },
+            "uses_default_kernelopts": { "default": True, 'order': 30 },
-            "uses_default_kernelopts": { "default": True },
+            "kernelopts": { "default": "", 'order': 30, "eval": \
-            "kernelopts": { "default": "", "eval": \
                 'value if not self.uses_default_kernelopts else default_kernelopts_pcidevs if len(self.pcidevs) > 0 else default_kernelopts' },
             "mac": { "attr": "_mac", "default": None },
             "include_in_backups": { "default": True },
@@ -263,8 +262,7 @@ class QubesVm(object):
                 'self.template.appmenus_templates_dir if self.template is not None else None' },
             "config_file_template": { "eval": "config_template_pv" },
             "icon_path": { "eval": 'self.dir_path + "/icon.png" if self.dir_path is not None else None' },
-            "kernels_dir": { 'eval': 'self.template.kernels_dir if self.template is not None else ' + \
+            "kernels_dir": { 'eval': 'qubes_kernels_base_dir + "/" + self.kernel if self.kernel is not None else ' + \
-                'qubes_kernels_base_dir + "/" + self.kernel if self.kernel is not None else ' + \
                 # for backward compatibility (or another rare case): kernel=None -> kernel in VM dir
                 'self.dir_path + "/" + default_kernels_subdir' },
             }
@@ -332,6 +330,10 @@ class QubesVm(object):
             qubes_host = QubesHost()
             self.vcpus = qubes_host.no_cpus
 
+        # Always set if meminfo-writer should be active or not
+        if 'meminfo-writer' not in self.services:
+            self.services['meminfo-writer'] = not (len(self.pcidevs) > 0)
+
         # Some additional checks for template based VM
         if self.template is not None:
             if not self.template.is_template():
@@ -2163,6 +2165,14 @@ class QubesHVm(QubesVm):
     def is_appvm(self):
         return True
 
+    def get_clone_attrs(self):
+        attrs = super(QubesHVm, self).get_clone_attrs()
+        attrs.remove('kernel')
+        attrs.remove('uses_default_kernel')
+        attrs.remove('kernelopts')
+        attrs.remove('uses_default_kernelopts')
+        return attrs
+
     def create_on_disk(self, verbose, source_template = None):
         if dry_run:
             return
@@ -2217,7 +2227,10 @@ class QubesHVm(QubesVm):
             if drive_path.startswith("hd:"):
                 type_mode = ",w"
                 drive_path = drive_path[3:]
-            backend_split = re.match(r"^([a-zA-Z0-9]*):(.*)", drive_path)
+            elif drive_path.startswith("cdrom:"):
+                type_mode = ":cdrom,r"
+                drive_path = drive_path[6:]
+            backend_split = re.match(r"^([a-zA-Z0-9-]*):(.*)", drive_path)
             if backend_split:
                 backend_domain = "," + backend_split.group(1)
                 drive_path = backend_split.group(2)
@@ -2368,8 +2381,6 @@ class QubesVmCollection(dict):
         qid = self.get_new_unused_qid()
         vm = QubesHVm (qid=qid, name=name,
                          netvm = self.get_default_netvm(),
-                         kernel = self.get_default_kernel(),
-                         uses_default_kernel = True,
                          label=label)
 
         if not self.verify_new_vm (vm):

dom0/qvm-core/qubesutils.py

@@ -78,7 +78,7 @@ def parse_size(size):
 
     size = size.strip().upper()
     if size.isdigit():
-        return size
+        return int(size)
 
     for unit, multiplier in units:
         if size.endswith(unit):
@@ -142,6 +142,9 @@ def block_name_to_majorminor(name):
     elif name.startswith("sr"):
         disk = False
         major = 11
+    elif name.startswith("loop"):
+        disk = False
+        major = 7
     elif name.startswith("md"):
         disk = False
         major = 9
@@ -185,7 +188,7 @@ def block_find_unused_frontend(vm = None):
             return block_devid_to_name(major << 8 | minor)
     return None
 
-def block_list(vm = None):
+def block_list(vm = None, system_disks = False):
     device_re = re.compile(r"^[a-z0-9]{1,8}$")
     # FIXME: any better idea of desc_re?
     desc_re = re.compile(r"^.{1,255}$")
@@ -229,6 +232,11 @@ def block_list(vm = None):
             if block_name_to_majorminor(device) == (0, 0):
                 print >> sys.stderr, "Unsupported device %s:%s" % (vm_name, device)
                 continue
+
+            if not system_disks:
+                if xid == '0' and device_desc.startswith(qubes_base_dir):
+                    continue
+
             visible_name = "%s:%s" % (vm_name, device)
             devices_list[visible_name] = {"name": visible_name, "xid":int(xid),
                 "vm": vm_name, "device":device, "size":int(device_size),
@@ -242,25 +250,34 @@ def block_check_attached(backend_vm, device, backend_xid = None):
     vm_list = xs.ls('', '/local/domain/%d/backend/vbd' % backend_xid)
     if vm_list is None:
         return None
-    device_majorminor = block_name_to_majorminor(device)
+    device_majorminor = None
+    try:
+        device_majorminor = block_name_to_majorminor(device)
+    except:
+        # Unknown devices will be compared directly - perhaps it is a filename?
+        pass
     for vm_xid in vm_list:
         for devid in xs.ls('', '/local/domain/%d/backend/vbd/%s' % (backend_xid, vm_xid)):
             (tmp_major, tmp_minor) = (0, 0)
             phys_device = xs.read('', '/local/domain/%d/backend/vbd/%s/%s/physical-device' % (backend_xid, vm_xid, devid))
+            dev_params = xs.read('', '/local/domain/%d/backend/vbd/%s/%s/params' % (backend_xid, vm_xid, devid))
             if phys_device and phys_device.find(':'):
                 (tmp_major, tmp_minor) = phys_device.split(":")
                 tmp_major = int(tmp_major, 16)
                 tmp_minor = int(tmp_minor, 16)
             else:
                 # perhaps not ready yet - check params
-                dev_params = xs.read('', '/local/domain/%d/backend/vbd/%s/%s/params' % (backend_xid, vm_xid, devid))
+                if not dev_params:
-                if not dev_params or not dev_params.startswith('/dev/'):
                     # Skip not-phy devices
                     continue
+                elif not dev_params.startswith('/dev/'):
+                    # will compare params directly
+                    pass
                 else:
                     (tmp_major, tmp_minor) = block_name_to_majorminor(dev_params.lstrip('/dev/'))
 
-            if (tmp_major, tmp_minor) == device_majorminor:
+            if (device_majorminor and (tmp_major, tmp_minor) == device_majorminor) or \
+               (device_majorminor is None and dev_params == device):
                 vm_name = xl_ctx.domid_to_name(int(vm_xid))
                 frontend = block_devid_to_name(int(devid))
                 return {"xid":int(vm_xid), "frontend": frontend, "devid": int(devid), "vm": vm_name}
@@ -290,7 +307,12 @@ def block_attach(vm, backend_vm, device, frontend=None, mode="w", auto_detach=Fa
         else:
             raise QubesException("Device %s from %s already connected to VM %s as %s" % (device, backend_vm.name, attached_vm['vm'], attached_vm['frontend']))
 
-    xl_cmd = [ '/usr/sbin/xl', 'block-attach', vm.name, 'phy:/dev/' + device, frontend, mode, str(backend_vm.xid) ]
+    if device.startswith('/'):
+        backend_dev = 'script:file:' + device
+    else:
+        backend_dev = 'phy:/dev/' + device
+
+    xl_cmd = [ '/usr/sbin/xl', 'block-attach', vm.name, backend_dev, frontend, mode, str(backend_vm.xid) ]
     subprocess.check_call(xl_cmd)
 
 def block_detach(vm, frontend = "xvdi", vm_xid = None):

dom0/qvm-tools/qvm-block

@@ -37,6 +37,8 @@ def main():
 
     parser = OptionParser (usage)
     parser.add_option ("-l", "--list", action="store_true", dest="do_list", default=False)
+    parser.add_option ("-A", "--attach-file", action="store", dest="file_path", default=None,
+                        help="Attach specified file instead of physical device (syntax: qvm-block -A file backend-VM frontend-VM)")
     parser.add_option ("-a", "--attach", action="store_true", dest="do_attach", default=False)
     parser.add_option ("-d", "--detach", action="store_true", dest="do_detach", default=False)
     parser.add_option ("-f", "--frontend", dest="frontend", 
@@ -45,11 +47,16 @@ def main():
                         help="Force read-only mode")
     parser.add_option ("--no-auto-detach", dest="auto_detach", action="store_false", default=True,
                         help="Fail when device already connected to other VM")
+    parser.add_option ("--show-system-disks", dest="system_disks", action="store_true", default=False,
+                        help="List also system disks")
 
     (options, args) = parser.parse_args ()
 
+    if options.file_path:
+        options.do_attach = True
+
     if options.do_list + options.do_attach + options.do_detach > 1:
-        print >> sys.stderr, "Only one of -l -a -d is allowed!"
+        print >> sys.stderr, "Only one of -l -a/-A -d is allowed!"
         exit (1)
 
     if options.do_attach or options.do_detach:
@@ -64,10 +71,16 @@ def main():
         vm = qvm_collection.get_vm_by_name(args[1])
         if vm is None:
             parser.error ("Invalid VM name: %s" % args[1])
-        dev_list = block_list()
+        if options.file_path:
-        if not args[0] in dev_list.keys():
+            dev = {}
-            parser.error ("Invalid device name: %s" % args[0])
+            dev['vm'] = args[0]
-        dev = dev_list[args[0]]
+            dev['device'] = options.file_path
+            dev['mode'] = 'w'
+        else:
+            dev_list = block_list()
+            if not args[0] in dev_list.keys():
+                parser.error ("Invalid device name: %s" % args[0])
+            dev = dev_list[args[0]]
         backend_vm = qvm_collection.get_vm_by_name(dev['vm'])
         assert backend_vm is not None
         kwargs = {}
@@ -102,7 +115,9 @@ def main():
             block_detach(None, attached_to['devid'], vm_xid=attached_to['xid'])
     else:
         # do_list
-        for dev in block_list().values():
+        kwargs = {}
+        kwargs['system_disks'] = options.system_disks
+        for dev in block_list(**kwargs).values():
             attached_to = block_check_attached(None, dev['device'], backend_xid = dev['xid'])
             attached_to_str = ""
             if attached_to:

dom0/qvm-tools/qvm-prefs

@@ -65,9 +65,7 @@ def do_list(vm):
     print fmt.format ("MAC", "%s%s" % (vm.mac, " (auto)" if vm._mac is None else ""))
 
     if hasattr(vm, 'kernel'):
-        if vm.template is not None:
+        if vm.uses_default_kernel:
-            print fmt.format ("kernel", "%s (from template)" % vm.kernel)
-        elif vm.uses_default_kernel:
             print fmt.format ("kernel", "%s (default)" % vm.kernel)
         else:
             print fmt.format ("kernel", vm.kernel)
@@ -162,10 +160,6 @@ def set_netvm(vms, vm, args):
     vm.netvm = netvm
 
 def set_kernel(vms, vm, args):
-    if vm.template is not None:
-        print >> sys.stderr, "Cannot set kernel for template-based VM. Set it for template instead."
-        return False
-
     if len (args) != 1:
         print >> sys.stderr, "Missing kernel version argument!"
         print >> sys.stderr, "Possible values:"

dom0/qvm-tools/qvm-start

@@ -38,6 +38,10 @@ def main():
     parser.add_option ("--console", action="store_true", dest="debug_console", default=False,
                       help="Attach debugging console to the newly started VM")
     parser.add_option ("--drive", dest="drive", default=None,
+                      help="Temporarily attach specified drive as CD/DVD or hard disk (can be specified with prefix 'hd:' or 'cdrom:', default is cdrom)")
+    parser.add_option ("--hddisk", dest="drive_hd", default=None,
+                      help="Temporarily attach specified drive as hard disk")
+    parser.add_option ("--cdrom", dest="drive_cdrom", default=None,
                       help="Temporarily attach specified drive as CD/DVD")
     parser.add_option ("--dvm", action="store_true", dest="preparing_dvm", default=False,
                       help="Do actions necessary when preparing DVM image")
@@ -59,6 +63,16 @@ def main():
         print >> sys.stderr, "A VM with the name '{0}' does not exist in the system.".format(vmname)
         exit(1)
 
+    if bool(options.drive_hd) + bool(options.drive_cdrom) + bool(options.drive) > 1:
+        print >> sys.stderr, "Only one of --drive, --cdrom, --hddisk can be specified"
+        exit(1)
+
+    if options.drive_hd:
+        options.drive = 'hd:' + options.drive_hd
+
+    if options.drive_cdrom:
+        options.drive = 'cdrom:' + options.drive_hd
+
     if options.drive:
         if hasattr(vm, 'drive'):
             vm.drive = options.drive

misc/block_add_change

@@ -16,6 +16,11 @@ if [ -n "`ls -A /sys/$DEVPATH/holders 2> /dev/null`" ]; then
     xenstore-rm "$XS_KEY"
     exit 0
 fi
+# ... and "empty" loop devices
+if [ "$MAJOR" -eq 7 -a ! -d /sys/$DEVPATH/loop ]; then
+    xenstore-rm "$XS_KEY"
+    exit 0
+fi
 
 # Special case for CD
 if [ "$ID_TYPE" = "cd" ]; then
@@ -26,6 +31,11 @@ if [ "$ID_TYPE" = "cd" ]; then
     fi
     MODE=r
 fi
+
+# Special description for loop devices
+if [ -d /sys/$DEVPATH/loop ]; then
+    DESC=$(cat /sys/$DEVPATH/loop/backing_file)
+fi
 xenstore-write "$XS_KEY/desc" "$DESC" "$XS_KEY/size" "$SIZE" "$XS_KEY/mode" "$MODE"
 
 # Make sure that block backend is loaded

misc/qubes_block.rules

@@ -6,9 +6,6 @@ SUBSYSTEM!="block", GOTO="qubes_block_end"
 # Skip xen-blkfront devices
 ENV{MAJOR}=="202", GOTO="qubes_block_end"
 
-# Skip loop devices
-ENV{MAJOR}=="7", GOTO="qubes_block_end"
-
 # Skip device-mapper devices
 ENV{MAJOR}=="253", GOTO="qubes_block_end"
 

rpm_spec/core-dom0.spec

@@ -92,6 +92,9 @@ cp qvm-core/__init__.py[co] $RPM_BUILD_ROOT%{python_sitearch}/qubes
 cp qmemman/qmemman*py $RPM_BUILD_ROOT%{python_sitearch}/qubes
 cp qmemman/qmemman*py[co] $RPM_BUILD_ROOT%{python_sitearch}/qubes
 
+mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/qubes
+cp misc/qmemman.conf $RPM_BUILD_ROOT%{_sysconfdir}/qubes/
+
 mkdir -p $RPM_BUILD_ROOT/usr/lib/qubes
 cp aux-tools/patch_appvm_initramfs.sh $RPM_BUILD_ROOT/usr/lib/qubes
 cp aux-tools/unbind_pci_device.sh $RPM_BUILD_ROOT/usr/lib/qubes
@@ -307,6 +310,7 @@ fi
 /etc/init.d/qubes_core
 /etc/init.d/qubes_netvm
 /etc/init.d/qubes_setupdvm
+%config(noreplace) %attr(0664,root,qubes) %{_sysconfdir}/qubes/qmemman.conf
 /usr/bin/qvm-*
 /usr/bin/qubes-*
 /usr/bin/qclipd

vm-systemd/qubes-sysinit.sh

@@ -4,6 +4,7 @@
 DEFAULT_ENABLED_NETVM="network-manager qubes-network"
 DEFAULT_ENABLED_PROXYVM="meminfo-writer qubes-network qubes-firewall qubes-netwatcher"
 DEFAULT_ENABLED_APPVM="meminfo-writer cups"
+DEFAULT_ENABLED_TEMPLATEVM=$DEFAULT_ENABLED_APPVM
 DEFAULT_ENABLED="meminfo-writer"
 
 XS_READ=/usr/bin/xenstore-read
@@ -30,6 +31,7 @@ TYPE=`$XS_READ qubes_vm_type 2> /dev/null`
 [ "$TYPE" == "AppVM" ] && DEFAULT_ENABLED=$DEFAULT_ENABLED_APPVM
 [ "$TYPE" == "NetVM" ] && DEFAULT_ENABLED=$DEFAULT_ENABLED_NETVM
 [ "$TYPE" == "ProxyVM" ] && DEFAULT_ENABLED=$DEFAULT_ENABLED_PROXYVM
+[ "$TYPE" == "TemplateVM" ] && DEFAULT_ENABLED=$DEFAULT_ENABLED_TEMPLATEVM
 
 # Enable default services
 for srv in $DEFAULT_ENABLED; do