Merge remote-tracking branch 'origin/master' into core3-devel

Conflicts:
	core/qubes.py
	doc/Makefile
	doc/manpages/qvm-prefs.rst
	doc/qvm-tools/qvm-add-appvm.rst
	doc/qvm-tools/qvm-backup-restore.rst
	doc/qvm-tools/qvm-backup.rst
	doc/qvm-tools/qvm-block.rst
	doc/qvm-tools/qvm-clone.rst
	doc/qvm-tools/qvm-firewall.rst
	doc/qvm-tools/qvm-ls.rst
	doc/qvm-tools/qvm-pci.rst
	doc/qvm-tools/qvm-run.rst
	doc/qvm-tools/qvm-shutdown.rst
	doc/qvm-tools/qvm-start.rst
	doc/qvm-tools/qvm-template-commit.rst
	qvm-tools/qvm-ls
	qvm-tools/qvm-prefs
	qvm-tools/qvm-remove
	tests/__init__.py
	vm-config/xen-vm-template.xml

core-modules/000QubesVm.py

@@ -312,7 +312,7 @@ class QubesVm(object):
 
         if not self.verify_name(self.name):
             msg = ("'%s' is invalid VM name (invalid characters, over 31 chars long, "
-                   "or one of 'none', 'true', 'false')") % self.name
+                   "ends with '-dm', or one of 'none', 'true', 'false')") % self.name
             if 'xml_element' in kwargs:
                 print >>sys.stderr, "WARNING: %s" % msg
             else:
@@ -569,6 +569,9 @@ class QubesVm(object):
             # avoid conflict when /var/lib/qubes/appvms is mounted on
             # separate partition
             return False
+        if name.endswith('-dm'):
+            # avoid conflict with device model stubdomain names for HVMs
+            return False
         return re.match(r"^[a-zA-Z][a-zA-Z0-9_.-]*$", name) is not None
 
     def pre_rename(self, new_name):
@@ -585,7 +588,7 @@ class QubesVm(object):
             raise QubesException("Cannot change name of running VM!")
 
         if not self.verify_name(name):
-            raise QubesException("Invalid characters in VM name")
+            raise QubesException("Invalid VM name")
 
         if self.installed_by_rpm:
             raise QubesException("Cannot rename VM installed by RPM -- first clone VM and then use yum to remove package.")
@@ -1783,8 +1786,8 @@ class QubesVm(object):
         # Avoid using environment variables for checking the current session,
         #  because this script may be called with cleared env (like with sudo).
         if subprocess.check_output(
-                ['xprop', '-root', '-notype', 'KDE_SESSION_VERSION']) == \
+                ['xprop', '-root', '-notype', 'KWIN_RUNNING']) == \
-                'KDE_SESSION_VERSION = 5\n':
+                'KWIN_RUNNING = 0x1\n':
             # native decoration plugins is used, so adjust window properties
             # accordingly
             guid_cmd += ['-T']  # prefix window titles with VM name
@@ -2012,11 +2015,17 @@ class QubesVm(object):
             # Run GUI daemon in "invisible" mode, so applications started by
             # prerun script will not disturb the user
             extra_guid_args = ['-I']
-        elif not os.path.exists('/var/run/shm.id'):
+        elif not os.path.exists('/var/run/qubes/shm.id') \
+                and not os.path.exists('/var/run/shm.id'):
             # Start GUI daemon only when shmoverride is loaded; unless
             # preparing DispVM, where it isn't needed because of "invisible"
             # mode
             start_guid = False
+        if start_guid and 'DISPLAY' not in os.environ:
+            if verbose:
+                print >> sys.stderr, \
+                    "WARNING: not starting GUI, because DISPLAY not set"
+            start_guid = False
 
         if start_guid:
             self.start_guid(verbose=verbose, notify_function=notify_function,

core-modules/01QubesDisposableVm.py

@@ -40,6 +40,7 @@ except ImportError:
     pass
 
 DISPID_STATE_FILE = '/var/run/qubes/dispid'
+GUID_SHMID_FILE = ['/var/run/qubes/shm.id', '/var/run/shm.id']
 
 class QubesDisposableVm(QubesVm):
     """
@@ -222,7 +223,8 @@ class QubesDisposableVm(QubesVm):
         if qmemman_present:
             qmemman_client.close()
 
-        if kwargs.get('start_guid', True) and os.path.exists('/var/run/shm.id'):
+        if kwargs.get('start_guid', True) and \
+                any(os.path.exists(x) for x in GUID_SHMID_FILE):
             self.start_guid(verbose=verbose, before_qrexec=True,
                     notify_function=kwargs.get('notify_function', None))
 
@@ -230,7 +232,8 @@ class QubesDisposableVm(QubesVm):
                 notify_function=kwargs.get('notify_function', None))
         print >>sys.stderr, "time=%s, qrexec done" % (str(time.time()))
 
-        if kwargs.get('start_guid', True) and os.path.exists('/var/run/shm.id'):
+        if kwargs.get('start_guid', True) and \
+                any(os.path.exists(x) for x in GUID_SHMID_FILE):
             self.start_guid(verbose=verbose,
                     notify_function=kwargs.get('notify_function', None))
         print >>sys.stderr, "time=%s, guid done" % (str(time.time()))

core/qubes.py

@@ -144,6 +144,23 @@ class QubesDaemonPidfile(object):
 
 ### Initialization code
 
+# Globally defined lables
+QubesVmLabels = {
+    "red":      QubesVmLabel(1, "0xcc0000", "red"     ),
+    "orange":   QubesVmLabel(2, "0xf57900", "orange"  ),
+    "yellow":   QubesVmLabel(3, "0xedd400", "yellow"  ),
+    "green":    QubesVmLabel(4, "0x73d216", "green"   ),
+    "gray":     QubesVmLabel(5, "0x555753", "gray"    ),
+    "blue":     QubesVmLabel(6, "0x3465a4", "blue"    ),
+    "purple":   QubesVmLabel(7, "0x75507b", "purple"  ),
+    "black":    QubesVmLabel(8, "0x000000", "black"   ),
+}
+
+QubesDispVmLabels = {
+    k: QubesVmLabel(index=v.index, color=v.color, name=v.name, dispvm=True)
+        for k, v in QubesVmLabels.iteritems()
+}
+
 defaults["appvm_label"] = QubesVmLabels["red"]
 defaults["template_label"] = QubesVmLabels["black"]
 defaults["servicevm_label"] = QubesVmLabels["red"]

core/qubesutils.py

@@ -332,7 +332,9 @@ def block_check_attached(qvmc, device):
                                         "VM '%s'" % (source.get('type'),
                                                      vm.name)
                     continue
-                if backend_name == device['vm'] and path == device['device']:
+                if backend_name == device['vm'] and (path == device['device']
+                        or not path.startswith('/dev/') and path == device[
+                        'desc']):
                     return {
                         "frontend": disk.find('target').get('dev'),
                         "vm": vm}
@@ -384,6 +386,8 @@ def block_attach(qvmc, vm, device, frontend=None, mode="w", auto_detach=False, w
     SubElement(disk, 'target').set('dev', frontend)
     if backend_vm.qid != 0:
         SubElement(disk, 'backenddomain').set('name', device['vm'])
+    if mode == "r":
+        SubElement(disk, 'readonly')
     vm.libvirt_domain.attachDevice(etree.tostring(disk,  encoding='utf-8'))
     try:
         # trigger watches to update device status

doc/qvm-tools/qvm-check.rst

@@ -0,0 +1,32 @@
+=========
+qvm-check
+=========
+
+NAME
+====
+qvm-check - Specify no state options to check if VM exists
+
+:Date:   2013-06-23
+
+SYNOPSIS
+========
+| qvm-check [options] <vm-name>
+
+OPTIONS
+=======
+-h, --help
+    Show this help message and exit
+-q, --quiet
+    Be quiet
+--running
+    Determine if VM is running
+--paused
+    Determine if VM is paused
+--template
+    Determine if VM is a template
+
+AUTHORS
+=======
+| Joanna Rutkowska <joanna at invisiblethingslab dot com>
+| Rafal Wojtczuk <rafal at invisiblethingslab dot com>
+| Marek Marczykowski <marmarek at invisiblethingslab dot com>

doc/qvm-tools/qvm-grow-root.rst

@@ -0,0 +1,26 @@
+=============
+qvm-grow-root
+=============
+
+NAME
+====
+qvm-grow-root - increase root storage capacity of a specified VM
+
+:Date:   2014-03-21
+
+SYNOPSIS
+========
+| qvm-grow-root <vm-name> <size>
+
+OPTIONS
+=======
+-h, --help
+    Show this help message and exit
+--allow-start
+    Allow VM to be started to complete the operation
+
+AUTHORS
+=======
+| Joanna Rutkowska <joanna at invisiblethingslab dot com>
+| Rafal Wojtczuk <rafal at invisiblethingslab dot com>
+| Marek Marczykowski <marmarek at invisiblethingslab dot com>

doc/qvm-tools/qvm-usb.rst

@@ -0,0 +1,36 @@
+=======
+qvm-usb
+=======
+
+NAME
+====
+qvm-usb - List/set VM USB devices
+
+:Date:   2013-03-16
+
+SYNOPSIS
+========
+| qvm-usb -l [options]
+| qvm-usb -a [options] <vm-name> <device-vm-name>:<device>
+| qvm-usb -d [options] <device-vm-name>:<device>
+
+OPTIONS
+=======
+-h, --help
+    Show this help message and exit
+-l, -list
+    List devices
+-a, --attach
+    Attach specified device to specified VM
+-d, --detach
+    Detach specified device
+--no-auto-detach
+    Fail when device already connected to other VM
+--force-root
+    Force to run, even with root privileges
+
+AUTHORS
+=======
+| Joanna Rutkowska <joanna at invisiblethingslab dot com>
+| Rafal Wojtczuk <rafal at invisiblethingslab dot com>
+| Marek Marczykowski <marmarek at invisiblethingslab dot com>

qubes/qmemman/algo.py

@@ -111,7 +111,7 @@ def refresh_meminfo_for_domain(domain, untrusted_xenstore_key):
 def prefmem(domain):
 #dom0 is special, as it must have large cache, for vbds. Thus, give it a special boost
     if domain.id == '0':
-        return min(domain.mem_used*CACHE_FACTOR + 350*1024*1024, domain.memory_maximum)
+        return min(domain.mem_used*CACHE_FACTOR + DOM0_MEM_BOOST, domain.memory_maximum)
     return max(min(domain.mem_used*CACHE_FACTOR, domain.memory_maximum), MIN_PREFMEM)
 
 def memory_needed(domain):

qubes/tests/__init__.py

@@ -829,7 +829,7 @@ class SystemTestsMixin(object):
                     "menuentry 'Default' {\n"
                     "  linux /vmlinuz root=/dev/xvda1 "
                     "rd.driver.blacklist=bochs_drm "
-                    "rd.driver.blacklist=uhci_hcd\n"
+                    "rd.driver.blacklist=uhci_hcd console=hvc0\n"
                     "  initrd /initrd\n"
                     "}"
                 )

qubes/tests/int/vm_qrexec_gui.py

@@ -1004,6 +1004,44 @@ class TC_00_AppVMMixin(qubes.tests.SystemTestsMixin):
         if vm_image != dom0_image:
             self.fail("Dom0 window doesn't match VM window content")
 
+class TC_10_Generic(qubes.tests.SystemTestsMixin, qubes.tests.QubesTestCase):
+    def setUp(self):
+        super(TC_10_Generic, self).setUp()
+        self.vm = self.qc.add_new_vm(
+            "QubesAppVm",
+            name=self.make_vm_name('vm'),
+            template=self.qc.get_default_template())
+        self.vm.create_on_disk(verbose=False)
+        self.save_and_reload_db()
+        self.qc.unlock_db()
+        self.vm = self.qc[self.vm.qid]
+
+    def test_000_anyvm_deny_dom0(self):
+        '''$anyvm in policy should not match dom0'''
+        policy = open("/etc/qubes-rpc/policy/test.AnyvmDeny", "w")
+        policy.write("%s $anyvm allow" % (self.vm.name,))
+        policy.close()
+        self.addCleanup(os.unlink, "/etc/qubes-rpc/policy/test.AnyvmDeny")
+
+        flagfile = '/tmp/test-anyvmdeny-flag'
+        if os.path.exists(flagfile):
+            os.remove(flagfile)
+        with open('/etc/qubes-rpc/test.AnyvmDeny', 'w') as f:
+            f.write('touch {}\n'.format(flagfile))
+            f.write('echo service output\n')
+        self.addCleanup(os.unlink, "/etc/qubes-rpc/test.AnyvmDeny")
+
+        self.vm.start(verbose=False)
+        p = self.vm.run("/usr/lib/qubes/qrexec-client-vm dom0 test.AnyvmDeny",
+                             passio_popen=True, passio_stderr=True)
+        (stdout, stderr) = p.communicate()
+        self.assertEqual(p.returncode, 1,
+            '$anyvm matched dom0, qrexec-client-vm output: {}'.
+                format(stdout + stderr))
+        self.assertFalse(os.path.exists(flagfile),
+            'Flag file created (service was run) even though should be denied,'
+            ' qrexec-client-vm output: {}'.format(stdout + stderr))
+
 
 def load_tests(loader, tests, pattern):
     try:

qvm-tools/qubes-hcl-report

@@ -22,6 +22,8 @@ VERSION=2.5
 COPY2VM="dom0"
 SUPPORT_FILES=0
 
+XL_DMESG_PREFIX_REGEX='^(XEN) \(\[[^]]*\] \)\?'
+
 while [ $# -gt 0 ]; do
   case "$1" in
 
@@ -76,6 +78,13 @@ sudo dmidecode > $TEMP_DIR/dmidecode
 xl info > $TEMP_DIR/xl-info
 xl dmesg > $TEMP_DIR/xl-dmesg
 
+if cat $TEMP_DIR/xl-dmesg | grep "$XL_DMESG_PREFIX_REGEX"'Xen version ' > /dev/null; then
+    XL_DMESG_INCOMPLETE=no
+else
+    XL_DMESG_INCOMPLETE=yes
+    echo -e 'WARNING: "xl dmesg" is incomplete. Some information are missing. Please reboot and try again.\n'
+fi
+
 
 BRAND=`cat $TEMP_DIR/dmidecode |grep -A9 "System Information" |grep "Manufacturer:" |cut -d ' ' -f2-`
 PRODUCT=`cat $TEMP_DIR/dmidecode |grep -A9 "System Information" |grep "Product Name:" |cut -d ' ' -f3-`
@@ -101,6 +110,7 @@ XEN_EXTRA=`cat $TEMP_DIR/xl-info |grep xen_extra |cut -d: -f2 |tr -d ' '`
 QUBES=`cat $TEMP_DIR/qubes-release |cut -d '(' -f2 |cut -d ')' -f1`
 XL_VTX=`cat $TEMP_DIR/xl-info |grep xen_caps | grep hvm`
 XL_VTD=`cat $TEMP_DIR/xl-info |grep virt_caps |grep hvm_directio`
+XL_HAP=`cat $TEMP_DIR/xl-dmesg |grep "$XL_DMESG_PREFIX_REGEX"'HVM: Hardware Assisted Paging (HAP) detected\( but disabled\)\?$'`
 PCRS=`find /sys/devices/ -name pcrs`
 
 FILENAME="Qubes-HCL-${BRAND//[^[:alnum:]]/_}-${PRODUCT//[^[:alnum:]]/_}-$DATE"
@@ -127,6 +137,20 @@ if [[ $XL_VTD ]]
 
 fi
 
+if [ $XL_DMESG_INCOMPLETE = yes ]; then
+    HAP=""
+    HAP_VERBOSE='Unknown ("xl dmesg" incomplete)'
+elif [ -n "$XL_HAP" ]; then
+    HAP="yes"
+    HAP_VERBOSE="Yes"
+    if [[ "$XL_HAP" =~ "disabled" ]]; then
+        HAP_VERBOSE="Yes (disabled)"
+    fi
+else
+    HAP="no"
+    HAP_VERBOSE="No"
+fi
+
 if [[ $PCRS ]]
   then
     # try tu run tcsd and: grep the logs, try get version info.
@@ -152,6 +176,7 @@ echo -e "Net:\n$NET\n"
 echo -e "SCSI:\n$SCSI\n"
 echo -e "HVM:\t\t$VTX"
 echo -e "I/O MMU:\t$VTD"
+echo -e "HAP/SLAT:\t$HAP_VERBOSE"
 echo -e "TPM:\t\t$TPM"
 echo
 
@@ -164,6 +189,8 @@ hvm:
   '$HVM'
 iommu:
   '$IOMMU'
+slat:
+  '$HAP'
 tpm:
   '$TPM_s'
 brand: |

qvm-tools/qvm-backup-restore

@@ -284,7 +284,7 @@ def main():
                 exit(1)
             else:
                 print >> sys.stderr, "Continuing as directed."
-        print >> sys.stderr, "NOTE: Before restoring the dom0 home directory, "
+        print >> sys.stderr, "NOTE: Before restoring the dom0 home directory, "\
                              "a new directory named "\
                              "'home-pre-restore-<current-time>' will be "\
                              "created inside the dom0 home directory. If any "\

qvm-tools/qvm-block

@@ -87,6 +87,7 @@ def main():
         if options.do_file_attach:
             dev = {}
             (dev['vm'], dev['device']) = args[1].split(":")
+            dev['desc'] = dev['device']
             dev['mode'] = 'w'
         else:
             dev_list = block_list(qvm_collection)

qvm-tools/qvm-check

@@ -52,30 +52,30 @@ Specify no state options to check if VM exists"""
     vm = qvm_collection.get_vm_by_name(vmname)
     if vm is None:
         if options.verbose:
-            print >> sys.stdout, "A VM with the name '{0}' does not exist in the system!".format(vmname)
+            print >> sys.stderr, "A VM with the name '{0}' does not exist in the system!".format(vmname)
         exit(1)
 
     elif options.running:
         vm_state = not vm.is_running()
         if options.verbose:
-            print >> sys.stdout, "A VM with the name {0} is {1}running.".format(vmname, "not " * vm_state)
+            print >> sys.stderr, "A VM with the name {0} is {1}running.".format(vmname, "not " * vm_state)
         exit(vm_state)
 
     elif options.paused:
         vm_state = not vm.is_paused()
         if options.verbose:
-            print >> sys.stdout, "A VM with the name {0} is {1}paused.".format(vmname, "not " * vm_state)
+            print >> sys.stderr, "A VM with the name {0} is {1}paused.".format(vmname, "not " * vm_state)
         exit(vm_state)
 
     elif options.template:
         vm_state = not vm.is_template()
         if options.verbose:
-            print >> sys.stdout, "A VM with the name {0} is {1}a template.".format(vmname, "not " * vm_state)
+            print >> sys.stderr, "A VM with the name {0} is {1}a template.".format(vmname, "not " * vm_state)
         exit(vm_state)
 
     else:
         if options.verbose:
-            print >> sys.stdout, "A VM with the name '{0}' does exist.".format(vmname)
+            print >> sys.stderr, "A VM with the name '{0}' does exist.".format(vmname)
         exit(0)
 
 main()

qvm-tools/qvm-create-default-dvm

@@ -1,5 +1,5 @@
 #!/bin/sh
-if [ $# != 1 -a $# != 2 ] ; then
+if [ $# != 1 -a $# != 2 -o $1 == "--help" -o $1 == "-h" ] ; then
 	echo 'Usage: qvm-create-default-dvm templatename|--default-template|--used-template [script-name|--default-script]'
 	exit 1
 fi

templates/libvirt/xen.xml

@@ -18,7 +18,7 @@
                 <type arch="x86_64" machine="xenpv">linux</type>
                 <kernel>{{ vm.storage.kernels_dir }}/vmlinuz</kernel>
                 <initrd>{{ vm.storage.kernels_dir }}/initramfs</initrd>
-                <cmdline>root=/dev/mapper/dmroot ro nomodeset console=hvc0 rd_NO_PLYMOUTH 3 {{ vm.kernelopts }}</cmdline>
+                <cmdline>root=/dev/mapper/dmroot ro nomodeset console=hvc0 rd_NO_PLYMOUTH rd.plymouth.enable=0 plymouth.enable=0 {{ vm.kernelopts }}</cmdline>
             {% endif %}
         {% endblock %}
     </os>

version

@@ -1 +1 @@
-3.2.8
+3.2.12