Merge remote-tracking branch 'origin/pull/88/head' into core3-devel

.travis.yml

@@ -7,8 +7,6 @@ python:
 install:
   - pip install --quiet -r ci/requirements.txt
   - git clone https://github.com/"${TRAVIS_REPO_SLUG%%/*}"/qubes-builder ~/qubes-builder
-# debootstrap in trusty is old...
-before_script: sudo ln -s sid /usr/share/debootstrap/scripts/stretch
 script:
   - PYTHONPATH=test-packages pylint --rcfile=ci/pylintrc qubes
   - ./run-tests --no-syslog

linux/system-config/block-snapshot

@@ -53,14 +53,21 @@ get_dev() {
 }
 
 get_dm_snapshot_name() {
+  local base cow cow2
+
   base=$1
   cow=$2
+  cow2=$3
 
-  echo snapshot-$(stat -c '%D:%i' "$base")-$(stat -c '%D:%i' "$cow")
+  name="snapshot-$(stat -c '%D:%i' "$base")-$(stat -c '%D:%i' "$cow")"
+  if [ -n "$cow2" ]; then
+    name="$name-$(stat -c '%D:%i' "$cow2")"
+  fi
+  echo "$name"
 }
 
 create_dm_snapshot() {
-  local base_dev cow_dev base_sz
+  local base_dev cow_dev base_sz base cow dm_devname
 
   dm_devname=$1
   base=$2
@@ -77,7 +84,7 @@ create_dm_snapshot() {
 }
 
 create_dm_snapshot_origin() {
-  local base_dev base_sz
+  local base_dev base_sz dm_devname base
 
   dm_devname=$1
   base=$2
@@ -103,8 +110,14 @@ case "$command" in
         fi
         echo $p > "$HOTPLUG_STORE-params"
         echo $t > "$HOTPLUG_STORE-type"
-        base=${p/:*/}
+        base=${p%%:*}
-        cow=${p/*:/}
+        cow=${p#*:}
+        cow2=${p##*:}
+        if [ "$cow" != "$cow2" ]; then
+          cow=${cow%:*}
+        else
+          cow2=""
+        fi
 
         if [ -L "$base" ]; then
           base=$(readlink -f "$base") || fatal "$base link does not exist."
@@ -114,6 +127,10 @@ case "$command" in
           cow=$(readlink -f "$cow") || fatal "$cow link does not exist."
         fi
 
+        if [ -L "$cow2" ]; then
+          cow2=$(readlink -f "$cow2") || fatal "$cow2 link does not exist."
+        fi
+
         # first ensure that snapshot device exists (to write somewhere changes from snapshot-origin)
         dm_devname=$(get_dm_snapshot_name "$base" "$cow")
 
@@ -122,6 +139,12 @@ case "$command" in
         # prepare snapshot device
         create_dm_snapshot $dm_devname "$base" "$cow"
 
+        if [ -n "$cow2" ]; then
+          dm_devname_full=$(get_dm_snapshot_name "$base" "$cow" "$cow2")
+          create_dm_snapshot "$dm_devname_full" "/dev/mapper/$dm_devname" "$cow2"
+          dm_devname="$dm_devname_full"
+        fi
+
         if [ "$t" == "snapshot" ]; then
           #that's all for snapshot, store name of prepared device
           xenstore_write "$XENBUS_PATH/node" "/dev/mapper/$dm_devname"
@@ -152,8 +175,14 @@ case "$command" in
     case $t in
       snapshot|origin)
         p=$3
-        base=${p/:*/}
+        base=${p%%:*}
-        cow=${p/*:/}
+        cow=${p#*:}
+        cow2=${p##*:}
+        if [ "$cow" != "$cow2" ]; then
+          cow=${cow%:*}
+        else
+          cow2=""
+        fi
 
         if [ -L "$base" ]; then
           base=$(readlink -f "$base") || fatal "$base link does not exist."
@@ -163,6 +192,10 @@ case "$command" in
           cow=$(readlink -f "$cow") || fatal "$cow link does not exist."
         fi
 
+        if [ -L "$cow2" ]; then
+          cow2=$(readlink -f "$cow2") || fatal "$cow2 link does not exist."
+        fi
+
         # first ensure that snapshot device exists (to write somewhere changes from snapshot-origin)
         dm_devname=$(get_dm_snapshot_name "$base" "$cow")
 
@@ -171,6 +204,12 @@ case "$command" in
         # prepare snapshot device
         create_dm_snapshot $dm_devname "$base" "$cow"
 
+        if [ -n "$cow2" ]; then
+          dm_devname_full=$(get_dm_snapshot_name "$base" "$cow" "$cow2")
+          create_dm_snapshot "$dm_devname_full" "/dev/mapper/$dm_devname" "$cow2"
+          dm_devname="$dm_devname_full"
+        fi
+
         if [ "$t" == "snapshot" ]; then
           #that's all for snapshot, store name of prepared device
           echo "/dev/mapper/$dm_devname"
@@ -232,7 +271,7 @@ case "$command" in
         fi
 
         # get list of used (loop) devices
-        deps="$(dmsetup deps $node | cut -d: -f2 | sed -e 's#(7, \([0-9]\+\))#/dev/loop\1#g')"
+        deps="$(dmsetup deps $node -o blkdevname | cut -d: -f2 | sed -e 's#(\([a-z0-9-]\+\))#/dev/\1#g')"
 
         # if this is origin
         if [ "${node/origin/}" != "$node" ]; then
@@ -241,7 +280,8 @@ case "$command" in
             use_count=$(dmsetup info $snap|grep Open|awk '{print $3}')
             if [ "$use_count" -eq 0 ]; then
               # unused snapshot - remove it
-              deps="$deps $(dmsetup deps $snap | cut -d: -f2 | sed -e 's#(7, \([0-9]\+\))#/dev/loop\1#g')"
+              deps="$deps $(dmsetup deps $snap -o blkdevname | cut -d: -f2 |\
+                   sed -e 's#(\([a-z0-9-]\+\))#/dev/\1#g')"
               log debug "Removing $snap"
               dmsetup remove $snap
             fi
@@ -265,11 +305,18 @@ case "$command" in
           dmsetup remove $node
         fi
 
-        # try to free loop devices
+        # try to free unused devices
         for dev in $deps; do
           if [ -b "$dev" ]; then
             log debug "Removing $dev"
-            losetup -d $dev 2> /dev/null || true
+            case $dev in
+              /dev/loop*)
+                losetup -d $dev 2> /dev/null || true
+                ;;
+              /dev/dm-*)
+                dmsetup remove $dev 2> /dev/null || true
+                ;;
+            esac
           fi
         done
 

qubes/ext/gui.py

@@ -135,8 +135,8 @@ class GUI(qubes.ext.Extension):
         GUI daemon securely displays windows from domain.
         ''' # pylint: disable=no-self-use,unused-argument
 
-        if not start_guid or preparing_dvm \
+
-                or not os.path.exists('/var/run/shm.id'):
+        if not start_guid or preparing_dvm:
             return
 
         if self.is_guid_running(vm):
@@ -150,6 +150,18 @@ class GUI(qubes.ext.Extension):
             vm.log.error('Not starting gui daemon, no DISPLAY set')
             return
 
+        display = os.getenv('DISPLAY')
+        if not display.startswith(':'):
+            vm.log.error('Expected local $DISPLAY, got \'{}\''.format(display))
+            return
+
+        display_num = display[1:].partition('.')[0]
+        shmid_path = '/var/run/qubes/shm.id.{}'.format(display_num)
+        if not os.path.exists(shmid_path):
+            vm.log.error(
+                'Not starting gui daemon, no {} file'.format(shmid_path))
+            return
+
         vm.log.info('Starting gui daemon')
 
         guid_cmd = [qubes.config.system_path['qubes_guid_path'],

qubes/storage/file.py

@@ -235,21 +235,34 @@ class FilePool(qubes.storage.Pool):
         create_dir_if_not_exists(vm_templates_path)
 
     def start(self, volume):
-        if volume._is_snapshot or volume._is_origin:
+        if volume._is_volatile:
-            _check_path(volume.path)
-            try:
-                _check_path(volume.path_cow)
-            except qubes.storage.StoragePoolException:
-                create_sparse_file(volume.path_cow, volume.size)
-                _check_path(volume.path_cow)
-        elif volume._is_volatile:
             self.reset(volume)
+        else:
+            _check_path(volume.path)
+            if volume.snap_on_start:
+                if not volume.save_on_stop:
+                    # make sure previous snapshot is removed - even if VM
+                    # shutdown routing wasn't called (power interrupt or so)
+                    _remove_if_exists(volume.path_cow)
+                try:
+                    _check_path(volume.path_cow)
+                except qubes.storage.StoragePoolException:
+                    create_sparse_file(volume.path_cow, volume.size)
+                    _check_path(volume.path_cow)
+                if hasattr(volume, 'path_source_cow'):
+                    try:
+                        _check_path(volume.path_source_cow)
+                    except qubes.storage.StoragePoolException:
+                        create_sparse_file(volume.path_source_cow, volume.size)
+                        _check_path(volume.path_source_cow)
         return volume
 
     def stop(self, volume):
         if volume.save_on_stop:
             self.commit(volume)
-        elif volume._is_volatile:
+        elif volume.snap_on_start:
+            _remove_if_exists(volume.path_cow)
+        else:
             _remove_if_exists(volume.path)
         return volume
 
@@ -316,6 +329,8 @@ class FileVolume(qubes.storage.Volume):
         if self._is_snapshot:
             self.path = os.path.join(self.dir_path, self.source + '.img')
             img_name = self.source + '-cow.img'
+            self.path_source_cow = os.path.join(self.dir_path, img_name)
+            img_name = self.vid + '-cow.img'
             self.path_cow = os.path.join(self.dir_path, img_name)
         elif self._is_volume or self._is_volatile:
             self.path = os.path.join(self.dir_path, self.vid + '.img')
@@ -347,6 +362,8 @@ class FileVolume(qubes.storage.Volume):
             the libvirt XML template as <disk>.
         '''
         path = self.path
+        if self._is_snapshot:
+            path += ":" + self.path_source_cow
         if self._is_origin or self._is_snapshot:
             path += ":" + self.path_cow
         return qubes.devices.BlockDevice(path, self.name, self.script, self.rw,
@@ -380,7 +397,7 @@ class FileVolume(qubes.storage.Volume):
     def _is_origin(self):
         ''' Internal helper. Useful for differentiating volume handling '''
         # pylint: disable=line-too-long
-        return not self.snap_on_start and self.save_on_stop and self.revisions_to_keep > 0  # NOQA
+        return self.save_on_stop and self.revisions_to_keep > 0  # NOQA
 
     @property
     def _is_snapshot(self):

qubes/tests/__init__.py

@@ -507,10 +507,20 @@ class SystemTestsMixin(object):
         elif isinstance(template, str):
             template = self.host_app.domains[template]
 
+        used_pools = [vol.pool for vol in template.volumes.values()]
+
+        for pool in used_pools:
+            if pool in self.app.pools:
+                continue
+            self.app.add_pool(**self.host_app.pools[pool].config)
+
         template_vm = self.app.add_new_vm(qubes.vm.templatevm.TemplateVM,
             name=template.name,
             uuid=template.uuid,
             label='black')
+        for name, volume in template_vm.volumes.items():
+            if volume.pool != template.volumes[name].pool:
+                template_vm.storage.init_volume(name, volume.config)
         self.app.default_template = template_vm
 
     def init_networking(self):
@@ -908,6 +918,7 @@ def load_tests(loader, tests, pattern): # pylint: disable=unused-argument
             # tool tests
             'qubes.tests.integ.tools.qubes_create',
             'qubes.tests.integ.tools.qvm_check',
+            'qubes.tests.integ.tools.qvm_features',
             'qubes.tests.integ.tools.qvm_firewall',
             'qubes.tests.integ.tools.qvm_prefs',
             'qubes.tests.integ.tools.qvm_run',

qubes/tests/integ/storage.py

@@ -211,6 +211,52 @@ class StorageTestMixin(qubes.tests.SystemTestsMixin):
         self.assertNotEqual(p.returncode, 0,
             'origin changes not visible in snapshot: {}'.format(stdout))
 
+    def test_004_snapshot_non_persistent(self):
+        '''Test snapshot volume non-persistence'''
+        size = 128 * 1024 * 1024
+        volume_config = {
+            'pool': self.pool.name,
+            'size': size,
+            'internal': False,
+            'save_on_stop': True,
+            'rw': True,
+        }
+        testvol = self.vm1.storage.init_volume('testvol', volume_config)
+        self.vm1.storage.get_pool(testvol).create(testvol)
+        volume_config = {
+            'pool': self.pool.name,
+            'size': size,
+            'internal': False,
+            'snap_on_start': True,
+            'source': testvol.vid,
+            'rw': True,
+        }
+        testvol_snap = self.vm2.storage.init_volume('testvol', volume_config)
+        self.vm2.storage.get_pool(testvol_snap).create(testvol_snap)
+        self.app.save()
+        self.vm2.start()
+
+        p = self.vm2.run(
+            'head -c {} /dev/zero | diff -q /dev/xvde -'.format(size),
+            user='root', passio_popen=True)
+        stdout, _ = p.communicate()
+        self.assertEqual(p.returncode, 0,
+            'snapshot image not clean: {}'.format(stdout))
+
+        self.vm2.run('echo test123 > /dev/xvde && sync', user='root', wait=True)
+        p.wait()
+        self.assertEqual(p.returncode, 0,
+            'Write to read-write snapshot volume failed')
+        self.vm2.shutdown(wait=True)
+        self.vm2.start()
+        p = self.vm2.run(
+            'head -c {} /dev/zero 2>&1 | diff -q /dev/xvde - 2>&1'.format(size),
+            user='root', passio_popen=True)
+        stdout, _ = p.communicate()
+        self.assertEqual(p.returncode, 0,
+            'changes on snapshot survived VM restart: {}'.format(
+                stdout))
+
 
 class StorageFile(StorageTestMixin, qubes.tests.QubesTestCase):
     def init_pool(self):

qubes/tests/integ/tools/qvm_features.py

@@ -0,0 +1,70 @@
+#!/usr/bin/python2
+# -*- encoding: utf8 -*-
+#
+# The Qubes OS Project, http://www.qubes-os.org
+#
+# Copyright (C) 2017 Marek Marczykowski-Górecki
+#                               <marmarek@invisiblethingslab.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+import qubes
+import qubes.tools.qvm_features
+
+import qubes.tests
+import qubes.tests.tools
+import qubes.vm.appvm
+
+
+class TC_00_qvm_features(qubes.tests.SystemTestsMixin,
+        qubes.tests.QubesTestCase):
+    def setUp(self):
+        super(TC_00_qvm_features, self).setUp()
+        self.init_default_template()
+
+        self.sharedopts = ['--qubesxml', qubes.tests.XMLPATH]
+
+        self.vm1 = self.app.add_new_vm(qubes.vm.appvm.AppVM,
+            name=self.make_vm_name('vm1'),
+            template=self.app.default_template,
+            label='red')
+        self.app.save()
+
+    def test_000_list(self):
+        self.assertEqual(0, qubes.tools.qvm_features.main(
+            self.sharedopts + [self.vm1.name]))
+
+        with self.assertRaises(SystemExit):
+            qubes.tools.qvm_features.main(
+                self.sharedopts + ['test-no-such-vm'])
+
+    def test_001_get_missing(self):
+        self.assertEqual(1, qubes.tools.qvm_features.main(
+            self.sharedopts + [self.vm1.name, 'no-such-feature']))
+
+    def test_002_set_and_get(self):
+        self.assertEqual(0, qubes.tools.qvm_features.main(
+            self.sharedopts + [self.vm1.name, 'test-feature', 'true']))
+        with qubes.tests.tools.StdoutBuffer() as buf:
+            self.assertEqual(0, qubes.tools.qvm_features.main(
+                self.sharedopts + [self.vm1.name, 'test-feature']))
+            self.assertEqual('true\n', buf.getvalue())
+
+    def test_003_set_and_list(self):
+        self.assertEqual(0, qubes.tools.qvm_features.main(
+            self.sharedopts + [self.vm1.name, 'test-feature', 'true']))
+        with qubes.tests.tools.StdoutBuffer() as buf:
+            self.assertEqual(0, qubes.tools.qvm_features.main(
+                self.sharedopts + [self.vm1.name]))
+            self.assertEqual('test-feature  true\n', buf.getvalue())

qubes/tests/storage_file.py

@@ -216,7 +216,7 @@ class TC_01_FileVolumes(qubes.tests.QubesTestCase):
                                  label='red')
 
         expected = vm.template.dir_path + '/root.img:' + vm.template.dir_path \
-            + '/root-cow.img'
+            + '/root-cow.img:' + vm.dir_path + '/root-cow.img'
         self.assertVolumePath(vm, 'root', expected, rw=False)
         expected = vm.dir_path + '/private.img:' + \
             vm.dir_path + '/private-cow.img'

qubes/tools/qvm_features.py

@@ -58,43 +58,43 @@ def main(args=None):
     '''
 
     args = parser.parse_args(args)
+    vm = args.domains[0]
 
     if args.request:
         # Request mode: instead of setting the features directly,
         # let the extensions handle them first.
-        args.vm.fire_event('feature-request', untrusted_features=args.features)
+        vm.fire_event('feature-request', untrusted_features=args.features)
-        return 0
 
-    if args.feature is None:
+    elif args.feature is None:
         if args.delete:
             parser.error('--unset requires a feature')
 
-        width = max(len(feature) for feature in args.vm.features)
+        # max doesn't like empty list
-        for feature in sorted(args.vm.features):
+        if vm.features:
-            print('{name:{width}s}  {value}'.format(
+            width = max(len(feature) for feature in vm.features)
-                name=feature, value=args.vm.features[feature], width=width))
+            for feature in sorted(vm.features):
+                print('{name:{width}s}  {value}'.format(
+                    name=feature, value=vm.features[feature], width=width))
 
-        return 0
+    elif args.delete:
-
-    if args.delete:
         if args.value is not None:
             parser.error('cannot both set and unset a value')
         try:
-            del args.vm.features[args.feature]
+            del vm.features[args.feature]
             args.app.save()
         except KeyError:
             pass
-        return 0
 
-    if args.value is None:
+    elif args.value is None:
         try:
-            print(args.vm.features[args.feature])
+            print(vm.features[args.feature])
             return 0
         except KeyError:
             return 1
+    else:
+        vm.features[args.feature] = args.value
+        args.app.save()
 
-    args.vm.features[args.feature] = args.value
-    args.app.save()
     return 0
 
 

qubes/vm/qubesvm.py

@@ -821,8 +821,12 @@ class QubesVM(qubes.vm.mix.net.NetVMMixin, qubes.vm.BaseVM):
         self._update_libvirt_domain()
 
         qmemman_client = self.request_memory(mem_required)
-
+        try:
-        self.libvirt_domain.createWithFlags(libvirt.VIR_DOMAIN_START_PAUSED)
+            self.libvirt_domain.createWithFlags(libvirt.VIR_DOMAIN_START_PAUSED)
+        except:
+            if qmemman_client:
+                qmemman_client.close()
+            raise
 
         try:
             self.fire_event('domain-spawn',

rpm_spec/core-dom0.spec

@@ -360,6 +360,7 @@ fi
 %{python3_sitelib}/qubes/tests/integ/tools/__pycache__/*
 %{python3_sitelib}/qubes/tests/integ/tools/__init__.py
 %{python3_sitelib}/qubes/tests/integ/tools/qubes_create.py
+%{python3_sitelib}/qubes/tests/integ/tools/qvm_features.py*
 %{python3_sitelib}/qubes/tests/integ/tools/qvm_firewall.py
 %{python3_sitelib}/qubes/tests/integ/tools/qvm_check.py
 %{python3_sitelib}/qubes/tests/integ/tools/qvm_prefs.py

templates/libvirt/xen.xml

@@ -115,9 +115,9 @@
                     type="stubdom"
                     {% if vm.netvm %}
                         cmdline="-net lwip,client_ip={{ vm.ip -}}
-                            ,server_ip={{ vm.secondary_dns -}}
+                            ,server_ip={{ vm.dns[1] -}}
                             ,dns={{ vm.netvm.gateway -}}
-                            ,gw={{ self.netvm.gateway -}}
+                            ,gw={{ vm.netvm.gateway -}}
                             ,netmask={{ vm.netmask }}"
                     {% endif %}
                     />