Implement new admin.vm.ImportWithSize API call

This should allow importing a volume and changing the size at the
same time, without performing the resize operation on original
volume first.

The internal API has been renamed to internal.vm.volume.ImportBegin
to avoid confusion, and for symmetry with ImportEnd.

See QubesOS/qubes-issues#5239.

Makefile

@@ -191,6 +191,8 @@ endif
 	cp qubes-rpc-policy/qubes.GetDate.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.GetDate
 	cp qubes-rpc-policy/qubes.ConnectTCP.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.ConnectTCP
 	cp qubes-rpc-policy/admin.vm.Console.policy $(DESTDIR)/etc/qubes-rpc/policy/admin.vm.Console
+	cp qubes-rpc-policy/admin.vm.volume.Import.policy $(DESTDIR)/etc/qubes-rpc/policy/admin.vm.volume.Import
+	cp qubes-rpc-policy/admin.vm.volume.ImportWithSize.policy $(DESTDIR)/etc/qubes-rpc/policy/admin.vm.volume.ImportWithSize
 	cp qubes-rpc-policy/policy.RegisterArgument.policy $(DESTDIR)/etc/qubes-rpc/policy/policy.RegisterArgument
 	cp qubes-rpc/qubes.FeaturesRequest $(DESTDIR)/etc/qubes-rpc/
 	cp qubes-rpc/qubes.GetDate $(DESTDIR)/etc/qubes-rpc/
@@ -208,6 +210,7 @@ endif
 			$(DESTDIR)/etc/qubes-rpc/$$method || exit 1; \
 	done
 	install qubes-rpc/admin.vm.volume.Import $(DESTDIR)/etc/qubes-rpc/
+	ln -s admin.vm.volume.Import $(DESTDIR)/etc/qubes-rpc/admin.vm.volume.ImportWithSize
 	install qubes-rpc/admin.vm.Console $(DESTDIR)/etc/qubes-rpc/
 	PYTHONPATH=.:test-packages qubes-rpc-policy/generate-admin-policy \
 		--destdir=$(DESTDIR)/etc/qubes-rpc/policy \
@@ -260,4 +263,3 @@ msi:
 	candle -arch x64 -dversion=$(VERSION) installer.wxs
 	light -b destinstdir -o core-admin.msm installer.wixobj
 	rm -rf destinstdir
-

qubes-rpc-policy/admin.vm.volume.Import.policy

@@ -0,0 +1,13 @@
+## Note that policy parsing stops at the first match.
+## Anything not specifically allowed here (or in included file) will be denied.
+
+## Please use a single # to start your custom comments
+
+## Add your entries here, make sure to append ",target=dom0" to all allow/ask actions
+
+## Include a common file for all admin.* methods to ease setting up
+## Management VM.
+## To allow only specific actions, edit specific policy file, like this one. To
+## allow all of them, edit appropriate /etc/qubes-rpc/include/admin-*.
+
+$include:include/admin-local-rwx

qubes-rpc-policy/admin.vm.volume.ImportWithSize.policy

@@ -0,0 +1,13 @@
+## Note that policy parsing stops at the first match.
+## Anything not specifically allowed here (or in included file) will be denied.
+
+## Please use a single # to start your custom comments
+
+## Add your entries here, make sure to append ",target=dom0" to all allow/ask actions
+
+## Include a common file for all admin.* methods to ease setting up
+## Management VM.
+## To allow only specific actions, edit specific policy file, like this one. To
+## allow all of them, edit appropriate /etc/qubes-rpc/include/admin-*.
+
+$include:include/admin-local-rwx

qubes-rpc/admin.vm.volume.Import

@@ -16,23 +16,36 @@
 #    2. Actual data import (done by this script, using dd)
 #    3. Report final result, produce final response to the caller (done by
 #       qubesd)
-#    
+#
 #    This way we do not pass all the data through qubesd, but still can
 #    control the process from there in a meaningful way. Note that the last
 #    part (second call to qubesd) may perform all kind of verification (like
 #    a signature check on the data, or so) and can also prevent VM from
 #    starting (hooking also domain-pre-start event) from not verified image.
+#
+# Note that this script implements two calls:
+#   - admin.vm.volume.Import
+#   - admin.vm.volume.ImportWithSize
+# In the case of admin.vm.ImportWithSize, the first line of payload is then
+# data size in bytes. This is so that we can already notify qubesd to create a
+# volume with the new size.
 
 set -e
 
 # use temporary file, because env variables deal poorly with \0 inside
 tmpfile=$(mktemp)
 trap "rm -f $tmpfile" EXIT
-qubesd-query -e \
+
-        "$QREXEC_REMOTE_DOMAIN" \
+requested_size=""
-        "admin.vm.volume.Import" \
+if [[ ${0##*/} == admin.vm.volume.ImportWithSize ]]; then
-        "$QREXEC_REQUESTED_TARGET" \
+    read requested_size
-        "$1" >$tmpfile
+fi
+
+echo -n "$requested_size" | qubesd-query -c /var/run/qubesd.internal.sock \
+    "$QREXEC_REMOTE_DOMAIN" \
+    "internal.vm.volume.ImportBegin" \
+    "$QREXEC_REQUESTED_TARGET" \
+    "$1" >$tmpfile
 
 # exit if qubesd returned an error (not '0\0')
 if [ "$(head -c 2 $tmpfile | xxd -p)" != "3000" ]; then

qubes/api/admin.py

@@ -486,36 +486,6 @@ class QubesAdminAPI(qubes.api.AbstractQubesAPI):
         finally:  # even if calling qubes.ResizeDisk inside the VM failed
             self.app.save()
 
-    @qubes.api.method('admin.vm.volume.Import', no_payload=True,
-        scope='local', write=True)
-    @asyncio.coroutine
-    def vm_volume_import(self):
-        """Import volume data.
-
-        Note that this function only returns a path to where data should be
-        written, actual importing is done by a script in /etc/qubes-rpc
-        When the script finish importing, it will trigger
-        internal.vm.volume.ImportEnd (with either b'ok' or b'fail' as a
-        payload) and response from that call will be actually send to the
-        caller.
-        """
-        self.enforce(self.arg in self.dest.volumes.keys())
-
-        self.fire_event_for_permission()
-
-        if not self.dest.is_halted():
-            raise qubes.exc.QubesVMNotHaltedError(self.dest)
-
-        path = yield from self.dest.storage.import_data(self.arg)
-        self.enforce(' ' not in path)
-        size = self.dest.volumes[self.arg].size
-
-        # when we know the action is allowed, inform extensions that it will
-        # be performed
-        self.dest.fire_event('domain-volume-import-begin', volume=self.arg)
-
-        return '{} {}'.format(size, path)
-
     @qubes.api.method('admin.vm.volume.Set.revisions_to_keep',
         scope='local', write=True)
     @asyncio.coroutine

qubes/api/internal.py

@@ -1,4 +1,4 @@
-# -*- encoding: utf8 -*-
+# -*- encoding: utf-8 -*-
 #
 # The Qubes OS Project, http://www.qubes-os.org
 #
@@ -56,6 +56,61 @@ class QubesInternalAPI(qubes.api.AbstractQubesAPI):
 
         return json.dumps(system_info)
 
+    @qubes.api.method('internal.vm.volume.ImportBegin',
+        scope='local', write=True)
+    @asyncio.coroutine
+    def vm_volume_import(self, untrusted_payload):
+        """Begin importing volume data. Payload is either size of new data
+        in bytes, or empty. If empty, the current volume's size will be used.
+        Returns size and path to where data should be written.
+
+        Triggered by scripts in /etc/qubes-rpc:
+        admin.vm.volume.Import, admin.vm.volume.ImportWithSize.
+
+        When the script finish importing, it will trigger
+        internal.vm.volume.ImportEnd (with either b'ok' or b'fail' as a
+        payload) and response from that call will be actually send to the
+        caller.
+        """
+        self.enforce(self.arg in self.dest.volumes.keys())
+
+        if untrusted_payload:
+            original_method = 'admin.vm.volume.ImportWithSize'
+        else:
+            original_method = 'admin.vm.volume.Import'
+        self.src.fire_event(
+            'admin-permission:' + original_method,
+            pre_event=True, dest=self.dest, arg=self.arg)
+
+        if not self.dest.is_halted():
+            raise qubes.exc.QubesVMNotHaltedError(self.dest)
+
+        requested_size = None
+        if untrusted_payload:
+            try:
+                untrusted_value = int(untrusted_payload.decode('ascii'))
+            except (UnicodeDecodeError, ValueError):
+                raise qubes.api.ProtocolError('Invalid value')
+            self.enforce(untrusted_value > 0)
+            requested_size = untrusted_value
+            del untrusted_value
+        del untrusted_payload
+
+        path = yield from self.dest.storage.import_data(
+            self.arg, requested_size)
+        self.enforce(' ' not in path)
+        if requested_size is None:
+            size = self.dest.volumes[self.arg].size
+        else:
+            size = requested_size
+
+        # when we know the action is allowed, inform extensions that it will
+        # be performed
+        self.dest.fire_event(
+            'domain-volume-import-begin', volume=self.arg, size=size)
+
+        return '{} {}'.format(size, path)
+
     @qubes.api.method('internal.vm.volume.ImportEnd')
     @asyncio.coroutine
     def vm_volume_import_end(self, untrusted_payload):

qubes/storage/__init__.py

@@ -188,7 +188,7 @@ class Volume:
         '''
         raise self._not_implemented("export")
 
-    def import_data(self):
+    def import_data(self, size):
         ''' Returns a path to overwrite volume data.
 
             This method is called after volume was already :py:meth:`create`-ed.
@@ -199,6 +199,8 @@ class Volume:
             on the fly), the returned path may be a pipe.
 
             This can be implemented as a coroutine.
+
+            :param int size: size of new data in bytes
         '''
         raise self._not_implemented("import_data")
 
@@ -624,14 +626,22 @@ class Storage:
         return self.vm.volumes[volume].export()
 
     @asyncio.coroutine
-    def import_data(self, volume):
+    def import_data(self, volume, size):
-        ''' Helper function to import volume data (pool.import_data(volume))'''
+        '''
+        Helper function to import volume data (pool.import_data(volume)).
+
+        :size: new size in bytes, or None if using old size
+        '''
+
         assert isinstance(volume, (Volume, str)), \
             "You need to pass a Volume or pool name as str"
-        if isinstance(volume, Volume):
+        if isinstance(volume, str):
-            ret = volume.import_data()
+            volume = self.vm.volumes[volume]
-        else:
+
-            ret = self.vm.volumes[volume].import_data()
+        if size is None:
+            size = volume.size
+
+        ret = volume.import_data(size)
         return (yield from qubes.utils.coro_maybe(ret))
 
     @asyncio.coroutine

qubes/storage/file.py

@@ -283,12 +283,12 @@ class FileVolume(qubes.storage.Volume):
             copy_file(src_volume.export(), self.path)
         return self
 
-    def import_data(self):
+    def import_data(self, size):
         if not self.save_on_stop:
             raise qubes.storage.StoragePoolException(
                 "Can not import into save_on_stop=False volume {!s}".format(
                     self))
-        create_sparse_file(self.path_import, self.size)
+        create_sparse_file(self.path_import, size)
         return self.path_import
 
     def import_data_end(self, success):

qubes/storage/lvm.py

@@ -560,7 +560,7 @@ class ThinVolume(qubes.storage.Volume):
 
     @locked
     @asyncio.coroutine
-    def import_data(self):
+    def import_data(self, size):
         ''' Returns an object that can be `open()`. '''
         if self.is_dirty():
             raise qubes.storage.StoragePoolException(
@@ -569,7 +569,7 @@ class ThinVolume(qubes.storage.Volume):
         self.abort_if_import_in_progress()
         # pylint: disable=protected-access
         cmd = ['create', self.pool._pool_id, self._vid_import.split('/')[1],
-               str(self.size)]
+               str(size)]
         yield from qubes_lvm_coro(cmd, self.log)
         yield from reset_cache_coro()
         devpath = '/dev/' + self._vid_import

qubes/storage/reflink.py

@@ -289,11 +289,11 @@ class ReflinkVolume(qubes.storage.Volume):
 
     @_coroutinized
     @_locked
-    def import_data(self):
+    def import_data(self, size):
         if not self.save_on_stop:
             raise NotImplementedError(
                 'Cannot import_data: {} is not save_on_stop'.format(self.vid))
-        _create_sparse_file(self._path_import, self._get_size())
+        _create_sparse_file(self._path_import, size)
         return self._path_import
 
     def _import_data_end(self, success):

qubes/tests/api_admin.py

@@ -1,4 +1,4 @@
-# -*- encoding: utf8 -*-
+# -*- encoding: utf-8 -*-
 #
 # The Qubes OS Project, http://www.qubes-os.org
 #
@@ -34,6 +34,7 @@ import qubes
 import qubes.devices
 import qubes.firewall
 import qubes.api.admin
+import qubes.api.internal
 import qubes.tests
 import qubes.storage
 
@@ -113,6 +114,13 @@ class AdminAPITestCase(qubes.tests.QubesTestCase):
             'admin-permission:' + method.decode('ascii'))
         return response
 
+    def call_internal_mgmt_func(self, method, dest, arg=b'', payload=b''):
+        mgmt_obj = qubes.api.internal.QubesInternalAPI(self.app, b'dom0', method, dest, arg)
+        loop = asyncio.get_event_loop()
+        response = loop.run_until_complete(
+            mgmt_obj.execute(untrusted_payload=payload))
+        return response
+
 
 class TC_00_VMs(AdminAPITestCase):
     def test_000_vm_list(self):
@@ -207,11 +215,9 @@ qid default=False type=int 2
 qrexec_timeout default=True type=int 60
 updateable default=True type=bool False
 kernelopts default=False type=str opt1\\nopt2\\nopt3\\\\opt4
-netvm default=True type=vm 
+netvm default=True type=vm \n'''
-'''
         self.assertEqual(value, expected)
 
-
     def test_030_vm_property_set_vm(self):
         netvm = self.app.add_new_vm('AppVM', label='red', name='test-net',
             template='test-template', provides_network=True)
@@ -1747,9 +1753,12 @@ netvm default=True type=vm
         self.assertFalse(mock_remove.called)
         self.assertFalse(self.app.save.called)
 
+    # Import tests
+    # (internal methods, normally called from qubes-rpc script)
+
     def test_510_vm_volume_import(self):
-        value = self.call_mgmt_func(b'admin.vm.volume.Import', b'test-vm1',
+        value = self.call_internal_mgmt_func(
-            b'private')
+            b'internal.vm.volume.ImportBegin', b'test-vm1', b'private')
         self.assertEqual(value, '{} {}'.format(
             2*2**30, '/tmp/qubes-test-dir/appvms/test-vm1/private-import.img'))
         self.assertFalse(self.app.save.called)
@@ -1758,8 +1767,34 @@ netvm default=True type=vm
         with unittest.mock.patch.object(
                 self.vm, 'get_power_state', lambda: 'Running'):
             with self.assertRaises(qubes.exc.QubesVMNotHaltedError):
-                self.call_mgmt_func(b'admin.vm.volume.Import', b'test-vm1',
+                self.call_internal_mgmt_func(
-                    b'private')
+                    b'internal.vm.volume.ImportBegin', b'test-vm1', b'private')
+
+    def test_512_vm_volume_import_with_size(self):
+        new_size = 4 * 2**30
+        file_name = '/tmp/qubes-test-dir/appvms/test-vm1/private-import.img'
+
+        value = self.call_internal_mgmt_func(
+            b'internal.vm.volume.ImportBegin', b'test-vm1',
+            b'private', payload=str(new_size).encode())
+        self.assertEqual(value, '{} {}'.format(
+            new_size, file_name))
+        self.assertFalse(self.app.save.called)
+
+        self.assertEqual(os.stat(file_name).st_size, new_size)
+
+    def test_515_vm_volume_import_fire_event(self):
+        self.call_internal_mgmt_func(
+            b'internal.vm.volume.ImportBegin', b'test-vm1', b'private')
+        self.assertEventFired(
+            self.emitter, 'admin-permission:admin.vm.volume.Import')
+
+    def test_516_vm_volume_import_fire_event_with_size(self):
+        self.call_internal_mgmt_func(
+            b'internal.vm.volume.ImportBegin', b'test-vm1', b'private',
+            b'123')
+        self.assertEventFired(
+            self.emitter, 'admin-permission:admin.vm.volume.ImportWithSize')
 
     def setup_for_clone(self):
         self.pool = unittest.mock.MagicMock()

qubes/tests/storage_file.py

@@ -332,7 +332,7 @@ class TC_01_FileVolumes(qubes.tests.QubesTestCase):
         vm = qubes.tests.storage.TestVM(self)
         volume = self.app.get_pool(self.POOL_NAME).init_volume(vm, config)
         volume.create()
-        import_path = volume.import_data()
+        import_path = volume.import_data(volume.size)
         self.assertNotEqual(volume.path, import_path)
         with open(import_path, 'w+') as import_file:
             import_file.write('test')
@@ -353,7 +353,7 @@ class TC_01_FileVolumes(qubes.tests.QubesTestCase):
         vm = qubes.tests.storage.TestVM(self)
         volume = self.app.get_pool(self.POOL_NAME).init_volume(vm, config)
         volume.create()
-        import_path = volume.import_data()
+        import_path = volume.import_data(volume.size)
         self.assertNotEqual(volume.path, import_path)
         with open(import_path, 'w+') as import_file:
             import_file.write('test')
@@ -376,7 +376,7 @@ class TC_01_FileVolumes(qubes.tests.QubesTestCase):
         volume.create()
         with open(volume.path, 'w') as vol_file:
             vol_file.write('test data')
-        import_path = volume.import_data()
+        import_path = volume.import_data(volume.size)
         self.assertNotEqual(volume.path, import_path)
         with open(import_path, 'w+'):
             pass
@@ -402,6 +402,30 @@ class TC_01_FileVolumes(qubes.tests.QubesTestCase):
         self.assertEqual(os.path.getsize(volume.path), new_size)
         self.assertEqual(volume.size, new_size)
 
+    def test_024_import_data_with_new_size(self):
+        config = {
+            'name': 'root',
+            'pool': self.POOL_NAME,
+            'save_on_stop': True,
+            'rw': True,
+            'size': 1024 * 1024,
+        }
+        new_size = 2 * 1024 * 1024
+
+        vm = qubes.tests.storage.TestVM(self)
+        volume = self.app.get_pool(self.POOL_NAME).init_volume(vm, config)
+        volume.create()
+        import_path = volume.import_data(new_size)
+        self.assertNotEqual(volume.path, import_path)
+        with open(import_path, 'r+b') as import_file:
+            import_file.write(b'test')
+        volume.import_data_end(True)
+        self.assertFalse(os.path.exists(import_path), import_path)
+        with open(volume.path, 'rb') as volume_file:
+            volume_data = volume_file.read()
+        self.assertEqual(volume_data.strip(b'\0'), b'test')
+        self.assertEqual(len(volume_data), new_size)
+
     def _get_loop_size(self, path):
         sudo = [] if os.getuid() == 0 else ['sudo']
         try:

qubes/tests/storage_lvm.py

@@ -698,7 +698,8 @@ class TC_00_ThinPool(ThinPoolBase):
         self.loop.run_until_complete(volume.create())
         current_uuid = self._get_lv_uuid(volume.path)
         self.assertFalse(volume.is_dirty())
-        import_path = self.loop.run_until_complete(volume.import_data())
+        import_path = self.loop.run_until_complete(
+            volume.import_data(volume.size))
         import_uuid = self._get_lv_uuid(import_path)
         self.assertNotEqual(current_uuid, import_uuid)
         # success - commit data
@@ -729,7 +730,8 @@ class TC_00_ThinPool(ThinPoolBase):
         self.loop.run_until_complete(volume.create())
         current_uuid = self._get_lv_uuid(volume.path)
         self.assertFalse(volume.is_dirty())
-        import_path = self.loop.run_until_complete(volume.import_data())
+        import_path = self.loop.run_until_complete(
+            volume.import_data(volume.size))
         import_uuid = self._get_lv_uuid(import_path)
         self.assertNotEqual(current_uuid, import_uuid)
         # fail - discard data
@@ -860,7 +862,8 @@ class TC_00_ThinPool(ThinPoolBase):
             'sudo', 'dd', 'if=/dev/urandom', 'of=' + volume.path, 'count=1', 'bs=1M'
         ))
         self.loop.run_until_complete(p.wait())
-        import_path = self.loop.run_until_complete(volume.import_data())
+        import_path = self.loop.run_until_complete(
+            volume.import_data(volume.size))
         self.assertNotEqual(volume.path, import_path)
         p = self.loop.run_until_complete(asyncio.create_subprocess_exec(
             'sudo', 'touch', import_path))
@@ -874,6 +877,28 @@ class TC_00_ThinPool(ThinPoolBase):
         volume_data, _ = self.loop.run_until_complete(p.communicate())
         self.assertEqual(volume_data.strip(b'\0'), b'')
 
+    def test_035_import_data_new_size(self):
+        ''' Test volume import'''
+        config = {
+            'name': 'root',
+            'pool': self.pool.name,
+            'save_on_stop': True,
+            'rw': True,
+            'revisions_to_keep': 2,
+            'size': qubes.config.defaults['root_img_size'],
+        }
+        new_size = 2 * qubes.config.defaults['root_img_size']
+
+        vm = qubes.tests.storage.TestVM(self)
+        volume = self.app.get_pool(self.pool.name).init_volume(vm, config)
+        self.loop.run_until_complete(volume.create())
+        self.loop.run_until_complete(
+            volume.import_data(new_size))
+        self.loop.run_until_complete(volume.import_data_end(True))
+        self.assertEqual(volume.size, new_size)
+
+        self.loop.run_until_complete(volume.remove())
+
     def test_040_volatile(self):
         '''Volatile volume test'''
         config = {

qubes/tests/storage_reflink.py

@@ -128,7 +128,7 @@ class TC_10_ReflinkPool(qubes.tests.QubesTestCase):
         self.loop.run_until_complete(volume.create())
         with open(volume.export(), 'w') as vol_file:
             vol_file.write('test data')
-        import_path = self.loop.run_until_complete(volume.import_data())
+        import_path = self.loop.run_until_complete(volume.import_data(volume.size))
         self.assertNotEqual(volume.path, import_path)
         with open(import_path, 'w+'):
             pass