From 63ac952803a0f6767ed64128f4e02b846647730b Mon Sep 17 00:00:00 2001 From: Pawel Marczewski Date: Thu, 16 Jan 2020 14:41:00 +0100 Subject: [PATCH] Implement new admin.vm.ImportWithSize API call This should allow importing a volume and changing the size at the same time, without performing the resize operation on original volume first. The internal API has been renamed to internal.vm.volume.ImportBegin to avoid confusion, and for symmetry with ImportEnd. See QubesOS/qubes-issues#5239. --- Makefile | 4 +- .../admin.vm.volume.Import.policy | 13 +++++ .../admin.vm.volume.ImportWithSize.policy | 13 +++++ qubes-rpc/admin.vm.volume.Import | 25 ++++++-- qubes/api/admin.py | 30 ---------- qubes/api/internal.py | 57 ++++++++++++++++++- qubes/storage/__init__.py | 24 +++++--- qubes/storage/file.py | 4 +- qubes/storage/lvm.py | 4 +- qubes/storage/reflink.py | 4 +- qubes/tests/api_admin.py | 51 ++++++++++++++--- qubes/tests/storage_file.py | 30 +++++++++- qubes/tests/storage_lvm.py | 31 +++++++++- qubes/tests/storage_reflink.py | 2 +- 14 files changed, 226 insertions(+), 66 deletions(-) create mode 100644 qubes-rpc-policy/admin.vm.volume.Import.policy create mode 100644 qubes-rpc-policy/admin.vm.volume.ImportWithSize.policy diff --git a/Makefile b/Makefile index df64f81e..78197e83 100644 --- a/Makefile +++ b/Makefile @@ -191,6 +191,8 @@ endif cp qubes-rpc-policy/qubes.GetDate.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.GetDate cp qubes-rpc-policy/qubes.ConnectTCP.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.ConnectTCP cp qubes-rpc-policy/admin.vm.Console.policy $(DESTDIR)/etc/qubes-rpc/policy/admin.vm.Console + cp qubes-rpc-policy/admin.vm.volume.Import.policy $(DESTDIR)/etc/qubes-rpc/policy/admin.vm.volume.Import + cp qubes-rpc-policy/admin.vm.volume.ImportWithSize.policy $(DESTDIR)/etc/qubes-rpc/policy/admin.vm.volume.ImportWithSize cp qubes-rpc-policy/policy.RegisterArgument.policy $(DESTDIR)/etc/qubes-rpc/policy/policy.RegisterArgument cp qubes-rpc/qubes.FeaturesRequest $(DESTDIR)/etc/qubes-rpc/ cp qubes-rpc/qubes.GetDate $(DESTDIR)/etc/qubes-rpc/ @@ -208,6 +210,7 @@ endif $(DESTDIR)/etc/qubes-rpc/$$method || exit 1; \ done install qubes-rpc/admin.vm.volume.Import $(DESTDIR)/etc/qubes-rpc/ + ln -s admin.vm.volume.Import $(DESTDIR)/etc/qubes-rpc/admin.vm.volume.ImportWithSize install qubes-rpc/admin.vm.Console $(DESTDIR)/etc/qubes-rpc/ PYTHONPATH=.:test-packages qubes-rpc-policy/generate-admin-policy \ --destdir=$(DESTDIR)/etc/qubes-rpc/policy \ @@ -260,4 +263,3 @@ msi: candle -arch x64 -dversion=$(VERSION) installer.wxs light -b destinstdir -o core-admin.msm installer.wixobj rm -rf destinstdir - diff --git a/qubes-rpc-policy/admin.vm.volume.Import.policy b/qubes-rpc-policy/admin.vm.volume.Import.policy new file mode 100644 index 00000000..c2b92162 --- /dev/null +++ b/qubes-rpc-policy/admin.vm.volume.Import.policy @@ -0,0 +1,13 @@ +## Note that policy parsing stops at the first match. +## Anything not specifically allowed here (or in included file) will be denied. + +## Please use a single # to start your custom comments + +## Add your entries here, make sure to append ",target=dom0" to all allow/ask actions + +## Include a common file for all admin.* methods to ease setting up +## Management VM. +## To allow only specific actions, edit specific policy file, like this one. To +## allow all of them, edit appropriate /etc/qubes-rpc/include/admin-*. + +$include:include/admin-local-rwx diff --git a/qubes-rpc-policy/admin.vm.volume.ImportWithSize.policy b/qubes-rpc-policy/admin.vm.volume.ImportWithSize.policy new file mode 100644 index 00000000..c2b92162 --- /dev/null +++ b/qubes-rpc-policy/admin.vm.volume.ImportWithSize.policy @@ -0,0 +1,13 @@ +## Note that policy parsing stops at the first match. +## Anything not specifically allowed here (or in included file) will be denied. + +## Please use a single # to start your custom comments + +## Add your entries here, make sure to append ",target=dom0" to all allow/ask actions + +## Include a common file for all admin.* methods to ease setting up +## Management VM. +## To allow only specific actions, edit specific policy file, like this one. To +## allow all of them, edit appropriate /etc/qubes-rpc/include/admin-*. + +$include:include/admin-local-rwx diff --git a/qubes-rpc/admin.vm.volume.Import b/qubes-rpc/admin.vm.volume.Import index f41ee987..db57c9e1 100755 --- a/qubes-rpc/admin.vm.volume.Import +++ b/qubes-rpc/admin.vm.volume.Import @@ -16,23 +16,36 @@ # 2. Actual data import (done by this script, using dd) # 3. Report final result, produce final response to the caller (done by # qubesd) -# +# # This way we do not pass all the data through qubesd, but still can # control the process from there in a meaningful way. Note that the last # part (second call to qubesd) may perform all kind of verification (like # a signature check on the data, or so) and can also prevent VM from # starting (hooking also domain-pre-start event) from not verified image. +# +# Note that this script implements two calls: +# - admin.vm.volume.Import +# - admin.vm.volume.ImportWithSize +# In the case of admin.vm.ImportWithSize, the first line of payload is then +# data size in bytes. This is so that we can already notify qubesd to create a +# volume with the new size. set -e # use temporary file, because env variables deal poorly with \0 inside tmpfile=$(mktemp) trap "rm -f $tmpfile" EXIT -qubesd-query -e \ - "$QREXEC_REMOTE_DOMAIN" \ - "admin.vm.volume.Import" \ - "$QREXEC_REQUESTED_TARGET" \ - "$1" >$tmpfile + +requested_size="" +if [[ ${0##*/} == admin.vm.volume.ImportWithSize ]]; then + read requested_size +fi + +echo -n "$requested_size" | qubesd-query -c /var/run/qubesd.internal.sock \ + "$QREXEC_REMOTE_DOMAIN" \ + "internal.vm.volume.ImportBegin" \ + "$QREXEC_REQUESTED_TARGET" \ + "$1" >$tmpfile # exit if qubesd returned an error (not '0\0') if [ "$(head -c 2 $tmpfile | xxd -p)" != "3000" ]; then diff --git a/qubes/api/admin.py b/qubes/api/admin.py index 17a3ecd8..8a7da0ea 100644 --- a/qubes/api/admin.py +++ b/qubes/api/admin.py @@ -486,36 +486,6 @@ class QubesAdminAPI(qubes.api.AbstractQubesAPI): finally: # even if calling qubes.ResizeDisk inside the VM failed self.app.save() - @qubes.api.method('admin.vm.volume.Import', no_payload=True, - scope='local', write=True) - @asyncio.coroutine - def vm_volume_import(self): - """Import volume data. - - Note that this function only returns a path to where data should be - written, actual importing is done by a script in /etc/qubes-rpc - When the script finish importing, it will trigger - internal.vm.volume.ImportEnd (with either b'ok' or b'fail' as a - payload) and response from that call will be actually send to the - caller. - """ - self.enforce(self.arg in self.dest.volumes.keys()) - - self.fire_event_for_permission() - - if not self.dest.is_halted(): - raise qubes.exc.QubesVMNotHaltedError(self.dest) - - path = yield from self.dest.storage.import_data(self.arg) - self.enforce(' ' not in path) - size = self.dest.volumes[self.arg].size - - # when we know the action is allowed, inform extensions that it will - # be performed - self.dest.fire_event('domain-volume-import-begin', volume=self.arg) - - return '{} {}'.format(size, path) - @qubes.api.method('admin.vm.volume.Set.revisions_to_keep', scope='local', write=True) @asyncio.coroutine diff --git a/qubes/api/internal.py b/qubes/api/internal.py index 09a05171..297edfcb 100644 --- a/qubes/api/internal.py +++ b/qubes/api/internal.py @@ -1,4 +1,4 @@ -# -*- encoding: utf8 -*- +# -*- encoding: utf-8 -*- # # The Qubes OS Project, http://www.qubes-os.org # @@ -56,6 +56,61 @@ class QubesInternalAPI(qubes.api.AbstractQubesAPI): return json.dumps(system_info) + @qubes.api.method('internal.vm.volume.ImportBegin', + scope='local', write=True) + @asyncio.coroutine + def vm_volume_import(self, untrusted_payload): + """Begin importing volume data. Payload is either size of new data + in bytes, or empty. If empty, the current volume's size will be used. + Returns size and path to where data should be written. + + Triggered by scripts in /etc/qubes-rpc: + admin.vm.volume.Import, admin.vm.volume.ImportWithSize. + + When the script finish importing, it will trigger + internal.vm.volume.ImportEnd (with either b'ok' or b'fail' as a + payload) and response from that call will be actually send to the + caller. + """ + self.enforce(self.arg in self.dest.volumes.keys()) + + if untrusted_payload: + original_method = 'admin.vm.volume.ImportWithSize' + else: + original_method = 'admin.vm.volume.Import' + self.src.fire_event( + 'admin-permission:' + original_method, + pre_event=True, dest=self.dest, arg=self.arg) + + if not self.dest.is_halted(): + raise qubes.exc.QubesVMNotHaltedError(self.dest) + + requested_size = None + if untrusted_payload: + try: + untrusted_value = int(untrusted_payload.decode('ascii')) + except (UnicodeDecodeError, ValueError): + raise qubes.api.ProtocolError('Invalid value') + self.enforce(untrusted_value > 0) + requested_size = untrusted_value + del untrusted_value + del untrusted_payload + + path = yield from self.dest.storage.import_data( + self.arg, requested_size) + self.enforce(' ' not in path) + if requested_size is None: + size = self.dest.volumes[self.arg].size + else: + size = requested_size + + # when we know the action is allowed, inform extensions that it will + # be performed + self.dest.fire_event( + 'domain-volume-import-begin', volume=self.arg, size=size) + + return '{} {}'.format(size, path) + @qubes.api.method('internal.vm.volume.ImportEnd') @asyncio.coroutine def vm_volume_import_end(self, untrusted_payload): diff --git a/qubes/storage/__init__.py b/qubes/storage/__init__.py index da518a99..f71c2a7b 100644 --- a/qubes/storage/__init__.py +++ b/qubes/storage/__init__.py @@ -188,7 +188,7 @@ class Volume: ''' raise self._not_implemented("export") - def import_data(self): + def import_data(self, size): ''' Returns a path to overwrite volume data. This method is called after volume was already :py:meth:`create`-ed. @@ -199,6 +199,8 @@ class Volume: on the fly), the returned path may be a pipe. This can be implemented as a coroutine. + + :param int size: size of new data in bytes ''' raise self._not_implemented("import_data") @@ -624,14 +626,22 @@ class Storage: return self.vm.volumes[volume].export() @asyncio.coroutine - def import_data(self, volume): - ''' Helper function to import volume data (pool.import_data(volume))''' + def import_data(self, volume, size): + ''' + Helper function to import volume data (pool.import_data(volume)). + + :size: new size in bytes, or None if using old size + ''' + assert isinstance(volume, (Volume, str)), \ "You need to pass a Volume or pool name as str" - if isinstance(volume, Volume): - ret = volume.import_data() - else: - ret = self.vm.volumes[volume].import_data() + if isinstance(volume, str): + volume = self.vm.volumes[volume] + + if size is None: + size = volume.size + + ret = volume.import_data(size) return (yield from qubes.utils.coro_maybe(ret)) @asyncio.coroutine diff --git a/qubes/storage/file.py b/qubes/storage/file.py index cd3ee9d7..c4ef05a5 100644 --- a/qubes/storage/file.py +++ b/qubes/storage/file.py @@ -283,12 +283,12 @@ class FileVolume(qubes.storage.Volume): copy_file(src_volume.export(), self.path) return self - def import_data(self): + def import_data(self, size): if not self.save_on_stop: raise qubes.storage.StoragePoolException( "Can not import into save_on_stop=False volume {!s}".format( self)) - create_sparse_file(self.path_import, self.size) + create_sparse_file(self.path_import, size) return self.path_import def import_data_end(self, success): diff --git a/qubes/storage/lvm.py b/qubes/storage/lvm.py index b0a3b52d..1e2f0e9d 100644 --- a/qubes/storage/lvm.py +++ b/qubes/storage/lvm.py @@ -560,7 +560,7 @@ class ThinVolume(qubes.storage.Volume): @locked @asyncio.coroutine - def import_data(self): + def import_data(self, size): ''' Returns an object that can be `open()`. ''' if self.is_dirty(): raise qubes.storage.StoragePoolException( @@ -569,7 +569,7 @@ class ThinVolume(qubes.storage.Volume): self.abort_if_import_in_progress() # pylint: disable=protected-access cmd = ['create', self.pool._pool_id, self._vid_import.split('/')[1], - str(self.size)] + str(size)] yield from qubes_lvm_coro(cmd, self.log) yield from reset_cache_coro() devpath = '/dev/' + self._vid_import diff --git a/qubes/storage/reflink.py b/qubes/storage/reflink.py index 795a7ffc..a7aa320c 100644 --- a/qubes/storage/reflink.py +++ b/qubes/storage/reflink.py @@ -289,11 +289,11 @@ class ReflinkVolume(qubes.storage.Volume): @_coroutinized @_locked - def import_data(self): + def import_data(self, size): if not self.save_on_stop: raise NotImplementedError( 'Cannot import_data: {} is not save_on_stop'.format(self.vid)) - _create_sparse_file(self._path_import, self._get_size()) + _create_sparse_file(self._path_import, size) return self._path_import def _import_data_end(self, success): diff --git a/qubes/tests/api_admin.py b/qubes/tests/api_admin.py index 0cd6faa2..185b487f 100644 --- a/qubes/tests/api_admin.py +++ b/qubes/tests/api_admin.py @@ -1,4 +1,4 @@ -# -*- encoding: utf8 -*- +# -*- encoding: utf-8 -*- # # The Qubes OS Project, http://www.qubes-os.org # @@ -34,6 +34,7 @@ import qubes import qubes.devices import qubes.firewall import qubes.api.admin +import qubes.api.internal import qubes.tests import qubes.storage @@ -113,6 +114,13 @@ class AdminAPITestCase(qubes.tests.QubesTestCase): 'admin-permission:' + method.decode('ascii')) return response + def call_internal_mgmt_func(self, method, dest, arg=b'', payload=b''): + mgmt_obj = qubes.api.internal.QubesInternalAPI(self.app, b'dom0', method, dest, arg) + loop = asyncio.get_event_loop() + response = loop.run_until_complete( + mgmt_obj.execute(untrusted_payload=payload)) + return response + class TC_00_VMs(AdminAPITestCase): def test_000_vm_list(self): @@ -207,11 +215,9 @@ qid default=False type=int 2 qrexec_timeout default=True type=int 60 updateable default=True type=bool False kernelopts default=False type=str opt1\\nopt2\\nopt3\\\\opt4 -netvm default=True type=vm -''' +netvm default=True type=vm \n''' self.assertEqual(value, expected) - def test_030_vm_property_set_vm(self): netvm = self.app.add_new_vm('AppVM', label='red', name='test-net', template='test-template', provides_network=True) @@ -1747,9 +1753,12 @@ netvm default=True type=vm self.assertFalse(mock_remove.called) self.assertFalse(self.app.save.called) + # Import tests + # (internal methods, normally called from qubes-rpc script) + def test_510_vm_volume_import(self): - value = self.call_mgmt_func(b'admin.vm.volume.Import', b'test-vm1', - b'private') + value = self.call_internal_mgmt_func( + b'internal.vm.volume.ImportBegin', b'test-vm1', b'private') self.assertEqual(value, '{} {}'.format( 2*2**30, '/tmp/qubes-test-dir/appvms/test-vm1/private-import.img')) self.assertFalse(self.app.save.called) @@ -1758,8 +1767,34 @@ netvm default=True type=vm with unittest.mock.patch.object( self.vm, 'get_power_state', lambda: 'Running'): with self.assertRaises(qubes.exc.QubesVMNotHaltedError): - self.call_mgmt_func(b'admin.vm.volume.Import', b'test-vm1', - b'private') + self.call_internal_mgmt_func( + b'internal.vm.volume.ImportBegin', b'test-vm1', b'private') + + def test_512_vm_volume_import_with_size(self): + new_size = 4 * 2**30 + file_name = '/tmp/qubes-test-dir/appvms/test-vm1/private-import.img' + + value = self.call_internal_mgmt_func( + b'internal.vm.volume.ImportBegin', b'test-vm1', + b'private', payload=str(new_size).encode()) + self.assertEqual(value, '{} {}'.format( + new_size, file_name)) + self.assertFalse(self.app.save.called) + + self.assertEqual(os.stat(file_name).st_size, new_size) + + def test_515_vm_volume_import_fire_event(self): + self.call_internal_mgmt_func( + b'internal.vm.volume.ImportBegin', b'test-vm1', b'private') + self.assertEventFired( + self.emitter, 'admin-permission:admin.vm.volume.Import') + + def test_516_vm_volume_import_fire_event_with_size(self): + self.call_internal_mgmt_func( + b'internal.vm.volume.ImportBegin', b'test-vm1', b'private', + b'123') + self.assertEventFired( + self.emitter, 'admin-permission:admin.vm.volume.ImportWithSize') def setup_for_clone(self): self.pool = unittest.mock.MagicMock() diff --git a/qubes/tests/storage_file.py b/qubes/tests/storage_file.py index 9742ffa1..bc3df8cb 100644 --- a/qubes/tests/storage_file.py +++ b/qubes/tests/storage_file.py @@ -332,7 +332,7 @@ class TC_01_FileVolumes(qubes.tests.QubesTestCase): vm = qubes.tests.storage.TestVM(self) volume = self.app.get_pool(self.POOL_NAME).init_volume(vm, config) volume.create() - import_path = volume.import_data() + import_path = volume.import_data(volume.size) self.assertNotEqual(volume.path, import_path) with open(import_path, 'w+') as import_file: import_file.write('test') @@ -353,7 +353,7 @@ class TC_01_FileVolumes(qubes.tests.QubesTestCase): vm = qubes.tests.storage.TestVM(self) volume = self.app.get_pool(self.POOL_NAME).init_volume(vm, config) volume.create() - import_path = volume.import_data() + import_path = volume.import_data(volume.size) self.assertNotEqual(volume.path, import_path) with open(import_path, 'w+') as import_file: import_file.write('test') @@ -376,7 +376,7 @@ class TC_01_FileVolumes(qubes.tests.QubesTestCase): volume.create() with open(volume.path, 'w') as vol_file: vol_file.write('test data') - import_path = volume.import_data() + import_path = volume.import_data(volume.size) self.assertNotEqual(volume.path, import_path) with open(import_path, 'w+'): pass @@ -402,6 +402,30 @@ class TC_01_FileVolumes(qubes.tests.QubesTestCase): self.assertEqual(os.path.getsize(volume.path), new_size) self.assertEqual(volume.size, new_size) + def test_024_import_data_with_new_size(self): + config = { + 'name': 'root', + 'pool': self.POOL_NAME, + 'save_on_stop': True, + 'rw': True, + 'size': 1024 * 1024, + } + new_size = 2 * 1024 * 1024 + + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.POOL_NAME).init_volume(vm, config) + volume.create() + import_path = volume.import_data(new_size) + self.assertNotEqual(volume.path, import_path) + with open(import_path, 'r+b') as import_file: + import_file.write(b'test') + volume.import_data_end(True) + self.assertFalse(os.path.exists(import_path), import_path) + with open(volume.path, 'rb') as volume_file: + volume_data = volume_file.read() + self.assertEqual(volume_data.strip(b'\0'), b'test') + self.assertEqual(len(volume_data), new_size) + def _get_loop_size(self, path): sudo = [] if os.getuid() == 0 else ['sudo'] try: diff --git a/qubes/tests/storage_lvm.py b/qubes/tests/storage_lvm.py index 563fdf9f..5646ac9f 100644 --- a/qubes/tests/storage_lvm.py +++ b/qubes/tests/storage_lvm.py @@ -698,7 +698,8 @@ class TC_00_ThinPool(ThinPoolBase): self.loop.run_until_complete(volume.create()) current_uuid = self._get_lv_uuid(volume.path) self.assertFalse(volume.is_dirty()) - import_path = self.loop.run_until_complete(volume.import_data()) + import_path = self.loop.run_until_complete( + volume.import_data(volume.size)) import_uuid = self._get_lv_uuid(import_path) self.assertNotEqual(current_uuid, import_uuid) # success - commit data @@ -729,7 +730,8 @@ class TC_00_ThinPool(ThinPoolBase): self.loop.run_until_complete(volume.create()) current_uuid = self._get_lv_uuid(volume.path) self.assertFalse(volume.is_dirty()) - import_path = self.loop.run_until_complete(volume.import_data()) + import_path = self.loop.run_until_complete( + volume.import_data(volume.size)) import_uuid = self._get_lv_uuid(import_path) self.assertNotEqual(current_uuid, import_uuid) # fail - discard data @@ -860,7 +862,8 @@ class TC_00_ThinPool(ThinPoolBase): 'sudo', 'dd', 'if=/dev/urandom', 'of=' + volume.path, 'count=1', 'bs=1M' )) self.loop.run_until_complete(p.wait()) - import_path = self.loop.run_until_complete(volume.import_data()) + import_path = self.loop.run_until_complete( + volume.import_data(volume.size)) self.assertNotEqual(volume.path, import_path) p = self.loop.run_until_complete(asyncio.create_subprocess_exec( 'sudo', 'touch', import_path)) @@ -874,6 +877,28 @@ class TC_00_ThinPool(ThinPoolBase): volume_data, _ = self.loop.run_until_complete(p.communicate()) self.assertEqual(volume_data.strip(b'\0'), b'') + def test_035_import_data_new_size(self): + ''' Test volume import''' + config = { + 'name': 'root', + 'pool': self.pool.name, + 'save_on_stop': True, + 'rw': True, + 'revisions_to_keep': 2, + 'size': qubes.config.defaults['root_img_size'], + } + new_size = 2 * qubes.config.defaults['root_img_size'] + + vm = qubes.tests.storage.TestVM(self) + volume = self.app.get_pool(self.pool.name).init_volume(vm, config) + self.loop.run_until_complete(volume.create()) + self.loop.run_until_complete( + volume.import_data(new_size)) + self.loop.run_until_complete(volume.import_data_end(True)) + self.assertEqual(volume.size, new_size) + + self.loop.run_until_complete(volume.remove()) + def test_040_volatile(self): '''Volatile volume test''' config = { diff --git a/qubes/tests/storage_reflink.py b/qubes/tests/storage_reflink.py index 9da68638..5483cb96 100644 --- a/qubes/tests/storage_reflink.py +++ b/qubes/tests/storage_reflink.py @@ -128,7 +128,7 @@ class TC_10_ReflinkPool(qubes.tests.QubesTestCase): self.loop.run_until_complete(volume.create()) with open(volume.export(), 'w') as vol_file: vol_file.write('test data') - import_path = self.loop.run_until_complete(volume.import_data()) + import_path = self.loop.run_until_complete(volume.import_data(volume.size)) self.assertNotEqual(volume.path, import_path) with open(import_path, 'w+'): pass