Qubes/core-admin
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Implement new admin.vm.ImportWithSize API call

Browse Source 
This should allow importing a volume and changing the size at the
same time, without performing the resize operation on original
volume first.

The internal API has been renamed to internal.vm.volume.ImportBegin
to avoid confusion, and for symmetry with ImportEnd.

See QubesOS/qubes-issues#5239.
This commit is contained in:
Pawel Marczewski 2020-01-16 14:41:00 +01:00
parent 309dd11b1d
commit 63ac952803
No known key found for this signature in database
GPG Key ID: DE42EE9B14F96465
14 changed files with 226 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@ -191,6 +191,8 @@ endif
cp qubes-rpc-policy/qubes.GetDate.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.GetDate
cp qubes-rpc-policy/qubes.ConnectTCP.policy $(DESTDIR)/etc/qubes-rpc/policy/qubes.ConnectTCP
cp qubes-rpc-policy/admin.vm.Console.policy $(DESTDIR)/etc/qubes-rpc/policy/admin.vm.Console
cp qubes-rpc-policy/admin.vm.volume.Import.policy $(DESTDIR)/etc/qubes-rpc/policy/admin.vm.volume.Import
cp qubes-rpc-policy/admin.vm.volume.ImportWithSize.policy $(DESTDIR)/etc/qubes-rpc/policy/admin.vm.volume.ImportWithSize
cp qubes-rpc-policy/policy.RegisterArgument.policy $(DESTDIR)/etc/qubes-rpc/policy/policy.RegisterArgument
cp qubes-rpc/qubes.FeaturesRequest $(DESTDIR)/etc/qubes-rpc/
cp qubes-rpc/qubes.GetDate $(DESTDIR)/etc/qubes-rpc/
@ -208,6 +210,7 @@ endif
$(DESTDIR)/etc/qubes-rpc/$$method || exit 1; \
done
install qubes-rpc/admin.vm.volume.Import $(DESTDIR)/etc/qubes-rpc/
ln -s admin.vm.volume.Import $(DESTDIR)/etc/qubes-rpc/admin.vm.volume.ImportWithSize
install qubes-rpc/admin.vm.Console $(DESTDIR)/etc/qubes-rpc/
PYTHONPATH=.:test-packages qubes-rpc-policy/generate-admin-policy \
--destdir=$(DESTDIR)/etc/qubes-rpc/policy \
@ -260,4 +263,3 @@ msi:
candle -arch x64 -dversion=$(VERSION) installer.wxs
light -b destinstdir -o core-admin.msm installer.wixobj
rm -rf destinstdir

13
qubes-rpc-policy/admin.vm.volume.Import.policy Normal file
View File

@ -0,0 +1,13 @@
## Note that policy parsing stops at the first match.
## Anything not specifically allowed here (or in included file) will be denied.
## Please use a single # to start your custom comments
## Add your entries here, make sure to append ",target=dom0" to all allow/ask actions
## Include a common file for all admin.* methods to ease setting up
## Management VM.
## To allow only specific actions, edit specific policy file, like this one. To
## allow all of them, edit appropriate /etc/qubes-rpc/include/admin-*.
$include:include/admin-local-rwx

13
qubes-rpc-policy/admin.vm.volume.ImportWithSize.policy Normal file
View File

@ -0,0 +1,13 @@
## Note that policy parsing stops at the first match.
## Anything not specifically allowed here (or in included file) will be denied.
## Please use a single # to start your custom comments
## Add your entries here, make sure to append ",target=dom0" to all allow/ask actions
## Include a common file for all admin.* methods to ease setting up
## Management VM.
## To allow only specific actions, edit specific policy file, like this one. To
## allow all of them, edit appropriate /etc/qubes-rpc/include/admin-*.
$include:include/admin-local-rwx

17
qubes-rpc/admin.vm.volume.Import
View File

@ -22,15 +22,28 @@
# part (second call to qubesd) may perform all kind of verification (like
# a signature check on the data, or so) and can also prevent VM from
# starting (hooking also domain-pre-start event) from not verified image.
#
# Note that this script implements two calls:
# - admin.vm.volume.Import
# - admin.vm.volume.ImportWithSize
# In the case of admin.vm.ImportWithSize, the first line of payload is then
# data size in bytes. This is so that we can already notify qubesd to create a
# volume with the new size.
set -e
# use temporary file, because env variables deal poorly with \0 inside
tmpfile=$(mktemp)
trap "rm -f $tmpfile" EXIT
qubesd-query -e \
requested_size=""
if [[ ${0##*/} == admin.vm.volume.ImportWithSize ]]; then
read requested_size
fi
echo -n "$requested_size" | qubesd-query -c /var/run/qubesd.internal.sock \
"$QREXEC_REMOTE_DOMAIN" \
"admin.vm.volume.Import" \
"internal.vm.volume.ImportBegin" \
"$QREXEC_REQUESTED_TARGET" \
"$1" >$tmpfile

30
qubes/api/admin.py
View File

@ -486,36 +486,6 @@ class QubesAdminAPI(qubes.api.AbstractQubesAPI):
finally: # even if calling qubes.ResizeDisk inside the VM failed
self.app.save()
@qubes.api.method('admin.vm.volume.Import', no_payload=True,
scope='local', write=True)
@asyncio.coroutine
def vm_volume_import(self):
"""Import volume data.
Note that this function only returns a path to where data should be
written, actual importing is done by a script in /etc/qubes-rpc
When the script finish importing, it will trigger
internal.vm.volume.ImportEnd (with either b'ok' or b'fail' as a
payload) and response from that call will be actually send to the
caller.
"""
self.enforce(self.arg in self.dest.volumes.keys())
self.fire_event_for_permission()
if not self.dest.is_halted():
raise qubes.exc.QubesVMNotHaltedError(self.dest)
path = yield from self.dest.storage.import_data(self.arg)
self.enforce(' ' not in path)
size = self.dest.volumes[self.arg].size
# when we know the action is allowed, inform extensions that it will
# be performed
self.dest.fire_event('domain-volume-import-begin', volume=self.arg)
return '{} {}'.format(size, path)
@qubes.api.method('admin.vm.volume.Set.revisions_to_keep',
scope='local', write=True)
@asyncio.coroutine

57
qubes/api/internal.py
View File

@ -1,4 +1,4 @@
# -*- encoding: utf8 -*-
# -*- encoding: utf-8 -*-
#
# The Qubes OS Project, http://www.qubes-os.org
#
@ -56,6 +56,61 @@ class QubesInternalAPI(qubes.api.AbstractQubesAPI):
return json.dumps(system_info)
@qubes.api.method('internal.vm.volume.ImportBegin',
scope='local', write=True)
@asyncio.coroutine
def vm_volume_import(self, untrusted_payload):
"""Begin importing volume data. Payload is either size of new data
in bytes, or empty. If empty, the current volume's size will be used.
Returns size and path to where data should be written.
Triggered by scripts in /etc/qubes-rpc:
admin.vm.volume.Import, admin.vm.volume.ImportWithSize.
When the script finish importing, it will trigger
internal.vm.volume.ImportEnd (with either b'ok' or b'fail' as a
payload) and response from that call will be actually send to the
caller.
"""
self.enforce(self.arg in self.dest.volumes.keys())
if untrusted_payload:
original_method = 'admin.vm.volume.ImportWithSize'
else:
original_method = 'admin.vm.volume.Import'
self.src.fire_event(
'admin-permission:' + original_method,
pre_event=True, dest=self.dest, arg=self.arg)
if not self.dest.is_halted():
raise qubes.exc.QubesVMNotHaltedError(self.dest)
requested_size = None
if untrusted_payload:
try:
untrusted_value = int(untrusted_payload.decode('ascii'))
except (UnicodeDecodeError, ValueError):
raise qubes.api.ProtocolError('Invalid value')
self.enforce(untrusted_value > 0)
requested_size = untrusted_value
del untrusted_value
del untrusted_payload
path = yield from self.dest.storage.import_data(
self.arg, requested_size)
self.enforce(' ' not in path)
if requested_size is None:
size = self.dest.volumes[self.arg].size
else:
size = requested_size
# when we know the action is allowed, inform extensions that it will
# be performed
self.dest.fire_event(
'domain-volume-import-begin', volume=self.arg, size=size)
return '{} {}'.format(size, path)
@qubes.api.method('internal.vm.volume.ImportEnd')
@asyncio.coroutine
def vm_volume_import_end(self, untrusted_payload):

24
qubes/storage/__init__.py
View File

@ -188,7 +188,7 @@ class Volume:
'''
raise self._not_implemented("export")
def import_data(self):
def import_data(self, size):
''' Returns a path to overwrite volume data.
This method is called after volume was already :py:meth:`create`-ed.
@ -199,6 +199,8 @@ class Volume:
on the fly), the returned path may be a pipe.
This can be implemented as a coroutine.
:param int size: size of new data in bytes
'''
raise self._not_implemented("import_data")
@ -624,14 +626,22 @@ class Storage:
return self.vm.volumes[volume].export()
@asyncio.coroutine
def import_data(self, volume):
''' Helper function to import volume data (pool.import_data(volume))'''
def import_data(self, volume, size):
'''
Helper function to import volume data (pool.import_data(volume)).
:size: new size in bytes, or None if using old size
'''
assert isinstance(volume, (Volume, str)), \
"You need to pass a Volume or pool name as str"
if isinstance(volume, Volume):
ret = volume.import_data()
else:
ret = self.vm.volumes[volume].import_data()
if isinstance(volume, str):
volume = self.vm.volumes[volume]
if size is None:
size = volume.size
ret = volume.import_data(size)
return (yield from qubes.utils.coro_maybe(ret))
@asyncio.coroutine

4
qubes/storage/file.py
View File

@ -283,12 +283,12 @@ class FileVolume(qubes.storage.Volume):
copy_file(src_volume.export(), self.path)
return self
def import_data(self):
def import_data(self, size):
if not self.save_on_stop:
raise qubes.storage.StoragePoolException(
"Can not import into save_on_stop=False volume {!s}".format(
self))
create_sparse_file(self.path_import, self.size)
create_sparse_file(self.path_import, size)
return self.path_import
def import_data_end(self, success):

4
qubes/storage/lvm.py
View File

@ -560,7 +560,7 @@ class ThinVolume(qubes.storage.Volume):
@locked
@asyncio.coroutine
def import_data(self):
def import_data(self, size):
''' Returns an object that can be `open()`. '''
if self.is_dirty():
raise qubes.storage.StoragePoolException(
@ -569,7 +569,7 @@ class ThinVolume(qubes.storage.Volume):
self.abort_if_import_in_progress()
# pylint: disable=protected-access
cmd = ['create', self.pool._pool_id, self._vid_import.split('/')[1],
str(self.size)]
str(size)]
yield from qubes_lvm_coro(cmd, self.log)
yield from reset_cache_coro()
devpath = '/dev/' + self._vid_import

4
qubes/storage/reflink.py
View File

@ -289,11 +289,11 @@ class ReflinkVolume(qubes.storage.Volume):
@_coroutinized
@_locked
def import_data(self):
def import_data(self, size):
if not self.save_on_stop:
raise NotImplementedError(
'Cannot import_data: {} is not save_on_stop'.format(self.vid))
_create_sparse_file(self._path_import, self._get_size())
_create_sparse_file(self._path_import, size)
return self._path_import
def _import_data_end(self, success):

51
qubes/tests/api_admin.py
View File

@ -1,4 +1,4 @@
# -*- encoding: utf8 -*-
# -*- encoding: utf-8 -*-
#
# The Qubes OS Project, http://www.qubes-os.org
#
@ -34,6 +34,7 @@ import qubes
import qubes.devices
import qubes.firewall
import qubes.api.admin
import qubes.api.internal
import qubes.tests
import qubes.storage
@ -113,6 +114,13 @@ class AdminAPITestCase(qubes.tests.QubesTestCase):
'admin-permission:' + method.decode('ascii'))
return response
def call_internal_mgmt_func(self, method, dest, arg=b'', payload=b''):
mgmt_obj = qubes.api.internal.QubesInternalAPI(self.app, b'dom0', method, dest, arg)
loop = asyncio.get_event_loop()
response = loop.run_until_complete(
mgmt_obj.execute(untrusted_payload=payload))
return response
class TC_00_VMs(AdminAPITestCase):
def test_000_vm_list(self):
@ -207,11 +215,9 @@ qid default=False type=int 2
qrexec_timeout default=True type=int 60
updateable default=True type=bool False
kernelopts default=False type=str opt1\\nopt2\\nopt3\\\\opt4
netvm default=True type=vm
'''
netvm default=True type=vm \n'''
self.assertEqual(value, expected)
def test_030_vm_property_set_vm(self):
netvm = self.app.add_new_vm('AppVM', label='red', name='test-net',
template='test-template', provides_network=True)
@ -1747,9 +1753,12 @@ netvm default=True type=vm
self.assertFalse(mock_remove.called)
self.assertFalse(self.app.save.called)
# Import tests
# (internal methods, normally called from qubes-rpc script)
def test_510_vm_volume_import(self):
value = self.call_mgmt_func(b'admin.vm.volume.Import', b'test-vm1',
b'private')
value = self.call_internal_mgmt_func(
b'internal.vm.volume.ImportBegin', b'test-vm1', b'private')
self.assertEqual(value, '{} {}'.format(
2*2**30, '/tmp/qubes-test-dir/appvms/test-vm1/private-import.img'))
self.assertFalse(self.app.save.called)
@ -1758,8 +1767,34 @@ netvm default=True type=vm
with unittest.mock.patch.object(
self.vm, 'get_power_state', lambda: 'Running'):
with self.assertRaises(qubes.exc.QubesVMNotHaltedError):
self.call_mgmt_func(b'admin.vm.volume.Import', b'test-vm1',
b'private')
self.call_internal_mgmt_func(
b'internal.vm.volume.ImportBegin', b'test-vm1', b'private')
def test_512_vm_volume_import_with_size(self):
new_size = 4 * 2**30
file_name = '/tmp/qubes-test-dir/appvms/test-vm1/private-import.img'
value = self.call_internal_mgmt_func(
b'internal.vm.volume.ImportBegin', b'test-vm1',
b'private', payload=str(new_size).encode())
self.assertEqual(value, '{} {}'.format(
new_size, file_name))
self.assertFalse(self.app.save.called)
self.assertEqual(os.stat(file_name).st_size, new_size)
def test_515_vm_volume_import_fire_event(self):
self.call_internal_mgmt_func(
b'internal.vm.volume.ImportBegin', b'test-vm1', b'private')
self.assertEventFired(
self.emitter, 'admin-permission:admin.vm.volume.Import')
def test_516_vm_volume_import_fire_event_with_size(self):
self.call_internal_mgmt_func(
b'internal.vm.volume.ImportBegin', b'test-vm1', b'private',
b'123')
self.assertEventFired(
self.emitter, 'admin-permission:admin.vm.volume.ImportWithSize')
def setup_for_clone(self):
self.pool = unittest.mock.MagicMock()

30
qubes/tests/storage_file.py
View File

@ -332,7 +332,7 @@ class TC_01_FileVolumes(qubes.tests.QubesTestCase):
vm = qubes.tests.storage.TestVM(self)
volume = self.app.get_pool(self.POOL_NAME).init_volume(vm, config)
volume.create()
import_path = volume.import_data()
import_path = volume.import_data(volume.size)
self.assertNotEqual(volume.path, import_path)
with open(import_path, 'w+') as import_file:
import_file.write('test')
@ -353,7 +353,7 @@ class TC_01_FileVolumes(qubes.tests.QubesTestCase):
vm = qubes.tests.storage.TestVM(self)
volume = self.app.get_pool(self.POOL_NAME).init_volume(vm, config)
volume.create()
import_path = volume.import_data()
import_path = volume.import_data(volume.size)
self.assertNotEqual(volume.path, import_path)
with open(import_path, 'w+') as import_file:
import_file.write('test')
@ -376,7 +376,7 @@ class TC_01_FileVolumes(qubes.tests.QubesTestCase):
volume.create()
with open(volume.path, 'w') as vol_file:
vol_file.write('test data')
import_path = volume.import_data()
import_path = volume.import_data(volume.size)
self.assertNotEqual(volume.path, import_path)
with open(import_path, 'w+'):
pass
@ -402,6 +402,30 @@ class TC_01_FileVolumes(qubes.tests.QubesTestCase):
self.assertEqual(os.path.getsize(volume.path), new_size)
self.assertEqual(volume.size, new_size)
def test_024_import_data_with_new_size(self):
config = {
'name': 'root',
'pool': self.POOL_NAME,
'save_on_stop': True,
'rw': True,
'size': 1024 * 1024,
}
new_size = 2 * 1024 * 1024
vm = qubes.tests.storage.TestVM(self)
volume = self.app.get_pool(self.POOL_NAME).init_volume(vm, config)
volume.create()
import_path = volume.import_data(new_size)
self.assertNotEqual(volume.path, import_path)
with open(import_path, 'r+b') as import_file:
import_file.write(b'test')
volume.import_data_end(True)
self.assertFalse(os.path.exists(import_path), import_path)
with open(volume.path, 'rb') as volume_file:
volume_data = volume_file.read()
self.assertEqual(volume_data.strip(b'\0'), b'test')
self.assertEqual(len(volume_data), new_size)
def _get_loop_size(self, path):
sudo = [] if os.getuid() == 0 else ['sudo']
try:

31
qubes/tests/storage_lvm.py
View File

@ -698,7 +698,8 @@ class TC_00_ThinPool(ThinPoolBase):
self.loop.run_until_complete(volume.create())
current_uuid = self._get_lv_uuid(volume.path)
self.assertFalse(volume.is_dirty())
import_path = self.loop.run_until_complete(volume.import_data())
import_path = self.loop.run_until_complete(
volume.import_data(volume.size))
import_uuid = self._get_lv_uuid(import_path)
self.assertNotEqual(current_uuid, import_uuid)
# success - commit data
@ -729,7 +730,8 @@ class TC_00_ThinPool(ThinPoolBase):
self.loop.run_until_complete(volume.create())
current_uuid = self._get_lv_uuid(volume.path)
self.assertFalse(volume.is_dirty())
import_path = self.loop.run_until_complete(volume.import_data())
import_path = self.loop.run_until_complete(
volume.import_data(volume.size))
import_uuid = self._get_lv_uuid(import_path)
self.assertNotEqual(current_uuid, import_uuid)
# fail - discard data
@ -860,7 +862,8 @@ class TC_00_ThinPool(ThinPoolBase):
'sudo', 'dd', 'if=/dev/urandom', 'of=' + volume.path, 'count=1', 'bs=1M'
))
self.loop.run_until_complete(p.wait())
import_path = self.loop.run_until_complete(volume.import_data())
import_path = self.loop.run_until_complete(
volume.import_data(volume.size))
self.assertNotEqual(volume.path, import_path)
p = self.loop.run_until_complete(asyncio.create_subprocess_exec(
'sudo', 'touch', import_path))
@ -874,6 +877,28 @@ class TC_00_ThinPool(ThinPoolBase):
volume_data, _ = self.loop.run_until_complete(p.communicate())
self.assertEqual(volume_data.strip(b'\0'), b'')
def test_035_import_data_new_size(self):
''' Test volume import'''
config = {
'name': 'root',
'pool': self.pool.name,
'save_on_stop': True,
'rw': True,
'revisions_to_keep': 2,
'size': qubes.config.defaults['root_img_size'],
}
new_size = 2 * qubes.config.defaults['root_img_size']
vm = qubes.tests.storage.TestVM(self)
volume = self.app.get_pool(self.pool.name).init_volume(vm, config)
self.loop.run_until_complete(volume.create())
self.loop.run_until_complete(
volume.import_data(new_size))
self.loop.run_until_complete(volume.import_data_end(True))
self.assertEqual(volume.size, new_size)
self.loop.run_until_complete(volume.remove())
def test_040_volatile(self):
'''Volatile volume test'''
config = {

2
qubes/tests/storage_reflink.py
View File

@ -128,7 +128,7 @@ class TC_10_ReflinkPool(qubes.tests.QubesTestCase):
self.loop.run_until_complete(volume.create())
with open(volume.export(), 'w') as vol_file:
vol_file.write('test data')
import_path = self.loop.run_until_complete(volume.import_data())
import_path = self.loop.run_until_complete(volume.import_data(volume.size))
self.assertNotEqual(volume.path, import_path)
with open(import_path, 'w+'):
pass