Qubes/core-admin-client
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

tests: restoring a backup bigger than available space in /var/tmp

Browse Source 
This test uses three tricks to test /var/tmp space monitoring:
1. Creates a big uncompressed backup (2GB file instead of few bytes)
2. Mount small tmpfs over /var/tmp (650MB - minimal space that should
   not deadlock the restore)
3. Artificially slow down data processing by adding sleep()

QubesOS/qubes-issues#4791
This commit is contained in:
Marek Marczykowski-Górecki 2021-01-05 04:25:14 +01:00
parent af13c198be
commit 0f42fd0580
No known key found for this signature in database
GPG Key ID: 063938BA42CFA724
2 changed files with 139 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
qubesadmin/tests/backup/__init__.py
View File

@ -169,17 +169,23 @@ class BackupTestCase(qubesadmin.tests.QubesTestCase):
def restore_backup(self, source=None, appvm=None, options=None,
expect_errors=None, manipulate_restore_info=None,
passphrase='qubes', force_compression_filter=None):
passphrase='qubes', force_compression_filter=None,
tmpdir=None):
if source is None:
backupfile = os.path.join(self.backupdir,
sorted(os.listdir(self.backupdir))[-1])
else:
backupfile = source
kwargs = {}
if tmpdir:
kwargs['tmpdir'] = tmpdir
with self.assertNotRaises(qubesadmin.exc.QubesException):
restore_op = qubesadmin.backup.restore.BackupRestore(
self.app, backupfile, appvm, passphrase,
force_compression_filter=force_compression_filter)
force_compression_filter=force_compression_filter,
**kwargs)
if options:
for key, value in options.items():
setattr(restore_op.options, key, value)
@ -215,6 +221,16 @@ class BackupTestCase(qubesadmin.tests.QubesTestCase):
f.truncate(size)
f.close()
def create_full_image(self, path, size, signature=b''):
f = open(path, "wb")
f.write(signature)
f.write(b'\0' * (SIGNATURE_LEN - len(signature)))
block_size = 1024 ** 2
f.write(b'\0' * (block_size - SIGNATURE_LEN))
for _ in range(size // block_size - 1):
f.write(b'\1' * block_size)
f.close()
def vm_checksum(self, vms):
hashes = {}
for vm in vms:

136
qubesadmin/tests/backup/backupcompatibility.py
View File

@ -775,13 +775,16 @@ class TC_00_QubesXML(qubesadmin.tests.QubesTestCase):
# backup code use multiprocessing, synchronize with main process
class AppProxy(object):
def __init__(self, app, sync_queue):
def __init__(self, app, sync_queue, delay_stream=0):
self._app = app
self._sync_queue = sync_queue
self._delay_stream = delay_stream
self.cache_enabled = False
def qubesd_call(self, dest, method, arg=None, payload=None,
payload_stream=None):
if payload_stream:
time.sleep(self._delay_stream)
signature_bin = payload_stream.read(512)
payload = signature_bin.split(b'\0', 1)[0]
subprocess.call(['cat'], stdin=payload_stream,
@ -792,9 +795,10 @@ class AppProxy(object):
class MockVolume(qubesadmin.storage.Volume):
def __init__(self, import_data_queue, *args, **kwargs):
def __init__(self, import_data_queue, delay_stream, *args, **kwargs):
super(MockVolume, self).__init__(*args, **kwargs)
self.app = AppProxy(self.app, import_data_queue)
self.app = AppProxy(self.app, import_data_queue,
delay_stream=delay_stream)
class MockFirewall(qubesadmin.firewall.Firewall):
def __init__(self, import_data_queue, *args, **kwargs):
@ -824,13 +828,17 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
with open(os.path.join(dir, name + ".desktop"), "w") as f:
f.write(template.format(name=name, comment=name, command=name))
def create_private_img(self, filename):
def create_private_img(self, filename, sparse=True):
signature = '/'.join(os.path.splitext(filename)[0].split('/')[-2:])
self.create_sparse(filename, 2*2**20, signature=signature.encode())
if sparse:
self.create_sparse(filename, 2*2**20, signature=signature.encode())
else:
self.create_full_image(filename, 2 * 2 ** 30,
signature=signature.encode())
#subprocess.check_call(["/usr/sbin/mkfs.ext4", "-q", "-F", filename])
def create_volatile_img(self, filename):
self.create_sparse(filename, 11.5*2**20)
self.create_sparse(filename, int(11.5*2**20))
# here used to be sfdisk call with "0,1024,S\n,10240,L\n" input,
# but since sfdisk folks like to change command arguments in
# incompatible way, have an partition table verbatim here
@ -1273,11 +1281,12 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
output.close()
def create_v4_backup(self, compressed="gzip"):
def create_v4_backup(self, compressed="gzip", big=False):
"""
Create "backup format 4" backup - used in R4.0
:param compressed: Should the backup be compressed
:param big: Should the backup include big(ish) VM?
:return:
"""
output = open(self.fullpath("backup.bin"), "w")
@ -1301,6 +1310,12 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
self.handle_v4_file("qubes.xml", "", output, compressed=compressed)
self.create_v4_files()
if big:
# make one AppVM non-sparse
self.create_private_img(
self.fullpath('appvms/test-work/private.img'),
sparse=False)
for vm_type in ["appvms", "vm-templates"]:
for vm_name in os.listdir(self.fullpath(vm_type)):
vm_dir = os.path.join(vm_type, vm_name)
@ -1470,6 +1485,17 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
def mock_appmenus(self, queue, vm, stream):
queue.put((vm.name, 'appmenus', None, stream.read()))
def cleanup_tmpdir(self, tmpdir: tempfile.TemporaryDirectory):
subprocess.run(['sudo', 'umount', tmpdir.name], check=True)
tmpdir.cleanup()
def create_limited_tmpdir(self, size):
d = tempfile.TemporaryDirectory()
subprocess.run(['sudo', 'mount', '-t', 'tmpfs', 'none', d.name, '-o',
'size={}'.format(size)], check=True)
self.addCleanup(self.cleanup_tmpdir, d)
return d.name
def test_210_r2(self):
self.create_v3_backup(False)
self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = (
@ -1504,7 +1530,7 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
patches = [
mock.patch('qubesadmin.storage.Volume',
functools.partial(MockVolume, qubesd_calls_queue)),
functools.partial(MockVolume, qubesd_calls_queue, 0)),
mock.patch(
'qubesadmin.backup.restore.BackupRestore._handle_appmenus_list',
functools.partial(self.mock_appmenus, qubesd_calls_queue)),
@ -1572,7 +1598,7 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
patches = [
mock.patch('qubesadmin.storage.Volume',
functools.partial(MockVolume, qubesd_calls_queue)),
functools.partial(MockVolume, qubesd_calls_queue, 0)),
mock.patch(
'qubesadmin.backup.restore.BackupRestore._handle_appmenus_list',
functools.partial(self.mock_appmenus, qubesd_calls_queue)),
@ -1639,7 +1665,7 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
patches = [
mock.patch('qubesadmin.storage.Volume',
functools.partial(MockVolume, qubesd_calls_queue)),
functools.partial(MockVolume, qubesd_calls_queue, 0)),
mock.patch(
'qubesadmin.backup.restore.BackupRestore._handle_appmenus_list',
functools.partial(self.mock_appmenus, qubesd_calls_queue)),
@ -1708,7 +1734,7 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
patches = [
mock.patch('qubesadmin.storage.Volume',
functools.partial(MockVolume, qubesd_calls_queue)),
functools.partial(MockVolume, qubesd_calls_queue, 0)),
mock.patch(
'qubesadmin.backup.restore.BackupRestore._handle_appmenus_list',
functools.partial(self.mock_appmenus, qubesd_calls_queue)),
@ -1779,7 +1805,7 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
patches = [
mock.patch('qubesadmin.storage.Volume',
functools.partial(MockVolume, qubesd_calls_queue)),
functools.partial(MockVolume, qubesd_calls_queue, 0)),
mock.patch(
'qubesadmin.backup.restore.BackupRestore._handle_appmenus_list',
functools.partial(self.mock_appmenus, qubesd_calls_queue)),
@ -1850,7 +1876,7 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
patches = [
mock.patch('qubesadmin.storage.Volume',
functools.partial(MockVolume, qubesd_calls_queue)),
functools.partial(MockVolume, qubesd_calls_queue, 0)),
mock.patch(
'qubesadmin.backup.restore.BackupRestore._handle_appmenus_list',
functools.partial(self.mock_appmenus, qubesd_calls_queue)),
@ -1893,7 +1919,7 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
patches = [
mock.patch('qubesadmin.storage.Volume',
functools.partial(MockVolume, qubesd_calls_queue)),
functools.partial(MockVolume, qubesd_calls_queue, 0)),
mock.patch(
'qubesadmin.backup.restore.BackupRestore._handle_appmenus_list',
functools.partial(self.mock_appmenus, qubesd_calls_queue)),
@ -1952,7 +1978,7 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
patches = [
mock.patch('qubesadmin.storage.Volume',
functools.partial(MockVolume, qubesd_calls_queue)),
functools.partial(MockVolume, qubesd_calls_queue, 0)),
mock.patch(
'qubesadmin.backup.restore.BackupRestore._handle_appmenus_list',
functools.partial(self.mock_appmenus, qubesd_calls_queue)),
@ -1985,6 +2011,86 @@ class TC_10_BackupCompatibility(qubesadmin.tests.backup.BackupTestCase):
self.assertDom0Restored(dummy_timestamp)
@unittest.skipUnless(spawn.find_executable('scrypt'),
"scrypt not installed")
def test_300_r4_no_space(self):
self.create_v4_backup("", big=True)
self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = (
b'0\0dom0 class=AdminVM state=Running\n'
b'fedora-25 class=TemplateVM state=Halted\n'
b'testvm class=AppVM state=Running\n'
b'sys-net class=AppVM state=Running\n'
)
self.app.expected_calls[
('dom0', 'admin.property.Get', 'default_template', None)] = \
b'0\0default=no type=vm fedora-25'
self.app.expected_calls[
('sys-net', 'admin.vm.property.Get', 'provides_network', None)] = \
b'0\0default=no type=bool True'
self.setup_expected_calls(parsed_qubes_xml_v4, templates_map={
'debian-8': 'fedora-25'
})
firewall_data = (
'action=accept specialtarget=dns\n'
'action=accept proto=icmp\n'
'action=accept proto=tcp dstports=22-22\n'
'action=accept proto=tcp dsthost=www.qubes-os.org '
'dstports=443-443\n'
'action=accept proto=tcp dst4=192.168.0.0/24\n'
'action=drop\n'
)
self.app.expected_calls[
('test-work', 'admin.vm.firewall.Set', None,
firewall_data.encode())] = b'0\0'
del self.app.expected_calls[
('test-work', 'admin.vm.volume.Resize', 'private', b'2097152')]
self.app.expected_calls[
('test-work', 'admin.vm.volume.Resize', 'private', b'2147483648')] = \
b'0\0'
qubesd_calls_queue = multiprocessing.Queue()
dummy_timestamp = time.strftime("test-%Y-%m-%d-%H%M%S")
patches = [
mock.patch('qubesadmin.storage.Volume',
functools.partial(MockVolume, qubesd_calls_queue, 30)),
mock.patch(
'qubesadmin.backup.restore.BackupRestore._handle_appmenus_list',
functools.partial(self.mock_appmenus, qubesd_calls_queue)),
mock.patch(
'qubesadmin.firewall.Firewall',
functools.partial(MockFirewall, qubesd_calls_queue)),
mock.patch(
'time.strftime',
return_value=dummy_timestamp),
mock.patch(
'qubesadmin.backup.restore.BackupRestore.check_disk_space')
]
small_tmpdir = self.create_limited_tmpdir('620M')
for patch in patches:
patch.start()
try:
self.restore_backup(self.fullpath("backup.bin"),
tmpdir=small_tmpdir,
options={
'use-default-template': True,
'use-default-netvm': True,
})
finally:
for patch in patches:
patch.stop()
# retrieve calls from other multiprocess.Process instances
while not qubesd_calls_queue.empty():
call_args = qubesd_calls_queue.get()
with contextlib.suppress(qubesadmin.exc.QubesException):
self.app.qubesd_call(*call_args)
qubesd_calls_queue.close()
self.assertAllCalled()
self.assertDom0Restored(dummy_timestamp)
class TC_11_BackupCompatibilityIntoLVM(TC_10_BackupCompatibility):
storage_pool = 'some-pool'