storage_reflink.py 7.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211
  1. #
  2. # The Qubes OS Project, https://www.qubes-os.org
  3. #
  4. # Copyright (C) 2018 Rusty Bird <rustybird@net-c.com>
  5. #
  6. # This library is free software; you can redistribute it and/or
  7. # modify it under the terms of the GNU Lesser General Public
  8. # License as published by the Free Software Foundation; either
  9. # version 2.1 of the License, or (at your option) any later version.
  10. #
  11. # This library is distributed in the hope that it will be useful,
  12. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14. # Lesser General Public License for more details.
  15. #
  16. # You should have received a copy of the GNU Lesser General Public
  17. # License along with this library; if not, see <https://www.gnu.org/licenses/>.
  18. #
  19. ''' Tests for the file-reflink storage driver '''
  20. # pylint: disable=protected-access
  21. # pylint: disable=invalid-name
  22. import os
  23. import shutil
  24. import subprocess
  25. import sys
  26. import qubes.tests
  27. import qubes.tests.storage
  28. from qubes.storage import reflink
  29. class TestApp(qubes.Qubes):
  30. ''' A Mock App object '''
  31. def __init__(self, *args, **kwargs): # pylint: disable=unused-argument
  32. super(TestApp, self).__init__('/tmp/qubes-test.xml', load=False,
  33. offline_mode=True, **kwargs)
  34. self.load_initial_values()
  35. class ReflinkMixin:
  36. def setUp(self, fs_type='btrfs'): # pylint: disable=arguments-differ
  37. super().setUp()
  38. self.test_dir = '/var/tmp/test-reflink-units-on-' + fs_type
  39. mkdir_fs(self.test_dir, fs_type, cleanup_via=self.addCleanup)
  40. def test_000_copy_file(self):
  41. source = os.path.join(self.test_dir, 'source-file')
  42. dest = os.path.join(self.test_dir, 'new-directory', 'dest-file')
  43. content = os.urandom(1024**2)
  44. with open(source, 'wb') as source_io:
  45. source_io.write(content)
  46. ficlone_succeeded = reflink._copy_file(source, dest)
  47. self.assertEqual(ficlone_succeeded, self.ficlone_supported)
  48. self.assertNotEqual(os.stat(source).st_ino, os.stat(dest).st_ino)
  49. with open(source, 'rb') as source_io:
  50. self.assertEqual(source_io.read(), content)
  51. with open(dest, 'rb') as dest_io:
  52. self.assertEqual(dest_io.read(), content)
  53. def test_001_create_and_resize_files_and_update_loopdevs(self):
  54. img_real = os.path.join(self.test_dir, 'img-real')
  55. img_sym = os.path.join(self.test_dir, 'img-sym')
  56. size_initial = 111 * 1024**2
  57. size_resized = 222 * 1024**2
  58. os.symlink(img_real, img_sym)
  59. reflink._create_sparse_file(img_real, size_initial)
  60. stat = os.stat(img_real)
  61. self.assertEqual(stat.st_blocks, 0)
  62. self.assertEqual(stat.st_size, size_initial)
  63. dev_from_real = setup_loopdev(img_real, cleanup_via=self.addCleanup)
  64. dev_from_sym = setup_loopdev(img_sym, cleanup_via=self.addCleanup)
  65. reflink._resize_file(img_real, size_resized)
  66. stat = os.stat(img_real)
  67. self.assertEqual(stat.st_blocks, 0)
  68. self.assertEqual(stat.st_size, size_resized)
  69. reflink_update_loopdev_sizes(os.path.join(self.test_dir, 'unrelated'))
  70. for dev in (dev_from_real, dev_from_sym):
  71. self.assertEqual(get_blockdev_size(dev), size_initial)
  72. reflink_update_loopdev_sizes(img_sym)
  73. for dev in (dev_from_real, dev_from_sym):
  74. self.assertEqual(get_blockdev_size(dev), size_resized)
  75. class TC_10_ReflinkPool(qubes.tests.QubesTestCase):
  76. def setUp(self):
  77. super().setUp()
  78. self.test_dir = '/var/tmp/test-reflink-units-on-btrfs'
  79. pool_conf = {
  80. 'driver': 'file-reflink',
  81. 'dir_path': self.test_dir,
  82. 'name': 'test-btrfs'
  83. }
  84. mkdir_fs(self.test_dir, 'btrfs', cleanup_via=self.addCleanup)
  85. self.app = TestApp()
  86. self.pool = self.loop.run_until_complete(self.app.add_pool(**pool_conf))
  87. self.app.default_pool = self.app.get_pool(pool_conf['name'])
  88. def tearDown(self) -> None:
  89. self.app.default_pool = 'varlibqubes'
  90. self.loop.run_until_complete(self.app.remove_pool(self.pool.name))
  91. del self.pool
  92. self.app.close()
  93. del self.app
  94. super(TC_10_ReflinkPool, self).tearDown()
  95. def test_012_import_data_empty(self):
  96. config = {
  97. 'name': 'root',
  98. 'pool': self.pool.name,
  99. 'save_on_stop': True,
  100. 'rw': True,
  101. 'size': 1024 * 1024,
  102. }
  103. vm = qubes.tests.storage.TestVM(self)
  104. volume = self.pool.init_volume(vm, config)
  105. self.loop.run_until_complete(volume.create())
  106. with open(volume.export(), 'w') as vol_file:
  107. vol_file.write('test data')
  108. import_path = self.loop.run_until_complete(volume.import_data())
  109. self.assertNotEqual(volume.path, import_path)
  110. with open(import_path, 'w+'):
  111. pass
  112. self.loop.run_until_complete(volume.import_data_end(True))
  113. self.assertFalse(os.path.exists(import_path), import_path)
  114. with open(volume.export()) as volume_file:
  115. volume_data = volume_file.read().strip('\0')
  116. self.assertNotEqual(volume_data, 'test data')
  117. class TC_00_ReflinkOnBtrfs(ReflinkMixin, qubes.tests.QubesTestCase):
  118. def setUp(self): # pylint: disable=arguments-differ
  119. super().setUp('btrfs')
  120. self.ficlone_supported = True
  121. class TC_01_ReflinkOnExt4(ReflinkMixin, qubes.tests.QubesTestCase):
  122. def setUp(self): # pylint: disable=arguments-differ
  123. super().setUp('ext4')
  124. self.ficlone_supported = False
  125. def setup_loopdev(img, cleanup_via=None):
  126. dev = str.strip(cmd('sudo', 'losetup', '-f', '--show', img).decode())
  127. if cleanup_via is not None:
  128. cleanup_via(detach_loopdev, dev)
  129. return dev
  130. def detach_loopdev(dev):
  131. cmd('sudo', 'losetup', '-d', dev)
  132. def get_fs_type(directory):
  133. # 'stat -f -c %T' would identify ext4 as 'ext2/ext3'
  134. return cmd('df', '--output=fstype', directory).decode().splitlines()[1]
  135. def mkdir_fs(directory, fs_type,
  136. accessible=True, max_size=100*1024**3, cleanup_via=None):
  137. os.mkdir(directory)
  138. if get_fs_type(directory) != fs_type:
  139. img = os.path.join(directory, 'img')
  140. with open(img, 'xb') as img_io:
  141. img_io.truncate(max_size)
  142. cmd('mkfs.' + fs_type, img)
  143. dev = setup_loopdev(img)
  144. os.remove(img)
  145. cmd('sudo', 'mount', dev, directory)
  146. detach_loopdev(dev)
  147. if accessible:
  148. cmd('sudo', 'chmod', '777', directory)
  149. else:
  150. cmd('sudo', 'chmod', '000', directory)
  151. cmd('sudo', 'chattr', '+i', directory) # cause EPERM on write as root
  152. if cleanup_via is not None:
  153. cleanup_via(rmtree_fs, directory)
  154. def rmtree_fs(directory):
  155. cmd('sudo', 'chattr', '-i', directory)
  156. cmd('sudo', 'chmod', '777', directory)
  157. if os.path.ismount(directory):
  158. cmd('sudo', 'umount', '-l', directory)
  159. # loop device and backing file are garbage collected automatically
  160. shutil.rmtree(directory)
  161. def get_blockdev_size(dev):
  162. return int(cmd('sudo', 'blockdev', '--getsize64', dev))
  163. def reflink_update_loopdev_sizes(img):
  164. env = [k + '=' + v for k, v in os.environ.items() # 'sudo -E' alone would
  165. if k.startswith('PYTHON')] # drop some of these
  166. code = ('from qubes.storage import reflink\n'
  167. 'reflink._update_loopdev_sizes(%r)' % img)
  168. cmd('sudo', '-E', 'env', *env, sys.executable, '-c', code)
  169. def cmd(*argv):
  170. p = subprocess.run(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  171. if p.returncode != 0:
  172. raise Exception(str(p)) # this will show stdout and stderr
  173. return p.stdout