file.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514
  1. #
  2. # The Qubes OS Project, https://www.qubes-os.org/
  3. #
  4. # Copyright (C) 2015 Joanna Rutkowska <joanna@invisiblethingslab.com>
  5. # Copyright (C) 2013-2015 Marek Marczykowski-Górecki
  6. # <marmarek@invisiblethingslab.com>
  7. # Copyright (C) 2015 Wojtek Porczyk <woju@invisiblethingslab.com>
  8. #
  9. # This library is free software; you can redistribute it and/or
  10. # modify it under the terms of the GNU Lesser General Public
  11. # License as published by the Free Software Foundation; either
  12. # version 2.1 of the License, or (at your option) any later version.
  13. #
  14. # This library is distributed in the hope that it will be useful,
  15. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  17. # Lesser General Public License for more details.
  18. #
  19. # You should have received a copy of the GNU Lesser General Public
  20. # License along with this library; if not, see <https://www.gnu.org/licenses/>.
  21. #
  22. ''' This module contains pool implementations backed by file images'''
  23. import asyncio
  24. import os
  25. import os.path
  26. import re
  27. import subprocess
  28. from contextlib import suppress
  29. import qubes.storage
  30. import qubes.utils
  31. BLKSIZE = 512
  32. class FilePool(qubes.storage.Pool):
  33. ''' File based 'original' disk implementation
  34. Volumes are stored in sparse files. Additionally device-mapper is used for
  35. applying copy-on-write layer.
  36. Quick reference on device-mapper layers:
  37. snap_on_start save_on_stop layout
  38. yes yes not supported
  39. no yes snapshot-origin(volume.img, volume-cow.img)
  40. yes no snapshot(
  41. snapshot(source.img, source-cow.img),
  42. volume-cow.img)
  43. no no volume.img directly
  44. ''' # pylint: disable=protected-access
  45. driver = 'file'
  46. def __init__(self, *, name, revisions_to_keep=1, dir_path):
  47. super().__init__(name=name, revisions_to_keep=revisions_to_keep)
  48. self.dir_path = os.path.normpath(dir_path)
  49. self._volumes = []
  50. @property
  51. def config(self):
  52. return {
  53. 'name': self.name,
  54. 'dir_path': self.dir_path,
  55. 'driver': FilePool.driver,
  56. 'revisions_to_keep': self.revisions_to_keep
  57. }
  58. def init_volume(self, vm, volume_config):
  59. if volume_config.get('snap_on_start', False) and \
  60. volume_config.get('save_on_stop', False):
  61. raise NotImplementedError(
  62. 'snap_on_start + save_on_stop not supported by file driver')
  63. volume_config['dir_path'] = self.dir_path
  64. if 'vid' not in volume_config:
  65. volume_config['vid'] = os.path.join(
  66. self._vid_prefix(vm), volume_config['name'])
  67. try:
  68. if not volume_config.get('save_on_stop', False):
  69. volume_config['revisions_to_keep'] = 0
  70. except KeyError:
  71. pass
  72. if 'revisions_to_keep' not in volume_config:
  73. volume_config['revisions_to_keep'] = self.revisions_to_keep
  74. volume_config['pool'] = self
  75. volume = FileVolume(**volume_config)
  76. self._volumes += [volume]
  77. return volume
  78. @property
  79. def revisions_to_keep(self):
  80. return self._revisions_to_keep
  81. @revisions_to_keep.setter
  82. def revisions_to_keep(self, value):
  83. value = int(value)
  84. if value > 1:
  85. raise NotImplementedError(
  86. 'FilePool supports maximum 1 volume revision to keep')
  87. self._revisions_to_keep = value
  88. def destroy(self):
  89. pass
  90. def setup(self):
  91. create_dir_if_not_exists(self.dir_path)
  92. appvms_path = os.path.join(self.dir_path, 'appvms')
  93. create_dir_if_not_exists(appvms_path)
  94. vm_templates_path = os.path.join(self.dir_path, 'vm-templates')
  95. create_dir_if_not_exists(vm_templates_path)
  96. @staticmethod
  97. def _vid_prefix(vm):
  98. ''' Helper to create a prefix for the vid for volume
  99. ''' # FIX Remove this if we drop the file backend
  100. import qubes.vm.templatevm # pylint: disable=redefined-outer-name
  101. import qubes.vm.dispvm # pylint: disable=redefined-outer-name
  102. if isinstance(vm, qubes.vm.templatevm.TemplateVM):
  103. subdir = 'vm-templates'
  104. else:
  105. subdir = 'appvms'
  106. return os.path.join(subdir, vm.name)
  107. def target_dir(self, vm):
  108. """ Returns the path to vmdir depending on the type of the VM.
  109. The default QubesOS file storage saves the vm images in three
  110. different directories depending on the ``QubesVM`` type:
  111. * ``appvms`` for ``QubesAppVm`` or ``QubesHvm``
  112. * ``vm-templates`` for ``QubesTemplateVm`` or ``QubesTemplateHvm``
  113. Args:
  114. vm: a QubesVM
  115. pool_dir: the root directory of the pool
  116. Returns:
  117. string (str) absolute path to the directory where the vm files
  118. are stored
  119. """
  120. return os.path.join(self.dir_path, self._vid_prefix(vm))
  121. def list_volumes(self):
  122. return self._volumes
  123. @property
  124. def size(self):
  125. try:
  126. statvfs = os.statvfs(self.dir_path)
  127. return statvfs.f_frsize * statvfs.f_blocks
  128. except FileNotFoundError:
  129. return 0
  130. @property
  131. def usage(self):
  132. try:
  133. statvfs = os.statvfs(self.dir_path)
  134. return statvfs.f_frsize * (statvfs.f_blocks - statvfs.f_bfree)
  135. except FileNotFoundError:
  136. return 0
  137. def included_in(self, app):
  138. ''' Check if there is pool containing this one - either as a
  139. filesystem or its LVM volume'''
  140. return qubes.storage.search_pool_containing_dir(
  141. [pool for pool in app.pools.values() if pool is not self],
  142. self.dir_path)
  143. class FileVolume(qubes.storage.Volume):
  144. ''' Parent class for the xen volumes implementation which expects a
  145. `target_dir` param on initialization. '''
  146. def __init__(self, dir_path, **kwargs):
  147. self.dir_path = dir_path
  148. assert self.dir_path, "dir_path not specified"
  149. self._revisions_to_keep = 0
  150. super().__init__(**kwargs)
  151. if self.snap_on_start:
  152. img_name = self.source.vid + '-cow.img'
  153. self.path_source_cow = os.path.join(self.dir_path, img_name)
  154. @property
  155. def revisions_to_keep(self):
  156. return self._revisions_to_keep
  157. @revisions_to_keep.setter
  158. def revisions_to_keep(self, value):
  159. if int(value) > 1:
  160. raise NotImplementedError(
  161. 'FileVolume supports maximum 1 volume revision to keep')
  162. self._revisions_to_keep = int(value)
  163. def create(self):
  164. assert isinstance(self.size, int) and self.size > 0, \
  165. 'Volume size must be > 0'
  166. if not self.snap_on_start:
  167. create_sparse_file(self.path, self.size)
  168. def remove(self):
  169. if not self.snap_on_start:
  170. _remove_if_exists(self.path)
  171. if self.snap_on_start or self.save_on_stop:
  172. _remove_if_exists(self.path_cow)
  173. def is_dirty(self):
  174. if not self.save_on_stop:
  175. return False
  176. if os.path.exists(self.path_cow):
  177. stat = os.stat(self.path_cow)
  178. return stat.st_blocks > 0
  179. return False
  180. def resize(self, size):
  181. ''' Expands volume, throws
  182. :py:class:`qubst.storage.qubes.storage.StoragePoolException` if
  183. given size is less than current_size
  184. ''' # pylint: disable=no-self-use
  185. if not self.rw:
  186. msg = 'Can not resize reađonly volume {!s}'.format(self)
  187. raise qubes.storage.StoragePoolException(msg)
  188. if size < self.size:
  189. raise qubes.storage.StoragePoolException(
  190. 'For your own safety, shrinking of %s is'
  191. ' disabled. If you really know what you'
  192. ' are doing, use `truncate` on %s manually.' %
  193. (self.name, self.vid))
  194. with open(self.path, 'a+b') as fd:
  195. fd.truncate(size)
  196. p = subprocess.Popen(['losetup', '--associated', self.path],
  197. stdout=subprocess.PIPE)
  198. result = p.communicate()
  199. m = re.match(r'^(/dev/loop\d+):\s', result[0].decode())
  200. if m is not None:
  201. loop_dev = m.group(1)
  202. # resize loop device
  203. subprocess.check_call(['losetup', '--set-capacity',
  204. loop_dev])
  205. self._size = size
  206. def commit(self):
  207. msg = 'Tried to commit a non commitable volume {!r}'.format(self)
  208. assert self.save_on_stop and self.rw, msg
  209. if os.path.exists(self.path_cow):
  210. if self.revisions_to_keep:
  211. old_path = self.path_cow + '.old'
  212. os.rename(self.path_cow, old_path)
  213. else:
  214. os.unlink(self.path_cow)
  215. create_sparse_file(self.path_cow, self.size)
  216. return self
  217. def export(self):
  218. # FIXME: this should rather return snapshot(self.path, self.path_cow)
  219. # if domain is running
  220. return self.path
  221. @asyncio.coroutine
  222. def import_volume(self, src_volume):
  223. if src_volume.snap_on_start:
  224. raise qubes.storage.StoragePoolException(
  225. "Can not import snapshot volume {!s} in to pool {!s} ".format(
  226. src_volume, self))
  227. if self.save_on_stop:
  228. _remove_if_exists(self.path)
  229. path = yield from qubes.utils.coro_maybe(src_volume.export())
  230. try:
  231. copy_file(path, self.path)
  232. finally:
  233. yield from qubes.utils.coro_maybe(src_volume.export_end(path))
  234. return self
  235. def import_data(self, size):
  236. if not self.save_on_stop:
  237. raise qubes.storage.StoragePoolException(
  238. "Can not import into save_on_stop=False volume {!s}".format(
  239. self))
  240. create_sparse_file(self.path_import, size)
  241. return self.path_import
  242. def import_data_end(self, success):
  243. if success:
  244. os.rename(self.path_import, self.path)
  245. else:
  246. os.unlink(self.path_import)
  247. return self
  248. def reset(self):
  249. ''' Remove and recreate a volatile volume '''
  250. assert not self.snap_on_start and not self.save_on_stop, \
  251. "Not a volatile volume"
  252. assert isinstance(self.size, int) and self.size > 0, \
  253. 'Volatile volume size must be > 0'
  254. _remove_if_exists(self.path)
  255. create_sparse_file(self.path, self.size)
  256. return self
  257. def start(self):
  258. if not self.save_on_stop and not self.snap_on_start:
  259. self.reset()
  260. else:
  261. if not self.save_on_stop:
  262. # make sure previous snapshot is removed - even if VM
  263. # shutdown routine wasn't called (power interrupt or so)
  264. _remove_if_exists(self.path_cow)
  265. if not os.path.exists(self.path_cow):
  266. create_sparse_file(self.path_cow, self.size)
  267. if not self.snap_on_start:
  268. _check_path(self.path)
  269. if hasattr(self, 'path_source_cow'):
  270. if not os.path.exists(self.path_source_cow):
  271. create_sparse_file(self.path_source_cow, self.size)
  272. return self
  273. def stop(self):
  274. if self.save_on_stop:
  275. self.commit()
  276. elif self.snap_on_start:
  277. _remove_if_exists(self.path_cow)
  278. else:
  279. _remove_if_exists(self.path)
  280. return self
  281. @property
  282. def path(self):
  283. if self.snap_on_start:
  284. return os.path.join(self.dir_path, self.source.vid + '.img')
  285. return os.path.join(self.dir_path, self.vid + '.img')
  286. @property
  287. def path_cow(self):
  288. img_name = self.vid + '-cow.img'
  289. return os.path.join(self.dir_path, img_name)
  290. @property
  291. def path_import(self):
  292. img_name = self.vid + '-import.img'
  293. return os.path.join(self.dir_path, img_name)
  294. def verify(self):
  295. ''' Verifies the volume. '''
  296. if not os.path.exists(self.path) and \
  297. (self.snap_on_start or self.save_on_stop):
  298. msg = 'Missing image file: {!s}.'.format(self.path)
  299. raise qubes.storage.StoragePoolException(msg)
  300. return True
  301. @property
  302. def script(self):
  303. if not self.snap_on_start and not self.save_on_stop:
  304. return None
  305. if not self.snap_on_start and self.save_on_stop:
  306. return 'block-origin'
  307. if self.snap_on_start:
  308. return 'block-snapshot'
  309. return None
  310. def block_device(self):
  311. ''' Return :py:class:`qubes.storage.BlockDevice` for serialization in
  312. the libvirt XML template as <disk>.
  313. '''
  314. path = self.path
  315. if self.snap_on_start:
  316. path += ":" + self.path_source_cow
  317. if self.snap_on_start or self.save_on_stop:
  318. path += ":" + self.path_cow
  319. return qubes.storage.BlockDevice(path, self.name, self.script, self.rw,
  320. self.domain, self.devtype)
  321. @property
  322. def revisions(self):
  323. if not hasattr(self, 'path_cow'):
  324. return {}
  325. old_revision = self.path_cow + '.old' # pylint: disable=no-member
  326. if not os.path.exists(old_revision):
  327. return {}
  328. seconds = os.path.getctime(old_revision)
  329. iso_date = qubes.storage.isodate(seconds).split('.', 1)[0]
  330. return {'old': iso_date}
  331. @property
  332. def size(self):
  333. with suppress(FileNotFoundError):
  334. self._size = os.path.getsize(self.path)
  335. return self._size
  336. @size.setter
  337. def size(self, _):
  338. raise qubes.storage.StoragePoolException(
  339. "You shouldn't use volume size setter, use resize method instead")
  340. @property
  341. def usage(self):
  342. ''' Returns the actualy used space '''
  343. usage = 0
  344. if self.save_on_stop or self.snap_on_start:
  345. usage = get_disk_usage(self.path_cow)
  346. if self.save_on_stop or not self.snap_on_start:
  347. usage += get_disk_usage(self.path)
  348. return usage
  349. def create_sparse_file(path, size):
  350. ''' Create an empty sparse file '''
  351. if os.path.exists(path):
  352. raise IOError("Volume %s already exists" % path)
  353. parent_dir = os.path.dirname(path)
  354. if not os.path.exists(parent_dir):
  355. os.makedirs(parent_dir)
  356. with open(path, 'a+b') as fh:
  357. fh.truncate(size)
  358. def get_disk_usage_one(st):
  359. '''Extract disk usage of one inode from its stat_result struct.
  360. If known, get real disk usage, as written to device by filesystem, not
  361. logical file size. Those values may be different for sparse files.
  362. :param os.stat_result st: stat result
  363. :returns: disk usage
  364. '''
  365. try:
  366. return st.st_blocks * BLKSIZE
  367. except AttributeError:
  368. return st.st_size
  369. def get_disk_usage(path):
  370. '''Get real disk usage of given path (file or directory).
  371. When *path* points to directory, then it is evaluated recursively.
  372. This function tries estimate real disk usage. See documentation of
  373. :py:func:`get_disk_usage_one`.
  374. :param str path: path to evaluate
  375. :returns: disk usage
  376. '''
  377. try:
  378. st = os.lstat(path)
  379. except OSError:
  380. return 0
  381. ret = get_disk_usage_one(st)
  382. # if path is not a directory, this is skipped
  383. for dirpath, dirnames, filenames in os.walk(path):
  384. for name in dirnames + filenames:
  385. ret += get_disk_usage_one(os.lstat(os.path.join(dirpath, name)))
  386. return ret
  387. def create_dir_if_not_exists(path):
  388. """ Check if a directory exists in if not create it.
  389. This method does not create any parent directories.
  390. """
  391. if not os.path.exists(path):
  392. os.mkdir(path)
  393. def copy_file(source, destination):
  394. '''Effective file copy, preserving sparse files etc.'''
  395. # We prefer to use Linux's cp, because it nicely handles sparse files
  396. assert os.path.exists(source), \
  397. "Missing the source %s to copy from" % source
  398. assert not os.path.exists(destination), \
  399. "Destination %s already exists" % destination
  400. parent_dir = os.path.dirname(destination)
  401. if not os.path.exists(parent_dir):
  402. os.makedirs(parent_dir)
  403. try:
  404. cmd = ['cp', '--sparse=always',
  405. '--reflink=auto', source, destination]
  406. subprocess.check_call(cmd)
  407. except subprocess.CalledProcessError:
  408. raise IOError('Error while copying {!r} to {!r}'.format(source,
  409. destination))
  410. def _remove_if_exists(path):
  411. ''' Removes a file if it exist, silently succeeds if file does not exist '''
  412. if os.path.exists(path):
  413. os.remove(path)
  414. def _check_path(path):
  415. ''' Raise an StoragePoolException if ``path`` does not exist'''
  416. if not os.path.exists(path):
  417. msg = 'Missing image file: %s' % path
  418. raise qubes.storage.StoragePoolException(msg)