file.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513
  1. #
  2. # The Qubes OS Project, https://www.qubes-os.org/
  3. #
  4. # Copyright (C) 2015 Joanna Rutkowska <joanna@invisiblethingslab.com>
  5. # Copyright (C) 2013-2015 Marek Marczykowski-Górecki
  6. # <marmarek@invisiblethingslab.com>
  7. # Copyright (C) 2015 Wojtek Porczyk <woju@invisiblethingslab.com>
  8. #
  9. # This library is free software; you can redistribute it and/or
  10. # modify it under the terms of the GNU Lesser General Public
  11. # License as published by the Free Software Foundation; either
  12. # version 2.1 of the License, or (at your option) any later version.
  13. #
  14. # This library is distributed in the hope that it will be useful,
  15. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  17. # Lesser General Public License for more details.
  18. #
  19. # You should have received a copy of the GNU Lesser General Public
  20. # License along with this library; if not, see <https://www.gnu.org/licenses/>.
  21. #
  22. ''' This module contains pool implementations backed by file images'''
  23. from __future__ import absolute_import
  24. import os
  25. import os.path
  26. import re
  27. import subprocess
  28. from contextlib import suppress
  29. import qubes.storage
  30. BLKSIZE = 512
  31. class FilePool(qubes.storage.Pool):
  32. ''' File based 'original' disk implementation
  33. Volumes are stored in sparse files. Additionally device-mapper is used for
  34. applying copy-on-write layer.
  35. Quick reference on device-mapper layers:
  36. snap_on_start save_on_stop layout
  37. yes yes not supported
  38. no yes snapshot-origin(volume.img, volume-cow.img)
  39. yes no snapshot(
  40. snapshot(source.img, source-cow.img),
  41. volume-cow.img)
  42. no no volume.img directly
  43. ''' # pylint: disable=protected-access
  44. driver = 'file'
  45. def __init__(self, revisions_to_keep=1, dir_path=None, **kwargs):
  46. self._revisions_to_keep = 0
  47. super(FilePool, self).__init__(revisions_to_keep=revisions_to_keep,
  48. **kwargs)
  49. assert dir_path, "No pool dir_path specified"
  50. self.dir_path = os.path.normpath(dir_path)
  51. self._volumes = []
  52. @property
  53. def config(self):
  54. return {
  55. 'name': self.name,
  56. 'dir_path': self.dir_path,
  57. 'driver': FilePool.driver,
  58. 'revisions_to_keep': self.revisions_to_keep
  59. }
  60. def init_volume(self, vm, volume_config):
  61. if volume_config.get('snap_on_start', False) and \
  62. volume_config.get('save_on_stop', False):
  63. raise NotImplementedError(
  64. 'snap_on_start + save_on_stop not supported by file driver')
  65. volume_config['dir_path'] = self.dir_path
  66. if 'vid' not in volume_config:
  67. volume_config['vid'] = os.path.join(
  68. self._vid_prefix(vm), volume_config['name'])
  69. try:
  70. if not volume_config.get('save_on_stop', False):
  71. volume_config['revisions_to_keep'] = 0
  72. except KeyError:
  73. pass
  74. if 'revisions_to_keep' not in volume_config:
  75. volume_config['revisions_to_keep'] = self.revisions_to_keep
  76. volume_config['pool'] = self
  77. volume = FileVolume(**volume_config)
  78. self._volumes += [volume]
  79. return volume
  80. @property
  81. def revisions_to_keep(self):
  82. return self._revisions_to_keep
  83. @revisions_to_keep.setter
  84. def revisions_to_keep(self, value):
  85. value = int(value)
  86. if value > 1:
  87. raise NotImplementedError(
  88. 'FilePool supports maximum 1 volume revision to keep')
  89. self._revisions_to_keep = value
  90. def destroy(self):
  91. pass
  92. def setup(self):
  93. create_dir_if_not_exists(self.dir_path)
  94. appvms_path = os.path.join(self.dir_path, 'appvms')
  95. create_dir_if_not_exists(appvms_path)
  96. vm_templates_path = os.path.join(self.dir_path, 'vm-templates')
  97. create_dir_if_not_exists(vm_templates_path)
  98. @staticmethod
  99. def _vid_prefix(vm):
  100. ''' Helper to create a prefix for the vid for volume
  101. ''' # FIX Remove this if we drop the file backend
  102. import qubes.vm.templatevm # pylint: disable=redefined-outer-name
  103. import qubes.vm.dispvm # pylint: disable=redefined-outer-name
  104. if isinstance(vm, qubes.vm.templatevm.TemplateVM):
  105. subdir = 'vm-templates'
  106. else:
  107. subdir = 'appvms'
  108. return os.path.join(subdir, vm.name)
  109. def target_dir(self, vm):
  110. """ Returns the path to vmdir depending on the type of the VM.
  111. The default QubesOS file storage saves the vm images in three
  112. different directories depending on the ``QubesVM`` type:
  113. * ``appvms`` for ``QubesAppVm`` or ``QubesHvm``
  114. * ``vm-templates`` for ``QubesTemplateVm`` or ``QubesTemplateHvm``
  115. Args:
  116. vm: a QubesVM
  117. pool_dir: the root directory of the pool
  118. Returns:
  119. string (str) absolute path to the directory where the vm files
  120. are stored
  121. """
  122. return os.path.join(self.dir_path, self._vid_prefix(vm))
  123. def list_volumes(self):
  124. return self._volumes
  125. @property
  126. def size(self):
  127. try:
  128. statvfs = os.statvfs(self.dir_path)
  129. return statvfs.f_frsize * statvfs.f_blocks
  130. except FileNotFoundError:
  131. return 0
  132. @property
  133. def usage(self):
  134. try:
  135. statvfs = os.statvfs(self.dir_path)
  136. return statvfs.f_frsize * (statvfs.f_blocks - statvfs.f_bfree)
  137. except FileNotFoundError:
  138. return 0
  139. def included_in(self, app):
  140. ''' Check if there is pool containing this one - either as a
  141. filesystem or its LVM volume'''
  142. return qubes.storage.search_pool_containing_dir(
  143. [pool for pool in app.pools.values() if pool is not self],
  144. self.dir_path)
  145. class FileVolume(qubes.storage.Volume):
  146. ''' Parent class for the xen volumes implementation which expects a
  147. `target_dir` param on initialization. '''
  148. def __init__(self, dir_path, **kwargs):
  149. self.dir_path = dir_path
  150. assert self.dir_path, "dir_path not specified"
  151. self._revisions_to_keep = 0
  152. super(FileVolume, self).__init__(**kwargs)
  153. if self.snap_on_start:
  154. img_name = self.source.vid + '-cow.img'
  155. self.path_source_cow = os.path.join(self.dir_path, img_name)
  156. @property
  157. def revisions_to_keep(self):
  158. return self._revisions_to_keep
  159. @revisions_to_keep.setter
  160. def revisions_to_keep(self, value):
  161. if int(value) > 1:
  162. raise NotImplementedError(
  163. 'FileVolume supports maximum 1 volume revision to keep')
  164. self._revisions_to_keep = int(value)
  165. def create(self):
  166. assert isinstance(self.size, int) and self.size > 0, \
  167. 'Volume size must be > 0'
  168. if not self.snap_on_start:
  169. create_sparse_file(self.path, self.size)
  170. def remove(self):
  171. if not self.snap_on_start:
  172. _remove_if_exists(self.path)
  173. if self.snap_on_start or self.save_on_stop:
  174. _remove_if_exists(self.path_cow)
  175. def is_dirty(self):
  176. if not self.save_on_stop:
  177. return False
  178. if os.path.exists(self.path_cow):
  179. stat = os.stat(self.path_cow)
  180. return stat.st_blocks > 0
  181. return False
  182. def resize(self, size):
  183. ''' Expands volume, throws
  184. :py:class:`qubst.storage.qubes.storage.StoragePoolException` if
  185. given size is less than current_size
  186. ''' # pylint: disable=no-self-use
  187. if not self.rw:
  188. msg = 'Can not resize reađonly volume {!s}'.format(self)
  189. raise qubes.storage.StoragePoolException(msg)
  190. if size < self.size:
  191. raise qubes.storage.StoragePoolException(
  192. 'For your own safety, shrinking of %s is'
  193. ' disabled. If you really know what you'
  194. ' are doing, use `truncate` on %s manually.' %
  195. (self.name, self.vid))
  196. with open(self.path, 'a+b') as fd:
  197. fd.truncate(size)
  198. p = subprocess.Popen(['losetup', '--associated', self.path],
  199. stdout=subprocess.PIPE)
  200. result = p.communicate()
  201. m = re.match(r'^(/dev/loop\d+):\s', result[0].decode())
  202. if m is not None:
  203. loop_dev = m.group(1)
  204. # resize loop device
  205. subprocess.check_call(['losetup', '--set-capacity',
  206. loop_dev])
  207. self._size = size
  208. def commit(self):
  209. msg = 'Tried to commit a non commitable volume {!r}'.format(self)
  210. assert self.save_on_stop and self.rw, msg
  211. if os.path.exists(self.path_cow):
  212. if self.revisions_to_keep:
  213. old_path = self.path_cow + '.old'
  214. os.rename(self.path_cow, old_path)
  215. else:
  216. os.unlink(self.path_cow)
  217. create_sparse_file(self.path_cow, self.size)
  218. return self
  219. def export(self):
  220. # FIXME: this should rather return snapshot(self.path, self.path_cow)
  221. # if domain is running
  222. return self.path
  223. def import_volume(self, src_volume):
  224. if src_volume.snap_on_start:
  225. raise qubes.storage.StoragePoolException(
  226. "Can not import snapshot volume {!s} in to pool {!s} ".format(
  227. src_volume, self))
  228. if self.save_on_stop:
  229. _remove_if_exists(self.path)
  230. copy_file(src_volume.export(), self.path)
  231. return self
  232. def import_data(self, size):
  233. if not self.save_on_stop:
  234. raise qubes.storage.StoragePoolException(
  235. "Can not import into save_on_stop=False volume {!s}".format(
  236. self))
  237. create_sparse_file(self.path_import, size)
  238. return self.path_import
  239. def import_data_end(self, success):
  240. if success:
  241. os.rename(self.path_import, self.path)
  242. else:
  243. os.unlink(self.path_import)
  244. return self
  245. def reset(self):
  246. ''' Remove and recreate a volatile volume '''
  247. assert not self.snap_on_start and not self.save_on_stop, \
  248. "Not a volatile volume"
  249. assert isinstance(self.size, int) and self.size > 0, \
  250. 'Volatile volume size must be > 0'
  251. _remove_if_exists(self.path)
  252. create_sparse_file(self.path, self.size)
  253. return self
  254. def start(self):
  255. if not self.save_on_stop and not self.snap_on_start:
  256. self.reset()
  257. else:
  258. if not self.save_on_stop:
  259. # make sure previous snapshot is removed - even if VM
  260. # shutdown routine wasn't called (power interrupt or so)
  261. _remove_if_exists(self.path_cow)
  262. if not os.path.exists(self.path_cow):
  263. create_sparse_file(self.path_cow, self.size)
  264. if not self.snap_on_start:
  265. _check_path(self.path)
  266. if hasattr(self, 'path_source_cow'):
  267. if not os.path.exists(self.path_source_cow):
  268. create_sparse_file(self.path_source_cow, self.size)
  269. return self
  270. def stop(self):
  271. if self.save_on_stop:
  272. self.commit()
  273. elif self.snap_on_start:
  274. _remove_if_exists(self.path_cow)
  275. else:
  276. _remove_if_exists(self.path)
  277. return self
  278. @property
  279. def path(self):
  280. if self.snap_on_start:
  281. return os.path.join(self.dir_path, self.source.vid + '.img')
  282. return os.path.join(self.dir_path, self.vid + '.img')
  283. @property
  284. def path_cow(self):
  285. img_name = self.vid + '-cow.img'
  286. return os.path.join(self.dir_path, img_name)
  287. @property
  288. def path_import(self):
  289. img_name = self.vid + '-import.img'
  290. return os.path.join(self.dir_path, img_name)
  291. def verify(self):
  292. ''' Verifies the volume. '''
  293. if not os.path.exists(self.path) and \
  294. (self.snap_on_start or self.save_on_stop):
  295. msg = 'Missing image file: {!s}.'.format(self.path)
  296. raise qubes.storage.StoragePoolException(msg)
  297. return True
  298. @property
  299. def script(self):
  300. if not self.snap_on_start and not self.save_on_stop:
  301. return None
  302. if not self.snap_on_start and self.save_on_stop:
  303. return 'block-origin'
  304. if self.snap_on_start:
  305. return 'block-snapshot'
  306. return None
  307. def block_device(self):
  308. ''' Return :py:class:`qubes.storage.BlockDevice` for serialization in
  309. the libvirt XML template as <disk>.
  310. '''
  311. path = self.path
  312. if self.snap_on_start:
  313. path += ":" + self.path_source_cow
  314. if self.snap_on_start or self.save_on_stop:
  315. path += ":" + self.path_cow
  316. return qubes.storage.BlockDevice(path, self.name, self.script, self.rw,
  317. self.domain, self.devtype)
  318. @property
  319. def revisions(self):
  320. if not hasattr(self, 'path_cow'):
  321. return {}
  322. old_revision = self.path_cow + '.old' # pylint: disable=no-member
  323. if not os.path.exists(old_revision):
  324. return {}
  325. seconds = os.path.getctime(old_revision)
  326. iso_date = qubes.storage.isodate(seconds).split('.', 1)[0]
  327. return {'old': iso_date}
  328. @property
  329. def size(self):
  330. with suppress(FileNotFoundError):
  331. self._size = os.path.getsize(self.path)
  332. return self._size
  333. @size.setter
  334. def size(self, _):
  335. raise qubes.storage.StoragePoolException(
  336. "You shouldn't use volume size setter, use resize method instead")
  337. @property
  338. def usage(self):
  339. ''' Returns the actualy used space '''
  340. usage = 0
  341. if self.save_on_stop or self.snap_on_start:
  342. usage = get_disk_usage(self.path_cow)
  343. if self.save_on_stop or not self.snap_on_start:
  344. usage += get_disk_usage(self.path)
  345. return usage
  346. def create_sparse_file(path, size):
  347. ''' Create an empty sparse file '''
  348. if os.path.exists(path):
  349. raise IOError("Volume %s already exists" % path)
  350. parent_dir = os.path.dirname(path)
  351. if not os.path.exists(parent_dir):
  352. os.makedirs(parent_dir)
  353. with open(path, 'a+b') as fh:
  354. fh.truncate(size)
  355. def get_disk_usage_one(st):
  356. '''Extract disk usage of one inode from its stat_result struct.
  357. If known, get real disk usage, as written to device by filesystem, not
  358. logical file size. Those values may be different for sparse files.
  359. :param os.stat_result st: stat result
  360. :returns: disk usage
  361. '''
  362. try:
  363. return st.st_blocks * BLKSIZE
  364. except AttributeError:
  365. return st.st_size
  366. def get_disk_usage(path):
  367. '''Get real disk usage of given path (file or directory).
  368. When *path* points to directory, then it is evaluated recursively.
  369. This function tries estimate real disk usage. See documentation of
  370. :py:func:`get_disk_usage_one`.
  371. :param str path: path to evaluate
  372. :returns: disk usage
  373. '''
  374. try:
  375. st = os.lstat(path)
  376. except OSError:
  377. return 0
  378. ret = get_disk_usage_one(st)
  379. # if path is not a directory, this is skipped
  380. for dirpath, dirnames, filenames in os.walk(path):
  381. for name in dirnames + filenames:
  382. ret += get_disk_usage_one(os.lstat(os.path.join(dirpath, name)))
  383. return ret
  384. def create_dir_if_not_exists(path):
  385. """ Check if a directory exists in if not create it.
  386. This method does not create any parent directories.
  387. """
  388. if not os.path.exists(path):
  389. os.mkdir(path)
  390. def copy_file(source, destination):
  391. '''Effective file copy, preserving sparse files etc.'''
  392. # We prefer to use Linux's cp, because it nicely handles sparse files
  393. assert os.path.exists(source), \
  394. "Missing the source %s to copy from" % source
  395. assert not os.path.exists(destination), \
  396. "Destination %s already exists" % destination
  397. parent_dir = os.path.dirname(destination)
  398. if not os.path.exists(parent_dir):
  399. os.makedirs(parent_dir)
  400. try:
  401. cmd = ['cp', '--sparse=always',
  402. '--reflink=auto', source, destination]
  403. subprocess.check_call(cmd)
  404. except subprocess.CalledProcessError:
  405. raise IOError('Error while copying {!r} to {!r}'.format(source,
  406. destination))
  407. def _remove_if_exists(path):
  408. ''' Removes a file if it exist, silently succeeds if file does not exist '''
  409. if os.path.exists(path):
  410. os.remove(path)
  411. def _check_path(path):
  412. ''' Raise an StoragePoolException if ``path`` does not exist'''
  413. if not os.path.exists(path):
  414. msg = 'Missing image file: %s' % path
  415. raise qubes.storage.StoragePoolException(msg)