lvm.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473
  1. #
  2. # The Qubes OS Project, http://www.qubes-os.org
  3. #
  4. # Copyright (C) 2016 Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
  5. #
  6. # This program is free software; you can redistribute it and/or modify
  7. # it under the terms of the GNU General Public License as published by
  8. # the Free Software Foundation; either version 2 of the License, or
  9. # (at your option) any later version.
  10. #
  11. # This program is distributed in the hope that it will be useful,
  12. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14. # GNU General Public License for more details.
  15. #
  16. # You should have received a copy of the GNU General Public License along
  17. # with this program; if not, write to the Free Software Foundation, Inc.,
  18. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  19. #
  20. ''' Driver for storing vm images in a LVM thin pool '''
  21. import logging
  22. import os
  23. import subprocess
  24. import qubes
  25. def check_lvm_version():
  26. #Check if lvm is very very old, like in Travis-CI
  27. try:
  28. lvm_help = subprocess.check_output(['lvm', 'lvcreate', '--help'],
  29. stderr=subprocess.DEVNULL).decode()
  30. return '--setactivationskip' not in lvm_help
  31. except subprocess.CalledProcessError:
  32. pass
  33. lvm_is_very_old = check_lvm_version()
  34. class ThinPool(qubes.storage.Pool):
  35. ''' LVM Thin based pool implementation
  36. ''' # pylint: disable=protected-access
  37. size_cache = None
  38. driver = 'lvm_thin'
  39. def __init__(self, volume_group, thin_pool, revisions_to_keep=1, **kwargs):
  40. super(ThinPool, self).__init__(revisions_to_keep=revisions_to_keep,
  41. **kwargs)
  42. self.volume_group = volume_group
  43. self.thin_pool = thin_pool
  44. self._pool_id = "{!s}/{!s}".format(volume_group, thin_pool)
  45. self.log = logging.getLogger('qube.storage.lvm.%s' % self._pool_id)
  46. def clone(self, source, target):
  47. cmd = ['clone', str(source), str(target)]
  48. qubes_lvm(cmd, self.log)
  49. return target
  50. def _commit(self, volume):
  51. msg = "Trying to commit {!s}, but it has save_on_stop == False"
  52. msg = msg.format(volume)
  53. assert volume.save_on_stop, msg
  54. msg = "Trying to commit {!s}, but it has rw == False"
  55. msg = msg.format(volume)
  56. assert volume.rw, msg
  57. assert hasattr(volume, '_vid_snap')
  58. try:
  59. cmd = ['remove', volume.vid + "-back"]
  60. qubes_lvm(cmd, self.log)
  61. except qubes.storage.StoragePoolException:
  62. pass
  63. cmd = ['clone', volume.vid, volume.vid + "-back"]
  64. qubes_lvm(cmd, self.log)
  65. cmd = ['remove', volume.vid]
  66. qubes_lvm(cmd, self.log)
  67. cmd = ['clone', volume._vid_snap, volume.vid]
  68. qubes_lvm(cmd, self.log)
  69. @property
  70. def config(self):
  71. return {
  72. 'name': self.name,
  73. 'volume_group': self.volume_group,
  74. 'thin_pool': self.thin_pool,
  75. 'driver': ThinPool.driver
  76. }
  77. def create(self, volume):
  78. assert volume.vid
  79. assert volume.size
  80. if volume.save_on_stop:
  81. if volume.source:
  82. cmd = ['clone', str(volume.source), volume.vid]
  83. else:
  84. cmd = [
  85. 'create',
  86. self._pool_id,
  87. volume.vid.split('/', 1)[1],
  88. str(volume.size)
  89. ]
  90. qubes_lvm(cmd, self.log)
  91. reset_cache()
  92. return volume
  93. def destroy(self):
  94. pass # TODO Should we remove an existing pool?
  95. def export(self, volume):
  96. ''' Returns an object that can be `open()`. '''
  97. devpath = '/dev/' + volume.vid
  98. if not os.access(devpath, os.R_OK):
  99. # FIXME: convert to udev rules, and drop after introducing qubesd
  100. subprocess.check_call(['sudo', 'chgrp', 'qubes', devpath])
  101. subprocess.check_call(['sudo', 'chmod', 'g+rw', devpath])
  102. return devpath
  103. def init_volume(self, vm, volume_config):
  104. ''' Initialize a :py:class:`qubes.storage.Volume` from `volume_config`.
  105. '''
  106. if 'vid' not in volume_config.keys():
  107. if vm and hasattr(vm, 'name'):
  108. vm_name = vm.name
  109. else:
  110. # for the future if we have volumes not belonging to a vm
  111. vm_name = qubes.utils.random_string()
  112. assert self.name
  113. volume_config['vid'] = "{!s}/{!s}-{!s}".format(
  114. self.volume_group, vm_name, volume_config['name'])
  115. volume_config['volume_group'] = self.volume_group
  116. return ThinVolume(**volume_config)
  117. def import_volume(self, dst_pool, dst_volume, src_pool, src_volume):
  118. if not src_volume.save_on_stop:
  119. return dst_volume
  120. src_path = src_pool.export(src_volume)
  121. # HACK: neat trick to speed up testing if you have same physical thin
  122. # pool assigned to two qubes-pools i.e: qubes_dom0 and test-lvm
  123. # pylint: disable=line-too-long
  124. if isinstance(src_pool, ThinPool) and src_pool.thin_pool == dst_pool.thin_pool: # NOQA
  125. return self.clone(src_volume, dst_volume)
  126. else:
  127. dst_volume = self.create(dst_volume)
  128. cmd = ['sudo', 'dd', 'if=' + src_path, 'of=/dev/' + dst_volume.vid,
  129. 'conv=sparse']
  130. subprocess.check_call(cmd)
  131. reset_cache()
  132. return dst_volume
  133. def is_dirty(self, volume):
  134. if volume.save_on_stop:
  135. return os.path.exists(volume.path + '-snap')
  136. return False
  137. def remove(self, volume):
  138. assert volume.vid
  139. if self.is_dirty(volume):
  140. cmd = ['remove', volume._vid_snap]
  141. qubes_lvm(cmd, self.log)
  142. if not os.path.exists(volume.path):
  143. return
  144. cmd = ['remove', volume.vid]
  145. qubes_lvm(cmd, self.log)
  146. reset_cache()
  147. def rename(self, volume, old_name, new_name):
  148. ''' Called when the domain changes its name '''
  149. new_vid = "{!s}/{!s}-{!s}".format(self.volume_group, new_name,
  150. volume.name)
  151. if volume.save_on_stop:
  152. cmd = ['clone', volume.vid, new_vid]
  153. qubes_lvm(cmd, self.log)
  154. cmd = ['remove', volume.vid]
  155. qubes_lvm(cmd, self.log)
  156. volume.vid = new_vid
  157. if volume.snap_on_start:
  158. volume._vid_snap = volume.vid + '-snap'
  159. reset_cache()
  160. return volume
  161. def revert(self, volume, revision=None):
  162. old_path = volume.path + '-back'
  163. if not os.path.exists(old_path):
  164. msg = "Volume {!s} has no {!s}".format(volume, old_path)
  165. raise qubes.storage.StoragePoolException(msg)
  166. cmd = ['remove', volume.vid]
  167. qubes_lvm(cmd, self.log)
  168. cmd = ['clone', volume.vid + '-back', volume.vid]
  169. qubes_lvm(cmd, self.log)
  170. reset_cache()
  171. return volume
  172. def resize(self, volume, size):
  173. ''' Expands volume, throws
  174. :py:class:`qubst.storage.qubes.storage.StoragePoolException` if
  175. given size is less than current_size
  176. '''
  177. if not volume.rw:
  178. msg = 'Can not resize reađonly volume {!s}'.format(volume)
  179. raise qubes.storage.StoragePoolException(msg)
  180. if size <= volume.size:
  181. raise qubes.storage.StoragePoolException(
  182. 'For your own safety, shrinking of %s is'
  183. ' disabled. If you really know what you'
  184. ' are doing, use `lvresize` on %s manually.' %
  185. (volume.name, volume.vid))
  186. cmd = ['extend', volume.vid, str(size)]
  187. qubes_lvm(cmd, self.log)
  188. reset_cache()
  189. def setup(self):
  190. pass # TODO Should we create a non existing pool?
  191. def start(self, volume):
  192. if volume.snap_on_start:
  193. if not volume.save_on_stop or not self.is_dirty(volume):
  194. self._snapshot(volume)
  195. elif not volume.save_on_stop:
  196. self._reset_volume(volume)
  197. reset_cache()
  198. return volume
  199. def stop(self, volume):
  200. if volume.save_on_stop and volume.snap_on_start:
  201. self._commit(volume)
  202. if volume.snap_on_start:
  203. cmd = ['remove', volume._vid_snap]
  204. qubes_lvm(cmd, self.log)
  205. elif not volume.save_on_stop:
  206. cmd = ['remove', volume.vid]
  207. qubes_lvm(cmd, self.log)
  208. reset_cache()
  209. return volume
  210. def _snapshot(self, volume):
  211. try:
  212. cmd = ['remove', volume._vid_snap]
  213. qubes_lvm(cmd, self.log)
  214. except: # pylint: disable=bare-except
  215. pass
  216. if volume.source is None:
  217. cmd = ['clone', volume.vid, volume._vid_snap]
  218. else:
  219. cmd = ['clone', str(volume.source), volume._vid_snap]
  220. qubes_lvm(cmd, self.log)
  221. def verify(self, volume):
  222. ''' Verifies the volume. '''
  223. try:
  224. vol_info = size_cache[volume.vid]
  225. return vol_info['attr'][4] == 'a'
  226. except KeyError:
  227. return False
  228. @property
  229. def volumes(self):
  230. ''' Return a list of volumes managed by this pool '''
  231. volumes = []
  232. for vid, vol_info in size_cache.items():
  233. if not vid.startswith(self.volume_group + '/'):
  234. continue
  235. if vol_info['pool_lv'] != self.thin_pool:
  236. continue
  237. if vid.endswith('-snap'):
  238. # implementation detail volume
  239. continue
  240. config = {
  241. 'pool': self.name,
  242. 'vid': vid,
  243. 'name': vid,
  244. 'volume_group': self.volume_group,
  245. 'rw': vol_info['attr'][1] == 'w',
  246. }
  247. volumes += [ThinVolume(**config)]
  248. return volumes
  249. def _reset_volume(self, volume):
  250. ''' Resets a volatile volume '''
  251. assert volume._is_volatile, \
  252. 'Expected a volatile volume, but got {!r}'.format(volume)
  253. self.log.debug('Resetting volatile ' + volume.vid)
  254. try:
  255. cmd = ['remove', volume.vid]
  256. qubes_lvm(cmd, self.log)
  257. except qubes.storage.StoragePoolException:
  258. pass
  259. cmd = ['create', self._pool_id, volume.vid.split('/')[1],
  260. str(volume.size)]
  261. qubes_lvm(cmd, self.log)
  262. def init_cache(log=logging.getLogger('qube.storage.lvm')):
  263. cmd = ['lvs', '--noheadings', '-o',
  264. 'vg_name,pool_lv,name,lv_size,data_percent,lv_attr',
  265. '--units', 'b', '--separator', ',']
  266. if os.getuid() != 0:
  267. cmd.insert(0, 'sudo')
  268. p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
  269. close_fds=True)
  270. out, err = p.communicate()
  271. return_code = p.returncode
  272. if return_code == 0 and err:
  273. log.warning(err)
  274. elif return_code != 0:
  275. raise qubes.storage.StoragePoolException(err)
  276. result = {}
  277. for line in out.splitlines():
  278. line = line.decode().strip()
  279. pool_name, pool_lv, name, size, usage_percent, attr = line.split(',', 5)
  280. if '' in [pool_name, pool_lv, name, size, usage_percent]:
  281. continue
  282. name = pool_name + "/" + name
  283. size = int(size[:-1])
  284. usage = int(size / 100 * float(usage_percent))
  285. result[name] = {'size': size, 'usage': usage, 'pool_lv': pool_lv,
  286. 'attr': attr}
  287. return result
  288. size_cache = init_cache()
  289. class ThinVolume(qubes.storage.Volume):
  290. ''' Default LVM thin volume implementation
  291. ''' # pylint: disable=too-few-public-methods
  292. def __init__(self, volume_group, size=0, **kwargs):
  293. self.volume_group = volume_group
  294. super(ThinVolume, self).__init__(size=size, **kwargs)
  295. if self.snap_on_start and self.source is None:
  296. msg = "snap_on_start specified on {!r} but no volume source set"
  297. msg = msg.format(self.name)
  298. raise qubes.storage.StoragePoolException(msg)
  299. elif not self.snap_on_start and self.source is not None:
  300. msg = "source specified on {!r} but no snap_on_start set"
  301. msg = msg.format(self.name)
  302. raise qubes.storage.StoragePoolException(msg)
  303. self.path = '/dev/' + self.vid
  304. if self.snap_on_start:
  305. self._vid_snap = self.vid + '-snap'
  306. self._size = size
  307. @property
  308. def revisions(self):
  309. path = self.path + '-back'
  310. if os.path.exists(path):
  311. seconds = os.path.getctime(path)
  312. iso_date = qubes.storage.isodate(seconds).split('.', 1)[0]
  313. return {iso_date: path}
  314. return {}
  315. @property
  316. def _is_origin(self):
  317. return not self.snap_on_start and self.save_on_stop
  318. @property
  319. def _is_origin_snapshot(self):
  320. return self.snap_on_start and self.save_on_stop
  321. @property
  322. def _is_snapshot(self):
  323. return self.snap_on_start and not self.save_on_stop
  324. @property
  325. def _is_volatile(self):
  326. return not self.snap_on_start and not self.save_on_stop
  327. @property
  328. def size(self):
  329. try:
  330. return qubes.storage.lvm.size_cache[self.vid]['size']
  331. except KeyError:
  332. return self._size
  333. @size.setter
  334. def size(self, _):
  335. raise qubes.storage.StoragePoolException(
  336. "You shouldn't use lvm size setter")
  337. def block_device(self):
  338. ''' Return :py:class:`qubes.storage.BlockDevice` for serialization in
  339. the libvirt XML template as <disk>.
  340. '''
  341. if self.snap_on_start:
  342. return qubes.storage.BlockDevice(
  343. '/dev/' + self._vid_snap, self.name, self.script,
  344. self.rw, self.domain, self.devtype)
  345. return super(ThinVolume, self).block_device()
  346. @property
  347. def usage(self): # lvm thin usage always returns at least the same usage as
  348. # the parent
  349. try:
  350. return qubes.storage.lvm.size_cache[self.vid]['usage']
  351. except KeyError:
  352. return 0
  353. def pool_exists(pool_id):
  354. ''' Return true if pool exists '''
  355. try:
  356. vol_info = size_cache[pool_id]
  357. return vol_info['attr'][0] == 't'
  358. except KeyError:
  359. return False
  360. def qubes_lvm(cmd, log=logging.getLogger('qube.storage.lvm')):
  361. ''' Call :program:`lvm` to execute an LVM operation '''
  362. action = cmd[0]
  363. if action == 'remove':
  364. lvm_cmd = ['lvremove', '-f', cmd[1]]
  365. elif action == 'clone':
  366. lvm_cmd = ['lvcreate', '-kn', '-ay', '-s', cmd[1], '-n', cmd[2]]
  367. elif action == 'create':
  368. lvm_cmd = ['lvcreate', '-T', cmd[1], '-kn', '-ay', '-n', cmd[2], '-V',
  369. str(cmd[3]) + 'B']
  370. elif action == 'extend':
  371. size = int(cmd[2]) / (1000 * 1000)
  372. lvm_cmd = ["lvextend", "-L%s" % size, cmd[1]]
  373. else:
  374. raise NotImplementedError('unsupported action: ' + action)
  375. if lvm_is_very_old:
  376. # old lvm in trusty image used there does not support -k option
  377. lvm_cmd = [x for x in lvm_cmd if x != '-kn']
  378. if os.getuid() != 0:
  379. cmd = ['sudo', 'lvm'] + lvm_cmd
  380. else:
  381. cmd = ['lvm'] + lvm_cmd
  382. p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
  383. close_fds=True)
  384. out, err = p.communicate()
  385. return_code = p.returncode
  386. if out:
  387. log.debug(out)
  388. if return_code == 0 and err:
  389. log.warning(err)
  390. elif return_code != 0:
  391. assert err, "Command exited unsuccessful, but printed nothing to stderr"
  392. raise qubes.storage.StoragePoolException(err)
  393. return True
  394. def reset_cache():
  395. qubes.storage.lvm.size_cache = init_cache()