lvm.py 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848
  1. #
  2. # The Qubes OS Project, http://www.qubes-os.org
  3. #
  4. # Copyright (C) 2016 Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
  5. #
  6. # This library is free software; you can redistribute it and/or
  7. # modify it under the terms of the GNU Lesser General Public
  8. # License as published by the Free Software Foundation; either
  9. # version 2.1 of the License, or (at your option) any later version.
  10. #
  11. # This library is distributed in the hope that it will be useful,
  12. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14. # Lesser General Public License for more details.
  15. #
  16. # You should have received a copy of the GNU Lesser General Public
  17. # License along with this library; if not, see <https://www.gnu.org/licenses/>.
  18. #
  19. ''' Driver for storing vm images in a LVM thin pool '''
  20. import logging
  21. import os
  22. import subprocess
  23. import time
  24. import asyncio
  25. import qubes
  26. import qubes.storage
  27. import qubes.utils
  28. def check_lvm_version():
  29. #Check if lvm is very very old, like in Travis-CI
  30. try:
  31. lvm_help = subprocess.check_output(['lvm', 'lvcreate', '--help'],
  32. stderr=subprocess.DEVNULL).decode()
  33. return '--setactivationskip' not in lvm_help
  34. except (subprocess.CalledProcessError, FileNotFoundError):
  35. pass
  36. lvm_is_very_old = check_lvm_version()
  37. class ThinPool(qubes.storage.Pool):
  38. ''' LVM Thin based pool implementation
  39. Volumes are stored as LVM thin volumes, in thin pool specified by
  40. *volume_group*/*thin_pool* arguments. LVM volume naming scheme:
  41. vm-{vm_name}-{volume_name}[-suffix]
  42. Where suffix can be one of:
  43. "-snap" - snapshot for currently running VM, at VM shutdown will be
  44. either discarded (if save_on_stop=False), or committed
  45. (if save_on_stop=True)
  46. "-{revision_id}" - volume revision - new revision is automatically
  47. created at each VM shutdown, *revisions_to_keep* control how many
  48. old revisions (in addition to the current one) should be stored
  49. "" (no suffix) - the most recent committed volume state; also volatile
  50. volume (snap_on_start=False, save_on_stop=False)
  51. On VM startup, new volume is created, depending on volume type,
  52. according to the table below:
  53. snap_on_start, save_on_stop
  54. False, False, - no suffix, fresh empty volume
  55. False, True, - "-snap", snapshot of last committed revision
  56. True , False, - "-snap", snapshot of last committed revision
  57. of source volume (from VM's template)
  58. True, True, - unsupported configuration
  59. Volume's revision_id format is "{timestamp}-back", where timestamp is in
  60. '%s' format (seconds since unix epoch)
  61. ''' # pylint: disable=protected-access
  62. size_cache = None
  63. driver = 'lvm_thin'
  64. def __init__(self, *, name, revisions_to_keep=1, volume_group, thin_pool):
  65. super().__init__(name=name, revisions_to_keep=revisions_to_keep)
  66. self.volume_group = volume_group
  67. self.thin_pool = thin_pool
  68. self._pool_id = "{!s}/{!s}".format(volume_group, thin_pool)
  69. self.log = logging.getLogger('qubes.storage.lvm.%s' % self._pool_id)
  70. self._volume_objects_cache = {}
  71. def __repr__(self):
  72. return '<{} at {:#x} name={!r} volume_group={!r} thin_pool={!r}>'.\
  73. format(
  74. type(self).__name__, id(self),
  75. self.name, self.volume_group, self.thin_pool)
  76. @property
  77. def config(self):
  78. return {
  79. 'name': self.name,
  80. 'volume_group': self.volume_group,
  81. 'thin_pool': self.thin_pool,
  82. 'driver': ThinPool.driver,
  83. 'revisions_to_keep': self.revisions_to_keep,
  84. }
  85. def destroy(self):
  86. pass # TODO Should we remove an existing pool?
  87. def init_volume(self, vm, volume_config):
  88. ''' Initialize a :py:class:`qubes.storage.Volume` from `volume_config`.
  89. '''
  90. if 'revisions_to_keep' not in volume_config.keys():
  91. volume_config['revisions_to_keep'] = self.revisions_to_keep
  92. if 'vid' not in volume_config.keys():
  93. if vm and hasattr(vm, 'name'):
  94. vm_name = vm.name
  95. else:
  96. # for the future if we have volumes not belonging to a vm
  97. vm_name = qubes.utils.random_string()
  98. assert self.name
  99. volume_config['vid'] = "{!s}/vm-{!s}-{!s}".format(
  100. self.volume_group, vm_name, volume_config['name'])
  101. volume_config['volume_group'] = self.volume_group
  102. volume_config['pool'] = self
  103. volume = ThinVolume(**volume_config)
  104. self._volume_objects_cache[volume_config['vid']] = volume
  105. return volume
  106. def setup(self):
  107. reset_cache()
  108. cache_key = self.volume_group + '/' + self.thin_pool
  109. if cache_key not in size_cache:
  110. raise qubes.storage.StoragePoolException(
  111. 'Thin pool {} does not exist'.format(cache_key))
  112. if size_cache[cache_key]['attr'][0] != 't':
  113. raise qubes.storage.StoragePoolException(
  114. 'Volume {} is not a thin pool'.format(cache_key))
  115. # TODO Should we create a non existing pool?
  116. def get_volume(self, vid):
  117. ''' Return a volume with given vid'''
  118. if vid in self._volume_objects_cache:
  119. return self._volume_objects_cache[vid]
  120. config = {
  121. 'pool': self,
  122. 'vid': vid,
  123. 'name': vid,
  124. 'volume_group': self.volume_group,
  125. }
  126. # don't cache this object, as it doesn't carry full configuration
  127. return ThinVolume(**config)
  128. def list_volumes(self):
  129. ''' Return a list of volumes managed by this pool '''
  130. volumes = []
  131. for vid, vol_info in size_cache.items():
  132. if not vid.startswith(self.volume_group + '/'):
  133. continue
  134. if vol_info['pool_lv'] != self.thin_pool:
  135. continue
  136. if vid.endswith('-snap') or vid.endswith('-import'):
  137. # implementation detail volume
  138. continue
  139. if vid.endswith('-back'):
  140. # old revisions
  141. continue
  142. volume = self.get_volume(vid)
  143. if volume in volumes:
  144. continue
  145. volumes.append(volume)
  146. return volumes
  147. @property
  148. def size(self):
  149. try:
  150. return qubes.storage.lvm.size_cache[
  151. self.volume_group + '/' + self.thin_pool]['size']
  152. except KeyError:
  153. return 0
  154. @property
  155. def usage(self):
  156. refresh_cache()
  157. try:
  158. return qubes.storage.lvm.size_cache[
  159. self.volume_group + '/' + self.thin_pool]['usage']
  160. except KeyError:
  161. return 0
  162. @property
  163. def usage_details(self):
  164. result = {}
  165. result['data_size'] = self.size
  166. result['data_usage'] = self.usage
  167. try:
  168. metadata_size = qubes.storage.lvm.size_cache[
  169. self.volume_group + '/' + self.thin_pool]['metadata_size']
  170. metadata_usage = qubes.storage.lvm.size_cache[
  171. self.volume_group + '/' + self.thin_pool]['metadata_usage']
  172. except KeyError:
  173. metadata_size = 0
  174. metadata_usage = 0
  175. result['metadata_size'] = metadata_size
  176. result['metadata_usage'] = metadata_usage
  177. return result
  178. _init_cache_cmd = ['lvs', '--noheadings', '-o',
  179. 'vg_name,pool_lv,name,lv_size,data_percent,lv_attr,origin,lv_metadata_size,'
  180. 'metadata_percent', '--units', 'b', '--separator', ';']
  181. def _parse_lvm_cache(lvm_output):
  182. result = {}
  183. for line in lvm_output.splitlines():
  184. line = line.decode().strip()
  185. pool_name, pool_lv, name, size, usage_percent, attr, \
  186. origin, metadata_size, metadata_percent = line.split(';', 8)
  187. if '' in [pool_name, name, size, usage_percent]:
  188. continue
  189. name = pool_name + "/" + name
  190. size = int(size[:-1]) # Remove 'B' suffix
  191. usage = int(size / 100 * float(usage_percent))
  192. if metadata_size:
  193. metadata_size = int(metadata_size[:-1])
  194. metadata_usage = int(metadata_size / 100 * float(metadata_percent))
  195. else:
  196. metadata_usage = None
  197. result[name] = {'size': size, 'usage': usage, 'pool_lv': pool_lv,
  198. 'attr': attr, 'origin': origin, 'metadata_size': metadata_size,
  199. 'metadata_usage': metadata_usage}
  200. return result
  201. def init_cache(log=logging.getLogger('qubes.storage.lvm')):
  202. cmd = _init_cache_cmd
  203. if os.getuid() != 0:
  204. cmd = ['sudo'] + cmd
  205. environ = os.environ.copy()
  206. environ['LC_ALL'] = 'C.utf8'
  207. p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
  208. close_fds=True, env=environ)
  209. out, err = p.communicate()
  210. return_code = p.returncode
  211. if return_code == 0 and err:
  212. log.warning(err)
  213. elif return_code != 0:
  214. raise qubes.storage.StoragePoolException(err)
  215. return _parse_lvm_cache(out)
  216. @asyncio.coroutine
  217. def init_cache_coro(log=logging.getLogger('qubes.storage.lvm')):
  218. cmd = _init_cache_cmd
  219. if os.getuid() != 0:
  220. cmd = ['sudo'] + cmd
  221. environ = os.environ.copy()
  222. environ['LC_ALL'] = 'C.utf8'
  223. p = yield from asyncio.create_subprocess_exec(*cmd,
  224. stdout=subprocess.PIPE,
  225. stderr=subprocess.PIPE,
  226. close_fds=True, env=environ)
  227. out, err = yield from p.communicate()
  228. return_code = p.returncode
  229. if return_code == 0 and err:
  230. log.warning(err)
  231. elif return_code != 0:
  232. raise qubes.storage.StoragePoolException(err)
  233. return _parse_lvm_cache(out)
  234. size_cache_time = 0
  235. size_cache = init_cache()
  236. def _revision_sort_key(revision):
  237. '''Sort key for revisions. Sort them by time
  238. :returns timestamp
  239. '''
  240. if isinstance(revision, tuple):
  241. revision = revision[0]
  242. if '-' in revision:
  243. revision = revision.split('-')[0]
  244. return int(revision)
  245. class ThinVolume(qubes.storage.Volume):
  246. ''' Default LVM thin volume implementation
  247. ''' # pylint: disable=too-few-public-methods
  248. def __init__(self, volume_group, **kwargs):
  249. self.volume_group = volume_group
  250. super().__init__(**kwargs)
  251. self.log = logging.getLogger('qubes.storage.lvm.%s' % str(self.pool))
  252. if self.snap_on_start or self.save_on_stop:
  253. self._vid_snap = self.vid + '-snap'
  254. if self.save_on_stop:
  255. self._vid_import = self.vid + '-import'
  256. @property
  257. def path(self):
  258. return '/dev/' + self._vid_current
  259. @property
  260. def _vid_current(self):
  261. if self.vid in size_cache:
  262. return self.vid
  263. vol_revisions = self.revisions
  264. if vol_revisions:
  265. last_revision = \
  266. max(vol_revisions.items(), key=_revision_sort_key)[0]
  267. return self.vid + '-' + last_revision
  268. # detached pool? return expected path
  269. return self.vid
  270. @property
  271. def revisions(self):
  272. name_prefix = self.vid + '-'
  273. revisions = {}
  274. for revision_vid in size_cache:
  275. if not revision_vid.startswith(name_prefix):
  276. continue
  277. if not revision_vid.endswith('-back'):
  278. continue
  279. revision_vid = revision_vid[len(name_prefix):]
  280. if revision_vid.count('-') > 1:
  281. # VM+volume name is a prefix of another VM, see #4680
  282. continue
  283. # get revision without suffix
  284. seconds = int(revision_vid.split('-')[0])
  285. iso_date = qubes.storage.isodate(seconds).split('.', 1)[0]
  286. revisions[revision_vid] = iso_date
  287. return revisions
  288. @property
  289. def size(self):
  290. try:
  291. if self.is_dirty():
  292. return qubes.storage.lvm.size_cache[self._vid_snap]['size']
  293. return qubes.storage.lvm.size_cache[self._vid_current]['size']
  294. except KeyError:
  295. return self._size
  296. @size.setter
  297. def size(self, _):
  298. raise qubes.storage.StoragePoolException(
  299. "You shouldn't use lvm size setter")
  300. @asyncio.coroutine
  301. def _reset(self):
  302. ''' Resets a volatile volume '''
  303. assert not self.snap_on_start and not self.save_on_stop, \
  304. "Not a volatile volume"
  305. self.log.debug('Resetting volatile %s', self.vid)
  306. try:
  307. cmd = ['remove', self.vid]
  308. yield from qubes_lvm_coro(cmd, self.log)
  309. except qubes.storage.StoragePoolException:
  310. pass
  311. # pylint: disable=protected-access
  312. cmd = ['create', self.pool._pool_id, self.vid.split('/')[1],
  313. str(self.size)]
  314. yield from qubes_lvm_coro(cmd, self.log)
  315. @asyncio.coroutine
  316. def _remove_revisions(self, revisions=None):
  317. '''Remove old volume revisions.
  318. If no revisions list is given, it removes old revisions according to
  319. :py:attr:`revisions_to_keep`
  320. :param revisions: list of revisions to remove
  321. '''
  322. if revisions is None:
  323. revisions = sorted(self.revisions.items(),
  324. key=_revision_sort_key)
  325. # pylint: disable=invalid-unary-operand-type
  326. revisions = revisions[:(-self.revisions_to_keep) or None]
  327. revisions = [rev_id for rev_id, _ in revisions]
  328. for rev_id in revisions:
  329. # safety check
  330. assert rev_id != self._vid_current
  331. try:
  332. cmd = ['remove', self.vid + '-' + rev_id]
  333. yield from qubes_lvm_coro(cmd, self.log)
  334. except qubes.storage.StoragePoolException:
  335. pass
  336. @asyncio.coroutine
  337. def _commit(self, vid_to_commit=None, keep=False):
  338. '''
  339. Commit temporary volume into current one. By default
  340. :py:attr:`_vid_snap` is used (which is created by :py:meth:`start()`),
  341. but can be overriden by *vid_to_commit* argument.
  342. :param vid_to_commit: LVM volume ID to commit into this one
  343. :param keep: whether to keep or not *vid_to_commit*.
  344. IOW use 'clone' or 'rename' methods.
  345. :return: None
  346. '''
  347. msg = "Trying to commit {!s}, but it has save_on_stop == False"
  348. msg = msg.format(self)
  349. assert self.save_on_stop, msg
  350. msg = "Trying to commit {!s}, but it has rw == False"
  351. msg = msg.format(self)
  352. assert self.rw, msg
  353. if vid_to_commit is None:
  354. assert hasattr(self, '_vid_snap')
  355. vid_to_commit = self._vid_snap
  356. assert self._lock.locked()
  357. if not os.path.exists('/dev/' + vid_to_commit):
  358. # nothing to commit
  359. return
  360. if self._vid_current == self.vid:
  361. cmd = ['rename', self.vid,
  362. '{}-{}-back'.format(self.vid, int(time.time()))]
  363. yield from qubes_lvm_coro(cmd, self.log)
  364. yield from reset_cache_coro()
  365. cmd = ['clone' if keep else 'rename',
  366. vid_to_commit,
  367. self.vid]
  368. yield from qubes_lvm_coro(cmd, self.log)
  369. yield from reset_cache_coro()
  370. # make sure the one we've committed right now is properly
  371. # detected as the current one - before removing anything
  372. assert self._vid_current == self.vid
  373. # and remove old snapshots, if needed
  374. yield from self._remove_revisions()
  375. @qubes.storage.Volume.locked
  376. @asyncio.coroutine
  377. def create(self):
  378. assert self.vid
  379. assert self.size
  380. if self.save_on_stop:
  381. if self.source:
  382. cmd = ['clone', self.source.path, self.vid]
  383. else:
  384. cmd = [
  385. 'create',
  386. self.pool._pool_id, # pylint: disable=protected-access
  387. self.vid.split('/', 1)[1],
  388. str(self.size)
  389. ]
  390. yield from qubes_lvm_coro(cmd, self.log)
  391. yield from reset_cache_coro()
  392. return self
  393. @qubes.storage.Volume.locked
  394. @asyncio.coroutine
  395. def remove(self):
  396. assert self.vid
  397. try:
  398. if os.path.exists('/dev/' + self._vid_snap):
  399. cmd = ['remove', self._vid_snap]
  400. yield from qubes_lvm_coro(cmd, self.log)
  401. except AttributeError:
  402. pass
  403. try:
  404. if os.path.exists('/dev/' + self._vid_import):
  405. cmd = ['remove', self._vid_import]
  406. yield from qubes_lvm_coro(cmd, self.log)
  407. except AttributeError:
  408. pass
  409. yield from self._remove_revisions(self.revisions.keys())
  410. if not os.path.exists(self.path):
  411. return
  412. cmd = ['remove', self.path]
  413. yield from qubes_lvm_coro(cmd, self.log)
  414. yield from reset_cache_coro()
  415. # pylint: disable=protected-access
  416. self.pool._volume_objects_cache.pop(self.vid, None)
  417. def export(self):
  418. ''' Returns an object that can be `open()`. '''
  419. # make sure the device node is available
  420. qubes_lvm(['activate', self.path], self.log)
  421. devpath = self.path
  422. return devpath
  423. @qubes.storage.Volume.locked
  424. @asyncio.coroutine
  425. def import_volume(self, src_volume):
  426. if not src_volume.save_on_stop:
  427. return self
  428. if self.is_dirty():
  429. raise qubes.storage.StoragePoolException(
  430. 'Cannot import to dirty volume {} -'
  431. ' start and stop a qube to cleanup'.format(self.vid))
  432. self.abort_if_import_in_progress()
  433. # HACK: neat trick to speed up testing if you have same physical thin
  434. # pool assigned to two qubes-pools i.e: qubes_dom0 and test-lvm
  435. # pylint: disable=line-too-long
  436. if isinstance(src_volume.pool, ThinPool) and \
  437. src_volume.pool.thin_pool == self.pool.thin_pool: # NOQA
  438. yield from self._commit(src_volume.path[len('/dev/'):], keep=True)
  439. else:
  440. cmd = ['create',
  441. self.pool._pool_id, # pylint: disable=protected-access
  442. self._vid_import.split('/')[1],
  443. str(src_volume.size)]
  444. yield from qubes_lvm_coro(cmd, self.log)
  445. src_path = yield from qubes.utils.coro_maybe(src_volume.export())
  446. try:
  447. cmd = ['dd', 'if=' + src_path, 'of=/dev/' + self._vid_import,
  448. 'conv=sparse', 'status=none', 'bs=128K']
  449. if not os.access('/dev/' + self._vid_import, os.W_OK) or \
  450. not os.access(src_path, os.R_OK):
  451. cmd.insert(0, 'sudo')
  452. p = yield from asyncio.create_subprocess_exec(*cmd)
  453. yield from p.wait()
  454. finally:
  455. yield from qubes.utils.coro_maybe(
  456. src_volume.export_end(src_path))
  457. if p.returncode != 0:
  458. cmd = ['remove', self._vid_import]
  459. yield from qubes_lvm_coro(cmd, self.log)
  460. raise qubes.storage.StoragePoolException(
  461. 'Failed to import volume {!r}, dd exit code: {}'.format(
  462. src_volume, p.returncode))
  463. yield from self._commit(self._vid_import)
  464. return self
  465. @qubes.storage.Volume.locked
  466. @asyncio.coroutine
  467. def import_data(self, size):
  468. ''' Returns an object that can be `open()`. '''
  469. if self.is_dirty():
  470. raise qubes.storage.StoragePoolException(
  471. 'Cannot import data to dirty volume {}, stop the qube first'.
  472. format(self.vid))
  473. self.abort_if_import_in_progress()
  474. # pylint: disable=protected-access
  475. cmd = ['create', self.pool._pool_id, self._vid_import.split('/')[1],
  476. str(size)]
  477. yield from qubes_lvm_coro(cmd, self.log)
  478. yield from reset_cache_coro()
  479. devpath = '/dev/' + self._vid_import
  480. return devpath
  481. @qubes.storage.Volume.locked
  482. @asyncio.coroutine
  483. def import_data_end(self, success):
  484. '''Either commit imported data, or discard temporary volume'''
  485. if not os.path.exists('/dev/' + self._vid_import):
  486. raise qubes.storage.StoragePoolException(
  487. 'No import operation in progress on {}'.format(self.vid))
  488. if success:
  489. yield from self._commit(self._vid_import)
  490. else:
  491. cmd = ['remove', self._vid_import]
  492. yield from qubes_lvm_coro(cmd, self.log)
  493. def abort_if_import_in_progress(self):
  494. try:
  495. devpath = '/dev/' + self._vid_import
  496. if os.path.exists(devpath):
  497. raise qubes.storage.StoragePoolException(
  498. 'Import operation in progress on {}'.format(self.vid))
  499. except AttributeError: # self._vid_import
  500. # no vid_import - import definitely not in progress
  501. pass
  502. def is_dirty(self):
  503. if self.save_on_stop:
  504. return os.path.exists('/dev/' + self._vid_snap)
  505. return False
  506. def is_outdated(self):
  507. if not self.snap_on_start:
  508. return False
  509. if self._vid_snap not in size_cache:
  510. return False
  511. return (size_cache[self._vid_snap]['origin'] !=
  512. self.source.path.split('/')[-1])
  513. @qubes.storage.Volume.locked
  514. @asyncio.coroutine
  515. def revert(self, revision=None):
  516. if self.is_dirty():
  517. raise qubes.storage.StoragePoolException(
  518. 'Cannot revert dirty volume {}, stop the qube first'.format(
  519. self.vid))
  520. self.abort_if_import_in_progress()
  521. if revision is None:
  522. revision = \
  523. max(self.revisions.items(), key=_revision_sort_key)[0]
  524. old_path = '/dev/' + self.vid + '-' + revision
  525. if not os.path.exists(old_path):
  526. msg = "Volume {!s} has no {!s}".format(self, old_path)
  527. raise qubes.storage.StoragePoolException(msg)
  528. if self.vid in size_cache:
  529. cmd = ['remove', self.vid]
  530. yield from qubes_lvm_coro(cmd, self.log)
  531. cmd = ['clone', self.vid + '-' + revision, self.vid]
  532. yield from qubes_lvm_coro(cmd, self.log)
  533. yield from reset_cache_coro()
  534. return self
  535. @qubes.storage.Volume.locked
  536. @asyncio.coroutine
  537. def resize(self, size):
  538. ''' Expands volume, throws
  539. :py:class:`qubst.storage.qubes.storage.StoragePoolException` if
  540. given size is less than current_size
  541. '''
  542. if not self.rw:
  543. msg = 'Can not resize reađonly volume {!s}'.format(self)
  544. raise qubes.storage.StoragePoolException(msg)
  545. if size < self.size:
  546. raise qubes.storage.StoragePoolException(
  547. 'For your own safety, shrinking of %s is'
  548. ' disabled (%d < %d). If you really know what you'
  549. ' are doing, use `lvresize` on %s manually.' %
  550. (self.name, size, self.size, self.vid))
  551. if size == self.size:
  552. return
  553. if self.is_dirty():
  554. cmd = ['extend', self._vid_snap, str(size)]
  555. yield from qubes_lvm_coro(cmd, self.log)
  556. elif hasattr(self, '_vid_import') and \
  557. os.path.exists('/dev/' + self._vid_import):
  558. cmd = ['extend', self._vid_import, str(size)]
  559. yield from qubes_lvm_coro(cmd, self.log)
  560. elif self.save_on_stop and not self.snap_on_start:
  561. cmd = ['extend', self._vid_current, str(size)]
  562. yield from qubes_lvm_coro(cmd, self.log)
  563. self._size = size
  564. yield from reset_cache_coro()
  565. @asyncio.coroutine
  566. def _snapshot(self):
  567. try:
  568. cmd = ['remove', self._vid_snap]
  569. yield from qubes_lvm_coro(cmd, self.log)
  570. except: # pylint: disable=bare-except
  571. pass
  572. if self.source is None:
  573. cmd = ['clone', self._vid_current, self._vid_snap]
  574. else:
  575. cmd = ['clone', self.source.path, self._vid_snap]
  576. yield from qubes_lvm_coro(cmd, self.log)
  577. @qubes.storage.Volume.locked
  578. @asyncio.coroutine
  579. def start(self):
  580. self.abort_if_import_in_progress()
  581. try:
  582. if self.snap_on_start or self.save_on_stop:
  583. if not self.save_on_stop or not self.is_dirty():
  584. yield from self._snapshot()
  585. else:
  586. yield from self._reset()
  587. finally:
  588. yield from reset_cache_coro()
  589. return self
  590. @qubes.storage.Volume.locked
  591. @asyncio.coroutine
  592. def stop(self):
  593. try:
  594. if self.save_on_stop:
  595. yield from self._commit()
  596. if self.snap_on_start and not self.save_on_stop:
  597. cmd = ['remove', self._vid_snap]
  598. yield from qubes_lvm_coro(cmd, self.log)
  599. elif not self.snap_on_start and not self.save_on_stop:
  600. cmd = ['remove', self.vid]
  601. yield from qubes_lvm_coro(cmd, self.log)
  602. finally:
  603. yield from reset_cache_coro()
  604. return self
  605. def verify(self):
  606. ''' Verifies the volume. '''
  607. if not self.save_on_stop and not self.snap_on_start:
  608. # volatile volumes don't need any files
  609. return True
  610. if self.source is not None:
  611. vid = self.source.path[len('/dev/'):]
  612. else:
  613. vid = self._vid_current
  614. try:
  615. vol_info = size_cache[vid]
  616. if vol_info['attr'][4] != 'a':
  617. raise qubes.storage.StoragePoolException(
  618. 'volume {} not active'.format(vid))
  619. except KeyError:
  620. raise qubes.storage.StoragePoolException(
  621. 'volume {} missing'.format(vid))
  622. return True
  623. def block_device(self):
  624. ''' Return :py:class:`qubes.storage.BlockDevice` for serialization in
  625. the libvirt XML template as <disk>.
  626. '''
  627. if self.snap_on_start or self.save_on_stop:
  628. return qubes.storage.BlockDevice(
  629. '/dev/' + self._vid_snap, self.name, self.script,
  630. self.rw, self.domain, self.devtype)
  631. return super().block_device()
  632. @property
  633. def usage(self): # lvm thin usage always returns at least the same usage as
  634. # the parent
  635. refresh_cache()
  636. try:
  637. return qubes.storage.lvm.size_cache[self._vid_current]['usage']
  638. except KeyError:
  639. return 0
  640. def pool_exists(pool_id):
  641. ''' Return true if pool exists '''
  642. try:
  643. vol_info = size_cache[pool_id]
  644. return vol_info['attr'][0] == 't'
  645. except KeyError:
  646. return False
  647. def _get_lvm_cmdline(cmd):
  648. ''' Build command line for :program:`lvm` call.
  649. The purpose of this function is to keep all the detailed lvm options in
  650. one place.
  651. :param cmd: array of str, where cmd[0] is action and the rest are arguments
  652. :return array of str appropriate for subprocess.Popen
  653. '''
  654. action = cmd[0]
  655. if action == 'remove':
  656. lvm_cmd = ['lvremove', '-f', cmd[1]]
  657. elif action == 'clone':
  658. lvm_cmd = ['lvcreate', '-kn', '-ay', '-s', cmd[1], '-n', cmd[2]]
  659. elif action == 'create':
  660. lvm_cmd = ['lvcreate', '-T', cmd[1], '-kn', '-ay', '-n', cmd[2], '-V',
  661. str(cmd[3]) + 'B']
  662. elif action == 'extend':
  663. size = int(cmd[2]) / (1024 * 1024)
  664. lvm_cmd = ["lvextend", "-L%s" % size, cmd[1]]
  665. elif action == 'activate':
  666. lvm_cmd = ['lvchange', '-ay', cmd[1]]
  667. elif action == 'rename':
  668. lvm_cmd = ['lvrename', cmd[1], cmd[2]]
  669. else:
  670. raise NotImplementedError('unsupported action: ' + action)
  671. if lvm_is_very_old:
  672. # old lvm in trusty image used there does not support -k option
  673. lvm_cmd = [x for x in lvm_cmd if x != '-kn']
  674. if os.getuid() != 0:
  675. cmd = ['sudo', 'lvm'] + lvm_cmd
  676. else:
  677. cmd = ['lvm'] + lvm_cmd
  678. return cmd
  679. def _process_lvm_output(returncode, stdout, stderr, log):
  680. '''Process output of LVM, determine if the call was successful and
  681. possibly log warnings.'''
  682. # Filter out warning about intended over-provisioning.
  683. # Upstream discussion about missing option to silence it:
  684. # https://bugzilla.redhat.com/1347008
  685. err = '\n'.join(line for line in stderr.decode().splitlines()
  686. if 'exceeds the size of thin pool' not in line)
  687. if stdout:
  688. log.debug(stdout)
  689. if returncode == 0 and err:
  690. log.warning(err)
  691. elif returncode != 0:
  692. assert err, "Command exited unsuccessful, but printed nothing to stderr"
  693. err = err.replace('%', '%%')
  694. raise qubes.storage.StoragePoolException(err)
  695. return True
  696. def qubes_lvm(cmd, log=logging.getLogger('qubes.storage.lvm')):
  697. ''' Call :program:`lvm` to execute an LVM operation '''
  698. # the only caller for this non-coroutine version is ThinVolume.export()
  699. cmd = _get_lvm_cmdline(cmd)
  700. environ = os.environ.copy()
  701. environ['LC_ALL'] = 'C.utf8'
  702. p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
  703. close_fds=True, env=environ)
  704. out, err = p.communicate()
  705. return _process_lvm_output(p.returncode, out, err, log)
  706. @asyncio.coroutine
  707. def qubes_lvm_coro(cmd, log=logging.getLogger('qubes.storage.lvm')):
  708. ''' Call :program:`lvm` to execute an LVM operation
  709. Coroutine version of :py:func:`qubes_lvm`'''
  710. environ = os.environ.copy()
  711. environ['LC_ALL'] = 'C.utf8'
  712. if cmd[0] == "remove":
  713. pre_cmd = ['blkdiscard', '/dev/'+cmd[1]]
  714. p = yield from asyncio.create_subprocess_exec(*pre_cmd,
  715. stdout=subprocess.DEVNULL,
  716. stderr=subprocess.DEVNULL,
  717. close_fds=True, env=environ)
  718. _, _ = yield from p.communicate()
  719. cmd = _get_lvm_cmdline(cmd)
  720. p = yield from asyncio.create_subprocess_exec(*cmd,
  721. stdout=subprocess.PIPE,
  722. stderr=subprocess.PIPE,
  723. close_fds=True, env=environ)
  724. out, err = yield from p.communicate()
  725. return _process_lvm_output(p.returncode, out, err, log)
  726. def reset_cache():
  727. qubes.storage.lvm.size_cache = init_cache()
  728. qubes.storage.lvm.size_cache_time = time.monotonic()
  729. @asyncio.coroutine
  730. def reset_cache_coro():
  731. qubes.storage.lvm.size_cache = yield from init_cache_coro()
  732. qubes.storage.lvm.size_cache_time = time.monotonic()
  733. def refresh_cache():
  734. '''Reset size cache, if it's older than 30sec '''
  735. if size_cache_time+30 < time.monotonic():
  736. reset_cache()