lvm.py 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832
  1. #
  2. # The Qubes OS Project, http://www.qubes-os.org
  3. #
  4. # Copyright (C) 2016 Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
  5. #
  6. # This library is free software; you can redistribute it and/or
  7. # modify it under the terms of the GNU Lesser General Public
  8. # License as published by the Free Software Foundation; either
  9. # version 2.1 of the License, or (at your option) any later version.
  10. #
  11. # This library is distributed in the hope that it will be useful,
  12. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14. # Lesser General Public License for more details.
  15. #
  16. # You should have received a copy of the GNU Lesser General Public
  17. # License along with this library; if not, see <https://www.gnu.org/licenses/>.
  18. #
  19. ''' Driver for storing vm images in a LVM thin pool '''
  20. import functools
  21. import logging
  22. import os
  23. import subprocess
  24. import time
  25. import asyncio
  26. import qubes
  27. import qubes.storage
  28. import qubes.utils
  29. def check_lvm_version():
  30. #Check if lvm is very very old, like in Travis-CI
  31. try:
  32. lvm_help = subprocess.check_output(['lvm', 'lvcreate', '--help'],
  33. stderr=subprocess.DEVNULL).decode()
  34. return '--setactivationskip' not in lvm_help
  35. except (subprocess.CalledProcessError, FileNotFoundError):
  36. pass
  37. lvm_is_very_old = check_lvm_version()
  38. class ThinPool(qubes.storage.Pool):
  39. ''' LVM Thin based pool implementation
  40. Volumes are stored as LVM thin volumes, in thin pool specified by
  41. *volume_group*/*thin_pool* arguments. LVM volume naming scheme:
  42. vm-{vm_name}-{volume_name}[-suffix]
  43. Where suffix can be one of:
  44. "-snap" - snapshot for currently running VM, at VM shutdown will be
  45. either discarded (if save_on_stop=False), or committed
  46. (if save_on_stop=True)
  47. "-{revision_id}" - volume revision - new revision is automatically
  48. created at each VM shutdown, *revisions_to_keep* control how many
  49. old revisions (in addition to the current one) should be stored
  50. "" (no suffix) - the most recent committed volume state; also volatile
  51. volume (snap_on_start=False, save_on_stop=False)
  52. On VM startup, new volume is created, depending on volume type,
  53. according to the table below:
  54. snap_on_start, save_on_stop
  55. False, False, - no suffix, fresh empty volume
  56. False, True, - "-snap", snapshot of last committed revision
  57. True , False, - "-snap", snapshot of last committed revision
  58. of source volume (from VM's template)
  59. True, True, - unsupported configuration
  60. Volume's revision_id format is "{timestamp}-back", where timestamp is in
  61. '%s' format (seconds since unix epoch)
  62. ''' # pylint: disable=protected-access
  63. size_cache = None
  64. driver = 'lvm_thin'
  65. def __init__(self, volume_group, thin_pool, revisions_to_keep=1, **kwargs):
  66. super(ThinPool, self).__init__(revisions_to_keep=revisions_to_keep,
  67. **kwargs)
  68. self.volume_group = volume_group
  69. self.thin_pool = thin_pool
  70. self._pool_id = "{!s}/{!s}".format(volume_group, thin_pool)
  71. self.log = logging.getLogger('qubes.storage.lvm.%s' % self._pool_id)
  72. self._volume_objects_cache = {}
  73. def __repr__(self):
  74. return '<{} at {:#x} name={!r} volume_group={!r} thin_pool={!r}>'.\
  75. format(
  76. type(self).__name__, id(self),
  77. self.name, self.volume_group, self.thin_pool)
  78. @property
  79. def config(self):
  80. return {
  81. 'name': self.name,
  82. 'volume_group': self.volume_group,
  83. 'thin_pool': self.thin_pool,
  84. 'driver': ThinPool.driver,
  85. 'revisions_to_keep': self.revisions_to_keep,
  86. }
  87. def destroy(self):
  88. pass # TODO Should we remove an existing pool?
  89. def init_volume(self, vm, volume_config):
  90. ''' Initialize a :py:class:`qubes.storage.Volume` from `volume_config`.
  91. '''
  92. if 'revisions_to_keep' not in volume_config.keys():
  93. volume_config['revisions_to_keep'] = self.revisions_to_keep
  94. if 'vid' not in volume_config.keys():
  95. if vm and hasattr(vm, 'name'):
  96. vm_name = vm.name
  97. else:
  98. # for the future if we have volumes not belonging to a vm
  99. vm_name = qubes.utils.random_string()
  100. assert self.name
  101. volume_config['vid'] = "{!s}/vm-{!s}-{!s}".format(
  102. self.volume_group, vm_name, volume_config['name'])
  103. volume_config['volume_group'] = self.volume_group
  104. volume_config['pool'] = self
  105. volume = ThinVolume(**volume_config)
  106. self._volume_objects_cache[volume_config['vid']] = volume
  107. return volume
  108. def setup(self):
  109. reset_cache()
  110. cache_key = self.volume_group + '/' + self.thin_pool
  111. if cache_key not in size_cache:
  112. raise qubes.storage.StoragePoolException(
  113. 'Thin pool {} does not exist'.format(cache_key))
  114. if size_cache[cache_key]['attr'][0] != 't':
  115. raise qubes.storage.StoragePoolException(
  116. 'Volume {} is not a thin pool'.format(cache_key))
  117. # TODO Should we create a non existing pool?
  118. def get_volume(self, vid):
  119. ''' Return a volume with given vid'''
  120. if vid in self._volume_objects_cache:
  121. return self._volume_objects_cache[vid]
  122. config = {
  123. 'pool': self,
  124. 'vid': vid,
  125. 'name': vid,
  126. 'volume_group': self.volume_group,
  127. }
  128. # don't cache this object, as it doesn't carry full configuration
  129. return ThinVolume(**config)
  130. def list_volumes(self):
  131. ''' Return a list of volumes managed by this pool '''
  132. volumes = []
  133. for vid, vol_info in size_cache.items():
  134. if not vid.startswith(self.volume_group + '/'):
  135. continue
  136. if vol_info['pool_lv'] != self.thin_pool:
  137. continue
  138. if vid.endswith('-snap') or vid.endswith('-import'):
  139. # implementation detail volume
  140. continue
  141. if vid.endswith('-back'):
  142. # old revisions
  143. continue
  144. volume = self.get_volume(vid)
  145. if volume in volumes:
  146. continue
  147. volumes.append(volume)
  148. return volumes
  149. @property
  150. def size(self):
  151. try:
  152. return qubes.storage.lvm.size_cache[
  153. self.volume_group + '/' + self.thin_pool]['size']
  154. except KeyError:
  155. return 0
  156. @property
  157. def usage(self):
  158. refresh_cache()
  159. try:
  160. return qubes.storage.lvm.size_cache[
  161. self.volume_group + '/' + self.thin_pool]['usage']
  162. except KeyError:
  163. return 0
  164. _init_cache_cmd = ['lvs', '--noheadings', '-o',
  165. 'vg_name,pool_lv,name,lv_size,data_percent,lv_attr,origin',
  166. '--units', 'b', '--separator', ';']
  167. def _parse_lvm_cache(lvm_output):
  168. result = {}
  169. for line in lvm_output.splitlines():
  170. line = line.decode().strip()
  171. pool_name, pool_lv, name, size, usage_percent, attr, \
  172. origin = line.split(';', 6)
  173. if '' in [pool_name, name, size, usage_percent]:
  174. continue
  175. name = pool_name + "/" + name
  176. size = int(size[:-1]) # Remove 'B' suffix
  177. usage = int(size / 100 * float(usage_percent))
  178. result[name] = {'size': size, 'usage': usage, 'pool_lv': pool_lv,
  179. 'attr': attr, 'origin': origin}
  180. return result
  181. def init_cache(log=logging.getLogger('qubes.storage.lvm')):
  182. cmd = _init_cache_cmd
  183. if os.getuid() != 0:
  184. cmd = ['sudo'] + cmd
  185. environ = os.environ.copy()
  186. environ['LC_ALL'] = 'C.utf8'
  187. p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
  188. close_fds=True, env=environ)
  189. out, err = p.communicate()
  190. return_code = p.returncode
  191. if return_code == 0 and err:
  192. log.warning(err)
  193. elif return_code != 0:
  194. raise qubes.storage.StoragePoolException(err)
  195. return _parse_lvm_cache(out)
  196. @asyncio.coroutine
  197. def init_cache_coro(log=logging.getLogger('qubes.storage.lvm')):
  198. cmd = _init_cache_cmd
  199. if os.getuid() != 0:
  200. cmd = ['sudo'] + cmd
  201. environ = os.environ.copy()
  202. environ['LC_ALL'] = 'C.utf8'
  203. p = yield from asyncio.create_subprocess_exec(*cmd,
  204. stdout=subprocess.PIPE,
  205. stderr=subprocess.PIPE,
  206. close_fds=True, env=environ)
  207. out, err = yield from p.communicate()
  208. return_code = p.returncode
  209. if return_code == 0 and err:
  210. log.warning(err)
  211. elif return_code != 0:
  212. raise qubes.storage.StoragePoolException(err)
  213. return _parse_lvm_cache(out)
  214. size_cache_time = 0
  215. size_cache = init_cache()
  216. def _revision_sort_key(revision):
  217. '''Sort key for revisions. Sort them by time
  218. :returns timestamp
  219. '''
  220. if isinstance(revision, tuple):
  221. revision = revision[0]
  222. if '-' in revision:
  223. revision = revision.split('-')[0]
  224. return int(revision)
  225. def locked(method):
  226. '''Decorator running given Volume's coroutine under a lock.
  227. Needs to be added after wrapping with @asyncio.coroutine, for example:
  228. >>>@locked
  229. >>>@asyncio.coroutine
  230. >>>def start(self):
  231. >>> pass
  232. '''
  233. @asyncio.coroutine
  234. @functools.wraps(method)
  235. def wrapper(self, *args, **kwargs):
  236. with (yield from self._lock): # pylint: disable=protected-access
  237. return (yield from method(self, *args, **kwargs))
  238. return wrapper
  239. class ThinVolume(qubes.storage.Volume):
  240. ''' Default LVM thin volume implementation
  241. ''' # pylint: disable=too-few-public-methods
  242. def __init__(self, volume_group, size=0, **kwargs):
  243. self.volume_group = volume_group
  244. super(ThinVolume, self).__init__(size=size, **kwargs)
  245. self.log = logging.getLogger('qubes.storage.lvm.%s' % str(self.pool))
  246. if self.snap_on_start or self.save_on_stop:
  247. self._vid_snap = self.vid + '-snap'
  248. if self.save_on_stop:
  249. self._vid_import = self.vid + '-import'
  250. self._size = size
  251. self._lock = asyncio.Lock()
  252. @property
  253. def path(self):
  254. return '/dev/' + self._vid_current
  255. @property
  256. def _vid_current(self):
  257. if self.vid in size_cache:
  258. return self.vid
  259. vol_revisions = self.revisions
  260. if vol_revisions:
  261. last_revision = \
  262. max(vol_revisions.items(), key=_revision_sort_key)[0]
  263. return self.vid + '-' + last_revision
  264. # detached pool? return expected path
  265. return self.vid
  266. @property
  267. def revisions(self):
  268. name_prefix = self.vid + '-'
  269. revisions = {}
  270. for revision_vid in size_cache:
  271. if not revision_vid.startswith(name_prefix):
  272. continue
  273. if not revision_vid.endswith('-back'):
  274. continue
  275. revision_vid = revision_vid[len(name_prefix):]
  276. if revision_vid.count('-') > 1:
  277. # VM+volume name is a prefix of another VM, see #4680
  278. continue
  279. # get revision without suffix
  280. seconds = int(revision_vid.split('-')[0])
  281. iso_date = qubes.storage.isodate(seconds).split('.', 1)[0]
  282. revisions[revision_vid] = iso_date
  283. return revisions
  284. @property
  285. def size(self):
  286. try:
  287. if self.is_dirty():
  288. return qubes.storage.lvm.size_cache[self._vid_snap]['size']
  289. return qubes.storage.lvm.size_cache[self._vid_current]['size']
  290. except KeyError:
  291. return self._size
  292. @size.setter
  293. def size(self, _):
  294. raise qubes.storage.StoragePoolException(
  295. "You shouldn't use lvm size setter")
  296. @asyncio.coroutine
  297. def _reset(self):
  298. ''' Resets a volatile volume '''
  299. assert not self.snap_on_start and not self.save_on_stop, \
  300. "Not a volatile volume"
  301. self.log.debug('Resetting volatile %s', self.vid)
  302. try:
  303. cmd = ['remove', self.vid]
  304. yield from qubes_lvm_coro(cmd, self.log)
  305. except qubes.storage.StoragePoolException:
  306. pass
  307. # pylint: disable=protected-access
  308. cmd = ['create', self.pool._pool_id, self.vid.split('/')[1],
  309. str(self.size)]
  310. yield from qubes_lvm_coro(cmd, self.log)
  311. @asyncio.coroutine
  312. def _remove_revisions(self, revisions=None):
  313. '''Remove old volume revisions.
  314. If no revisions list is given, it removes old revisions according to
  315. :py:attr:`revisions_to_keep`
  316. :param revisions: list of revisions to remove
  317. '''
  318. if revisions is None:
  319. revisions = sorted(self.revisions.items(),
  320. key=_revision_sort_key)
  321. # pylint: disable=invalid-unary-operand-type
  322. revisions = revisions[:(-self.revisions_to_keep) or None]
  323. revisions = [rev_id for rev_id, _ in revisions]
  324. for rev_id in revisions:
  325. # safety check
  326. assert rev_id != self._vid_current
  327. try:
  328. cmd = ['remove', self.vid + '-' + rev_id]
  329. yield from qubes_lvm_coro(cmd, self.log)
  330. except qubes.storage.StoragePoolException:
  331. pass
  332. @asyncio.coroutine
  333. def _commit(self, vid_to_commit=None, keep=False):
  334. '''
  335. Commit temporary volume into current one. By default
  336. :py:attr:`_vid_snap` is used (which is created by :py:meth:`start()`),
  337. but can be overriden by *vid_to_commit* argument.
  338. :param vid_to_commit: LVM volume ID to commit into this one
  339. :param keep: whether to keep or not *vid_to_commit*.
  340. IOW use 'clone' or 'rename' methods.
  341. :return: None
  342. '''
  343. msg = "Trying to commit {!s}, but it has save_on_stop == False"
  344. msg = msg.format(self)
  345. assert self.save_on_stop, msg
  346. msg = "Trying to commit {!s}, but it has rw == False"
  347. msg = msg.format(self)
  348. assert self.rw, msg
  349. if vid_to_commit is None:
  350. assert hasattr(self, '_vid_snap')
  351. vid_to_commit = self._vid_snap
  352. assert self._lock.locked()
  353. if not os.path.exists('/dev/' + vid_to_commit):
  354. # nothing to commit
  355. return
  356. if self._vid_current == self.vid:
  357. cmd = ['rename', self.vid,
  358. '{}-{}-back'.format(self.vid, int(time.time()))]
  359. yield from qubes_lvm_coro(cmd, self.log)
  360. yield from reset_cache_coro()
  361. cmd = ['clone' if keep else 'rename',
  362. vid_to_commit,
  363. self.vid]
  364. yield from qubes_lvm_coro(cmd, self.log)
  365. yield from reset_cache_coro()
  366. # make sure the one we've committed right now is properly
  367. # detected as the current one - before removing anything
  368. assert self._vid_current == self.vid
  369. # and remove old snapshots, if needed
  370. yield from self._remove_revisions()
  371. @locked
  372. @asyncio.coroutine
  373. def create(self):
  374. assert self.vid
  375. assert self.size
  376. if self.save_on_stop:
  377. if self.source:
  378. cmd = ['clone', self.source.path, self.vid]
  379. else:
  380. cmd = [
  381. 'create',
  382. self.pool._pool_id, # pylint: disable=protected-access
  383. self.vid.split('/', 1)[1],
  384. str(self.size)
  385. ]
  386. yield from qubes_lvm_coro(cmd, self.log)
  387. yield from reset_cache_coro()
  388. return self
  389. @locked
  390. @asyncio.coroutine
  391. def remove(self):
  392. assert self.vid
  393. try:
  394. if os.path.exists('/dev/' + self._vid_snap):
  395. cmd = ['remove', self._vid_snap]
  396. yield from qubes_lvm_coro(cmd, self.log)
  397. except AttributeError:
  398. pass
  399. try:
  400. if os.path.exists('/dev/' + self._vid_import):
  401. cmd = ['remove', self._vid_import]
  402. yield from qubes_lvm_coro(cmd, self.log)
  403. except AttributeError:
  404. pass
  405. yield from self._remove_revisions(self.revisions.keys())
  406. if not os.path.exists(self.path):
  407. return
  408. cmd = ['remove', self.path]
  409. yield from qubes_lvm_coro(cmd, self.log)
  410. yield from reset_cache_coro()
  411. # pylint: disable=protected-access
  412. self.pool._volume_objects_cache.pop(self.vid, None)
  413. def export(self):
  414. ''' Returns an object that can be `open()`. '''
  415. # make sure the device node is available
  416. qubes_lvm(['activate', self.path], self.log)
  417. devpath = self.path
  418. return devpath
  419. @locked
  420. @asyncio.coroutine
  421. def import_volume(self, src_volume):
  422. if not src_volume.save_on_stop:
  423. return self
  424. if self.is_dirty():
  425. raise qubes.storage.StoragePoolException(
  426. 'Cannot import to dirty volume {} -'
  427. ' start and stop a qube to cleanup'.format(self.vid))
  428. self.abort_if_import_in_progress()
  429. # HACK: neat trick to speed up testing if you have same physical thin
  430. # pool assigned to two qubes-pools i.e: qubes_dom0 and test-lvm
  431. # pylint: disable=line-too-long
  432. if isinstance(src_volume.pool, ThinPool) and \
  433. src_volume.pool.thin_pool == self.pool.thin_pool: # NOQA
  434. yield from self._commit(src_volume.path[len('/dev/'):], keep=True)
  435. else:
  436. cmd = ['create',
  437. self.pool._pool_id, # pylint: disable=protected-access
  438. self._vid_import.split('/')[1],
  439. str(src_volume.size)]
  440. yield from qubes_lvm_coro(cmd, self.log)
  441. src_path = src_volume.export()
  442. cmd = ['dd', 'if=' + src_path, 'of=/dev/' + self._vid_import,
  443. 'conv=sparse', 'status=none']
  444. if not os.access('/dev/' + self._vid_import, os.W_OK) or \
  445. not os.access(src_path, os.R_OK):
  446. cmd.insert(0, 'sudo')
  447. p = yield from asyncio.create_subprocess_exec(*cmd)
  448. yield from p.wait()
  449. if p.returncode != 0:
  450. cmd = ['remove', self._vid_import]
  451. yield from qubes_lvm_coro(cmd, self.log)
  452. raise qubes.storage.StoragePoolException(
  453. 'Failed to import volume {!r}, dd exit code: {}'.format(
  454. src_volume, p.returncode))
  455. yield from self._commit(self._vid_import)
  456. return self
  457. @locked
  458. @asyncio.coroutine
  459. def import_data(self):
  460. ''' Returns an object that can be `open()`. '''
  461. if self.is_dirty():
  462. raise qubes.storage.StoragePoolException(
  463. 'Cannot import data to dirty volume {}, stop the qube first'.
  464. format(self.vid))
  465. self.abort_if_import_in_progress()
  466. # pylint: disable=protected-access
  467. cmd = ['create', self.pool._pool_id, self._vid_import.split('/')[1],
  468. str(self.size)]
  469. yield from qubes_lvm_coro(cmd, self.log)
  470. yield from reset_cache_coro()
  471. devpath = '/dev/' + self._vid_import
  472. return devpath
  473. @locked
  474. @asyncio.coroutine
  475. def import_data_end(self, success):
  476. '''Either commit imported data, or discard temporary volume'''
  477. if not os.path.exists('/dev/' + self._vid_import):
  478. raise qubes.storage.StoragePoolException(
  479. 'No import operation in progress on {}'.format(self.vid))
  480. if success:
  481. yield from self._commit(self._vid_import)
  482. else:
  483. cmd = ['remove', self._vid_import]
  484. yield from qubes_lvm_coro(cmd, self.log)
  485. def abort_if_import_in_progress(self):
  486. try:
  487. devpath = '/dev/' + self._vid_import
  488. if os.path.exists(devpath):
  489. raise qubes.storage.StoragePoolException(
  490. 'Import operation in progress on {}'.format(self.vid))
  491. except AttributeError: # self._vid_import
  492. # no vid_import - import definitely not in progress
  493. pass
  494. def is_dirty(self):
  495. if self.save_on_stop:
  496. return os.path.exists('/dev/' + self._vid_snap)
  497. return False
  498. def is_outdated(self):
  499. if not self.snap_on_start:
  500. return False
  501. if self._vid_snap not in size_cache:
  502. return False
  503. return (size_cache[self._vid_snap]['origin'] !=
  504. self.source.path.split('/')[-1])
  505. @locked
  506. @asyncio.coroutine
  507. def revert(self, revision=None):
  508. if self.is_dirty():
  509. raise qubes.storage.StoragePoolException(
  510. 'Cannot revert dirty volume {}, stop the qube first'.format(
  511. self.vid))
  512. self.abort_if_import_in_progress()
  513. if revision is None:
  514. revision = \
  515. max(self.revisions.items(), key=_revision_sort_key)[0]
  516. old_path = '/dev/' + self.vid + '-' + revision
  517. if not os.path.exists(old_path):
  518. msg = "Volume {!s} has no {!s}".format(self, old_path)
  519. raise qubes.storage.StoragePoolException(msg)
  520. if self.vid in size_cache:
  521. cmd = ['remove', self.vid]
  522. yield from qubes_lvm_coro(cmd, self.log)
  523. cmd = ['clone', self.vid + '-' + revision, self.vid]
  524. yield from qubes_lvm_coro(cmd, self.log)
  525. yield from reset_cache_coro()
  526. return self
  527. @locked
  528. @asyncio.coroutine
  529. def resize(self, size):
  530. ''' Expands volume, throws
  531. :py:class:`qubst.storage.qubes.storage.StoragePoolException` if
  532. given size is less than current_size
  533. '''
  534. if not self.rw:
  535. msg = 'Can not resize reađonly volume {!s}'.format(self)
  536. raise qubes.storage.StoragePoolException(msg)
  537. if size < self.size:
  538. raise qubes.storage.StoragePoolException(
  539. 'For your own safety, shrinking of %s is'
  540. ' disabled (%d < %d). If you really know what you'
  541. ' are doing, use `lvresize` on %s manually.' %
  542. (self.name, size, self.size, self.vid))
  543. if size == self.size:
  544. return
  545. if self.is_dirty():
  546. cmd = ['extend', self._vid_snap, str(size)]
  547. yield from qubes_lvm_coro(cmd, self.log)
  548. elif hasattr(self, '_vid_import') and \
  549. os.path.exists('/dev/' + self._vid_import):
  550. cmd = ['extend', self._vid_import, str(size)]
  551. yield from qubes_lvm_coro(cmd, self.log)
  552. elif self.save_on_stop or not self.snap_on_start:
  553. cmd = ['extend', self._vid_current, str(size)]
  554. yield from qubes_lvm_coro(cmd, self.log)
  555. yield from reset_cache_coro()
  556. @asyncio.coroutine
  557. def _snapshot(self):
  558. try:
  559. cmd = ['remove', self._vid_snap]
  560. yield from qubes_lvm_coro(cmd, self.log)
  561. except: # pylint: disable=bare-except
  562. pass
  563. if self.source is None:
  564. cmd = ['clone', self._vid_current, self._vid_snap]
  565. else:
  566. cmd = ['clone', self.source.path, self._vid_snap]
  567. yield from qubes_lvm_coro(cmd, self.log)
  568. @locked
  569. @asyncio.coroutine
  570. def start(self):
  571. self.abort_if_import_in_progress()
  572. try:
  573. if self.snap_on_start or self.save_on_stop:
  574. if not self.save_on_stop or not self.is_dirty():
  575. yield from self._snapshot()
  576. else:
  577. yield from self._reset()
  578. finally:
  579. yield from reset_cache_coro()
  580. return self
  581. @locked
  582. @asyncio.coroutine
  583. def stop(self):
  584. try:
  585. if self.save_on_stop:
  586. yield from self._commit()
  587. if self.snap_on_start and not self.save_on_stop:
  588. cmd = ['remove', self._vid_snap]
  589. yield from qubes_lvm_coro(cmd, self.log)
  590. elif not self.snap_on_start and not self.save_on_stop:
  591. cmd = ['remove', self.vid]
  592. yield from qubes_lvm_coro(cmd, self.log)
  593. finally:
  594. yield from reset_cache_coro()
  595. return self
  596. def verify(self):
  597. ''' Verifies the volume. '''
  598. if not self.save_on_stop and not self.snap_on_start:
  599. # volatile volumes don't need any files
  600. return True
  601. if self.source is not None:
  602. vid = self.source.path[len('/dev/'):]
  603. else:
  604. vid = self._vid_current
  605. try:
  606. vol_info = size_cache[vid]
  607. if vol_info['attr'][4] != 'a':
  608. raise qubes.storage.StoragePoolException(
  609. 'volume {} not active'.format(vid))
  610. except KeyError:
  611. raise qubes.storage.StoragePoolException(
  612. 'volume {} missing'.format(vid))
  613. return True
  614. def block_device(self):
  615. ''' Return :py:class:`qubes.storage.BlockDevice` for serialization in
  616. the libvirt XML template as <disk>.
  617. '''
  618. if self.snap_on_start or self.save_on_stop:
  619. return qubes.storage.BlockDevice(
  620. '/dev/' + self._vid_snap, self.name, self.script,
  621. self.rw, self.domain, self.devtype)
  622. return super(ThinVolume, self).block_device()
  623. @property
  624. def usage(self): # lvm thin usage always returns at least the same usage as
  625. # the parent
  626. refresh_cache()
  627. try:
  628. return qubes.storage.lvm.size_cache[self._vid_current]['usage']
  629. except KeyError:
  630. return 0
  631. def pool_exists(pool_id):
  632. ''' Return true if pool exists '''
  633. try:
  634. vol_info = size_cache[pool_id]
  635. return vol_info['attr'][0] == 't'
  636. except KeyError:
  637. return False
  638. def _get_lvm_cmdline(cmd):
  639. ''' Build command line for :program:`lvm` call.
  640. The purpose of this function is to keep all the detailed lvm options in
  641. one place.
  642. :param cmd: array of str, where cmd[0] is action and the rest are arguments
  643. :return array of str appropriate for subprocess.Popen
  644. '''
  645. action = cmd[0]
  646. if action == 'remove':
  647. lvm_cmd = ['lvremove', '-f', cmd[1]]
  648. elif action == 'clone':
  649. lvm_cmd = ['lvcreate', '-kn', '-ay', '-s', cmd[1], '-n', cmd[2]]
  650. elif action == 'create':
  651. lvm_cmd = ['lvcreate', '-T', cmd[1], '-kn', '-ay', '-n', cmd[2], '-V',
  652. str(cmd[3]) + 'B']
  653. elif action == 'extend':
  654. size = int(cmd[2]) / (1024 * 1024)
  655. lvm_cmd = ["lvextend", "-L%s" % size, cmd[1]]
  656. elif action == 'activate':
  657. lvm_cmd = ['lvchange', '-ay', cmd[1]]
  658. elif action == 'rename':
  659. lvm_cmd = ['lvrename', cmd[1], cmd[2]]
  660. else:
  661. raise NotImplementedError('unsupported action: ' + action)
  662. if lvm_is_very_old:
  663. # old lvm in trusty image used there does not support -k option
  664. lvm_cmd = [x for x in lvm_cmd if x != '-kn']
  665. if os.getuid() != 0:
  666. cmd = ['sudo', 'lvm'] + lvm_cmd
  667. else:
  668. cmd = ['lvm'] + lvm_cmd
  669. return cmd
  670. def _process_lvm_output(returncode, stdout, stderr, log):
  671. '''Process output of LVM, determine if the call was successful and
  672. possibly log warnings.'''
  673. # Filter out warning about intended over-provisioning.
  674. # Upstream discussion about missing option to silence it:
  675. # https://bugzilla.redhat.com/1347008
  676. err = '\n'.join(line for line in stderr.decode().splitlines()
  677. if 'exceeds the size of thin pool' not in line)
  678. if stdout:
  679. log.debug(stdout)
  680. if returncode == 0 and err:
  681. log.warning(err)
  682. elif returncode != 0:
  683. assert err, "Command exited unsuccessful, but printed nothing to stderr"
  684. err = err.replace('%', '%%')
  685. raise qubes.storage.StoragePoolException(err)
  686. return True
  687. def qubes_lvm(cmd, log=logging.getLogger('qubes.storage.lvm')):
  688. ''' Call :program:`lvm` to execute an LVM operation '''
  689. # the only caller for this non-coroutine version is ThinVolume.export()
  690. cmd = _get_lvm_cmdline(cmd)
  691. environ = os.environ.copy()
  692. environ['LC_ALL'] = 'C.utf8'
  693. p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
  694. close_fds=True, env=environ)
  695. out, err = p.communicate()
  696. return _process_lvm_output(p.returncode, out, err, log)
  697. @asyncio.coroutine
  698. def qubes_lvm_coro(cmd, log=logging.getLogger('qubes.storage.lvm')):
  699. ''' Call :program:`lvm` to execute an LVM operation
  700. Coroutine version of :py:func:`qubes_lvm`'''
  701. cmd = _get_lvm_cmdline(cmd)
  702. environ = os.environ.copy()
  703. environ['LC_ALL'] = 'C.utf8'
  704. p = yield from asyncio.create_subprocess_exec(*cmd,
  705. stdout=subprocess.PIPE,
  706. stderr=subprocess.PIPE,
  707. close_fds=True, env=environ)
  708. out, err = yield from p.communicate()
  709. return _process_lvm_output(p.returncode, out, err, log)
  710. def reset_cache():
  711. qubes.storage.lvm.size_cache = init_cache()
  712. qubes.storage.lvm.size_cache_time = time.monotonic()
  713. @asyncio.coroutine
  714. def reset_cache_coro():
  715. qubes.storage.lvm.size_cache = yield from init_cache_coro()
  716. qubes.storage.lvm.size_cache_time = time.monotonic()
  717. def refresh_cache():
  718. '''Reset size cache, if it's older than 30sec '''
  719. if size_cache_time+30 < time.monotonic():
  720. reset_cache()